From f92fafcf7fa5bbf28443795d697929c700822d70 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Wed, 2 Sep 2020 10:17:16 -0400 Subject: [PATCH 001/136] Add versions endpoint to spec, fix versions model objects. --- daemon/algod/api/algod.oas2.json | 160 +++++---- daemon/algod/api/algod.oas3.yml | 144 ++++---- .../api/server/v2/generated/private/routes.go | 250 +++++++------- .../api/server/v2/generated/private/types.go | 33 +- .../algod/api/server/v2/generated/routes.go | 315 +++++++++--------- daemon/algod/api/server/v2/generated/types.go | 33 +- 6 files changed, 493 insertions(+), 442 deletions(-) diff --git a/daemon/algod/api/algod.oas2.json b/daemon/algod/api/algod.oas2.json index 8210a5d66b..27948c9a92 100644 --- a/daemon/algod/api/algod.oas2.json +++ b/daemon/algod/api/algod.oas2.json @@ -96,6 +96,26 @@ } } }, + "/versions": { + "get": { + "description": "Retrieves the current version", + "tags": [ + "common" + ], + "produces": [ + "application/json" + ], + "schemes": [ + "http" + ], + "operationId": "GetVersion", + "responses": { + "200": { + "$ref": "#/responses/VersionsResponse" + } + } + } + }, "/v2/accounts/{address}": { "get": { "description": "Given a specific account public key, this call returns the accounts status, balance and spendable amounts", @@ -412,7 +432,6 @@ ], "responses": { "200": { - "description": "(empty)", "schema": { "type": "object" } @@ -1605,68 +1624,6 @@ } } }, - "Version": { - "description": "Note that we annotate this as a model so that legacy clients\ncan directly import a swagger generated Version model.", - "type": "object", - "required": [ - "build", - "genesis-hash", - "genesis-id", - "versions" - ], - "properties": { - "build": { - "$ref": "#/definitions/VersionBuild" - }, - "genesis-hash": { - "type": "string", - "format": "byte" - }, - "genesis-id": { - "type": "string" - }, - "versions": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "x-go-package": "github.com/algorand/go-algorand/daemon/algod/api/spec/common" - }, - "VersionBuild": { - "description": "the current algod build version information.", - "type": "object", - "required": [ - "branch", - "build-number", - "channel", - "commit-hash", - "major", - "minor" - ], - "properties": { - "branch": { - "type": "string" - }, - "build-number": { - "type": "integer" - }, - "channel": { - "type": "string" - }, - "commit-hash": { - "type": "string", - "format": "byte" - }, - "major": { - "type": "integer" - }, - "minor": { - "type": "integer" - } - } - }, "DryrunRequest": { "description": "Request data type for dryrun endpoint. Given the Transactions and simulated ledger state upload, run TEAL scripts and return debugging information.", "type": "object", @@ -1748,6 +1705,73 @@ "x-algorand-format": "uint64" } } + }, + "Version": { + "description": "algod version information.", + "type": "object", + "title": "Version contains the current algod version.", + "required": [ + "versions", + "genesis_id", + "genesis_hash_b64", + "build" + ], + "properties": { + "build": { + "$ref": "#/definitions/BuildVersion" + }, + "genesis_hash_b64": { + "type": "string", + "format": "byte" + }, + "genesis_id": { + "type": "string" + }, + "versions": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "BuildVersion": { + "tags": [ + "common" + ], + "type": "object", + "title": "BuildVersion contains the current algod build version information.", + "required": [ + "major", + "minor", + "build_number", + "commit_hash", + "branch", + "channel" + ], + "properties": { + "branch": { + "type": "string" + }, + "build_number": { + "type": "integer", + "format": "int64" + }, + "channel": { + "type": "string" + }, + "commit_hash": { + "type": "string" + }, + "major": { + "type": "integer", + "format": "int64" + }, + "minor": { + "type": "integer", + "format": "int64" + } + } } }, "parameters": { @@ -1922,7 +1946,6 @@ }, "responses": { "AccountResponse": { - "description": "(empty)", "schema": { "$ref": "#/definitions/Account" } @@ -1952,7 +1975,6 @@ "tags": [ "private" ], - "description": "(empty)", "schema": { "description": "An catchpoint start response.", "type": "object", @@ -1971,7 +1993,6 @@ "tags": [ "private" ], - "description": "(empty)", "schema": { "description": "An catchpoint abort response.", "type": "object", @@ -1987,7 +2008,6 @@ } }, "NodeStatusResponse": { - "description": "(empty)", "schema": { "description": "NodeStatus contains the information about a node status", "type": "object", @@ -2288,6 +2308,12 @@ } } } + }, + "VersionsResponse": { + "description": "VersionsResponse is the response to 'GET /versions'", + "schema": { + "$ref": "#/definitions/Version" + } } }, "securityDefinitions": { diff --git a/daemon/algod/api/algod.oas3.yml b/daemon/algod/api/algod.oas3.yml index 688ee9c26d..f59dd055ba 100644 --- a/daemon/algod/api/algod.oas3.yml +++ b/daemon/algod/api/algod.oas3.yml @@ -230,8 +230,7 @@ "$ref": "#/components/schemas/Account" } } - }, - "description": "(empty)" + } }, "ApplicationResponse": { "content": { @@ -297,8 +296,7 @@ "type": "object" } } - }, - "description": "(empty)" + } }, "CatchpointStartResponse": { "content": { @@ -317,8 +315,7 @@ "type": "object" } } - }, - "description": "(empty)" + } }, "CompileResponse": { "content": { @@ -450,8 +447,7 @@ "type": "object" } } - }, - "description": "(empty)" + } }, "PendingTransactionResponse": { "content": { @@ -644,6 +640,16 @@ } }, "description": "TransactionParams contains the parameters that help a client construct a new transaction." + }, + "VersionsResponse": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Version" + } + } + }, + "description": "VersionsResponse is the response to 'GET /versions'" } }, "schemas": { @@ -1000,6 +1006,41 @@ ], "type": "object" }, + "BuildVersion": { + "properties": { + "branch": { + "type": "string" + }, + "build_number": { + "format": "int64", + "type": "integer" + }, + "channel": { + "type": "string" + }, + "commit_hash": { + "type": "string" + }, + "major": { + "format": "int64", + "type": "integer" + }, + "minor": { + "format": "int64", + "type": "integer" + } + }, + "required": [ + "branch", + "build_number", + "channel", + "commit_hash", + "major", + "minor" + ], + "title": "BuildVersion contains the current algod build version information.", + "type": "object" + }, "DryrunRequest": { "description": "Request data type for dryrun endpoint. Given the Transactions and simulated ledger state upload, run TEAL scripts and return debugging information.", "properties": { @@ -1274,17 +1315,17 @@ "type": "object" }, "Version": { - "description": "Note that we annotate this as a model so that legacy clients\ncan directly import a swagger generated Version model.", + "description": "algod version information.", "properties": { "build": { - "$ref": "#/components/schemas/VersionBuild" + "$ref": "#/components/schemas/BuildVersion" }, - "genesis-hash": { + "genesis_hash_b64": { "format": "byte", "pattern": "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$", "type": "string" }, - "genesis-id": { + "genesis_id": { "type": "string" }, "versions": { @@ -1296,45 +1337,11 @@ }, "required": [ "build", - "genesis-hash", - "genesis-id", + "genesis_hash_b64", + "genesis_id", "versions" ], - "type": "object", - "x-go-package": "github.com/algorand/go-algorand/daemon/algod/api/spec/common" - }, - "VersionBuild": { - "description": "the current algod build version information.", - "properties": { - "branch": { - "type": "string" - }, - "build-number": { - "type": "integer" - }, - "channel": { - "type": "string" - }, - "commit-hash": { - "format": "byte", - "pattern": "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$", - "type": "string" - }, - "major": { - "type": "integer" - }, - "minor": { - "type": "integer" - } - }, - "required": [ - "branch", - "build-number", - "channel", - "commit-hash", - "major", - "minor" - ], + "title": "Version contains the current algod version.", "type": "object" } }, @@ -1464,8 +1471,7 @@ "$ref": "#/components/schemas/Account" } } - }, - "description": "(empty)" + } }, "400": { "content": { @@ -2010,8 +2016,7 @@ "type": "object" } } - }, - "description": "(empty)" + } }, "400": { "content": { @@ -2089,8 +2094,7 @@ "type": "object" } } - }, - "description": "(empty)" + } }, "400": { "content": { @@ -2281,8 +2285,7 @@ "type": "object" } } - }, - "description": "(empty)" + } } }, "tags": [ @@ -2370,8 +2373,7 @@ "type": "object" } } - }, - "description": "(empty)" + } }, "401": { "content": { @@ -2494,8 +2496,7 @@ "type": "object" } } - }, - "description": "(empty)" + } }, "400": { "content": { @@ -3230,6 +3231,27 @@ }, "summary": "Get a specific pending transaction." } + }, + "/versions": { + "get": { + "description": "Retrieves the current version", + "operationId": "GetVersion", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Version" + } + } + }, + "description": "VersionsResponse is the response to 'GET /versions'" + } + }, + "tags": [ + "common" + ] + } } }, "security": [ diff --git a/daemon/algod/api/server/v2/generated/private/routes.go b/daemon/algod/api/server/v2/generated/private/routes.go index 5cfcc95a5a..6be3262f14 100644 --- a/daemon/algod/api/server/v2/generated/private/routes.go +++ b/daemon/algod/api/server/v2/generated/private/routes.go @@ -235,131 +235,131 @@ func RegisterHandlers(router interface { // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9f3fbNhLgV8Fp970mOVFyfnU3fq9vz036w9c0zYvdvbuNc1uIHEmoSYAlQMtqzt/9", - "3gwAEiRBSXa82eu7/SuxAAwGg5nBzGAw/DhJVVEqCdLoyfHHSckrXoCBiv7iaapqaRKR4V8Z6LQSpRFK", - "To59G9OmEnI1mU4E/lpys55MJ5IX0PbB8dNJBb/VooJscmyqGqYTna6h4AjYbEvs3UC6TlYqcSBOLIjT", - "V5ObHQ08yyrQeojlTzLfMiHTvM6AmYpLzVNs0mwjzJqZtdDMDWZCMiWBqSUz605nthSQZ3rmF/lbDdU2", - "WKWbfHxJNy2KSaVyGOL5UhULIcFjBQ1SzYYwo1gGS+q05obhDIir72gU08CrdM2WqtqDqkUixBdkXUyO", - "3080yAwq2q0UxBX9d1kB/A6J4dUKzOTDNLa4pYEqMaKILO3UUb8CXedGM+pLa1yJK5AMR83Yj7U2bAGM", - "S/bu25fs6dOnL3AhBTcGMsdko6tqZw/XZIdPjicZN+Cbh7zG85WquMySpv+7b1/S/GdugYf24lpDXFhO", - "sIWdvhpbgB8YYSEhDaxoHzrcjyMiQtH+vIClquDAPbGd73VTwvn/rbuScpOuSyWkiewLo1Zmm6M6LBi+", - "S4c1CHT6l0ipCoG+P0pefPj4ePr46OZP70+Sf7g/nz+9OXD5Lxu4eygQ7ZjWVQUy3SarCjhJy5rLIT3e", - "OX7Qa1XnGVvzK9p8XpCqd2MZjrWq84rnNfKJSCt1kq+UZtyxUQZLXueG+YlZLXNUUwjNcTsTmpWVuhIZ", - "ZFPUvpu1SNcs5dqCoH5sI/IcebDWkI3xWnx1O4TpJiQJ4nUnetCC/t8lRruuPZSAa9IGSZorDYlRe44n", - "f+JwmbHwQGnPKn27w4qdr4HR5NhgD1uinUSezvMtM7SvGeOaceaPpikTS7ZVNdvQ5uTiksa71SDVCoZE", - "o83pnKMovGPkGxAjQryFUjlwScTzcjckmVyKVV2BZps1mLU78yrQpZIamFr8CqnBbf/vZz+9YapiP4LW", - "fAVveXrJQKYqG99jN2nsBP9VK9zwQq9Knl7Gj+tcFCKC8o/8WhR1wWRdLKDC/fLng1GsAlNXcgwhC3EP", - "nxX8ejjpeVXLlDa3nbZjqCErCV3mfDtjp0tW8OuvjqYOHc14nrMSZCbkiplrOWqk4dz70UsqVcvsABvG", - "4IYFp6YuIRVLARlroOzAxE2zDx8hb4dPa1kF6Hggo+g0s+xBR8J1hGdQdLGFlXwFAcvM2M9Oc1GrUZcg", - "GwXHFltqKiu4EqrWzaARHGnq3ea1VAaSsoKliPDYmSMHag/bx6nXwhk4qZKGCwkZal5CWhmwmmgUp2DC", - "3c7M8IhecA1fPhs7wNvWA3d/qfq7vnPHD9pt6pRYkYyci9jqBDZuNnXGH+D8hXNrsUrsz4ONFKtzPEqW", - "Iqdj5lfcP0+GWpMS6BDCHzxarCQ3dQXHF/IR/sUSdma4zHiV4S+F/enHOjfiTKzwp9z+9FqtRHomViPE", - "bHCNelM0rLD/ILy4OjbXUafhtVKXdRkuKO14pYstO301tskW5m0Z86RxZUOv4vzaexq3HWGum40cQXKU", - "diXHjpewrQCx5emS/rleEj/xZfU7/lOWeYymyMDuoKWggAsWvHO/4U8o8mB9AoQiUo5EndPxefwxQOjP", - "FSwnx5M/zdtIydy26rmDa2fs7t4DKEqzfYhUOGnh3z8G7cgYFkEzE9LuGnWdWl/x/vFBqFFMyIDt4fB1", - "rtLLO+FQVqqEygi7vwuEM5QgAs/WwDOoWMYNn7XOlrW/RuSABn5P48h7gipy9P1E/+E5w2aUTm68WYcm", - "rdBo3KkgAJWhJWjPFzsTdiALVbHCGn8MjbZbYfmyndwq7kbTvndk+dCHFtmdb6y9yWiEXwQuvfUmTxaq", - "uhu/9BhBstZHZhyhNlYxrry7s9S1LhNHn4idbTv0ALVhyaG6DSnUB38IrQLJbqlzZvi/gDoaod4HdbqA", - "Phd1VFGKHO5Bvtdcr4eLQ0Pp6RN29v3J88dP/vnk+Zd40peVWlW8YIutAc0euPOJabPN4eFwxXRQ1LmJ", - "Q//ymffEunD3Uo4QbmAfQrdzQE1iKcZs3AGxe1Vtq1reAwmhqlQVsZ2JpYxKVZ5cQaWFioRB3roezPVA", - "vWXt997vFlu24Zrh3OTW1TKDahajPPprZBoYKPS+g8WCPr+WLW0cQF5VfDvYAbveyOrcvIfsSZf43kvQ", - "rIQqMdeSZbCoV+GZxpaVKhhnGQ0kBfpGZXBmuKn1PWiHFliLDG5EiAJfqNowzqTKUNCxc1xvjMREKRhD", - "MSQTqiKztufVAtDKTnm9WhuG5qmKbW07MOGp3ZSEzhY94kI2vr/tZaez8ba8Ap5t2QJAMrVwfprzIGmR", - "nMI7xt/cOK3VotX4Fh28ykqloDVkibum2ouav/KiTTY7yER4E77NJEwrtuTVHXE1yvB8D57UZ4itbq0P", - "59sOsT5s+l3715883EVeoatqmQBNHRTuHAyMkXAvTepy5FrDnXbnokCRYJJLpSFVMtNRYDnXJtknCtip", - "cyTjtgbcF+N+AjzivL/m2lj3WciMzDYrwjQPjaEpxhEe1dII+e9eQQ9hp6h7pK51o611XZaqMpDF1iDh", - "esdcb+C6mUstA9jNkWAUqzXsgzxGpQC+I5ZdiSUQNy5+08SXhoujUDnq1m2UlB0kWkLsQuTM9wqoG4Z2", - "RxBBG78ZSYwjdI9zmnjydKKNKkvUSSapZTNujExntveJ+bntO2QublpdmSnA2Y3HyWG+sZS1Qf01R3uJ", - "ILOCX6K+J+vH+vlDnFEYEy1kCskuzkexPMNeoQjsEdIRg9RdGwaz9YSjx79Rphtlgj27MLbgW1rHb23U", - "+ryN6NyDgfAKDBe5boyAJjTezkJR9H6GA1psFaQgTb5FHl6KqrAXUXR2aP+bNTEyN4u9cmnFUmasgg2v", - "Mt9j6LEEi0mEzOA6rnV5J26RwTUTcaSXzczCsNRfE8kQwCyqANzF2w4UXMDiLpPj0Pi09lrJUknHLhyp", - "AQWjEGmluL1HxMXYw9M0V2UVFByxoxstd9iPzynkKrHXlpFj07b7a00fTg55Jg7X88moxDessVkD3ZSg", - "Gu8RMeQ2dN9Aw9hCVrla8DxBoxaSDHKzNxyFxjK8op54fqp0OLyL8sXF+zy7uPjAXmNfsp+BXcJ2Tre7", - "LF1zuYI25B7yqbWM4RrSOlT1PTIe5Oy4uGIX+667M52USuVJ49b1rwgG6r9P90uRXkLGUE+QMepOpS+6", - "O4STsAfI4rq5RNmst97OLUuQkD2cMXYiGek2F1voWSC9yeUXZtf81zRrVtN9LpeMFjm7kHH33d4Gf6JM", - "eTC7JcmmR33iVBbI7onMtRwRJ76hywwEF5XPnRHDMxoZHDmDEzZgKovFIafad5QzxDu7LDJyQtpTRdeL", - "QlDiUNBtiprT3+UOvVhhZoydk+5AL0LDFVQ8p6wI7YOpQrNCoDOq6zQFyI4vZNLBJFWFm/hB+1+rli7q", - "o6OnwI4e9sdog+ajc5isDPTHfsWOpraJyMW+YheTi8kAUgWFuoLMOo0hX9tRe8H+lwbuhfxpoJhZwbfW", - "3fSyyHS9XIpUWKLnCvX6SvWsQKmoBSpED9Bp00yYKR1lRFGynu2+tAIYt1ruI64RgYp2Mx6lqO38DV6X", - "dzSDa57iKjkpmS3bIKM0fDY0PowqkxBANPy6Y0YXGNcdPX5HuRvqc+tl78bvvOdnd8gRsOtsvy09IEYU", - "g0PE/4SVCndduFwdn9CRC20GSDqHn25FGoaMHDoz9r9UzVJO8lvWBhpfS1XkwJBjizPQGevndJZaSyHI", - "oQAbBqGWR4/6C3/0yO250GwJG5/ghh375Hj0yAqB0uaTJaDHmtenEQOKgs94mkaSktdcr2d7A9EE96D4", - "cwD69JWfkIRJazpibqYTdIHz7T0IvAXEKnD2nu4Eg7RtVcswmc7tn95qA8UwommH/nPEEn3nPbfBSatk", - "LiQkhZKwjeaPCwk/UmP0nCYWGRlMwjo2tu/ZdvDvodWd55Dd/FT60m4HLPG2Se27h83vw+0Fs8M0QrIy", - "IS8ZZ2kuKFCopDZVnZoLySlw0TODemzhwzHjoayXvks8dhYJbTlQF5JrpGETzoheciwhEqj8FsBHtHS9", - "WoHumUVsCXAhXS8hWS2FobnIqkzshpVQ0W3UzPZES2DJc4q8/Q6VYovadFUvZTtZy8ZG1nEappYXkhuW", - "A9eG/Sjk+TWB8x6O5xkJZqOqy4YKIx4aSNBCJ/ELu+9s6/dcr/3ysaNXNm6wDR4j/DYlamugk079vx/8", - "7fj9SfIPnvx+lLz4r/MPH5/dPHw0+PHJzVdf/Z/uT09vvnr4tz/HdsrjHsvFcZifvnJmyekrOnvaoPoA", - "988WFC6ETKJMhu5CISSldPZ4iz3AE9Qz0MM2PO92/UKaa4mMdMVzkaELfBd26Ku4gSxa6ehxTWcjejE+", - "v9YPMXdnpZKSp5d0Dz5ZCbOuF7NUFXNvjs1XqjHN5hmHQklqy+a8FHN0b+dXj/ccjZ+gr1hEXVG2m/X5", - "gzSliFnqbp46HhJCtK81bLofegivYCmkwPbjC5lxw+cLrkWq57WG6muec5nCbKXYMXMgX3HDybHuhenG", - "HlRR0MNhU9aLXKTsMjzfWn4fizZdXLxHql9cfBjcGg1PIzdVPIJHEyQbYdaqNokLdY47520AgyDbYNeu", - "WafMwbbb7EKpDv5IVLEsdRKEmeLLL8sclx+cmZrRIEpSYtqoymsWVDcuUID7+0a5e7OKb3wKeY3O8C8F", - "L98LaT6wxDm1J2VJMSwKIv3iBBi17raEwwNRLYotsJjzQgu3VsqtE9cI6Jkd5SOzOk45bCLSUR8UtTbQ", - "dlc6IajvVY6be2cyBTCi1KnNOkGZiq5KI2uRPAQP//gKFYy/6EJfFJnPPURZAEvXkF5CRtF8CrxNO8P9", - "/bJT115khbZvR2x+GiU4k4+1AFaXGXcHGpfbfqapBmN8eu07uITtuWrzo2+TWnoznbhIeYI8MyYgJdIj", - "0Kxq2RUXH23vbb67sKBodlkyGzC2qX+eLY4bvvBjxgXIqvt7EJ4YUzRk2MHvJa8ihLDMP0KCOywU4X0S", - "60fD07wyIhWlXf9hAe+3nTEIZJ9Sj6pxtexr64EyjWpv2zlZcB1X3IAtuB8oQ/1UDj+TDVfYmydG748d", - "4y5yCK5qtJNsXpEF4ZdtH1SOoRbnEqhke5p6NLoUCY/ttbvrE1ftDR/d8R5ywO296UEu8pfzohvTFThv", - "Dld8NLw+mvh/Gty4B+/JmrR+r9j6wjBtnnjYp90+/d/n/PtE/8n0Vkn704lLrIpth5J0umeQw4q7aDKl", - "bDlGcah9oYMNQjx+Wi7R52dJ7PKea61SYS8YW13u5gA0/h4xZqMV7GAIMTYO0KYwHAFmb1Qom3J1GyQl", - "CIrbcQ+bAnjB37A/jNW+sXdm5V7zb6g7WiGatm9g7DYOQyrTSVQljVnmnV7MdlnAwD+IsSiqpmGQYRjK", - "0JADHcdJR7Mml7HQE1oVQGx45ocF5jp7IJZ4yD8MorEVrNChbZ1AlFYf1fi8jviVMpAsRaVNQv5ndHnY", - "6VtNxuC32DWufjqkYvaRrsji2oemvYRtkom8ju+2m/eHVzjtm8Zv0fXiErZ0yABP12xBj8rxFOpMj312", - "TG0TWHYu+LVd8Gt+b+s9jJewK05cKWV6c/xBuKqnT3YJU4QBY8wx3LVRku5QL8EV/1C3BMkFNhGBkhZm", - "u7z1gTDdOk1iVPNaSNG1BIbuzlXYbBqbMBO8yR4mKI/IAC9LkV33fGcLNc7jNMVtDHVr8Q+oQLvrgO2h", - "QOAnx/L1KvC+vt3S4My0r+sHuUv7KdPPmAoUQjiV0L42zJBQyNqU4rKPVufA8x9g+3fsS8uZ3Ewnn+by", - "x2jtIO6h9dtme6N0psCsdQE7kbNbkpyXZaWueJ64NyBjrFmpK8ea1N0/GfnMqi7ufp9/c/L6rUOfUsKA", - "Vy4TateqqF/5h1kVesSxdKjzIDJC1qr3na0hFmx+83AvDKb47LWOLYdazDGXFa/mgAtF0QVXlvH7ob2h", - "kjDj7U6S2UmZ+9TIXJg/d68iP5CwOIe2O7xHL4Rz7agGUNiCF5op2c8aQDOOvExil4JvcRdtYHaoIGRd", - "JCgCic5FGg8dyIVGKZJ1Qc8jtgYYdR4xCBFiLUbC57IWASzspg+4fukhGcwRJSaFdXbQbqFcpbJait9q", - "YCIDabCpcllEHWFB2fCJscMjLZ6E6wC7PNwG/Kec8whq7IQnJHYf8mGUN5J67Z0+v9AmPI0/BMG5W1zS", - "hDMOjqUdFyyOPxw32+vjdTdaGxYWG+ogZAxbhGJ/VTMfOlhbREfmiFYpG9XYJ+PampKrD9fTrVomdEOF", - "bBPeeK5VBEwtN1zaokM4ztLQjdZg/XYctVEVvRDSEL32FTpZVup3iHuTS9yoSGKTIyWZbDR6Fnl50Vei", - "TWSkLSfn6RviMcraY9ZU0Mi6l2gjEk5cHoSvKVPTB5m4tGxtCyR17kPjwhHmMMwt/FY4HM6DvI+cbxY8", - "VhMAjRrE6aS9KOmEw4xifrDfBd0kKDveC+5cmr7CPqspoWqzD4fPIu9ooPyxWD6DVBQ8j0dHM6J+92Fl", - "JlbCVpmqNQRljBwgW57PcpErBWWvolrSnC7Z0TQolOZ2IxNXQotFDtTjse2x4JpOrSbk2QzB5YE0a03d", - "nxzQfV3LrILMrLUlrFasMSLtiwEff16A2QBIdkT9Hr9gDyjyrsUVPEQqOltkcvz4BeU52D+OYoedKye3", - "S69kpFj+h1MscT6mqwcLAw8pB3UWfeJla4COq7Ad0mSHHiJL1NNpvf2yVHDJVxC/US324GTH0m5S4K5H", - "F5nZAnbaVGrLhInPD4ajfhrJdUL1Z9FwCegFCpBRTKsC+amtUWQn9eBsNTxXH8Tj5RvpmqP0Dwl6Tuvn", - "DdLaszy2arqMesML6JJ1yrh9CUlvIdwLWqcQZyOFGaC6ik9SjWywPzfdWPZAKpkUKDvZwzaLLuC/aF0C", - "ZXgendZ43dXPXNkN+lBTC6Eko4StO4TlgU66M4nrKr5OXuNUP7977Q6GQlWxIgOtNnSHRAWmEnAVldh+", - "NlhjmTTHhad8zEDxpRh+q0Gb2MMbarD5M+S34RloyzAwkBmdIDNmH6og2p2nBqS5RVHnNm0dshVUzqmv", - "y1zxbMoQzvk3J6+ZnVW7x470QILKQKzso6eGRJEwUvB8/zavwMbSbQ6HszsPAVetDb2p1YYXZSw9EXuc", - "+w6UA3nFRe6vtEmlhdSZsVf2NNFeV9lJ2sd+rJnO8W++UvTKmxvD0zWp6Y5Ss0IS9f0Orl/iM3x1UA+w", - "Ka3WvIq379eM8iVMbAWTKVN4lm6EtjVN4Qq6GZFNerAzE3yGZHd5VS2l5ZS4ztuRvn4Xsnvk7GWRD3NE", - "MesR/paqS6u6SuG25VzOaFT0MUy/NsygEKCE7PxaNgW3fK3qlEslRUpPUYIqqg3Krj7qIXG4A17t9F0w", - "L+JOQiPCFa1I01xHOyqO1qjxitARbhiECFpxUy132D8NFeJE52IFRjvNBtnUVx1yvoGQGlyVAyqVG+hJ", - "dPH6d1LRcHn7rvqWbEQpZSNH4LfYRsefcGkgl0LSK0NHNpdxYq13Kt9o0GUQhq0UaLee7isa/R7HzM6v", - "5Sli/GHmyz0SDBuWxGXbOPgQ1ImPirsoNPZ9iX0ZhSDbnzvpa3bSk7J0k8Y0gW52OFY3aZTAkchq4kNb", - "AXEb+CG0Hey28zqLzlNkNLiiYDiUdA4PGGPkrfI36ChZjrJPHu01cjSHXsgIGq+FhLYYaeSASKNHAm0M", - "yevIOJ1W3KTrg3XaOfCcou8xhaaNC0d8KqjeBhNJaI1+jvFtbKtnjSiOpkOb4c7ltqmBitwdGBMvqfiy", - "I+SwFhZZVc6IyihRqFcdK6Y4UHH7enPdA2AoBkObyA43FbeSc5uTaCyxORMaTdxikUdSI141jUGFOMrB", - "Wmzp39hL0fEVuMuaO1c2oIG3ti93VxnIce8TLVZ33JV2/D1uS08Gwj2Kcf83qFbCh2uDR79W8TT1Eela", - "WPn6nuRUNMnOXZ4lRRejQ1CScbcjNF5ccUqqcSQ55F37tI9b7WvjTWMpIuloRhM3Ll3RcLar3IetfBiD", - "YO+2bMVF+xWEqLM5dp9lr7OweTD6MLthYIUR7J0E9RelQ4R+8JkQrOTCBVNbERlS1uVMDbPYDsmmaDe4", - "vwiXiURAYiu5Y+LQQbI3pFJEsMPr5j3sedkhqX1h0LMkVQX3TNrgCL0laYcX6Ycuj9ZBHFNrGK7z4A3o", - "0HaE9ocQvtULQ+KOi7NZHCLO8URtHE76xBLEPyUYapPPpg06BVvdvLFd//torTv7logbtgHGpVQkUS7q", - "xjgrVAY5067GRg4rnm7d6z99IVMuWSYqoEIVoqCaa5zpDV+toKJno7ZMqo9NELTIbtUiz/axjYPxNfWN", - "vMb9d76nHQqxRfZW5kR/a2mhu9+PNtP8q96MpqoobGigQ/7oy8nmORYFXQj9tk7grtjhouLSeiIDChGU", - "4EsNkTpday4l5NHR9m7i38QhBf9VjeBcCBlv6rOAJUyPDO2auyv0U3r4kVIK04mGtK6E2VL+kPdMxD+j", - "udHfNfLrqsw3t7DuEtB++MSFx1tpb79V8Z2ydZ8LdJfIdTBU/eSba16UOTg9+tUXi7/A078+y46ePv7L", - "4q9Hz49SePb8xdERf/GMP37x9DE8+evzZ0fwePnli8WT7MmzJ4tnT559+fxF+vTZ48WzL1/85Qv/oQiL", - "aPsRhv9J5QSSk7enyTki224UL8UPsLUvopE7fckHnpLmhoKLfHLsf/pvXk5QgIJv27lfJ+62YbI2ptTH", - "8/lms5mFQ+YrqseXGFWn67mfZ1hs5u1pE9C3SQckSzZWi4JO54UwOWWaUNu7b87O2cnb01mrDibHk6PZ", - "0ewxVQApQfJSTI4nT+kn4vo17ft8DTw3KBk308m8AFOJVLu/nAqfuWoX+NPVk7mPAM4/uqv1m11t3dwG", - "92AlGBC8eJx/7JROzEK49B5w/tHnfQRNtg7v/CMFGIPfXSHN+ce2su2N5e4cYpEeX+Gr7U6Vu6jovra/", - "IkP7u0mhu9WFm905zXBXcNTLpspv+M3R9/+ffqHvQ++DJU+Ojv7ziQUqk/rslpTY6dd04gCReb/mGfN3", - "jDT3488396mkVySoqJhVxDfTyfPPufpTiaLAc0Y9g0yTIUv8LC+l2kjfE0/Nuih4tfXirTvKwtf0Jt3M", - "V5oqDVbiihuYfKBSlrFL3RGlQ9+yuLXSoQ90/EfpfC6l88f+csl/lM4fTemcWaVwuNJxhpBN9pjbimit", - "feTfLQ4f83UtuzHN5Qx99oCiyhI2D13CiAUbeRjaXM6rzEaQfHEfn9rkZp0NNNs7B7TzBvkH2Op9au58", - "DeyX9hvtv1ACJl3VTJmq2C88z4Pf6FOb3oSdjXzwvXkseOjX3m9upjG0lgA+HZTSPl1RT1T3l+CflVoa", - "dK5zhxkQbX21JYx+9NWWoQo1m2PBx0dHR7GXFX2cXbTLYkzptxuV5HAF+XCrx5DovS7d9YnE0Q9VDB8F", - "h15nhOv8F4Wbd8KjX4zsvnS9DXavlPzCsA0XrrZ4UFnGfg6kEMZ/TNWmVLkUvubsiH+AM0GQu7/P+6lH", - "3B+vSOfNDmWn17XJ1EaOKy5638NzlyBLKauNs20U8wAaTTVj/it4+dZ/3pVxSu5Stel+ddkXjOjVIm5K", - "Gq2EpAlIymkWmwnOgzxL96WIoRI8c5i9sR/W6Om96McnLY5xuY8J/afy0uEGyM499IVHOn/PURTQ2LNf", - "6UmIckO33wDP5y7dp/ervZQPfuzWIY78Om8eXUUb+8GMWOv8o7l28Yog8EZb1oTc3n9AylM6r9vNNo50", - "PJ/TzfdaaTOfoObpxpjCxg8NUT96FvDEvflw838DAAD//9ssXxsJhQAA", + "H4sIAAAAAAAC/+x9bXPcNpLwX8Ezu1V+eYYz8lt2rarUnmI7iS6O47KU3N1avgRD9swgIgGGACVNfPrv", + "V90ASJAEZ0ayznep3U/SkECj0ehudDcazY+TVBWlkiCNnhx+nJS84gUYqOgXT1NVS5OIDH9loNNKlEYo", + "OTn075g2lZCryXQi8GnJzXoynUheQNsG+08nFfxWiwqyyaGpaphOdLqGgiNgsymxdQPpKlmpxIE4siCO", + "X06ut7zgWVaB1kMsf5D5hgmZ5nUGzFRcap7iK80uhVkzsxaauc5MSKYkMLVkZt1pzJYC8kzP/CR/q6Ha", + "BLN0g49P6bpFMalUDkM8X6hiISR4rKBBqlkQZhTLYEmN1twwHAFx9Q2NYhp4la7ZUlU7ULVIhPiCrIvJ", + "4fuJBplBRauVgrigf5cVwO+QGF6twEw+TGOTWxqoEiOKyNSOHfUr0HVuNKO2NMeVuADJsNeMfV9rwxbA", + "uGTvvn7Bnjx58hwnUnBjIHNMNjqrdvRwTrb75HCScQP+9ZDXeL5SFZdZ0rR/9/ULGv/ETXDfVlxriAvL", + "Eb5hxy/HJuA7RlhISAMrWocO92OPiFC0jxewVBXsuSa28Z0uSjj+/+qqpNyk61IJaSLrwugts6+jOizo", + "vk2HNQh02pdIqQqBvj9Inn/4+Gj66OD6T++Pkr+7n8+eXO85/RcN3B0UiDZM66oCmW6SVQWcpGXN5ZAe", + "7xw/6LWq84yt+QUtPi9I1bu+DPta1XnB8xr5RKSVOspXSjPu2CiDJa9zw/zArJY5qimE5ridCc3KSl2I", + "DLIpat/LtUjXLOXagqB27FLkOfJgrSEb47X47LYI03VIEsTrVvSgCf3fJUY7rx2UgCvSBkmaKw2JUTu2", + "J7/jcJmxcENp9yp9s82Kna6B0eD4wm62RDuJPJ3nG2ZoXTPGNePMb01TJpZso2p2SYuTi3Pq72aDVCsY", + "Eo0Wp7OPovCOkW9AjAjxFkrlwCURz8vdkGRyKVZ1BZpdrsGs3Z5XgS6V1MDU4ldIDS77v5788Iapin0P", + "WvMVvOXpOQOZqmx8jd2gsR38V61wwQu9Knl6Ht+uc1GICMrf8ytR1AWTdbGACtfL7w9GsQpMXckxhCzE", + "HXxW8KvhoKdVLVNa3HbYjqGGrCR0mfPNjB0vWcGvvjyYOnQ043nOSpCZkCtmruSokYZj70YvqVQtsz1s", + "GIMLFuyauoRULAVkrIGyBRM3zC58hLwZPq1lFaDjgYyi04yyAx0JVxGeQdHFN6zkKwhYZsZ+dJqL3hp1", + "DrJRcGyxoVdlBRdC1brpNIIjDb3dvJbKQFJWsBQRHjtx5EDtYds49Vo4AydV0nAhIUPNS0grA1YTjeIU", + "DLjdmRlu0Quu4YunYxt4+3bP1V+q/qpvXfG9VpsaJVYkI/sivnUCGzebOv33cP7CsbVYJfbxYCHF6hS3", + "kqXIaZv5FdfPk6HWpAQ6hPAbjxYryU1dweGZfIi/WMJODJcZrzJ8UthH39e5ESdihY9y++i1Won0RKxG", + "iNngGvWmqFth/yC8uDo2V1Gn4bVS53UZTijteKWLDTt+ObbIFuZNGfOocWVDr+L0ynsaN+1hrpqFHEFy", + "lHYlx4bnsKkAseXpkv5cLYmf+LL6Hf+UZR6jKTKw22gpKOCCBe/cM3yEIg/WJ0AoIuVI1Dltn4cfA4T+", + "XMFycjj507yNlMztWz13cHHE6+nkqIVz9yO1Pe38eo5M+5oJaVeHmk6tT3j3+CDUKCZkqPZw+CpX6fmt", + "cCgrVUJlhF3HBcIZSgqBZ2vgGVQs44bPWqfK2lkj/E4dv6V+5CVBFdnifqB/eM7wNUohN958Q9NVaDTi", + "VBBoytDis/uIHQkbkCWqWGGNPIbG2Y2wfNEObhV0o1HfO7J86EOLrM4ra1cy6uEngVNvvcajhapuxy89", + "RpCs9YUZR6iN9Ysz764sNa3LxNEnYk/bBj1AbfhxqFZDCvXBx2jVocKJ4f8DVNAI9S6o0AV011RQRSly", + "uAN5XXO9Hk4CDZwnj9nJt0fPHj3++fGzL3CHLiu1qnjBFhsDmt13+wrTZpPDg+HMSMHXuYlD/+Kp96C6", + "cHdSiBBuYO8jUaeAmsFSjNl4AWL3stpUtbwDEkJVqSpi8xLrGJWqPLmASgsVCV+8dS2Ya4F6yNrdvecW", + "W3bJNcOxyR2rZQbVLEZ59LNoSzdQ6F0bhQV9eiVb2jiAvKr4ZrACdr6R2blx91mTLvG9da9ZCVViriTL", + "YFGvwj2KLStVMM4y6kgK8Y3K4MRwU+s70AItsBYZXIgQBb5QtWGcSZWhQGPjuH4YiWVSEIViPyZUOWZt", + "958FoHWc8nq1NgzNShVb2rZjwlO7KAntFXrE9Wt8dtvKDmfjZHkFPNuwBYBkauH8K+f50SQ5hWWMP3Fx", + "2qlFq/EJOniVlUpBa8gSd7y0EzV/VEWLbLaQifAmfJtBmFZsyatb4mqU4fkOPKnNEFvdWhPOJx1ivd/w", + "29avP3i4irxCF9MyAZouKNw5GBgj4U6a1OXIcYTb1U5FgSLBJJdKQ6pkpqPAcq5NsksUsFFn68VlDbgv", + "xv0EeMTpfs21sW6vkBmZYVaEaRzqQ0OMIzyqpRHyT15BD2GnqHukrnWjrXVdlqoykMXmIOFqy1hv4KoZ", + "Sy0D2M2WYBSrNeyCPEalAL4jlp2JJRA3Lu7SxIWGk6MQN+rWTZSUHSRaQmxD5MS3CqgbhmRHEEGbvelJ", + "jCN0j3OaOPB0oo0qS9RJJqll02+MTCe29ZH5sW07ZC5uWl2ZKcDRjcfJYX5pKWuD8WuO9hJBZgU/R31P", + "1o/1z4c4ozAmWsgUkm2cj2J5gq1CEdghpCOGpzvuC0brCUePf6NMN8oEO1ZhbMIjVvBbG1U+bSMud2AI", + "vATDRa6bzb4JXbejUJS7n4GAllkFKUiTb5BXl6Iq7EER7RHaP7OmROZGsUcirfjJjFVwyavMtxh6IMFk", + "EiEzuIprV96JN2RwxUQc6WUzsjAs9cc4MgQwiwq6OxjbgoILNNxmcOwaH9Ye+1gq6diBIL1AAShEWilu", + "z/lwMnaTNM1RVgUFR+zoxMlt6uNjCrlK7LFiZHu07/2xow/3hjwTh+v5ZFSyG9a4XAOdZKC67hEx5DZ0", + "00DD2ERWuVrwPEHjFZIMcrMzjIRGMbyklrhPqnTYvYvy2dn7PDs7+8BeY1uyk4Gdw2ZOp68sXXO5gjYk", + "HvKptYDhCtI6VOk9Mu7l1Li4Xxf7rlsznZRK5UnjvvVD+AM136f7uUjPIWOoJ8jodLvPve4K4SDsPrK4", + "bg45Ltcbb8+WJUjIHswYO5IMitJsXKygZ2n0Bpf3zLbxr2jUrKbzVi4ZTXJ2JuNuuj2t/USZ8mC2S5JN", + "X/rEoSyQ7QOZKzkiTvySDhsQXFQ+t0b6TqhnsOUMdtKAqSwW+/jD31BOD++sssjI2Wh3FV0vCkGJPUGz", + "KWpOf9Y69FaFmTF2SroDvQUNF1DxnLIWtA+CCs0KgU6nrtMUIDs8k0kHk1QVbuD77b9WLZ3VBwdPgB08", + "6PfRBs1E5xhZGej3/ZIdTO0rIhf7kp1NziYDSBUU6gIy6xyGfG177QT7/xq4Z/KHgWJmBd9Yt9LLItP1", + "cilSYYmeK9TrK9Wz9qSiN1AheoDOmWbCTGkrI4qSlWzXpRXASdRquYv4RQQq2se4laK28ydsXd7RDK54", + "irPkpGQ27BIZpeGzofFhVJmEAKLh1C0juoC27ujxW8rdUJ9bb3o7fqc9f7pDjoBdZ7tt5gExohjsI/5H", + "rFS46sLl0viEi1xoM0DSOfZ0mtEwZGTTmbH/UDVLOclvWRtofCpVkaNCDiyOQHusH9NZai2FIIcCbLiD", + "3jx82J/4w4duzYVmS7j0CWjYsE+Ohw+tEChtPlkCeqx5dRwxoCjIjLtpJGl4zfV6tjPgTHD3ijMHoI9f", + "+gFJmLSmLeZ6OkFXN9/cgcBbQKwCZ+/pTtBH27dqGSa7ufXTG22gGEYubdefRyzRd95DG+y0SuZCQlIo", + "CZtofreQ8D29jO7TxCIjnUlYx/r2PdgO/j20uuPss5qfSl9a7YAl3japd3ew+H24vaB1mOZHVibkJeMs", + "zQUFBJXUpqpTcyY5BSh6ZlCPLXzYZTxk9cI3icfIIiEsB+pMco00bMIW0cOMJUQCkl8D+MiVrlcr0D2z", + "iC0BzqRrJSSrpTA0FlmViV2wEio6dZrZlmgJLHlOEbbfoVJsUZuu6qVsJGvZ2Ag6DsPU8kxyw3Lg2rDv", + "hTy9InDew/E8I8Fcquq8ocKIhwYStNBJ/GDuG/v2W67XfvrY0Csb19kGiRF+m7K0MdBJd/7P+387fH+U", + "/J0nvx8kz////MPHp9cPHg4ePr7+8sv/6j56cv3lg7/9ObZSHvdYrozD/PilM0uOX9Le0wbPB7h/tuBv", + "IWQSZTJ0FwohKeWyx1vsPu6gnoEetGF4t+pn0lxJZKQLnosMXeDbsENfxQ1k0UpHj2s6C9GL5fm5foi5", + "OyuVlDw9p3PtyUqYdb2YpaqYe3NsvlKNaTbPOBRK0rtszksxR/d2fvFox9b4CfqKRdTV9XTitI6+8wwa", + "Bzg2of6YTRTd/zaK3fvm1Smbu5XS92zinAUdZDxFLGh3GNZx5nDy9uKHzRxEZ+YlLIUU+P7wTGbc8PmC", + "a5Hqea2h+ornXKYwWyl2yBzIl9xwigH0Iopjd7MoPuOwKetFLlJ2Hm7FrWiOBcbOzt4jg5ydfRgcZA03", + "TjdUPNhIAySXwqxVbRIXlR2PI7SxFoJs43LbRp0yB9typIv6OvgjAdCy1EkQEYtPvyxznH7AhppRJ8qD", + "YtqoyitB1IwupoHr+0a5o7yKX/ps9Br99l8KXr4X0nxgifO/j8qSwm0U7/rF6RrkyU0J+8fMWhRbYDE/", + "iyZuDaob58YR0BPbyweRdZxy+IpIR21QK7QxwdvSCUF9q3Jc3FuTKYARpU5t1gnKVHRWGlmL5CG4Q8hX", + "qAv92Ru6zch87k7LAli6hvQcMjp4oBjhtNPdH3m7ncWLrND2GopNgaNcaXIHF8DqMuNu7+Vy009a1WCM", + "z9R9B+ewOVVtqvVNslSvpxMX1E+QZ8YEpER6BJuAWnbFxR8M9Bbfna1Q4L0smY1t2+xCzxaHDV/4PuMC", + "ZHemOxCeGFM0ZNjC7yWvIoSwzD9CgltMFOF9EutHI+m8MiIVpZ3/frH5t50+CGSXUo+qcbXsa+uBMo1q", + "b9s4WXAdV9yAb3A9UIb62SV+JBtZsYdkjK4yO8Zd5BCcKmkn2bwiY8dP297NHEMtziVQyXY39Wh0KRJu", + "22t3LCku2sNIOnbeZ4PbeSiFXOTzBUQ3/Cxw3Bwu+OhJwOgdguMgCSC4mtbcEPCKrS8M0+a2iL0l7m8S", + "+OsD/s7AZHqj/P/pxOV6xZZDSdrdM8hhxV3gm7LIHKM41O7pYIEQjx+Wy1xIYEksn4BrrVJhz0JbXe7G", + "ADT+HjJmAytsbwgxNg7QpoghAWZvVCibcnUTJCUICjFyD5tijcFv2B1xa6/rO7Nyp/k31B2tEE3b6zR2", + "GYfRn+kkqpLGLPNOK2abLGDgysRYFFXTMB4yjLpoyIG246SjWZPzWJQMrQogNjzx3QJznd0XS9zkHwSB", + "4wpW6Hu3/ipKqw/AfN6YwYUykCxFpU1CrnJ0etjoa03G4NfYNK5+OqRi9r6vyOLah4Y9h02SibyOr7Yb", + "97uXOOybxm/R9eIcNrTJAE/XbEH303EX6gyPbbYMbXNqtk74tZ3wa35n892Pl7ApDlwpZXpj/EG4qqdP", + "tglThAFjzDFctVGSblEvQTbCULcEeRA2Z4LyK2bbvPWBMN04o2NU81pI0bkEhu7WWdjEH5vbE1zvHuZM", + "j8gAL0uRXfV8Zws1zuM0xE0MdWvxD6hAq+uA7aBA4CfHUggr8L6+XdJgz7QX9QdpVrsp00/uChRCOJTQ", + "vszMkFDI2pSNs4tWp8Dz72DzE7al6Uyup5NPc/ljtHYQd9D6bbO8UTpTDNm6gJ3I2Q1JzsuyUhc8T9y1", + "lDHWrNSFY01q7m+xfGZVF3e/T18dvX7r0KfsNeCVS9raNitqV/5hZoUecSxz6zSIjJC16n1na4gFi9/c", + "DQyDKT7RrmPLoRZzzGXFq9ngQlF0wZVl/ChrZ6gkTM67lWR2svs+NTIXpvrdqcgPJCzOoe0K79AL4Vhb", + "CgsUtnaGZkr2ExzQjCMvk9il4BtcRRuYHSoIWRcJikCic5HGQwdyoVGKZF3QjY2NAUaNRwxChFiLkfC5", + "rEUAC5vpPU6KekgGY0SJSWGdLbRbKFf0rJbitxqYyEAafFW5hKeOsKBs+Bze4ZYWzxd2gF3KcAP+U/Z5", + "BDW2wxMS2zf5MMobyRL3Tp+faBOexgdBcO4GhzThiINtacsBi+MPx832pHvdjdaGNcqGOggZw9az2F0g", + "zYcO1hbRkTGiBc9GNfbRuLamPPD99XSrlgndUCHb3DyeaxUBU8tLLm39Iuxnaeh6a7B+O/a6VBVdWtIQ", + "PaEWOllW6neIe5NLXKhIDpYjJZls1HsWuQzSV6JNZKStTOfpG+Ixytpj1lTwknUP0UYknLg8CF9TUqkP", + "MnFp2drWWuoc3caFI0y3mFv4rXA4nAcpKjm/XPBY2QE0ahCno/agpBMOM4r5zn4VdJNL7XgvOHNp2gp7", + "06eEqk2UHN7UvKWB8sdi+QxSUfA8Hh3NiPrdu56ZWAlbsKrWEFREcoBspT/LRa6qlD2KaklzvGQH06Dm", + "mluNTFwILRY5UItHtsWCa9q1mpBn0wWnB9KsNTV/vEfzdS2zCjKz1pawWrHGiLSXG3z8eQHmEkCyA2r3", + "6Dm7T5F3LS7gAVLR2SKTw0fPKSXD/jiIbXauMt02vZKRYvk3p1jifExHDxYGblIO6ix668yWEx1XYVuk", + "yXbdR5aopdN6u2Wp4JKvIH6iWuzAyfal1aTAXY8uMrO18LSp1IYJEx8fDEf9NJKWherPouFy5QsUIKOY", + "VgXyU1vuyA7qwdnCeq4EicfLv6RjjtLfeeg5rZ83SGv38tis6TDqDS+gS9Yp4/ZyJl3bcJd6nUKcjdSK", + "gOoiPkg1ssB+33R92X2pZFKg7GQP2oS/gP+ipRKU4Xl0WON1Vz9zZTvofU0thJKMErbuEJYHOunWJK6r", + "+Dx5jUP9+O612xgKVcXqHrTa0G0SFZhKwEVUYvuJa41l0mwXnvIxA+WrWuTZT226aa/EUMVluo7GPxfY", + "8ee2JlpDdkv16BW8NZcS8ig4K8s/e5mPaKVf1b7jFELu2bZfOshOtze5FvEumh4pPyCSV5gcBwip2s2/", + "axJH8pXKGI3TXrJuGWF4Ryooo/JbDdrE7nPRC5vrRD422iu2igcDmdFuP2P2/hPi0rnBQrusKOrc3oaA", + "bAWVC8DUZa54NmUI5/TV0WtmR9XuDi3du6EqIit7l64zi55vFVR/uMnlwrHUqP3hbM8ZwVlrQ1eyteFF", + "Gct6xRanvgGl1l5wkfv0A9p+QurM2Eu782u/r9hB2jukrBnO6RriCfzHGJ6uaUvtbEDjLL9/+RvPlToo", + "A9lU1GuKKthrkUb5Cji2AM6UKbR7LoW2pWzhArqJtk3WuTPpfOJtd3pVLaXllPj+tOVWxG3I7pGzB3s+", + "JBXFrEf4G24zWtVVCjetBnRCvaJ3rPqlhQb1HyVkp1eyqb/mS5SnXCopUrrhFBTPbVB2ZXH3iZnucRms", + "7y57EXcSGhGuaEGjJnXAUXG0xJFXhI5ww4BR8BYX1XKH/Wmo/io6gisw2mk2yKa+aJXz44TU4IpkUIXk", + "QE+iO94/P4webbTX9W/IRpT+N2KufI3vyFQRLmXnXEi6vOrI5rKDrKdFVTsNunfCsJUC7ebTvZyl32Of", + "2emVPEaMP8x8lU+CYUPIOG17ZjEEdeRPMNyJAbZ9gW0ZhYvbx51UQzvoUVm6QWOaQDcrHCu7NUrgSBQ8", + "8WHIgLgN/BDaFnbbevRI+ykyGlzQwQWUtA8PGGPkCvwrdGotR9mbtPbIP3o1Q8gIGq+FhLYGbWSDSKNb", + "Ai0MyetIP51W3FgTcC+ddgo8p5OSmELTxoWOPhVUb4GJJDRHP8b4MrbF10YUR9OgNdy43DSlb5G7A2Pi", + "BdXcdoQcllIjq8oZURkldfWKq8UUBypuX5awuwEMxWBoE9nupuJWcm6yE40loWdCoztSLPJIGsvL5mVQ", + "YJDy5RYb+hu7gDw+A3ewduuCGdTxxvbl9uIVOa59osXqlqvS9r/DZenJQLhGMe5/hWolvLczuEtuFU9z", + "rYaO8JUv90pORZOY3uVZUnRRp62t3LndaR2vwTkl1TiSyPOuvTHKrfa1scGxdJ50NPuMG5daajjbVkXG", + "Fs6MQbDnkLZgp/34RTQwMHb2aI8e8fWg9352w8AKI9hbCeoPtYcIfeezVljJhQt8tyIypKzLbxtmHO6T", + "+dIucH8SLmuMgMRmcsskr71kb0iliGCHqQE72PO8Q1J7G6RnSaoK7pi0wRZ6Q9IOkx72nR7Ngzim1jCc", + "594L0KHtCO33IXyrF4bEHRdns9hHnONJ9did9IkliL/2MdQmn00bdOr9unFjq/7TWPTAesgjgaoeTWuR", + "Z7sWtxN2bK8zU2Dt58UXTzvRu895ofpneyA/FDd3t/QmG39/EYgwkbl2Bg+GCgKKe8QSXbdI5JBqQaV1", + "JcyGcne8pSl+juYlfwPSVT12ReSbE1B3AGe/X+JC06umdfvJiW+ULQNdoPlLpqChIimvrnhR5uDk4st7", + "i7/Ak78+zQ6ePPrL4q8Hzw5SePrs+cEBf/6UP3r+5BE8/uuzpwfwaPnF88Xj7PHTx4unj59+8ex5+uTp", + "o8XTL57/5Z7/3oNFtP2Wwr9T1YHk6O1xcorItjThpfgONvaeMbKxv8HMU5JEKLjIJ4f+0b94CZulqgg+", + "UeeeTlykf7I2ptSH8/nl5eUs7DJfUdm+xKg6Xc/9OMOaNG+PmwCtPfCnFbWxN2QFWlTHCkf07t2rk1N2", + "9PZ41jLM5HByMDuYPaJCISVIXorJ4eQJPSLpWdO6z9fAc4Me3fV0Mi/AVCLV7pe+5KsVVDN3eRsfXTye", + "+4jO/KM71r7e9q6bV+AuiwQdgtuG84+dCotZCJfu4s0/+pyL4JUtyzv/SAGj4Lmrqzn/2Ba6vbbcnUPM", + "c/eFwNrmVOCLaupr+xQZ2p8LCt0tNtysznGGq4K9XjRFf8NPh77/B/3Q3ofed0ceHxz8g31B4ekNZ7zV", + "Hu34b5E6CV/xjPmzIRr70ecb+1jSTQ1USMwq3Ovp5NnnnP2xRJbnOaOWQTbHcOl/lOdSXUrfEnfHuih4", + "tfFirDtKwZfyJh3MV5oKD1biAp3yD1TZMnYYN6Jc6FMVN1Yu9P2NfyqXz6Vc/hgfJvmncvmjKZcTK/z7", + "Kxdn2NjD+LkthNbaO/4O4PBiXNdSG9NQznBn9ynqJ+HygTvQt2Ajlyybw1OVWQ/fF8rxaUJu1NlAg71z", + "QDv3eb+Djd6lzk7XwH5pP53+CyUzUih9ylTFfuF5HjyjL2B6k3Q28h325uLdvh9hv76extBaAvjUSkqh", + "dLU8Ua2fg7+iaWnQOW4bnlC3ZdWWMPotVlt9KtRgjgUfHRwcxFJb+ji7aITFmFJZL1WSwwXkw6UeQ6J3", + "U3PblwtHv0MxvGAbepERrvMf+m3u3I5+yLF7a/Qm2L1U8p5hl1y4kuJBlRb7tY9CGP+NU5vy4tLhmj0i", + "/l3MBEFu/2zup25lf7zanNdblJ1e1yZTl3JccdFdGZ67ZFNK/2ycZ6OYB9BoqhnzH63LN/6rq4xT8o2q", + "TfdjyL74Qq8EcVMeaCUkDUBSTqPYrGoe5Cy6D0EMleCJw+yN/W5GT+9FvwlpcYzLfUzoP5WXhobG1rXy", + "xTo6v+fI8mi82Y/tJEShobtugOdzl3bRe2oPR4OH3TLDkafz5qJS9GU/CBF7O/9oroI4QxvvC+NntFJN", + "5Oz9ByQ4ZcS6RWzDQYfzOR1IrpU28wkqnG6oKHz5oaHxR7/yntbXH67/OwAA//+hhD2Il4QAAA==", } // GetSwagger returns the Swagger specification corresponding to the generated code diff --git a/daemon/algod/api/server/v2/generated/private/types.go b/daemon/algod/api/server/v2/generated/private/types.go index bb3626a7ae..3a44a1a51f 100644 --- a/daemon/algod/api/server/v2/generated/private/types.go +++ b/daemon/algod/api/server/v2/generated/private/types.go @@ -227,6 +227,16 @@ type AssetParams struct { Url *string `json:"url,omitempty"` } +// BuildVersion defines model for BuildVersion. +type BuildVersion struct { + Branch string `json:"branch"` + BuildNumber uint64 `json:"build_number"` + Channel string `json:"channel"` + CommitHash string `json:"commit_hash"` + Major uint64 `json:"major"` + Minor uint64 `json:"minor"` +} + // DryrunRequest defines model for DryrunRequest. type DryrunRequest struct { Accounts []Account `json:"accounts"` @@ -340,22 +350,10 @@ type TealValue struct { // Version defines model for Version. type Version struct { - - // the current algod build version information. - Build VersionBuild `json:"build"` - GenesisHash []byte `json:"genesis-hash"` - GenesisId string `json:"genesis-id"` - Versions []string `json:"versions"` -} - -// VersionBuild defines model for VersionBuild. -type VersionBuild struct { - Branch string `json:"branch"` - BuildNumber uint64 `json:"build-number"` - Channel string `json:"channel"` - CommitHash []byte `json:"commit-hash"` - Major uint64 `json:"major"` - Minor uint64 `json:"minor"` + Build BuildVersion `json:"build"` + GenesisHashB64 []byte `json:"genesis_hash_b64"` + GenesisId string `json:"genesis_id"` + Versions []string `json:"versions"` } // AccountId defines model for account-id. @@ -616,6 +614,9 @@ type TransactionParametersResponse struct { MinFee uint64 `json:"min-fee"` } +// VersionsResponse defines model for VersionsResponse. +type VersionsResponse Version + // RegisterParticipationKeysParams defines parameters for RegisterParticipationKeys. type RegisterParticipationKeysParams struct { diff --git a/daemon/algod/api/server/v2/generated/routes.go b/daemon/algod/api/server/v2/generated/routes.go index b2c2dab120..db41c576ba 100644 --- a/daemon/algod/api/server/v2/generated/routes.go +++ b/daemon/algod/api/server/v2/generated/routes.go @@ -561,163 +561,164 @@ func RegisterHandlers(router interface { // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9/XPbOJLov4LTXdUkOdFyvuY2rpq650nmw2+TTCr27O27OG8PIlsS1iTABUBLmjz/", - "76/QAEiQBCX5I8l4Vj8lFoEG0OhvNBqfRqkoSsGBazU6+jQqqaQFaJD4F01TUXGdsMz8lYFKJSs1E3x0", - "5L8RpSXj89F4xMyvJdWL0XjEaQFNG9N/PJLwj4pJyEZHWlYwHql0AQU1gPW6NK1rSKtkLhIH4tiCOHk1", - "utrwgWaZBKX6s/yF52vCeJpXGRAtKVc0NZ8UWTK9IHrBFHGdCeNEcCBiRvSi1ZjMGOSZOvCL/EcFch2s", - "0g0+vKSrZoqJFDn05/lSFFPGwc8K6knVG0K0IBnMsNGCamJGMHP1DbUgCqhMF2Qm5Jap2kmE8wVeFaOj", - "DyMFPAOJu5UCu8T/ziTAb5BoKuegRx/HscXNNMhEsyKytBOHfQmqyrUi2BbXOGeXwInpdUDeVEqTKRDK", - "yfsfX5KnT5++MAspqNaQOSIbXFUzergm2310NMqoBv+5T2s0nwtJeZbU7d//+BLHP3UL3LUVVQrizHJs", - "vpCTV0ML8B0jJMS4hjnuQ4v6TY8IUzQ/T2EmJOy4J7bxnW5KOP5X3ZWU6nRRCsZ1ZF8IfiX2c1SGBd03", - "ybB6Aq32pcGUNEA/HCYvPn56PH58ePWvH46T/3Z/Pn96tePyX9Zwt2Ag2jCtpASerpO5BIrcsqC8j4/3", - "jh7UQlR5Rhb0EjefFijqXV9i+lrReUnzytAJS6U4zudCEerIKIMZrXJN/MCk4rkRUwaao3bCFCmluGQZ", - "ZGMjfZcLli5ISpUFge3IkuW5ocFKQTZEa/HVbWCmqxAlZl43wgcu6PeLjGZdWzABK5QGSZoLBYkWW9ST", - "1ziUZyRUKI2uUtdTVuRsAQQHNx+sskXccUPTeb4mGvc1I1QRSrxqGhM2I2tRkSVuTs4usL9bjcFaQQzS", - "cHNaetQw7xD6esiIIG8qRA6UI/I83/VRxmdsXklQZLkAvXA6T4IqBVdAxPTvkGqz7f/79Je3REjyBpSi", - "c3hH0wsCPBXZ8B67QWMa/O9KmA0v1Lyk6UVcXeesYJEpv6ErVlQF4VUxBWn2y+sHLYgEXUk+NCELcQud", - "FXTVH/RMVjzFzW2GbRlqhpSYKnO6PiAnM1LQ1XeHYzcdRWiekxJ4xvic6BUfNNLM2Nunl0hR8WwHG0ab", - "DQu0piohZTMGGamhbJiJG2bbfBi/3nwayyqYjgcyOJ16lC3T4bCK0IxhXfOFlHQOAckckF+d5MKvWlwA", - "rwUcma7xUynhkolK1Z0G5ohDbzavudCQlBJmLEJjpw4dRnrYNk68Fs7ASQXXlHHIjOTFSQsNVhINzikY", - "cLMz01fRU6rg22dDCrz5uuPuz0R31zfu+E67jY0Sy5IRvWi+OoaNm02t/js4f+HYis0T+3NvI9n8zKiS", - "GctRzfzd7J9HQ6VQCLQQ4RWPYnNOdSXh6Jw/Mn+RhJxqyjMqM/NLYX96U+WanbK5+Sm3P70Wc5aesvkA", - "Muu5Rr0p7FbYfwy8uDjWq6jT8FqIi6oMF5S2vNLpmpy8GtpkC/O6hHlcu7KhV3G28p7GdXvoVb2RA5Mc", - "xF1JTcMLWEsws6XpDP9ZzZCe6Ez+Zv4pyzyGU0PATtFiUMAFC96738xPhuXB+gQGCkupQeoE1efRp2BC", - "/yZhNjoa/eukiZRM7Fc1cXDtiO3dewBFqdcPDRaOG/h3P4OmZ2wWwWfCuN01bDq2vuLdz8dAjc4EDdjO", - "HL7PRXpxozmUUpQgNbP7OzVw+hyE4MkCaAaSZFTTg8bZsvbXAB9gx5+xH3pPICOq7xf8D82J+Wy4k2pv", - "1hmTlilj3IkgAJUZS9DqFzuSaYAWqiCFNf6IMdquNcuXzeBWcNeS9oNDy8cutMju/GDtTYI9/CLM0htv", - "8ngq5M3opUMInDQ+MqEGam0Vm5W3dxabVmXi8BOxs22DDqAmLNkXtyGGuuB3wVXA2Q12TjX9DNhRBupd", - "YKcN6EthRxQly+EO+HtB1aK/OGMoPX1CTn8+fv74yd+ePP/WaPpSirmkBZmuNSjywOknovQ6h4f9FaOi", - "qHIdh/7tM++JteFuxRxOuIa9C97OwEgSizFi4w5mdq/kWlb8DlAIUgoZsZ2RpLRIRZ5cglRMRMIg71wL", - "4loYuWXt987vdrZkSRUxY6NbV/EM5EEM88ZfQ9NAQ6G2KRYL+mzFG9w4gFRKuu7tgF1vZHVu3F32pI18", - "7yUoUoJM9IqTDKbVPNRpZCZFQSjJsCMK0Lcig1NNdaXuQDo0wJrJmI0Ip0CnotKEEi4yw+imcVxuDMRE", - "MRiDMSQdiiK9sPpqCsbKTmk1X2hizFMR29qmY0JTuykJ6hY14ELWvr9tZYez8bZcAs3WZArAiZg6P815", - "kLhIiuEd7U9unNRqplX7Fq15lVKkoBRkiTum2jo1f+SFm6w3oAnnjfOtByFKkBmVN5yrFprmW+aJbfqz", - "VY314Xzb/qx3G37T/nUHD3eRSuOqWiIwpo5h7hw0DKFwK06qcuBYw2m7M1YYliCccqEgFTxTUWA5VTrZ", - "xgqmUUslm20NqC9G/Qh4wHl/TZW27jPjGZptloVxHOyDQwxPeFBKG8h/8QK6Dzs1soerStXSWlVlKaSG", - "LLYGDqsNY72FVT2WmAWwa5WgBakUbIM8hKUAvkOWXYlFENUuflPHl/qLw1C5ka3rKCpbk2gQsWkip75V", - "gN0wtDswEWPj1z2RcJjqUE4dTx6PlBZlaWSSTipe9xtC06ltfax/bdr2iYvqRlZmAszo2s/JzXxpMWuD", - "+gtq7CWETAp6YeQ9Wj/Wz+/P2TBjohhPIdlE+YYtT02rkAW2MOmAQeqODYPROszRod8o0Q0SwZZdGFrw", - "Na3jdzZqfdZEdO7AQHgFmrJc1UZAHRpvRsEoejfDwVhsElLgOl8bGp4xWdiDKNQdyv9mTYzMjWKPXBq2", - "5BmRsKQy8y36HkuwmITxDFZxqUtbcYsMVoTFJz2rR2aapP6YiIcADqICwB28bZiCC1jcZHDTNT6sPVay", - "WFKxA0f8YBijYKkU1J4jmsVY5anrozIJBTWzwxMtp+yHx2R8nthjy4jatN/9saYPJ4c0E4fr6WSQ42vS", - "WC4AT0qMGO8gMaQ2476BgqGFzHMxpXlijFpIMsj11nCUMZbhFbY0+lOk/e7tKZ+ff8iz8/OP5LVpi/Yz", - "kAtYT/B0l6QLyufQhNxDOrWWMawgrUJR30HjTs6Oiyu2Z992d8ajUog8qd267hFBT/x38X7B0gvIiJET", - "aIw6rfRNe4fMIOSBIXFVH6IsF2tv55YlcMgeHhByzAnKNhdb6FggncH5N3rT+CscNavwPJdygos8OOdx", - "992eBt+SpzyYzZxk06NuOZQFsnkgveID7ESXeJhhwEX5c2PE8BR7Biqnp2EDorKz2EWr/YQ5Q7S1yyxD", - "J6TRKqqaFgwTh4JmYyM5/Vlu34tl+oCQM5QdxotQcAmS5pgVoXwwlSlSMOOMqipNAbKjc560ZpKKwg38", - "oPmvFUvn1eHhUyCHD7t9lDbmo3OYLA90+35HDsf2E6KLfEfOR+ejHiQJhbiEzDqNIV3bXlvB/ksN95z/", - "0hPMpKBr6256XiSqms1YyizSc2Hk+lx0rEAu8AtIMz0wTpsiTI9RlSFG0Xq2+9IwYNxquYu4RgSqsZuN", - "KjXSzp/gtWlHEVjR1KySopBZk6UhlJrO+saHFmUSAoiGXzeM6ALjqiXHb8h3fXluvezN8zvr+NktdATk", - "erDdlu4hIzqDXdj/mJTC7DpzuTo+oSNnSvcm6Rx+PBWpCTKidA7I/xEVSSnyb1lpqH0tIdGBQcfWjIA6", - "1o/pLLUGQ5BDATYMgl8ePeou/NEjt+dMkRksfYKbadhFx6NHlgmE0rfmgA5prk4iBhQGn402jSQlL6ha", - "HGwNRCPcneLPAeiTV35AZCalUMVcjUfGBc7Xd8DwFhCR4Ow91QoGKftVzMJkOrd/aq00FP2Ipu36twFL", - "9L333HqaVvCccUgKwWEdzR9nHN7gx6ieRhIZ6IzMOtS369m25t+ZVnucXXbztvjF3Q5I4l2d2ncHm9+F", - "2wlmh2mEaGVCXhJK0pxhoFBwpWWV6nNOMXDRMYM6ZOHDMcOhrJe+STx2FgltOVDnnCqDwzqcET3kmEEk", - "UPkjgI9oqWo+B9Uxi8gM4Jy7VoyTijONY6FVmdgNK0HiadSBbWksgRnNMfL2G0hBppVui17MdrKWjY2s", - "m2GImJ1zqkkOVGnyhvGzFYLzHo6nGQ56KeRFjYUBDw04KKaS+IHdT/brz1Qt/PJNQy9sXGcbPDbwm5So", - "tYZWOvX/ffCfRx+Ok/+myW+HyYt/n3z89Ozq4aPej0+uvvvu/7V/enr13cP//LfYTvm5x3Jx3MxPXjmz", - "5OQV6p4mqN6b+xcLCheMJ1EiM+5CwTimdHZoizwwGtQT0MMmPO92/ZzrFTeEdElzlhkX+Cbk0BVxPV60", - "3NGhmtZGdGJ8fq0fY+7OXCQlTS/wHHw0Z3pRTQ9SUUy8OTaZi9o0m2QUCsHxWzahJZsY93Zy+XiLaryF", - "vCIRcYXZbtbnD9KUImapO3lqeUgGor2tYdP9jIfwCmaMM/P96JxnVNPJlCqWqkmlQH5Pc8pTOJgLckQc", - "yFdUU3SsO2G6oQtVGPRwsymrac5SchHqt4beh6JN5+cfDNbPzz/2To362sgNFY/g4QDJkumFqHTiQp3D", - "znkTwEDINti1adQxcbDtNrtQqoM/EFUsS5UEYab48ssyN8sPdKYi2AmTlIjSQnrJYsSNCxSY/X0r3LmZ", - "pEufQl4ZZ/h/Clp+YFx/JIlzao/LEmNYGET6H8fARuquS9g9ENVMsQEWc15w4dZKuXbiGgI9tb18ZFbF", - "MWc+IeqwjWG1JtB2UzwZUD+L3GzujdEUwIhip9KLxPBUdFXKkBbyQ3Dxj86NgPEHXcYXNcTnLqJMgaQL", - "SC8gw2g+Bt7Gre7+fNmJa8+yTNm7IzY/DROc0ceaAqnKjDqFRvm6m2mqQGufXvseLmB9Jpr86Oukll6N", - "Ry5SnhiaGWKQ0uAjkKxi1mYXH23vbL47sMBodlkSGzC2qX+eLI5quvB9hhnIivs7YJ4YUdRo2EDvJZUR", - "RFjiH0DBDRZq4N2K9KPhaSo1S1lp179bwPtdq48Bsk2oR8W4mHWldU+YRqW3bZxMqYoLbjBfzH4YHuqm", - "cviRbLjCnjwRvH/sCHeaQ3BUoxxnU4kWhF+2vVA5NLU4lYDkjTb102hjJFTbC3fWxy6bEz48491FwW09", - "6TFU5A/nWTumy8y4OVzSwfD6YOL/SXDiHtwnq9P6vWDrMsO4vuJhr3b79H+f8+8T/UfjayXtj0cusSq2", - "HYKjds8ghzl10WRM2XKE4qb2jQo2yMzjl9nM+PwkiR3eU6VEyuwBYyPL3RhgjL9HhNhoBdkZQoyMg2lj", - "GA4Bk7ci5E0+v84kOTCM21EPGwN4wd+wPYzV3LF3ZuVW868vOxomGjd3YOw29kMq41FUJA1Z5q1WxDaZ", - "Qs8/iJGoEU39IEM/lKEgB1THSUuyJhex0JOxKgDJ8NR3C8x18oDNjJJ/GERjJcyNQ9s4gYZbfVTjyzri", - "l0JDMmNS6QT9z+jyTKMfFRqDP5qmcfHTQhWxl3RZFpc+OOwFrJOM5VV8t924f35lhn1b+y2qml7AGpUM", - "0HRBpnip3Gih1vCmzYahbQLLxgW/tgt+Te9svbvRkmlqBpZC6M4Y94SqOvJkEzNFCDBGHP1dG0TpBvES", - "HPH3ZUuQXGATETBp4WCTt95jpmunSQxKXgspupbA0N24CptNYxNmgjvZ/QTlAR6gZcmyVcd3tlDjNI5D", - "XMdQtxZ/Dwu4uw7YFgwEfnIsX0+C9/XtlgY6096u7+UubcdMN2MqEAjhUEz52jB9RBnSxhSXbbg6A5r/", - "GdZ/MW1xOaOr8eh2Ln8M1w7iFly/q7c3imcMzFoXsBU5uybKaVlKcUnzxN0BGSJNKS4daWJzf2XkC4u6", - "uPt99sPx63du+pgSBlS6TKhNq8J25b1ZlfGIY+lQZ0FkBK1V7ztbQyzY/PriXhhM8dlrLVvOSDFHXJa9", - "agUXsqILrszi50NbQyVhxtuNOLOVMnfbyFyYP3enLN/jsDiFNju8RS6EY22oBlDYgheKCN7NGjBmHHqZ", - "SC4FXZtdtIHZvoDgVZEYFkhUztJ46IBPleEiXhV4PWKtgWDjAYPQQKzYQPicVyyAZZqpHY5fOpMMxogi", - "E8M6G3A3Fa5SWcXZPyogLAOuzSfpsohazGJ4wyfG9lVaPAnXAXZ5uDX42+h5A2pIw+MkNiv5MMobSb32", - "Tp9faB2eNj8EwblrHNKEI/bU0oYDFkcfjprt8fGiHa0NC4v1ZZAhDFuEYntVMx86WNiJDowRrVI2KLGP", - "h6U1JlfvLqcbsYzTDQWyTXijuRIRMBVfUm6LDpl+FoeutwLrt5teSyHxhpCC6LEvU8lMit8g7k3OzEZF", - "EpscKtFkw94HkZsXXSFaR0aacnIev+E8Bkl7yJoKPpL2IdoAhyOVB+FrzNT0QSbKLVnbAkmt89A4c4Q5", - "DBMLv2EON+de3kdOl1MaqwlgjBozp+PmoKQVDtOC+M5+F1SdoOxoLzhzqdsye62mBNlkH/avRd7QQLlf", - "JJ9Bygqax6OjGWK/fbEyY3Nmq0xVCoIyRg6QLc9nqciVgrJHUQ1qTmbkcBwUSnO7kbFLptg0B2zx2LaY", - "UoVaqw551l3M8oDrhcLmT3Zovqh4JiHTC2URqwSpjUh7Y8DHn6eglwCcHGK7xy/IA4y8K3YJDw0WnS0y", - "Onr8AvMc7B+HMWXnysltkisZCpb/coIlTsd49GBhGCXloB5Er3jZGqDDImwDN9muu/AStnRSbzsvFZTT", - "OcRPVIstc7J9cTcxcNfBC89sATulpVgTpuPjg6ZGPg3kOhnxZ6fhEtALw0BaECUKQ09NjSI7qAdnq+G5", - "+iB+Xv4jHnOU/iJBx2n9skFaq8tjq8bDqLe0gDZax4Tam5B4F8LdoHUC8WCgMAPIy/ggcmCDvd50fckD", - "LnhSGN7JHjZZdAH9ResSCE3z6LDay65u5spm0LuaWgZKMojYqoVYGsikG6O4kvF10soM9ev7104xFELG", - "igw00tApCQlaMriMcmw3G6y2TGp14TEfM1B8KYZ/VKB07OINfrD5M+i3GR1oyzAQ4BlqkANiL6qYabeu", - "GqDkZkWV27R1yOYgnVNflbmg2ZgYOGc/HL8mdlTlLjviBQksAzG3l55qFEXCSMH1/evcAhtKt9kdzuY8", - "BLNqpfFOrdK0KGPpiabFmW+AOZCXlOX+SBtFWoidA/LKahPlZZUdpLnsR+rhHP3mc4G3vKnWNF2gmG4J", - "NcskUd9v5/olPsNXBfUA69Jq9a14e39NC1/CxFYwGRNhdOmSKVvTFC6hnRFZpwc7M8FnSLaXJyvOLaXE", - "Zd6G9PWboN1Pzh4W+TBHdGYdxF9TdClRyRSuW87lFHtFL8N0a8P0CgFyyM5WvC645WtVp5QLzlK8ihJU", - "Ua2n7Oqj7hKH2+HWTtcF8yzuODTCXNGKNPVxtMPiYI0aLwgd4vpBiOCr2VRLHfZPjYU4jXMxB62cZINs", - "7KsOOd+AcQWuygGWyg3kpHHxumdS0XB5c6/6mmSEKWUDKvBH8w3VH3NpIBeM4y1DhzaXcWKtdyzfqI3L", - "wDSZC1BuPe1bNOqD6XNwtuInZsYfD3y5R4Rhw5Jm2TYO3gd17KPiLgpt2r40bQmGIJufW+lrdtDjsnSD", - "xiSBqnc4VjdpEMGRyGriQ1sBcmv4IbQN5LbxOAv1qSE0uMRgOJSoh3uEMXBX+QfjKFmKslce7TFyNIee", - "8cg0XjMOTTHSiIJIoyoBNwb5daCfSiXV6WJnmXYGNMfoe0ygKe3CEbcF1dlgRAmu0Y8xvI1N9awBwVE3", - "aDLcKV/XNVANdQfGxEssvuwQ2a+FhVaVM6IyTBTqVMeKCQ4juH29ubYC6LNB3yay3bWklnOuo4mGEpsz", - "poyJW0zzSGrEq/pjUCEOc7Cma/w3dlN0eAXusObGlQ2w47Xty81VBnKz94li8xvuStP/DrelwwPhHsWo", - "/wcjVsKLa71Lv1bw1PUR8VhY+Pqe6FTUyc5tmkVBF8NDUJJxsyM0XFxxjKJxIDnkfXO1j1rpa+NNQyki", - "6WBGE9UuXVFTsqnch618GINgz7ZsxUX7CkLU2Rw6z7LHWeZzr/dudkPPCkPYGxHqD0r7E/qzz4QgJWUu", - "mNqwSB+zLmeqn8W2SzZFs8HdRbhMJAQSW8kNE4d24r0+liKMHR43byHPixZK7Q2DjiUpJNwxagMVek3U", - "9g/Sd10ergMpplLQX+fOG9DC7QDud0F8Ixf6yB1mZz3dhZ3jidqmO8oTixB/laAvTb6YNGgVbHXjxnb9", - "L4O17uxdIqrJEgjlXCBHuagboaQQGeREuRobOcxpuna3/9Q5TyknGZOAhSpYgTXXKFFLOp+DxGujtkyq", - "j00gtMhuVSzPtpGNg/E9to3cxv2a92n7TGwney1zoru1uNDN90frYT7XndFUFIUNDbTQH705WV/HwqAL", - "Tr+pE7gpdjiVlFtPpIchhBK81BCp07WgnEMe7W3PJr4ShRT072JgzgXj8U9dErCI6aChWXN7hX5IDz9S", - "SmE8UpBWkuk15g95z4T9LZob/VPNv67KfH0K6w4B7cMnLjzecHvzVsVPwtZ9Loy7hK6DxuonP6xoUebg", - "5Oh330z/A57+6Vl2+PTxf0z/dPj8MIVnz18cHtIXz+jjF08fw5M/PX92CI9n376YPsmePHsyffbk2bfP", - "X6RPnz2ePvv2xX984x+KsBNtHmH4K5YTSI7fnSRnZrLNRtGS/RnW9ka0oU5f8oGmKLmhoCwfHfmf/pfn", - "E8NAwdt27teRO20YLbQu1dFkslwuD8IukznW40u0qNLFxI/TLzbz7qQO6NukA+QlG6s1jI76gukcM03w", - "2/sfTs/I8buTg0YcjI5GhweHB4+xAkgJnJZsdDR6ij8h1S9w3ycLoLk2nHE1Hk0K0JKlyv3lRPiBq3Zh", - "frp8MvERwMknd7R+ZeDMY7lUvmpWHYHu36seWzVjvNq6SlZwhUi5m0VjMrVZQ8QVauMZxohtRohRfjV6", - "TrLg7czgMYZx6+nPD/foNatYCafYBfXY+6R1bvvw+zTBE37+2b7nf7qKHG997Lw58uTw8DO8MzJuQfF4", - "ueMHS57d4dTbvvetF9AF11vGG5obeoL6bTq7oMf3dkEnHG+XGAFGrIC+Go+e3+MdOuGGoWhOsGWQ0NIX", - "kb/yCy6W3Lc0yrkqCirXqHqDa+2h7XQ1KIrbqWTufuCwfIagyFhwpbh1JDJdezobE1XXeC4lE8aEwJcc", - "M0glUFT4QuJJYlOuzF2cBFvU+s3xX/Hc4c3xX20dwOgrd8HwtiZmW7j/BDpSTu/7dfNS00ZJ/7XE5/h3", - "+zDg/dGFt1VB+6KM97Yo4w5Ce7+7+5Kb97bk5v02SVd1GjAlXPCEY4mFSyBBxGNvo/6ubdTnh0/v7WpO", - "QV6yFMgZFKWQVLJ8TX7ldS7Z7UzwWuZUPMju2yh/egXmGys6MN+Dck+TT613I7LtQZXW/fCsVWacxt/E", - "DCrhuNzUcXPplfLM5gD5U3419pc/MW5nb1nb/Rj3roYexIz04JDu+/XJq13s8taagvtwMdu8ha/rvcD7", - "WSMZN36v9HNqgN48vqcZ8cnGn1k27yZMnx0++3IzCHfhrdDkR0xP/Mwi/bPGCeJkFQgbLKk2+eSvzu0g", - "YNy11LZo6T5yGxMqhkPH7gaBq8Rcv69h5IkVhPZmcF9qmBF2lRf9m7MxSdHcFvy9yIhrvSG8lwt7uXBj", - "udAlqEYi2BcMJ58wNTsUBz2WxGeY/0AHKEEtPykKX0xGkBnodOFeiO4cVkfEik9pH5Ypmy453lq+7N8H", - "v8374Ds4JHsEf5kH2O9z4CPQliQhb9EcQgb32fp/xLDH59TIn3tBbwUHAiumsManpcX9cWNtLmA5AESK", - "fw4hrL9fmw7uldLJp+bZ4KsmQ8ReL51Yy3+TXWHfcBnd6ZnO/t2de/Duztf3Km7FIZ3VSgjfPgZ3vbrh", - "Fl8itF83s51E5ZqrRaUzsQxSrppSzIOc5F/Bv0NO2j/Fv3+Kf/8U//4p/v1T/Pun+PdP8d/vp/i/vsF1", - "XQerG8T7jF5P24QNTJnGhLN/T5aU6WQmpFVPCdZxiwRQ26P/F2XaVQ90vpUWRliA0dBYCc4KGgcnqLuj", - "wlwW98SGf+ucFZFDVzPUj0LuFK9tgqBaELMwUnHNfBY+PsXk7bnfX/Bzb6nuLdW9pbq3VPeW6t5S3Vuq", - "fyxL9eskO5Ak8YLaZ7LG8ljJ/bSm71Gq6JfM7WwM7Nq8RoPcmMOGvzcegmig+cRVlsPzYqEGs6nCKnWp", - "GY5xUuYUyzGvtL/Tg5WYv33mkyHqeku2UIWRQabB0yfk9Ofj54+f/O3J82/r58XbbR/4yrFKr3Nbfrnt", - "KZwBzV+6uVthAkp/L7J1Z1/N9CY40/aONtfoGacyUsos8sh0FwdaYDlDV5uv50xc3WmCRLyGcR+f21A5", - "UMc3Sn2btnNr+Vh3nd/B3kWKmj316CSuDNpXlagEZ+TIrJEe//Ti80biyqMxykbIhGNDYVmVAr495uhn", - "lZhGc+CJY/JkKrK1f6jC1UhsiTRbvG5Yov2wgrQynIEzcUT9QD10zzxiEc4whhEtHhzUVwaE5/Ks+lLK", - "lknbKKRuvnntosu3Pqrvgtv00D55ICSZS1GVD+2LBXyNzmlRUr724RdjT2HVZnx0E9OL7lYs1hUre0Jt", - "96LDoU2Pd8W6v1u0kCVVvuJwZksOx8sudQvjbsd4U/ZxW0Edu95oidqBgrT9TfS77BIb65BTCTLRKx4p", - "FNkpC/lPn9N7H+XvOykumXEVo+LMhnd1lL0PtophGQgglMOd+5peELel43u6DG9/7iohV4mz2W5t0C3A", - "vvPlDZzI5VajnKSgWUoVJiG6ytyf2djTq5OIp43TxCIFs94lLaMtt5f0R7g7mWIB6Ob5KLxFrJTNwv6q", - "hllTQ+TY5Xy2sLGXEn8UJ/d7z3yKUCLpssucQbX8HcQUXeoVj0qpSfM+XTRHKWCI+kGrOzwB6oFvHwQF", - "L0fZkwjIS0JdCUMMTmpZpfqcUwz6hS929Q+JfChz2DB66ZvE486RsLADdc4pvrFShwKjBtIMYrXjAbz9", - "par5HJTuSOIZwDl3rRhv3nMpWCpFYjP1SpAo0Q9sy4KuyYzmGLX+DaQgU2OyhxdfMVSmNMtzdyplhiFi", - "ds6xUKQR+m+YMc8MOB9NqU9a3SsN4Yvw/ZB0t8RjvzydYupnqhZ++T4igoEb+9kevHz5J4TaBSKjMz95", - "5YpSnLzCe8bNgVRv7l/sQKVgPIkSmdH47ly3S1vkgXvQCgnoYXO05Xb9nBvTWAv7Xnvzmuz1yKEb+O7x", - "ouWOzQUzW/Fxv9bPVTzz8vEW++AW8opExNVec/9xwtPdFw/rjTdGbG/vB/TyHdQA+30X/tqa6LIvs7Uv", - "s7UvxLQvs7Xf3X2ZrX0Rqn0Rqn/WIlQHGy3EySe92qUsTAiVZfahVgmpHbkW4GGzVgGZ/hkg0weEnOEr", - "rNToALgESXN8fFv56+xMkYLNF5qoKk0BsqNznrRmYmvgm4EfNP+1bu55dXj4FMjhQ9LuYsMWgeDtd0VL", - "FT/Z55O+I+ej81EXkIRCXIIrJoGtswqPZW2nrVD/xYE957/I3sYVdG1DKwtalmCUmqpmM5Yyi/BcGFdg", - "Ljr5bFzgF5BmcmDkqSJM27pdiE3MA3RZJ9S9DhUzufva/Ro11Y87xBJPJTdkd80Ku/++S3ndfxbz+hVo", - "ynJVZ7hHvCn0a7qUtaSqYdxapox9YrTyv7nDZzdKzi4gzDnFg/4llZlvEX31rqnU5l917AeW2iWsMlh5", - "g6A76Vk9MtO26JRxN3vvEPXjWq4Q1IYpuGo5Nxl84EXsq/EozYWCxGJJxd4zwg9GEmEslmIolro3rv0z", - "twaGYWZqZifxConNZB8ek/F5Yh9IiISo7Xf3gEIdi+tEviNwPZ0MprPWpGEf1kZp00ViSG0z4m6SD4R/", - "7YuBNhnixu8Gdrr3nmTKs/Pzj+S1LXKIrx9dwHpiXyZJF5TPQdU4CunUXvuwGSxBHnMHjXf3VqHRGsnA", - "K6Mn/dzmLt4vWHoBGTFywj+rPmDCkwd1xTZ8Rnq5WPtLHFYNPTwg5JgTTNz1L0q3I82dwfk3etP4q1Bx", - "tjVSJN8uBXYJ8pY85cFs5iQFhuFuOZQFsnkgveID7ESXEYd21xI+Ef+1400GRGVncRdhgb1W2mulvVba", - "a6W9Vtprpc+mlXohmPsfpOj2uXmUogvp7sIUXz1Q8QcqG7ivEPg7W1CYutkqAXyL2G39AGLMCraT8G9y", - "Yhitfo3zw8erj+abvPQRtuaJyaPJBM2IhVB6Mroaf+o8Pxl+NLKTzi0EF8EqJbvEip4fr/5/AAAA//9U", - "hyTJJOUAAA==", + "H4sIAAAAAAAC/+x9/XfbNpbov4LV7jlNuqLlfHUmPqdnn5ukrd8kaU7szs7bOK8DkVcSxiTAAUBLap7/", + "93dwAZAgCUryR9K6o58Si8AFcHG/cXHxaZSKohQcuFajo0+jkkpagAaJf9E0FRXXCcvMXxmoVLJSM8FH", + "R/4bUVoyPh+NR8z8WlK9GI1HnBbQtDH9xyMJ/6yYhGx0pGUF45FKF1BQA1ivS9O6hrRK5iJxII4tiJOX", + "o6sNH2iWSVCqP8ufeL4mjKd5lQHRknJFU/NJkSXTC6IXTBHXmTBOBAciZkQvWo3JjEGeqQO/yH9WINfB", + "Kt3gw0u6aqaYSJFDf54vRDFlHPysoJ5UvSFEC5LBDBstqCZmBDNX31ALooDKdEFmQm6Zqp1EOF/gVTE6", + "+jBSwDOQuFspsEv870wC/AqJpnIOevRxHFvcTINMNCsiSztx2Jegqlwrgm1xjXN2CZyYXgfkTaU0mQKh", + "nLz//gV58uTJc7OQgmoNmSOywVU1o4drst1HR6OMavCf+7RG87mQlGdJ3f799y9w/FO3wF1bUaUgzizH", + "5gs5eTm0AN8xQkKMa5jjPrSo3/SIMEXz8xRmQsKOe2Ib3+mmhOP/pruSUp0uSsG4juwLwa/Efo7KsKD7", + "JhlWT6DVvjSYkgboh8Pk+cdPj8aPDq/+/cNx8j/uz2dPrnZc/osa7hYMRBumlZTA03Uyl0CRWxaU9/Hx", + "3tGDWogqz8iCXuLm0wJFvetLTF8rOi9pXhk6YakUx/lcKEIdGWUwo1WuiR+YVDw3YspAc9ROmCKlFJcs", + "g2xspO9ywdIFSamyILAdWbI8NzRYKciGaC2+ug3MdBWixMzrRvjABf1+kdGsawsmYIXSIElzoSDRYot6", + "8hqH8oyECqXRVep6yoqcLYDg4OaDVbaIO25oOs/XROO+ZoQqQolXTWPCZmQtKrLEzcnZBfZ3qzFYK4hB", + "Gm5OS48a5h1CXw8ZEeRNhciBckSe57s+yviMzSsJiiwXoBdO50lQpeAKiJj+A1Jttv1/n/70lghJ3oBS", + "dA7vaHpBgKciG95jN2hMg/9DCbPhhZqXNL2Iq+ucFSwy5Td0xYqqILwqpiDNfnn9oAWRoCvJhyZkIW6h", + "s4Ku+oOeyYqnuLnNsC1DzZASU2VO1wfkZEYKuvr2cOymowjNc1ICzxifE73ig0aaGXv79BIpKp7tYMNo", + "s2GB1lQlpGzGICM1lA0zccNsmw/j15tPY1kF0/FABqdTj7JlOhxWEZoxrGu+kJLOISCZA/Kzk1z4VYsL", + "4LWAI9M1fiolXDJRqbrTwBxx6M3mNRcaklLCjEVo7NShw0gP28aJ18IZOKngmjIOmZG8OGmhwUqiwTkF", + "A252ZvoqekoVfPN0SIE3X3fc/Zno7vrGHd9pt7FRYlkyohfNV8ewcbOp1X8H5y8cW7F5Yn/ubSSbnxlV", + "MmM5qpl/mP3zaKgUCoEWIrziUWzOqa4kHJ3zr81fJCGnmvKMysz8Utif3lS5Zqdsbn7K7U+vxZylp2w+", + "gMx6rlFvCrsV9h8DLy6O9SrqNLwW4qIqwwWlLa90uiYnL4c22cK8LmEe165s6FWcrbyncd0eelVv5MAk", + "B3FXUtPwAtYSzGxpOsN/VjOkJzqTv5p/yjKP4dQQsFO0GBRwwYL37jfzk2F5sD6BgcJSapA6QfV59CmY", + "0H9ImI2ORv8+aSIlE/tVTRxcM+LVeHTcwLn7kZqedn0dR6b5TBi3u4NNx9YnvPv5GKjRmaCh2pnDd7lI", + "L240h1KKEqRmdh+nBk6fUxA8WQDNQJKManrQOFXWzhqgd+z4I/ZDLwlkRMX9hP+hOTGfDRdS7c03Y7oy", + "ZYw4EQSaMmPxWT1iRzIN0BIVpLBGHjHG2bVm+aIZ3AroWqJ+cGj52IUW2Z1X1q4k2MMvwiy98RqPp0Le", + "jF46hMBJ4wsTaqDW1q9ZeXtnsWlVJg4/EXvaNugAasKPfbEaYqgLPoarFhZONf0MWFAG6l1goQ3orrEg", + "ipLlcAf8uqBq0V+EMXCePCanPx4/e/T4l8fPvjEaupRiLmlBpmsNijxweoUovc7hYX9lKOCrXMehf/PU", + "e1BtuFsxhBOuYe/CUWdgJIPFGLHxAjO7l3ItK34HKAQphYzYvEg6WqQiTy5BKiYi4Yt3rgVxLYwcsnZ3", + "53c7W7Kkipix0R2reAbyIIZ542ehStdQqG2KwoI+W/EGNw4glZKueztg1xtZnRt3lz1pI99b94qUIBO9", + "4iSDaTUPdRSZSVEQSjLsiALxrcjgVFNdqTuQAg2wZjJmI8Ip0KmoNKGEi8wwtGkclw8DsUwMomDsR4ci", + "Ry+s/pmCsY5TWs0XmhizUsS2tumY0NRuSoK6Qg24frXPblvZ4WycLJdAszWZAnAips6/cp4fLpJiWEb7", + "ExcnnZpp1T5Ba16lFCkoBVnijpe2Ts0fVeEm6w1ownnjfOtBiBJkRuUN56qFpvmWeWKb/mxVY004n7Q/", + "692G37R/3cHDXaTSuJiWCIzpYpg7Bw1DKNyKk6ocOI5wWu2MFYYlCKdcKEgFz1QUWE6VTraxgmnUUr1m", + "WwPqi1E/Ah5wul9Tpa3by3iGZphlYRwH++AQwxMelNIG8l+9gO7DTo3s4apStbRWVVkKqSGLrYHDasNY", + "b2FVjyVmAexaJWhBKgXbIA9hKYDvkGVXYhFEtYu71HGh/uIwxG1k6zqKytYkGkRsmsipbxVgNwzJDkzE", + "2Ox1TyQcpjqUU8eBxyOlRVkamaSTitf9htB0alsf65+btn3iorqRlZkAM7r2c3IzX1rM2mD8ghp7CSGT", + "gl4YeY/Wj/XP+3M2zJgoxlNINlG+YctT0ypkgS1MOmB4uuO+YLQOc3ToN0p0g0SwZReGFjxgBb+zUeWz", + "JuJyB4bAS9CU5apW9nXouhkFo9zdDARjmUlIget8bWh1xmRhD4pQRyj/mzUlMjeKPRJp2I9nRMKSysy3", + "6HsgwWISxjNYxaUrbcUbMlgRFp/0rB6ZaZL6YxweAjiIMro7GNswBRdouMngpmt8WHvsY7GkYgeC+MEw", + "QMFSKag95zOLsUpS10dZEgpqZocnTk6pD4/J+Dyxx4oR9Wi/+2NHH+4NaSYO19PJIGfXpLFcAJ5kGHHd", + "QWJIbcZNAwVDC5nnYkrzxBivkGSQ661hJGMUw0tsafSkSPvd21M+P/+QZ+fnH8lr0xbtZCAXsJ7g6StJ", + "F5TPoQmJh3RqLWBYQVqFIr2Dxp2cGhf3a8++7daMR6UQeVK7b90Qfk/Md/F+wdILyIiRE2h0Ou3zVXuH", + "zCDkgSFxVR9yLBdrb8+WJXDIHh4QcswJFKVeu1hBx9LoDM6/0pvGX+GoWYXnrZQTXOTBOY+76fa09pY8", + "5cFs5iSbvnTLoSyQzQPpFR9gJ7rEwwYDLsqfGyN9p9gzUDk9TRoQlZ3FLv7wD5jTQ1u7zDJ0Nhqtoqpp", + "wTCxJ2g2NpLTn7X2vVWmDwg5Q9lhvAUFlyBpjlkLygdBmSIFM06nqtIUIDs650lrJqko3MAPmv9asXRe", + "HR4+AXL4sNtHaWMmOsfI8kC377fkcGw/IbrIt+R8dD7qQZJQiEvIrHMY0rXttRXsv9Vwz/lPPcFMCrq2", + "bqXnRaKq2YylzCI9F0auz0XH2uMCv4A00wPjnCnC9BhVGWIUrWS7Lw0DjqJWy13ELyJQjX1sVKmRdv6E", + "rU07isCKpmaVFIXMmiwNodR01jc+tCiTEEA0nLphRBfQVi05fkO+68tz601vnt9Zx59uoSMg14PtNnMP", + "GdEZ7ML+x6QUZteZy6XxCRc5U7o3SefY42lGTZARpXNA/o+oSEqRf8tKQ+1TCYmOCjqwZgTUsX5MZ6k1", + "GIIcCrDhDvzy9dfdhX/9tdtzpsgMlj4BzTTsouPrry0TCKVvzQEd0lydRAwoDDIbbRpJGl5QtTjYGnBG", + "uDvFmQPQJy/9gMhMSqGKuRqPjKubr++A4S0gIsHZe6oV9FH2q5iFyW5u/9RaaSj6kUvb9ZcBS/S999B6", + "mlbwnHFICsFhHc3vZhze4MeonkYSGeiMzDrUt+vBtubfmVZ7nF1287b4xd0OSOJdnXp3B5vfhdsJWodp", + "fmhlQl4SStKcYUBQcKVllepzTjFA0TGDOmThwy7DIasXvkk8RhYJYTlQ55wqg8M6bBE9zJhBJCD5PYCP", + "XKlqPgfVMYvIDOCcu1aMk4ozjWOhVZnYDStB4qnTgW1pLIEZzTHC9itIQaaVbotezEaylo2NoJthiJid", + "c6pJDlRp8obxsxWC8x6OpxkOeinkRY2FAQ8NOCimkvjB3A/2649ULfzyTUMvbFxnGyQ28JuUpbWGVrrz", + "/33wX0cfjpP/ocmvh8nz/5x8/PT06uHXvR8fX3377f9r//Tk6tuH//UfsZ3yc4/lyriZn7x0ZsnJS9Q9", + "TfC8N/cvFvwtGE+iRGbchYJxTLns0BZ5YDSoJ6CHTRje7fo51ytuCOmS5iwzLvBNyKEr4nq8aLmjQzWt", + "jejE8vxaP8bcnblISppe4Ln2aM70opoepKKYeHNsMhe1aTbJKBSC47dsQks2Me7t5PLRFtV4C3lFIuLq", + "ajxyUkfdeQaNAxxbUHfMOoru/9aCfPXDqzMycTulvrKJcxZ0kPEUsaDdYVjLmTOLtxc/bOagcWZewoxx", + "Zr4fnfOMajqZUsVSNakUyO9oTnkKB3NBjogD+ZJqijGATkRx6G4WxmfcbMpqmrOUXISquGHNocDY+fkH", + "QyDn5x97B1l9xemGigcbcYBkyfRCVDpxUdnhOEITa0HINi63adQxcbAtRbqor4M/EAAtS5UEEbH48ssy", + "N8sPyFAR7IR5UERpIb0QNJLRxTTM/r4V7ihP0qXPRq+M3/73gpYfGNcfSeL87+OyxHAbxrv+7mSNocl1", + "CbvHzJopNsBifhYu3BpU186NQ6CntpcPIqs45swnRB22MVKhiQneFE8G1I8iN5t7YzQFMKLYqfQiMTwV", + "XZUypIX8ENwhpHMjC/3Zm3GbDfG5Oy1TIOkC0gvI8OABY4TjVnd/5O00i2dZpuw1FJsCh7nS6A5OgVRl", + "Rp3upXzdTVpVoLXP1H0PF7A+E02q9XWyVK/GIxfUTwzNDDFIafARKAExa7OLPxjobL47W8HAe1kSG9u2", + "2YWeLI5quvB9hhnIaqY7YJ4YUdRo2EDvJZURRFjiH0DBDRZq4N2K9KORdCo1S1lp179bbP5dq48Bsk2o", + "R8W4mHWldU+YRqW3bZxMqYoLbjBfzH4YHupml/iRbGTFHpIRvMrsCHeaQ3CqpBxnU4nGjl+2vZs5NLU4", + "lYDkjTb102hjJFTbC3csyS6bw0g8dt5FwW09lDJU5PMFWDv8zMy4OVzSwZOAwTsEJ0ESQHA1rb4h4AVb", + "lxnG9W0Re0vc3yTw1wf8nYHR+Fr5/+ORy/WKbYfgqN0zyGFOXeAbs8gcobipfaWCDTLz+Gk2yxkHksTy", + "CahSImX2LLSR5W4MMMbf14TYwArZGUKMjINpY8QQAZO3IuRNPr/OJDkwDDFSDxtjjcHfsD3i1lzXd2bl", + "VvOvLzsaJho312nsNvajP+NRVCQNWeatVsQ2mULPlYmRqBFN/XhIP+qiIAdUx0lLsiYXsSiZsSoAyfDU", + "dwvMdfKAzYySfxgEjiXMje/d+KuGW30A5svGDC6FhmTGpNIJusrR5ZlG3ys0Br83TePip4UqYu/7siwu", + "fXDYC1gnGcur+G67cf/y0gz7tvZbVDW9gDUqGaDpgkzxfrrRQq3hTZsNQ9ucmo0Lfm0X/Jre2Xp3oyXT", + "1AwshdCdMe4JVXXkySZmihBgjDj6uzaI0g3iJchG6MuWIA/C5kxgfsXBJm+9x0zXzugYlLwWUnQtgaG7", + "cRU28cfm9gTXu/s50wM8QMuSZauO72yhxmkch7iOoW4t/h4WcHcdsC0YCPzkWAqhBO/r2y0NdKa9qN9L", + "s9qOmW5yVyAQwqGY8mVm+ogypI3ZONtwdQY0/wus/2ra4nJGV+PR7Vz+GK4dxC24fldvbxTPGEO2LmAr", + "cnZNlNOylOKS5om7ljJEmlJcOtLE5v4WyxcWdXH3++zV8et3bvqYvQZUuqStTavCduW9WZXxiGOZW2dB", + "ZAStVe87W0Ms2Pz6bmAYTPGJdi1bzkgxR1yWvWoFF7KiC67M4kdZW0MlYXLejTizld1328hcmOp3pyzf", + "47A4hTY7vEUuhGNtKCxQ2NoZigjeTXAwZhx6mUguBV2bXbSB2b6A4FWRGBZIVM7SeOiAT5XhIl4VeGNj", + "rYFg4wGD0ECs2ED4nFcsgGWaqR1OijqTDMaIIhPDOhtwNxWu6FnF2T8rICwDrs0n6RKeWsxieMPn8PZV", + "Wjxf2AF2KcM1+NvoeQNqSMPjJDYr+TDKG8kS906fX2gdnjY/BMG5axzShCP21NKGAxZHH46a7Un3oh2t", + "DWuU9WWQIQxbz2J7gTQfOljYiQ6MES14Niixj4elNeaB7y6nG7GM0w0Fss3No7kSETAVX1Ju6xeZfhaH", + "rrcC67ebXksh8dKSgugJNVPJTIpfIe5NzsxGRXKwHCrRZMPeB5HLIF0hWkdGmsp0Hr/hPAZJe8iaCj6S", + "9iHaAIcjlQfha0wq9UEmyi1Z21pLraPbOHOE6RYTC79hDjfnXopKTpdTGis7YIwaM6fj5qCkFQ7TgvjO", + "fhdUnUvtaC84c6nbMnvTpwTZJEr2b2re0EC5XySfQcoKmsejoxliv33XM2NzZgtWVQqCikgOkK30Z6nI", + "VZWyR1ENak5m5HAc1Fxzu5GxS6bYNAds8ci2mFKFWqsOedZdzPKA64XC5o93aL6oeCYh0wtlEasEqY1I", + "e7nBx5+noJcAnBxiu0fPyQOMvCt2CQ8NFp0tMjp69BxTMuwfhzFl5yrTbZIrGQqW/3aCJU7HePRgYRgl", + "5aAeRG+d2XKiwyJsAzfZrrvwErZ0Um87LxWU0znET1SLLXOyfXE3MXDXwQvPbC08paVYE6bj44OmRj4N", + "pGUZ8Wen4XLlC8NAWhAlCkNPTbkjO6gHZwvruRIkfl7+Ix5zlP7OQ8dp/bJBWqvLY6vGw6i3tIA2WseE", + "2suZeG3DXep1AvFgoFYEyMv4IHJgg73edH3JAy54UhjeyR42CX8B/UVLJQhN8+iw2suububKZtC7mloG", + "SjKI2KqFWBrIpBujuJLxddLKDPXz+9dOMRRCxuoeNNLQKQkJWjK4jHJsN3GttkxqdeExHzNQvqtYnv21", + "STftlBiSlKeLaPxzajr+0tREq9FusR69gregnEMeBWd5+RfP8xGp9A+x6zgF4zu27ZYOssvtLK6ZeHua", + "flJ+QINepnMzQIjVdv5dnTiSz0VGcJzmknVDCP07UkEZlX9WoHTsPhd+sLlO6GMbe8VW8SDAM9T2B8Te", + "fzJzad1gQS3Liiq3tyEgm4N0AZiqzAXNxsTAOXt1/JrYUZW7Q4v3brCKyNzepWutouNbBdUfrnO5cCg1", + "anc4m3NGzKqVxivZStOijGW9mhZnvgGm1l5Slvv0A1Q/IXYOyEur+ZXXK3aQ5g4pqYdzsgZpwvxHa5ou", + "UKW2FNAwye9e/sZTpQrKQNYV9eqiCvZapBa+Ao4tgDMmwtg9S6ZsKVu4hHaibZ117kw6n3jbXp6sOLeU", + "EtdPG25F3ATtfnL2YM+HpKIz6yD+mmpGiUqmcN1qQKfYK3rHqltaqFf/kUN2tuJ1/TVfojylXHCW4g2n", + "oHhuPWVXFneXmOkOl8G67rJnccehEeaKFjSqUwccFgdLHHlB6BDXDxgFX82mWuqwf2qsv2ocwTlo5SQb", + "ZGNftMr5cYwrcEUysEJyICeNO949P4webTTX9a9JRpj+N2CufG++oanCXMrOBeN4edWhzWUHWU8Lq3Zq", + "494xTeYClFtP+3KW+mD6HJyt+ImZ8ccDX+UTYdgQslm2PbPogzr2JxjuxMC0fWHaEgwXNz+3Ug3toMdl", + "6QaNSQJV73Cs7NYggiNR8MSHIQPk1vBDaBvIbePRI+pTQ2hwiQcXUKIe7hHGwBX4V8aptRRlb9LaI//o", + "1QzGI9N4zTg0NWgjCiKNqgTcGOTXgX4qlVRbE3AnmXYGNMeTkphAU9qFjm4LqrPBiBJcox9jeBub4msD", + "gqNu0BhulK/r0reGugNj4gXW3HaI7JdSQ6vKGVEZJnV1iqvFBIcR3L4sYVsB9NmgbxPZ7lpSyznX0URD", + "SegZU8YdKaZ5JI3lZf0xKDCI+XLTNf4bu4A8vAJ3sHbjghnY8dr25ebiFbnZ+0Sx+Q13pel/h9vS4YFw", + "j2LU/8qIlfDeTu8uuRU89bUaPMIXvtwrOhV1YnqbZlHQRZ22pnLnZqd1uAbnGEXjQCLP++bGKLXS18YG", + "h9J50sHsM6pdaqmmZFMVGVs4MwbBnkPagp328YtoYGDo7NEePZrPvd672Q09Kwxhb0SoP9TuT+gvPmuF", + "lJS5wHfDIn3Muvy2fsbhLpkvzQZ3F+GyxhBIbCU3TPLaiff6WIowdpgasIU8L1ootbdBOpakkHDHqA1U", + "6DVR20962HV5uA6kmEpBf507b0ALtwO43wXxjVzoI3eYnfV0F3aOJ9Wb7ihPLEL8tY++NPli0qBV79eN", + "G9v1vw5FD6yHPBCo6uC0Ynm2bXNbYcfmOjMG1n6ZfvO0Fb37kheqf7EH8n12c3dLr6P4u5uAiImstTV4", + "MFQQUNwhlui6RSKHWAsqrSTTa8zd8ZYm+yWal/wDcFf12BWRr09A3QGcfb/EhabndevmyYkfhC0DXRjz", + "F01BjUVSXq1oUebg+OLbr6Z/gid/fpodPnn0p+mfD58dpvD02fPDQ/r8KX30/MkjePznZ08P4dHsm+fT", + "x9njp4+nTx8//ebZ8/TJ00fTp988/9NX/r0HO9HmLYW/YdWB5PjdSXJmJtvghJbsL7C294wNGfsbzDRF", + "ToSCsnx05H/6X57DDlJRBE/UuV9HLtI/WmhdqqPJZLlcHoRdJnMs25doUaWLiR+nX5Pm3UkdoLUH/rij", + "NvZmSAE31ZHCMX57/+r0jBy/OzloCGZ0NDo8ODx4hIVCSuC0ZKOj0RP8Cblngfs+WQDNtfHorsajSQFa", + "slS5v9SSzucgD9zlbfPT5eOJj+hMPrlj7SsDZx7LY/LFteqIYv9O89iGKIyXUhfTCq7vKHerZ0ymNmOH", + "uHpuPMOYn83GMMKsRs9JFjyBGby1MG694PnhHj1KFav0FLscHntmtM4rH35mJniJz7++9+zPV5GjpY+d", + "p0MeHx5+hudCxi0oHi83fHfk6R1Ose0z3XqiXXA9OfCG5oZuoH5KboQLenRvF3TC8QaHEVTECuKr8ejZ", + "Pd6hE24Yh+YEWwZJI31R+DO/4GLJfUujhKuioHKNKja4Oh4aU1eDIredruXu4A3LYQhqjgXXdluh7Ona", + "09mYqLq0cymZMKYCPryYQSqBomIXEk+Amupl7nIi2FrWb47/hvHiN8d/s2UBo4/SBcPbEpltIf4D6Eh1", + "ve/WzcNKGyX6byUmx7/bd/zuj867rarZ12i8tzUadxDa+93dV+C8txU477dJuqpTbSnhgiccyxhcAgkC", + "WXsb9Xdtoz47fHJvV3MK8pKlQM6gKIWkkuVr8jOvc4BuZ4LXMqfiQVbWRvnTqzffWNGB+R6UVJp8aj0j", + "kW0PnrTuYGetquM0/rRlUG3G5X+Om4ullGc2d8Ofzqqxv2CJ8Tl7k9nux7h3/fIgZqQHhyvfrU9e7mKX", + "t9YU3DmL2eYtfF3vwdzPGrG48bOjn1MD9ObxHc2ITxL9zLJ5N2H69PDpl5tBuAtvhSbfY1rZZxbpnzVO", + "ECerQNhg2bLJJ389bQcB465+tkVL963amFAxHDp2WfquMHP93IaRJ1YQ2tu3falhRthVXvRvp8YkRXMj", + "7/ciI671FPBeLuzlwo3lQpegGolgHy6cfMKU2lAc9FgSX1P+Ax2UBPXypCh8wRZBZqDThXvouXN6PfRe", + "/0aZsuki4a3ly/6Z79s8872DQ7JH8Jd5R/0+Bz4CbUkS8hbNIWRwn2X9Rwx7fE6N/LkX9FZwILBiCuto", + "WlrcHzfW5gJeuUek+NcRwhr3tengHiedfGpeC75qMkHstcCJtfw32RX2SZfRnZ7p7J/huQfP8Pz2XsWt", + "OKSzWgnhk8fgrsU23OLLcPZrU7aTpVxztah0JpZBalVT7niQk/zj93fISfsX+Pcv8O9f4N+/wL9/gX//", + "Av/+Bf778QL/b29YXdeR6gbrPqN30zZVA5OlMdXs35MlZTqZCWnVUII10SKB0vbo/02ZdpX4nA+lhREK", + "YDQxVlWzAsXBCeqiqDBnxT1X4Z84Z0XkcNUM9b2QO8Vlm2CnFsQsjFRcM59Vj88aebvt9xfk3Fuke4t0", + "b5HuLdK9Rbq3SPcW6f20SH+b5AWSJF4g+8zUWF4qGd1Lq/kepX5+yVzNxpCuzWg0vI3Za/h446GGBppP", + "XIUvPP8VajA7KqwWlprhGCdlTrGE8Ur7OzpYvfibpz65oa57YwsGGFljGjx5TE5/PH726PEvj599U78e", + "3m77wFdbVXqd25LFbY/gDGj+ws3dCg1Q+juRrTv7aqY3wZm2d7S5Rs84lZGSUpE3pLs40ALLyrkaaT2n", + "4epOEx7idX/7+NyGyoHat1Hq27SdW0uuumqgDvYuRy1mTz06iStH9ZtKVIIzcmTWSI9/efF5I3Hl0Rhl", + "I2TCsaGwrEoB3+ty9LNKTKM58MQxeTIV2do/7uBq1bVEmi0iNizRXq0grQxn4EwcUT9QD93TiFgMMYxV", + "RIu4BjWJAeG5vKm+lLLlqjYKqZtvXrv47a2P3rvgNr2jTx4ISeZSVOVDW+Wfr9EJLUrK1z7MYuwmrJ6L", + "D1ViutDdisW6cmBPqO1e/DW03fHuV/d3ixaypMpXfs1s6dd4+ZtugdLtGG/K720rl2LXGy0VOlAYtL+J", + "fpddomIdWipBJnrFIwX7OuX5/uVzdO+j/H0nxSUzLmFUnNkwro6y98FWMSwDAYRyuHP/0gvitnR8T5fh", + "bc5dJeQqcTbbrQ26Bdi3sbyBE7msapSTFDRLqcKkQlch+TMbe3p1EvGocZpYdGDWu3RltOX2MvgIdydT", + "LADdPLmEt4KVraf02xpmTU2QY5fD2cLGXkr8UZzc7zzzKUKJpMsucwZVy3cQU3SpVzwqpSbNm27RnKOA", + "IepHoO7wpKcHvn3gE7y2ZE8cIC8JJWnOMJguuNKySvU5pxjcC1+56h8G+ZDlsGH0wjeJx5cj4V8H6pxT", + "fJekDvlFDaQZxGp4A3j7S1XzOSjdkcQzgHPuWjHevIFSsFSKxGbelSBRoh/YlgVdkxnNMTr9K0hBpsZk", + "Dy+yYqhMaZbn7vTJDEPE7JxTTXIwQv8NM+aZAeejKfWJqquWH76i3g89uyp2Aw/3/GC//kjVwi/fR0Qw", + "cGM/2wOWL//sjp977D0+N/OTl67IxMlLvDfcHDz15v7FDk4KxpMokRmN785vu7RFHrhHoJCAHjZHWG7X", + "z7kxjbWwb5w3L7Bejxy6Ae4eL1ru6FBNayM6cXC/1o+xCw5zkRgHEKsNj+ZML6opFuDzFx8mc1Ffgphk", + "FArB8Vs2oSWbqBLSyeWjLfbBLeQViYirveb+44Snu68E1htvjNje3g/o5Tuo6fX7LuS1NaFlXzZrXzZr", + "X1hpXzZrv7v7sln7olL7olL/qkWlDjZaiJNPerVLmZcQKsvs46YSUjtyLcDDZq2CMP0zQKYPCDnDl0up", + "0QFwCZLm+GC18tfTmSIFmy80UVWaAmRH5zxpzcQ+oWkGftD817q559Xh4RMghw9Ju4sNWwSCt98VLVX8", + "ZJ+x+Zacj85HXUASCnEJrjgEts4qPJa1nbZC/TcH9pz/JHsbV9C1Da0saFmCUWqqms1YyizCc2Fcgbno", + "5K1xgV9AmsmBkaeKMG3rcCE2Md/PZZ1Q90pPzOTua/dr1EI/7hBLPGXckN01K+b+5y7lcv9VzOuXoCnL", + "VZ3JHvGm0K/pUtaSqoZxa5ky9gnQyv/mDp/dKDm7gDC3FA/6l1RmvkX09bGm8pp/XS/yon6rJFUGK28Q", + "dCc9q0dmzdP7/fdg+nEtV9hpwxRc9ZubDD7wivTVeJTmQkFisaRi78rgByOJMBZLMRRL3bvQ/rlRA8Mw", + "MzWzk3hVxGasD4/J+DyxDxtEQtT2u3v4oI7FdSLfEbieTgbTVmvSsI9Ro7TpIjGkthlxN8MHwr/25Tab", + "DHHj99s63XtP4+TZ+flH8toWLcR3nS5gPbEviqQLyuegahyFdGqvd9gMliBfuYPGu3szzmiNZOC1x5N+", + "DnMX7xcsvYCMGDnhnyIfMOHJg7oCGz7nu1ys/WUNq4YeHhByzAkUpV77l33bkebO4PwrvWn8Vag42xop", + "km+XArsEeUue8mA2c5ICw3C3HMoC2TyQXvEBdqLLiEO7a0meiP/a8SYDorKzuIuwwF4r7bXSXivttdJe", + "K+210mfTSr0QzP0PUnT73DxK0YV0d2GK3zxQ8QcqA7iv+Pc7W1CYutkq6XuL2G39cGHMCnZR2eYp0PBp", + "TYyq1Y9qfvh49dF8k5c+4Na8FHk0maBVsRBKT0ZX40+dVyTDj0aU0rmF4AJapWSXWLDz49X/DwAA///D", + "WmxWsuQAAA==", } // GetSwagger returns the Swagger specification corresponding to the generated code diff --git a/daemon/algod/api/server/v2/generated/types.go b/daemon/algod/api/server/v2/generated/types.go index c3b918387f..9b31e0f89e 100644 --- a/daemon/algod/api/server/v2/generated/types.go +++ b/daemon/algod/api/server/v2/generated/types.go @@ -227,6 +227,16 @@ type AssetParams struct { Url *string `json:"url,omitempty"` } +// BuildVersion defines model for BuildVersion. +type BuildVersion struct { + Branch string `json:"branch"` + BuildNumber uint64 `json:"build_number"` + Channel string `json:"channel"` + CommitHash string `json:"commit_hash"` + Major uint64 `json:"major"` + Minor uint64 `json:"minor"` +} + // DryrunRequest defines model for DryrunRequest. type DryrunRequest struct { Accounts []Account `json:"accounts"` @@ -340,22 +350,10 @@ type TealValue struct { // Version defines model for Version. type Version struct { - - // the current algod build version information. - Build VersionBuild `json:"build"` - GenesisHash []byte `json:"genesis-hash"` - GenesisId string `json:"genesis-id"` - Versions []string `json:"versions"` -} - -// VersionBuild defines model for VersionBuild. -type VersionBuild struct { - Branch string `json:"branch"` - BuildNumber uint64 `json:"build-number"` - Channel string `json:"channel"` - CommitHash []byte `json:"commit-hash"` - Major uint64 `json:"major"` - Minor uint64 `json:"minor"` + Build BuildVersion `json:"build"` + GenesisHashB64 []byte `json:"genesis_hash_b64"` + GenesisId string `json:"genesis_id"` + Versions []string `json:"versions"` } // AccountId defines model for account-id. @@ -616,6 +614,9 @@ type TransactionParametersResponse struct { MinFee uint64 `json:"min-fee"` } +// VersionsResponse defines model for VersionsResponse. +type VersionsResponse Version + // AccountInformationParams defines parameters for AccountInformation. type AccountInformationParams struct { From 108c4742d71dc9053bcb1c03805d7c4f01b8e28e Mon Sep 17 00:00:00 2001 From: Will Winder Date: Wed, 2 Sep 2020 12:00:44 -0400 Subject: [PATCH 002/136] Auth Middleware Tests (#1438) * Auth middleware unit tests. * Add pprof test to e2e rest test script. --- .../algod/api/server/lib/middlewares/auth.go | 10 +- .../api/server/lib/middlewares/auth_test.go | 188 ++++++++++++++++++ daemon/algod/api/server/router.go | 3 +- test/scripts/e2e_subs/rest.sh | 30 +++ 4 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 daemon/algod/api/server/lib/middlewares/auth_test.go diff --git a/daemon/algod/api/server/lib/middlewares/auth.go b/daemon/algod/api/server/lib/middlewares/auth.go index e168f00a1f..52a386480a 100644 --- a/daemon/algod/api/server/lib/middlewares/auth.go +++ b/daemon/algod/api/server/lib/middlewares/auth.go @@ -25,7 +25,13 @@ import ( "github.com/labstack/echo/v4" ) +// TokenPathParam is the name of the path parameter used by URLAuthPrefix +const TokenPathParam = "token" +// URLAuthPrefix is the echo formatted url/path param which can be used for supplying an API token. +const URLAuthPrefix = "/urlAuth/:" + TokenPathParam const urlAuthFormatter = "/urlAuth/%s" +// InvalidTokenMessage is the message set when an invalid / missing token is found. +const InvalidTokenMessage = "Invalid API Token" // AuthMiddleware provides some data to the handler. type AuthMiddleware struct { @@ -71,7 +77,7 @@ func (auth *AuthMiddleware) handler(next echo.HandlerFunc) echo.HandlerFunc { } // Handle debug routes with /urlAuth/:token prefix. - if ctx.Param("token") != "" { + if ctx.Param(TokenPathParam) != "" { // For debug routes, we place the apiToken in the path itself providedToken = []byte(ctx.Param("token")) @@ -93,6 +99,6 @@ func (auth *AuthMiddleware) handler(next echo.HandlerFunc) echo.HandlerFunc { } } - return echo.NewHTTPError(http.StatusUnauthorized, "Invalid API Token") + return echo.NewHTTPError(http.StatusUnauthorized, InvalidTokenMessage) } } diff --git a/daemon/algod/api/server/lib/middlewares/auth_test.go b/daemon/algod/api/server/lib/middlewares/auth_test.go new file mode 100644 index 0000000000..30e3bbd13c --- /dev/null +++ b/daemon/algod/api/server/lib/middlewares/auth_test.go @@ -0,0 +1,188 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package middlewares + +import ( + "errors" + "net/http" + "testing" + + "github.com/labstack/echo/v4" + "github.com/stretchr/testify/require" +) + +var errSuccess = errors.New("unexpected success") +var invalidTokenError = echo.NewHTTPError(http.StatusUnauthorized, InvalidTokenMessage) +var e = echo.New() +var testAPIHeader = "API-Header-Whatever" + +// success is the "next" handler, it is only called when auth allows the request to continue +func success(ctx echo.Context) error { + return errSuccess +} + +func TestAuth(t *testing.T) { + tokens := []string{"token1", "token2"} + + tests := []struct { + name string + url string + header string + token string + method string + expectResponse error + finalPath string + }{ + { + "Valid token (1)", + "N/A", + testAPIHeader, + tokens[0], + "GET", + errSuccess, + "", + }, + { + "Valid token (2)", + "N/A", + testAPIHeader, + tokens[1], + "GET", + errSuccess, + "", + }, + { + "Valid token Bearer Format (1)", + "N/A", + "Authorization", + "Bearer " + tokens[0], + "GET", + errSuccess, + "", + }, + { + "Valid token Bearer Format (2)", + "N/A", + "Authorization", + "Bearer " + tokens[1], + "GET", + errSuccess, + "", + }, + { + "Invalid token", + "N/A", + testAPIHeader, + "invalid_token", + "GET", + invalidTokenError, + "", + }, + { + "Invalid token Bearer Format", + "N/A", + "Authorization", + "Bearer invalid_token", + "GET", + invalidTokenError, + "", + }, + { + "Missing token", + "N/A", + "", + "", + "GET", + invalidTokenError, + "", + }, + { + "Invalid token + OPTIONS", + "N/A", + testAPIHeader, + "invalid_token", + "OPTIONS", + errSuccess, + "", + }, + { + "Invalid bearer token + OPTIONS", + "N/A", + "Authorization", + "Bearer invalid_token", + "OPTIONS", + errSuccess, + "", + }, + { + "Token in url (1)", + "http://my-node.com:80/urlAuth/" + tokens[0] + "/v2/status", + "", + tokens[0], + "GET", + errSuccess, + "/v2/status", + }, + { + "Token in url (2)", + "http://my-node.com:80/urlAuth/" + tokens[1] + "/v2/status", + "", + tokens[1], + "GET", + errSuccess, + "/v2/status", + }, + { + "Invalid token in url", + "http://my-node.com:80/urlAuth/invalid_token/v2/status", + "", + "invalid_token", + "GET", + invalidTokenError, + "", + }, + } + + authFn := MakeAuth(testAPIHeader, tokens) + handler := authFn(success) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req, _ := http.NewRequest(test.method, test.url, nil) + if test.header != "" { + req.Header.Set(test.header, test.token) + } + ctx := e.NewContext(req, nil) + + // There is no router to update the context based on the url, so do it manually. + if test.header == "" && test.token != "" { + ctx.SetParamNames(TokenPathParam) + ctx.SetParamValues(test.token) + } + ctx.SetPath("") + + err := handler(ctx) + require.Equal(t, test.expectResponse, err, test.name) + + // In some cases the auth rewrites the path, make sure the path has been rewritten + if test.finalPath != "" { + require.Equal(t, test.finalPath, ctx.Path()) + require.Equal(t, test.finalPath, ctx.Request().URL.Path) + } + }) + } +} diff --git a/daemon/algod/api/server/router.go b/daemon/algod/api/server/router.go index 6067b21687..f7cca11a87 100644 --- a/daemon/algod/api/server/router.go +++ b/daemon/algod/api/server/router.go @@ -60,6 +60,7 @@ package server import ( + "fmt" "net" "net/http" @@ -130,7 +131,7 @@ func NewRouter(logger logging.Logger, node *node.AlgorandFullNode, shutdown <-ch // The auth middleware removes /urlAuth/:token so that it can be routed correctly. if node.Config().EnableProfiler { e.GET("/debug/pprof/*", echo.WrapHandler(http.DefaultServeMux), adminAuthenticator) - e.GET("/urlAuth/:token/debug/pprof/*", echo.WrapHandler(http.DefaultServeMux), adminAuthenticator) + e.GET(fmt.Sprintf("%s/debug/pprof/*", middlewares.URLAuthPrefix), echo.WrapHandler(http.DefaultServeMux), adminAuthenticator) } // Registering common routes (no auth) registerHandlers(e, "", common.Routes, ctx) diff --git a/test/scripts/e2e_subs/rest.sh b/test/scripts/e2e_subs/rest.sh index 037579205f..89ebda358f 100755 --- a/test/scripts/e2e_subs/rest.sh +++ b/test/scripts/e2e_subs/rest.sh @@ -16,6 +16,9 @@ PUB_TOKEN=$(cat "$ALGORAND_DATA"/algod.token) ADMIN_TOKEN=$(cat "$ALGORAND_DATA"/algod.admin.token) NET=$(cat "$ALGORAND_DATA"/algod.net) +PRIMARY_NET=$(cat "$ALGORAND_DATA2"/algod.net) +PRIMARY_ADMIN_TOKEN=$(cat "$ALGORAND_DATA2"/algod.admin.token) + function base_call { curl -o "$3" -w "%{http_code}" -q -s -H "Authorization: Bearer $1" "$NET$2" @@ -112,7 +115,34 @@ function test_assets_endpoint { call_and_verify "Asset invalid parameter" "/v2/assets/$ASSET_ID?this-should-fail=200" 400 'parameter detected: this-should-fail' } +function pprof_test { + # URL Auth - valid + CODE=$(curl -o "${TEMPDIR}/curl_out.txt" -w "%{http_code}" -q -s "$PRIMARY_NET/urlAuth/$PRIMARY_ADMIN_TOKEN/debug/pprof/block") + if [[ "$CODE" != "200" ]]; then + fail_and_exit "Call pprof with valid token" "/urlAuth/:token/debug/pprof" "Invalid exit code expected 200 (actual $CODE)" + fi + + # URL Auth - invalid + CODE=$(curl -o "${TEMPDIR}/curl_out.txt" -w "%{http_code}" -q -s "$PRIMARY_NET/urlAuth/invalid_token/debug/pprof/block") + if [[ "$CODE" != "401" ]]; then + fail_and_exit "Call pprof with invalid token" "/urlAuth/invalid_token/debug/pprof" "Invalid exit code expected 401 (actual $CODE)" + fi + + # Header Auth - valid + CODE=$(curl -o "${TEMPDIR}/curl_out.txt" -w "%{http_code}" -q -s "$PRIMARY_NET/debug/pprof/block" -H "Authorization: Bearer $PRIMARY_ADMIN_TOKEN") + if [[ "$CODE" != "200" ]]; then + fail_and_exit "Call pprof with valid token" "/debug/pprof" "Invalid exit code expected 200 (actual $CODE)" + fi + + # Header Auth - invalid + CODE=$(curl -o "${TEMPDIR}/curl_out.txt" -w "%{http_code}" -q -s "$PRIMARY_NET/debug/pprof/block" -H "Authorization: Bearer invalid_token") + if [[ "$CODE" != "401" ]]; then + fail_and_exit "Call pprof with invalid token" "/debug/pprof" "Invalid exit code expected 401 (actual $CODE)" + fi +} + # Run the tests. test_applications_endpoint test_assets_endpoint +pprof_test From ffb4e69f8d999a8ab865ff64e811fbab58c87857 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Wed, 2 Sep 2020 11:23:14 -0400 Subject: [PATCH 003/136] Add genesis.json endpoint. --- cmd/algod/main.go | 2 +- daemon/algod/api/server/common/handlers.go | 21 +++++++++++++++++++++ daemon/algod/api/server/lib/common.go | 3 +++ daemon/algod/server.go | 5 ++++- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/cmd/algod/main.go b/cmd/algod/main.go index 4e8cfd832c..227b72d931 100644 --- a/cmd/algod/main.go +++ b/cmd/algod/main.go @@ -289,7 +289,7 @@ func main() { } } - err = s.Initialize(cfg, phonebookAddresses) + err = s.Initialize(cfg, phonebookAddresses, string(genesisText[:])) if err != nil { fmt.Fprintln(os.Stderr, err) log.Error(err) diff --git a/daemon/algod/api/server/common/handlers.go b/daemon/algod/api/server/common/handlers.go index 64d8986a92..4883ca2030 100644 --- a/daemon/algod/api/server/common/handlers.go +++ b/daemon/algod/api/server/common/handlers.go @@ -27,6 +27,27 @@ import ( "github.com/algorand/go-algorand/daemon/algod/api/spec/common" ) +// GenesisJSON is an httpHandler for route GET /genesis.json +func GenesisJSON(ctx lib.ReqContext, context echo.Context) { + // swagger:operation GET /genesis.json GenesisJSON + //--- + // Summary: Gets the genesis information + // Description: Returns the entire genesis file in json. + // Produces: + // - application/json + // Schemes: + // - http + // Responses: + // 200: + // description: The current genesis information + // schema: {type: string} + // default: { description: Unknown Error } + w := context.Response().Writer + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(lib.GenesisJSONText)) +} + // SwaggerJSON is an httpHandler for route GET /swagger.json func SwaggerJSON(ctx lib.ReqContext, context echo.Context) { // swagger:operation GET /swagger.json SwaggerJSON diff --git a/daemon/algod/api/server/lib/common.go b/daemon/algod/api/server/lib/common.go index 3562564763..1078acfcd6 100644 --- a/daemon/algod/api/server/lib/common.go +++ b/daemon/algod/api/server/lib/common.go @@ -28,6 +28,9 @@ import ( // SwaggerSpecJSON is autogenerated from swagger.json, and bundled in with a script on build. var SwaggerSpecJSON string +// GenesisJSON is initialized when the node starts. +var GenesisJSONText string + // HandlerFunc defines a wrapper for http.HandlerFunc that includes a context type HandlerFunc func(ReqContext, echo.Context) diff --git a/daemon/algod/server.go b/daemon/algod/server.go index c51c520c2c..ab5e629056 100644 --- a/daemon/algod/server.go +++ b/daemon/algod/server.go @@ -34,6 +34,7 @@ import ( "github.com/algorand/go-algorand/config" apiServer "github.com/algorand/go-algorand/daemon/algod/api/server" + "github.com/algorand/go-algorand/daemon/algod/api/server/lib" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" @@ -59,10 +60,12 @@ type Server struct { } // Initialize creates a Node instance with applicable network services -func (s *Server) Initialize(cfg config.Local, phonebookAddresses []string) error { +func (s *Server) Initialize(cfg config.Local, phonebookAddresses []string, genesisText string) error { // set up node s.log = logging.Base() + lib.GenesisJSONText = genesisText + liveLog := filepath.Join(s.RootPath, "node.log") archive := filepath.Join(s.RootPath, cfg.LogArchiveName) fmt.Println("Logging to: ", liveLog) From 4c1cd34c6c7de743e5f211ff18799e5cfb41aa53 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Wed, 2 Sep 2020 11:23:52 -0400 Subject: [PATCH 004/136] Update spec with genesis.json endpoint. --- daemon/algod/api/algod.oas2.json | 27 ++++++ daemon/algod/api/algod.oas3.yml | 26 ++++++ daemon/algod/api/server/common/routes.go | 7 ++ .../api/server/v2/generated/private/routes.go | 26 +++--- .../algod/api/server/v2/generated/routes.go | 90 +++++++++---------- 5 files changed, 118 insertions(+), 58 deletions(-) diff --git a/daemon/algod/api/algod.oas2.json b/daemon/algod/api/algod.oas2.json index 8210a5d66b..a089c8d6ea 100644 --- a/daemon/algod/api/algod.oas2.json +++ b/daemon/algod/api/algod.oas2.json @@ -69,6 +69,33 @@ } } }, + "/genesis.json": { + "get": { + "description": "Returns the entire genesis file in json.", + "tags": [ + "common" + ], + "produces": [ + "application/json" + ], + "schemes": [ + "http" + ], + "summary": "Gets the genesis information.", + "operationId": "SwaggerJSON", + "responses": { + "200": { + "description": "The genesis file in json.", + "schema": { + "type": "string" + } + }, + "default": { + "description": "Unknown Error" + } + } + } + }, "/swagger.json": { "get": { "description": "Returns the entire swagger spec in json.", diff --git a/daemon/algod/api/algod.oas3.yml b/daemon/algod/api/algod.oas3.yml index 688ee9c26d..0029b14126 100644 --- a/daemon/algod/api/algod.oas3.yml +++ b/daemon/algod/api/algod.oas3.yml @@ -1359,6 +1359,32 @@ }, "openapi": "3.0.1", "paths": { + "/genesis.json": { + "get": { + "description": "Returns the entire genesis file in json.", + "operationId": "SwaggerJSON", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "string" + } + } + }, + "description": "The genesis file in json." + }, + "default": { + "content": {}, + "description": "Unknown Error" + } + }, + "summary": "Gets the genesis information.", + "tags": [ + "common" + ] + } + }, "/health": { "get": { "operationId": "HealthCheck", diff --git a/daemon/algod/api/server/common/routes.go b/daemon/algod/api/server/common/routes.go index 9e0229271a..d974dc7159 100644 --- a/daemon/algod/api/server/common/routes.go +++ b/daemon/algod/api/server/common/routes.go @@ -47,4 +47,11 @@ var Routes = lib.Routes{ Path: "/swagger.json", HandlerFunc: SwaggerJSON, }, + + lib.Route{ + Name: "genesis.json", + Method: "GET", + Path: "/genesis.json", + HandlerFunc: GenesisJSON, + }, } diff --git a/daemon/algod/api/server/v2/generated/private/routes.go b/daemon/algod/api/server/v2/generated/private/routes.go index 5cfcc95a5a..574a86f030 100644 --- a/daemon/algod/api/server/v2/generated/private/routes.go +++ b/daemon/algod/api/server/v2/generated/private/routes.go @@ -347,19 +347,19 @@ var swaggerSpec = []string{ "4q9Hz49SePb8xdERf/GMP37x9DE8+evzZ0fwePnli8WT7MmzJ4tnT559+fxF+vTZ48WzL1/85Qv/oQiL", "aPsRhv9J5QSSk7enyTki224UL8UPsLUvopE7fckHnpLmhoKLfHLsf/pvXk5QgIJv27lfJ+62YbI2ptTH", "8/lms5mFQ+YrqseXGFWn67mfZ1hs5u1pE9C3SQckSzZWi4JO54UwOWWaUNu7b87O2cnb01mrDibHk6PZ", - "0ewxVQApQfJSTI4nT+kn4vo17ft8DTw3KBk308m8AFOJVLu/nAqfuWoX+NPVk7mPAM4/uqv1m11t3dwG", - "92AlGBC8eJx/7JROzEK49B5w/tHnfQRNtg7v/CMFGIPfXSHN+ce2su2N5e4cYpEeX+Gr7U6Vu6jovra/", - "IkP7u0mhu9WFm905zXBXcNTLpspv+M3R9/+ffqHvQ++DJU+Ojv7ziQUqk/rslpTY6dd04gCReb/mGfN3", - "jDT3488396mkVySoqJhVxDfTyfPPufpTiaLAc0Y9g0yTIUv8LC+l2kjfE0/Nuih4tfXirTvKwtf0Jt3M", - "V5oqDVbiihuYfKBSlrFL3RGlQ9+yuLXSoQ90/EfpfC6l88f+csl/lM4fTemcWaVwuNJxhpBN9pjbimit", - "feTfLQ4f83UtuzHN5Qx99oCiyhI2D13CiAUbeRjaXM6rzEaQfHEfn9rkZp0NNNs7B7TzBvkH2Op9au58", - "DeyX9hvtv1ACJl3VTJmq2C88z4Pf6FOb3oSdjXzwvXkseOjX3m9upjG0lgA+HZTSPl1RT1T3l+CflVoa", - "dK5zhxkQbX21JYx+9NWWoQo1m2PBx0dHR7GXFX2cXbTLYkzptxuV5HAF+XCrx5DovS7d9YnE0Q9VDB8F", - "h15nhOv8F4Wbd8KjX4zsvnS9DXavlPzCsA0XrrZ4UFnGfg6kEMZ/TNWmVLkUvubsiH+AM0GQu7/P+6lH", - "3B+vSOfNDmWn17XJ1EaOKy5638NzlyBLKauNs20U8wAaTTVj/it4+dZ/3pVxSu5Stel+ddkXjOjVIm5K", - "Gq2EpAlIymkWmwnOgzxL96WIoRI8c5i9sR/W6Om96McnLY5xuY8J/afy0uEGyM499IVHOn/PURTQ2LNf", - "6UmIckO33wDP5y7dp/ervZQPfuzWIY78Om8eXUUb+8GMWOv8o7l28Yog8EZb1oTc3n9AylM6r9vNNo50", - "PJ/TzfdaaTOfoObpxpjCxg8NUT96FvDEvflw838DAAD//9ssXxsJhQAA", + "0ewxVQApQfJSTI4nT+kn4vo17fvcqZKZK2lxM53M18Bzs3Z/FGAqkWr3l9PqYe+rJ3MfFJx/dLftN7va", + "uukO7g1LMCB4BDn/2KmmmIVw6Yng/KNPBQmabGne+UeKOQa/u9qa849tsdsby/A5xII/vuhX252KeVEd", + "fm1/RR7315VCdwsONxt2muFG4aiXTeHf8DOk7/8//Wjfh943TJ4cHf3nqwtUOfXZLSmx09XphAYi837N", + "M+avHWnux59v7lNJD0tQdzGrm2+mk+efc/WnEkWB54x6BsknQ5b4WV5KtZG+Jx6kdVHwauvFW3eUhS/z", + "TeqarzQVH6zEFTcw+UDVLWP3vCNKhz5vcWulQ9/s+I/S+VxK54/9MZP/KJ0/mtI5s0rhcKXjDCGb/zG3", + "RdJa+8g/ZRy+7+tadmOay9n+7AEFmiVsHrocEgs28la0ua9XmQ0q+Xo/PtvJzTobaLZ3DmjnWfIPsNX7", + "1Nz5Gtgv7Wfbf6GcTLq9mTJVsV94nge/0dc3vQk7G/kGfPN+8NAPwN/cTGNoLQF8hihlgro6n6juL8G/", + "NLU06NzwDpMi2pJrSxj9DqytTBVqNseCj4+OjmKPLfo4uwCYxZgycjcqyeEK8uFWjyHRe3C666uJo9+u", + "GL4TDh3RCNf5jww3T4dHPyLZffx6G+xeKfmFYRsuXLnxoNiM/UJIIYz/vqrNsnJZfc3ZEf8mZ4Igd3+y", + "91OPuD9e3c6bHcpOr2uTqY0cV1z05IfnLmeWslgb/9so5gE0mmrG/Ifx8q3/4ivjlO+latP9ELOvIdEr", + "T9xUOVoJSROQlNMsNjmcB6mX7uMRQyV45jB7Y7+10dN70e9RWhzjch8T+k/lpcMNkJ176GuRdP6eoyig", + "sWc/3JMQ5YZuvwGez10GUO9Xe08f/NgtTRz5dd68w4o29oMZsdb5R3Pt4hVBLI62rInCvf+AlKcMX7eb", + "bWjpeD6ny/C10mY+Qc3TDTuFjR8aon70LOCJe/Ph5v8GAAD//1KcL6cchQAA", } // GetSwagger returns the Swagger specification corresponding to the generated code diff --git a/daemon/algod/api/server/v2/generated/routes.go b/daemon/algod/api/server/v2/generated/routes.go index b2c2dab120..a20b7719e1 100644 --- a/daemon/algod/api/server/v2/generated/routes.go +++ b/daemon/algod/api/server/v2/generated/routes.go @@ -673,51 +673,51 @@ var swaggerSpec = []string{ "5Oh330z/A57+6Vl2+PTxf0z/dPj8MIVnz18cHtIXz+jjF08fw5M/PX92CI9n376YPsmePHsyffbk2bfP", "X6RPnz2ePvv2xX984x+KsBNtHmH4K5YTSI7fnSRnZrLNRtGS/RnW9ka0oU5f8oGmKLmhoCwfHfmf/pfn", "E8NAwdt27teRO20YLbQu1dFkslwuD8IukznW40u0qNLFxI/TLzbz7qQO6NukA+QlG6s1jI76gukcM03w", - "2/sfTs/I8buTg0YcjI5GhweHB4+xAkgJnJZsdDR6ij8h1S9w3ycLoLk2nHE1Hk0K0JKlyv3lRPiBq3Zh", - "frp8MvERwMknd7R+ZeDMY7lUvmpWHYHu36seWzVjvNq6SlZwhUi5m0VjMrVZQ8QVauMZxohtRohRfjV6", - "TrLg7czgMYZx6+nPD/foNatYCafYBfXY+6R1bvvw+zTBE37+2b7nf7qKHG997Lw58uTw8DO8MzJuQfF4", - "ueMHS57d4dTbvvetF9AF11vGG5obeoL6bTq7oMf3dkEnHG+XGAFGrIC+Go+e3+MdOuGGoWhOsGWQ0NIX", - "kb/yCy6W3Lc0yrkqCirXqHqDa+2h7XQ1KIrbqWTufuCwfIagyFhwpbh1JDJdezobE1XXeC4lE8aEwJcc", - "M0glUFT4QuJJYlOuzF2cBFvU+s3xX/Hc4c3xX20dwOgrd8HwtiZmW7j/BDpSTu/7dfNS00ZJ/7XE5/h3", - "+zDg/dGFt1VB+6KM97Yo4w5Ce7+7+5Kb97bk5v02SVd1GjAlXPCEY4mFSyBBxGNvo/6ubdTnh0/v7WpO", - "QV6yFMgZFKWQVLJ8TX7ldS7Z7UzwWuZUPMju2yh/egXmGys6MN+Dck+TT613I7LtQZXW/fCsVWacxt/E", - "DCrhuNzUcXPplfLM5gD5U3419pc/MW5nb1nb/Rj3roYexIz04JDu+/XJq13s8taagvtwMdu8ha/rvcD7", - "WSMZN36v9HNqgN48vqcZ8cnGn1k27yZMnx0++3IzCHfhrdDkR0xP/Mwi/bPGCeJkFQgbLKk2+eSvzu0g", - "YNy11LZo6T5yGxMqhkPH7gaBq8Rcv69h5IkVhPZmcF9qmBF2lRf9m7MxSdHcFvy9yIhrvSG8lwt7uXBj", - "udAlqEYi2BcMJ58wNTsUBz2WxGeY/0AHKEEtPykKX0xGkBnodOFeiO4cVkfEik9pH5Ypmy453lq+7N8H", - "v8374Ds4JHsEf5kH2O9z4CPQliQhb9EcQgb32fp/xLDH59TIn3tBbwUHAiumsManpcX9cWNtLmA5AESK", - "fw4hrL9fmw7uldLJp+bZ4KsmQ8ReL51Yy3+TXWHfcBnd6ZnO/t2de/Duztf3Km7FIZ3VSgjfPgZ3vbrh", - "Fl8itF83s51E5ZqrRaUzsQxSrppSzIOc5F/Bv0NO2j/Fv3+Kf/8U//4p/v1T/Pun+PdP8d/vp/i/vsF1", - "XQerG8T7jF5P24QNTJnGhLN/T5aU6WQmpFVPCdZxiwRQ26P/F2XaVQ90vpUWRliA0dBYCc4KGgcnqLuj", - "wlwW98SGf+ucFZFDVzPUj0LuFK9tgqBaELMwUnHNfBY+PsXk7bnfX/Bzb6nuLdW9pbq3VPeW6t5S3Vuq", - "fyxL9eskO5Ak8YLaZ7LG8ljJ/bSm71Gq6JfM7WwM7Nq8RoPcmMOGvzcegmig+cRVlsPzYqEGs6nCKnWp", - "GY5xUuYUyzGvtL/Tg5WYv33mkyHqeku2UIWRQabB0yfk9Ofj54+f/O3J82/r58XbbR/4yrFKr3Nbfrnt", - "KZwBzV+6uVthAkp/L7J1Z1/N9CY40/aONtfoGacyUsos8sh0FwdaYDlDV5uv50xc3WmCRLyGcR+f21A5", - "UMc3Sn2btnNr+Vh3nd/B3kWKmj316CSuDNpXlagEZ+TIrJEe//Ti80biyqMxykbIhGNDYVmVAr495uhn", - "lZhGc+CJY/JkKrK1f6jC1UhsiTRbvG5Yov2wgrQynIEzcUT9QD10zzxiEc4whhEtHhzUVwaE5/Ks+lLK", - "lknbKKRuvnntosu3Pqrvgtv00D55ICSZS1GVD+2LBXyNzmlRUr724RdjT2HVZnx0E9OL7lYs1hUre0Jt", - "96LDoU2Pd8W6v1u0kCVVvuJwZksOx8sudQvjbsd4U/ZxW0Edu95oidqBgrT9TfS77BIb65BTCTLRKx4p", - "FNkpC/lPn9N7H+XvOykumXEVo+LMhnd1lL0PtophGQgglMOd+5peELel43u6DG9/7iohV4mz2W5t0C3A", - "vvPlDZzI5VajnKSgWUoVJiG6ytyf2djTq5OIp43TxCIFs94lLaMtt5f0R7g7mWIB6Ob5KLxFrJTNwv6q", - "hllTQ+TY5Xy2sLGXEn8UJ/d7z3yKUCLpssucQbX8HcQUXeoVj0qpSfM+XTRHKWCI+kGrOzwB6oFvHwQF", - "L0fZkwjIS0JdCUMMTmpZpfqcUwz6hS929Q+JfChz2DB66ZvE486RsLADdc4pvrFShwKjBtIMYrXjAbz9", - "par5HJTuSOIZwDl3rRhv3nMpWCpFYjP1SpAo0Q9sy4KuyYzmGLX+DaQgU2OyhxdfMVSmNMtzdyplhiFi", - "ds6xUKQR+m+YMc8MOB9NqU9a3SsN4Yvw/ZB0t8RjvzydYupnqhZ++T4igoEb+9kevHz5J4TaBSKjMz95", - "5YpSnLzCe8bNgVRv7l/sQKVgPIkSmdH47ly3S1vkgXvQCgnoYXO05Xb9nBvTWAv7Xnvzmuz1yKEb+O7x", - "ouWOzQUzW/Fxv9bPVTzz8vEW++AW8opExNVec/9xwtPdFw/rjTdGbG/vB/TyHdQA+30X/tqa6LIvs7Uv", - "s7UvxLQvs7Xf3X2ZrX0Rqn0Rqn/WIlQHGy3EySe92qUsTAiVZfahVgmpHbkW4GGzVgGZ/hkg0weEnOEr", - "rNToALgESXN8fFv56+xMkYLNF5qoKk0BsqNznrRmYmvgm4EfNP+1bu55dXj4FMjhQ9LuYsMWgeDtd0VL", - "FT/Z55O+I+ej81EXkIRCXIIrJoGtswqPZW2nrVD/xYE957/I3sYVdG1DKwtalmCUmqpmM5Yyi/BcGFdg", - "Ljr5bFzgF5BmcmDkqSJM27pdiE3MA3RZJ9S9DhUzufva/Ro11Y87xBJPJTdkd80Ku/++S3ndfxbz+hVo", - "ynJVZ7hHvCn0a7qUtaSqYdxapox9YrTyv7nDZzdKzi4gzDnFg/4llZlvEX31rqnU5l917AeW2iWsMlh5", - "g6A76Vk9MtO26JRxN3vvEPXjWq4Q1IYpuGo5Nxl84EXsq/EozYWCxGJJxd4zwg9GEmEslmIolro3rv0z", - "twaGYWZqZifxConNZB8ek/F5Yh9IiISo7Xf3gEIdi+tEviNwPZ0MprPWpGEf1kZp00ViSG0z4m6SD4R/", - "7YuBNhnixu8Gdrr3nmTKs/Pzj+S1LXKIrx9dwHpiXyZJF5TPQdU4CunUXvuwGSxBHnMHjXf3VqHRGsnA", - "K6Mn/dzmLt4vWHoBGTFywj+rPmDCkwd1xTZ8Rnq5WPtLHFYNPTwg5JgTTNz1L0q3I82dwfk3etP4q1Bx", - "tjVSJN8uBXYJ8pY85cFs5iQFhuFuOZQFsnkgveID7ESXEYd21xI+Ef+1400GRGVncRdhgb1W2mulvVba", - "a6W9Vtprpc+mlXohmPsfpOj2uXmUogvp7sIUXz1Q8QcqG7ivEPg7W1CYutkqAXyL2G39AGLMCraT8G9y", - "Yhitfo3zw8erj+abvPQRtuaJyaPJBM2IhVB6Mroaf+o8Pxl+NLKTzi0EF8EqJbvEip4fr/5/AAAA//9U", - "hyTJJOUAAA==", + "2/sfTs/I8buTg0YcjI5GhweHB4+xAkgJnJZsdDR6ij8h1S9w3ydOlBy4khZX49FkATTXC/dHAVqyVLm/", + "nFQPW18+mfig4OSTO22/MqDnsfQqX0irDkr3r1qPreYxjm5dOCu4VaTcZaMxmdpEIuJqt/EMw8Y2ScTo", + "wxpjJ1nwnGbwPsO49Rroh3v0wFWsqlPsznrsydI63X34yZrgVT//kt/zP11FTrw+dp4heXJ4+BmeHhm3", + "oHi83PEbJs/ucOptd/zWC+iC6y3jDc0NPUH9XJ1d0ON7u6ATjhdOjEwjVmZfjUfP7/EOnXDDUDQn2DLI", + "cemLyF/5BRdL7lsafV0VBZVr1MbBTffQnLoaFMXt7DJ3ZXBYPkNQdyy4Zdw6JZmuPZ2NiarLPpeSCWNV", + "4OOOGaQSKNoAQuLhYlPBzN2lBFvn+s3xX/Eo4s3xX21pwOjDd8HwtkxmW7j/BDpSYe/7dfN400ZJ/7XE", + "5/h3+1bg/dGFt1VB+zqN97ZO4w5Ce7+7+yqc97YK5/02SVd1ZjAlXPCEY9WFSyBBEGRvo/6ubdTnh0/v", + "7WpOQV6yFMgZFKWQVLJ8TX7ldXrZ7UzwWuZUPEj42yh/ejXnGys6MN+DClCTT62nJLLtQZXWlfGsVXmc", + "xp/JDIrjuHTVcXMPlvLMpgX5g3819vdBMZRnL17b/Rj3bosexIz04Nzu+/XJq13s8taagityMdu8ha/r", + "Pcr7WSMZN37C9HNqgN48vqcZ8fnHn1k27yZMnx0++3IzCHfhrdDkR8xY/Mwi/bPGCeJkFQgbrLI2+eRv", + "0+0gYNxN1bZo6b57GxMqhkPH7lKBK85cP7lh5IkVhPaycF9qmBF2lRf9y7QxSdFcIPy9yIhrPSu8lwt7", + "uXBjudAlqEYi2EcNJ58wWzsUBz2WxJeZ/0AHKEF5PykKX19GkBnodOEeje6cX0fEis9yH5Ypm+493lq+", + "7J8Mv82T4Ts4JHsEf5k32e9z4CPQliQhb9EcQgb3Cfx/xLDH59TIn3tBbwUHAiumsOynpcX9cWNtLmCF", + "AESKfyEhLMlfmw7u4dLJp+Yl4asmQ8TeOJ1Yy3+TXWGfdRnd6ZnO/imee/AUz9f3Km7FIZ3VSgifQwZ3", + "47rhFl81tF9Ks51E5ZqrRaUzsQxSrprqzIOc5B/Gv0NO2r/Ov3+df/86//51/v3r/PvX+fev89/v1/m/", + "vsF1XQerG8T7jF5P24QNTJnGhLN/T5aU6WQmpFVPCZZ2iwRQ26P/F2XaFRR0vpUWRliA0dBYHM4KGgcn", + "KMWjwlwW9+qGf/6cFZFDVzPUj0LuFK9tgqBaELMwUnHNfBY+vs7k7bnfX/Bzb6nuLdW9pbq3VPeW6t5S", + "3VuqfyxL9eskO5Ak8YLaZ7LG8ljJ/bSm71Gq6JfM7WwM7Nq8RoPcmMOGvzcegmig+cQVm8PzYqEGs6nC", + "wnWpGY5xUuYUKzSvtL/Tg8WZv33mkyHqEky2doWRQabB0yfk9Ofj54+f/O3J82/rF8fbbR/4YrJKr3Nb", + "kbntKZwBzV+6uVthAkp/L7J1Z1/N9CY40/aONjfrGacyUt0s8u50FwdaYIVDV66v50xc3WmCRLyscR+f", + "21A5UNo3Sn2btnNrRVl3w9/B3kWKmj316CSuMtpXlagEZ+TIrJEe//Ti80biyqMxykbIhGNDYVmVAj5H", + "5uhnlZhGc+CJY/JkKrK1f7vClU1siTRbz25Yov2wgrQynIEzcUT9QD10Lz9iXc4whhGtJxyUXAaE5/Ks", + "+lLKVk7bKKRuvnntOsy3Pqrvgtv09j55ICSZS1GVD+0jBnyNzmlRUr724RdjT2EhZ3yHE9OL7lYs1kUs", + "e0Jt9zrEoU2Pd8W6v1u0kCVVvghxZqsQxysxdWvlbsd4UwlyW40du95o1dqBGrX9TfS77BIb65BTCTLR", + "Kx6pHdmpFPlPn9N7H+XvOykumXEVo+LMhnd1lL0PtophGQgglMOd+5peELel43u6DG9/7iohV4mz2W5t", + "0C3APv3lDZzI5VajnKSgWUoVJiG6Yt2f2djTq5OIp43TxCIFs94lLaMtt1f5R7g7mWIB6OZFKbxFrJTN", + "wv6qhllTQ+TY5Xy2sLGXEn8UJ/d7z3yKUCLpssucQQH9HcQUXeoVj0qpSfNkXTRHKWCI+o2rOzwB6oFv", + "HwQFj0nZkwjIS0JdVUMMTmpZpfqcUwz6hY949Q+JfChz2DB66ZvE486RsLADdc4pPrtShwKjBtIMYuXk", + "Abz9par5HJTuSOIZwDl3rRhvnngpWCpFYjP1SpAo0Q9sy4KuyYzmGLX+DaQgU2OyhxdfMVSmNMtzdypl", + "hiFids6xdqQR+m+YMc8MOB9NqU9a3cMN4SPx/ZB0t+pjv2KdYupnqhZ++T4igoEb+9kevHz5V4XaNSOj", + "Mz955YpSnLzCe8bNgVRv7l/sQKVgPIkSmdH47ly3S1vkgXvjCgnoYXO05Xb9nBvTWAv7hHvzwOz1yKEb", + "+O7xouWOzTU0W/Fxv9bPVU/z8vEW++AW8opExNVec/9xwtPdRxDrjTdGbG/vB/TyHdQA+30X/tqa6LIv", + "s7Uvs7UvxLQvs7Xf3X2ZrX0Rqn0Rqn/WIlQHGy3EySe92qUsTAiVZfbtVgmpHbkW4GGzVgGZ/hkg0weE", + "nOHDrNToALgESXN8j1v56+xMkYLNF5qoKk0BsqNznrRmYsvim4EfNP+1bu55dXj4FMjhQ9LuYsMWgeDt", + "d0VLFT/ZF5W+I+ej81EXkIRCXIIrJoGtswqPZW2nrVD/xYE957/I3sYVdG1DKwtalmCUmqpmM5Yyi/Bc", + "GFdgLjr5bFzgF5BmcmDkqSJM27pdiE3MA3RZJ9Q9GBUzufva/Ro11Y87xBJPJTdkd80Ku/++S3ndfxbz", + "+hVoynJVZ7hHvCn0a7qUtaSqYdxapox9YrTyv7nDZzdKzi4gzDnFg/4llZlvEX0Ir6nU5h967AeW2iWs", + "Mlh5g6A76Vk9MtO26JRxN3tPE/XjWq4Q1IYpuGo5Nxl84JHsq/EozYWCxGJJxZ44wg9GEmEslmIolrpn", + "r/3LtwaGYWZqZifxConNZB8ek/F5Yh9IiISo7Xf3gEIdi+tEviNwPZ0MprPWpGHf2kZp00ViSG0z4m6S", + "D4R/7SOCNhnixk8Jdrr3XmnKs/Pzj+S1LXKIDyJdwHpiHytJF5TPQdU4CunUXvuwGSxBHnMHjXf3fKHR", + "GsnAw6Mn/dzmLt4vWHoBGTFywr+0PmDCkwd1xTZ8WXq5WPtLHFYNPTwg5JgTTNz1j0y3I82dwfk3etP4", + "q1BxtjVSJN8uBXYJ8pY85cFs5iQFhuFuOZQFsnkgveID7ESXEYd21xI+Ef+1400GRGVncRdhgb1W2mul", + "vVbaa6W9Vtprpc+mlXohmPsfpOj2uXmUogvp7sIUXz1Q8QcqG7ivEPg7W1CYutkqAXyL2G39AGLMCraT", + "8M90YhitfqDzw8erj+abvPQRtubVyaPJBM2IhVB6Mroaf+q8SBl+NLKTzi0EF8EqJbvEip4fr/5/AAAA", + "///NnzqKN+UAAA==", } // GetSwagger returns the Swagger specification corresponding to the generated code From febe0b64261d156dcef0cc214ca1a3c896b00527 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Wed, 2 Sep 2020 11:59:56 -0400 Subject: [PATCH 005/136] Add e2e test. --- test/scripts/e2e_subs/rest.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/scripts/e2e_subs/rest.sh b/test/scripts/e2e_subs/rest.sh index 89ebda358f..64f4049517 100755 --- a/test/scripts/e2e_subs/rest.sh +++ b/test/scripts/e2e_subs/rest.sh @@ -141,8 +141,18 @@ function pprof_test { fi } +function test_genesis_endpoint { + call_and_verify "There should be a genesis.json endpoint." "/genesis.json" 200 ' + "id": "v1", + "network": "tbd", + "proto": "future", + "rwd": "7777777777777777777777777777777777777777777777777774MSJUVU" +}' +} + # Run the tests. test_applications_endpoint test_assets_endpoint pprof_test +test_genesis_endpoint From 6c707c0f6048180f28cfca660713f818bcc48a76 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Wed, 2 Sep 2020 12:40:50 -0400 Subject: [PATCH 006/136] Fixes for make fmt && make lint --- daemon/algod/api/server/lib/common.go | 2 +- daemon/algod/api/server/lib/middlewares/auth.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/daemon/algod/api/server/lib/common.go b/daemon/algod/api/server/lib/common.go index 1078acfcd6..5c5d1a1286 100644 --- a/daemon/algod/api/server/lib/common.go +++ b/daemon/algod/api/server/lib/common.go @@ -28,7 +28,7 @@ import ( // SwaggerSpecJSON is autogenerated from swagger.json, and bundled in with a script on build. var SwaggerSpecJSON string -// GenesisJSON is initialized when the node starts. +// GenesisJSONText is initialized when the node starts. var GenesisJSONText string // HandlerFunc defines a wrapper for http.HandlerFunc that includes a context diff --git a/daemon/algod/api/server/lib/middlewares/auth.go b/daemon/algod/api/server/lib/middlewares/auth.go index 52a386480a..cd5a38ed08 100644 --- a/daemon/algod/api/server/lib/middlewares/auth.go +++ b/daemon/algod/api/server/lib/middlewares/auth.go @@ -27,9 +27,11 @@ import ( // TokenPathParam is the name of the path parameter used by URLAuthPrefix const TokenPathParam = "token" + // URLAuthPrefix is the echo formatted url/path param which can be used for supplying an API token. const URLAuthPrefix = "/urlAuth/:" + TokenPathParam const urlAuthFormatter = "/urlAuth/%s" + // InvalidTokenMessage is the message set when an invalid / missing token is found. const InvalidTokenMessage = "Invalid API Token" From 23382bac3508ac00cf5d9765280bca47c7067958 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Wed, 2 Sep 2020 14:31:21 -0400 Subject: [PATCH 007/136] Minor code simplification. --- cmd/algod/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/algod/main.go b/cmd/algod/main.go index 227b72d931..fb22c66a89 100644 --- a/cmd/algod/main.go +++ b/cmd/algod/main.go @@ -289,7 +289,7 @@ func main() { } } - err = s.Initialize(cfg, phonebookAddresses, string(genesisText[:])) + err = s.Initialize(cfg, phonebookAddresses, string(genesisText)) if err != nil { fmt.Fprintln(os.Stderr, err) log.Error(err) From 107e9c0ec7d489eea425cdd66ea17e7253adf8c2 Mon Sep 17 00:00:00 2001 From: egieseke Date: Thu, 3 Sep 2020 10:50:29 -0400 Subject: [PATCH 008/136] Recipe for network testing (#1322) Added a new recipe for performing network disruption tests. --- netdeploy/remote/deployedNetwork.go | 5 + netdeploy/remote/hostConfig.go | 1 + netdeploy/remote/topology.go | 1 + .../recipes/network-partition/Makefile | 12 + .../recipes/network-partition/gen_topology.py | 71 + .../recipes/network-partition/genesis.json | 1013 +++++++ .../recipes/network-partition/net.json | 2564 +++++++++++++++++ .../recipes/network-partition/node.json | 22 + .../network-partition/nonPartNode.json | 5 + .../recipes/network-partition/recipe.json | 7 + .../recipes/network-partition/relay.json | 11 + .../recipes/network-partition/topology.json | 154 + 12 files changed, 3866 insertions(+) create mode 100644 test/testdata/deployednettemplates/recipes/network-partition/Makefile create mode 100644 test/testdata/deployednettemplates/recipes/network-partition/gen_topology.py create mode 100644 test/testdata/deployednettemplates/recipes/network-partition/genesis.json create mode 100644 test/testdata/deployednettemplates/recipes/network-partition/net.json create mode 100644 test/testdata/deployednettemplates/recipes/network-partition/node.json create mode 100644 test/testdata/deployednettemplates/recipes/network-partition/nonPartNode.json create mode 100644 test/testdata/deployednettemplates/recipes/network-partition/recipe.json create mode 100644 test/testdata/deployednettemplates/recipes/network-partition/relay.json create mode 100644 test/testdata/deployednettemplates/recipes/network-partition/topology.json diff --git a/netdeploy/remote/deployedNetwork.go b/netdeploy/remote/deployedNetwork.go index 5e10ebeb99..8ce5c2583e 100644 --- a/netdeploy/remote/deployedNetwork.go +++ b/netdeploy/remote/deployedNetwork.go @@ -360,6 +360,7 @@ type cloudHostConfiguration struct { type cloudHostSpec struct { Name string + Group string Provider string Region string InstanceType string @@ -404,6 +405,9 @@ func (cfg DeployedNetwork) GenerateCloudTemplate(templates HostTemplates, target if err != nil { return } + + hostSpec.Group = strings.TrimSpace(cloudHost.Group) + topology.Hosts = append(topology.Hosts, hostSpec) } @@ -488,6 +492,7 @@ func createHostSpec(host HostConfig, template cloudHost) (hostSpec cloudHostSpec } hostSpec.Name = host.Name + hostSpec.Group = host.Group hostSpec.Provider = template.Provider hostSpec.Region = template.Region hostSpec.InstanceType = template.BaseConfiguration diff --git a/netdeploy/remote/hostConfig.go b/netdeploy/remote/hostConfig.go index c620a91d36..f3f15313aa 100644 --- a/netdeploy/remote/hostConfig.go +++ b/netdeploy/remote/hostConfig.go @@ -19,5 +19,6 @@ package remote // HostConfig represents the configuration of a single deployed Host type HostConfig struct { Name string + Group string Nodes []NodeConfig } diff --git a/netdeploy/remote/topology.go b/netdeploy/remote/topology.go index b5b9eb34e6..7abda88fc7 100644 --- a/netdeploy/remote/topology.go +++ b/netdeploy/remote/topology.go @@ -23,6 +23,7 @@ import ( type cloudHostType struct { Name string + Group string Template string } diff --git a/test/testdata/deployednettemplates/recipes/network-partition/Makefile b/test/testdata/deployednettemplates/recipes/network-partition/Makefile new file mode 100644 index 0000000000..24226bc5b6 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/network-partition/Makefile @@ -0,0 +1,12 @@ +PARAMS=-w 100 -R 8 -N 20 -n 100 -H 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json + +all: net.json genesis.json + +net.json: node.json nonPartNode.json ${GOPATH}/bin/netgoal + netgoal generate -t net -r /tmp/wat -o net.json ${PARAMS} + +genesis.json: ${GOPATH}/bin/netgoal + netgoal generate -t genesis -r /tmp/wat -o genesis.json ${PARAMS} + +clean: + rm -f net.json genesis.json diff --git a/test/testdata/deployednettemplates/recipes/network-partition/gen_topology.py b/test/testdata/deployednettemplates/recipes/network-partition/gen_topology.py new file mode 100644 index 0000000000..3de7bea638 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/network-partition/gen_topology.py @@ -0,0 +1,71 @@ +import random + +node_types = {"R":8, "N":20, "NPN":2} +node_size = {"R":"-m5d.4xl", "N":"-m5d.4xl", "NPN":"-m5d.4xl"} +partitions = {"A":50, "B":20, "C":10, "D":10, "E":5, "F":5} +regions = [ + "AWS-US-EAST-2", + "AWS-US-WEST-1" +] + +def gen_topology(ranges): + f = open("topology.json", "w") + f.write("{ \"Hosts\":\n [") + node_groups = {} + + region_count = len(regions) + first = True + for x in node_types: + node_type = x + node_count = node_types[x] + region_size = node_size[x] + for i in range(node_count): + node_name = node_type + str(i+1) + region = regions[i%region_count] + # randomly assign the node to a partition + partition = get_partition(ranges) + node_groups.setdefault(partition,[]).append(node_name); + if (first ): + first = False + else: + f.write(",") + f.write ("\n {\n \"Name\": \"" + node_name + "\",\n \"Group\": \"" + partition + "\",\n \"Template\": \"" + region + region_size + "\"\n }" ) + + f.write("\n ]\n}\n") + f.close() + + for node_group in node_groups: + f = open("group_" + node_group + ".txt", "w") + for node in node_groups[node_group]: + f.write(node +"\n") + f.close() + + +def get_partition(ranges): + random_value = random.randint(1,100) + for partition_name in ranges: + partition_value = ranges[partition_name] + if random_value >= partition_value['start'] and random_value <= partition_value['end'] : + return partition_name + print("error, partition not found for random_value ", random_value) + exit(1) + +def get_ranges(): + ranges = {} + start_pos = 1; + for name, size in partitions.items(): + if (start_pos > 100) : + print("error, range exceeded 100") + exit(1) + end_pos = start_pos + size - 1 + ranges[name] = {"start": start_pos, "end": end_pos} + start_pos = end_pos + 1 + print(ranges) + return ranges + + +# create the group ranges based on group percent size +ranges = get_ranges() + +# gen the topology.json file based and assign groups +gen_topology(ranges) diff --git a/test/testdata/deployednettemplates/recipes/network-partition/genesis.json b/test/testdata/deployednettemplates/recipes/network-partition/genesis.json new file mode 100644 index 0000000000..c1f7c184e9 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/network-partition/genesis.json @@ -0,0 +1,1013 @@ +{ + "NetworkName": "", + "VersionModifier": "", + "ConsensusProtocol": "", + "FirstPartKeyRound": 0, + "LastPartKeyRound": 3000000, + "PartKeyDilution": 0, + "Wallets": [ + { + "Name": "Wallet1", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet2", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet3", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet4", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet5", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet6", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet7", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet8", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet9", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet10", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet11", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet12", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet13", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet14", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet15", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet16", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet17", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet18", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet19", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet20", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet21", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet22", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet23", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet24", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet25", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet26", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet27", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet28", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet29", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet30", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet31", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet32", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet33", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet34", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet35", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet36", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet37", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet38", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet39", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet40", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet41", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet42", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet43", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet44", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet45", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet46", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet47", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet48", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet49", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet50", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet51", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet52", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet53", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet54", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet55", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet56", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet57", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet58", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet59", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet60", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet61", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet62", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet63", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet64", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet65", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet66", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet67", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet68", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet69", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet70", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet71", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet72", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet73", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet74", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet75", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet76", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet77", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet78", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet79", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet80", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet81", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet82", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet83", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet84", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet85", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet86", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet87", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet88", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet89", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet90", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet91", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet92", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet93", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet94", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet95", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet96", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet97", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet98", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet99", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet100", + "Stake": 0.5, + "Online": true + }, + { + "Name": "Wallet101", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet102", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet103", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet104", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet105", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet106", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet107", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet108", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet109", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet110", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet111", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet112", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet113", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet114", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet115", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet116", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet117", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet118", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet119", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet120", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet121", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet122", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet123", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet124", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet125", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet126", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet127", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet128", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet129", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet130", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet131", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet132", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet133", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet134", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet135", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet136", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet137", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet138", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet139", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet140", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet141", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet142", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet143", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet144", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet145", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet146", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet147", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet148", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet149", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet150", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet151", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet152", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet153", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet154", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet155", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet156", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet157", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet158", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet159", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet160", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet161", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet162", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet163", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet164", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet165", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet166", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet167", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet168", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet169", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet170", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet171", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet172", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet173", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet174", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet175", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet176", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet177", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet178", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet179", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet180", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet181", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet182", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet183", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet184", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet185", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet186", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet187", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet188", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet189", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet190", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet191", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet192", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet193", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet194", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet195", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet196", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet197", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet198", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet199", + "Stake": 0.5, + "Online": false + }, + { + "Name": "Wallet200", + "Stake": 0.5, + "Online": false + } + ], + "FeeSink": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ", + "RewardsPool": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ", + "Comment": "" +} diff --git a/test/testdata/deployednettemplates/recipes/network-partition/net.json b/test/testdata/deployednettemplates/recipes/network-partition/net.json new file mode 100644 index 0000000000..3b2b5df750 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/network-partition/net.json @@ -0,0 +1,2564 @@ +{ + "Hosts": [ + { + "Name": "R1", + "Group": "", + "Nodes": [ + { + "Name": "relay1", + "Wallets": null, + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"EnableIncomingMessageFilter\": true, \"CadaverSizeTarget\": 0, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "R2", + "Group": "", + "Nodes": [ + { + "Name": "relay2", + "Wallets": null, + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"EnableIncomingMessageFilter\": true, \"CadaverSizeTarget\": 0, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "R3", + "Group": "", + "Nodes": [ + { + "Name": "relay3", + "Wallets": null, + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"EnableIncomingMessageFilter\": true, \"CadaverSizeTarget\": 0, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "R4", + "Group": "", + "Nodes": [ + { + "Name": "relay4", + "Wallets": null, + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"EnableIncomingMessageFilter\": true, \"CadaverSizeTarget\": 0, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "R5", + "Group": "", + "Nodes": [ + { + "Name": "relay5", + "Wallets": null, + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"EnableIncomingMessageFilter\": true, \"CadaverSizeTarget\": 0, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "R6", + "Group": "", + "Nodes": [ + { + "Name": "relay6", + "Wallets": null, + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"EnableIncomingMessageFilter\": true, \"CadaverSizeTarget\": 0, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "R7", + "Group": "", + "Nodes": [ + { + "Name": "relay7", + "Wallets": null, + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"EnableIncomingMessageFilter\": true, \"CadaverSizeTarget\": 0, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "R8", + "Group": "", + "Nodes": [ + { + "Name": "relay8", + "Wallets": null, + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"EnableIncomingMessageFilter\": true, \"CadaverSizeTarget\": 0, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N1", + "Group": "", + "Nodes": [ + { + "Name": "node1", + "Wallets": [ + { + "Name": "Wallet1", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node21", + "Wallets": [ + { + "Name": "Wallet2", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node41", + "Wallets": [ + { + "Name": "Wallet3", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node61", + "Wallets": [ + { + "Name": "Wallet4", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node81", + "Wallets": [ + { + "Name": "Wallet5", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N2", + "Group": "", + "Nodes": [ + { + "Name": "node2", + "Wallets": [ + { + "Name": "Wallet6", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node22", + "Wallets": [ + { + "Name": "Wallet7", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node42", + "Wallets": [ + { + "Name": "Wallet8", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node62", + "Wallets": [ + { + "Name": "Wallet9", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node82", + "Wallets": [ + { + "Name": "Wallet10", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N3", + "Group": "", + "Nodes": [ + { + "Name": "node3", + "Wallets": [ + { + "Name": "Wallet11", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node23", + "Wallets": [ + { + "Name": "Wallet12", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node43", + "Wallets": [ + { + "Name": "Wallet13", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node63", + "Wallets": [ + { + "Name": "Wallet14", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node83", + "Wallets": [ + { + "Name": "Wallet15", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N4", + "Group": "", + "Nodes": [ + { + "Name": "node4", + "Wallets": [ + { + "Name": "Wallet16", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node24", + "Wallets": [ + { + "Name": "Wallet17", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node44", + "Wallets": [ + { + "Name": "Wallet18", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node64", + "Wallets": [ + { + "Name": "Wallet19", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node84", + "Wallets": [ + { + "Name": "Wallet20", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N5", + "Group": "", + "Nodes": [ + { + "Name": "node5", + "Wallets": [ + { + "Name": "Wallet21", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node25", + "Wallets": [ + { + "Name": "Wallet22", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node45", + "Wallets": [ + { + "Name": "Wallet23", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node65", + "Wallets": [ + { + "Name": "Wallet24", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node85", + "Wallets": [ + { + "Name": "Wallet25", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N6", + "Group": "", + "Nodes": [ + { + "Name": "node6", + "Wallets": [ + { + "Name": "Wallet26", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node26", + "Wallets": [ + { + "Name": "Wallet27", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node46", + "Wallets": [ + { + "Name": "Wallet28", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node66", + "Wallets": [ + { + "Name": "Wallet29", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node86", + "Wallets": [ + { + "Name": "Wallet30", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N7", + "Group": "", + "Nodes": [ + { + "Name": "node7", + "Wallets": [ + { + "Name": "Wallet31", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node27", + "Wallets": [ + { + "Name": "Wallet32", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node47", + "Wallets": [ + { + "Name": "Wallet33", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node67", + "Wallets": [ + { + "Name": "Wallet34", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node87", + "Wallets": [ + { + "Name": "Wallet35", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N8", + "Group": "", + "Nodes": [ + { + "Name": "node8", + "Wallets": [ + { + "Name": "Wallet36", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node28", + "Wallets": [ + { + "Name": "Wallet37", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node48", + "Wallets": [ + { + "Name": "Wallet38", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node68", + "Wallets": [ + { + "Name": "Wallet39", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node88", + "Wallets": [ + { + "Name": "Wallet40", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N9", + "Group": "", + "Nodes": [ + { + "Name": "node9", + "Wallets": [ + { + "Name": "Wallet41", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node29", + "Wallets": [ + { + "Name": "Wallet42", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node49", + "Wallets": [ + { + "Name": "Wallet43", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node69", + "Wallets": [ + { + "Name": "Wallet44", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node89", + "Wallets": [ + { + "Name": "Wallet45", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N10", + "Group": "", + "Nodes": [ + { + "Name": "node10", + "Wallets": [ + { + "Name": "Wallet46", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node30", + "Wallets": [ + { + "Name": "Wallet47", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node50", + "Wallets": [ + { + "Name": "Wallet48", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node70", + "Wallets": [ + { + "Name": "Wallet49", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node90", + "Wallets": [ + { + "Name": "Wallet50", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N11", + "Group": "", + "Nodes": [ + { + "Name": "node11", + "Wallets": [ + { + "Name": "Wallet51", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node31", + "Wallets": [ + { + "Name": "Wallet52", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node51", + "Wallets": [ + { + "Name": "Wallet53", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node71", + "Wallets": [ + { + "Name": "Wallet54", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node91", + "Wallets": [ + { + "Name": "Wallet55", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N12", + "Group": "", + "Nodes": [ + { + "Name": "node12", + "Wallets": [ + { + "Name": "Wallet56", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node32", + "Wallets": [ + { + "Name": "Wallet57", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node52", + "Wallets": [ + { + "Name": "Wallet58", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node72", + "Wallets": [ + { + "Name": "Wallet59", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node92", + "Wallets": [ + { + "Name": "Wallet60", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N13", + "Group": "", + "Nodes": [ + { + "Name": "node13", + "Wallets": [ + { + "Name": "Wallet61", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node33", + "Wallets": [ + { + "Name": "Wallet62", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node53", + "Wallets": [ + { + "Name": "Wallet63", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node73", + "Wallets": [ + { + "Name": "Wallet64", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node93", + "Wallets": [ + { + "Name": "Wallet65", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N14", + "Group": "", + "Nodes": [ + { + "Name": "node14", + "Wallets": [ + { + "Name": "Wallet66", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node34", + "Wallets": [ + { + "Name": "Wallet67", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node54", + "Wallets": [ + { + "Name": "Wallet68", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node74", + "Wallets": [ + { + "Name": "Wallet69", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node94", + "Wallets": [ + { + "Name": "Wallet70", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N15", + "Group": "", + "Nodes": [ + { + "Name": "node15", + "Wallets": [ + { + "Name": "Wallet71", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node35", + "Wallets": [ + { + "Name": "Wallet72", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node55", + "Wallets": [ + { + "Name": "Wallet73", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node75", + "Wallets": [ + { + "Name": "Wallet74", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node95", + "Wallets": [ + { + "Name": "Wallet75", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N16", + "Group": "", + "Nodes": [ + { + "Name": "node16", + "Wallets": [ + { + "Name": "Wallet76", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node36", + "Wallets": [ + { + "Name": "Wallet77", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node56", + "Wallets": [ + { + "Name": "Wallet78", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node76", + "Wallets": [ + { + "Name": "Wallet79", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node96", + "Wallets": [ + { + "Name": "Wallet80", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N17", + "Group": "", + "Nodes": [ + { + "Name": "node17", + "Wallets": [ + { + "Name": "Wallet81", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node37", + "Wallets": [ + { + "Name": "Wallet82", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node57", + "Wallets": [ + { + "Name": "Wallet83", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node77", + "Wallets": [ + { + "Name": "Wallet84", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node97", + "Wallets": [ + { + "Name": "Wallet85", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N18", + "Group": "", + "Nodes": [ + { + "Name": "node18", + "Wallets": [ + { + "Name": "Wallet86", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node38", + "Wallets": [ + { + "Name": "Wallet87", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node58", + "Wallets": [ + { + "Name": "Wallet88", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node78", + "Wallets": [ + { + "Name": "Wallet89", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node98", + "Wallets": [ + { + "Name": "Wallet90", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N19", + "Group": "", + "Nodes": [ + { + "Name": "node19", + "Wallets": [ + { + "Name": "Wallet91", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node39", + "Wallets": [ + { + "Name": "Wallet92", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node59", + "Wallets": [ + { + "Name": "Wallet93", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node79", + "Wallets": [ + { + "Name": "Wallet94", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node99", + "Wallets": [ + { + "Name": "Wallet95", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "N20", + "Group": "", + "Nodes": [ + { + "Name": "node20", + "Wallets": [ + { + "Name": "Wallet96", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node40", + "Wallets": [ + { + "Name": "Wallet97", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node60", + "Wallets": [ + { + "Name": "Wallet98", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node80", + "Wallets": [ + { + "Name": "Wallet99", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + }, + { + "Name": "node100", + "Wallets": [ + { + "Name": "Wallet100", + "ParticipationOnly": false + } + ], + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" + } + ] + }, + { + "Name": "NPN1", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode1", + "Wallets": [ + { + "Name": "Wallet101", + "ParticipationOnly": false + }, + { + "Name": "Wallet111", + "ParticipationOnly": false + }, + { + "Name": "Wallet121", + "ParticipationOnly": false + }, + { + "Name": "Wallet131", + "ParticipationOnly": false + }, + { + "Name": "Wallet141", + "ParticipationOnly": false + }, + { + "Name": "Wallet151", + "ParticipationOnly": false + }, + { + "Name": "Wallet161", + "ParticipationOnly": false + }, + { + "Name": "Wallet171", + "ParticipationOnly": false + }, + { + "Name": "Wallet181", + "ParticipationOnly": false + }, + { + "Name": "Wallet191", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4 }" + } + ] + }, + { + "Name": "NPN2", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode2", + "Wallets": [ + { + "Name": "Wallet102", + "ParticipationOnly": false + }, + { + "Name": "Wallet112", + "ParticipationOnly": false + }, + { + "Name": "Wallet122", + "ParticipationOnly": false + }, + { + "Name": "Wallet132", + "ParticipationOnly": false + }, + { + "Name": "Wallet142", + "ParticipationOnly": false + }, + { + "Name": "Wallet152", + "ParticipationOnly": false + }, + { + "Name": "Wallet162", + "ParticipationOnly": false + }, + { + "Name": "Wallet172", + "ParticipationOnly": false + }, + { + "Name": "Wallet182", + "ParticipationOnly": false + }, + { + "Name": "Wallet192", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4 }" + } + ] + }, + { + "Name": "NPN3", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode3", + "Wallets": [ + { + "Name": "Wallet103", + "ParticipationOnly": false + }, + { + "Name": "Wallet113", + "ParticipationOnly": false + }, + { + "Name": "Wallet123", + "ParticipationOnly": false + }, + { + "Name": "Wallet133", + "ParticipationOnly": false + }, + { + "Name": "Wallet143", + "ParticipationOnly": false + }, + { + "Name": "Wallet153", + "ParticipationOnly": false + }, + { + "Name": "Wallet163", + "ParticipationOnly": false + }, + { + "Name": "Wallet173", + "ParticipationOnly": false + }, + { + "Name": "Wallet183", + "ParticipationOnly": false + }, + { + "Name": "Wallet193", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4 }" + } + ] + }, + { + "Name": "NPN4", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode4", + "Wallets": [ + { + "Name": "Wallet104", + "ParticipationOnly": false + }, + { + "Name": "Wallet114", + "ParticipationOnly": false + }, + { + "Name": "Wallet124", + "ParticipationOnly": false + }, + { + "Name": "Wallet134", + "ParticipationOnly": false + }, + { + "Name": "Wallet144", + "ParticipationOnly": false + }, + { + "Name": "Wallet154", + "ParticipationOnly": false + }, + { + "Name": "Wallet164", + "ParticipationOnly": false + }, + { + "Name": "Wallet174", + "ParticipationOnly": false + }, + { + "Name": "Wallet184", + "ParticipationOnly": false + }, + { + "Name": "Wallet194", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4 }" + } + ] + }, + { + "Name": "NPN5", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode5", + "Wallets": [ + { + "Name": "Wallet105", + "ParticipationOnly": false + }, + { + "Name": "Wallet115", + "ParticipationOnly": false + }, + { + "Name": "Wallet125", + "ParticipationOnly": false + }, + { + "Name": "Wallet135", + "ParticipationOnly": false + }, + { + "Name": "Wallet145", + "ParticipationOnly": false + }, + { + "Name": "Wallet155", + "ParticipationOnly": false + }, + { + "Name": "Wallet165", + "ParticipationOnly": false + }, + { + "Name": "Wallet175", + "ParticipationOnly": false + }, + { + "Name": "Wallet185", + "ParticipationOnly": false + }, + { + "Name": "Wallet195", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4 }" + } + ] + }, + { + "Name": "NPN6", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode6", + "Wallets": [ + { + "Name": "Wallet106", + "ParticipationOnly": false + }, + { + "Name": "Wallet116", + "ParticipationOnly": false + }, + { + "Name": "Wallet126", + "ParticipationOnly": false + }, + { + "Name": "Wallet136", + "ParticipationOnly": false + }, + { + "Name": "Wallet146", + "ParticipationOnly": false + }, + { + "Name": "Wallet156", + "ParticipationOnly": false + }, + { + "Name": "Wallet166", + "ParticipationOnly": false + }, + { + "Name": "Wallet176", + "ParticipationOnly": false + }, + { + "Name": "Wallet186", + "ParticipationOnly": false + }, + { + "Name": "Wallet196", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4 }" + } + ] + }, + { + "Name": "NPN7", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode7", + "Wallets": [ + { + "Name": "Wallet107", + "ParticipationOnly": false + }, + { + "Name": "Wallet117", + "ParticipationOnly": false + }, + { + "Name": "Wallet127", + "ParticipationOnly": false + }, + { + "Name": "Wallet137", + "ParticipationOnly": false + }, + { + "Name": "Wallet147", + "ParticipationOnly": false + }, + { + "Name": "Wallet157", + "ParticipationOnly": false + }, + { + "Name": "Wallet167", + "ParticipationOnly": false + }, + { + "Name": "Wallet177", + "ParticipationOnly": false + }, + { + "Name": "Wallet187", + "ParticipationOnly": false + }, + { + "Name": "Wallet197", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4 }" + } + ] + }, + { + "Name": "NPN8", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode8", + "Wallets": [ + { + "Name": "Wallet108", + "ParticipationOnly": false + }, + { + "Name": "Wallet118", + "ParticipationOnly": false + }, + { + "Name": "Wallet128", + "ParticipationOnly": false + }, + { + "Name": "Wallet138", + "ParticipationOnly": false + }, + { + "Name": "Wallet148", + "ParticipationOnly": false + }, + { + "Name": "Wallet158", + "ParticipationOnly": false + }, + { + "Name": "Wallet168", + "ParticipationOnly": false + }, + { + "Name": "Wallet178", + "ParticipationOnly": false + }, + { + "Name": "Wallet188", + "ParticipationOnly": false + }, + { + "Name": "Wallet198", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4 }" + } + ] + }, + { + "Name": "NPN9", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode9", + "Wallets": [ + { + "Name": "Wallet109", + "ParticipationOnly": false + }, + { + "Name": "Wallet119", + "ParticipationOnly": false + }, + { + "Name": "Wallet129", + "ParticipationOnly": false + }, + { + "Name": "Wallet139", + "ParticipationOnly": false + }, + { + "Name": "Wallet149", + "ParticipationOnly": false + }, + { + "Name": "Wallet159", + "ParticipationOnly": false + }, + { + "Name": "Wallet169", + "ParticipationOnly": false + }, + { + "Name": "Wallet179", + "ParticipationOnly": false + }, + { + "Name": "Wallet189", + "ParticipationOnly": false + }, + { + "Name": "Wallet199", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4 }" + } + ] + }, + { + "Name": "NPN10", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode10", + "Wallets": [ + { + "Name": "Wallet110", + "ParticipationOnly": false + }, + { + "Name": "Wallet120", + "ParticipationOnly": false + }, + { + "Name": "Wallet130", + "ParticipationOnly": false + }, + { + "Name": "Wallet140", + "ParticipationOnly": false + }, + { + "Name": "Wallet150", + "ParticipationOnly": false + }, + { + "Name": "Wallet160", + "ParticipationOnly": false + }, + { + "Name": "Wallet170", + "ParticipationOnly": false + }, + { + "Name": "Wallet180", + "ParticipationOnly": false + }, + { + "Name": "Wallet190", + "ParticipationOnly": false + }, + { + "Name": "Wallet200", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4 }" + } + ] + } + ] +} diff --git a/test/testdata/deployednettemplates/recipes/network-partition/node.json b/test/testdata/deployednettemplates/recipes/network-partition/node.json new file mode 100644 index 0000000000..4386fa3fd7 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/network-partition/node.json @@ -0,0 +1,22 @@ +{ + "APIToken": "{{APIToken}}", + "EnableBlockStats": false, + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }", + "AltConfigs": [ + { + "APIToken": "{{APIToken}}", + "EnableBlockStats": true, + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }", + "FractionApply": 0.2 + } + ] +} + diff --git a/test/testdata/deployednettemplates/recipes/network-partition/nonPartNode.json b/test/testdata/deployednettemplates/recipes/network-partition/nonPartNode.json new file mode 100644 index 0000000000..3825bb420b --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/network-partition/nonPartNode.json @@ -0,0 +1,5 @@ +{ + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4 }" +} diff --git a/test/testdata/deployednettemplates/recipes/network-partition/recipe.json b/test/testdata/deployednettemplates/recipes/network-partition/recipe.json new file mode 100644 index 0000000000..a2f88f63b4 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/network-partition/recipe.json @@ -0,0 +1,7 @@ +{ + "GenesisFile":"genesis.json", + "NetworkFile":"net.json", + "ConfigFile": "../../configs/reference.json", + "HostTemplatesFile": "../../hosttemplates/hosttemplates.json", + "TopologyFile": "topology.json" +} diff --git a/test/testdata/deployednettemplates/recipes/network-partition/relay.json b/test/testdata/deployednettemplates/recipes/network-partition/relay.json new file mode 100644 index 0000000000..25bb6b5a26 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/network-partition/relay.json @@ -0,0 +1,11 @@ +{ + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableBlockStats": true, + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"EnableIncomingMessageFilter\": true, \"CadaverSizeTarget\": 0, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true }" +} diff --git a/test/testdata/deployednettemplates/recipes/network-partition/topology.json b/test/testdata/deployednettemplates/recipes/network-partition/topology.json new file mode 100644 index 0000000000..e050f67368 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/network-partition/topology.json @@ -0,0 +1,154 @@ +{ "Hosts": + [ + { + "Name": "R1", + "Group": "F", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "R2", + "Group": "B", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "R3", + "Group": "A", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "R4", + "Group": "A", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "R5", + "Group": "A", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "R6", + "Group": "B", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "R7", + "Group": "A", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "R8", + "Group": "B", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "N1", + "Group": "B", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "N2", + "Group": "A", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "N3", + "Group": "B", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "N4", + "Group": "B", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "N5", + "Group": "A", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "N6", + "Group": "E", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "N7", + "Group": "A", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "N8", + "Group": "A", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "N9", + "Group": "B", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "N10", + "Group": "A", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "N11", + "Group": "E", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "N12", + "Group": "B", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "N13", + "Group": "A", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "N14", + "Group": "A", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "N15", + "Group": "B", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "N16", + "Group": "A", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "N17", + "Group": "B", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "N18", + "Group": "A", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "N19", + "Group": "C", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "N20", + "Group": "D", + "Template": "AWS-US-WEST-1-m5d.4xl" + }, + { + "Name": "NPN1", + "Group": "A", + "Template": "AWS-US-EAST-2-m5d.4xl" + }, + { + "Name": "NPN2", + "Group": "D", + "Template": "AWS-US-WEST-1-m5d.4xl" + } + ] +} From 7ab606ee04244678ae4a314eb73ddbab5587c934 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Thu, 3 Sep 2020 16:12:33 -0400 Subject: [PATCH 009/136] Improve versions description. --- daemon/algod/api/algod.oas2.json | 2 +- daemon/algod/api/algod.oas3.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/algod/api/algod.oas2.json b/daemon/algod/api/algod.oas2.json index 27948c9a92..3cb0f39308 100644 --- a/daemon/algod/api/algod.oas2.json +++ b/daemon/algod/api/algod.oas2.json @@ -98,7 +98,7 @@ }, "/versions": { "get": { - "description": "Retrieves the current version", + "description": "Retrieves the API version, binary build versions, and genesis information.", "tags": [ "common" ], diff --git a/daemon/algod/api/algod.oas3.yml b/daemon/algod/api/algod.oas3.yml index f59dd055ba..38624c60a4 100644 --- a/daemon/algod/api/algod.oas3.yml +++ b/daemon/algod/api/algod.oas3.yml @@ -3234,7 +3234,7 @@ }, "/versions": { "get": { - "description": "Retrieves the current version", + "description": "Retrieves the API version, binary build versions, and genesis information.", "operationId": "GetVersion", "responses": { "200": { From 0b6774fdc7851cf40f246da8119213236af3f339 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Thu, 3 Sep 2020 16:40:18 -0400 Subject: [PATCH 010/136] Update algod.oas2.json --- daemon/algod/api/algod.oas2.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/algod/api/algod.oas2.json b/daemon/algod/api/algod.oas2.json index 3cb0f39308..a15c7c4f85 100644 --- a/daemon/algod/api/algod.oas2.json +++ b/daemon/algod/api/algod.oas2.json @@ -98,7 +98,7 @@ }, "/versions": { "get": { - "description": "Retrieves the API version, binary build versions, and genesis information.", + "description": "Retrieves the supported API version, binary build versions, and genesis information.", "tags": [ "common" ], From ff5d1f1095a1ce5ce10757ca35c2f71aaf85e224 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Thu, 3 Sep 2020 16:46:06 -0400 Subject: [PATCH 011/136] Allow max and assetIndex to be optional. Closes #1445 --- .../algod/api/server/v1/handlers/handlers.go | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/daemon/algod/api/server/v1/handlers/handlers.go b/daemon/algod/api/server/v1/handlers/handlers.go index 313d8eb20f..163bf056a2 100644 --- a/daemon/algod/api/server/v1/handlers/handlers.go +++ b/daemon/algod/api/server/v1/handlers/handlers.go @@ -1321,20 +1321,28 @@ func Assets(ctx lib.ReqContext, context echo.Context) { const maxAssetsToList = 100 + var err error + var max int64 = maxAssetsToList + var assetIdx int64 = 0 + // Parse max assets to fetch from db - max, err := strconv.ParseInt(r.FormValue("max"), 10, 64) - if err != nil || max < 0 || max > maxAssetsToList { - err := fmt.Errorf(errFailedParsingMaxAssetsToList, 0, maxAssetsToList) - lib.ErrorResponse(w, http.StatusBadRequest, err, err.Error(), ctx.Log) - return + if r.PostFormValue("max") != "" { + max, err = strconv.ParseInt(r.FormValue("max"), 10, 64) + if err != nil || max < 0 || max > maxAssetsToList { + err := fmt.Errorf(errFailedParsingMaxAssetsToList, 0, maxAssetsToList) + lib.ErrorResponse(w, http.StatusBadRequest, err, err.Error(), ctx.Log) + return + } } // Parse maximum asset idx - assetIdx, err := strconv.ParseInt(r.FormValue("assetIdx"), 10, 64) - if err != nil || assetIdx < 0 { - errs := errFailedParsingAssetIdx - lib.ErrorResponse(w, http.StatusBadRequest, errors.New(errs), errs, ctx.Log) - return + if r.PostFormValue("assetIdx") != "" { + assetIdx, err = strconv.ParseInt(r.FormValue("assetIdx"), 10, 64) + if err != nil || assetIdx < 0 { + errs := errFailedParsingAssetIdx + lib.ErrorResponse(w, http.StatusBadRequest, errors.New(errs), errs, ctx.Log) + return + } } // If assetIdx is 0, we want the most recent assets, so make it intmax From 29ef275ed059973040ddc5ef559c73e245770577 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Thu, 3 Sep 2020 16:51:19 -0400 Subject: [PATCH 012/136] Remove .json from the new /genesis.json endpoint. --- daemon/algod/api/algod.oas2.json | 2 +- daemon/algod/api/algod.oas3.yml | 2 +- daemon/algod/api/server/common/handlers.go | 4 +- daemon/algod/api/server/common/routes.go | 4 +- .../api/server/v2/generated/private/routes.go | 26 +++--- .../algod/api/server/v2/generated/routes.go | 90 +++++++++---------- test/scripts/e2e_subs/rest.sh | 2 +- 7 files changed, 65 insertions(+), 65 deletions(-) diff --git a/daemon/algod/api/algod.oas2.json b/daemon/algod/api/algod.oas2.json index a089c8d6ea..6c116c07bd 100644 --- a/daemon/algod/api/algod.oas2.json +++ b/daemon/algod/api/algod.oas2.json @@ -69,7 +69,7 @@ } } }, - "/genesis.json": { + "/genesis": { "get": { "description": "Returns the entire genesis file in json.", "tags": [ diff --git a/daemon/algod/api/algod.oas3.yml b/daemon/algod/api/algod.oas3.yml index 0029b14126..b44790315b 100644 --- a/daemon/algod/api/algod.oas3.yml +++ b/daemon/algod/api/algod.oas3.yml @@ -1359,7 +1359,7 @@ }, "openapi": "3.0.1", "paths": { - "/genesis.json": { + "/genesis": { "get": { "description": "Returns the entire genesis file in json.", "operationId": "SwaggerJSON", diff --git a/daemon/algod/api/server/common/handlers.go b/daemon/algod/api/server/common/handlers.go index 4883ca2030..556c754258 100644 --- a/daemon/algod/api/server/common/handlers.go +++ b/daemon/algod/api/server/common/handlers.go @@ -27,9 +27,9 @@ import ( "github.com/algorand/go-algorand/daemon/algod/api/spec/common" ) -// GenesisJSON is an httpHandler for route GET /genesis.json +// GenesisJSON is an httpHandler for route GET /genesis func GenesisJSON(ctx lib.ReqContext, context echo.Context) { - // swagger:operation GET /genesis.json GenesisJSON + // swagger:operation GET /genesis GenesisJSON //--- // Summary: Gets the genesis information // Description: Returns the entire genesis file in json. diff --git a/daemon/algod/api/server/common/routes.go b/daemon/algod/api/server/common/routes.go index d974dc7159..927a6546c7 100644 --- a/daemon/algod/api/server/common/routes.go +++ b/daemon/algod/api/server/common/routes.go @@ -49,9 +49,9 @@ var Routes = lib.Routes{ }, lib.Route{ - Name: "genesis.json", + Name: "genesis", Method: "GET", - Path: "/genesis.json", + Path: "/genesis", HandlerFunc: GenesisJSON, }, } diff --git a/daemon/algod/api/server/v2/generated/private/routes.go b/daemon/algod/api/server/v2/generated/private/routes.go index 574a86f030..c2dbf63c18 100644 --- a/daemon/algod/api/server/v2/generated/private/routes.go +++ b/daemon/algod/api/server/v2/generated/private/routes.go @@ -347,19 +347,19 @@ var swaggerSpec = []string{ "4q9Hz49SePb8xdERf/GMP37x9DE8+evzZ0fwePnli8WT7MmzJ4tnT559+fxF+vTZ48WzL1/85Qv/oQiL", "aPsRhv9J5QSSk7enyTki224UL8UPsLUvopE7fckHnpLmhoKLfHLsf/pvXk5QgIJv27lfJ+62YbI2ptTH", "8/lms5mFQ+YrqseXGFWn67mfZ1hs5u1pE9C3SQckSzZWi4JO54UwOWWaUNu7b87O2cnb01mrDibHk6PZ", - "0ewxVQApQfJSTI4nT+kn4vo17fvcqZKZK2lxM53M18Bzs3Z/FGAqkWr3l9PqYe+rJ3MfFJx/dLftN7va", - "uukO7g1LMCB4BDn/2KmmmIVw6Yng/KNPBQmabGne+UeKOQa/u9qa849tsdsby/A5xII/vuhX252KeVEd", - "fm1/RR7315VCdwsONxt2muFG4aiXTeHf8DOk7/8//Wjfh943TJ4cHf3nqwtUOfXZLSmx09XphAYi837N", - "M+avHWnux59v7lNJD0tQdzGrm2+mk+efc/WnEkWB54x6BsknQ5b4WV5KtZG+Jx6kdVHwauvFW3eUhS/z", - "TeqarzQVH6zEFTcw+UDVLWP3vCNKhz5vcWulQ9/s+I/S+VxK54/9MZP/KJ0/mtI5s0rhcKXjDCGb/zG3", - "RdJa+8g/ZRy+7+tadmOay9n+7AEFmiVsHrocEgs28la0ua9XmQ0q+Xo/PtvJzTobaLZ3DmjnWfIPsNX7", - "1Nz5Gtgv7Wfbf6GcTLq9mTJVsV94nge/0dc3vQk7G/kGfPN+8NAPwN/cTGNoLQF8hihlgro6n6juL8G/", - "NLU06NzwDpMi2pJrSxj9DqytTBVqNseCj4+OjmKPLfo4uwCYxZgycjcqyeEK8uFWjyHRe3C666uJo9+u", - "GL4TDh3RCNf5jww3T4dHPyLZffx6G+xeKfmFYRsuXLnxoNiM/UJIIYz/vqrNsnJZfc3ZEf8mZ4Igd3+y", - "91OPuD9e3c6bHcpOr2uTqY0cV1z05IfnLmeWslgb/9so5gE0mmrG/Ifx8q3/4ivjlO+latP9ELOvIdEr", - "T9xUOVoJSROQlNMsNjmcB6mX7uMRQyV45jB7Y7+10dN70e9RWhzjch8T+k/lpcMNkJ176GuRdP6eoyig", - "sWc/3JMQ5YZuvwGez10GUO9Xe08f/NgtTRz5dd68w4o29oMZsdb5R3Pt4hVBLI62rInCvf+AlKcMX7eb", - "bWjpeD6ny/C10mY+Qc3TDTuFjR8aon70LOCJe/Ph5v8GAAD//1KcL6cchQAA", + "0ewxVQApQfJSTI4nT+kn4vo17fvcqZLJ8ceb6WS+Bp6btfujAFOJ1Dc5hT5ztS/wp6sncx8PnH90F+03", + "u9q6mQ7u+UowIHj/OP/YKaSYhXDpdeD8o88CCZpsVd75Rwo3Br+7sprzj22d2xvL6znE4j6+3lfbnep4", + "UQl+bX9F9vY3lUJ3aw03e3Wa4R7hqJdNzd/wC6Tv/z/9Xt+H3udLnhwd/eeDC1Q09dktKbHTy+lEBSLz", + "fs0z5m8cae7Hn2/uU0lvSlBtMauWb6aT559z9acSRYHnjHoGeSdDlvhZXkq1kb4nnqF1UfBq68Vbd5SF", + "r/BNmpqvNNUdrMQVNzD5QIUtY1e8I0qHvmxxa6VDn+v4j9L5XErnj/0dk/8onT+a0jmzSuFwpeMMIZv6", + "Mbf10Vr7yL9iHD7t61p2Y5rLmf3sAcWYJWweuvQRCzbyTLS5qleZjSf5Uj8+0cnNOhtotncOaOdF8g+w", + "1fvU3Pka2C/tF9t/oXRMuriZMlWxX3ieB7/Rhze9CTsb+fx783Tw0G+/39xMY2gtAXxyKCWBuhKfqO4v", + "wT8ytTToXO4O8yHaamtLGP0ErC1KFWo2x4KPj46OYu8s+ji72JfFmJJxNyrJ4Qry4VaPIdF7a7rrg4mj", + "n60YPhEOfdAI1/nvCzevhke/H9l993ob7F4p+YVhGy5cpfGgzoz9OEghjP+0qk2wcgl9zdkR/xxngiB3", + "f633U4+4P17Jzpsdyk6va5OpjRxXXPTah+cuXZYSWBvX2yjmATSaasb8N/Hyrf/YK+OU6qVq0/0Gsy8f", + "0atM3BQ4WglJE5CU0yw2L5wHWZfuuxFDJXjmMHtjP7PR03vRT1FaHONyHxP6T+Wlww2QnXvoy5B0/p6j", + "KKCxZ7/ZkxDlhm6/AZ7PXfJP71d7RR/82K1KHPl13jzBijb2gxmx1vlHc+3iFUEYjrasCcC9/4CUp+Re", + "t5ttVOl4Pqd78LXSZj5BzdONOIWNHxqifvQs4Il78+Hm/wYAAP//K8QBZBeFAAA=", } // GetSwagger returns the Swagger specification corresponding to the generated code diff --git a/daemon/algod/api/server/v2/generated/routes.go b/daemon/algod/api/server/v2/generated/routes.go index a20b7719e1..6e00b2c6ef 100644 --- a/daemon/algod/api/server/v2/generated/routes.go +++ b/daemon/algod/api/server/v2/generated/routes.go @@ -673,51 +673,51 @@ var swaggerSpec = []string{ "5Oh330z/A57+6Vl2+PTxf0z/dPj8MIVnz18cHtIXz+jjF08fw5M/PX92CI9n376YPsmePHsyffbk2bfP", "X6RPnz2ePvv2xX984x+KsBNtHmH4K5YTSI7fnSRnZrLNRtGS/RnW9ka0oU5f8oGmKLmhoCwfHfmf/pfn", "E8NAwdt27teRO20YLbQu1dFkslwuD8IukznW40u0qNLFxI/TLzbz7qQO6NukA+QlG6s1jI76gukcM03w", - "2/sfTs/I8buTg0YcjI5GhweHB4+xAkgJnJZsdDR6ij8h1S9w3ydOlBy4khZX49FkATTXC/dHAVqyVLm/", - "nFQPW18+mfig4OSTO22/MqDnsfQqX0irDkr3r1qPreYxjm5dOCu4VaTcZaMxmdpEIuJqt/EMw8Y2ScTo", - "wxpjJ1nwnGbwPsO49Rroh3v0wFWsqlPsznrsydI63X34yZrgVT//kt/zP11FTrw+dp4heXJ4+BmeHhm3", - "oHi83PEbJs/ucOptd/zWC+iC6y3jDc0NPUH9XJ1d0ON7u6ATjhdOjEwjVmZfjUfP7/EOnXDDUDQn2DLI", - "cemLyF/5BRdL7lsafV0VBZVr1MbBTffQnLoaFMXt7DJ3ZXBYPkNQdyy4Zdw6JZmuPZ2NiarLPpeSCWNV", - "4OOOGaQSKNoAQuLhYlPBzN2lBFvn+s3xX/Eo4s3xX21pwOjDd8HwtkxmW7j/BDpSYe/7dfN400ZJ/7XE", - "5/h3+1bg/dGFt1VB+zqN97ZO4w5Ce7+7+yqc97YK5/02SVd1ZjAlXPCEY9WFSyBBEGRvo/6ubdTnh0/v", - "7WpOQV6yFMgZFKWQVLJ8TX7ldXrZ7UzwWuZUPEj42yh/ejXnGys6MN+DClCTT62nJLLtQZXWlfGsVXmc", - "xp/JDIrjuHTVcXMPlvLMpgX5g3819vdBMZRnL17b/Rj3bosexIz04Nzu+/XJq13s8taagityMdu8ha/r", - "Pcr7WSMZN37C9HNqgN48vqcZ8fnHn1k27yZMnx0++3IzCHfhrdDkR8xY/Mwi/bPGCeJkFQgbrLI2+eRv", - "0+0gYNxN1bZo6b57GxMqhkPH7lKBK85cP7lh5IkVhPaycF9qmBF2lRf9y7QxSdFcIPy9yIhrPSu8lwt7", - "uXBjudAlqEYi2EcNJ58wWzsUBz2WxJeZ/0AHKEF5PykKX19GkBnodOEeje6cX0fEis9yH5Ypm+493lq+", - "7J8Mv82T4Ts4JHsEf5k32e9z4CPQliQhb9EcQgb3Cfx/xLDH59TIn3tBbwUHAiumsOynpcX9cWNtLmCF", - "AESKfyEhLMlfmw7u4dLJp+Yl4asmQ8TeOJ1Yy3+TXWGfdRnd6ZnO/imee/AUz9f3Km7FIZ3VSgifQwZ3", - "47rhFl81tF9Ks51E5ZqrRaUzsQxSrprqzIOc5B/Gv0NO2r/Ov3+df/86//51/v3r/PvX+fev89/v1/m/", - "vsF1XQerG8T7jF5P24QNTJnGhLN/T5aU6WQmpFVPCZZ2iwRQ26P/F2XaFRR0vpUWRliA0dBYHM4KGgcn", - "KMWjwlwW9+qGf/6cFZFDVzPUj0LuFK9tgqBaELMwUnHNfBY+vs7k7bnfX/Bzb6nuLdW9pbq3VPeW6t5S", - "3VuqfyxL9eskO5Ak8YLaZ7LG8ljJ/bSm71Gq6JfM7WwM7Nq8RoPcmMOGvzcegmig+cQVm8PzYqEGs6nC", - "wnWpGY5xUuYUKzSvtL/Tg8WZv33mkyHqEky2doWRQabB0yfk9Ofj54+f/O3J82/rF8fbbR/4YrJKr3Nb", - "kbntKZwBzV+6uVthAkp/L7J1Z1/N9CY40/aONjfrGacyUt0s8u50FwdaYIVDV66v50xc3WmCRLyscR+f", - "21A5UNo3Sn2btnNrRVl3w9/B3kWKmj316CSuMtpXlagEZ+TIrJEe//Ti80biyqMxykbIhGNDYVmVAj5H", - "5uhnlZhGc+CJY/JkKrK1f7vClU1siTRbz25Yov2wgrQynIEzcUT9QD10Lz9iXc4whhGtJxyUXAaE5/Ks", - "+lLKVk7bKKRuvnntOsy3Pqrvgtv09j55ICSZS1GVD+0jBnyNzmlRUr724RdjT2EhZ3yHE9OL7lYs1kUs", - "e0Jt9zrEoU2Pd8W6v1u0kCVVvghxZqsQxysxdWvlbsd4UwlyW40du95o1dqBGrX9TfS77BIb65BTCTLR", - "Kx6pHdmpFPlPn9N7H+XvOykumXEVo+LMhnd1lL0PtophGQgglMOd+5peELel43u6DG9/7iohV4mz2W5t", - "0C3APv3lDZzI5VajnKSgWUoVJiG6Yt2f2djTq5OIp43TxCIFs94lLaMtt1f5R7g7mWIB6OZFKbxFrJTN", - "wv6qhllTQ+TY5Xy2sLGXEn8UJ/d7z3yKUCLpssucQQH9HcQUXeoVj0qpSfNkXTRHKWCI+o2rOzwB6oFv", - "HwQFj0nZkwjIS0JdVUMMTmpZpfqcUwz6hY949Q+JfChz2DB66ZvE486RsLADdc4pPrtShwKjBtIMYuXk", - "Abz9par5HJTuSOIZwDl3rRhvnngpWCpFYjP1SpAo0Q9sy4KuyYzmGLX+DaQgU2OyhxdfMVSmNMtzdypl", - "hiFids6xdqQR+m+YMc8MOB9NqU9a3cMN4SPx/ZB0t+pjv2KdYupnqhZ++T4igoEb+9kevHz5V4XaNSOj", - "Mz955YpSnLzCe8bNgVRv7l/sQKVgPIkSmdH47ly3S1vkgXvjCgnoYXO05Xb9nBvTWAv7hHvzwOz1yKEb", - "+O7xouWOzTU0W/Fxv9bPVU/z8vEW++AW8opExNVec/9xwtPdRxDrjTdGbG/vB/TyHdQA+30X/tqa6LIv", - "s7Uvs7UvxLQvs7Xf3X2ZrX0Rqn0Rqn/WIlQHGy3EySe92qUsTAiVZfbtVgmpHbkW4GGzVgGZ/hkg0weE", - "nOHDrNToALgESXN8j1v56+xMkYLNF5qoKk0BsqNznrRmYsvim4EfNP+1bu55dXj4FMjhQ9LuYsMWgeDt", - "d0VLFT/ZF5W+I+ej81EXkIRCXIIrJoGtswqPZW2nrVD/xYE957/I3sYVdG1DKwtalmCUmqpmM5Yyi/Bc", - "GFdgLjr5bFzgF5BmcmDkqSJM27pdiE3MA3RZJ9Q9GBUzufva/Ro11Y87xBJPJTdkd80Ku/++S3ndfxbz", - "+hVoynJVZ7hHvCn0a7qUtaSqYdxapox9YrTyv7nDZzdKzi4gzDnFg/4llZlvEX0Ir6nU5h967AeW2iWs", - "Mlh5g6A76Vk9MtO26JRxN3tPE/XjWq4Q1IYpuGo5Nxl84JHsq/EozYWCxGJJxZ44wg9GEmEslmIolrpn", - "r/3LtwaGYWZqZifxConNZB8ek/F5Yh9IiISo7Xf3gEIdi+tEviNwPZ0MprPWpGHf2kZp00ViSG0z4m6S", - "D4R/7SOCNhnixk8Jdrr3XmnKs/Pzj+S1LXKIDyJdwHpiHytJF5TPQdU4CunUXvuwGSxBHnMHjXf3fKHR", - "GsnAw6Mn/dzmLt4vWHoBGTFywr+0PmDCkwd1xTZ8WXq5WPtLHFYNPTwg5JgTTNz1j0y3I82dwfk3etP4", - "q1BxtjVSJN8uBXYJ8pY85cFs5iQFhuFuOZQFsnkgveID7ESXEYd21xI+Ef+1400GRGVncRdhgb1W2mul", - "vVbaa6W9Vtprpc+mlXohmPsfpOj2uXmUogvp7sIUXz1Q8QcqG7ivEPg7W1CYutkqAXyL2G39AGLMCraT", - "8M90YhitfqDzw8erj+abvPQRtubVyaPJBM2IhVB6Mroaf+q8SBl+NLKTzi0EF8EqJbvEip4fr/5/AAAA", - "///NnzqKN+UAAA==", + "2/sfTs/I8buTg0YcjI5GhweHB4+xAkgJnJZsdDR6ij8h1S9w3ydOlIyOPl2NR5MF0Fwv3B8FaMlS/8kJ", + "9ANX+8L8dPlk4uOBk0/uoP3KQJ3HMqt8Da06Ht2/ZT22Ssf4uHXNrOBCkXL3jMZkanOIiCvbxjOMGNv8", + "EKMKa2SdZMFLmsHTDOPWQ6Af7tHbVrGCTrHr6rHXSutM9+HXaoIH/fwjfs//dBU57PrYeYHkyeHhZ3h1", + "ZNyC4vFyx8+XPLvDqbc98VsvoAuut4w3NDf0BPVLdXZBj+/tgk443jUx4oxYcX01Hj2/xzt0wg1D0Zxg", + "yyC9pS8if+UXXCy5b2lUdVUUVK5REQeX3ENL6mpQFLcTy9xtwWH5DEHJseCCceuAZLr2dDYmqq74XEom", + "jEGB7zpmkEqgqP6FxHPFpniZu0YJtsT1m+O/4inEm+O/2qqA0TfvguFthcy2cP8JdKS43vfr5t2mjZL+", + "a4nP8e/2mcD7owtvq4L2JRrvbYnGHYT2fnf3BTjvbQHO+22SruqkYEq44AnHgguXQIL4x95G/V3bqM8P", + "n97b1ZyCvGQpkDMoSiGpZPma/MrrzLLbmeC1zKl4kOu3Uf70ys03VnRgvgfFnyafWq9IZNuDKq3b4lmr", + "6DiNv5AZ1MVxmarj5gos5ZnNCPJn/mrsr4JiFM/eubb7Me5dFD2IGenBkd3365NXu9jlrTUFt+NitnkL", + "X9d7j/ezRjJu/Hrp59QAvXl8TzPiU48/s2zeTZg+O3z25WYQ7sJbocmPmKz4mUX6Z40TxMkqEDZYYG3y", + "yV+k20HAuEuqbdHSffI2JlQMh47dfQJXl7l+bcPIEysI7T3hvtQwI+wqL/r3aGOSork7+HuREdd6UXgv", + "F/Zy4cZyoUtQjUSw7xlOPmGidigOeiyJjzL/gQ5Qgsp+UhS+tIwgM9Dpwr0X3Tm6jogVn+A+LFM2XXm8", + "tXzZvxZ+m9fCd3BI9gj+Ms+x3+fAR6AtSULeojmEDO5z9/+IYY/PqZE/94LeCg4EVkxhxU9Li/vjxtpc", + "wOIAiBT/OEJYjb82HdybpZNPzSPCV02GiL1sOrGW/ya7wr7oMrrTM539Kzz34BWer+9V3IpDOquVEL6E", + "DO6ydcMtvmBov4pmO4nKNVeLSmdiGaRcNYWZBznJv4l/h5y0f5h//zD//mH+/cP8+4f59w/z7x/mv98P", + "8399g+u6DlY3iPcZvZ62CRuYMo0JZ/+eLCnTyUxIq54SrOoWCaC2R/8vyrSrJeh8Ky2MsACjobEunBU0", + "Dk5QhUeFuSzuwQ3/8jkrIoeuZqgfhdwpXtsEQbUgZmGk4pr5LHx8mMnbc7+/4OfeUt1bqntLdW+p7i3V", + "vaW6t1T/WJbq10l2IEniBbXPZI3lsZL7aU3fo1TRL5nb2RjYtXmNBrkxhw1/bzwE0UDziaszh+fFQg1m", + "U4U161IzHOOkzCkWZ15pf6cH6zJ/+8wnQ9TVl2zZCiODTIOnT8jpz8fPHz/525Pn39aPjbfbPvB1ZJVe", + "57YYc9tTOAOav3Rzt8IElP5eZOvOvprpTXCm7R1tLtUzTmWksFnkyekuDrTA4oauUl/Pmbi60wSJeEXj", + "Pj63oXKgqm+U+jZt59Zisu5yv4O9ixQ1e+rRSVxRtK8qUQnOyJFZIz3+6cXnjcSVR2OUjZAJx4bCsioF", + "fInM0c8qMY3mwBPH5MlUZGv/bIWrmNgSabaU3bBE+2EFaWU4A2fiiPqBeugefcSSnGEMI1pKOKi2DAjP", + "5Vn1pZQtmrZRSN1889olmG99VN8Ft+nZffJASDKXoiof2vcL+Bqd06KkfO3DL8aewhrO+AQnphfdrVis", + "61f2hNruJYhDmx7vinV/t2ghS6p8/eHMFiCOF2HqlsndjvGmCOS28jp2vdGCtQPlafub6HfZJTbWIacS", + "ZKJXPFI2slMk8p8+p/c+yt93Ulwy4ypGxZkN7+ooex9sFcMyEEAohzv3Nb0gbkvH93QZ3v7cVUKuEmez", + "3dqgW4B99csbOJHLrUY5SUGzlCpMQnR1uj+zsadXJxFPG6eJRQpmvUtaRltuL/CPcHcyxQLQzWNSeItY", + "KZuF/VUNs6aGyLHL+WxhYy8l/ihO7vee+RShRNJllzmD2vk7iCm61CselVKT5rW6aI5SwBD181Z3eALU", + "A98+CArekbInEZCXhLqChhic1LJK9TmnGPQL3+/qHxL5UOawYfTSN4nHnSNhYQfqnFN8caUOBUYNpBnE", + "KskDePtLVfM5KN2RxDOAc+5aMd687lKwVIrEZuqVIFGiH9iWBV2TGc0xav0bSEGmxmQPL75iqExplufu", + "VMoMQ8TsnGPZSCP03zBjnhlwPppSn7S6NxvC9+H7Ieluwcd+sTrF1M9ULfzyfUQEAzf2sz14+fIPCrXL", + "RUZnfvLKFaU4eYX3jJsDqd7cv9iBSsF4EiUyo/HduW6XtsgD97wVEtDD5mjL7fo5N6axFvb19uZt2euR", + "Qzfw3eNFyx2by2e24uN+rZ+rlObl4y32wS3kFYmIq73m/uOEp7vvH9Ybb4zY3t4P6OU7qAH2+y78tTXR", + "ZV9ma19ma1+IaV9ma7+7+zJb+yJU+yJU/6xFqA42WoiTT3q1S1mYECrL7LOtElI7ci3Aw2atAjL9M0Cm", + "Dwg5wzdZqdEBcAmS5vgUt/LX2ZkiBZsvNFFVmgJkR+c8ac3EVsQ3Az9o/mvd3PPq8PApkMOHpN3Fhi0C", + "wdvvipYqfrKPKX1Hzkfnoy4gCYW4BFdMAltnFR7L2k5bof6LA3vOf5G9jSvo2oZWFrQswSg1Vc1mLGUW", + "4bkwrsBcdPLZuMAvIM3kwMhTRZi2dbsQm5gH6LJOqHsrKmZy97X7NWqqH3eIJZ5KbsjumhV2/32X8rr/", + "LOb1K9CU5arOcI94U+jXdClrSVXDuLVMGfvEaOV/c4fPbpScXUCYc4oH/UsqM98i+gZeU6nNv/HYDyy1", + "S1hlsPIGQXfSs3pkpm3RKeNu9l4l6se1XCGoDVNw1XJuMvjA+9hX41GaCwWJxZKKvW6EH4wkwlgsxVAs", + "dS9e+0dvDQzDzNTMTuIVEpvJPjwm4/PEPpAQCVHb7+4BhToW14l8R+B6OhlMZ61Jwz6zjdKmi8SQ2mbE", + "3SQfCP/a9wNtMsSNXxHsdO890JRn5+cfyWtb5BDfQrqA9cS+U5IuKJ+DqnEU0qm99mEzWII85g4a7+7l", + "QqM1koE3R0/6uc1dvF+w9AIyYuSEf2R9wIQnD+qKbfio9HKx9pc4rBp6eEDIMSeYuOvfl25HmjuD82/0", + "pvFXoeJsa6RIvl0K7BLkLXnKg9nMSQoMw91yKAtk80B6xQfYiS4jDu2uJXwi/mvHmwyIys7iLsICe620", + "10p7rbTXSnuttNdKn00r9UIw9z9I0e1z8yhFF9LdhSm+eqDiD1Q2cF8h8He2oDB1s1UC+Bax2/oBxJgV", + "bCfhX+jEMFr9NueHj1cfzTd56SNszYOTR5MJmhELofRkdDX+1HmMMvxoZCedWwguglVKdokVPT9e/f8A", + "AAD//2QRP+Uy5QAA", } // GetSwagger returns the Swagger specification corresponding to the generated code diff --git a/test/scripts/e2e_subs/rest.sh b/test/scripts/e2e_subs/rest.sh index 64f4049517..3f915406b0 100755 --- a/test/scripts/e2e_subs/rest.sh +++ b/test/scripts/e2e_subs/rest.sh @@ -142,7 +142,7 @@ function pprof_test { } function test_genesis_endpoint { - call_and_verify "There should be a genesis.json endpoint." "/genesis.json" 200 ' + call_and_verify "There should be a genesis endpoint." "/genesis" 200 ' "id": "v1", "network": "tbd", "proto": "future", From a6c82751f742d403b7496118a8c7ad58e16b4d5d Mon Sep 17 00:00:00 2001 From: Will Winder Date: Thu, 3 Sep 2020 22:44:52 -0400 Subject: [PATCH 013/136] No need to return an error when requesting the same catchpoint multiple times (#1439) No need to return an error when requesting the same catchpoint multiple times. --- daemon/algod/api/algod.oas2.json | 4 + daemon/algod/api/algod.oas3.yml | 20 ++ daemon/algod/api/client/restClient.go | 4 +- .../api/server/v2/generated/private/routes.go | 250 +++++++++--------- daemon/algod/api/server/v2/handlers.go | 13 +- .../algod/api/server/v2/test/handlers_test.go | 31 ++- daemon/algod/api/server/v2/test/helpers.go | 17 +- node/error.go | 55 ++++ node/node.go | 6 +- 9 files changed, 254 insertions(+), 146 deletions(-) create mode 100644 node/error.go diff --git a/daemon/algod/api/algod.oas2.json b/daemon/algod/api/algod.oas2.json index 8210a5d66b..9af4c9ce9d 100644 --- a/daemon/algod/api/algod.oas2.json +++ b/daemon/algod/api/algod.oas2.json @@ -926,6 +926,10 @@ "description": "OK", "$ref": "#/responses/CatchpointStartResponse" }, + "201": { + "description": "OK", + "$ref": "#/responses/CatchpointStartResponse" + }, "400": { "description": "Bad Request", "schema": { diff --git a/daemon/algod/api/algod.oas3.yml b/daemon/algod/api/algod.oas3.yml index 688ee9c26d..c99599c54e 100644 --- a/daemon/algod/api/algod.oas3.yml +++ b/daemon/algod/api/algod.oas3.yml @@ -2092,6 +2092,26 @@ }, "description": "(empty)" }, + "201": { + "content": { + "application/json": { + "schema": { + "description": "An catchpoint start response.", + "properties": { + "catchup-message": { + "description": "Catchup start response string", + "type": "string" + } + }, + "required": [ + "catchup-message" + ], + "type": "object" + } + } + }, + "description": "(empty)" + }, "400": { "content": { "application/json": { diff --git a/daemon/algod/api/client/restClient.go b/daemon/algod/api/client/restClient.go index 9db7b17d51..f7d8449d8e 100644 --- a/daemon/algod/api/client/restClient.go +++ b/daemon/algod/api/client/restClient.go @@ -87,11 +87,11 @@ func (client *RestClient) SetAPIVersionAffinity(affinity APIVersion) (previousAf return } -// extractError checks if the response signifies an error (for now, StatusCode != 200). +// extractError checks if the response signifies an error (for now, StatusCode != 200 or StatusCode != 201). // If so, it returns the error. // Otherwise, it returns nil. func extractError(resp *http.Response) error { - if resp.StatusCode == 200 { + if resp.StatusCode == 200 || resp.StatusCode == 201 { return nil } diff --git a/daemon/algod/api/server/v2/generated/private/routes.go b/daemon/algod/api/server/v2/generated/private/routes.go index 5cfcc95a5a..9a1075386d 100644 --- a/daemon/algod/api/server/v2/generated/private/routes.go +++ b/daemon/algod/api/server/v2/generated/private/routes.go @@ -235,131 +235,131 @@ func RegisterHandlers(router interface { // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9f3fbNhLgV8Fp970mOVFyfnU3fq9vz036w9c0zYvdvbuNc1uIHEmoSYAlQMtqzt/9", - "3gwAEiRBSXa82eu7/SuxAAwGg5nBzGAw/DhJVVEqCdLoyfHHSckrXoCBiv7iaapqaRKR4V8Z6LQSpRFK", - "To59G9OmEnI1mU4E/lpys55MJ5IX0PbB8dNJBb/VooJscmyqGqYTna6h4AjYbEvs3UC6TlYqcSBOLIjT", - "V5ObHQ08yyrQeojlTzLfMiHTvM6AmYpLzVNs0mwjzJqZtdDMDWZCMiWBqSUz605nthSQZ3rmF/lbDdU2", - "WKWbfHxJNy2KSaVyGOL5UhULIcFjBQ1SzYYwo1gGS+q05obhDIir72gU08CrdM2WqtqDqkUixBdkXUyO", - "3080yAwq2q0UxBX9d1kB/A6J4dUKzOTDNLa4pYEqMaKILO3UUb8CXedGM+pLa1yJK5AMR83Yj7U2bAGM", - "S/bu25fs6dOnL3AhBTcGMsdko6tqZw/XZIdPjicZN+Cbh7zG85WquMySpv+7b1/S/GdugYf24lpDXFhO", - "sIWdvhpbgB8YYSEhDaxoHzrcjyMiQtH+vIClquDAPbGd73VTwvn/rbuScpOuSyWkiewLo1Zmm6M6LBi+", - "S4c1CHT6l0ipCoG+P0pefPj4ePr46OZP70+Sf7g/nz+9OXD5Lxu4eygQ7ZjWVQUy3SarCjhJy5rLIT3e", - "OX7Qa1XnGVvzK9p8XpCqd2MZjrWq84rnNfKJSCt1kq+UZtyxUQZLXueG+YlZLXNUUwjNcTsTmpWVuhIZ", - "ZFPUvpu1SNcs5dqCoH5sI/IcebDWkI3xWnx1O4TpJiQJ4nUnetCC/t8lRruuPZSAa9IGSZorDYlRe44n", - "f+JwmbHwQGnPKn27w4qdr4HR5NhgD1uinUSezvMtM7SvGeOaceaPpikTS7ZVNdvQ5uTiksa71SDVCoZE", - "o83pnKMovGPkGxAjQryFUjlwScTzcjckmVyKVV2BZps1mLU78yrQpZIamFr8CqnBbf/vZz+9YapiP4LW", - "fAVveXrJQKYqG99jN2nsBP9VK9zwQq9Knl7Gj+tcFCKC8o/8WhR1wWRdLKDC/fLng1GsAlNXcgwhC3EP", - "nxX8ejjpeVXLlDa3nbZjqCErCV3mfDtjp0tW8OuvjqYOHc14nrMSZCbkiplrOWqk4dz70UsqVcvsABvG", - "4IYFp6YuIRVLARlroOzAxE2zDx8hb4dPa1kF6Hggo+g0s+xBR8J1hGdQdLGFlXwFAcvM2M9Oc1GrUZcg", - "GwXHFltqKiu4EqrWzaARHGnq3ea1VAaSsoKliPDYmSMHag/bx6nXwhk4qZKGCwkZal5CWhmwmmgUp2DC", - "3c7M8IhecA1fPhs7wNvWA3d/qfq7vnPHD9pt6pRYkYyci9jqBDZuNnXGH+D8hXNrsUrsz4ONFKtzPEqW", - "Iqdj5lfcP0+GWpMS6BDCHzxarCQ3dQXHF/IR/sUSdma4zHiV4S+F/enHOjfiTKzwp9z+9FqtRHomViPE", - "bHCNelM0rLD/ILy4OjbXUafhtVKXdRkuKO14pYstO301tskW5m0Z86RxZUOv4vzaexq3HWGum40cQXKU", - "diXHjpewrQCx5emS/rleEj/xZfU7/lOWeYymyMDuoKWggAsWvHO/4U8o8mB9AoQiUo5EndPxefwxQOjP", - "FSwnx5M/zdtIydy26rmDa2fs7t4DKEqzfYhUOGnh3z8G7cgYFkEzE9LuGnWdWl/x/vFBqFFMyIDt4fB1", - "rtLLO+FQVqqEygi7vwuEM5QgAs/WwDOoWMYNn7XOlrW/RuSABn5P48h7gipy9P1E/+E5w2aUTm68WYcm", - "rdBo3KkgAJWhJWjPFzsTdiALVbHCGn8MjbZbYfmyndwq7kbTvndk+dCHFtmdb6y9yWiEXwQuvfUmTxaq", - "uhu/9BhBstZHZhyhNlYxrry7s9S1LhNHn4idbTv0ALVhyaG6DSnUB38IrQLJbqlzZvi/gDoaod4HdbqA", - "Phd1VFGKHO5Bvtdcr4eLQ0Pp6RN29v3J88dP/vnk+Zd40peVWlW8YIutAc0euPOJabPN4eFwxXRQ1LmJ", - "Q//ymffEunD3Uo4QbmAfQrdzQE1iKcZs3AGxe1Vtq1reAwmhqlQVsZ2JpYxKVZ5cQaWFioRB3roezPVA", - "vWXt997vFlu24Zrh3OTW1TKDahajPPprZBoYKPS+g8WCPr+WLW0cQF5VfDvYAbveyOrcvIfsSZf43kvQ", - "rIQqMdeSZbCoV+GZxpaVKhhnGQ0kBfpGZXBmuKn1PWiHFliLDG5EiAJfqNowzqTKUNCxc1xvjMREKRhD", - "MSQTqiKztufVAtDKTnm9WhuG5qmKbW07MOGp3ZSEzhY94kI2vr/tZaez8ba8Ap5t2QJAMrVwfprzIGmR", - "nMI7xt/cOK3VotX4Fh28ykqloDVkibum2ouav/KiTTY7yER4E77NJEwrtuTVHXE1yvB8D57UZ4itbq0P", - "59sOsT5s+l3715883EVeoatqmQBNHRTuHAyMkXAvTepy5FrDnXbnokCRYJJLpSFVMtNRYDnXJtknCtip", - "cyTjtgbcF+N+AjzivL/m2lj3WciMzDYrwjQPjaEpxhEe1dII+e9eQQ9hp6h7pK51o611XZaqMpDF1iDh", - "esdcb+C6mUstA9jNkWAUqzXsgzxGpQC+I5ZdiSUQNy5+08SXhoujUDnq1m2UlB0kWkLsQuTM9wqoG4Z2", - "RxBBG78ZSYwjdI9zmnjydKKNKkvUSSapZTNujExntveJ+bntO2QublpdmSnA2Y3HyWG+sZS1Qf01R3uJ", - "ILOCX6K+J+vH+vlDnFEYEy1kCskuzkexPMNeoQjsEdIRg9RdGwaz9YSjx79Rphtlgj27MLbgW1rHb23U", - "+ryN6NyDgfAKDBe5boyAJjTezkJR9H6GA1psFaQgTb5FHl6KqrAXUXR2aP+bNTEyN4u9cmnFUmasgg2v", - "Mt9j6LEEi0mEzOA6rnV5J26RwTUTcaSXzczCsNRfE8kQwCyqANzF2w4UXMDiLpPj0Pi09lrJUknHLhyp", - "AQWjEGmluL1HxMXYw9M0V2UVFByxoxstd9iPzynkKrHXlpFj07b7a00fTg55Jg7X88moxDessVkD3ZSg", - "Gu8RMeQ2dN9Aw9hCVrla8DxBoxaSDHKzNxyFxjK8op54fqp0OLyL8sXF+zy7uPjAXmNfsp+BXcJ2Tre7", - "LF1zuYI25B7yqbWM4RrSOlT1PTIe5Oy4uGIX+667M52USuVJ49b1rwgG6r9P90uRXkLGUE+QMepOpS+6", - "O4STsAfI4rq5RNmst97OLUuQkD2cMXYiGek2F1voWSC9yeUXZtf81zRrVtN9LpeMFjm7kHH33d4Gf6JM", - "eTC7JcmmR33iVBbI7onMtRwRJ76hywwEF5XPnRHDMxoZHDmDEzZgKovFIafad5QzxDu7LDJyQtpTRdeL", - "QlDiUNBtiprT3+UOvVhhZoydk+5AL0LDFVQ8p6wI7YOpQrNCoDOq6zQFyI4vZNLBJFWFm/hB+1+rli7q", - "o6OnwI4e9sdog+ajc5isDPTHfsWOpraJyMW+YheTi8kAUgWFuoLMOo0hX9tRe8H+lwbuhfxpoJhZwbfW", - "3fSyyHS9XIpUWKLnCvX6SvWsQKmoBSpED9Bp00yYKR1lRFGynu2+tAIYt1ruI64RgYp2Mx6lqO38DV6X", - "dzSDa57iKjkpmS3bIKM0fDY0PowqkxBANPy6Y0YXGNcdPX5HuRvqc+tl78bvvOdnd8gRsOtsvy09IEYU", - "g0PE/4SVCndduFwdn9CRC20GSDqHn25FGoaMHDoz9r9UzVJO8lvWBhpfS1XkwJBjizPQGevndJZaSyHI", - "oQAbBqGWR4/6C3/0yO250GwJG5/ghh375Hj0yAqB0uaTJaDHmtenEQOKgs94mkaSktdcr2d7A9EE96D4", - "cwD69JWfkIRJazpibqYTdIHz7T0IvAXEKnD2nu4Eg7RtVcswmc7tn95qA8UwommH/nPEEn3nPbfBSatk", - "LiQkhZKwjeaPCwk/UmP0nCYWGRlMwjo2tu/ZdvDvodWd55Dd/FT60m4HLPG2Se27h83vw+0Fs8M0QrIy", - "IS8ZZ2kuKFCopDZVnZoLySlw0TODemzhwzHjoayXvks8dhYJbTlQF5JrpGETzoheciwhEqj8FsBHtHS9", - "WoHumUVsCXAhXS8hWS2FobnIqkzshpVQ0W3UzPZES2DJc4q8/Q6VYovadFUvZTtZy8ZG1nEappYXkhuW", - "A9eG/Sjk+TWB8x6O5xkJZqOqy4YKIx4aSNBCJ/ELu+9s6/dcr/3ysaNXNm6wDR4j/DYlamugk079vx/8", - "7fj9SfIPnvx+lLz4r/MPH5/dPHw0+PHJzVdf/Z/uT09vvnr4tz/HdsrjHsvFcZifvnJmyekrOnvaoPoA", - "988WFC6ETKJMhu5CISSldPZ4iz3AE9Qz0MM2PO92/UKaa4mMdMVzkaELfBd26Ku4gSxa6ehxTWcjejE+", - "v9YPMXdnpZKSp5d0Dz5ZCbOuF7NUFXNvjs1XqjHN5hmHQklqy+a8FHN0b+dXj/ccjZ+gr1hEXVG2m/X5", - "gzSliFnqbp46HhJCtK81bLofegivYCmkwPbjC5lxw+cLrkWq57WG6muec5nCbKXYMXMgX3HDybHuhenG", - "HlRR0MNhU9aLXKTsMjzfWn4fizZdXLxHql9cfBjcGg1PIzdVPIJHEyQbYdaqNokLdY47520AgyDbYNeu", - "WafMwbbb7EKpDv5IVLEsdRKEmeLLL8sclx+cmZrRIEpSYtqoymsWVDcuUID7+0a5e7OKb3wKeY3O8C8F", - "L98LaT6wxDm1J2VJMSwKIv3iBBi17raEwwNRLYotsJjzQgu3VsqtE9cI6Jkd5SOzOk45bCLSUR8UtTbQ", - "dlc6IajvVY6be2cyBTCi1KnNOkGZiq5KI2uRPAQP//gKFYy/6EJfFJnPPURZAEvXkF5CRtF8CrxNO8P9", - "/bJT115khbZvR2x+GiU4k4+1AFaXGXcHGpfbfqapBmN8eu07uITtuWrzo2+TWnoznbhIeYI8MyYgJdIj", - "0Kxq2RUXH23vbb67sKBodlkyGzC2qX+eLY4bvvBjxgXIqvt7EJ4YUzRk2MHvJa8ihLDMP0KCOywU4X0S", - "60fD07wyIhWlXf9hAe+3nTEIZJ9Sj6pxtexr64EyjWpv2zlZcB1X3IAtuB8oQ/1UDj+TDVfYmydG748d", - "4y5yCK5qtJNsXpEF4ZdtH1SOoRbnEqhke5p6NLoUCY/ttbvrE1ftDR/d8R5ywO296UEu8pfzohvTFThv", - "Dld8NLw+mvh/Gty4B+/JmrR+r9j6wjBtnnjYp90+/d/n/PtE/8n0Vkn704lLrIpth5J0umeQw4q7aDKl", - "bDlGcah9oYMNQjx+Wi7R52dJ7PKea61SYS8YW13u5gA0/h4xZqMV7GAIMTYO0KYwHAFmb1Qom3J1GyQl", - "CIrbcQ+bAnjB37A/jNW+sXdm5V7zb6g7WiGatm9g7DYOQyrTSVQljVnmnV7MdlnAwD+IsSiqpmGQYRjK", - "0JADHcdJR7Mml7HQE1oVQGx45ocF5jp7IJZ4yD8MorEVrNChbZ1AlFYf1fi8jviVMpAsRaVNQv5ndHnY", - "6VtNxuC32DWufjqkYvaRrsji2oemvYRtkom8ju+2m/eHVzjtm8Zv0fXiErZ0yABP12xBj8rxFOpMj312", - "TG0TWHYu+LVd8Gt+b+s9jJewK05cKWV6c/xBuKqnT3YJU4QBY8wx3LVRku5QL8EV/1C3BMkFNhGBkhZm", - "u7z1gTDdOk1iVPNaSNG1BIbuzlXYbBqbMBO8yR4mKI/IAC9LkV33fGcLNc7jNMVtDHVr8Q+oQLvrgO2h", - "QOAnx/L1KvC+vt3S4My0r+sHuUv7KdPPmAoUQjiV0L42zJBQyNqU4rKPVufA8x9g+3fsS8uZ3Ewnn+by", - "x2jtIO6h9dtme6N0psCsdQE7kbNbkpyXZaWueJ64NyBjrFmpK8ea1N0/GfnMqi7ufp9/c/L6rUOfUsKA", - "Vy4TateqqF/5h1kVesSxdKjzIDJC1qr3na0hFmx+83AvDKb47LWOLYdazDGXFa/mgAtF0QVXlvH7ob2h", - "kjDj7U6S2UmZ+9TIXJg/d68iP5CwOIe2O7xHL4Rz7agGUNiCF5op2c8aQDOOvExil4JvcRdtYHaoIGRd", - "JCgCic5FGg8dyIVGKZJ1Qc8jtgYYdR4xCBFiLUbC57IWASzspg+4fukhGcwRJSaFdXbQbqFcpbJait9q", - "YCIDabCpcllEHWFB2fCJscMjLZ6E6wC7PNwG/Kec8whq7IQnJHYf8mGUN5J67Z0+v9AmPI0/BMG5W1zS", - "hDMOjqUdFyyOPxw32+vjdTdaGxYWG+ogZAxbhGJ/VTMfOlhbREfmiFYpG9XYJ+PampKrD9fTrVomdEOF", - "bBPeeK5VBEwtN1zaokM4ztLQjdZg/XYctVEVvRDSEL32FTpZVup3iHuTS9yoSGKTIyWZbDR6Fnl50Vei", - "TWSkLSfn6RviMcraY9ZU0Mi6l2gjEk5cHoSvKVPTB5m4tGxtCyR17kPjwhHmMMwt/FY4HM6DvI+cbxY8", - "VhMAjRrE6aS9KOmEw4xifrDfBd0kKDveC+5cmr7CPqspoWqzD4fPIu9ooPyxWD6DVBQ8j0dHM6J+92Fl", - "JlbCVpmqNQRljBwgW57PcpErBWWvolrSnC7Z0TQolOZ2IxNXQotFDtTjse2x4JpOrSbk2QzB5YE0a03d", - "nxzQfV3LrILMrLUlrFasMSLtiwEff16A2QBIdkT9Hr9gDyjyrsUVPEQqOltkcvz4BeU52D+OYoedKye3", - "S69kpFj+h1MscT6mqwcLAw8pB3UWfeJla4COq7Ad0mSHHiJL1NNpvf2yVHDJVxC/US324GTH0m5S4K5H", - "F5nZAnbaVGrLhInPD4ajfhrJdUL1Z9FwCegFCpBRTKsC+amtUWQn9eBsNTxXH8Tj5RvpmqP0Dwl6Tuvn", - "DdLaszy2arqMesML6JJ1yrh9CUlvIdwLWqcQZyOFGaC6ik9SjWywPzfdWPZAKpkUKDvZwzaLLuC/aF0C", - "ZXgendZ43dXPXNkN+lBTC6Eko4StO4TlgU66M4nrKr5OXuNUP7977Q6GQlWxIgOtNnSHRAWmEnAVldh+", - "NlhjmTTHhad8zEDxpRh+q0Gb2MMbarD5M+S34RloyzAwkBmdIDNmH6og2p2nBqS5RVHnNm0dshVUzqmv", - "y1zxbMoQzvk3J6+ZnVW7x470QILKQKzso6eGRJEwUvB8/zavwMbSbQ6HszsPAVetDb2p1YYXZSw9EXuc", - "+w6UA3nFRe6vtEmlhdSZsVf2NNFeV9lJ2sd+rJnO8W++UvTKmxvD0zWp6Y5Ss0IS9f0Orl/iM3x1UA+w", - "Ka3WvIq379eM8iVMbAWTKVN4lm6EtjVN4Qq6GZFNerAzE3yGZHd5VS2l5ZS4ztuRvn4Xsnvk7GWRD3NE", - "MesR/paqS6u6SuG25VzOaFT0MUy/NsygEKCE7PxaNgW3fK3qlEslRUpPUYIqqg3Krj7qIXG4A17t9F0w", - "L+JOQiPCFa1I01xHOyqO1qjxitARbhiECFpxUy132D8NFeJE52IFRjvNBtnUVx1yvoGQGlyVAyqVG+hJ", - "dPH6d1LRcHn7rvqWbEQpZSNH4LfYRsefcGkgl0LSK0NHNpdxYq13Kt9o0GUQhq0UaLee7isa/R7HzM6v", - "5Sli/GHmyz0SDBuWxGXbOPgQ1ImPirsoNPZ9iX0ZhSDbnzvpa3bSk7J0k8Y0gW52OFY3aZTAkchq4kNb", - "AXEb+CG0Hey28zqLzlNkNLiiYDiUdA4PGGPkrfI36ChZjrJPHu01cjSHXsgIGq+FhLYYaeSASKNHAm0M", - "yevIOJ1W3KTrg3XaOfCcou8xhaaNC0d8KqjeBhNJaI1+jvFtbKtnjSiOpkOb4c7ltqmBitwdGBMvqfiy", - "I+SwFhZZVc6IyihRqFcdK6Y4UHH7enPdA2AoBkObyA43FbeSc5uTaCyxORMaTdxikUdSI141jUGFOMrB", - "Wmzp39hL0fEVuMuaO1c2oIG3ti93VxnIce8TLVZ33JV2/D1uS08Gwj2Kcf83qFbCh2uDR79W8TT1Eela", - "WPn6nuRUNMnOXZ4lRRejQ1CScbcjNF5ccUqqcSQ55F37tI9b7WvjTWMpIuloRhM3Ll3RcLar3IetfBiD", - "YO+2bMVF+xWEqLM5dp9lr7OweTD6MLthYIUR7J0E9RelQ4R+8JkQrOTCBVNbERlS1uVMDbPYDsmmaDe4", - "vwiXiURAYiu5Y+LQQbI3pFJEsMPr5j3sedkhqX1h0LMkVQX3TNrgCL0laYcX6Ycuj9ZBHFNrGK7z4A3o", - "0HaE9ocQvtULQ+KOi7NZHCLO8URtHE76xBLEPyUYapPPpg06BVvdvLFd//torTv7logbtgHGpVQkUS7q", - "xjgrVAY5067GRg4rnm7d6z99IVMuWSYqoEIVoqCaa5zpDV+toKJno7ZMqo9NELTIbtUiz/axjYPxNfWN", - "vMb9d76nHQqxRfZW5kR/a2mhu9+PNtP8q96MpqoobGigQ/7oy8nmORYFXQj9tk7grtjhouLSeiIDChGU", - "4EsNkTpday4l5NHR9m7i38QhBf9VjeBcCBlv6rOAJUyPDO2auyv0U3r4kVIK04mGtK6E2VL+kPdMxD+j", - "udHfNfLrqsw3t7DuEtB++MSFx1tpb79V8Z2ydZ8LdJfIdTBU/eSba16UOTg9+tUXi7/A078+y46ePv7L", - "4q9Hz49SePb8xdERf/GMP37x9DE8+evzZ0fwePnli8WT7MmzJ4tnT559+fxF+vTZ48WzL1/85Qv/oQiL", - "aPsRhv9J5QSSk7enyTki224UL8UPsLUvopE7fckHnpLmhoKLfHLsf/pvXk5QgIJv27lfJ+62YbI2ptTH", - "8/lms5mFQ+YrqseXGFWn67mfZ1hs5u1pE9C3SQckSzZWi4JO54UwOWWaUNu7b87O2cnb01mrDibHk6PZ", - "0ewxVQApQfJSTI4nT+kn4vo17ft8DTw3KBk308m8AFOJVLu/nAqfuWoX+NPVk7mPAM4/uqv1m11t3dwG", - "92AlGBC8eJx/7JROzEK49B5w/tHnfQRNtg7v/CMFGIPfXSHN+ce2su2N5e4cYpEeX+Gr7U6Vu6jovra/", - "IkP7u0mhu9WFm905zXBXcNTLpspv+M3R9/+ffqHvQ++DJU+Ojv7ziQUqk/rslpTY6dd04gCReb/mGfN3", - "jDT3488396mkVySoqJhVxDfTyfPPufpTiaLAc0Y9g0yTIUv8LC+l2kjfE0/Nuih4tfXirTvKwtf0Jt3M", - "V5oqDVbiihuYfKBSlrFL3RGlQ9+yuLXSoQ90/EfpfC6l88f+csl/lM4fTemcWaVwuNJxhpBN9pjbimit", - "feTfLQ4f83UtuzHN5Qx99oCiyhI2D13CiAUbeRjaXM6rzEaQfHEfn9rkZp0NNNs7B7TzBvkH2Op9au58", - "DeyX9hvtv1ACJl3VTJmq2C88z4Pf6FOb3oSdjXzwvXkseOjX3m9upjG0lgA+HZTSPl1RT1T3l+CflVoa", - "dK5zhxkQbX21JYx+9NWWoQo1m2PBx0dHR7GXFX2cXbTLYkzptxuV5HAF+XCrx5DovS7d9YnE0Q9VDB8F", - "h15nhOv8F4Wbd8KjX4zsvnS9DXavlPzCsA0XrrZ4UFnGfg6kEMZ/TNWmVLkUvubsiH+AM0GQu7/P+6lH", - "3B+vSOfNDmWn17XJ1EaOKy5638NzlyBLKauNs20U8wAaTTVj/it4+dZ/3pVxSu5Stel+ddkXjOjVIm5K", - "Gq2EpAlIymkWmwnOgzxL96WIoRI8c5i9sR/W6Om96McnLY5xuY8J/afy0uEGyM499IVHOn/PURTQ2LNf", - "6UmIckO33wDP5y7dp/ervZQPfuzWIY78Om8eXUUb+8GMWOv8o7l28Yog8EZb1oTc3n9AylM6r9vNNo50", - "PJ/TzfdaaTOfoObpxpjCxg8NUT96FvDEvflw838DAAD//9ssXxsJhQAA", + "H4sIAAAAAAAC/+x9/3fbNvLgv4LTZ99rkhMl51t34/f69tykX3xN07zY3bvbOLeFyJGEmgRYArSs5vy/", + "35sBQIIkKMmON3u9z/6UWAAGg8HMYGYwGH6cpKoolQRp9OT446TkFS/AQEV/8TRVtTSJyPCvDHRaidII", + "JSfHvo1pUwm5mkwnAn8tuVlPphPJC2j74PjppILfalFBNjk2VQ3TiU7XUHAEbLYl9m4gXScrlTgQJxbE", + "6avJzY4GnmUVaD3E8ieZb5mQaV5nwEzFpeYpNmm2EWbNzFpo5gYzIZmSwNSSmXWnM1sKyDM984v8rYZq", + "G6zSTT6+pJsWxaRSOQzxfKmKhZDgsYIGqWZDmFEsgyV1WnPDcAbE1Xc0imngVbpmS1XtQdUiEeILsi4m", + "x+8nGmQGFe1WCuKK/rusAH6HxPBqBWbyYRpb3NJAlRhRRJZ26qhfga5zoxn1pTWuxBVIhqNm7MdaG7YA", + "xiV79+1L9vTp0xe4kIIbA5ljstFVtbOHa7LDJ8eTjBvwzUNe4/lKVVxmSdP/3bcvaf4zt8BDe3GtIS4s", + "J9jCTl+NLcAPjLCQkAZWtA8d7scREaFof17AUlVw4J7Yzve6KeH8/9JdSblJ16US0kT2hVErs81RHRYM", + "36XDGgQ6/UukVIVA3x8lLz58fDx9fHTzH+9Pkr+7P58/vTlw+S8buHsoEO2Y1lUFMt0mqwo4ScuayyE9", + "3jl+0GtV5xlb8yvafF6QqndjGY61qvOK5zXyiUgrdZKvlGbcsVEGS17nhvmJWS1zVFMIzXE7E5qVlboS", + "GWRT1L6btUjXLOXagqB+bCPyHHmw1pCN8Vp8dTuE6SYkCeJ1J3rQgv7fJUa7rj2UgGvSBkmaKw2JUXuO", + "J3/icJmx8EBpzyp9u8OKna+B0eTYYA9bop1Ens7zLTO0rxnjmnHmj6YpE0u2VTXb0Obk4pLGu9Ug1QqG", + "RKPN6ZyjKLxj5BsQI0K8hVI5cEnE83I3JJlcilVdgWabNZi1O/Mq0KWSGpha/AqpwW3/72c/vWGqYj+C", + "1nwFb3l6yUCmKhvfYzdp7AT/VSvc8EKvSp5exo/rXBQigvKP/FoUdcFkXSygwv3y54NRrAJTV3IMIQtx", + "D58V/Ho46XlVy5Q2t522Y6ghKwld5nw7Y6dLVvDrr46mDh3NeJ6zEmQm5IqZazlqpOHc+9FLKlXL7AAb", + "xuCGBaemLiEVSwEZa6DswMRNsw8fIW+HT2tZBeh4IKPoNLPsQUfCdYRnUHSxhZV8BQHLzNjPTnNRq1GX", + "IBsFxxZbaioruBKq1s2gERxp6t3mtVQGkrKCpYjw2JkjB2oP28ep18IZOKmShgsJGWpeQloZsJpoFKdg", + "wt3OzPCIXnANXz4bO8Db1gN3f6n6u75zxw/abeqUWJGMnIvY6gQ2bjZ1xh/g/IVza7FK7M+DjRSrczxK", + "liKnY+ZX3D9PhlqTEugQwh88WqwkN3UFxxfyEf7FEnZmuMx4leEvhf3pxzo34kys8Kfc/vRarUR6JlYj", + "xGxwjXpTNKyw/yC8uDo211Gn4bVSl3UZLijteKWLLTt9NbbJFuZtGfOkcWVDr+L82nsatx1hrpuNHEFy", + "lHYlx46XsK0AseXpkv65XhI/8WX1O/5TlnmMpsjA7qCloIALFrxzv+FPKPJgfQKEIlKORJ3T8Xn8MUDo", + "TxUsJ8eT/5i3kZK5bdVzB9fO2N29B1CUZvsQqXDSwr9/DNqRMSyCZiak3TXqOrW+4v3jg1CjmJAB28Ph", + "61yll3fCoaxUCZURdn8XCGcoQQSerYFnULGMGz5rnS1rf43IAQ38nsaR9wRV5Oj7if7Dc4bNKJ3ceLMO", + "TVqh0bhTQQAqQ0vQni92JuxAFqpihTX+GBptt8LyZTu5VdyNpn3vyPKhDy2yO99Ye5PRCL8IXHrrTZ4s", + "VHU3fukxgmStj8w4Qm2sYlx5d2epa10mjj4RO9t26AFqw5JDdRtSqA/+EFoFkt1S58zwfwJ1NEK9D+p0", + "AX0u6qiiFDncg3yvuV4PF4eG0tMn7Oz7k+ePn/zjyfMv8aQvK7WqeMEWWwOaPXDnE9Nmm8PD4YrpoKhz", + "E4f+5TPviXXh7qUcIdzAPoRu54CaxFKM2bgDYveq2la1vAcSQlWpKmI7E0sZlao8uYJKCxUJg7x1PZjr", + "gXrL2u+93y22bMM1w7nJratlBtUsRnn018g0MFDofQeLBX1+LVvaOIC8qvh2sAN2vZHVuXkP2ZMu8b2X", + "oFkJVWKuJctgUa/CM40tK1UwzjIaSAr0jcrgzHBT63vQDi2wFhnciBAFvlC1YZxJlaGgY+e43hiJiVIw", + "hmJIJlRFZm3PqwWglZ3yerU2DM1TFdvadmDCU7spCZ0tesSFbHx/28tOZ+NteQU827IFgGRq4fw050HS", + "IjmFd4y/uXFaq0Wr8S06eJWVSkFryBJ3TbUXNX/lRZtsdpCJ8CZ8m0mYVmzJqzviapTh+R48qc8QW91a", + "H863HWJ92PS79q8/ebiLvEJX1TIBmjoo3DkYGCPhXprU5ci1hjvtzkWBIsEkl0pDqmSmo8Byrk2yTxSw", + "U+dIxm0NuC/G/QR4xHl/zbWx7rOQGZltVoRpHhpDU4wjPKqlEfLfvIIewk5R90hd60Zb67osVWUgi61B", + "wvWOud7AdTOXWgawmyPBKFZr2Ad5jEoBfEcsuxJLIG5c/KaJLw0XR6Fy1K3bKCk7SLSE2IXIme8VUDcM", + "7Y4ggjZ+M5IYR+ge5zTx5OlEG1WWqJNMUstm3BiZzmzvE/Nz23fIXNy0ujJTgLMbj5PDfGMpa4P6a472", + "EkFmBb9EfU/Wj/XzhzijMCZayBSSXZyPYnmGvUIR2COkIwapuzYMZusJR49/o0w3ygR7dmFswbe0jt/a", + "qPV5G9G5BwPhFRguct0YAU1ovJ2Fouj9DAe02CpIQZp8izy8FFVhL6Lo7ND+N2tiZG4We+XSiqXMWAUb", + "XmW+x9BjCRaTCJnBdVzr8k7cIoNrJuJIL5uZhWGpvyaSIYBZVAG4i7cdKLiAxV0mx6Hxae21kqWSjl04", + "UgMKRiHSSnF7j4iLsYenaa7KKig4Ykc3Wu6wH59TyFViry0jx6Zt99eaPpwc8kwcrueTUYlvWGOzBrop", + "QTXeI2LIbei+gYaxhaxyteB5gkYtJBnkZm84Co1leEU98fxU6XB4F+WLi/d5dnHxgb3GvmQ/A7uE7Zxu", + "d1m65nIFbcg95FNrGcM1pHWo6ntkPMjZcXHFLvZdd2c6KZXKk8at618RDNR/n+6XIr2EjKGeIGPUnUpf", + "dHcIJ2EPkMV1c4myWW+9nVuWICF7OGPsRDLSbS620LNAepPLL8yu+a9p1qym+1wuGS1ydiHj7ru9Df5E", + "mfJgdkuSTY/6xKkskN0TmWs5Ik58Q5cZCC4qnzsjhmc0MjhyBidswFQWi0NOte8oZ4h3dllk5IS0p4qu", + "F4WgxKGg2xQ1p7/LHXqxwswYOyfdgV6EhiuoeE5ZEdoHU4VmhUBnVNdpCpAdX8ikg0mqCjfxg/a/Vi1d", + "1EdHT4EdPeyP0QbNR+cwWRnoj/2KHU1tE5GLfcUuJheTAaQKCnUFmXUaQ762o/aC/S8N3Av500Axs4Jv", + "rbvpZZHperkUqbBEzxXq9ZXqWYFSUQtUiB6g06aZMFM6yoiiZD3bfWkFMG613EdcIwIV7WY8SlHb+Ru8", + "Lu9oBtc8xVVyUjJbtkFGafhsaHwYVSYhgGj4dceMLjCuO3r8jnI31OfWy96N33nPz+6QI2DX2X5bekCM", + "KAaHiP8JKxXuunC5Oj6hIxfaDJB0Dj/dijQMGTl0Zux/qZqlnOS3rA00vpaqyIEhxxZnoDPWz+kstZZC", + "kEMBNgxCLY8e9Rf+6JHbc6HZEjY+wQ079snx6JEVAqXNJ0tAjzWvTyMGFAWf8TSNJCWvuV7P9gaiCe5B", + "8ecA9OkrPyEJk9Z0xNxMJ+gC59t7EHgLiFXg7D3dCQZp26qWYTKd2z+91QaKYUTTDv3HiCX6zntug5NW", + "yVxISAolYRvNHxcSfqTG6DlNLDIymIR1bGzfs+3g30OrO88hu/mp9KXdDljibZPadw+b34fbC2aHaYRk", + "ZUJeMs7SXFCgUEltqjo1F5JT4KJnBvXYwodjxkNZL32XeOwsEtpyoC4k10jDJpwRveRYQiRQ+S2Aj2jp", + "erUC3TOL2BLgQrpeQrJaCkNzkVWZ2A0roaLbqJntiZbAkucUefsdKsUWtemqXsp2spaNjazjNEwtLyQ3", + "LAeuDftRyPNrAuc9HM8zEsxGVZcNFUY8NJCghU7iF3bf2dbvuV775WNHr2zcYBs8RvhtStTWQCed+n8/", + "+Ovx+5Pk7zz5/Sh58V/nHz4+u3n4aPDjk5uvvvo/3Z+e3nz18K9/iu2Uxz2Wi+MwP33lzJLTV3T2tEH1", + "Ae6fLShcCJlEmQzdhUJISuns8RZ7gCeoZ6CHbXje7fqFNNcSGemK5yJDF/gu7NBXcQNZtNLR45rORvRi", + "fH6tH2LuzkolJU8v6R58shJmXS9mqSrm3hybr1Rjms0zDoWS1JbNeSnm6N7Orx7vORo/QV+xiLqibDfr", + "8wdpShGz1N08dTwkhGhfa9h0P/QQXsFSSIHtxxcy44bPF1yLVM9rDdXXPOcyhdlKsWPmQL7ihpNj3QvT", + "jT2ooqCHw6asF7lI2WV4vrX8PhZturh4j1S/uPgwuDUankZuqngEjyZINsKsVW0SF+ocd87bAAZBtsGu", + "XbNOmYNtt9mFUh38kahiWeokCDPFl1+WOS4/ODM1o0GUpMS0UZXXLKhuXKAA9/eNcvdmFd/4FPIaneFf", + "Cl6+F9J8YIlzak/KkmJYFET6xQkwat1tCYcHoloUW2Ax54UWbq2UWyeuEdAzO8pHZnWccthEpKM+KGpt", + "oO2udEJQ36scN/fOZApgRKlTm3WCMhVdlUbWInkIHv7xFSoYf9GFvigyn3uIsgCWriG9hIyi+RR4m3aG", + "+/tlp669yApt347Y/DRKcCYfawGsLjPuDjQut/1MUw3G+PTad3AJ23PV5kffJrX0ZjpxkfIEeWZMQEqk", + "R6BZ1bIrLj7a3tt8d2FB0eyyZDZgbFP/PFscN3zhx4wLkFX39yA8MaZoyLCD30teRQhhmX+EBHdYKML7", + "JNaPhqd5ZUQqSrv+wwLebztjEMg+pR5V42rZ19YDZRrV3rZzsuA6rrgBW3A/UIb6qRx+JhuusDdPjN4f", + "O8Zd5BBc1Wgn2bwiC8Iv2z6oHEMtziVQyfY09Wh0KRIe22t31yeu2hs+uuM95IDbe9ODXOQv50U3pitw", + "3hyu+Gh4fTTx/zS4cQ/ekzVp/V6x9YVh2jzxsE+7ffq/z/n3if6T6a2S9qcTl1gV2w4l6XTPIIcVd9Fk", + "StlyjOJQ+0IHG4R4/LRcos/PktjlPddapcJeMLa63M0BaPw9YsxGK9jBEGJsHKBNYTgCzN6oUDbl6jZI", + "ShAUt+MeNgXwgr9hfxirfWPvzMq95t9Qd7RCNG3fwNhtHIZUppOoShqzzDu9mO2ygIF/EGNRVE3DIMMw", + "lKEhBzqOk45mTS5joSe0KoDY8MwPC8x19kAs8ZB/GERjK1ihQ9s6gSitPqrxeR3xK2UgWYpKm4T8z+jy", + "sNO3mozBb7FrXP10SMXsI12RxbUPTXsJ2yQTeR3fbTfvD69w2jeN36LrxSVs6ZABnq7Zgh6V4ynUmR77", + "7JjaJrDsXPBru+DX/N7WexgvYVecuFLK9Ob4g3BVT5/sEqYIA8aYY7hroyTdoV6CK/6hbgmSC2wiAiUt", + "zHZ56wNhunWaxKjmtZCiawkM3Z2rsNk0NmEmeJM9TFAekQFeliK77vnOFmqcx2mK2xjq1uIfUIF21wHb", + "Q4HAT47l61XgfX27pcGZaV/XD3KX9lOmnzEVKIRwKqF9bZghoZC1KcVlH63Ogec/wPZv2JeWM7mZTj7N", + "5Y/R2kHcQ+u3zfZG6UyBWesCdiJntyQ5L8tKXfE8cW9AxlizUleONam7fzLymVVd3P0+/+bk9VuHPqWE", + "Aa9cJtSuVVG/8g+zKvSIY+lQ50FkhKxV7ztbQyzY/ObhXhhM8dlrHVsOtZhjLitezQEXiqILrizj90N7", + "QyVhxtudJLOTMvepkbkwf+5eRX4gYXEObXd4j14I59pRDaCwBS80U7KfNYBmHHmZxC4F3+Iu2sDsUEHI", + "ukhQBBKdizQeOpALjVIk64KeR2wNMOo8YhAixFqMhM9lLQJY2E0fcP3SQzKYI0pMCuvsoN1CuUpltRS/", + "1cBEBtJgU+WyiDrCgrLhE2OHR1o8CdcBdnm4DfhPOecR1NgJT0jsPuTDKG8k9do7fX6hTXgafwiCc7e4", + "pAlnHBxLOy5YHH84brbXx+tutDYsLDbUQcgYtgjF/qpmPnSwtoiOzBGtUjaqsU/GtTUlVx+up1u1TOiG", + "CtkmvPFcqwiYWm64tEWHcJyloRutwfrtOGqjKnohpCF67St0sqzU7xD3Jpe4UZHEJkdKMtlo9Czy8qKv", + "RJvISFtOztM3xGOUtcesqaCRdS/RRiScuDwIX1Ompg8ycWnZ2hZI6tyHxoUjzGGYW/itcDicB3kfOd8s", + "eKwmABo1iNNJe1HSCYcZxfxgvwu6SVB2vBfcuTR9hX1WU0LVZh8On0Xe0UD5Y7F8BqkoeB6PjmZE/e7D", + "ykyshK0yVWsIyhg5QLY8n+UiVwrKXkW1pDldsqNpUCjN7UYmroQWixyox2PbY8E1nVpNyLMZgssDadaa", + "uj85oPu6llkFmVlrS1itWGNE2hcDPv68ALMBkOyI+j1+wR5Q5F2LK3iIVHS2yOT48QvKc7B/HMUOO1dO", + "bpdeyUix/A+nWOJ8TFcPFgYeUg7qLPrEy9YAHVdhO6TJDj1Elqin03r7Zangkq8gfqNa7MHJjqXdpMBd", + "jy4yswXstKnUlgkTnx8MR/00kuuE6s+i4RLQCxQgo5hWBfJTW6PITurB2Wp4rj6Ix8s30jVH6R8S9JzW", + "zxuktWd5bNV0GfWGF9Al65Rx+xKS3kK4F7ROIc5GCjNAdRWfpBrZYH9uurHsgVQyKVB2sodtFl3Af9G6", + "BMrwPDqt8bqrn7myG/ShphZCSUYJW3cIywOddGcS11V8nbzGqX5+99odDIWqYkUGWm3oDokKTCXgKiqx", + "/WywxjJpjgtP+ZiB4ksx/FaDNrGHN9Rg82fIb8Mz0JZhYCAzOkFmzD5UQbQ7Tw1Ic4uizm3aOmQrqJxT", + "X5e54tmUIZzzb05eMzurdo8d6YEElYFY2UdPDYkiYaTg+f5tXoGNpdscDmd3HgKuWht6U6sNL8pYeiL2", + "OPcdKAfyiovcX2mTSgupM2Ov7Gmiva6yk7SP/VgznePffKXolTc3hqdrUtMdpWaFJOr7HVy/xGf46qAe", + "YFNarXkVb9+vGeVLmNgKJlOm8CzdCG1rmsIVdDMim/RgZyb4DMnu8qpaSsspcZ23I339LmT3yNnLIh/m", + "iGLWI/wtVZdWdZXCbcu5nNGo6GOYfm2YQSFACdn5tWwKbvla1SmXSoqUnqIEVVQblF191EPicAe82um7", + "YF7EnYRGhCtakaa5jnZUHK1R4xWhI9wwCBG04qZa7rB/GirEic7FCox2mg2yqa865HwDITW4KgdUKjfQ", + "k+ji9e+kouHy9l31LdmIUspGjsBvsY2OP+HSQC6FpFeGjmwu48Ra71S+0aDLIAxbKdBuPd1XNPo9jpmd", + "X8tTxPjDzJd7JBg2LInLtnHwIagTHxV3UWjs+xL7MgpBtj930tfspCdl6SaNaQLd7HCsbtIogSOR1cSH", + "tgLiNvBDaDvYbed1Fp2nyGhwRcFwKOkcHjDGyFvlb9BRshxlnzzaa+RoDr2QETReCwltMdLIAZFGjwTa", + "GJLXkXE6rbhJ1wfrtHPgOUXfYwpNGxeO+FRQvQ0mktAa/Rzj29hWzxpRHE2HNsOdy21TAxW5OzAmXlLx", + "ZUfIYS0ssqqcEZVRolCvOlZMcaDi9vXmugfAUAyGNpEdbipuJec2J9FYYnMmNJq4xSKPpEa8ahqDCnGU", + "g7XY0r+xl6LjK3CXNXeubEADb21f7q4ykOPeJ1qs7rgr7fh73JaeDIR7FOP+b1CthA/XBo9+reJp6iPS", + "tbDy9T3JqWiSnbs8S4ouRoegJONuR2i8uOKUVONIcsi79mkft9rXxpvGUkTS0Ywmbly6ouFsV7kPW/kw", + "BsHebdmKi/YrCFFnc+w+y15nYfNg9GF2w8AKI9g7CeovSocI/eAzIVjJhQumtiIypKzLmRpmsR2STdFu", + "cH8RLhOJgMRWcsfEoYNkb0iliGCH18172POyQ1L7wqBnSaoK7pm0wRF6S9IOL9IPXR6tgzim1jBc58Eb", + "0KHtCO0PIXyrF4bEHRdnszhEnOOJ2jic9IkliH9KMNQmn00bdAq2unlju/630Vp39i0RN2wDjEupSKJc", + "1I1xVqgMcqZdjY0cVjzdutd/+kKmXLJMVECFKkRBNdc40xu+WkFFz0ZtmVQfmyBokd2qRZ7tYxsH42vq", + "G3mN+698TzsUYovsrcyJ/tbSQne/H22m+We9GU1VUdjQQIf80ZeTzXMsCroQ+m2dwF2xw0XFpfVEBhQi", + "KMGXGiJ1utZcSsijo+3dxL+IQwr+qxrBuRAy3tRnAUuYHhnaNXdX6Kf08COlFKYTDWldCbOl/CHvmYh/", + "RHOjv2vk11WZb25h3SWg/fCJC4+30t5+q+I7Zes+F+guketgqPrJN9e8KHNwevSrLxZ/hqd/eZYdPX38", + "58Vfjp4fpfDs+YujI/7iGX/84uljePKX58+O4PHyyxeLJ9mTZ08Wz548+/L5i/Tps8eLZ1+++PMX/kMR", + "FtH2Iwz/k8oJJCdvT5NzRLbdKF6KH2BrX0Qjd/qSDzwlzQ0FF/nk2P/037ycoAAF37Zzv07cbcNkbUyp", + "j+fzzWYzC4fMV1SPLzGqTtdzP8+w2Mzb0yagb5MOSJZsrBYFnc4LYXLKNKG2d9+cnbOTt6ezVh1MjidH", + "s6PZY6oAUoLkpZgcT57ST8T1a9r3+Rp4blAybqaTeQGmEql2fzkVPnPVLvCnqydzHwGcf3RX6ze72rq5", + "De7BSjAgePE4/9gpnZiFcOk94Pyjz/sImmwd3vlHCjAGv7tCmvOPbWXbG8vdOcQiPb7CV9udKndR0X1t", + "f0WG9neTQnerCze7c5rhruCol02V3/Cbo+//k36h70PvgyVPjo7+/YkFKpP67JaU2OnXdOIAkXm/5hnz", + "d4w09+PPN/eppFckqKiYVcQ308nzz7n6U4miwHNGPYNMkyFL/CwvpdpI3xNPzbooeLX14q07ysLX9Cbd", + "zFeaKg1W4oobmHygUpaxS90RpUPfsri10qEPdPxb6XwupfPH/nLJk1sK/v+/lPi3+v2jqd8zqx4PV7/O", + "JLRpL3NbG661FP0LzuGzxq6NO6bDncvDHlB8XcLmoUudsWAjT2SbNAWV2ViaL3Pkk7zcrLOBjn/ngHZe", + "Y/8AW71P4Z+vgf3Sfq3+F0pFpUurKVMV+4XnefAbfXTUG/OzkU/fN88mD/3u/c3NNIbWEsAnxlICrCtv", + "igffJfgHtpYGnYvtYS5IW2luCaOfv7UFuULN5ljw8dHRUeyNSR9nF/ezGFMi8kYlOVxBPtzqMSR672x3", + "fSxy9JMdw+fRof8d4Tr/beXmxfTotzO7b35vg90rJb8wbMOFq7Ie1NixH0YphPGflbXJZS6ZsTk74p8i", + "TRDk7i8Vf+ph/8crV3qzQ9npdW0ytZHjioteOvHcpQpT8m4TdjCKeQCNppox/z3AfOs/dMs4pbmp2nS/", + "P+1LZ/SqMjfFnVZC0gQk5TSLzYnnQcap+2bGUAmeOcze2E+M9PRe9DOcFse43MeE/lN56XADZOce+hIs", + "nb/nKApo9trvFSVEuWEAxADP5y7xqferTU8IfuxWZI78Om+en0Ub+2GdWOv8o7l2kZsgBElb1gQf339A", + "ylNis9vNNqJ2PJ9TDsBaaTOfoObpRtvCxg8NUT96FvDEvflw838DAAD//9vVYYoThgAA", } // GetSwagger returns the Swagger specification corresponding to the generated code diff --git a/daemon/algod/api/server/v2/handlers.go b/daemon/algod/api/server/v2/handlers.go index 192fbbe528..e071bd3331 100644 --- a/daemon/algod/api/server/v2/handlers.go +++ b/daemon/algod/api/server/v2/handlers.go @@ -563,12 +563,21 @@ func (v2 *Handlers) startCatchup(ctx echo.Context, catchpoint string) error { return badRequest(ctx, err, errFailedToParseCatchpoint, v2.Log) } + // Select 200/201, or return an error + var code int err = v2.Node.StartCatchup(catchpoint) - if err != nil { + switch err.(type) { + case nil: + code = http.StatusCreated + case *node.CatchpointAlreadyInProgressError: + code = http.StatusOK + case *node.CatchpointUnableToStartError: + return badRequest(ctx, err, err.Error(), v2.Log) + default: return internalError(ctx, err, fmt.Sprintf(errFailedToStartCatchup, err), v2.Log) } - return ctx.JSON(http.StatusOK, private.CatchpointStartResponse{ + return ctx.JSON(code, private.CatchpointStartResponse{ CatchupMessage: catchpoint, }) } diff --git a/daemon/algod/api/server/v2/test/handlers_test.go b/daemon/algod/api/server/v2/test/handlers_test.go index eead3fd01c..2114deda77 100644 --- a/daemon/algod/api/server/v2/test/handlers_test.go +++ b/daemon/algod/api/server/v2/test/handlers_test.go @@ -18,6 +18,8 @@ package test import ( "bytes" + "errors" + "github.com/algorand/go-algorand/node" "io" "net/http" "net/http/httptest" @@ -42,7 +44,7 @@ func setupTestForMethodGet(t *testing.T) (v2.Handlers, echo.Context, *httptest.R numTransactions := 1 offlineAccounts := true mockLedger, rootkeys, _, stxns, releasefunc := testingenv(t, numAccounts, numTransactions, offlineAccounts) - mockNode := makeMockNode(mockLedger, t.Name()) + mockNode := makeMockNode(mockLedger, t.Name(), nil) dummyShutdownChan := make(chan struct{}) handler := v2.Handlers{ Node: mockNode, @@ -232,7 +234,7 @@ func postTransactionTest(t *testing.T, txnToUse, expectedCode int) { mockLedger, _, _, stxns, releasefunc := testingenv(t, numAccounts, numTransactions, offlineAccounts) defer releasefunc() dummyShutdownChan := make(chan struct{}) - mockNode := makeMockNode(mockLedger, t.Name()) + mockNode := makeMockNode(mockLedger, t.Name(), nil) handler := v2.Handlers{ Node: mockNode, Log: logging.Base(), @@ -260,14 +262,14 @@ func TestPostTransaction(t *testing.T) { postTransactionTest(t, 0, 200) } -func startCatchupTest(t *testing.T, catchpoint string, expectedCode int) { +func startCatchupTest(t *testing.T, catchpoint string, nodeError error, expectedCode int) { numAccounts := 1 numTransactions := 1 offlineAccounts := true mockLedger, _, _, _, releasefunc := testingenv(t, numAccounts, numTransactions, offlineAccounts) defer releasefunc() dummyShutdownChan := make(chan struct{}) - mockNode := makeMockNode(mockLedger, t.Name()) + mockNode := makeMockNode(mockLedger, t.Name(), nodeError) handler := v2.Handlers{ Node: mockNode, Log: logging.Base(), @@ -286,9 +288,18 @@ func TestStartCatchup(t *testing.T) { t.Parallel() goodCatchPoint := "5894690#DVFRZUYHEFKRLK5N6DNJRR4IABEVN2D6H76F3ZSEPIE6MKXMQWQA" - startCatchupTest(t, goodCatchPoint, 200) + startCatchupTest(t, goodCatchPoint, nil, 201) + + inProgressError := node.MakeCatchpointAlreadyInProgressError("error") + startCatchupTest(t, goodCatchPoint, inProgressError, 200) + + unableToStartError := node.MakeCatchpointUnableToStartError("error") + startCatchupTest(t, goodCatchPoint, unableToStartError, 400) + + startCatchupTest(t, goodCatchPoint, errors.New("anothing else is internal"), 500) + badCatchPoint := "bad catchpoint" - startCatchupTest(t, badCatchPoint, 400) + startCatchupTest(t, badCatchPoint, nil, 400) } func abortCatchupTest(t *testing.T, catchpoint string, expectedCode int) { @@ -298,7 +309,7 @@ func abortCatchupTest(t *testing.T, catchpoint string, expectedCode int) { mockLedger, _, _, _, releasefunc := testingenv(t, numAccounts, numTransactions, offlineAccounts) defer releasefunc() dummyShutdownChan := make(chan struct{}) - mockNode := makeMockNode(mockLedger, t.Name()) + mockNode := makeMockNode(mockLedger, t.Name(), nil) handler := v2.Handlers{ Node: mockNode, Log: logging.Base(), @@ -308,7 +319,7 @@ func abortCatchupTest(t *testing.T, catchpoint string, expectedCode int) { req := httptest.NewRequest(http.MethodDelete, "/", nil) rec := httptest.NewRecorder() c := e.NewContext(req, rec) - err := handler.StartCatchup(c, catchpoint) + err := handler.AbortCatchup(c, catchpoint) require.NoError(t, err) require.Equal(t, expectedCode, rec.Code) } @@ -329,7 +340,7 @@ func tealCompileTest(t *testing.T, bytesToUse []byte, expectedCode int, enableDe mockLedger, _, _, _, releasefunc := testingenv(t, numAccounts, numTransactions, offlineAccounts) defer releasefunc() dummyShutdownChan := make(chan struct{}) - mockNode := makeMockNode(mockLedger, t.Name()) + mockNode := makeMockNode(mockLedger, t.Name(), nil) mockNode.config.EnableDeveloperAPI = enableDeveloperAPI handler := v2.Handlers{ Node: &mockNode, @@ -368,7 +379,7 @@ func tealDryrunTest( mockLedger, _, _, _, releasefunc := testingenv(t, numAccounts, numTransactions, offlineAccounts) defer releasefunc() dummyShutdownChan := make(chan struct{}) - mockNode := makeMockNode(mockLedger, t.Name()) + mockNode := makeMockNode(mockLedger, t.Name(), nil) mockNode.config.EnableDeveloperAPI = enableDeveloperAPI handler := v2.Handlers{ Node: &mockNode, diff --git a/daemon/algod/api/server/v2/test/helpers.go b/daemon/algod/api/server/v2/test/helpers.go index 34d12883c1..dd283dd921 100644 --- a/daemon/algod/api/server/v2/test/helpers.go +++ b/daemon/algod/api/server/v2/test/helpers.go @@ -81,10 +81,15 @@ type mockNode struct { ledger *data.Ledger genesisID string config config.Local + err error } -func makeMockNode(ledger *data.Ledger, genesisID string) mockNode { - return mockNode{ledger: ledger, genesisID: genesisID, config: config.GetDefaultLocal()} +func makeMockNode(ledger *data.Ledger, genesisID string, nodeError error) mockNode { + return mockNode{ + ledger: ledger, + genesisID: genesisID, + config: config.GetDefaultLocal(), + err: nodeError} } func (m mockNode) Ledger() *data.Ledger { @@ -104,7 +109,7 @@ func (m mockNode) GenesisHash() crypto.Digest { } func (m mockNode) BroadcastSignedTxGroup(txgroup []transactions.SignedTxn) error { - return nil + return m.err } func (m mockNode) GetPendingTransaction(txID transactions.Txid) (res node.TxnWithStatus, found bool) { @@ -114,7 +119,7 @@ func (m mockNode) GetPendingTransaction(txID transactions.Txid) (res node.TxnWit } func (m mockNode) GetPendingTxnsFromPool() ([]transactions.SignedTxn, error) { - return nil, nil + return nil, m.err } func (m mockNode) SuggestedFee() basics.MicroAlgos { @@ -168,11 +173,11 @@ func (m mockNode) AssembleBlock(round basics.Round, deadline time.Time) (agreeme } func (m mockNode) StartCatchup(catchpoint string) error { - return nil + return m.err } func (m mockNode) AbortCatchup(catchpoint string) error { - return nil + return m.err } ////// mock ledger testing environment follows diff --git a/node/error.go b/node/error.go new file mode 100644 index 0000000000..d3aa5a6624 --- /dev/null +++ b/node/error.go @@ -0,0 +1,55 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package node + +// Catchpoint already in progress error + +// CatchpointAlreadyInProgressError indicates that the requested catchpoint is already running +type CatchpointAlreadyInProgressError struct { + message string +} + +// MakeCatchpointAlreadyInProgressError creates the error +func MakeCatchpointAlreadyInProgressError(text string) *CatchpointAlreadyInProgressError { + return &CatchpointAlreadyInProgressError{ + message: text, + } +} + +// Error satisfies builtin interface `error` +func (e *CatchpointAlreadyInProgressError) Error() string { + return e.message +} + +// Catchpoint unable to start error + +// CatchpointUnableToStartError indicates that the requested catchpoint cannot be started +type CatchpointUnableToStartError struct { + message string +} + +// MakeCatchpointUnableToStartError creates the error +func MakeCatchpointUnableToStartError(text string) *CatchpointUnableToStartError { + return &CatchpointUnableToStartError{ + message: text, + } +} + +// Error satisfies builtin interface `error` +func (e *CatchpointUnableToStartError) Error() string { + return e.message +} diff --git a/node/node.go b/node/node.go index 3fcb008ae5..2c691e42cd 100644 --- a/node/node.go +++ b/node/node.go @@ -839,7 +839,11 @@ func (node *AlgorandFullNode) StartCatchup(catchpoint string) error { } if node.catchpointCatchupService != nil { stats := node.catchpointCatchupService.GetStatistics() - return fmt.Errorf("unable to start catchpoint catchup for '%s' - already catching up '%s'", catchpoint, stats.CatchpointLabel) + // No need to return an error + if catchpoint == stats.CatchpointLabel { + return MakeCatchpointAlreadyInProgressError(fmt.Sprintf("the requested catchpoint '%s' is already in progress, suppressing error", catchpoint)) + } + return MakeCatchpointUnableToStartError(fmt.Sprintf("unable to start catchpoint catchup for '%s' - already catching up '%s'", catchpoint, stats.CatchpointLabel)) } var err error node.catchpointCatchupService, err = catchup.MakeNewCatchpointCatchupService(catchpoint, node, node.log, node.net, node.ledger.Ledger, node.config) From 4e8530daa6ba00f0f6de29e86fc33ed7f4b9fde3 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Fri, 4 Sep 2020 09:27:26 -0400 Subject: [PATCH 014/136] Fix genesis endpoint operation ID. --- daemon/algod/api/algod.oas2.json | 2 +- daemon/algod/api/algod.oas3.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/algod/api/algod.oas2.json b/daemon/algod/api/algod.oas2.json index 6c116c07bd..b2449dca7f 100644 --- a/daemon/algod/api/algod.oas2.json +++ b/daemon/algod/api/algod.oas2.json @@ -82,7 +82,7 @@ "http" ], "summary": "Gets the genesis information.", - "operationId": "SwaggerJSON", + "operationId": "GetGenesis", "responses": { "200": { "description": "The genesis file in json.", diff --git a/daemon/algod/api/algod.oas3.yml b/daemon/algod/api/algod.oas3.yml index b44790315b..69a0f3fc47 100644 --- a/daemon/algod/api/algod.oas3.yml +++ b/daemon/algod/api/algod.oas3.yml @@ -1362,7 +1362,7 @@ "/genesis": { "get": { "description": "Returns the entire genesis file in json.", - "operationId": "SwaggerJSON", + "operationId": "GetGenesis", "responses": { "200": { "content": { From 09253dd64fe66d8dcd54d8fc2336784fd4656c22 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Fri, 4 Sep 2020 09:37:55 -0400 Subject: [PATCH 015/136] finish merge... --- daemon/algod/api/algod.oas3.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/daemon/algod/api/algod.oas3.yml b/daemon/algod/api/algod.oas3.yml index 6fe16a5249..fb6e66a5c0 100644 --- a/daemon/algod/api/algod.oas3.yml +++ b/daemon/algod/api/algod.oas3.yml @@ -2113,8 +2113,7 @@ "type": "object" } } - }, - "description": "(empty)" + } }, "400": { "content": { @@ -3254,7 +3253,7 @@ }, "/versions": { "get": { - "description": "Retrieves the API version, binary build versions, and genesis information.", + "description": "Retrieves the supported API version, binary build versions, and genesis information.", "operationId": "GetVersion", "responses": { "200": { From bcc035060c1c75899ff7ad652e34e07b5f6857a3 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Fri, 4 Sep 2020 10:41:32 -0400 Subject: [PATCH 016/136] Minor catchpoint error handling code cleanup (#1484) Improve error object creation for catchpoint errors --- .../algod/api/server/v2/test/handlers_test.go | 6 ++--- node/error.go | 25 +++++++++++++------ node/node.go | 4 +-- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/daemon/algod/api/server/v2/test/handlers_test.go b/daemon/algod/api/server/v2/test/handlers_test.go index 2114deda77..0998d8449f 100644 --- a/daemon/algod/api/server/v2/test/handlers_test.go +++ b/daemon/algod/api/server/v2/test/handlers_test.go @@ -19,7 +19,6 @@ package test import ( "bytes" "errors" - "github.com/algorand/go-algorand/node" "io" "net/http" "net/http/httptest" @@ -36,6 +35,7 @@ import ( "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/transactions/logic" "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/node" "github.com/algorand/go-algorand/protocol" ) @@ -290,10 +290,10 @@ func TestStartCatchup(t *testing.T) { goodCatchPoint := "5894690#DVFRZUYHEFKRLK5N6DNJRR4IABEVN2D6H76F3ZSEPIE6MKXMQWQA" startCatchupTest(t, goodCatchPoint, nil, 201) - inProgressError := node.MakeCatchpointAlreadyInProgressError("error") + inProgressError := node.MakeCatchpointAlreadyInProgressError("catchpoint") startCatchupTest(t, goodCatchPoint, inProgressError, 200) - unableToStartError := node.MakeCatchpointUnableToStartError("error") + unableToStartError := node.MakeCatchpointUnableToStartError("running", "requested") startCatchupTest(t, goodCatchPoint, unableToStartError, 400) startCatchupTest(t, goodCatchPoint, errors.New("anothing else is internal"), 500) diff --git a/node/error.go b/node/error.go index d3aa5a6624..4afc08bda3 100644 --- a/node/error.go +++ b/node/error.go @@ -16,40 +16,49 @@ package node +import ( + "fmt" +) + // Catchpoint already in progress error // CatchpointAlreadyInProgressError indicates that the requested catchpoint is already running type CatchpointAlreadyInProgressError struct { - message string + catchpoint string } // MakeCatchpointAlreadyInProgressError creates the error -func MakeCatchpointAlreadyInProgressError(text string) *CatchpointAlreadyInProgressError { +func MakeCatchpointAlreadyInProgressError(catchpoint string) *CatchpointAlreadyInProgressError { return &CatchpointAlreadyInProgressError{ - message: text, + catchpoint: catchpoint, } } // Error satisfies builtin interface `error` func (e *CatchpointAlreadyInProgressError) Error() string { - return e.message + return fmt.Sprintf("the requested catchpoint '%s' is already in progress, suppressing error", e.catchpoint) } // Catchpoint unable to start error // CatchpointUnableToStartError indicates that the requested catchpoint cannot be started type CatchpointUnableToStartError struct { - message string + catchpointRunning string + catchpointRequested string } // MakeCatchpointUnableToStartError creates the error -func MakeCatchpointUnableToStartError(text string) *CatchpointUnableToStartError { +func MakeCatchpointUnableToStartError(catchpointRunning, catchpointRequested string) *CatchpointUnableToStartError { return &CatchpointUnableToStartError{ - message: text, + catchpointRunning: catchpointRunning, + catchpointRequested: catchpointRequested, } } // Error satisfies builtin interface `error` func (e *CatchpointUnableToStartError) Error() string { - return e.message + return fmt.Sprintf( + "unable to start catchpoint catchup for '%s' - already catching up '%s'", + e.catchpointRequested, + e.catchpointRunning) } diff --git a/node/node.go b/node/node.go index 2c691e42cd..20e6ece7ff 100644 --- a/node/node.go +++ b/node/node.go @@ -841,9 +841,9 @@ func (node *AlgorandFullNode) StartCatchup(catchpoint string) error { stats := node.catchpointCatchupService.GetStatistics() // No need to return an error if catchpoint == stats.CatchpointLabel { - return MakeCatchpointAlreadyInProgressError(fmt.Sprintf("the requested catchpoint '%s' is already in progress, suppressing error", catchpoint)) + return MakeCatchpointAlreadyInProgressError(catchpoint) } - return MakeCatchpointUnableToStartError(fmt.Sprintf("unable to start catchpoint catchup for '%s' - already catching up '%s'", catchpoint, stats.CatchpointLabel)) + return MakeCatchpointUnableToStartError(stats.CatchpointLabel, catchpoint) } var err error node.catchpointCatchupService, err = catchup.MakeNewCatchpointCatchupService(catchpoint, node, node.log, node.net, node.ledger.Ledger, node.config) From 423e9592367b91bf6fc1cfc177b7ac8c3f24f312 Mon Sep 17 00:00:00 2001 From: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> Date: Fri, 4 Sep 2020 13:51:08 -0400 Subject: [PATCH 017/136] Simplify dryrunning and debugging group txn (#1405) Extend clerk dryrun so it can dump dryrun state object suitable for REST, tealdbg and python tests --- cmd/goal/clerk.go | 17 ++- cmd/tealdbg/main.go | 4 +- daemon/algod/api/server/v2/dryrun.go | 8 +- libgoal/libgoal.go | 103 +++++++++--------- .../cli/goal/expect/goalDryrunRestTest.exp | 23 ++++ 5 files changed, 96 insertions(+), 59 deletions(-) diff --git a/cmd/goal/clerk.go b/cmd/goal/clerk.go index c685c07539..5e91a9b18e 100644 --- a/cmd/goal/clerk.go +++ b/cmd/goal/clerk.go @@ -128,6 +128,9 @@ func init() { dryrunCmd.Flags().StringVarP(&txFilename, "txfile", "t", "", "transaction or transaction-group to test") dryrunCmd.Flags().StringVarP(&protoVersion, "proto", "P", "", "consensus protocol version id string") + dryrunCmd.Flags().BoolVar(&dumpForDryrun, "dryrun-dump", false, "Dump in dryrun format acceptable by dryrun REST api instead of running") + dryrunCmd.Flags().Var(&dumpForDryrunFormat, "dryrun-dump-format", "Dryrun dump format: "+dumpForDryrunFormat.AllowedString()) + dryrunCmd.Flags().StringVarP(&outFilename, "outfile", "o", "", "Filename for writing dryrun state object") dryrunCmd.MarkFlagRequired("txfile") dryrunRemoteCmd.Flags().StringVarP(&txFilename, "dryrun-state", "D", "", "dryrun request object to run") @@ -979,10 +982,22 @@ var dryrunCmd = &cobra.Command{ for i, st := range stxns { txgroup[i] = st } + proto, params := getProto(protoVersion) + if dumpForDryrun { + // Write dryrun data to file + dataDir := ensureSingleDataDir() + client := ensureFullClient(dataDir) + data, err := libgoal.MakeDryrunStateBytes(client, nil, txgroup, string(proto), dumpForDryrunFormat.String()) + if err != nil { + reportErrorf(err.Error()) + } + writeFile(outFilename, data, 0600) + return + } + if timeStamp <= 0 { timeStamp = time.Now().Unix() } - _, params := getProto(protoVersion) for i, txn := range txgroup { if txn.Lsig.Blank() { continue diff --git a/cmd/tealdbg/main.go b/cmd/tealdbg/main.go index 1715c80af0..1399d3071d 100644 --- a/cmd/tealdbg/main.go +++ b/cmd/tealdbg/main.go @@ -184,8 +184,8 @@ func debugLocal(args []string) { // program can be set either directly // or with SignedTxn.Lsig.Logic, // or with BalanceRecord.AppParams.ApprovalProgram - if len(args) == 0 && (len(txnFile) == 0 || len(balanceFile) == 0) && len(ddrFile) == 0 { - log.Fatalln("No program to debug: must specify program(s), or transaction(s) and a balance record(s), or dryrun-req object") + if len(args) == 0 && len(txnFile) == 0 && len(ddrFile) == 0 { + log.Fatalln("No program to debug: must specify program(s), or transaction(s), or dryrun-req object") } if len(args) == 0 && groupIndex != 0 { diff --git a/daemon/algod/api/server/v2/dryrun.go b/daemon/algod/api/server/v2/dryrun.go index a2c62a921b..c1b0b5f414 100644 --- a/daemon/algod/api/server/v2/dryrun.go +++ b/daemon/algod/api/server/v2/dryrun.go @@ -411,12 +411,11 @@ func doDryrunRequest(dr *DryrunRequest, proto *config.ConsensusParams, response GroupIndex: ti, //Logger: nil, // TODO: capture logs, send them back } - var result *generated.DryrunTxnResult + var result generated.DryrunTxnResult if len(stxn.Lsig.Logic) > 0 { var debug dryrunDebugReceiver ep.Debugger = &debug pass, err := logic.Eval(stxn.Lsig.Logic, ep) - result = new(generated.DryrunTxnResult) var messages []string result.Disassembly = debug.lines result.LogicSigTrace = &debug.history @@ -460,9 +459,6 @@ func doDryrunRequest(dr *DryrunRequest, proto *config.ConsensusParams, response } } var messages []string - if result == nil { - result = new(generated.DryrunTxnResult) - } if !ok { messages = make([]string, 1) messages[0] = fmt.Sprintf("uploaded state did not include app id %d referenced in txn[%d]", appIdx, ti) @@ -507,7 +503,7 @@ func doDryrunRequest(dr *DryrunRequest, proto *config.ConsensusParams, response } result.AppCallMessages = &messages } - response.Txns[ti] = *result + response.Txns[ti] = result } } diff --git a/libgoal/libgoal.go b/libgoal/libgoal.go index 1b465a134b..94e36cac64 100644 --- a/libgoal/libgoal.go +++ b/libgoal/libgoal.go @@ -926,7 +926,7 @@ func MakeDryrunStateBytes(client Client, txnOrStxn interface{}, other []transact } } -// MakeDryrunState function creates DryrunRequest data structure and serializes it into a file +// MakeDryrunState function creates v2.DryrunRequest data structure func MakeDryrunState(client Client, txnOrStxn interface{}, other []transactions.SignedTxn, proto string) (dr v2.DryrunRequest, err error) { gdr, err := MakeDryrunStateGenerated(client, txnOrStxn, other, proto) if err != nil { @@ -935,78 +935,81 @@ func MakeDryrunState(client Client, txnOrStxn interface{}, other []transactions. return v2.DryrunRequestFromGenerated(&gdr) } -// MakeDryrunStateGenerated function creates DryrunRequest data structure and serializes it into a file +// MakeDryrunStateGenerated function creates generatedV2.DryrunRequest data structure func MakeDryrunStateGenerated(client Client, txnOrStxn interface{}, other []transactions.SignedTxn, proto string) (dr generatedV2.DryrunRequest, err error) { var txns []transactions.SignedTxn - var tx *transactions.Transaction - if txn, ok := txnOrStxn.(transactions.Transaction); ok { - tx = &txn + if txnOrStxn == nil { + // empty input do nothing + } else if txn, ok := txnOrStxn.(transactions.Transaction); ok { txns = append(txns, transactions.SignedTxn{Txn: txn}) } else if stxn, ok := txnOrStxn.(transactions.SignedTxn); ok { - tx = &stxn.Txn txns = append(txns, stxn) } else { err = fmt.Errorf("unsupported txn type") return } + txns = append(txns, other...) for i := range txns { enc := protocol.EncodeJSON(&txns[i]) dr.Txns = append(dr.Txns, enc) } - if tx.Type == protocol.ApplicationCallTx { - apps := []basics.AppIndex{tx.ApplicationID} - apps = append(apps, tx.ForeignApps...) - for _, appIdx := range apps { - var appParams generatedV2.ApplicationParams - if appIdx == 0 { - // if it is an app create txn then use params from the txn - appParams.ApprovalProgram = tx.ApprovalProgram - appParams.ClearStateProgram = tx.ClearStateProgram - appParams.GlobalStateSchema = &generatedV2.ApplicationStateSchema{ - NumUint: tx.GlobalStateSchema.NumUint, - NumByteSlice: tx.GlobalStateSchema.NumByteSlice, - } - appParams.LocalStateSchema = &generatedV2.ApplicationStateSchema{ - NumUint: tx.LocalStateSchema.NumUint, - NumByteSlice: tx.LocalStateSchema.NumByteSlice, + for _, txn := range txns { + tx := txn.Txn + if tx.Type == protocol.ApplicationCallTx { + apps := []basics.AppIndex{tx.ApplicationID} + apps = append(apps, tx.ForeignApps...) + for _, appIdx := range apps { + var appParams generatedV2.ApplicationParams + if appIdx == 0 { + // if it is an app create txn then use params from the txn + appParams.ApprovalProgram = tx.ApprovalProgram + appParams.ClearStateProgram = tx.ClearStateProgram + appParams.GlobalStateSchema = &generatedV2.ApplicationStateSchema{ + NumUint: tx.GlobalStateSchema.NumUint, + NumByteSlice: tx.GlobalStateSchema.NumByteSlice, + } + appParams.LocalStateSchema = &generatedV2.ApplicationStateSchema{ + NumUint: tx.LocalStateSchema.NumUint, + NumByteSlice: tx.LocalStateSchema.NumByteSlice, + } + appParams.Creator = tx.Sender.String() + // zero is not acceptable by ledger in dryrun/debugger + appIdx = defaultAppIdx + } else { + // otherwise need to fetch app state + var app generatedV2.Application + if app, err = client.ApplicationInformation(uint64(tx.ApplicationID)); err != nil { + return + } + appParams = app.Params } - appParams.Creator = tx.Sender.String() - // zero is not acceptable by ledger in dryrun/debugger - appIdx = defaultAppIdx - } else { - // otherwise need to fetch app state - var app generatedV2.Application - if app, err = client.ApplicationInformation(uint64(tx.ApplicationID)); err != nil { + dr.Apps = append(dr.Apps, generatedV2.Application{ + Id: uint64(appIdx), + Params: appParams, + }) + } + + accounts := append(tx.Accounts, tx.Sender) + for _, acc := range accounts { + var info generatedV2.Account + if info, err = client.AccountInformationV2(acc.String()); err != nil { return } - appParams = app.Params + dr.Accounts = append(dr.Accounts, info) } - dr.Apps = append(dr.Apps, generatedV2.Application{ - Id: uint64(appIdx), - Params: appParams, - }) - } - accounts := append(tx.Accounts, tx.Sender) - for _, acc := range accounts { - var info generatedV2.Account - if info, err = client.AccountInformationV2(acc.String()); err != nil { + dr.ProtocolVersion = proto + if dr.Round, err = client.CurrentRound(); err != nil { return } - dr.Accounts = append(dr.Accounts, info) - } - - dr.ProtocolVersion = proto - if dr.Round, err = client.CurrentRound(); err != nil { - return - } - var b v1.Block - if b, err = client.Block(dr.Round); err != nil { - return + var b v1.Block + if b, err = client.Block(dr.Round); err != nil { + return + } + dr.LatestTimestamp = uint64(b.Timestamp) } - dr.LatestTimestamp = uint64(b.Timestamp) } return } diff --git a/test/e2e-go/cli/goal/expect/goalDryrunRestTest.exp b/test/e2e-go/cli/goal/expect/goalDryrunRestTest.exp index baaf242635..d9cff62725 100644 --- a/test/e2e-go/cli/goal/expect/goalDryrunRestTest.exp +++ b/test/e2e-go/cli/goal/expect/goalDryrunRestTest.exp @@ -101,6 +101,27 @@ if { [catch { timeout { ::AlgorandGoal::Abort "goal app create timeout" } } + # atomic transfer + set DRREQ_FILE_4 "$TEST_ROOT_DIR/atomic-tran-drreq.msgp" + set AT_TX1_FILE "$TEST_ROOT_DIR/atomic-tran-tx1.mspg" + set AT_TX2_FILE "$TEST_ROOT_DIR/atomic-tran-tx2.mspg" + set AT_COMBINED_FILE "$TEST_ROOT_DIR/atomic-tran-comb.mspg" + set AT_GROUPPED_FILE "$TEST_ROOT_DIR/atomic-tran-group.mspg" + spawn goal clerk send --from $PRIMARY_ACCOUNT_ADDRESS --to $PRIMARY_ACCOUNT_ADDRESS -a 1 --fee 1000 -d $TEST_PRIMARY_NODE_DIR -o $AT_TX1_FILE + expect { + timeout { ::AlgorandGoal::Abort "goal clerk send timeout" } + } + spawn goal app create --creator $PRIMARY_ACCOUNT_ADDRESS --approval-prog $TEAL_PROG_FILE --clear-prog $TEAL_PROG_FILE --global-byteslices 0 --global-ints 0 --local-byteslices 0 --local-ints 0 -d $TEST_PRIMARY_NODE_DIR -o $AT_TX2_FILE + expect { + timeout { ::AlgorandGoal::Abort "goal app create timeout" } + } + exec cat $AT_TX1_FILE $AT_TX2_FILE > $AT_COMBINED_FILE + exec goal clerk group -i $AT_COMBINED_FILE -o $AT_GROUPPED_FILE + spawn goal clerk dryrun -t $AT_GROUPPED_FILE -d $TEST_PRIMARY_NODE_DIR -o $DRREQ_FILE_4 --dryrun-dump --dryrun-dump-format=msgp + expect { + timeout { ::AlgorandGoal::Abort "goal clerk dryrun timeout" } + } + # invalid app set INVALID_FILE_1 "$TEST_ROOT_DIR/invalid-app.json" set INVALID_FILE_1_ID [open $INVALID_FILE_1 "w"] @@ -111,6 +132,8 @@ if { [catch { TestGoalDryrun $DRREQ_FILE_1 $TEST_PRIMARY_NODE_DIR TestGoalDryrun $DRREQ_FILE_2 $TEST_PRIMARY_NODE_DIR TestGoalDryrun $DRREQ_FILE_3 $TEST_PRIMARY_NODE_DIR + TestGoalDryrun $DRREQ_FILE_4 $TEST_PRIMARY_NODE_DIR + TestGoalDryrunExitCode $DRREQ_FILE_3 $TEST_PRIMARY_NODE_DIR 0 "PASS" TestGoalDryrunExitCode "" $TEST_PRIMARY_NODE_DIR 1 "Cannot read file : open : no such file or directory" TestGoalDryrunExitCode $INVALID_FILE_1 $TEST_PRIMARY_NODE_DIR 1 "dryrun-remote: HTTP 400 Bad Request:" From 0f6f1910a223326f3e2b17550bb385567aee7641 Mon Sep 17 00:00:00 2001 From: algonautshant <55754073+algonautshant@users.noreply.github.com> Date: Fri, 4 Sep 2020 14:02:51 -0400 Subject: [PATCH 018/136] Add unit tests to logic debugger (#1466) Add unit tests to logic debugger --- data/transactions/logic/debugger.go | 1 + data/transactions/logic/debugger_test.go | 50 +++++++++++++++++++++ data/transactions/logic/eval_test.go | 36 ++++++++++++++++ data/transactions/logic/opcodes.go | 2 - data/transactions/logic/opcodes_test.go | 55 +++++++++++++++++++++++- 5 files changed, 141 insertions(+), 3 deletions(-) diff --git a/data/transactions/logic/debugger.go b/data/transactions/logic/debugger.go index 5e72754542..809136509f 100644 --- a/data/transactions/logic/debugger.go +++ b/data/transactions/logic/debugger.go @@ -185,6 +185,7 @@ func stackValueToTealValue(sv *stackValue) basics.TealValue { } } +// valueDeltaToValueDelta converts delta's bytes to base64 in a new struct func valueDeltaToValueDelta(vd *basics.ValueDelta) basics.ValueDelta { return basics.ValueDelta{ Action: vd.Action, diff --git a/data/transactions/logic/debugger_test.go b/data/transactions/logic/debugger_test.go index 8a2fba90e6..978ed2a4a3 100644 --- a/data/transactions/logic/debugger_test.go +++ b/data/transactions/logic/debugger_test.go @@ -18,8 +18,10 @@ package logic import ( "os" + "encoding/base64" "testing" + "github.com/algorand/go-algorand/data/basics" "github.com/stretchr/testify/require" ) @@ -125,3 +127,51 @@ func TestDebuggerHook(t *testing.T) { require.Greater(t, testDbg.update, 1) require.Equal(t, 1, len(testDbg.state.Stack)) } + +func TestLineToPC(t *testing.T) { + dState := DebugState{ + Disassembly: "abc\ndef\nghi", + PCOffset: []PCOffset{{PC: 1, Offset: 4}, {PC: 2, Offset: 8}, {PC: 3, Offset: 12}}, + } + pc := dState.LineToPC(0) + require.Equal(t, 0, pc) + + pc = dState.LineToPC(1) + require.Equal(t, 1, pc) + + pc = dState.LineToPC(2) + require.Equal(t, 2, pc) + + pc = dState.LineToPC(3) + require.Equal(t, 3, pc) + + pc = dState.LineToPC(4) + require.Equal(t, 0, pc) + + pc = dState.LineToPC(-1) + require.Equal(t, 0, pc) + + pc = dState.LineToPC(0x7fffffff) + require.Equal(t, 0, pc) + + dState.PCOffset = []PCOffset{} + pc = dState.LineToPC(1) + require.Equal(t, 0, pc) + + dState.PCOffset = []PCOffset{{PC: 1, Offset: 0}} + pc = dState.LineToPC(1) + require.Equal(t, 0, pc) +} + +func TestValueDeltaToValueDelta(t *testing.T) { + vDelta := basics.ValueDelta{ + Action: basics.SetUintAction, + Bytes: "some string", + Uint: uint64(0xffffffff), + } + ans := valueDeltaToValueDelta(&vDelta) + require.Equal(t, vDelta.Action, ans.Action) + require.NotEqual(t, vDelta.Bytes, ans.Bytes) + require.Equal(t, base64.StdEncoding.EncodeToString([]byte(vDelta.Bytes)), ans.Bytes) + require.Equal(t, vDelta.Uint, ans.Uint) +} diff --git a/data/transactions/logic/eval_test.go b/data/transactions/logic/eval_test.go index 4b5df059cf..1446e13958 100644 --- a/data/transactions/logic/eval_test.go +++ b/data/transactions/logic/eval_test.go @@ -110,6 +110,42 @@ func TestEmptyProgram(t *testing.T) { require.False(t, pass) } +// TestMinTealVersionParamEval tests eval/check reading the MinTealVersion from the param +func TestMinTealVersionParamEvalCheck(t *testing.T) { + t.Parallel() + params := defaultEvalParams(nil, nil) + version2 := uint64(rekeyingEnabledVersion) + params.MinTealVersion = &version2 + program := make([]byte, binary.MaxVarintLen64) + // set the teal program version to 1 + binary.PutUvarint(program, 1) + + _, err := Check(program, params) + require.Contains(t, err.Error(), fmt.Sprintf("program version must be >= %d", appsEnabledVersion)) + + // If the param is read correctly, the eval should fail + pass, err := Eval(program, params) + require.Error(t, err) + require.Contains(t, err.Error(), fmt.Sprintf("program version must be >= %d", appsEnabledVersion)) + require.False(t, pass) +} + +func TestTxnFieldToTealValue(t *testing.T) { + + txn := transactions.Transaction{} + groupIndex := 0 + field := FirstValid + values := [6]uint64{0, 1, 2, 0xffffffff, 0xffffffffffffffff} + + for _, value := range values { + txn.FirstValid = basics.Round(value) + tealValue, err := TxnFieldToTealValue(&txn, groupIndex, field) + require.NoError(t, err) + require.Equal(t, basics.TealUintType, tealValue.Type) + require.Equal(t, value, tealValue.Uint) + } +} + func TestWrongProtoVersion(t *testing.T) { t.Parallel() for v := uint64(1); v <= AssemblerMaxVersion; v++ { diff --git a/data/transactions/logic/opcodes.go b/data/transactions/logic/opcodes.go index 83e57ce4a1..6f41c0089d 100644 --- a/data/transactions/logic/opcodes.go +++ b/data/transactions/logic/opcodes.go @@ -174,8 +174,6 @@ func (a sortByOpcode) Less(i, j int) bool { return a[i].Opcode < a[j].Opcode } // OpcodesByVersion returns list of opcodes available in a specific version of TEAL // by copying v1 opcodes to v2 to create a full list. -// This function must be used for documentation only because it modifies opcode versions -// to the first introduced for the opcodes updated in later versions. func OpcodesByVersion(version uint64) []OpSpec { // for updated opcodes use the lowest version opcode was introduced in maxOpcode := 0 diff --git a/data/transactions/logic/opcodes_test.go b/data/transactions/logic/opcodes_test.go index 7364cd7197..f2031142b9 100644 --- a/data/transactions/logic/opcodes_test.go +++ b/data/transactions/logic/opcodes_test.go @@ -32,8 +32,57 @@ func TestOpSpecs(t *testing.T) { } } +func (os *OpSpec) equals(oso *OpSpec) bool { + if os.Opcode != oso.Opcode { + return false + } + if os.Name != oso.Name { + return false + } + if !reflect.DeepEqual(os.Args, oso.Args) { + return false + } + if !reflect.DeepEqual(os.Returns, oso.Returns) { + return false + } + if os.Version != oso.Version { + return false + } + if os.Modes != oso.Modes { + return false + } + + return true +} + +func TestOpcodesByVersionReordered(t *testing.T) { + + // Make a copy to restore to the original + OpSpecsOrig := make([]OpSpec, len(OpSpecs)) + for idx, opspec := range OpSpecs { + cp := opspec + OpSpecsOrig[idx] = cp + } + defer func() { + OpSpecs = OpSpecsOrig + }() + + // To test the case where a newer version opcode is before an older version + // Change the order of opcode 0x01 so that version 2 comes before version 1 + tmp := OpSpecs[1] + OpSpecs[1] = OpSpecs[4] + OpSpecs[4] = tmp + + t.Run("TestOpcodesByVersion", TestOpcodesByVersion) +} + func TestOpcodesByVersion(t *testing.T) { - t.Parallel() + // Make a copy of the OpSpecs to check if OpcodesByVersion will change it + OpSpecs2 := make([]OpSpec, len(OpSpecs)) + for idx, opspec := range OpSpecs { + cp := opspec + OpSpecs2[idx] = cp + } opSpecs := make([][]OpSpec, 2) for v := uint64(1); v <= LogicVersion; v++ { @@ -59,6 +108,10 @@ func TestOpcodesByVersion(t *testing.T) { }) } require.Greater(t, len(opSpecs[1]), len(opSpecs[0])) + + for idx, opspec := range OpSpecs { + require.True(t, opspec.equals(&OpSpecs2[idx])) + } } func TestOpcodesVersioningV2(t *testing.T) { From da3c4562320872646eb433a9ddb684e5fb2648ae Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Fri, 4 Sep 2020 14:04:39 -0400 Subject: [PATCH 019/136] Add support for algorelay deleting unused SRV and DNS entries Add support for algorelay deleting unused SRV and DNS entries --- cmd/algorelay/relayCmd.go | 165 +++++++++++++++++++------ tools/network/cloudflare/cloudflare.go | 2 +- 2 files changed, 128 insertions(+), 39 deletions(-) diff --git a/cmd/algorelay/relayCmd.go b/cmd/algorelay/relayCmd.go index 3bebad9c1c..616c296df7 100644 --- a/cmd/algorelay/relayCmd.go +++ b/cmd/algorelay/relayCmd.go @@ -228,6 +228,7 @@ var checkCmd = &cobra.Command{ if enabledRelaysDNSAlias[alias] { continue } + if relaysDNSAlias[alias] { fmt.Printf("WARN : disabled relay %s has a _algobootstrap entry\n", bootstrap) } else { @@ -291,11 +292,6 @@ var updateCmd = &cobra.Command{ continue } - if !relay.CheckSuccess { - fmt.Printf("[%d] OK: Skipping NotSuccessful %s\n", relay.ID, relay.Address) - // Don't output results if skipped - continue - } const checkOnly = false name, port, err := ensureRelayStatus(checkOnly, relay, nameDomainArg, srvDomainArg, defaultPortArg, context) if err != nil { @@ -307,7 +303,11 @@ var updateCmd = &cobra.Command{ }) anyUpdateError = true } else { - fmt.Printf("[%d] OK: %s -> %s:%d\n", relay.ID, relay.Address, name, port) + if relay.CheckSuccess { + fmt.Printf("[%d] OK: %s -> %s:%d\n", relay.ID, relay.Address, name, port) + } else { + fmt.Printf("[%d] OK: %s removed ( if it was there )\n", relay.ID, relay.Address) + } results = append(results, checkResult{ ID: relay.ID, Success: true, @@ -380,6 +380,7 @@ func ensureRelayStatus(checkOnly bool, relay eb.Relay, nameDomain string, srvDom } targetDomainAlias := relay.DNSAlias + "." + nameDomain + if topmost != targetDomainAlias { if checkOnly { err = fmt.Errorf("topmost DNS name is not the assigned DNS Alias (wanted: %s, found %s)", @@ -387,19 +388,35 @@ func ensureRelayStatus(checkOnly bool, relay eb.Relay, nameDomain string, srvDom return } - // Add A/CNAME for the DNSAlias assigned - err = addDNSRecord(targetDomainAlias, topmost, ctx.nameZoneID) + if relay.CheckSuccess { + // Add A/CNAME for the DNSAlias assigned + err = addDNSRecord(targetDomainAlias, topmost, ctx.nameZoneID) + if err != nil { + return + } + fmt.Printf("[%d] Added DNS Record: %s -> %s\n", relay.ID, targetDomainAlias, topmost) + + // Update our state + names = append(names, targetDomainAlias) + topmost = targetDomainAlias + } else { + // remove entry. + err = deleteDNSRecord(targetDomainAlias, topmost, ctx.nameZoneID) + if err != nil { + return + } + fmt.Printf("[%d] Removed DNS Record: %s -> %s\n", relay.ID, targetDomainAlias, topmost) + } + } else if !relay.CheckSuccess { + // remove entry. + err = deleteDNSRecord(targetDomainAlias, names[0], ctx.nameZoneID) if err != nil { return } - fmt.Printf("[%d] Added DNS Record: %s -> %s\n", relay.ID, targetDomainAlias, topmost) - - // Update our state - names = append(names, targetDomainAlias) - topmost = targetDomainAlias + fmt.Printf("[%d] Removed DNS Record: %s -> %s\n", relay.ID, targetDomainAlias, names[0]) } - var ensureEntry = func(use string, entries map[string]uint16, port uint16) error { + var ensureEntry = func(use string, entries map[string]uint16, port uint16) (matchingEntries int, err error) { type srvMatch struct { name string port uint16 @@ -415,50 +432,85 @@ func ensureRelayStatus(checkOnly bool, relay eb.Relay, nameDomain string, srvDom } if len(matches) == 0 { - return fmt.Errorf("no %s SRV entries found mapping to %s in '%s'", use, target, srvDomain) + return 0, fmt.Errorf("no %s SRV entries found mapping to %s in '%s'", use, target, srvDomain) } if len(matches) > 1 { - return fmt.Errorf("multiple %s SRV entries found in the chain mapping to %s", use, target) + return len(matches), fmt.Errorf("multiple %s SRV entries found in the chain mapping to %s", use, target) } if matches[0].name != topmost || matches[0].port != port { - return fmt.Errorf("existing %s SRV record mapped to intermediate DNS name or wrong port (wanted %s:%d, found %s:%d)", + return len(matches), fmt.Errorf("existing %s SRV record mapped to intermediate DNS name or wrong port (wanted %s:%d, found %s:%d)", use, topmost, port, matches[0].name, matches[0].port) } - return nil + return len(matches), nil } - err = ensureEntry("bootstrap", ctx.bootstrap.entries, port) - if err != nil { - if checkOnly { - return - } - - // Add SRV entry to map to our DNSAlias - err = addSRVRecord(ctx.bootstrap.networkName, topmost, port, ctx.bootstrap.shortName, ctx.srvZoneID) - if err != nil { - return - } - fmt.Printf("[%d] Added boostrap SRV Record: %s:%d\n", relay.ID, targetDomainAlias, port) - } - - err = ensureEntry("metrics", ctx.metrics.entries, metricsPort) - if relay.MetricsEnabled { + var matchCount int + matchCount, err = ensureEntry("algobootstrap", ctx.bootstrap.entries, port) + if relay.CheckSuccess { if err != nil { if checkOnly { return } - // Add SRV entry for metrics - err = addSRVRecord(ctx.metrics.networkName, topmost, metricsPort, ctx.metrics.shortName, ctx.srvZoneID) + // Add SRV entry to map to our DNSAlias + err = addSRVRecord(ctx.bootstrap.networkName, topmost, port, ctx.bootstrap.shortName, ctx.srvZoneID) if err != nil { return } - fmt.Printf("[%d] Added metrics SRV Record: %s:%d\n", relay.ID, targetDomainAlias, metricsPort) + fmt.Printf("[%d] Added boostrap SRV Record: %s:%d\n", relay.ID, targetDomainAlias, port) + } + } else { + if matchCount > 0 { + err = clearSRVRecord(ctx.bootstrap.networkName, topmost, ctx.bootstrap.shortName, ctx.srvZoneID) + if err != nil { + return + } + fmt.Printf("[%d] Removed boostrap SRV Record: %s\n", relay.ID, targetDomainAlias) + } + } + + matchCount, err = ensureEntry("metrics", ctx.metrics.entries, metricsPort) + if relay.MetricsEnabled { + if relay.CheckSuccess { + if err != nil { + if checkOnly { + return + } + + // Add SRV entry for metrics + err = addSRVRecord(ctx.metrics.networkName, topmost, metricsPort, ctx.metrics.shortName, ctx.srvZoneID) + if err != nil { + return + } + fmt.Printf("[%d] Added metrics SRV Record: %s:%d\n", relay.ID, targetDomainAlias, metricsPort) + } + } else { + if matchCount > 0 { + // metrics are enabled, but we should delete the entry since it failed the success test. + err = clearSRVRecord(ctx.metrics.networkName, topmost, ctx.metrics.shortName, ctx.srvZoneID) + if err != nil { + return + } + fmt.Printf("[%d] Removed metrics SRV Record: %s\n", relay.ID, targetDomainAlias) + } else { + err = nil + } } } else if err == nil { - err = fmt.Errorf("metrics should not be registered for %s but it is", target) + if checkOnly { + err = fmt.Errorf("metrics should not be registered for %s but it is", target) + return + } + if matchCount > 0 { + // delete the metric entry. + err = clearSRVRecord(ctx.metrics.networkName, topmost, ctx.metrics.shortName, ctx.srvZoneID) + if err != nil { + return + } + fmt.Printf("[%d] Removed metrics SRV Record: %s\n", relay.ID, targetDomainAlias) + } } else { // If metrics are not enabled, then we SHOULD get an error. // Since this isn't actually an error, reset to nil @@ -575,3 +627,40 @@ func addSRVRecord(srvNetwork string, target string, port uint16, serviceShortNam return cloudflareDNS.SetSRVRecord(context.Background(), srvNetwork, target, cloudflare.AutomaticTTL, priority, uint(port), serviceShortName, "_tcp", weight) } + +func clearSRVRecord(srvNetwork string, target string, serviceShortName string, cfZoneID string) error { + cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfAuthKey) + return cloudflareDNS.ClearSRVRecord(context.Background(), srvNetwork, target, serviceShortName, "_tcp") +} + +func deleteDNSRecord(from string, to string, cfZoneID string) (err error) { + isIP := net.ParseIP(to) != nil + var recordType string + if isIP { + recordType = "A" + } else { + recordType = "CNAME" + } + + cloudflareDNS := cloudflare.NewDNS(cfZoneID, cfEmail, cfAuthKey) + + var records []cloudflare.DNSRecordResponseEntry + records, err = cloudflareDNS.ListDNSRecord(context.Background(), recordType, "", "", "", "", "") + if err != nil { + return + } + + for _, record := range records { + // Error if duplicates found + recordFrom := strings.ToLower(record.Name) + recordTarget := strings.ToLower(record.Content) + if from == recordFrom && recordTarget == to { + // delete the entry + err = cloudflareDNS.DeleteDNSRecord(context.Background(), record.ID) + if err != nil { + return err + } + } + } + return nil +} diff --git a/tools/network/cloudflare/cloudflare.go b/tools/network/cloudflare/cloudflare.go index 80c4f13544..122fe00c22 100644 --- a/tools/network/cloudflare/cloudflare.go +++ b/tools/network/cloudflare/cloudflare.go @@ -103,7 +103,7 @@ func (d *DNS) ClearSRVRecord(ctx context.Context, name string, target string, se return err } if len(entries) == 0 { - fmt.Printf("No SRV entry for '%s'='%s'.\n", name, target) + fmt.Printf("No SRV entry for '[%s.%s.]%s'='%s'.\n", service, protocol, name, target) return nil } From 8fd86a225714e10e014828e2f4037d6e521194ed Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Fri, 4 Sep 2020 16:44:43 -0400 Subject: [PATCH 020/136] Add a benchmark to test restoring from catchpoint performance. (#1485) Add a benchmark to test restoring from catchpoint performance --- ledger/catchupaccessor_test.go | 111 +++++++++++++++++++++++++++++++++ ledger/ledger_test.go | 8 +-- 2 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 ledger/catchupaccessor_test.go diff --git a/ledger/catchupaccessor_test.go b/ledger/catchupaccessor_test.go new file mode 100644 index 0000000000..75c3a2d93b --- /dev/null +++ b/ledger/catchupaccessor_test.go @@ -0,0 +1,111 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package ledger + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" +) + +func benchmarkRestoringFromCatchpointFileHelper(b *testing.B) { + genesisInitState, _ := testGenerateInitState(b, protocol.ConsensusCurrentVersion) + const inMem = false + log := logging.TestingLog(b) + cfg := config.GetDefaultLocal() + cfg.Archival = false + log.SetLevel(logging.Warn) + dbBaseFileName := strings.Replace(b.Name(), "/", "_", -1) + l, err := OpenLedger(log, dbBaseFileName, inMem, genesisInitState, cfg) + require.NoError(b, err, "could not open ledger") + defer func() { + l.Close() + os.Remove(dbBaseFileName + ".block.sqlite") + os.Remove(dbBaseFileName + ".tracker.sqlite") + }() + + catchpointAccessor := MakeCatchpointCatchupAccessor(l, log) + catchpointAccessor.ResetStagingBalances(context.Background(), true) + + accountsCount := uint64(b.N) + fileHeader := CatchpointFileHeader{ + Version: catchpointFileVersion, + BalancesRound: basics.Round(0), + BlocksRound: basics.Round(0), + Totals: AccountTotals{}, + TotalAccounts: accountsCount, + TotalChunks: (accountsCount + BalancesPerCatchpointFileChunk - 1) / BalancesPerCatchpointFileChunk, + Catchpoint: "", + BlockHeaderDigest: crypto.Digest{}, + } + encodedFileHeader := protocol.Encode(&fileHeader) + var progress CatchpointCatchupAccessorProgress + err = catchpointAccessor.ProgressStagingBalances(context.Background(), "content.msgpack", encodedFileHeader, &progress) + require.NoError(b, err) + + b.ResetTimer() + accounts := uint64(0) + var last64KStart time.Time + for accounts < accountsCount { + // generate a chunk; + chunkSize := accountsCount - accounts + if chunkSize > BalancesPerCatchpointFileChunk { + chunkSize = BalancesPerCatchpointFileChunk + } + if accounts >= accountsCount-64*1024 && last64KStart.IsZero() { + last64KStart = time.Now() + } + var balances catchpointFileBalancesChunk + balances.Balances = make([]encodedBalanceRecord, chunkSize) + for i := uint64(0); i < chunkSize; i++ { + var randomAccount encodedBalanceRecord + accountData := basics.AccountData{} + accountData.MicroAlgos.Raw = crypto.RandUint63() + randomAccount.AccountData = protocol.Encode(&accountData) + crypto.RandBytes(randomAccount.Address[:]) + balances.Balances[i] = randomAccount + } + err = catchpointAccessor.ProgressStagingBalances(context.Background(), "balances.XX.msgpack", protocol.Encode(&balances), &progress) + require.NoError(b, err) + accounts += chunkSize + } + if !last64KStart.IsZero() { + last64KDuration := time.Now().Sub(last64KStart) + b.Logf("Last 64K\t%v\n", last64KDuration) + } +} + +func BenchmarkRestoringFromCatchpointFile(b *testing.B) { + benchSizes := []int{1024 * 100, 1024 * 200, 1024 * 400} + for _, size := range benchSizes { + b.Run(fmt.Sprintf("Restore-%d", size), func(b *testing.B) { + b.N = size + benchmarkRestoringFromCatchpointFileHelper(b) + }) + } +} diff --git a/ledger/ledger_test.go b/ledger/ledger_test.go index 1eb3070cd1..2772f5f151 100644 --- a/ledger/ledger_test.go +++ b/ledger/ledger_test.go @@ -61,7 +61,7 @@ func sign(secrets map[basics.Address]*crypto.SignatureSecrets, t transactions.Tr } } -func testGenerateInitState(t *testing.T, proto protocol.ConsensusVersion) (genesisInitState InitState, initKeys map[basics.Address]*crypto.SignatureSecrets) { +func testGenerateInitState(tb testing.TB, proto protocol.ConsensusVersion) (genesisInitState InitState, initKeys map[basics.Address]*crypto.SignatureSecrets) { params := config.Consensus[proto] poolAddr := testPoolAddr sinkAddr := testSinkAddr @@ -95,7 +95,7 @@ func testGenerateInitState(t *testing.T, proto protocol.ConsensusVersion) (genes initBlock := bookkeeping.Block{ BlockHeader: bookkeeping.BlockHeader{ - GenesisID: t.Name(), + GenesisID: tb.Name(), Round: 0, RewardsState: bookkeeping.RewardsState{ RewardsRate: initialRewardsPerRound, @@ -109,12 +109,12 @@ func testGenerateInitState(t *testing.T, proto protocol.ConsensusVersion) (genes }, } if params.SupportGenesisHash { - initBlock.BlockHeader.GenesisHash = crypto.Hash([]byte(t.Name())) + initBlock.BlockHeader.GenesisHash = crypto.Hash([]byte(tb.Name())) } genesisInitState.Block = initBlock genesisInitState.Accounts = initAccounts - genesisInitState.GenesisHash = crypto.Hash([]byte(t.Name())) + genesisInitState.GenesisHash = crypto.Hash([]byte(tb.Name())) return } From 17bfc380f2e00eff643e6b396a3c2234c22b4648 Mon Sep 17 00:00:00 2001 From: Rotem Hemo Date: Mon, 7 Sep 2020 17:33:42 +0300 Subject: [PATCH 021/136] -- -- typo --- daemon/algod/api/algod.oas2.json | 2 +- daemon/algod/api/algod.oas3.yml | 2 +- go.mod | 14 ++++++------ go.sum | 37 ++++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/daemon/algod/api/algod.oas2.json b/daemon/algod/api/algod.oas2.json index 29432eea52..ea226dc89d 100644 --- a/daemon/algod/api/algod.oas2.json +++ b/daemon/algod/api/algod.oas2.json @@ -98,7 +98,7 @@ }, "/versions": { "get": { - "description": "Retrieves the supported API version, binary build versions, and genesis information.", + "description": "Retrieves the supported API versions, binary build versions, and genesis information.", "tags": [ "common" ], diff --git a/daemon/algod/api/algod.oas3.yml b/daemon/algod/api/algod.oas3.yml index fb6e66a5c0..b218d4e1d1 100644 --- a/daemon/algod/api/algod.oas3.yml +++ b/daemon/algod/api/algod.oas3.yml @@ -3253,7 +3253,7 @@ }, "/versions": { "get": { - "description": "Retrieves the supported API version, binary build versions, and genesis information.", + "description": "Retrieves the supported API versions, binary build versions, and genesis information.", "operationId": "GetVersion", "responses": { "200": { diff --git a/go.mod b/go.mod index 64f8dd5c61..4bb9aedb2c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,8 @@ require ( github.com/fatih/color v1.7.0 github.com/fortytw2/leaktest v1.3.0 // indirect github.com/gen2brain/beeep v0.0.0-20180718162406-4e430518395f - github.com/getkin/kin-openapi v0.14.0 + github.com/getkin/kin-openapi v0.22.0 + github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f // indirect github.com/gofrs/flock v0.7.0 github.com/google/go-querystring v1.0.0 @@ -28,8 +29,9 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmoiron/sqlx v1.2.0 github.com/karalabe/hid v0.0.0-20181128192157-d815e0c1a2e2 - github.com/labstack/echo/v4 v4.1.16 + github.com/labstack/echo/v4 v4.1.17 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 // indirect + github.com/matryer/moq v0.1.3 // indirect github.com/mattn/go-sqlite3 v1.10.0 github.com/miekg/dns v1.1.27 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect @@ -43,11 +45,11 @@ require ( github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect github.com/stretchr/testify v1.6.1 - golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 - golang.org/x/net v0.0.0-20200602114024-627f9648deb9 - golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a // indirect - golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 + golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a + golang.org/x/net v0.0.0-20200904194848-62affa334b73 + golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f golang.org/x/text v0.3.3 // indirect + golang.org/x/tools v0.0.0-20200904185747-39188db58858 // indirect google.golang.org/appengine v1.6.1 // indirect gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect diff --git a/go.sum b/go.sum index 90b932a0ee..6ffd738e6d 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,7 @@ github.com/aws/aws-sdk-go v1.16.5 h1:NVxzZXIuwX828VcJrpNxxWjur1tlOBISdMdDdHIKHcc github.com/aws/aws-sdk-go v1.16.5/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/cpuguy83/go-md2man v1.0.8 h1:DwoNytLphI8hzS2Af4D0dfaEaiSq2bN05mEm4R6vf8M= github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY= +github.com/cyberdelia/templates v0.0.0-20191230040416-20a325f050d4 h1:Fphwr1XDjkTR/KFbrrkLfY6D2CEOlHqFGomQQrxcHFs= github.com/cyberdelia/templates v0.0.0-20191230040416-20a325f050d4/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -35,9 +36,14 @@ github.com/gen2brain/beeep v0.0.0-20180718162406-4e430518395f/go.mod h1:GprdPCZg github.com/getkin/kin-openapi v0.3.1/go.mod h1:W8dhxZgpE84ciM+VIItFqkmZ4eHtuomrdIHtASQIqi0= github.com/getkin/kin-openapi v0.14.0 h1:hqwQL7kze/adt0wB+0UJR2nJm+gfUHqM0Gu4D8nByVc= github.com/getkin/kin-openapi v0.14.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= +github.com/getkin/kin-openapi v0.22.0 h1:J5IFyKd/5yuB6AZAgwK0CMBKnabWcmkowtsl6bRkz4s= +github.com/getkin/kin-openapi v0.22.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-chi/chi v4.1.1+incompatible h1:MmTgB0R8Bt/jccxp+t6S/1VGIKdJw5J74CK/c9tTfA4= github.com/go-chi/chi v4.1.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= +github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f h1:zlOR3rOlPAVvtfuxGKoghCmop5B0TRyu/ZieziZuGiM= @@ -79,16 +85,23 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o= github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= +github.com/labstack/echo/v4 v4.1.17 h1:PQIBaRplyRy3OjwILGkPg89JRtH2x5bssi59G2EL3fo= +github.com/labstack/echo/v4 v4.1.17/go.mod h1:Tn2yRQL/UclUalpb5rPdXDevbkJ+lp/2svdyFBg6CHQ= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matryer/moq v0.0.0-20200310130814-7721994d1b54 h1:p8zN0Xu28xyEkPpqLbFXAnjdgBVvTJCpfOtoDf/+/RQ= github.com/matryer/moq v0.0.0-20200310130814-7721994d1b54/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= +github.com/matryer/moq v0.1.3 h1:+fW3u2jmlPw59a3V6spZKOLCcvrDKzPjMsRvUhnZ/c0= +github.com/matryer/moq v0.1.3/go.mod h1:9RtPYjTnH1bSBIkpvtHkFN7nbWAnO7oRpdJkEIn6UtE= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -140,7 +153,11 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4= github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -148,8 +165,14 @@ golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -160,11 +183,16 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -178,6 +206,9 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -188,11 +219,17 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200423205358-59e73619c742 h1:9OGWpORUXvk8AsaBJlpzzDx7Srv/rSK6rvjcsJq4rJo= golang.org/x/tools v0.0.0-20200423205358-59e73619c742/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200815165600-90abf76919f3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858 h1:xLt+iB5ksWcZVxqc+g9K41ZHy+6MKWfXCDsjSThnsPA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= From 7f01897a2bef7a261de43d5acc13dea7fb86f3c3 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Tue, 8 Sep 2020 13:44:55 -0400 Subject: [PATCH 022/136] Optimize Merkle trie memory consumption (#1481) Compact merkle trie nodes memory. Changing the underlying data structure is expected to reduce the practical memory utilization during a fast catchup by 50-60 percent. --- crypto/merkletrie/bitset.go | 44 +++++ crypto/merkletrie/bitset_test.go | 71 +++++++ crypto/merkletrie/cache.go | 28 +++ crypto/merkletrie/cache_test.go | 2 +- crypto/merkletrie/committer_test.go | 13 +- crypto/merkletrie/node.go | 294 +++++++++++++--------------- crypto/merkletrie/node_test.go | 120 ++++++++++++ crypto/merkletrie/trie.go | 5 +- crypto/merkletrie/trie_test.go | 2 +- 9 files changed, 412 insertions(+), 167 deletions(-) create mode 100644 crypto/merkletrie/bitset.go create mode 100644 crypto/merkletrie/bitset_test.go create mode 100644 crypto/merkletrie/node_test.go diff --git a/crypto/merkletrie/bitset.go b/crypto/merkletrie/bitset.go new file mode 100644 index 0000000000..84600f112b --- /dev/null +++ b/crypto/merkletrie/bitset.go @@ -0,0 +1,44 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package merkletrie + +// bitset is used as a 256 bits bitmask storage. The simplistic implementation is designed +// explicitly to reduce memory utilization to the minimum required. +type bitset struct { + d [4]uint64 +} + +// SetBit sets the given bit in the bitset. +func (b *bitset) SetBit(bit byte) { + b.d[bit/64] |= 1 << (bit & 63) +} + +// ClearBit clears the given bit in the bitset. +func (b *bitset) ClearBit(bit byte) { + // the &^ is the go and-not operator + b.d[bit/64] &^= 1 << (bit & 63) +} + +// Bit tests the given bit in the bitset. +func (b *bitset) Bit(bit byte) bool { + return (b.d[bit/64] & (1 << (bit & 63))) != 0 +} + +// IsZero tests to see if all the bits in the bitset are set to zero. +func (b *bitset) IsZero() bool { + return b.d[0] == 0 && b.d[1] == 0 && b.d[2] == 0 && b.d[3] == 0 +} diff --git a/crypto/merkletrie/bitset_test.go b/crypto/merkletrie/bitset_test.go new file mode 100644 index 0000000000..cc96bccadb --- /dev/null +++ b/crypto/merkletrie/bitset_test.go @@ -0,0 +1,71 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package merkletrie + +import ( + "math/bits" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBitSet(t *testing.T) { + var a, b bitset + + // set the bits in a different order, and see if we're ending with the same set. + for i := 0; i <= 255; i += 2 { + a.SetBit(byte(i)) + b.SetBit(byte(256 - i)) + } + require.Equal(t, a, b) + + for i := 0; i <= 255; i += 2 { + require.NotZero(t, a.Bit(byte(i))) + } + + for i := 1; i <= 255; i += 2 { + require.Zero(t, a.Bit(byte(i))) + } + + // clear the bits at different order, and testing that the bits were cleared correctly. + for i := 0; i <= 255; i += 32 { + a.ClearBit(byte(i)) + b.ClearBit(byte(256 - i)) + } + require.Equal(t, a, b) + + for i := 0; i <= 255; i += 32 { + require.Zero(t, a.Bit(byte(i))) + } + + // clear all bits ( some would get cleared more than once ) + for i := 0; i <= 255; i += 2 { + a.ClearBit(byte(i)) + } + + // check that the bitset is zero. + require.True(t, a.IsZero()) +} + +// TestBitSetOneBit test that only one bit is being set when we call SetBit +func TestBitSetOneBit(t *testing.T) { + for i := 0; i < 256; i++ { + var a bitset + a.SetBit(byte(i)) + require.Equal(t, 1, bits.OnesCount64(a.d[0])+bits.OnesCount64(a.d[1])+bits.OnesCount64(a.d[2])+bits.OnesCount64(a.d[3])) + } +} diff --git a/crypto/merkletrie/cache.go b/crypto/merkletrie/cache.go index 2950a4cad8..2547a21e72 100644 --- a/crypto/merkletrie/cache.go +++ b/crypto/merkletrie/cache.go @@ -118,6 +118,34 @@ func (mtc *merkleTrieCache) allocateNewNode() (pnode *node, nid storedNodeIdenti return newNode, nextID } +// refurbishNode releases a given node and reallocate a new node while avoiding changing the underlaying buffer. +func (mtc *merkleTrieCache) refurbishNode(nid storedNodeIdentifier) (nextID storedNodeIdentifier) { + page := uint64(nid) / uint64(mtc.nodesPerPage) + pNode := mtc.pageToNIDsPtr[page][nid] + if mtc.txCreatedNodeIDs[nid] { + delete(mtc.txCreatedNodeIDs, nid) + delete(mtc.pageToNIDsPtr[page], nid) + if len(mtc.pageToNIDsPtr[page]) == 0 { + delete(mtc.pageToNIDsPtr, page) + } + mtc.cachedNodeCount-- + } else { + mtc.txDeletedNodeIDs[nid] = true + } + + nextID = mtc.mt.nextNodeID + mtc.mt.nextNodeID++ + page = uint64(nextID) / uint64(mtc.nodesPerPage) + if mtc.pageToNIDsPtr[page] == nil { + mtc.pageToNIDsPtr[page] = make(map[storedNodeIdentifier]*node, mtc.nodesPerPage) + } + mtc.pageToNIDsPtr[page][nextID] = pNode + mtc.cachedNodeCount++ + mtc.txCreatedNodeIDs[nextID] = true + mtc.modified = true + return nextID +} + // getNode retrieves the given node by its identifier, loading the page if it // cannot be found in cache, and returning an error if it's not in cache nor in committer. func (mtc *merkleTrieCache) getNode(nid storedNodeIdentifier) (pnode *node, err error) { diff --git a/crypto/merkletrie/cache_test.go b/crypto/merkletrie/cache_test.go index 889f1759f5..710ff81328 100644 --- a/crypto/merkletrie/cache_test.go +++ b/crypto/merkletrie/cache_test.go @@ -286,7 +286,7 @@ func (mt *Trie) TestDeleteRollback(d []byte) (bool, error) { return false, err } mt.cache.beginTransaction() - if pnode.leaf { + if pnode.leaf() { // remove the root. mt.cache.deleteNode(mt.root) mt.root = storedNodeIdentifierNull diff --git a/crypto/merkletrie/committer_test.go b/crypto/merkletrie/committer_test.go index 87cbb93260..8ff7cd6ea9 100644 --- a/crypto/merkletrie/committer_test.go +++ b/crypto/merkletrie/committer_test.go @@ -83,17 +83,14 @@ func TestInMemoryCommitter(t *testing.T) { } func (n *node) getChildren() (list []storedNodeIdentifier) { - if n.leaf { + if n.leaf() { return []storedNodeIdentifier{} } - i := n.firstChild - for { - list = append(list, n.children[i]) - if i == n.childrenNext[i] { - return - } - i = n.childrenNext[i] + for _, child := range n.children { + list = append(list, child.id) + } + return } func TestNoRedundentPages(t *testing.T) { diff --git a/crypto/merkletrie/node.go b/crypto/merkletrie/node.go index 0c96821a87..b1a1a2300e 100644 --- a/crypto/merkletrie/node.go +++ b/crypto/merkletrie/node.go @@ -19,32 +19,46 @@ package merkletrie import ( "bytes" "encoding/binary" + "sort" + "unsafe" "github.com/algorand/go-algorand/crypto" ) +type childEntry struct { + id storedNodeIdentifier + hashIndex byte +} type node struct { hash []byte - children []storedNodeIdentifier - childrenNext []byte - firstChild byte - leaf bool + children []childEntry + childrenMask bitset +} + +// leaf returns whether the current node is a leaf node, or a non-leaf node +func (n *node) leaf() bool { + return len(n.children) == 0 } +// these sizing constants are being used exclusively in node.stats() +var sliceSize int = int(unsafe.Sizeof([]byte{})) +var bitsetSize int = int(unsafe.Sizeof(bitset{})) +var childEntrySize int = int(unsafe.Sizeof(childEntry{})) + +// stats recursively update the provided Stats structure with the current node information func (n *node) stats(cache *merkleTrieCache, stats *Stats, depth int) (err error) { stats.nodesCount++ - if n.leaf { + if n.leaf() { stats.leafCount++ if depth > stats.depth { stats.depth = depth } - stats.size += 4 + len(n.hash) + 1 + stats.size += sliceSize + len(n.hash) + bitsetSize return nil } - i := n.firstChild - stats.size += 4 + len(n.hash) + cap(n.children)*8 + cap(n.childrenNext) + 1 - for { - childNode, err := cache.getNode(n.children[i]) + stats.size += sliceSize + len(n.hash) + sliceSize + len(n.children)*childEntrySize + bitsetSize + for _, child := range n.children { + childNode, err := cache.getNode(child.id) if err != nil { return err } @@ -52,23 +66,27 @@ func (n *node) stats(cache *merkleTrieCache, stats *Stats, depth int) (err error if err != nil { return err } - if i == n.childrenNext[i] { - break - } - i = n.childrenNext[i] } return nil } +// indexOf returns the index into the children array of the first child whose hashIndex field is less or equal to b +// it's being used in conjuction with the bitset, so we test only the equality path ( i.e. get the index of the +// child that has hashIndex of value x ) +func (n *node) indexOf(b byte) byte { + // find the child using binary search: + return byte(sort.Search(len(n.children), func(i int) bool { return n.children[i].hashIndex >= b })) +} + // find searches the trie for the element, recursively. func (n *node) find(cache *merkleTrieCache, d []byte) (bool, error) { - if n.leaf { + if n.leaf() { return 0 == bytes.Compare(d, n.hash), nil } - childNodeID := n.children[d[0]] - if childNodeID == storedNodeIdentifierNull { + if n.childrenMask.Bit(d[0]) == false { return false, nil } + childNodeID := n.children[n.indexOf(d[0])].id childNode, err := cache.getNode(childNodeID) if err != nil { return false, err @@ -81,7 +99,7 @@ func (n *node) find(cache *merkleTrieCache, d []byte) (bool, error) { func (n *node) add(cache *merkleTrieCache, d []byte, path []byte) (nodeID storedNodeIdentifier, err error) { // allocate a new node to replace the current one. var pnode *node - if n.leaf { + if n.leaf() { // find the diff index: idiff := 0 for ; n.hash[idiff] == d[idiff]; idiff++ { @@ -90,38 +108,48 @@ func (n *node) add(cache *merkleTrieCache, d []byte, path []byte) (nodeID stored curChildNode, curChildNodeID := cache.allocateNewNode() newChildNode, newChildNodeID := cache.allocateNewNode() - curChildNode.leaf = true curChildNode.hash = n.hash[idiff+1:] - newChildNode.leaf = true newChildNode.hash = d[idiff+1:] pnode, nodeID = cache.allocateNewNode() - pnode.leaf = false - pnode.children = make([]storedNodeIdentifier, 256) - pnode.childrenNext = make([]byte, 256) + pnode.childrenMask.SetBit(n.hash[idiff]) + pnode.childrenMask.SetBit(d[idiff]) - pnode.children[n.hash[idiff]] = curChildNodeID - pnode.children[d[idiff]] = newChildNodeID if n.hash[idiff] < d[idiff] { - pnode.firstChild = n.hash[idiff] - pnode.childrenNext[pnode.firstChild] = d[idiff] - pnode.childrenNext[d[idiff]] = d[idiff] + pnode.children = []childEntry{ + childEntry{ + id: curChildNodeID, + hashIndex: n.hash[idiff], + }, + childEntry{ + id: newChildNodeID, + hashIndex: d[idiff], + }, + } } else { - pnode.firstChild = d[idiff] - pnode.childrenNext[pnode.firstChild] = n.hash[idiff] - pnode.childrenNext[n.hash[idiff]] = n.hash[idiff] + pnode.children = []childEntry{ + childEntry{ + id: newChildNodeID, + hashIndex: d[idiff], + }, + childEntry{ + id: curChildNodeID, + hashIndex: n.hash[idiff], + }, + } } pnode.hash = append(path, d[:idiff]...) for i := idiff - 1; i >= 0; i-- { // create a parent node for pnode. pnode2, nodeID2 := cache.allocateNewNode() - pnode2.leaf = false - pnode2.children = make([]storedNodeIdentifier, 256) - pnode2.childrenNext = make([]byte, 256) - pnode2.children[d[i]] = nodeID - pnode2.firstChild = d[i] - pnode2.childrenNext[d[i]] = d[i] + pnode2.childrenMask.SetBit(d[i]) + pnode2.children = []childEntry{ + childEntry{ + id: nodeID, + hashIndex: d[i], + }, + } pnode2.hash = append(path, d[:i]...) pnode = pnode2 @@ -130,41 +158,48 @@ func (n *node) add(cache *merkleTrieCache, d []byte, path []byte) (nodeID stored return nodeID, nil } - if n.children[d[0]] == storedNodeIdentifierNull { + if n.childrenMask.Bit(d[0]) == false { // no such child. var childNode *node var childNodeID storedNodeIdentifier childNode, childNodeID = cache.allocateNewNode() - childNode.leaf = true childNode.hash = d[1:] - pnode, nodeID = n.duplicate(cache) - pnode.children[d[0]] = childNodeID + pnode, nodeID = cache.allocateNewNode() + pnode.childrenMask = n.childrenMask + pnode.childrenMask.SetBit(d[0]) - if pnode.firstChild > d[0] { - pnode.childrenNext[d[0]] = pnode.firstChild - pnode.firstChild = d[0] + pnode.children = make([]childEntry, len(n.children)+1, len(n.children)+1) + if d[0] > n.children[len(n.children)-1].hashIndex { + // the new entry comes after all the existing ones. + for i, child := range n.children { + pnode.children[i] = child + } + pnode.children[len(pnode.children)-1] = childEntry{ + id: childNodeID, + hashIndex: d[0], + } } else { - // iterate on all the entries. - i := pnode.firstChild - for { - if pnode.childrenNext[i] < d[0] { - if pnode.childrenNext[i] == i { - pnode.childrenNext[i] = d[0] - pnode.childrenNext[d[0]] = d[0] - break + for i, child := range n.children { + if d[0] < child.hashIndex { + pnode.children[i] = childEntry{ + hashIndex: d[0], + id: childNodeID, + } + // copy the rest of the items. + for ; i < len(n.children); i++ { + pnode.children[i+1] = n.children[i] } - } else { - pnode.childrenNext[d[0]] = pnode.childrenNext[i] - pnode.childrenNext[i] = d[0] break } - i = pnode.childrenNext[i] + pnode.children[i] = child } } } else { // there is already a child there. - childNode, err := cache.getNode(n.children[d[0]]) + curNodeIndex := n.indexOf(d[0]) + curNodeID := n.children[curNodeIndex].id + childNode, err := cache.getNode(curNodeID) if err != nil { return storedNodeIdentifierNull, err } @@ -172,9 +207,12 @@ func (n *node) add(cache *merkleTrieCache, d []byte, path []byte) (nodeID stored if err != nil { return storedNodeIdentifierNull, err } - pnode, nodeID = n.duplicate(cache) - cache.deleteNode(n.children[d[0]]) - pnode.children[d[0]] = updatedChild + + pnode, nodeID = childNode, cache.refurbishNode(curNodeID) + pnode.childrenMask = n.childrenMask + pnode.children = make([]childEntry, len(n.children), len(n.children)) + copy(pnode.children, n.children) + pnode.children[curNodeIndex].id = updatedChild } pnode.hash = path return nodeID, nil @@ -186,31 +224,26 @@ func (n *node) add(cache *merkleTrieCache, d []byte, path []byte) (nodeID stored // 1. all node id allocations are done in incremental monolitic order, from the bottom up. // 2. hash calculations are being doing in node id incremental ordering func (n *node) calculateHash(cache *merkleTrieCache) error { - if n.leaf { + if n.leaf() { return nil } path := n.hash hashAccumulator := make([]byte, 0, 64*256) // we can have up to 256 elements, so preallocate sufficient storage; append would expand the storage if it won't be enough. hashAccumulator = append(hashAccumulator, byte(len(path))) // we add this string length before the actual string so it could get "decoded"; in practice, it makes a good domain separator. hashAccumulator = append(hashAccumulator, path...) - i := n.firstChild - for { - childNode, err := cache.getNode(n.children[i]) + for _, child := range n.children { + childNode, err := cache.getNode(child.id) if err != nil { return err } - if childNode.leaf { + if childNode.leaf() { hashAccumulator = append(hashAccumulator, byte(0)) } else { hashAccumulator = append(hashAccumulator, byte(1)) } hashAccumulator = append(hashAccumulator, byte(len(childNode.hash))) // we add this string length before the actual string so it could get "decoded"; in practice, it makes a good domain separator. - hashAccumulator = append(hashAccumulator, i) // adding the first byte of the child + hashAccumulator = append(hashAccumulator, child.hashIndex) // adding the first byte of the child hashAccumulator = append(hashAccumulator, childNode.hash...) // adding the reminder of the child - if n.childrenNext[i] == i { - break - } - i = n.childrenNext[i] } hash := crypto.Hash(hashAccumulator) n.hash = hash[:] @@ -223,99 +256,59 @@ func (n *node) calculateHash(cache *merkleTrieCache) error { func (n *node) remove(cache *merkleTrieCache, key []byte, path []byte) (nodeID storedNodeIdentifier, err error) { // allocate a new node to replace the current one. var pnode, childNode *node - childNodeID := n.children[key[0]] + childIndex := n.indexOf(key[0]) + childNodeID := n.children[childIndex].id childNode, err = cache.getNode(childNodeID) if err != nil { return } - if childNode.leaf { - pnode, nodeID = n.duplicate(cache) + if childNode.leaf() { + pnode, nodeID = childNode, cache.refurbishNode(childNodeID) + pnode.childrenMask = n.childrenMask // we are guaranteed to have other children, because our tree forbids nodes that have exactly one leaf child and no other children. - // we have one or more children, see if it's the first child: - if pnode.firstChild == key[0] { - // we're removing the first child. - next := pnode.childrenNext[pnode.firstChild] - pnode.children[pnode.firstChild] = storedNodeIdentifierNull - pnode.childrenNext[pnode.firstChild] = 0 - pnode.firstChild = next - } else { - // wer're removing a child off the list ( known to be there, and not to be the first ) - i := pnode.firstChild - for { - if pnode.childrenNext[i] == key[0] { - // is this the last item ? - if pnode.childrenNext[key[0]] == key[0] { - // yes, it's the last item. - pnode.childrenNext[i] = i - // clear out pointers. - pnode.childrenNext[key[0]] = 0 - pnode.children[key[0]] = storedNodeIdentifierNull - break - } - pnode.childrenNext[i] = pnode.childrenNext[key[0]] - // clear out pointers - pnode.childrenNext[key[0]] = 0 - pnode.children[key[0]] = storedNodeIdentifierNull - break - } - i = pnode.childrenNext[i] - } - } + pnode.children = make([]childEntry, len(n.children)-1, len(n.children)-1) + copy(pnode.children, append(n.children[:childIndex], n.children[childIndex+1:]...)) + pnode.childrenMask.ClearBit(key[0]) } else { var updatedChildNodeID storedNodeIdentifier updatedChildNodeID, err = childNode.remove(cache, key[1:], append(path, key[0])) if err != nil { return storedNodeIdentifierNull, err } - pnode, nodeID = n.duplicate(cache) - pnode.children[key[0]] = updatedChildNodeID + + pnode, nodeID = childNode, cache.refurbishNode(childNodeID) + pnode.childrenMask = n.childrenMask + pnode.children = make([]childEntry, len(n.children), len(n.children)) + copy(pnode.children, n.children) + pnode.children[childIndex].id = updatedChildNodeID } - cache.deleteNode(childNodeID) // at this point, we might end up with a single leaf child. collapse that. - if pnode.childrenNext[pnode.firstChild] == pnode.firstChild { - childNode, err = cache.getNode(pnode.children[pnode.firstChild]) + if len(pnode.children) == 1 { + childNode, err = cache.getNode(pnode.children[0].id) if err != nil { return } - if childNode.leaf { + if childNode.leaf() { // convert current node into a leaf. - pnode.leaf = true - pnode.hash = append([]byte{pnode.firstChild}, childNode.hash...) - cache.deleteNode(pnode.children[pnode.firstChild]) + pnode.hash = append([]byte{pnode.children[0].hashIndex}, childNode.hash...) + cache.deleteNode(pnode.children[0].id) + pnode.childrenMask.ClearBit(pnode.children[0].hashIndex) pnode.children = nil - pnode.childrenNext = nil - pnode.firstChild = 0 } } - if !pnode.leaf { + if !pnode.leaf() { pnode.hash = path } return nodeID, nil } -// duplicate creates a copy of the current node -func (n *node) duplicate(cache *merkleTrieCache) (pnode *node, nodeID storedNodeIdentifier) { - pnode, nodeID = cache.allocateNewNode() - pnode.firstChild = n.firstChild - pnode.hash = n.hash // the hash is safe for just copy without duplicate, since it's always being reallocated upon change. - pnode.leaf = n.leaf - if !pnode.leaf { - pnode.children = make([]storedNodeIdentifier, 256) - pnode.childrenNext = make([]byte, 256) - // copy the elements starting the first known entry. - copy(pnode.children[n.firstChild:], n.children[n.firstChild:]) - copy(pnode.childrenNext[n.firstChild:], n.childrenNext[n.firstChild:]) - } - return -} - // serialize the content of the node into the buffer, and return the number of bytes consumed in the process. func (n *node) serialize(buf []byte) int { w := binary.PutUvarint(buf[:], uint64(len(n.hash))) copy(buf[w:], n.hash) w += len(n.hash) - if n.leaf { + if n.leaf() { buf[w] = 0 // leaf return w + 1 } @@ -323,18 +316,13 @@ func (n *node) serialize(buf []byte) int { buf[w] = 1 // non-leaf w++ // store all the children, and terminate with a null. - i := n.firstChild - for { - buf[w] = i + for _, child := range n.children { + buf[w] = child.hashIndex w++ - x := binary.PutUvarint(buf[w:], uint64(n.children[i])) + x := binary.PutUvarint(buf[w:], uint64(child.id)) w += x - if i == n.childrenNext[i] { - break - } - i = n.childrenNext[i] } - buf[w] = i + buf[w] = n.children[len(n.children)-1].hashIndex w++ return w } @@ -349,36 +337,34 @@ func deserializeNode(buf []byte) (n *node, s int) { n.hash = make([]byte, hashLength) copy(n.hash, buf[hashLength2:hashLength2+int(hashLength)]) s = hashLength2 + int(hashLength) - n.leaf = (buf[s] == 0) + isLeaf := (buf[s] == 0) s++ - if n.leaf { + if isLeaf { return } - n.children = make([]storedNodeIdentifier, 256) - n.childrenNext = make([]byte, 256) + var childEntries [256]childEntry first := true prevChildIndex := byte(0) - + i := 0 for { childIndex := buf[s] s++ if childIndex <= prevChildIndex && !first { break } - if first { - first = false - n.firstChild = childIndex - } else { - n.childrenNext[prevChildIndex] = childIndex - } + first = false nodeID, nodeIDLength := binary.Uvarint(buf[s:]) if nodeIDLength <= 0 { return nil, nodeIDLength } s += nodeIDLength - n.children[childIndex] = storedNodeIdentifier(nodeID) + + childEntries[i] = childEntry{hashIndex: childIndex, id: storedNodeIdentifier(nodeID)} + n.childrenMask.SetBit(childIndex) prevChildIndex = childIndex + i++ } - n.childrenNext[prevChildIndex] = prevChildIndex + n.children = make([]childEntry, i, i) + copy(n.children[:], childEntries[:i]) return } diff --git a/crypto/merkletrie/node_test.go b/crypto/merkletrie/node_test.go new file mode 100644 index 0000000000..1fbd02ab0e --- /dev/null +++ b/crypto/merkletrie/node_test.go @@ -0,0 +1,120 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package merkletrie + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/crypto" +) + +// TestNodeSerialization tests the serialization and deserialization of nodes. +func TestNodeSerialization(t *testing.T) { + var memoryCommitter InMemoryCommitter + mt1, _ := MakeTrie(&memoryCommitter, 1000) + // create 1024 hashes. + leafsCount := 1024 + hashes := make([]crypto.Digest, leafsCount) + for i := 0; i < len(hashes); i++ { + hashes[i] = crypto.Hash([]byte{byte(i % 256), byte((i / 256) % 256), byte(i / 65536)}) + } + + for i := 0; i < len(hashes); i++ { + mt1.Add(hashes[i][:]) + } + for _, page := range mt1.cache.pageToNIDsPtr { + for _, pnode := range page { + buf := make([]byte, 10000) + consumedWrite := pnode.serialize(buf[:]) + outNode, consumedRead := deserializeNode(buf[:]) + require.Equal(t, consumedWrite, consumedRead) + require.Equal(t, pnode.leaf(), outNode.leaf()) + require.Equal(t, len(pnode.children), len(outNode.children)) + reencodedBuffer := make([]byte, 10000) + renecodedConsumedWrite := outNode.serialize(reencodedBuffer[:]) + require.Equal(t, consumedWrite, renecodedConsumedWrite) + require.Equal(t, buf, reencodedBuffer) + } + } +} + +func (n *node) leafUsingChildrenMask() bool { + return n.childrenMask.IsZero() +} + +func (n *node) leafUsingChildrenLength() bool { + return len(n.children) == 0 +} + +func BenchmarkNodeLeafImplementation(b *testing.B) { + b.Run("leaf-ChildrenMask", func(b *testing.B) { + var memoryCommitter InMemoryCommitter + mt1, _ := MakeTrie(&memoryCommitter, defaultTestEvictSize) + // create 100000 hashes. + leafsCount := 100000 + hashes := make([]crypto.Digest, leafsCount) + for i := 0; i < len(hashes); i++ { + hashes[i] = crypto.Hash([]byte{byte(i % 256), byte((i / 256) % 256), byte(i / 65536)}) + } + + for i := 0; i < len(hashes); i++ { + mt1.Add(hashes[i][:]) + } + b.ResetTimer() + k := 0 + for { + for _, pageMap := range mt1.cache.pageToNIDsPtr { + for _, pnode := range pageMap { + pnode.leafUsingChildrenMask() + k++ + if k > b.N { + return + } + } + } + } + }) + b.Run("leaf-ChildrenLength", func(b *testing.B) { + var memoryCommitter InMemoryCommitter + mt1, _ := MakeTrie(&memoryCommitter, defaultTestEvictSize) + // create 100000 hashes. + leafsCount := 100000 + hashes := make([]crypto.Digest, leafsCount) + for i := 0; i < len(hashes); i++ { + hashes[i] = crypto.Hash([]byte{byte(i % 256), byte((i / 256) % 256), byte(i / 65536)}) + } + + for i := 0; i < len(hashes); i++ { + mt1.Add(hashes[i][:]) + } + b.ResetTimer() + k := 0 + for { + for _, pageMap := range mt1.cache.pageToNIDsPtr { + for _, pnode := range pageMap { + pnode.leafUsingChildrenLength() + k++ + if k > b.N { + return + } + } + } + } + }) +} diff --git a/crypto/merkletrie/trie.go b/crypto/merkletrie/trie.go index 05562382dd..b373a99f6b 100644 --- a/crypto/merkletrie/trie.go +++ b/crypto/merkletrie/trie.go @@ -117,7 +117,7 @@ func (mt *Trie) RootHash() (crypto.Digest, error) { return crypto.Digest{}, err } - if pnode.leaf { + if pnode.leaf() { return crypto.Hash(append([]byte{0}, pnode.hash...)), nil } return crypto.Hash(append([]byte{1}, pnode.hash...)), nil @@ -132,7 +132,6 @@ func (mt *Trie) Add(d []byte) (bool, error) { mt.cache.beginTransaction() pnode, mt.root = mt.cache.allocateNewNode() mt.cache.commitTransaction() - pnode.leaf = true pnode.hash = d mt.elementLength = len(d) return true, nil @@ -179,7 +178,7 @@ func (mt *Trie) Delete(d []byte) (bool, error) { return false, err } mt.cache.beginTransaction() - if pnode.leaf { + if pnode.leaf() { // remove the root. mt.cache.deleteNode(mt.root) mt.root = storedNodeIdentifierNull diff --git a/crypto/merkletrie/trie_test.go b/crypto/merkletrie/trie_test.go index 171e23529e..a7146816b2 100644 --- a/crypto/merkletrie/trie_test.go +++ b/crypto/merkletrie/trie_test.go @@ -50,7 +50,7 @@ func TestAddingAndRemoving(t *testing.T) { require.Equal(t, len(hashes), int(stats.leafCount)) require.Equal(t, 4, int(stats.depth)) require.Equal(t, 10915, int(stats.nodesCount)) - require.Equal(t, 2490656, int(stats.size)) + require.Equal(t, 1135745, int(stats.size)) require.True(t, int(stats.nodesCount) > len(hashes)) require.True(t, int(stats.nodesCount) < 2*len(hashes)) From 399078c5eb2d26400ac83acd77d0ceaad1f3fb22 Mon Sep 17 00:00:00 2001 From: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> Date: Tue, 8 Sep 2020 19:38:09 -0400 Subject: [PATCH 023/136] Add unit tests for equal functions in teal.go (#1486) Add unit tests for equal functions in teal.go --- data/basics/teal_test.go | 112 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/data/basics/teal_test.go b/data/basics/teal_test.go index 0de5553f5c..090bae9a41 100644 --- a/data/basics/teal_test.go +++ b/data/basics/teal_test.go @@ -111,3 +111,115 @@ func TestSatisfiesSchema(t *testing.T) { err = tkv.SatisfiesSchema(schema) a.NoError(err) } + +func TestStateDeltaEqual(t *testing.T) { + a := require.New(t) + + var d1 StateDelta = nil + var d2 StateDelta = nil + a.True(d1.Equal(d2)) + + d2 = StateDelta{} + a.True(d1.Equal(d2)) + + d2 = StateDelta{"test": {Action: SetUintAction, Uint: 0}} + a.False(d1.Equal(d2)) + + d1 = StateDelta{} + d2 = StateDelta{} + a.True(d1.Equal(d2)) + + d2 = StateDelta{"test": {Action: SetUintAction, Uint: 0}} + a.False(d1.Equal(d2)) + + d1 = StateDelta{"test2": {Action: SetBytesAction, Uint: 0}} + a.False(d1.Equal(d2)) + + d1 = StateDelta{"test": {Action: SetUintAction, Uint: 0}} + d2 = StateDelta{"test": {Action: SetUintAction, Uint: 0}} + a.True(d1.Equal(d2)) + + d1 = StateDelta{"test": {Action: SetBytesAction, Bytes: "val"}} + d2 = StateDelta{"test": {Action: SetBytesAction, Bytes: "val"}} + a.True(d1.Equal(d2)) + + d2 = StateDelta{"test": {Action: SetBytesAction, Bytes: "val1"}} + a.False(d1.Equal(d2)) +} + +func TestEvalDeltaEqual(t *testing.T) { + a := require.New(t) + + d1 := EvalDelta{} + d2 := EvalDelta{} + a.True(d1.Equal(d2)) + + d2 = EvalDelta{ + GlobalDelta: nil, + LocalDeltas: nil, + } + a.True(d1.Equal(d2)) + + d2 = EvalDelta{ + GlobalDelta: StateDelta{}, + LocalDeltas: map[uint64]StateDelta{}, + } + a.True(d1.Equal(d2)) + + d2 = EvalDelta{ + GlobalDelta: StateDelta{"test": {Action: SetUintAction, Uint: 0}}, + } + a.False(d1.Equal(d2)) + + d1 = EvalDelta{ + GlobalDelta: StateDelta{"test": {Action: SetUintAction, Uint: 0}}, + } + a.True(d1.Equal(d2)) + + d2 = EvalDelta{ + LocalDeltas: map[uint64]StateDelta{ + 0: {"test": {Action: SetUintAction, Uint: 0}}, + }, + } + a.False(d1.Equal(d2)) + + d1 = EvalDelta{ + LocalDeltas: map[uint64]StateDelta{ + 0: {"test": {Action: SetUintAction, Uint: 1}}, + }, + } + a.False(d1.Equal(d2)) + + d2 = EvalDelta{ + LocalDeltas: map[uint64]StateDelta{ + 0: {"test": {Action: SetUintAction, Uint: 1}}, + }, + } + a.True(d1.Equal(d2)) + + d1 = EvalDelta{ + LocalDeltas: map[uint64]StateDelta{ + 0: {"test": {Action: SetBytesAction, Bytes: "val"}}, + }, + } + d2 = EvalDelta{ + LocalDeltas: map[uint64]StateDelta{ + 0: {"test": {Action: SetBytesAction, Bytes: "val"}}, + }, + } + a.True(d1.Equal(d2)) + + d2 = EvalDelta{ + LocalDeltas: map[uint64]StateDelta{ + 0: {"test": {Action: SetBytesAction, Bytes: "val1"}}, + }, + } + a.False(d1.Equal(d2)) + + d2 = EvalDelta{ + LocalDeltas: map[uint64]StateDelta{ + 1: {"test": {Action: SetBytesAction, Bytes: "val"}}, + }, + } + a.False(d1.Equal(d2)) +} From 6bc1c0156dee3b8ede9efbe0feae81c13ef64a6e Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Tue, 8 Sep 2020 21:08:52 -0400 Subject: [PATCH 024/136] core crypto primitives for compact certificates (#1359) Implement core crypto primitives for compact certificates --- Makefile | 2 +- crypto/compactcert/bigfloat.go | 186 ++++ crypto/compactcert/bigfloat_test.go | 162 ++++ crypto/compactcert/builder.go | 246 ++++++ crypto/compactcert/builder_test.go | 239 +++++ crypto/compactcert/common.go | 103 +++ crypto/compactcert/common_test.go | 99 +++ crypto/compactcert/msgp_gen.go | 1271 +++++++++++++++++++++++++++ crypto/compactcert/msgp_gen_test.go | 322 +++++++ crypto/compactcert/structs.go | 115 +++ crypto/compactcert/verifier.go | 96 ++ protocol/hash.go | 3 + 12 files changed, 2843 insertions(+), 1 deletion(-) create mode 100644 crypto/compactcert/bigfloat.go create mode 100644 crypto/compactcert/bigfloat_test.go create mode 100644 crypto/compactcert/builder.go create mode 100644 crypto/compactcert/builder_test.go create mode 100644 crypto/compactcert/common.go create mode 100644 crypto/compactcert/common_test.go create mode 100644 crypto/compactcert/msgp_gen.go create mode 100644 crypto/compactcert/msgp_gen_test.go create mode 100644 crypto/compactcert/structs.go create mode 100644 crypto/compactcert/verifier.go diff --git a/Makefile b/Makefile index 5293d7c97c..a5975b5d39 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \ UNIT_TEST_SOURCES := $(sort $(shell GO111MODULE=off go list ./... | grep -v /go-algorand/test/ )) ALGOD_API_PACKAGES := $(sort $(shell GO111MODULE=off cd daemon/algod/api; go list ./... )) -MSGP_GENERATE := ./protocol ./crypto ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./auction ./agreement ./rpcs ./node ./ledger +MSGP_GENERATE := ./protocol ./crypto ./crypto/compactcert ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./auction ./agreement ./rpcs ./node ./ledger default: build diff --git a/crypto/compactcert/bigfloat.go b/crypto/compactcert/bigfloat.go new file mode 100644 index 0000000000..a11aabf9de --- /dev/null +++ b/crypto/compactcert/bigfloat.go @@ -0,0 +1,186 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package compactcert + +import ( + "fmt" + "math/bits" +) + +// A bigFloat represents the number mantissa*2^exp, which must be non-zero. +// +// A canonical representation is one where the highest bit of mantissa is +// set. Every operation enforces canonicality of results. +// +// We use 32-bit values here to avoid requiring a 64bit-by-64bit-to-128bit +// multiply operation for anyone that needs to implement this (even though +// Go has this operation, as bits.Mul64). +type bigFloat struct { + mantissa uint32 + exp int32 +} + +// Each bigFloat is associated with a rounding mode (up, away from zero, or +// down, towards zero). This is reflected by these two types of bigFloat. +type bigFloatUp struct { + bigFloat +} + +type bigFloatDn struct { + bigFloat +} + +// canonicalize() ensures that the bigFloat is canonical. +func (a *bigFloat) canonicalize() { + if a.mantissa == 0 { + // Just to avoid infinite loops in some error case. + return + } + + for (a.mantissa & (1 << 31)) == 0 { + a.mantissa = a.mantissa << 1 + a.exp = a.exp - 1 + } +} + +// doRoundUp adds one to the mantissa of a canonical bigFloat +// to implement the rounding-up when there are leftover low bits. +func (a *bigFloatUp) doRoundUp() { + if a.mantissa == (1<<32)-1 { + a.mantissa = 1 << 31 + a.exp++ + } else { + a.mantissa++ + } +} + +// geRaw returns whether a>=b. The Raw suffix indicates that +// this comparison does not take rounding into account, and might +// not be true if done with arbitrary-precision numbers. +func (a *bigFloat) geRaw(b *bigFloat) bool { + if a.exp > b.exp { + return true + } + + if a.exp < b.exp { + return false + } + + return a.mantissa >= b.mantissa +} + +// ge returns whether a>=b. It requires that a was computed with +// rounding-down and b was computed with rounding-up, so that if +// ge returns true, the arbitrary-precision computation would have +// also been >=. +func (a *bigFloatDn) ge(b *bigFloatUp) bool { + return a.geRaw(&b.bigFloat) +} + +// setu64Dn sets the value to the supplied uint64 (which might get +// rounded down in the process). x must not be zero. truncated +// returns whether any non-zero bits were truncated (rounded down). +func (a *bigFloat) setu64Dn(x uint64) (truncated bool, err error) { + if x == 0 { + return false, fmt.Errorf("bigFloat cannot be zero") + } + + e := int32(0) + + for x >= (1 << 32) { + if (x & 1) != 0 { + truncated = true + } + + x = x >> 1 + e = e + 1 + } + + a.mantissa = uint32(x) + a.exp = e + a.canonicalize() + return +} + +// setu64 calls setu64Dn and implements rounding based on the type. +func (a *bigFloatUp) setu64(x uint64) error { + truncated, err := a.setu64Dn(x) + if truncated { + a.doRoundUp() + } + return err +} + +func (a *bigFloatDn) setu64(x uint64) error { + _, err := a.setu64Dn(x) + return err +} + +// setu32 sets the value to the supplied uint32. +func (a *bigFloat) setu32(x uint32) error { + if x == 0 { + return fmt.Errorf("bigFloat cannot be zero") + } + + a.mantissa = x + a.exp = 0 + a.canonicalize() + return nil +} + +// setpow2 sets the value to 2^x. +func (a *bigFloat) setpow2(x int32) { + a.mantissa = 1 + a.exp = x + a.canonicalize() +} + +// mulDn sets a to the product a*b, keeping the most significant 32 bits +// of the product's mantissa. The return value indicates if any non-zero +// bits were discarded (rounded down). +func (a *bigFloat) mulDn(b *bigFloat) bool { + hi, lo := bits.Mul32(a.mantissa, b.mantissa) + + a.mantissa = hi + a.exp = a.exp + b.exp + 32 + + if (a.mantissa & (1 << 31)) == 0 { + a.mantissa = (a.mantissa << 1) | (lo >> 31) + a.exp = a.exp - 1 + lo = lo << 1 + } + + return lo != 0 +} + +// mul calls mulDn and implements appropriate rounding. +// Types prevent multiplying two values with different rounding types. +func (a *bigFloatUp) mul(b *bigFloatUp) { + truncated := a.mulDn(&b.bigFloat) + if truncated { + a.doRoundUp() + } +} + +func (a *bigFloatDn) mul(b *bigFloatDn) { + a.mulDn(&b.bigFloat) +} + +// String returns a string representation of a. +func (a *bigFloat) String() string { + return fmt.Sprintf("%d*2^%d", a.mantissa, a.exp) +} diff --git a/crypto/compactcert/bigfloat_test.go b/crypto/compactcert/bigfloat_test.go new file mode 100644 index 0000000000..c7c78e5749 --- /dev/null +++ b/crypto/compactcert/bigfloat_test.go @@ -0,0 +1,162 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package compactcert + +import ( + "math/big" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/crypto" +) + +func rand32() uint32 { + return uint32(crypto.RandUint64() & 0xffffffff) +} + +func TestBigFloatRounding(t *testing.T) { + a := &bigFloatDn{} + b := &bigFloatUp{} + + a.setu64(1 << 63) + b.setu64(1 << 63) + + require.True(t, a.geRaw(&b.bigFloat)) + require.True(t, b.geRaw(&a.bigFloat)) + + a.mul(a) + b.mul(b) + + require.True(t, a.geRaw(&b.bigFloat)) + require.True(t, b.geRaw(&a.bigFloat)) + + a.setu64((1 << 64) - 1) + b.setu64((1 << 64) - 1) + + require.False(t, a.geRaw(&b.bigFloat)) + require.True(t, b.geRaw(&a.bigFloat)) + + a.setu32((1 << 32) - 1) + b.setu32((1 << 32) - 1) + + a.mul(a) + b.mul(b) + + require.False(t, a.geRaw(&b.bigFloat)) + require.True(t, b.geRaw(&a.bigFloat)) +} + +func TestBigFloat(t *testing.T) { + a := &bigFloatDn{} + b := &bigFloatDn{} + + a.setu64(1) + require.Equal(t, a.mantissa, uint32(1<<31)) + require.Equal(t, a.exp, int32(-31)) + + a.setu32(1) + require.Equal(t, a.mantissa, uint32(1<<31)) + require.Equal(t, a.exp, int32(-31)) + + for i := int32(-256); i < 256; i++ { + a.setpow2(i) + require.Equal(t, a.mantissa, uint32(1<<31)) + require.Equal(t, a.exp, i-31) + } + + for i := 0; i < 8192; i++ { + x := rand32() + a.setu32(x) + require.True(t, a.exp <= 0) + require.Equal(t, x, a.mantissa>>(-a.exp)) + } + + for i := 0; i < 8192; i++ { + x := uint64(rand32()) + a.setu64(x) + if a.exp <= 0 { + require.Equal(t, x, uint64(a.mantissa>>(-a.exp))) + } + if a.exp >= 0 { + require.Equal(t, x>>a.exp, uint64(a.mantissa)) + } + } + + for i := 0; i < 8192; i++ { + x := crypto.RandUint64() + a.setu64(x) + if a.exp <= 0 { + require.Equal(t, x, uint64(a.mantissa>>(-a.exp))) + } + if a.exp >= 0 { + require.Equal(t, x>>a.exp, uint64(a.mantissa)) + } + } + + for i := 0; i < 8192; i++ { + x := rand32() + y := rand32() + a.setu64(uint64(x)) + b.setu64(uint64(y)) + + require.Equal(t, x >= y, a.geRaw(&b.bigFloat)) + require.Equal(t, x < y, b.geRaw(&a.bigFloat)) + require.True(t, a.geRaw(&a.bigFloat)) + require.True(t, b.geRaw(&b.bigFloat)) + } + + xx := &big.Int{} + yy := &big.Int{} + + for i := 0; i < 8192; i++ { + x := rand32() + y := rand32() + a.setu64(uint64(x)) + b.setu64(uint64(y)) + a.mul(b) + + xx.SetUint64(uint64(x)) + yy.SetUint64(uint64(y)) + xx.Mul(xx, yy) + if a.exp > 0 { + xx.Rsh(xx, uint(a.exp)) + } + if a.exp < 0 { + xx.Lsh(xx, uint(-a.exp)) + } + require.Equal(t, a.mantissa, uint32(xx.Uint64())) + } +} + +func BenchmarkBigFloatMulUp(b *testing.B) { + a := &bigFloatUp{} + a.setu32((1 << 32) - 1) + + for i := 0; i < b.N; i++ { + a.mul(a) + } +} + +func BenchmarkBigFloatMulDn(b *testing.B) { + a := &bigFloatDn{} + a.setu32((1 << 32) - 1) + + for i := 0; i < b.N; i++ { + a.mul(a) + } +} diff --git a/crypto/compactcert/builder.go b/crypto/compactcert/builder.go new file mode 100644 index 0000000000..a735627f4f --- /dev/null +++ b/crypto/compactcert/builder.go @@ -0,0 +1,246 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package compactcert + +import ( + "fmt" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklearray" + "github.com/algorand/go-algorand/data/basics" +) + +//msgp:ignore sigslot +type sigslot struct { + // Weight is the weight of the participant signing this message. + // This information is tracked here for convenience, but it does + // not appear in the commitment to the sigs array; it comes from + // the Weight field of the corresponding participant. + Weight uint64 + + // Include the parts of the sigslot that form the commitment to + // the sigs array. + sigslotCommit +} + +// Builder keeps track of signatures on a message and eventually produces +// a compact certificate for that message. +type Builder struct { + Params + + sigs []sigslot // Indexed by pos in participants + sigsHasValidL bool // The L values in sigs are consistent with weights + signedWeight uint64 // Total weight of signatures so far + participants []Participant + parttree *merklearray.Tree + + // Cached cert, if Build() was called and no subsequent + // Add() calls were made. + cert *Cert +} + +// MkBuilder constructs an empty builder (with no signatures). The message +// to be signed, as well as other security parameters, are specified in +// param. The participants that will sign the message are in part and +// parttree. +func MkBuilder(param Params, part []Participant, parttree *merklearray.Tree) (*Builder, error) { + npart := len(part) + + b := &Builder{ + Params: param, + sigs: make([]sigslot, npart), + sigsHasValidL: false, + signedWeight: 0, + participants: part, + parttree: parttree, + } + + return b, nil +} + +// Present checks if the builder already contains a signature at a particular +// offset. +func (b *Builder) Present(pos uint64) bool { + return b.sigs[pos].Weight != 0 +} + +// Add a signature to the set of signatures available for building a certificate. +// verifySig should be set to true in production; setting it to false is useful +// for benchmarking to avoid the cost of signature checks. +func (b *Builder) Add(pos uint64, sig crypto.OneTimeSignature, verifySig bool) error { + if b.Present(pos) { + return fmt.Errorf("position %d already added", pos) + } + + // Check participants array + if pos >= uint64(len(b.participants)) { + return fmt.Errorf("pos %d >= len(participants) %d", pos, len(b.participants)) + } + + p := b.participants[pos] + + if p.Weight == 0 { + return fmt.Errorf("position %d has zero weight", pos) + } + + // Check signature + ephID := basics.OneTimeIDForRound(b.SigRound, p.KeyDilution) + if verifySig && !p.PK.Verify(ephID, b.Msg, sig) { + return fmt.Errorf("signature does not verify under ID %v", ephID) + } + + // Remember the signature + b.sigs[pos].Weight = p.Weight + b.sigs[pos].Sig.OneTimeSignature = sig + b.signedWeight += p.Weight + b.cert = nil + b.sigsHasValidL = false + return nil +} + +// Ready returns whether the certificate is ready to be built. +func (b *Builder) Ready() bool { + return b.signedWeight > b.Params.ProvenWeight +} + +// SignedWeight returns the total weight of signatures added so far. +func (b *Builder) SignedWeight() uint64 { + return b.signedWeight +} + +//msgp:ignore sigsToCommit +type sigsToCommit []sigslot + +func (sc sigsToCommit) Length() uint64 { + return uint64(len(sc)) +} + +func (sc sigsToCommit) Get(pos uint64) (crypto.Hashable, error) { + if pos >= uint64(len(sc)) { + return nil, fmt.Errorf("pos %d past end %d", pos, len(sc)) + } + + return &sc[pos].sigslotCommit, nil +} + +// coinIndex returns the position pos in the sigs array such that the sum +// of all signature weights before pos is less than or equal to coinWeight, +// but the sum of all signature weights up to and including pos exceeds +// coinWeight. +// +// coinIndex works by doing a binary search on the sigs array. +func (b *Builder) coinIndex(coinWeight uint64) (uint64, error) { + if !b.sigsHasValidL { + return 0, fmt.Errorf("coinIndex: need valid L values") + } + + lo := uint64(0) + hi := uint64(len(b.sigs)) + +again: + if lo >= hi { + return 0, fmt.Errorf("coinIndex: lo %d >= hi %d", lo, hi) + } + + mid := (lo + hi) / 2 + if coinWeight < b.sigs[mid].L { + hi = mid + goto again + } + + if coinWeight < b.sigs[mid].L+b.sigs[mid].Weight { + return mid, nil + } + + lo = mid + 1 + goto again +} + +// Build returns a compact certificate, if the builder has accumulated +// enough signatures to construct it. +func (b *Builder) Build() (*Cert, error) { + if b.cert != nil { + return b.cert, nil + } + + if b.signedWeight <= b.Params.ProvenWeight { + return nil, fmt.Errorf("not enough signed weight: %d <= %d", b.signedWeight, b.Params.ProvenWeight) + } + + // Commit to the sigs array + for i := 1; i < len(b.sigs); i++ { + b.sigs[i].L = b.sigs[i-1].L + b.sigs[i-1].Weight + } + b.sigsHasValidL = true + + sigtree, err := merklearray.Build(sigsToCommit(b.sigs)) + if err != nil { + return nil, err + } + + // Reveal sufficient number of signatures + c := &Cert{ + SigCommit: sigtree.Root(), + SignedWeight: b.signedWeight, + Reveals: make(map[uint64]Reveal), + } + + nr, err := b.numReveals(b.signedWeight) + if err != nil { + return nil, err + } + + var proofPositions []uint64 + + for j := uint64(0); j < nr; j++ { + coin := hashCoin(j, c.SigCommit, c.SignedWeight) + pos, err := b.coinIndex(coin) + if err != nil { + return nil, err + } + + if pos >= uint64(len(b.participants)) { + return nil, fmt.Errorf("pos %d >= len(participants) %d", pos, len(b.participants)) + } + + // If we already revealed pos, no need to do it again + _, alreadyRevealed := c.Reveals[pos] + if alreadyRevealed { + continue + } + + // Generate the reveal for pos + c.Reveals[pos] = Reveal{ + SigSlot: b.sigs[pos].sigslotCommit, + Part: b.participants[pos], + } + + proofPositions = append(proofPositions, pos) + } + + c.SigProofs, err = sigtree.Prove(proofPositions) + if err != nil { + return nil, err + } + + c.PartProofs, err = b.parttree.Prove(proofPositions) + if err != nil { + return nil, err + } + + return c, nil +} diff --git a/crypto/compactcert/builder_test.go b/crypto/compactcert/builder_test.go new file mode 100644 index 0000000000..82b01de827 --- /dev/null +++ b/crypto/compactcert/builder_test.go @@ -0,0 +1,239 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package compactcert + +import ( + "fmt" + "testing" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklearray" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/protocol" + + "github.com/stretchr/testify/require" +) + +type TestMessage string + +func (m TestMessage) ToBeHashed() (protocol.HashID, []byte) { + return protocol.Message, []byte(m) +} + +type PartCommit struct { + participants []Participant +} + +func (pc PartCommit) Length() uint64 { + return uint64(len(pc.participants)) +} + +func (pc PartCommit) Get(pos uint64) (crypto.Hashable, error) { + if pos >= uint64(len(pc.participants)) { + return nil, fmt.Errorf("pos %d >= len %d", pos, len(pc.participants)) + } + + return pc.participants[pos], nil +} + +func TestBuildVerify(t *testing.T) { + // Doing a full test of 1M accounts takes too much CPU time in CI. + doLargeTest := false + + totalWeight := 10000000 + npartHi := 10 + npartLo := 9990 + + if doLargeTest { + npartHi *= 100 + npartLo *= 100 + } + + npart := npartHi + npartLo + + param := Params{ + Msg: TestMessage("hello world"), + ProvenWeight: uint64(totalWeight / 2), + SigRound: 0, + SecKQ: 128, + } + + // Share the key; we allow the same vote key to appear in multiple accounts.. + key := crypto.GenerateOneTimeSignatureSecrets(0, 1) + + var parts []Participant + var sigs []crypto.OneTimeSignature + for i := 0; i < npartHi; i++ { + part := Participant{ + PK: key.OneTimeSignatureVerifier, + Weight: uint64(totalWeight / 2 / npartHi), + KeyDilution: 10000, + } + + parts = append(parts, part) + } + + for i := 0; i < npartLo; i++ { + part := Participant{ + PK: key.OneTimeSignatureVerifier, + Weight: uint64(totalWeight / 2 / npartLo), + KeyDilution: 10000, + } + + parts = append(parts, part) + } + + ephID := basics.OneTimeIDForRound(0, parts[0].KeyDilution) + sig := key.Sign(ephID, param.Msg) + + for i := 0; i < npart; i++ { + sigs = append(sigs, sig) + } + + partcom, err := merklearray.Build(PartCommit{parts}) + if err != nil { + t.Error(err) + } + + b, err := MkBuilder(param, parts, partcom) + if err != nil { + t.Error(err) + } + + for i := 0; i < npart; i++ { + err = b.Add(uint64(i), sigs[i], !doLargeTest) + if err != nil { + t.Error(err) + } + } + + cert, err := b.Build() + if err != nil { + t.Error(err) + } + + var someReveal Reveal + for _, rev := range cert.Reveals { + someReveal = rev + break + } + + certenc := protocol.Encode(cert) + fmt.Printf("Cert size:\n") + fmt.Printf(" %6d elems sigproofs\n", len(cert.SigProofs)) + fmt.Printf(" %6d bytes sigproofs\n", len(protocol.EncodeReflect(cert.SigProofs))) + fmt.Printf(" %6d bytes partproofs\n", len(protocol.EncodeReflect(cert.PartProofs))) + fmt.Printf(" %6d bytes sigproof per reveal\n", len(protocol.EncodeReflect(cert.SigProofs))/len(cert.Reveals)) + fmt.Printf(" %6d reveals:\n", len(cert.Reveals)) + fmt.Printf(" %6d bytes reveals[*] participant\n", len(protocol.Encode(&someReveal.Part))) + fmt.Printf(" %6d bytes reveals[*] sigslot\n", len(protocol.Encode(&someReveal.SigSlot))) + fmt.Printf(" %6d bytes reveals[*] total\n", len(protocol.Encode(&someReveal))) + fmt.Printf(" %6d bytes total\n", len(certenc)) + + verif := MkVerifier(param, partcom.Root()) + err = verif.Verify(cert) + if err != nil { + t.Error(err) + } +} + +func BenchmarkBuildVerify(b *testing.B) { + totalWeight := 1000000 + npart := 10000 + + param := Params{ + Msg: TestMessage("hello world"), + ProvenWeight: uint64(totalWeight / 2), + SigRound: 0, + SecKQ: 128, + } + + var parts []Participant + var partkeys []*crypto.OneTimeSignatureSecrets + var sigs []crypto.OneTimeSignature + for i := 0; i < npart; i++ { + key := crypto.GenerateOneTimeSignatureSecrets(0, 1) + part := Participant{ + PK: key.OneTimeSignatureVerifier, + Weight: uint64(totalWeight / npart), + KeyDilution: 10000, + } + + ephID := basics.OneTimeIDForRound(0, part.KeyDilution) + sig := key.Sign(ephID, param.Msg) + + partkeys = append(partkeys, key) + sigs = append(sigs, sig) + parts = append(parts, part) + } + + var cert *Cert + partcom, err := merklearray.Build(PartCommit{parts}) + if err != nil { + b.Error(err) + } + + b.Run("AddBuild", func(b *testing.B) { + for i := 0; i < b.N; i++ { + builder, err := MkBuilder(param, parts, partcom) + if err != nil { + b.Error(err) + } + + for i := 0; i < npart; i++ { + err = builder.Add(uint64(i), sigs[i], true) + if err != nil { + b.Error(err) + } + } + + cert, err = builder.Build() + if err != nil { + b.Error(err) + } + } + }) + + b.Run("Verify", func(b *testing.B) { + for i := 0; i < b.N; i++ { + verif := MkVerifier(param, partcom.Root()) + err = verif.Verify(cert) + if err != nil { + b.Error(err) + } + } + }) +} + +func TestCoinIndex(t *testing.T) { + n := 1000 + b := &Builder{ + sigs: make([]sigslot, n), + sigsHasValidL: true, + } + + for i := 0; i < n; i++ { + b.sigs[i].L = uint64(i) + b.sigs[i].Weight = 1 + } + + for i := 0; i < n; i++ { + pos, err := b.coinIndex(uint64(i)) + require.NoError(t, err) + require.Equal(t, pos, uint64(i)) + } +} diff --git a/crypto/compactcert/common.go b/crypto/compactcert/common.go new file mode 100644 index 0000000000..604dbcf475 --- /dev/null +++ b/crypto/compactcert/common.go @@ -0,0 +1,103 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package compactcert + +import ( + "encoding/binary" + "fmt" + "math/big" + + "github.com/algorand/go-algorand/crypto" +) + +// hashCoin returns a number in [0, signedWeight) with a nearly uniform +// distribution, "randomized" by all of the supplied arguments. +func hashCoin(j uint64, sigcom crypto.Digest, signedWeight uint64) uint64 { + hashinput := make([]byte, 16+crypto.DigestSize) + binary.LittleEndian.PutUint64(hashinput[0:], j) + binary.LittleEndian.PutUint64(hashinput[8:], signedWeight) + copy(hashinput[16:], sigcom[:]) + h := crypto.Hash(hashinput) + + i := &big.Int{} + i.SetBytes(h[:]) + + w := &big.Int{} + w.SetUint64(signedWeight) + + res := &big.Int{} + res.Mod(i, w) + return res.Uint64() +} + +// numReveals computes the number of reveals necessary to achieve the desired +// security parameters. See section 8 of the ``Compact Certificates'' +// document for the analysis. +// +// numReveals is the smallest number that satisfies +// +// 2^-k >= 2^q * (provenWeight / signedWeight) ^ numReveals +// +// which is equivalent to the following: +// +// signedWeight ^ numReveals >= 2^(k+q) * provenWeight ^ numReveals +// +// To ensure that rounding errors do not reduce the security parameter, +// we compute the left-hand side with rounding-down, and compute the +// right-hand side with rounding-up. +func numReveals(signedWeight uint64, provenWeight uint64, secKQ uint64, bound uint64) (uint64, error) { + n := uint64(0) + + sw := &bigFloatDn{} + err := sw.setu64(signedWeight) + if err != nil { + return 0, err + } + + pw := &bigFloatUp{} + err = pw.setu64(provenWeight) + if err != nil { + return 0, err + } + + lhs := &bigFloatDn{} + err = lhs.setu64(1) + if err != nil { + return 0, err + } + + rhs := &bigFloatUp{} + rhs.setpow2(int32(secKQ)) + + for { + if lhs.ge(rhs) { + return n, nil + } + + if n >= bound { + return 0, fmt.Errorf("numReveals(%d, %d, %d) > %d", signedWeight, provenWeight, secKQ, bound) + } + + lhs.mul(sw) + rhs.mul(pw) + n++ + } +} + +func (p Params) numReveals(signedWeight uint64) (uint64, error) { + return numReveals(signedWeight, p.ProvenWeight, p.SecKQ, maxReveals) +} diff --git a/crypto/compactcert/common_test.go b/crypto/compactcert/common_test.go new file mode 100644 index 0000000000..aaefd10512 --- /dev/null +++ b/crypto/compactcert/common_test.go @@ -0,0 +1,99 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package compactcert + +import ( + "testing" + + "github.com/algorand/go-algorand/crypto" +) + +func TestHashCoin(t *testing.T) { + var slots [32]uint64 + var sigcom [32]byte + + crypto.RandBytes(sigcom[:]) + + for j := uint64(0); j < 1000; j++ { + coin := hashCoin(j, sigcom, uint64(len(slots))) + if coin >= uint64(len(slots)) { + t.Errorf("hashCoin out of bounds") + } + + slots[coin]++ + } + + for i, count := range slots { + if count < 3 { + t.Errorf("slot %d too low: %d", i, count) + } + if count > 100 { + t.Errorf("slot %d too high: %d", i, count) + } + } +} + +func BenchmarkHashCoin(b *testing.B) { + var sigcom [32]byte + crypto.RandBytes(sigcom[:]) + + for i := 0; i < b.N; i++ { + hashCoin(uint64(i), sigcom, 1024) + } +} + +func TestNumReveals(t *testing.T) { + billion := uint64(1000 * 1000 * 1000) + microalgo := uint64(1000 * 1000) + provenWeight := 2 * billion * microalgo + secKQ := uint64(128) + bound := uint64(1000) + + for i := uint64(3); i < 10; i++ { + signedWeight := i * billion * microalgo + n, err := numReveals(signedWeight, provenWeight, secKQ, bound) + if err != nil { + t.Error(err) + } + + if n < 50 || n > 300 { + t.Errorf("numReveals(%d, %d, %d) = %d looks suspect", + signedWeight, provenWeight, secKQ, n) + } + } +} + +func BenchmarkNumReveals(b *testing.B) { + billion := uint64(1000 * 1000 * 1000) + microalgo := uint64(1000 * 1000) + provenWeight := 100 * billion * microalgo + signedWeight := 110 * billion * microalgo + secKQ := uint64(128) + bound := uint64(1000) + + nr, err := numReveals(signedWeight, provenWeight, secKQ, bound) + if nr < 900 { + b.Errorf("numReveals(%d, %d, %d) = %d < 900", signedWeight, provenWeight, secKQ, nr) + } + + for i := 0; i < b.N; i++ { + _, err = numReveals(signedWeight, provenWeight, secKQ, bound) + if err != nil { + b.Error(err) + } + } +} diff --git a/crypto/compactcert/msgp_gen.go b/crypto/compactcert/msgp_gen.go new file mode 100644 index 0000000000..4c12f3d9a9 --- /dev/null +++ b/crypto/compactcert/msgp_gen.go @@ -0,0 +1,1271 @@ +package compactcert + +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +import ( + "sort" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/msgp/msgp" +) + +// The following msgp objects are implemented in this file: +// Cert +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// +// CompactOneTimeSignature +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// +// Participant +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// +// Reveal +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// +// sigslotCommit +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// + +// MarshalMsg implements msgp.Marshaler +func (z *Cert) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0005Len := uint32(5) + var zb0005Mask uint8 /* 6 bits */ + if len((*z).PartProofs) == 0 { + zb0005Len-- + zb0005Mask |= 0x1 + } + if len((*z).SigProofs) == 0 { + zb0005Len-- + zb0005Mask |= 0x2 + } + if (*z).SigCommit.MsgIsZero() { + zb0005Len-- + zb0005Mask |= 0x8 + } + if len((*z).Reveals) == 0 { + zb0005Len-- + zb0005Mask |= 0x10 + } + if (*z).SignedWeight == 0 { + zb0005Len-- + zb0005Mask |= 0x20 + } + // variable map header, size zb0005Len + o = append(o, 0x80|uint8(zb0005Len)) + if zb0005Len != 0 { + if (zb0005Mask & 0x1) == 0 { // if not empty + // string "P" + o = append(o, 0xa1, 0x50) + if (*z).PartProofs == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len((*z).PartProofs))) + } + for zb0002 := range (*z).PartProofs { + o, err = (*z).PartProofs[zb0002].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "PartProofs", zb0002) + return + } + } + } + if (zb0005Mask & 0x2) == 0 { // if not empty + // string "S" + o = append(o, 0xa1, 0x53) + if (*z).SigProofs == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len((*z).SigProofs))) + } + for zb0001 := range (*z).SigProofs { + o, err = (*z).SigProofs[zb0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "SigProofs", zb0001) + return + } + } + } + if (zb0005Mask & 0x8) == 0 { // if not empty + // string "c" + o = append(o, 0xa1, 0x63) + o, err = (*z).SigCommit.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "SigCommit") + return + } + } + if (zb0005Mask & 0x10) == 0 { // if not empty + // string "r" + o = append(o, 0xa1, 0x72) + if (*z).Reveals == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendMapHeader(o, uint32(len((*z).Reveals))) + } + zb0003_keys := make([]uint64, 0, len((*z).Reveals)) + for zb0003 := range (*z).Reveals { + zb0003_keys = append(zb0003_keys, zb0003) + } + sort.Sort(SortUint64(zb0003_keys)) + for _, zb0003 := range zb0003_keys { + zb0004 := (*z).Reveals[zb0003] + _ = zb0004 + o = msgp.AppendUint64(o, zb0003) + o, err = zb0004.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Reveals", zb0003) + return + } + } + } + if (zb0005Mask & 0x20) == 0 { // if not empty + // string "w" + o = append(o, 0xa1, 0x77) + o = msgp.AppendUint64(o, (*z).SignedWeight) + } + } + return +} + +func (_ *Cert) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*Cert) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Cert) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0005 int + var zb0006 bool + zb0005, zb0006, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0005, zb0006, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0005 > 0 { + zb0005-- + bts, err = (*z).SigCommit.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigCommit") + return + } + } + if zb0005 > 0 { + zb0005-- + (*z).SignedWeight, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SignedWeight") + return + } + } + if zb0005 > 0 { + zb0005-- + var zb0007 int + var zb0008 bool + zb0007, zb0008, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigProofs") + return + } + if zb0007 > maxProofDigests { + err = msgp.ErrOverflow(uint64(zb0007), uint64(maxProofDigests)) + err = msgp.WrapError(err, "struct-from-array", "SigProofs") + return + } + if zb0008 { + (*z).SigProofs = nil + } else if (*z).SigProofs != nil && cap((*z).SigProofs) >= zb0007 { + (*z).SigProofs = ((*z).SigProofs)[:zb0007] + } else { + (*z).SigProofs = make([]crypto.Digest, zb0007) + } + for zb0001 := range (*z).SigProofs { + bts, err = (*z).SigProofs[zb0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigProofs", zb0001) + return + } + } + } + if zb0005 > 0 { + zb0005-- + var zb0009 int + var zb0010 bool + zb0009, zb0010, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "PartProofs") + return + } + if zb0009 > maxProofDigests { + err = msgp.ErrOverflow(uint64(zb0009), uint64(maxProofDigests)) + err = msgp.WrapError(err, "struct-from-array", "PartProofs") + return + } + if zb0010 { + (*z).PartProofs = nil + } else if (*z).PartProofs != nil && cap((*z).PartProofs) >= zb0009 { + (*z).PartProofs = ((*z).PartProofs)[:zb0009] + } else { + (*z).PartProofs = make([]crypto.Digest, zb0009) + } + for zb0002 := range (*z).PartProofs { + bts, err = (*z).PartProofs[zb0002].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "PartProofs", zb0002) + return + } + } + } + if zb0005 > 0 { + zb0005-- + var zb0011 int + var zb0012 bool + zb0011, zb0012, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Reveals") + return + } + if zb0011 > maxReveals { + err = msgp.ErrOverflow(uint64(zb0011), uint64(maxReveals)) + err = msgp.WrapError(err, "struct-from-array", "Reveals") + return + } + if zb0012 { + (*z).Reveals = nil + } else if (*z).Reveals == nil { + (*z).Reveals = make(map[uint64]Reveal, zb0011) + } + for zb0011 > 0 { + var zb0003 uint64 + var zb0004 Reveal + zb0011-- + zb0003, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Reveals") + return + } + bts, err = zb0004.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Reveals", zb0003) + return + } + (*z).Reveals[zb0003] = zb0004 + } + } + if zb0005 > 0 { + err = msgp.ErrTooManyArrayFields(zb0005) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0006 { + (*z) = Cert{} + } + for zb0005 > 0 { + zb0005-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "c": + bts, err = (*z).SigCommit.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "SigCommit") + return + } + case "w": + (*z).SignedWeight, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "SignedWeight") + return + } + case "S": + var zb0013 int + var zb0014 bool + zb0013, zb0014, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SigProofs") + return + } + if zb0013 > maxProofDigests { + err = msgp.ErrOverflow(uint64(zb0013), uint64(maxProofDigests)) + err = msgp.WrapError(err, "SigProofs") + return + } + if zb0014 { + (*z).SigProofs = nil + } else if (*z).SigProofs != nil && cap((*z).SigProofs) >= zb0013 { + (*z).SigProofs = ((*z).SigProofs)[:zb0013] + } else { + (*z).SigProofs = make([]crypto.Digest, zb0013) + } + for zb0001 := range (*z).SigProofs { + bts, err = (*z).SigProofs[zb0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "SigProofs", zb0001) + return + } + } + case "P": + var zb0015 int + var zb0016 bool + zb0015, zb0016, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartProofs") + return + } + if zb0015 > maxProofDigests { + err = msgp.ErrOverflow(uint64(zb0015), uint64(maxProofDigests)) + err = msgp.WrapError(err, "PartProofs") + return + } + if zb0016 { + (*z).PartProofs = nil + } else if (*z).PartProofs != nil && cap((*z).PartProofs) >= zb0015 { + (*z).PartProofs = ((*z).PartProofs)[:zb0015] + } else { + (*z).PartProofs = make([]crypto.Digest, zb0015) + } + for zb0002 := range (*z).PartProofs { + bts, err = (*z).PartProofs[zb0002].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "PartProofs", zb0002) + return + } + } + case "r": + var zb0017 int + var zb0018 bool + zb0017, zb0018, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Reveals") + return + } + if zb0017 > maxReveals { + err = msgp.ErrOverflow(uint64(zb0017), uint64(maxReveals)) + err = msgp.WrapError(err, "Reveals") + return + } + if zb0018 { + (*z).Reveals = nil + } else if (*z).Reveals == nil { + (*z).Reveals = make(map[uint64]Reveal, zb0017) + } + for zb0017 > 0 { + var zb0003 uint64 + var zb0004 Reveal + zb0017-- + zb0003, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Reveals") + return + } + bts, err = zb0004.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Reveals", zb0003) + return + } + (*z).Reveals[zb0003] = zb0004 + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *Cert) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*Cert) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *Cert) Msgsize() (s int) { + s = 1 + 2 + (*z).SigCommit.Msgsize() + 2 + msgp.Uint64Size + 2 + msgp.ArrayHeaderSize + for zb0001 := range (*z).SigProofs { + s += (*z).SigProofs[zb0001].Msgsize() + } + s += 2 + msgp.ArrayHeaderSize + for zb0002 := range (*z).PartProofs { + s += (*z).PartProofs[zb0002].Msgsize() + } + s += 2 + msgp.MapHeaderSize + if (*z).Reveals != nil { + for zb0003, zb0004 := range (*z).Reveals { + _ = zb0003 + _ = zb0004 + s += 0 + msgp.Uint64Size + zb0004.Msgsize() + } + } + return +} + +// MsgIsZero returns whether this is a zero value +func (z *Cert) MsgIsZero() bool { + return ((*z).SigCommit.MsgIsZero()) && ((*z).SignedWeight == 0) && (len((*z).SigProofs) == 0) && (len((*z).PartProofs) == 0) && (len((*z).Reveals) == 0) +} + +// MarshalMsg implements msgp.Marshaler +func (z *CompactOneTimeSignature) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(6) + var zb0001Mask uint8 /* 8 bits */ + if (*z).OneTimeSignature.PK.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + if (*z).OneTimeSignature.PK1Sig.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x8 + } + if (*z).OneTimeSignature.PK2.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x10 + } + if (*z).OneTimeSignature.PK2Sig.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x20 + } + if (*z).OneTimeSignature.PKSigOld.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x40 + } + if (*z).OneTimeSignature.Sig.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x80 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "p" + o = append(o, 0xa1, 0x70) + o, err = (*z).OneTimeSignature.PK.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "PK") + return + } + } + if (zb0001Mask & 0x8) == 0 { // if not empty + // string "p1s" + o = append(o, 0xa3, 0x70, 0x31, 0x73) + o, err = (*z).OneTimeSignature.PK1Sig.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "PK1Sig") + return + } + } + if (zb0001Mask & 0x10) == 0 { // if not empty + // string "p2" + o = append(o, 0xa2, 0x70, 0x32) + o, err = (*z).OneTimeSignature.PK2.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "PK2") + return + } + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // string "p2s" + o = append(o, 0xa3, 0x70, 0x32, 0x73) + o, err = (*z).OneTimeSignature.PK2Sig.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "PK2Sig") + return + } + } + if (zb0001Mask & 0x40) == 0 { // if not empty + // string "ps" + o = append(o, 0xa2, 0x70, 0x73) + o, err = (*z).OneTimeSignature.PKSigOld.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "PKSigOld") + return + } + } + if (zb0001Mask & 0x80) == 0 { // if not empty + // string "s" + o = append(o, 0xa1, 0x73) + o, err = (*z).OneTimeSignature.Sig.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Sig") + return + } + } + } + return +} + +func (_ *CompactOneTimeSignature) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*CompactOneTimeSignature) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *CompactOneTimeSignature) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).OneTimeSignature.Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Sig") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).OneTimeSignature.PK.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "PK") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).OneTimeSignature.PKSigOld.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "PKSigOld") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).OneTimeSignature.PK2.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "PK2") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).OneTimeSignature.PK1Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "PK1Sig") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).OneTimeSignature.PK2Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "PK2Sig") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = CompactOneTimeSignature{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "s": + bts, err = (*z).OneTimeSignature.Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Sig") + return + } + case "p": + bts, err = (*z).OneTimeSignature.PK.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "PK") + return + } + case "ps": + bts, err = (*z).OneTimeSignature.PKSigOld.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "PKSigOld") + return + } + case "p2": + bts, err = (*z).OneTimeSignature.PK2.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "PK2") + return + } + case "p1s": + bts, err = (*z).OneTimeSignature.PK1Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "PK1Sig") + return + } + case "p2s": + bts, err = (*z).OneTimeSignature.PK2Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "PK2Sig") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *CompactOneTimeSignature) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*CompactOneTimeSignature) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *CompactOneTimeSignature) Msgsize() (s int) { + s = 1 + 2 + (*z).OneTimeSignature.Sig.Msgsize() + 2 + (*z).OneTimeSignature.PK.Msgsize() + 3 + (*z).OneTimeSignature.PKSigOld.Msgsize() + 3 + (*z).OneTimeSignature.PK2.Msgsize() + 4 + (*z).OneTimeSignature.PK1Sig.Msgsize() + 4 + (*z).OneTimeSignature.PK2Sig.Msgsize() + return +} + +// MsgIsZero returns whether this is a zero value +func (z *CompactOneTimeSignature) MsgIsZero() bool { + return ((*z).OneTimeSignature.Sig.MsgIsZero()) && ((*z).OneTimeSignature.PK.MsgIsZero()) && ((*z).OneTimeSignature.PKSigOld.MsgIsZero()) && ((*z).OneTimeSignature.PK2.MsgIsZero()) && ((*z).OneTimeSignature.PK1Sig.MsgIsZero()) && ((*z).OneTimeSignature.PK2Sig.MsgIsZero()) +} + +// MarshalMsg implements msgp.Marshaler +func (z *Participant) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(3) + var zb0001Mask uint8 /* 4 bits */ + if (*z).KeyDilution == 0 { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).PK.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + if (*z).Weight == 0 { + zb0001Len-- + zb0001Mask |= 0x8 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "d" + o = append(o, 0xa1, 0x64) + o = msgp.AppendUint64(o, (*z).KeyDilution) + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "p" + o = append(o, 0xa1, 0x70) + o, err = (*z).PK.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "PK") + return + } + } + if (zb0001Mask & 0x8) == 0 { // if not empty + // string "w" + o = append(o, 0xa1, 0x77) + o = msgp.AppendUint64(o, (*z).Weight) + } + } + return +} + +func (_ *Participant) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*Participant) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Participant) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).PK.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "PK") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).Weight, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Weight") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).KeyDilution, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "KeyDilution") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = Participant{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "p": + bts, err = (*z).PK.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "PK") + return + } + case "w": + (*z).Weight, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Weight") + return + } + case "d": + (*z).KeyDilution, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "KeyDilution") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *Participant) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*Participant) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *Participant) Msgsize() (s int) { + s = 1 + 2 + (*z).PK.Msgsize() + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + return +} + +// MsgIsZero returns whether this is a zero value +func (z *Participant) MsgIsZero() bool { + return ((*z).PK.MsgIsZero()) && ((*z).Weight == 0) && ((*z).KeyDilution == 0) +} + +// MarshalMsg implements msgp.Marshaler +func (z *Reveal) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 3 bits */ + if (*z).Part.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + if ((*z).SigSlot.Sig.MsgIsZero()) && ((*z).SigSlot.L == 0) { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "p" + o = append(o, 0xa1, 0x70) + o, err = (*z).Part.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Part") + return + } + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "s" + o = append(o, 0xa1, 0x73) + // omitempty: check for empty values + zb0002Len := uint32(2) + var zb0002Mask uint8 /* 3 bits */ + if (*z).SigSlot.L == 0 { + zb0002Len-- + zb0002Mask |= 0x2 + } + if (*z).SigSlot.Sig.MsgIsZero() { + zb0002Len-- + zb0002Mask |= 0x4 + } + // variable map header, size zb0002Len + o = append(o, 0x80|uint8(zb0002Len)) + if (zb0002Mask & 0x2) == 0 { // if not empty + // string "l" + o = append(o, 0xa1, 0x6c) + o = msgp.AppendUint64(o, (*z).SigSlot.L) + } + if (zb0002Mask & 0x4) == 0 { // if not empty + // string "s" + o = append(o, 0xa1, 0x73) + o, err = (*z).SigSlot.Sig.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "SigSlot", "Sig") + return + } + } + } + } + return +} + +func (_ *Reveal) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*Reveal) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Reveal) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + var zb0003 int + var zb0004 bool + zb0003, zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0003, zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigSlot") + return + } + if zb0003 > 0 { + zb0003-- + bts, err = (*z).SigSlot.Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigSlot", "struct-from-array", "Sig") + return + } + } + if zb0003 > 0 { + zb0003-- + (*z).SigSlot.L, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigSlot", "struct-from-array", "L") + return + } + } + if zb0003 > 0 { + err = msgp.ErrTooManyArrayFields(zb0003) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigSlot", "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigSlot") + return + } + if zb0004 { + (*z).SigSlot = sigslotCommit{} + } + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigSlot") + return + } + switch string(field) { + case "s": + bts, err = (*z).SigSlot.Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigSlot", "Sig") + return + } + case "l": + (*z).SigSlot.L, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigSlot", "L") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "SigSlot") + return + } + } + } + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Part.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Part") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = Reveal{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "s": + var zb0005 int + var zb0006 bool + zb0005, zb0006, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0005, zb0006, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SigSlot") + return + } + if zb0005 > 0 { + zb0005-- + bts, err = (*z).SigSlot.Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "SigSlot", "struct-from-array", "Sig") + return + } + } + if zb0005 > 0 { + zb0005-- + (*z).SigSlot.L, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "SigSlot", "struct-from-array", "L") + return + } + } + if zb0005 > 0 { + err = msgp.ErrTooManyArrayFields(zb0005) + if err != nil { + err = msgp.WrapError(err, "SigSlot", "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "SigSlot") + return + } + if zb0006 { + (*z).SigSlot = sigslotCommit{} + } + for zb0005 > 0 { + zb0005-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "SigSlot") + return + } + switch string(field) { + case "s": + bts, err = (*z).SigSlot.Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "SigSlot", "Sig") + return + } + case "l": + (*z).SigSlot.L, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "SigSlot", "L") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "SigSlot") + return + } + } + } + } + case "p": + bts, err = (*z).Part.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Part") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *Reveal) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*Reveal) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *Reveal) Msgsize() (s int) { + s = 1 + 2 + 1 + 2 + (*z).SigSlot.Sig.Msgsize() + 2 + msgp.Uint64Size + 2 + (*z).Part.Msgsize() + return +} + +// MsgIsZero returns whether this is a zero value +func (z *Reveal) MsgIsZero() bool { + return (((*z).SigSlot.Sig.MsgIsZero()) && ((*z).SigSlot.L == 0)) && ((*z).Part.MsgIsZero()) +} + +// MarshalMsg implements msgp.Marshaler +func (z *sigslotCommit) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 3 bits */ + if (*z).L == 0 { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).Sig.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "l" + o = append(o, 0xa1, 0x6c) + o = msgp.AppendUint64(o, (*z).L) + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "s" + o = append(o, 0xa1, 0x73) + o, err = (*z).Sig.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Sig") + return + } + } + } + return +} + +func (_ *sigslotCommit) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*sigslotCommit) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *sigslotCommit) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Sig") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).L, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "L") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = sigslotCommit{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "s": + bts, err = (*z).Sig.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Sig") + return + } + case "l": + (*z).L, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "L") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *sigslotCommit) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*sigslotCommit) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *sigslotCommit) Msgsize() (s int) { + s = 1 + 2 + (*z).Sig.Msgsize() + 2 + msgp.Uint64Size + return +} + +// MsgIsZero returns whether this is a zero value +func (z *sigslotCommit) MsgIsZero() bool { + return ((*z).Sig.MsgIsZero()) && ((*z).L == 0) +} diff --git a/crypto/compactcert/msgp_gen_test.go b/crypto/compactcert/msgp_gen_test.go new file mode 100644 index 0000000000..28f0e9d163 --- /dev/null +++ b/crypto/compactcert/msgp_gen_test.go @@ -0,0 +1,322 @@ +// +build !skip_msgp_testing + +package compactcert + +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +import ( + "testing" + + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/msgp/msgp" +) + +func TestMarshalUnmarshalCert(t *testing.T) { + v := Cert{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingCert(t *testing.T) { + protocol.RunEncodingTest(t, &Cert{}) +} + +func BenchmarkMarshalMsgCert(b *testing.B) { + v := Cert{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgCert(b *testing.B) { + v := Cert{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalCert(b *testing.B) { + v := Cert{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalCompactOneTimeSignature(t *testing.T) { + v := CompactOneTimeSignature{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingCompactOneTimeSignature(t *testing.T) { + protocol.RunEncodingTest(t, &CompactOneTimeSignature{}) +} + +func BenchmarkMarshalMsgCompactOneTimeSignature(b *testing.B) { + v := CompactOneTimeSignature{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgCompactOneTimeSignature(b *testing.B) { + v := CompactOneTimeSignature{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalCompactOneTimeSignature(b *testing.B) { + v := CompactOneTimeSignature{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalParticipant(t *testing.T) { + v := Participant{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingParticipant(t *testing.T) { + protocol.RunEncodingTest(t, &Participant{}) +} + +func BenchmarkMarshalMsgParticipant(b *testing.B) { + v := Participant{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgParticipant(b *testing.B) { + v := Participant{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalParticipant(b *testing.B) { + v := Participant{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReveal(t *testing.T) { + v := Reveal{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingReveal(t *testing.T) { + protocol.RunEncodingTest(t, &Reveal{}) +} + +func BenchmarkMarshalMsgReveal(b *testing.B) { + v := Reveal{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReveal(b *testing.B) { + v := Reveal{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReveal(b *testing.B) { + v := Reveal{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalsigslotCommit(t *testing.T) { + v := sigslotCommit{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingsigslotCommit(t *testing.T) { + protocol.RunEncodingTest(t, &sigslotCommit{}) +} + +func BenchmarkMarshalMsgsigslotCommit(b *testing.B) { + v := sigslotCommit{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgsigslotCommit(b *testing.B) { + v := sigslotCommit{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalsigslotCommit(b *testing.B) { + v := sigslotCommit{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/crypto/compactcert/structs.go b/crypto/compactcert/structs.go new file mode 100644 index 0000000000..37f756073b --- /dev/null +++ b/crypto/compactcert/structs.go @@ -0,0 +1,115 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package compactcert + +import ( + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/protocol" +) + +// Params defines common parameters for the verifier and builder. +type Params struct { + Msg crypto.Hashable // Message to be cerified + ProvenWeight uint64 // Weight threshold proven by the certificate + SigRound basics.Round // Ephemeral signature round to expect + SecKQ uint64 // Security parameter (k+q) from analysis document +} + +// A Participant corresponds to an account whose AccountData.Status +// is Online, and for which the expected sigRound satisfies +// AccountData.VoteFirstValid <= sigRound <= AccountData.VoteLastValid. +// +// In the Algorand ledger, it is possible for multiple accounts to have +// the same PK. Thus, the PK is not necessarily unique among Participants. +// However, each account will produce a unique Participant struct, to avoid +// potential DoS attacks where one account claims to have the same VoteID PK +// as another account. +type Participant struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + // PK is AccountData.VoteID. + PK crypto.OneTimeSignatureVerifier `codec:"p"` + + // Weight is AccountData.MicroAlgos. + Weight uint64 `codec:"w"` + + // KeyDilution is AccountData.KeyDilution() with the protocol for sigRound + // as expected by the Builder. + KeyDilution uint64 `codec:"d"` +} + +// ToBeHashed implements the crypto.Hashable interface. +func (p Participant) ToBeHashed() (protocol.HashID, []byte) { + return protocol.CompactCertPart, protocol.Encode(&p) +} + +// CompactOneTimeSignature is crypto.OneTimeSignature with omitempty +type CompactOneTimeSignature struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + crypto.OneTimeSignature +} + +// A sigslotCommit is a single slot in the sigs array that forms the certificate. +type sigslotCommit struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + // Sig is a signature by the participant on the expected message. + Sig CompactOneTimeSignature `codec:"s"` + + // L is the total weight of signatures in lower-numbered slots. + // This is initialized once the builder has collected a sufficient + // number of signatures. + L uint64 `codec:"l"` +} + +func (ssc sigslotCommit) ToBeHashed() (protocol.HashID, []byte) { + return protocol.CompactCertSig, protocol.Encode(&ssc) +} + +// Reveal is a single array position revealed as part of a compact +// certificate. It reveals an element of the signature array and +// the corresponding element of the participants array. +type Reveal struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + SigSlot sigslotCommit `codec:"s"` + Part Participant `codec:"p"` +} + +// maxReveals is a bound on allocation and on numReveals to limit log computation +const maxReveals = 1024 +const maxProofDigests = 20 * maxReveals + +// Cert represents a compact certificate. +type Cert struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + SigCommit crypto.Digest `codec:"c"` + SignedWeight uint64 `codec:"w"` + SigProofs []crypto.Digest `codec:"S,allocbound=maxProofDigests"` + PartProofs []crypto.Digest `codec:"P,allocbound=maxProofDigests"` + + // Reveals is a sparse map from the position being revealed + // to the corresponding elements from the sigs and participants + // arrays. + Reveals map[uint64]Reveal `codec:"r,allocbound=maxReveals"` +} + +// SortUint64 implements sorting by uint64 keys for +// canonical encoding of maps in msgpack format. +type SortUint64 = basics.SortUint64 diff --git a/crypto/compactcert/verifier.go b/crypto/compactcert/verifier.go new file mode 100644 index 0000000000..cfb7a7f792 --- /dev/null +++ b/crypto/compactcert/verifier.go @@ -0,0 +1,96 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package compactcert + +import ( + "fmt" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklearray" + "github.com/algorand/go-algorand/data/basics" +) + +// Verifier is used to verify a compact certificate. +type Verifier struct { + Params + + partcom crypto.Digest +} + +// MkVerifier constructs a verifier to check the compact certificate +// on the message specified in p, with partcom specifying the Merkle +// root of the participants that must sign the message. +func MkVerifier(p Params, partcom crypto.Digest) *Verifier { + return &Verifier{ + Params: p, + partcom: partcom, + } +} + +// Verify checks if c is a valid compact certificate for the message +// and participants that were used to construct the Verifier. +func (v *Verifier) Verify(c *Cert) error { + if c.SignedWeight <= v.ProvenWeight { + return fmt.Errorf("cert signed weight %d <= proven weight %d", c.SignedWeight, v.ProvenWeight) + } + + // Verify all of the reveals + sigs := make(map[uint64]crypto.Hashable) + parts := make(map[uint64]crypto.Hashable) + for pos, r := range c.Reveals { + sigs[pos] = r.SigSlot + parts[pos] = r.Part + + ephID := basics.OneTimeIDForRound(v.SigRound, r.Part.KeyDilution) + if !r.Part.PK.Verify(ephID, v.Msg, r.SigSlot.Sig.OneTimeSignature) { + return fmt.Errorf("signature in reveal pos %d does not verify", pos) + } + } + + err := merklearray.Verify(c.SigCommit, sigs, c.SigProofs) + if err != nil { + return err + } + + err = merklearray.Verify(v.partcom, parts, c.PartProofs) + if err != nil { + return err + } + + // Verify that the reveals contain the right coins + nr, err := v.numReveals(c.SignedWeight) + if err != nil { + return err + } + + for j := uint64(0); j < nr; j++ { + coin := hashCoin(j, c.SigCommit, c.SignedWeight) + matchingReveal := false + for _, r := range c.Reveals { + if r.SigSlot.L <= coin && coin < r.SigSlot.L+r.Part.Weight { + matchingReveal = true + break + } + } + + if !matchingReveal { + return fmt.Errorf("no reveal for coin %d at %d", j, coin) + } + } + + return nil +} diff --git a/protocol/hash.go b/protocol/hash.go index d3f821690e..4ab2786e69 100644 --- a/protocol/hash.go +++ b/protocol/hash.go @@ -28,6 +28,9 @@ const ( AuctionParams HashID = "aP" AuctionSettlement HashID = "aS" + CompactCertPart HashID = "ccp" + CompactCertSig HashID = "ccs" + AgreementSelector HashID = "AS" BlockHeader HashID = "BH" BalanceRecord HashID = "BR" From 4849f5167294b185615c9750f8aa53864375df81 Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Tue, 8 Sep 2020 21:29:51 -0400 Subject: [PATCH 025/136] implement tracking of top-N online accounts in the ledger (#1361) Implement tracking of top-N online accounts in the ledger --- data/basics/overflow.go | 19 +++ data/basics/userBalance.go | 43 ++++++ ledger/accountdb.go | 223 +++++++++++++++++++++++++++----- ledger/accountdb_test.go | 51 +++++++- ledger/acctupdates.go | 130 ++++++++++++++++++- ledger/acctupdates_test.go | 14 +- ledger/archival_test.go | 4 + ledger/catchpointwriter_test.go | 4 +- ledger/catchupaccessor.go | 3 +- ledger/ledger.go | 5 + ledger/onlineacct.go | 89 +++++++++++++ ledger/tracker.go | 2 + 12 files changed, 549 insertions(+), 38 deletions(-) create mode 100644 ledger/onlineacct.go diff --git a/data/basics/overflow.go b/data/basics/overflow.go index 3e28d17aa4..e344a2ada0 100644 --- a/data/basics/overflow.go +++ b/data/basics/overflow.go @@ -18,6 +18,7 @@ package basics import ( "math" + "math/big" ) // OverflowTracker is used to track when an operation causes an overflow @@ -163,3 +164,21 @@ func (t *OverflowTracker) SubR(a Round, b Round) Round { func (t *OverflowTracker) ScalarMulA(a MicroAlgos, b uint64) MicroAlgos { return MicroAlgos{Raw: t.Mul(a.Raw, b)} } + +// Muldiv computes a*b/c. The overflow flag indicates that +// the result was 2^64 or greater. +func Muldiv(a uint64, b uint64, c uint64) (res uint64, overflow bool) { + var aa big.Int + aa.SetUint64(a) + + var bb big.Int + bb.SetUint64(b) + + var cc big.Int + cc.SetUint64(c) + + aa.Mul(&aa, &bb) + aa.Div(&aa, &cc) + + return aa.Uint64(), !aa.IsUint64() +} diff --git a/data/basics/userBalance.go b/data/basics/userBalance.go index 1f56369d20..7ec30b836a 100644 --- a/data/basics/userBalance.go +++ b/data/basics/userBalance.go @@ -462,6 +462,49 @@ func (u AccountData) IsZero() bool { return reflect.DeepEqual(u, AccountData{}) } +// NormalizedOnlineBalance returns a ``normalized'' balance for this account. +// +// The normalization compensates for rewards that have not yet been applied, +// by computing a balance normalized to round 0. To normalize, we estimate +// the microalgo balance that an account should have had at round 0, in order +// to end up at its current balance when rewards are included. +// +// The benefit of the normalization procedure is that an account's normalized +// balance does not change over time (unlike the actual algo balance that includes +// rewards). This makes it possible to compare normalized balances between two +// accounts, to sort them, and get results that are close to what we would get +// if we computed the exact algo balance of the accounts at a given round number. +// +// The normalization can lead to some inconsistencies in comparisons between +// account balances, because the growth rate of rewards for accounts depends +// on how recently the account has been touched (our rewards do not implement +// compounding). However, online accounts have to periodically renew +// participation keys, so the scale of the inconsistency is small. +func (u AccountData) NormalizedOnlineBalance(proto config.ConsensusParams) uint64 { + if u.Status != Online { + return 0 + } + + // If this account had one RewardUnit of microAlgos in round 0, it would + // have perRewardUnit microAlgos at the account's current rewards level. + perRewardUnit := u.RewardsBase + proto.RewardUnit + + // To normalize, we compute, mathematically, + // u.MicroAlgos / perRewardUnit * proto.RewardUnit, as + // (u.MicroAlgos * proto.RewardUnit) / perRewardUnit. + norm, overflowed := Muldiv(u.MicroAlgos.ToUint64(), proto.RewardUnit, perRewardUnit) + + // Mathematically should be impossible to overflow + // because perRewardUnit >= proto.RewardUnit, as long + // as u.RewardBase isn't huge enough to cause overflow.. + if overflowed { + logging.Base().Panicf("overflow computing normalized balance %d * %d / (%d + %d)", + u.MicroAlgos.ToUint64(), proto.RewardUnit, u.RewardsBase, proto.RewardUnit) + } + + return norm +} + // BalanceRecord pairs an account's address with its associated data. type BalanceRecord struct { _struct struct{} `codec:",omitempty,omitemptyarray"` diff --git a/ledger/accountdb.go b/ledger/accountdb.go index c6f148dacb..9a98c0b54b 100644 --- a/ledger/accountdb.go +++ b/ledger/accountdb.go @@ -87,6 +87,18 @@ var creatablesMigration = []string{ `ALTER TABLE assetcreators ADD COLUMN ctype INTEGER DEFAULT 0`, } +func createNormalizedOnlineBalanceIndex(idxname string, tablename string) string { + return fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s + ON %s ( normalizedonlinebalance, address, data ) + WHERE normalizedonlinebalance>0`, idxname, tablename) +} + +var createOnlineAccountIndex = []string{ + `ALTER TABLE accountbase + ADD COLUMN normalizedonlinebalance INTEGER`, + createNormalizedOnlineBalanceIndex("onlineaccountbals", "accountbase"), +} + var accountsResetExprs = []string{ `DROP TABLE IF EXISTS acctrounds`, `DROP TABLE IF EXISTS accounttotals`, @@ -100,7 +112,7 @@ var accountsResetExprs = []string{ // accountDBVersion is the database version that this binary would know how to support and how to upgrade to. // details about the content of each of the versions can be found in the upgrade functions upgradeDatabaseSchemaXXXX // and their descriptions. -var accountDBVersion = int32(3) +var accountDBVersion = int32(4) type accountDelta struct { old basics.AccountData @@ -136,17 +148,26 @@ func writeCatchpointStagingCreatable(ctx context.Context, tx *sql.Tx, addr basic return nil } -func writeCatchpointStagingBalances(ctx context.Context, tx *sql.Tx, bals []encodedBalanceRecord) error { - insertStmt, err := tx.PrepareContext(ctx, "INSERT INTO catchpointbalances(address, data) VALUES(?, ?)") +func writeCatchpointStagingBalances(ctx context.Context, tx *sql.Tx, bals []encodedBalanceRecord, proto config.ConsensusParams) error { + insertStmt, err := tx.PrepareContext(ctx, "INSERT INTO catchpointbalances(address, normalizedonlinebalance, data) VALUES(?, ?, ?)") if err != nil { return err } for _, balance := range bals { - result, err := insertStmt.ExecContext(ctx, balance.Address[:], balance.AccountData) + var data basics.AccountData + err = protocol.Decode(balance.AccountData, &data) if err != nil { return err } + + normBalance := data.NormalizedOnlineBalance(proto) + + result, err := insertStmt.ExecContext(ctx, balance.Address[:], normBalance, balance.AccountData) + if err != nil { + return err + } + aff, err := result.RowsAffected() if err != nil { return err @@ -154,42 +175,68 @@ func writeCatchpointStagingBalances(ctx context.Context, tx *sql.Tx, bals []enco if aff != 1 { return fmt.Errorf("number of affected record in insert was expected to be one, but was %d", aff) } - } return nil } func resetCatchpointStagingBalances(ctx context.Context, tx *sql.Tx, newCatchup bool) (err error) { - s := "DROP TABLE IF EXISTS catchpointbalances;" - s += "DROP TABLE IF EXISTS catchpointassetcreators;" - s += "DROP TABLE IF EXISTS catchpointaccounthashes;" - s += "DELETE FROM accounttotals where id='catchpointStaging';" + s := []string{ + "DROP TABLE IF EXISTS catchpointbalances", + "DROP TABLE IF EXISTS catchpointassetcreators", + "DROP TABLE IF EXISTS catchpointaccounthashes", + "DELETE FROM accounttotals where id='catchpointStaging'", + } + if newCatchup { - s += "CREATE TABLE IF NOT EXISTS catchpointassetcreators(asset integer primary key, creator blob, ctype integer);" - s += "CREATE TABLE IF NOT EXISTS catchpointbalances(address blob primary key, data blob);" - s += "CREATE TABLE IF NOT EXISTS catchpointaccounthashes(id integer primary key, data blob);" + // SQLite has no way to rename an existing index. So, we need + // to cook up a fresh name for the index, which will be kept + // around after we rename the table from "catchpointbalances" + // to "accountbase". To construct a unique index name, we + // use the current time. + idxname := fmt.Sprintf("onlineaccountbals%d", time.Now().UnixNano()) + + s = append(s, + "CREATE TABLE IF NOT EXISTS catchpointassetcreators (asset integer primary key, creator blob, ctype integer)", + "CREATE TABLE IF NOT EXISTS catchpointbalances (address blob primary key, data blob, normalizedonlinebalance integer)", + "CREATE TABLE IF NOT EXISTS catchpointaccounthashes (id integer primary key, data blob)", + createNormalizedOnlineBalanceIndex(idxname, "catchpointbalances"), + ) } - _, err = tx.Exec(s) - return err + + for _, stmt := range s { + _, err = tx.Exec(stmt) + if err != nil { + return err + } + } + + return nil } // applyCatchpointStagingBalances switches the staged catchpoint catchup tables onto the actual // tables and update the correct balance round. This is the final step in switching onto the new catchpoint round. func applyCatchpointStagingBalances(ctx context.Context, tx *sql.Tx, balancesRound basics.Round) (err error) { - s := "ALTER TABLE accountbase RENAME TO accountbase_old;" - s += "ALTER TABLE assetcreators RENAME TO assetcreators_old;" - s += "ALTER TABLE accounthashes RENAME TO accounthashes_old;" - s += "ALTER TABLE catchpointbalances RENAME TO accountbase;" - s += "ALTER TABLE catchpointassetcreators RENAME TO assetcreators;" - s += "ALTER TABLE catchpointaccounthashes RENAME TO accounthashes;" - s += "DROP TABLE IF EXISTS accountbase_old;" - s += "DROP TABLE IF EXISTS assetcreators_old;" - s += "DROP TABLE IF EXISTS accounthashes_old;" - - _, err = tx.Exec(s) - if err != nil { - return err + stmts := []string{ + "ALTER TABLE accountbase RENAME TO accountbase_old", + "ALTER TABLE assetcreators RENAME TO assetcreators_old", + "ALTER TABLE accounthashes RENAME TO accounthashes_old", + + "ALTER TABLE catchpointbalances RENAME TO accountbase", + "ALTER TABLE catchpointassetcreators RENAME TO assetcreators", + "ALTER TABLE catchpointaccounthashes RENAME TO accounthashes", + + "DROP TABLE IF EXISTS accountbase_old", + "DROP TABLE IF EXISTS assetcreators_old", + "DROP TABLE IF EXISTS accounthashes_old", + } + + for _, stmt := range stmts { + _, err = tx.Exec(stmt) + if err != nil { + return err + } } + _, err = tx.Exec("INSERT OR REPLACE INTO acctrounds(id, rnd) VALUES('acctbase', ?)", balancesRound) if err != nil { return err @@ -269,6 +316,76 @@ func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData return nil } +// accountsAddNormalizedBalance adds the normalizedonlinebalance column +// to the accountbase table. +func accountsAddNormalizedBalance(tx *sql.Tx, proto config.ConsensusParams) error { + var exists bool + err := tx.QueryRow("SELECT 1 FROM pragma_table_info('accountbase') WHERE name='normalizedonlinebalance'").Scan(&exists) + if err == nil { + // Already exists. + return nil + } + if err != sql.ErrNoRows { + return err + } + + for _, stmt := range createOnlineAccountIndex { + _, err := tx.Exec(stmt) + if err != nil { + return err + } + } + + rows, err := tx.Query("SELECT address, data FROM accountbase") + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var addrbuf []byte + var buf []byte + err = rows.Scan(&addrbuf, &buf) + if err != nil { + return err + } + + var data basics.AccountData + err = protocol.Decode(buf, &data) + if err != nil { + return err + } + + normBalance := data.NormalizedOnlineBalance(proto) + if normBalance > 0 { + _, err = tx.Exec("UPDATE accountbase SET normalizedonlinebalance=? WHERE address=?", normBalance, addrbuf) + if err != nil { + return err + } + } + } + + return rows.Err() +} + +// accountDataToOnline returns the part of the AccountData that matters +// for online accounts (to answer top-N queries). We store a subset of +// the full AccountData because we need to store a large number of these +// in memory (say, 1M), and storing that many AccountData could easily +// cause us to run out of memory. +func accountDataToOnline(address basics.Address, ad *basics.AccountData, proto config.ConsensusParams) *onlineAccount { + return &onlineAccount{ + Address: address, + MicroAlgos: ad.MicroAlgos, + RewardsBase: ad.RewardsBase, + NormalizedOnlineBalance: ad.NormalizedOnlineBalance(proto), + VoteID: ad.VoteID, + VoteFirstValid: ad.VoteFirstValid, + VoteLastValid: ad.VoteLastValid, + VoteKeyDilution: ad.VoteKeyDilution, + } +} + func resetAccountHashes(tx *sql.Tx) (err error) { _, err = tx.Exec(`DELETE FROM accounthashes`) return @@ -554,6 +671,50 @@ func (qs *accountsDbQueries) close() { } } +// accountsOnlineTop returns the top n online accounts starting at position offset +// (that is, the top offset'th account through the top offset+n-1'th account). +// +// The accounts are sorted by their normalized balance and address. The normalized +// balance has to do with the reward parts of online account balances. See the +// normalization procedure in AccountData.NormalizedOnlineBalance(). +// +// Note that this does not check if the accounts have a vote key valid for any +// particular round (past, present, or future). +func accountsOnlineTop(tx *sql.Tx, offset, n uint64, proto config.ConsensusParams) (map[basics.Address]*onlineAccount, error) { + rows, err := tx.Query("SELECT address, data FROM accountbase WHERE normalizedonlinebalance>0 ORDER BY normalizedonlinebalance DESC, address DESC LIMIT ? OFFSET ?", n, offset) + if err != nil { + return nil, err + } + defer rows.Close() + + res := make(map[basics.Address]*onlineAccount, n) + for rows.Next() { + var addrbuf []byte + var buf []byte + err = rows.Scan(&addrbuf, &buf) + if err != nil { + return nil, err + } + + var data basics.AccountData + err = protocol.Decode(buf, &data) + if err != nil { + return nil, err + } + + var addr basics.Address + if len(addrbuf) != len(addr) { + err = fmt.Errorf("Account DB address length mismatch: %d != %d", len(addrbuf), len(addr)) + return nil, err + } + + copy(addr[:], addrbuf) + res[addr] = accountDataToOnline(addr, &data, proto) + } + + return res, rows.Err() +} + func accountsAll(tx *sql.Tx) (bals map[basics.Address]basics.AccountData, err error) { rows, err := tx.Query("SELECT address, data FROM accountbase") if err != nil { @@ -619,7 +780,7 @@ func accountsPutTotals(tx *sql.Tx, totals AccountTotals, catchpointStaging bool) } // accountsNewRound updates the accountbase and assetcreators by applying the provided deltas to the accounts / creatables. -func accountsNewRound(tx *sql.Tx, updates map[basics.Address]accountDelta, creatables map[basics.CreatableIndex]modifiedCreatable) (err error) { +func accountsNewRound(tx *sql.Tx, updates map[basics.Address]accountDelta, creatables map[basics.CreatableIndex]modifiedCreatable, proto config.ConsensusParams) (err error) { var insertCreatableIdxStmt, deleteCreatableIdxStmt, deleteStmt, replaceStmt *sql.Stmt @@ -629,7 +790,7 @@ func accountsNewRound(tx *sql.Tx, updates map[basics.Address]accountDelta, creat } defer deleteStmt.Close() - replaceStmt, err = tx.Prepare("REPLACE INTO accountbase (address, data) VALUES (?, ?)") + replaceStmt, err = tx.Prepare("REPLACE INTO accountbase (address, normalizedonlinebalance, data) VALUES (?, ?, ?)") if err != nil { return } @@ -640,12 +801,12 @@ func accountsNewRound(tx *sql.Tx, updates map[basics.Address]accountDelta, creat // prune empty accounts _, err = deleteStmt.Exec(addr[:]) } else { - _, err = replaceStmt.Exec(addr[:], protocol.Encode(&data.new)) + normBalance := data.new.NormalizedOnlineBalance(proto) + _, err = replaceStmt.Exec(addr[:], normBalance, protocol.Encode(&data.new)) } if err != nil { return } - } if len(creatables) > 0 { diff --git a/ledger/accountdb_test.go b/ledger/accountdb_test.go index 9d9374847b..9e73b4b106 100644 --- a/ledger/accountdb_test.go +++ b/ledger/accountdb_test.go @@ -17,9 +17,11 @@ package ledger import ( + "bytes" "context" "database/sql" "fmt" + "sort" "testing" "github.com/stretchr/testify/require" @@ -52,7 +54,8 @@ func randomAccountData(rewardsLevel uint64) basics.AccountData { } data.RewardsBase = rewardsLevel - + data.VoteFirstValid = 0 + data.VoteLastValid = 1000 return data } @@ -335,6 +338,10 @@ func checkAccounts(t *testing.T, tx *sql.Tx, rnd basics.Round, accts map[basics. require.NoError(t, err) defer aq.close() + proto := config.Consensus[protocol.ConsensusCurrentVersion] + err = accountsAddNormalizedBalance(tx, proto) + require.NoError(t, err) + var totalOnline, totalOffline, totalNotPart uint64 for addr, data := range accts { @@ -369,6 +376,46 @@ func checkAccounts(t *testing.T, tx *sql.Tx, rnd basics.Round, accts map[basics. d, err := aq.lookup(randomAddress()) require.NoError(t, err) require.Equal(t, d, basics.AccountData{}) + + onlineAccounts := make(map[basics.Address]*onlineAccount) + for addr, data := range accts { + if data.Status == basics.Online { + onlineAccounts[addr] = accountDataToOnline(addr, &data, proto) + } + } + + for i := 0; i < len(onlineAccounts); i++ { + dbtop, err := accountsOnlineTop(tx, 0, uint64(i), proto) + require.NoError(t, err) + require.Equal(t, i, len(dbtop)) + + // Compute the top-N accounts ourselves + var testtop []onlineAccount + for _, data := range onlineAccounts { + testtop = append(testtop, *data) + } + + sort.Slice(testtop, func(i, j int) bool { + ibal := testtop[i].NormalizedOnlineBalance + jbal := testtop[j].NormalizedOnlineBalance + if ibal > jbal { + return true + } + if ibal < jbal { + return false + } + return bytes.Compare(testtop[i].Address[:], testtop[j].Address[:]) > 0 + }) + + for j := 0; j < i; j++ { + _, ok := dbtop[testtop[j].Address] + require.True(t, ok) + } + } + + top, err := accountsOnlineTop(tx, 0, uint64(len(onlineAccounts)+1), proto) + require.NoError(t, err) + require.Equal(t, len(top), len(onlineAccounts)) } func TestAccountDBInit(t *testing.T) { @@ -475,7 +522,7 @@ func TestAccountDBRound(t *testing.T) { accts = newaccts ctbsWithDeletes := randomCreatableSampling(i, ctbsList, randomCtbs, expectedDbImage, numElementsPerSegement) - err = accountsNewRound(tx, updates, ctbsWithDeletes) + err = accountsNewRound(tx, updates, ctbsWithDeletes, proto) require.NoError(t, err) err = totalsNewRounds(tx, []map[basics.Address]accountDelta{updates}, []AccountTotals{{}}, []config.ConsensusParams{proto}) require.NoError(t, err) diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index fc2928c036..78d05fe8b6 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -17,6 +17,7 @@ package ledger import ( + "container/heap" "context" "database/sql" "encoding/hex" @@ -400,6 +401,109 @@ func (au *accountUpdates) listCreatables(maxCreatableIdx basics.CreatableIndex, return res, nil } +// onlineTop returns the top n online accounts, sorted by their normalized +// balance and address, whose voting keys are valid in voteRnd. See the +// normalization description in AccountData.NormalizedOnlineBalance(). +func (au *accountUpdates) onlineTop(rnd basics.Round, voteRnd basics.Round, n uint64) ([]*onlineAccount, error) { + au.accountsMu.RLock() + defer au.accountsMu.RUnlock() + offset, err := au.roundOffset(rnd) + if err != nil { + return nil, err + } + + proto := au.protos[offset] + + // Determine how many accounts have been modified in-memory, + // so that we obtain enough top accounts from disk (accountdb). + // If the *onlineAccount is nil, that means the account is offline + // as of the most recent change to that account, or its vote key + // is not valid in voteRnd. Otherwise, the *onlineAccount is the + // representation of the most recent state of the account, and it + // is online and can vote in voteRnd. + modifiedAccounts := make(map[basics.Address]*onlineAccount) + for o := uint64(0); o < offset; o++ { + for addr, d := range au.deltas[o] { + if d.new.Status != basics.Online { + modifiedAccounts[addr] = nil + continue + } + + if !(d.new.VoteFirstValid <= voteRnd && voteRnd <= d.new.VoteLastValid) { + modifiedAccounts[addr] = nil + continue + } + + modifiedAccounts[addr] = accountDataToOnline(addr, &d.new, proto) + } + } + + // Build up a set of candidate accounts. Start by loading the + // top N + len(modifiedAccounts) accounts from disk (accountdb). + // This ensures that, even if the worst case if all in-memory + // changes are deleting the top accounts in accountdb, we still + // will have top N left. + // + // Keep asking for more accounts until we get the desired number, + // or there are no more accounts left. + candidates := make(map[basics.Address]*onlineAccount) + batchOffset := uint64(0) + batchSize := uint64(1024) + for uint64(len(candidates)) < n+uint64(len(modifiedAccounts)) { + var accts map[basics.Address]*onlineAccount + err = au.dbs.rdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { + accts, err = accountsOnlineTop(tx, batchOffset, batchSize, proto) + return + }) + if err != nil { + return nil, err + } + + for addr, data := range accts { + if !(data.VoteFirstValid <= voteRnd && voteRnd <= data.VoteLastValid) { + continue + } + candidates[addr] = data + } + + // If we got fewer than batchSize accounts, there are no + // more accounts to look at. + if uint64(len(accts)) < batchSize { + break + } + + batchOffset += batchSize + } + + // Now update the candidates based on the in-memory deltas. + for addr, oa := range modifiedAccounts { + if oa == nil { + delete(candidates, addr) + } else { + candidates[addr] = oa + } + } + + // Get the top N accounts from the candidate set, by inserting all of + // the accounts into a heap and then pulling out N elements from the + // heap. + topHeap := &onlineTopHeap{ + accts: nil, + } + + for _, data := range candidates { + heap.Push(topHeap, data) + } + + var res []*onlineAccount + for topHeap.Len() > 0 && uint64(len(res)) < n { + acct := heap.Pop(topHeap).(*onlineAccount) + res = append(res, acct) + } + + return res, nil +} + // GetLastCatchpointLabel retrieves the last catchpoint label that was stored to the database. func (au *accountUpdates) GetLastCatchpointLabel() string { au.accountsMu.RLock() @@ -838,6 +942,12 @@ func (au *accountUpdates) accountsInitialize(ctx context.Context, tx *sql.Tx) (b au.log.Warnf("accountsInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 2 : %v", err) return 0, err } + case 3: + dbVersion, err = au.upgradeDatabaseSchema3(ctx, tx) + if err != nil { + au.log.Warnf("accountsInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 3 : %v", err) + return 0, err + } default: return 0, fmt.Errorf("accountsInitialize unable to upgrade database from schema version %d", dbVersion) } @@ -1074,6 +1184,22 @@ func (au *accountUpdates) upgradeDatabaseSchema2(ctx context.Context, tx *sql.Tx return 3, nil } +// upgradeDatabaseSchema3 upgrades the database schema from version 3 to version 4, +// adding the normalizedonlinebalance column to the accountbase table. +func (au *accountUpdates) upgradeDatabaseSchema3(ctx context.Context, tx *sql.Tx) (updatedDBVersion int32, err error) { + err = accountsAddNormalizedBalance(tx, au.ledger.GenesisProto()) + if err != nil { + return 0, err + } + + // update version + _, err = db.SetUserVersion(ctx, tx, 4) + if err != nil { + return 0, fmt.Errorf("accountsInitialize unable to update database schema version from 3 to 4: %v", err) + } + return 4, nil +} + // deleteStoredCatchpoints iterates over the storedcatchpoints table and deletes all the files stored on disk. // once all the files have been deleted, it would go ahead and remove the entries from the table. func (au *accountUpdates) deleteStoredCatchpoints(ctx context.Context, dbQueries *accountsDbQueries) (err error) { @@ -1418,6 +1544,8 @@ func (au *accountUpdates) commitRound(offset uint64, dbRound basics.Round, lookb beforeUpdatingBalancesTime := time.Now() var trieBalancesHash crypto.Digest + genesisProto := au.ledger.GenesisProto() + err := au.dbs.wdb.AtomicCommitWriteLock(func(ctx context.Context, tx *sql.Tx) (err error) { treeTargetRound := basics.Round(0) if au.catchpointInterval > 0 { @@ -1438,7 +1566,7 @@ func (au *accountUpdates) commitRound(offset uint64, dbRound basics.Round, lookb treeTargetRound = dbRound + basics.Round(offset) } for i := uint64(0); i < offset; i++ { - err = accountsNewRound(tx, deltas[i], creatableDeltas[i]) + err = accountsNewRound(tx, deltas[i], creatableDeltas[i], genesisProto) if err != nil { return err } diff --git a/ledger/acctupdates_test.go b/ledger/acctupdates_test.go index ff198f30f1..fc0c7789d3 100644 --- a/ledger/acctupdates_test.go +++ b/ledger/acctupdates_test.go @@ -110,6 +110,13 @@ func (ml *mockLedgerForTracker) GenesisHash() crypto.Digest { return crypto.Digest{} } +func (ml *mockLedgerForTracker) GenesisProto() config.ConsensusParams { + if len(ml.blocks) > 0 { + return config.Consensus[ml.blocks[0].block.CurrentProtocol] + } + return config.Consensus[protocol.ConsensusCurrentVersion] +} + // this function used to be in acctupdates.go, but we were never using it for production purposes. This // function has a conceptual flaw in that it attempts to load the entire balances into memory. This might // not work if we have large number of balances. On these unit testing, however, it's not the case, and it's @@ -978,6 +985,9 @@ func TestListCreatables(t *testing.T) { err = accountsInit(tx, accts, proto) require.NoError(t, err) + err = accountsAddNormalizedBalance(tx, proto) + require.NoError(t, err) + au := &accountUpdates{} au.accountsq, err = accountsDbInit(tx, tx) require.NoError(t, err) @@ -997,7 +1007,7 @@ func TestListCreatables(t *testing.T) { // ******* No deletes ******* // sync with the database var updates map[basics.Address]accountDelta - err = accountsNewRound(tx, updates, ctbsWithDeletes) + err = accountsNewRound(tx, updates, ctbsWithDeletes, proto) require.NoError(t, err) // nothing left in cache au.creatables = make(map[basics.CreatableIndex]modifiedCreatable) @@ -1013,7 +1023,7 @@ func TestListCreatables(t *testing.T) { // ******* Results are obtained from the database and from the cache ******* // ******* Deletes are in the database and in the cache ******* // sync with the database. This has deletes synced to the database. - err = accountsNewRound(tx, updates, au.creatables) + err = accountsNewRound(tx, updates, au.creatables, proto) require.NoError(t, err) // get new creatables in the cache. There will be deletes in the cache from the previous batch. au.creatables = randomCreatableSampling(3, ctbsList, randomCtbs, diff --git a/ledger/archival_test.go b/ledger/archival_test.go index ddb32b4957..beb9d51d78 100644 --- a/ledger/archival_test.go +++ b/ledger/archival_test.go @@ -89,6 +89,10 @@ func (wl *wrappedLedger) GenesisHash() crypto.Digest { return wl.l.GenesisHash() } +func (wl *wrappedLedger) GenesisProto() config.ConsensusParams { + return wl.l.GenesisProto() +} + func getInitState() (genesisInitState InitState) { blk := bookkeeping.Block{} blk.CurrentProtocol = protocol.ConsensusCurrentVersion diff --git a/ledger/catchpointwriter_test.go b/ledger/catchpointwriter_test.go index b9672da107..41d0bb5ed6 100644 --- a/ledger/catchpointwriter_test.go +++ b/ledger/catchpointwriter_test.go @@ -269,7 +269,9 @@ func TestFullCatchpointWriter(t *testing.T) { require.NoError(t, err) // create a ledger. - l, err := OpenLedger(ml.log, "TestFullCatchpointWriter", true, InitState{}, conf) + var initState InitState + initState.Block.CurrentProtocol = protocol.ConsensusCurrentVersion + l, err := OpenLedger(ml.log, "TestFullCatchpointWriter", true, initState, conf) require.NoError(t, err) defer l.Close() accessor := MakeCatchpointCatchupAccessor(l, l.log) diff --git a/ledger/catchupaccessor.go b/ledger/catchupaccessor.go index af7236b180..7c5e2be602 100644 --- a/ledger/catchupaccessor.go +++ b/ledger/catchupaccessor.go @@ -290,6 +290,7 @@ func (c *CatchpointCatchupAccessorImpl) processStagingBalances(ctx context.Conte return fmt.Errorf("processStagingBalances received a chunk with no accounts") } + proto := c.ledger.GenesisProto() wdb := c.ledger.trackerDB().wdb err = wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { // create the merkle trie for the balances @@ -308,7 +309,7 @@ func (c *CatchpointCatchupAccessorImpl) processStagingBalances(ctx context.Conte progress.cachedTrie.SetCommitter(mc) } - err = writeCatchpointStagingBalances(ctx, tx, balances.Balances) + err = writeCatchpointStagingBalances(ctx, tx, balances.Balances, proto) if err != nil { return } diff --git a/ledger/ledger.go b/ledger/ledger.go index 8c6181c93a..f7d6c82bb4 100644 --- a/ledger/ledger.go +++ b/ledger/ledger.go @@ -546,6 +546,11 @@ func (l *Ledger) GenesisHash() crypto.Digest { return l.genesisHash } +// GenesisProto returns the initial protocol for this ledger. +func (l *Ledger) GenesisProto() config.ConsensusParams { + return l.genesisProto +} + // GetCatchpointCatchupState returns the current state of the catchpoint catchup. func (l *Ledger) GetCatchpointCatchupState(ctx context.Context) (state CatchpointCatchupState, err error) { return MakeCatchpointCatchupAccessor(l, l.log).GetState(ctx) diff --git a/ledger/onlineacct.go b/ledger/onlineacct.go new file mode 100644 index 0000000000..489d8624cc --- /dev/null +++ b/ledger/onlineacct.go @@ -0,0 +1,89 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package ledger + +import ( + "bytes" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" +) + +// An onlineAccount corresponds to an account whose AccountData.Status +// is Online. This is used for a Merkle tree commitment of online +// accounts, which is subsequently used to validate participants for +// a compact certificate. +type onlineAccount struct { + // These are a subset of the fields from the corresponding AccountData. + Address basics.Address + MicroAlgos basics.MicroAlgos + RewardsBase uint64 + NormalizedOnlineBalance uint64 + VoteID crypto.OneTimeSignatureVerifier + VoteFirstValid basics.Round + VoteLastValid basics.Round + VoteKeyDilution uint64 +} + +// onlineTopHeap implements heap.Interface for tracking top N online accounts. +type onlineTopHeap struct { + accts []*onlineAccount +} + +// Len implements sort.Interface +func (h *onlineTopHeap) Len() int { + return len(h.accts) +} + +// Less implements sort.Interface +func (h *onlineTopHeap) Less(i, j int) bool { + // For the heap, "less" means the element is returned earlier by Pop(), + // so we actually implement "greater-than" here. + ibal := h.accts[i].NormalizedOnlineBalance + jbal := h.accts[j].NormalizedOnlineBalance + + if ibal > jbal { + return true + } + if ibal < jbal { + return false + } + + bcmp := bytes.Compare(h.accts[i].Address[:], h.accts[j].Address[:]) + if bcmp > 0 { + return true + } + + return false +} + +// Swap implements sort.Interface +func (h *onlineTopHeap) Swap(i, j int) { + h.accts[i], h.accts[j] = h.accts[j], h.accts[i] +} + +// Push implements heap.Interface +func (h *onlineTopHeap) Push(x interface{}) { + h.accts = append(h.accts, x.(*onlineAccount)) +} + +// Pop implements heap.Interface +func (h *onlineTopHeap) Pop() interface{} { + res := h.accts[len(h.accts)-1] + h.accts = h.accts[:len(h.accts)-1] + return res +} diff --git a/ledger/tracker.go b/ledger/tracker.go index 91cabfe260..f756139217 100644 --- a/ledger/tracker.go +++ b/ledger/tracker.go @@ -20,6 +20,7 @@ import ( "fmt" "reflect" + "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" @@ -92,6 +93,7 @@ type ledgerForTracker interface { Block(basics.Round) (bookkeeping.Block, error) BlockHdr(basics.Round) (bookkeeping.BlockHeader, error) GenesisHash() crypto.Digest + GenesisProto() config.ConsensusParams } type trackerRegistry struct { From b741465b01987db5c75b452d5d765a80721e04a6 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Tue, 8 Sep 2020 23:04:03 -0400 Subject: [PATCH 026/136] Add a benchmark for writeCatchpointStagingBalances (#1491) Add a benchmark for the db function writeCatchpointStagingBalances --- ledger/accountdb_test.go | 88 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/ledger/accountdb_test.go b/ledger/accountdb_test.go index 9e73b4b106..6a835e682d 100644 --- a/ledger/accountdb_test.go +++ b/ledger/accountdb_test.go @@ -21,14 +21,18 @@ import ( "context" "database/sql" "fmt" + "os" "sort" + "strings" "testing" + "time" "github.com/stretchr/testify/require" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" ) @@ -592,7 +596,7 @@ func randomCreatableSampling(iteration int, crtbsList []basics.CreatableIndex, ctb := creatables[crtbsList[i]] if ctb.created && // Always delete the first element, to make sure at least one - // element is always deleted. + // element is always deleted. (i == delSegmentStart || 1 == (crypto.RandUint64()%2)) { ctb.created = false newSample[crtbsList[i]] = ctb @@ -832,3 +836,85 @@ func TestAccountsDbQueriesCreateClose(t *testing.T) { qs.close() require.Nil(t, qs.listCreatablesStmt) } + +func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B) { + proto := config.Consensus[protocol.ConsensusCurrentVersion] + genesisInitState, _ := testGenerateInitState(b, protocol.ConsensusCurrentVersion) + const inMem = false + log := logging.TestingLog(b) + cfg := config.GetDefaultLocal() + cfg.Archival = false + log.SetLevel(logging.Warn) + dbBaseFileName := strings.Replace(b.Name(), "/", "_", -1) + l, err := OpenLedger(log, dbBaseFileName, inMem, genesisInitState, cfg) + require.NoError(b, err, "could not open ledger") + defer func() { + l.Close() + os.Remove(dbBaseFileName + ".block.sqlite") + os.Remove(dbBaseFileName + ".tracker.sqlite") + }() + catchpointAccessor := MakeCatchpointCatchupAccessor(l, log) + catchpointAccessor.ResetStagingBalances(context.Background(), true) + targetAccountsCount := uint64(b.N) + accountsLoaded := uint64(0) + var last64KStart time.Time + last64KSize := uint64(0) + last64KAccountCreationTime := time.Duration(0) + accountsWritingStarted := time.Now() + accountsGenerationDuration := time.Duration(0) + b.ResetTimer() + for accountsLoaded < targetAccountsCount { + b.StopTimer() + balancesLoopStart := time.Now() + // generate a chunk; + chunkSize := targetAccountsCount - accountsLoaded + if chunkSize > BalancesPerCatchpointFileChunk { + chunkSize = BalancesPerCatchpointFileChunk + } + last64KSize += chunkSize + if accountsLoaded >= targetAccountsCount-64*1024 && last64KStart.IsZero() { + last64KStart = time.Now() + last64KSize = chunkSize + last64KAccountCreationTime = time.Duration(0) + } + var balances catchpointFileBalancesChunk + balances.Balances = make([]encodedBalanceRecord, chunkSize) + for i := uint64(0); i < chunkSize; i++ { + var randomAccount encodedBalanceRecord + accountData := basics.AccountData{RewardsBase: accountsLoaded + i} + accountData.MicroAlgos.Raw = crypto.RandUint63() + randomAccount.AccountData = protocol.Encode(&accountData) + crypto.RandBytes(randomAccount.Address[:]) + balances.Balances[i] = randomAccount + } + balanceLoopDuration := time.Now().Sub(balancesLoopStart) + last64KAccountCreationTime += balanceLoopDuration + accountsGenerationDuration += balanceLoopDuration + b.StartTimer() + err = l.trackerDBs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { + err = writeCatchpointStagingBalances(ctx, tx, balances.Balances, proto) + return + }) + + require.NoError(b, err) + accountsLoaded += chunkSize + } + if !last64KStart.IsZero() { + last64KDuration := time.Now().Sub(last64KStart) - last64KAccountCreationTime + fmt.Printf("%-74s%-7d (last 64k) %-6d ns/account %d accounts/sec\n", b.Name(), last64KSize, (last64KDuration / time.Duration(last64KSize)).Nanoseconds(), int(float64(last64KSize)/float64(last64KDuration.Seconds()))) + } + stats, err := l.trackerDBs.wdb.Vacuum(context.Background()) + require.NoError(b, err) + fmt.Printf("%-74sdb fragmentation %.1f%%\n", b.Name(), float32(stats.PagesBefore-stats.PagesAfter)*100/float32(stats.PagesBefore)) + b.ReportMetric(float64(b.N)/float64((time.Now().Sub(accountsWritingStarted)-accountsGenerationDuration).Seconds()), "accounts/sec") +} + +func BenchmarkWriteCatchpointStagingBalances(b *testing.B) { + benchSizes := []int{1024 * 100, 1024 * 200, 1024 * 400} + for _, size := range benchSizes { + b.Run(fmt.Sprintf("Restore-%d", size), func(b *testing.B) { + b.N = size + benchmarkWriteCatchpointStagingBalancesSub(b) + }) + } +} From c58f4e919ef325ad1f0b25eb6de6f92c29ca562b Mon Sep 17 00:00:00 2001 From: algonautshant <55754073+algonautshant@users.noreply.github.com> Date: Wed, 9 Sep 2020 11:33:47 -0400 Subject: [PATCH 027/136] Fix e2e test : removing close from eof pattern match (#1493) Fix e2e test : avoid calling close redundently --- .../e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp b/test/e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp index 25378849f0..1f912fa51c 100644 --- a/test/e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp +++ b/test/e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp @@ -107,7 +107,7 @@ proc goalAppAccountAddress { TEST_ALGO_DIR TEST_DATA_DIR} { expect { timeout { puts timeout; ::AlgorandGoal::Abort "\n Failed to see expected output" } "*committed in round*" {puts "app call successful"; close} - eof {close; ::AlgorandGoal::Abort "app call failed" } + eof {::AlgorandGoal::Abort "app call failed" } } puts "Checking the results" @@ -124,18 +124,17 @@ proc goalAppAccountAddress { TEST_ALGO_DIR TEST_DATA_DIR} { expect { timeout { puts timeout; ::AlgorandGoal::Abort "\n Failed to see expected output" } "*$EXPECTED_OUTPUT*" {puts "Local state read correctly"; close} - eof {close; ::AlgorandGoal::Abort "app read failed" } + eof {::AlgorandGoal::Abort "app read failed" } } # check the local state of account 2 spawn goal app read --app-id $APP_ID --local --guess-format \ --from $ACCOUNT_2_ADDRESS -w $WALLET_1_NAME -d $TEST_PRIMARY_NODE_DIR - expect { timeout { puts timeout; ::AlgorandGoal::Abort "\n Failed to see expected output" } "Please enter the password for wallet '$WALLET_1_NAME':" {send "$WALLET_1_PASSWORD\r" ; exp_continue} "*$EXPECTED_OUTPUT*" {puts "Local state read correctly"; close} - eof {close; ::AlgorandGoal::Abort "app read failed" } + eof {::AlgorandGoal::Abort "app read failed" } } # call the app with a missing app-account. It should fail @@ -148,7 +147,7 @@ proc goalAppAccountAddress { TEST_ALGO_DIR TEST_DATA_DIR} { timeout { puts timeout; ::AlgorandGoal::Abort "\n Failed to see expected output" } "*Couldn't broadcast tx with algod: HTTP 400 Bad Request: TransactionPool.Remember: transaction*invalid Accounts index 4*" \ {puts "Error received successfully "; close} - eof {close; ::AlgorandGoal::Abort "failed to get the expected error" } + eof {::AlgorandGoal::Abort "failed to get the expected error" } } # Shutdown the network From bdc298ca1b2c84e5b285f3cf058f6cf1c7ade832 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Wed, 9 Sep 2020 15:29:45 -0400 Subject: [PATCH 028/136] Improve BenchmarkWriteCatchpointStagingBalances benchmark - compared ordered/random insertions (#1494) Improve db insertion benchmark - add ordered insertions --- ledger/accountdb_test.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/ledger/accountdb_test.go b/ledger/accountdb_test.go index 6a835e682d..3aa00d69c9 100644 --- a/ledger/accountdb_test.go +++ b/ledger/accountdb_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "database/sql" + "encoding/binary" "fmt" "os" "sort" @@ -837,7 +838,7 @@ func TestAccountsDbQueriesCreateClose(t *testing.T) { require.Nil(t, qs.listCreatablesStmt) } -func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B) { +func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B, ascendingOrder bool) { proto := config.Consensus[protocol.ConsensusCurrentVersion] genesisInitState, _ := testGenerateInitState(b, protocol.ConsensusCurrentVersion) const inMem = false @@ -885,6 +886,9 @@ func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B) { accountData.MicroAlgos.Raw = crypto.RandUint63() randomAccount.AccountData = protocol.Encode(&accountData) crypto.RandBytes(randomAccount.Address[:]) + if ascendingOrder { + binary.LittleEndian.PutUint64(randomAccount.Address[:], accountsLoaded+i) + } balances.Balances[i] = randomAccount } balanceLoopDuration := time.Now().Sub(balancesLoopStart) @@ -901,20 +905,26 @@ func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B) { } if !last64KStart.IsZero() { last64KDuration := time.Now().Sub(last64KStart) - last64KAccountCreationTime - fmt.Printf("%-74s%-7d (last 64k) %-6d ns/account %d accounts/sec\n", b.Name(), last64KSize, (last64KDuration / time.Duration(last64KSize)).Nanoseconds(), int(float64(last64KSize)/float64(last64KDuration.Seconds()))) + fmt.Printf("%-82s%-7d (last 64k) %-6d ns/account %d accounts/sec\n", b.Name(), last64KSize, (last64KDuration / time.Duration(last64KSize)).Nanoseconds(), int(float64(last64KSize)/float64(last64KDuration.Seconds()))) } stats, err := l.trackerDBs.wdb.Vacuum(context.Background()) require.NoError(b, err) - fmt.Printf("%-74sdb fragmentation %.1f%%\n", b.Name(), float32(stats.PagesBefore-stats.PagesAfter)*100/float32(stats.PagesBefore)) + fmt.Printf("%-82sdb fragmentation %.1f%%\n", b.Name(), float32(stats.PagesBefore-stats.PagesAfter)*100/float32(stats.PagesBefore)) b.ReportMetric(float64(b.N)/float64((time.Now().Sub(accountsWritingStarted)-accountsGenerationDuration).Seconds()), "accounts/sec") } func BenchmarkWriteCatchpointStagingBalances(b *testing.B) { benchSizes := []int{1024 * 100, 1024 * 200, 1024 * 400} for _, size := range benchSizes { - b.Run(fmt.Sprintf("Restore-%d", size), func(b *testing.B) { + b.Run(fmt.Sprintf("RandomInsertOrder-%d", size), func(b *testing.B) { + b.N = size + benchmarkWriteCatchpointStagingBalancesSub(b, false) + }) + } + for _, size := range benchSizes { + b.Run(fmt.Sprintf("AscendingInsertOrder-%d", size), func(b *testing.B) { b.N = size - benchmarkWriteCatchpointStagingBalancesSub(b) + benchmarkWriteCatchpointStagingBalancesSub(b, true) }) } } From b238b965cff8f3f6b5583674c3ce1501657d0de3 Mon Sep 17 00:00:00 2001 From: algonautshant <55754073+algonautshant@users.noreply.github.com> Date: Thu, 10 Sep 2020 12:25:30 -0400 Subject: [PATCH 029/136] Adding ledger tests (#1495) Adding ledger unit tests to cover ledger.GetLastCatchpointLabel, ledger.ledger.ListAssets, ledger.ListApplications --- ledger/ledger_test.go | 136 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/ledger/ledger_test.go b/ledger/ledger_test.go index 2772f5f151..868907a5d6 100644 --- a/ledger/ledger_test.go +++ b/ledger/ledger_test.go @@ -1078,3 +1078,139 @@ func TestLedgerReload(t *testing.T) { } } } + +// TestGetLastCatchpointLabel tests ledger.GetLastCatchpointLabel is returning the correct value. +func TestGetLastCatchpointLabel(t *testing.T) { + + //initLedger + genesisInitState, _ := testGenerateInitState(t, protocol.ConsensusCurrentVersion) + const inMem = true + log := logging.TestingLog(t) + cfg := config.GetDefaultLocal() + cfg.Archival = true + ledger, err := OpenLedger(log, t.Name(), inMem, genesisInitState, cfg) + require.NoError(t, err, "could not open ledger") + defer ledger.Close() + + // set some value + lastCatchpointLabel := "someCatchpointLabel" + ledger.accts.lastCatchpointLabel = lastCatchpointLabel + + // verify the value is returned + require.Equal(t, lastCatchpointLabel, ledger.GetLastCatchpointLabel()) +} + +// generate at least 3 asset and 3 app creatables, and return the ids +// of the asset/app with at least 3 elements less or equal. +func generateCreatables(numElementsPerSegement int) ( + randomCtbs map[basics.CreatableIndex]modifiedCreatable, + assetID3, + appID3 basics.CreatableIndex, + err error) { + + _, randomCtbs = randomCreatables(numElementsPerSegement) + asCounter3 := 0 + apCounter3 := 0 + + for x := 0; x < 10; x++ { + // find the assetid greater than at least 2 assetids + for cID, crtble := range randomCtbs { + switch crtble.ctype { + case basics.AssetCreatable: + if assetID3 == 0 { + assetID3 = cID + continue + } + asCounter3++ + if assetID3 < cID { + assetID3 = cID + } + case basics.AppCreatable: + if appID3 == 0 { + appID3 = cID + continue + } + apCounter3++ + if appID3 < cID { + appID3 = cID + } + } + if asCounter3 >= 3 && apCounter3 >= 3 { + // found at least 3rd smallest of both + break + } + } + + // there should be at least 3 asset and 3 app creatables generated. + // In the rare event this does not happen, repeat... up to 10 times (x) + if asCounter3 >= 3 && apCounter3 >= 3 { + break + } + } + if asCounter3 < 3 && apCounter3 < 3 { + return nil, 0, 0, fmt.Errorf("could not generate 3 apps and 3 assets") + } + return +} + +// TestListAssetsAndApplications tests the ledger.ListAssets and ledger.ListApplications +// interfaces. The detailed test on the correctness of these functions is given in: +// TestListCreatables (acctupdates_test.go) +func TestListAssetsAndApplications(t *testing.T) { + + numElementsPerSegement := 10 // This is multiplied by 10. see randomCreatables + + //initLedger + genesisInitState, _ := testGenerateInitState(t, protocol.ConsensusCurrentVersion) + const inMem = true + log := logging.TestingLog(t) + cfg := config.GetDefaultLocal() + cfg.Archival = true + ledger, err := OpenLedger(log, t.Name(), inMem, genesisInitState, cfg) + require.NoError(t, err, "could not open ledger") + defer ledger.Close() + + // ******* All results are obtained from the cache. Empty database ******* + // ******* No deletes ******* + // get random data. Inital batch, no deletes + randomCtbs, maxAsset, maxApp, err := generateCreatables(numElementsPerSegement) + require.NoError(t, err) + + // set the cache + ledger.accts.creatables = randomCtbs + + // Test ListAssets + // Check the number of results limit + results, err := ledger.ListAssets(basics.AssetIndex(maxAsset), 2) + require.NoError(t, err) + require.Equal(t, 2, len(results)) + // Check the max asset id limit + results, err = ledger.ListAssets(basics.AssetIndex(maxAsset), 100) + assetCount := 0 + for id, ctb := range randomCtbs { + if ctb.ctype == basics.AssetCreatable && + ctb.created && + id <= maxAsset { + assetCount++ + } + } + require.Equal(t, assetCount, len(results)) + + // Test ListApplications + // Check the number of results limit + ledger.accts.creatables = randomCtbs + results, err = ledger.ListApplications(basics.AppIndex(maxApp), 2) + require.NoError(t, err) + require.Equal(t, 2, len(results)) + // Check the max application id limit + results, err = ledger.ListApplications(basics.AppIndex(maxApp), 100) + appCount := 0 + for id, ctb := range randomCtbs { + if ctb.ctype == basics.AppCreatable && + ctb.created && + id <= maxApp { + appCount++ + } + } + require.Equal(t, appCount, len(results)) +} From 5c6b67da56e4f931d612b228722b3a8c27c73398 Mon Sep 17 00:00:00 2001 From: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> Date: Thu, 10 Sep 2020 17:14:16 -0400 Subject: [PATCH 030/136] Add txn.Accounts and txn.ApplicationArgs support to tealdbg (#1497) * Display addresses as standard base32 strings rather than hex-encoded blobs * Add previews for array fields * Fix logic.TxnFieldToTealValue test --- cmd/tealdbg/cdtState.go | 125 ++++++++++++++++++++++++++- data/transactions/logic/eval.go | 4 +- data/transactions/logic/eval_test.go | 32 ++++++- 3 files changed, 156 insertions(+), 5 deletions(-) diff --git a/cmd/tealdbg/cdtState.go b/cmd/tealdbg/cdtState.go index f78a3c3251..33e65b2d5e 100644 --- a/cmd/tealdbg/cdtState.go +++ b/cmd/tealdbg/cdtState.go @@ -66,6 +66,29 @@ type cdtStateUpdate struct { appState } +type typeHint int + +const ( + noHint typeHint = iota + addressHint +) + +var txnFileTypeHints = map[logic.TxnField]typeHint{ + logic.Sender: addressHint, + logic.Receiver: addressHint, + logic.CloseRemainderTo: addressHint, + logic.AssetSender: addressHint, + logic.AssetReceiver: addressHint, + logic.AssetCloseTo: addressHint, + logic.Accounts: addressHint, + logic.RekeyTo: addressHint, + logic.ConfigAssetManager: addressHint, + logic.ConfigAssetReserve: addressHint, + logic.ConfigAssetFreeze: addressHint, + logic.ConfigAssetClawback: addressHint, + logic.FreezeAssetAccount: addressHint, +} + func (s *cdtState) Init(disassembly string, proto *config.ConsensusParams, txnGroup []transactions.SignedTxn, groupIndex int, globals []basics.TealValue) { s.disassembly = disassembly s.proto = proto @@ -95,6 +118,7 @@ const scratchObjID = "scratchObjID" const tealErrorID = "tealErrorID" const appGlobalObjID = "appGlobalObjID" const appLocalsObjID = "appLocalsObjID" +const txnArrayFieldObjID = "txnArrayField" type objectDescFn func(s *cdtState, preview bool) []RuntimePropertyDescriptor @@ -144,6 +168,8 @@ func (s *cdtState) getObjectDescriptor(objID string, preview bool) (desc []Runti return makeAppLocalsKV(s, addr, appID), nil } else if addr, ok := decodeAppLocalsAddr(objID); ok { return makeAppLocalState(s, addr), nil + } else if idx, field, ok := decodeTxnArrayField(objID); ok { + return makeTxnArrayField(s, idx, field), nil } // might be nested object in array, parse and call err = fmt.Errorf("unk object id: %s", objID) @@ -340,12 +366,13 @@ func prepareTxn(txn *transactions.Transaction, groupIndex int) []fieldDesc { } var value string var valType string = "string" - tv, err := logic.TxnFieldToTealValue(txn, groupIndex, logic.TxnField(field)) + tv, err := logic.TxnFieldToTealValue(txn, groupIndex, logic.TxnField(field), 0) if err != nil { value = err.Error() valType = "undefined" } else { - value = tv.String() + hint := txnFileTypeHints[logic.TxnField(field)] + value = tealValueToString(&tv, hint) valType = tealTypeMap[tv.Type] } result = append(result, fieldDesc{name, value, valType}) @@ -381,6 +408,15 @@ func tealValueToFieldDesc(name string, tv basics.TealValue) fieldDesc { return fieldDesc{name, value, valType} } +func tealValueToString(tv *basics.TealValue, hint typeHint) string { + if hint == addressHint { + var a basics.Address + copy(a[:], []byte(tv.Bytes)) + return a.String() + } + return tv.String() +} + func prepareArray(array []basics.TealValue) []fieldDesc { result := make([]fieldDesc, 0, len(logic.TxnFieldNames)) for i := 0; i < len(array); i++ { @@ -567,6 +603,29 @@ func decodeAppLocalsAppID(objID string) (string, uint64, bool) { return "", 0, false } +var txnArrayFieldPrefix = fmt.Sprintf("%s__", txnArrayFieldObjID) + +func encodeTxnArrayField(groupIndex int, field int) string { + return fmt.Sprintf("%s%d_%d", txnArrayFieldPrefix, groupIndex, field) +} + +func decodeTxnArrayField(objID string) (int, int, bool) { + if strings.HasPrefix(objID, txnArrayFieldPrefix) { + encoded := objID[len(txnArrayFieldPrefix):] + parts := strings.Split(encoded, "_") + var groupIndex, fieldIndex int64 + var err error + if groupIndex, err = strconv.ParseInt(parts[0], 10, 32); err != nil { + return 0, 0, false + } + if fieldIndex, err = strconv.ParseInt(parts[1], 10, 32); err != nil { + return 0, 0, false + } + return int(groupIndex), int(fieldIndex), true + } + return 0, 0, false +} + func makeGlobalScope(s *cdtState, preview bool) (desc []RuntimePropertyDescriptor) { globals := makeObject("globals", globalsObjID) if preview { @@ -652,6 +711,68 @@ func makeTxnImpl(txn *transactions.Transaction, groupIndex int, preview bool) (d for _, field := range fields { desc = append(desc, makePrimitive(field)) } + + for _, fieldIdx := range []logic.TxnField{logic.ApplicationArgs, logic.Accounts} { + fieldID := encodeTxnArrayField(groupIndex, int(fieldIdx)) + var length int + switch logic.TxnField(fieldIdx) { + case logic.Accounts: + length = len(txn.Accounts) + case logic.ApplicationArgs: + length = len(txn.ApplicationArgs) + } + field := makeArray(logic.TxnFieldNames[fieldIdx], length, fieldID) + if preview { + elems := txnFieldToArrayFieldDesc(txn, groupIndex, logic.TxnField(fieldIdx), length) + prop := makePreview(elems) + p := RuntimeObjectPreview{ + Type: "object", + Subtype: "array", + Description: fmt.Sprintf("Array(%d)", length), + Overflow: true, + Properties: prop, + } + field.Value.Preview = &p + } + desc = append(desc, field) + } + + return +} + +func txnFieldToArrayFieldDesc(txn *transactions.Transaction, groupIndex int, field logic.TxnField, length int) (desc []fieldDesc) { + for i := 0; i < length; i++ { + tv, err := logic.TxnFieldToTealValue(txn, groupIndex, field, uint64(i)) + if err != nil { + return []fieldDesc{} + } + name := strconv.Itoa(i) + hint := txnFileTypeHints[field] + value := tealValueToString(&tv, hint) + valType := tealTypeMap[tv.Type] + desc = append(desc, fieldDesc{name, value, valType}) + } + return +} + +func makeTxnArrayField(s *cdtState, groupIndex int, fieldIdx int) (desc []RuntimePropertyDescriptor) { + if len(s.txnGroup) > 0 && s.groupIndex < len(s.txnGroup) && s.groupIndex >= 0 && fieldIdx >= 0 && fieldIdx < len(logic.TxnFieldNames) { + txn := s.txnGroup[groupIndex].Txn + var length int + switch logic.TxnField(fieldIdx) { + case logic.Accounts: + length = len(txn.Accounts) + case logic.ApplicationArgs: + length = len(txn.ApplicationArgs) + } + + elems := txnFieldToArrayFieldDesc(&txn, groupIndex, logic.TxnField(fieldIdx), length) + for _, elem := range elems { + desc = append(desc, makePrimitive(elem)) + } + + desc = append(desc, makePrimitive(fieldDesc{Name: "length", Value: strconv.Itoa(length), Type: "number"})) + } return } diff --git a/data/transactions/logic/eval.go b/data/transactions/logic/eval.go index fee178a6d5..159e25c6df 100644 --- a/data/transactions/logic/eval.go +++ b/data/transactions/logic/eval.go @@ -1249,9 +1249,9 @@ func (cx *evalContext) assetParamsEnumToValue(params *basics.AssetParams, field } // TxnFieldToTealValue is a thin wrapper for txnFieldToStack for external use -func TxnFieldToTealValue(txn *transactions.Transaction, groupIndex int, field TxnField) (basics.TealValue, error) { +func TxnFieldToTealValue(txn *transactions.Transaction, groupIndex int, field TxnField, arrayFieldIdx uint64) (basics.TealValue, error) { cx := evalContext{EvalParams: EvalParams{GroupIndex: groupIndex}} - sv, err := cx.txnFieldToStack(txn, field, 0, groupIndex) + sv, err := cx.txnFieldToStack(txn, field, arrayFieldIdx, groupIndex) return sv.toTealValue(), err } diff --git a/data/transactions/logic/eval_test.go b/data/transactions/logic/eval_test.go index 1446e13958..80d9ee6431 100644 --- a/data/transactions/logic/eval_test.go +++ b/data/transactions/logic/eval_test.go @@ -139,11 +139,41 @@ func TestTxnFieldToTealValue(t *testing.T) { for _, value := range values { txn.FirstValid = basics.Round(value) - tealValue, err := TxnFieldToTealValue(&txn, groupIndex, field) + tealValue, err := TxnFieldToTealValue(&txn, groupIndex, field, 0) require.NoError(t, err) require.Equal(t, basics.TealUintType, tealValue.Type) require.Equal(t, value, tealValue.Uint) } + + // check arrayFieldIdx is ignored for non-arrays + field = FirstValid + value := uint64(1) + txn.FirstValid = basics.Round(value) + tealValue, err := TxnFieldToTealValue(&txn, groupIndex, field, 10) + require.NoError(t, err) + require.Equal(t, basics.TealUintType, tealValue.Type) + require.Equal(t, value, tealValue.Uint) + + // check arrayFieldIdx is taken into account for arrays + field = Accounts + sender := basics.Address{} + addr, _ := basics.UnmarshalChecksumAddress("DFPKC2SJP3OTFVJFMCD356YB7BOT4SJZTGWLIPPFEWL3ZABUFLTOY6ILYE") + txn.Accounts = []basics.Address{addr} + tealValue, err = TxnFieldToTealValue(&txn, groupIndex, field, 0) + require.NoError(t, err) + require.Equal(t, basics.TealBytesType, tealValue.Type) + require.Equal(t, string(sender[:]), tealValue.Bytes) + + tealValue, err = TxnFieldToTealValue(&txn, groupIndex, field, 1) + require.NoError(t, err) + require.Equal(t, basics.TealBytesType, tealValue.Type) + require.Equal(t, string(addr[:]), tealValue.Bytes) + + tealValue, err = TxnFieldToTealValue(&txn, groupIndex, field, 100) + require.Error(t, err) + require.Equal(t, basics.TealUintType, tealValue.Type) + require.Equal(t, uint64(0), tealValue.Uint) + require.Equal(t, "", tealValue.Bytes) } func TestWrongProtoVersion(t *testing.T) { From b1a884c2bc90be630711ae8fa219ee93ebb0d44a Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Thu, 10 Sep 2020 17:28:31 -0400 Subject: [PATCH 031/136] Improve RestoringFromCatchpointFile benchmark precision (#1499) Improve RestoringFromCatchpointFile benchmark precision --- ledger/catchupaccessor_test.go | 35 +++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/ledger/catchupaccessor_test.go b/ledger/catchupaccessor_test.go index 75c3a2d93b..7c4c279bab 100644 --- a/ledger/catchupaccessor_test.go +++ b/ledger/catchupaccessor_test.go @@ -18,6 +18,7 @@ package ledger import ( "context" + "encoding/binary" "fmt" "os" "strings" @@ -41,6 +42,9 @@ func benchmarkRestoringFromCatchpointFileHelper(b *testing.B) { cfg.Archival = false log.SetLevel(logging.Warn) dbBaseFileName := strings.Replace(b.Name(), "/", "_", -1) + // delete database files, in case they were left there by previous iterations of this test. + os.Remove(dbBaseFileName + ".block.sqlite") + os.Remove(dbBaseFileName + ".tracker.sqlite") l, err := OpenLedger(log, dbBaseFileName, inMem, genesisInitState, cfg) require.NoError(b, err, "could not open ledger") defer func() { @@ -68,17 +72,18 @@ func benchmarkRestoringFromCatchpointFileHelper(b *testing.B) { err = catchpointAccessor.ProgressStagingBalances(context.Background(), "content.msgpack", encodedFileHeader, &progress) require.NoError(b, err) - b.ResetTimer() + // pre-create all encoded chunks. accounts := uint64(0) - var last64KStart time.Time + encodedAccountChunks := make([][]byte, 0, accountsCount/BalancesPerCatchpointFileChunk+1) + last64KIndex := -1 for accounts < accountsCount { // generate a chunk; chunkSize := accountsCount - accounts if chunkSize > BalancesPerCatchpointFileChunk { chunkSize = BalancesPerCatchpointFileChunk } - if accounts >= accountsCount-64*1024 && last64KStart.IsZero() { - last64KStart = time.Now() + if accounts >= accountsCount-64*1024 && last64KIndex == -1 { + last64KIndex = len(encodedAccountChunks) } var balances catchpointFileBalancesChunk balances.Balances = make([]encodedBalanceRecord, chunkSize) @@ -88,15 +93,31 @@ func benchmarkRestoringFromCatchpointFileHelper(b *testing.B) { accountData.MicroAlgos.Raw = crypto.RandUint63() randomAccount.AccountData = protocol.Encode(&accountData) crypto.RandBytes(randomAccount.Address[:]) + binary.LittleEndian.PutUint64(randomAccount.Address[:], accounts+i) balances.Balances[i] = randomAccount } - err = catchpointAccessor.ProgressStagingBalances(context.Background(), "balances.XX.msgpack", protocol.Encode(&balances), &progress) - require.NoError(b, err) + encodedAccountChunks = append(encodedAccountChunks, protocol.Encode(&balances)) accounts += chunkSize } + + b.ResetTimer() + accounts = uint64(0) + var last64KStart time.Time + for len(encodedAccountChunks) > 0 { + encodedAccounts := encodedAccountChunks[0] + encodedAccountChunks = encodedAccountChunks[1:] + + if last64KIndex == 0 { + last64KStart = time.Now() + } + + err = catchpointAccessor.ProgressStagingBalances(context.Background(), "balances.XX.msgpack", encodedAccounts, &progress) + require.NoError(b, err) + last64KIndex-- + } if !last64KStart.IsZero() { last64KDuration := time.Now().Sub(last64KStart) - b.Logf("Last 64K\t%v\n", last64KDuration) + b.ReportMetric(float64(last64KDuration.Nanoseconds())/float64(64*1024), "ns/last_64k_account") } } From 23b740c6a96493969efb7e1d35110117719849d2 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Fri, 11 Sep 2020 14:27:32 -0400 Subject: [PATCH 032/136] Fix asset flags. --- cmd/goal/asset.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/goal/asset.go b/cmd/goal/asset.go index 66e88f06ac..d6ce6d9414 100644 --- a/cmd/goal/asset.go +++ b/cmd/goal/asset.go @@ -110,7 +110,7 @@ func init() { addTxnFlags(freezeAssetCmd) infoAssetCmd.Flags().Uint64Var(&assetID, "assetid", 0, "ID of the asset to look up") - infoAssetCmd.Flags().StringVar(&assetUnitName, "asset", "", "Unit name of the asset to look up") + infoAssetCmd.Flags().StringVar(&assetUnitName, "unit", "", "Unit name of the asset to look up") infoAssetCmd.Flags().StringVar(&assetCreator, "creator", "", "Account address of the asset creator") } @@ -125,21 +125,21 @@ var assetCmd = &cobra.Command{ } func lookupAssetID(cmd *cobra.Command, creator string, client libgoal.Client) { - if cmd.Flags().Changed("assetid") && cmd.Flags().Changed("asset") { - reportErrorf("Only one of [-assetid] or [-asset and -creator] can be specified") + if cmd.Flags().Changed("assetid") && cmd.Flags().Changed("unit") { + reportErrorf("Only one of [--assetid] or [--unit and --creator] can be specified") } if cmd.Flags().Changed("assetid") { return } - if !cmd.Flags().Changed("asset") { - reportErrorf("Either [-assetid] or [-asset and -creator] must be specified") + if !cmd.Flags().Changed("unit") { + reportErrorf("Either [--assetid] or [--unit and --creator] must be specified") } if !cmd.Flags().Changed("creator") { reportErrorf("Asset creator must be specified if finding asset by name. " + - "Use the asset's integer identifier (-assetid) if the " + + "Use the asset's integer identifier [--assetid] if the " + "creator account is unknown.") } From 4afd8de4321982ac58747503b6263ba50d98e845 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Fri, 11 Sep 2020 16:38:19 -0400 Subject: [PATCH 033/136] e2e test updates. --- test/scripts/e2e_subs/e2e-app-real-assets-round.sh | 2 +- test/scripts/e2e_subs/limit-swap-test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/scripts/e2e_subs/e2e-app-real-assets-round.sh b/test/scripts/e2e_subs/e2e-app-real-assets-round.sh index 6793d6bb53..6401042c5e 100755 --- a/test/scripts/e2e_subs/e2e-app-real-assets-round.sh +++ b/test/scripts/e2e_subs/e2e-app-real-assets-round.sh @@ -18,7 +18,7 @@ ACCOUNT=$(${gcmd} account list|awk '{ print $3 }') # Create an ASA in account ${gcmd} asset create --creator ${ACCOUNT} --name bogocoin --unitname bogo --total 1337 -ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --asset bogo|grep 'Asset ID'|awk '{ print $3 }') +ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --unit bogo|grep 'Asset ID'|awk '{ print $3 }') # Create app that reads asset balance and checks asset details and checks round ROUND=$(goal node status | grep 'Last committed' | awk '{ print $4 }') diff --git a/test/scripts/e2e_subs/limit-swap-test.sh b/test/scripts/e2e_subs/limit-swap-test.sh index 52907b47ff..463128fb8e 100755 --- a/test/scripts/e2e_subs/limit-swap-test.sh +++ b/test/scripts/e2e_subs/limit-swap-test.sh @@ -16,7 +16,7 @@ ZERO_ADDRESS=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ ${gcmd} asset create --creator ${ACCOUNT} --name bogocoin --unitname bogo --total 1000000000000 -ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --asset bogo|grep 'Asset ID'|awk '{ print $3 }') +ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --unit bogo|grep 'Asset ID'|awk '{ print $3 }') # Asset ID: 5 From cd76f729d6b1207a27c82004729544e9fd5f2cac Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Mon, 14 Sep 2020 11:22:17 -0400 Subject: [PATCH 034/136] commit to Merkle tree of online participants in block header (#1447) commit to Merkle tree of online participants in block header --- agreement/msgp_gen.go | 577 +++++++++++++++++++++++++---------- config/consensus.go | 28 ++ data/basics/userBalance.go | 8 + data/bookkeeping/block.go | 18 ++ data/bookkeeping/msgp_gen.go | 354 +++++++++++++++------ ledger/acctupdates.go | 18 ++ ledger/eval.go | 46 +++ ledger/ledger.go | 9 + ledger/voters.go | 351 +++++++++++++++++++++ 9 files changed, 1146 insertions(+), 263 deletions(-) create mode 100644 ledger/voters.go diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 1d0b921f7a..7a54da1676 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -1414,121 +1414,160 @@ func (z period) MsgIsZero() bool { func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0001Len := uint32(26) - var zb0001Mask uint32 /* 31 bits */ - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel == 0 { + zb0001Len := uint32(29) + var zb0001Mask uint64 /* 34 bits */ + if (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x10 } - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x20 } - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsResidue == 0 { + if (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x40 } - if (*z).unauthenticatedProposal.Block.BlockHeader.GenesisID == "" { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel == 0 { zb0001Len-- zb0001Mask |= 0x80 } - if (*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x100 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsResidue == 0 { zb0001Len-- zb0001Mask |= 0x200 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.GenesisID == "" { zb0001Len-- zb0001Mask |= 0x400 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x800 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0 { + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x1000 } - if (*z).unauthenticatedProposal.OriginalPeriod == 0 { + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2000 } - if (*z).unauthenticatedProposal.OriginalProposer.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x4000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.Branch.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0 { zb0001Len-- zb0001Mask |= 0x8000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero() { + if (*z).unauthenticatedProposal.OriginalPeriod == 0 { zb0001Len-- zb0001Mask |= 0x10000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRate == 0 { + if (*z).unauthenticatedProposal.OriginalProposer.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x20000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.Round.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.Branch.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x40000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x80000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRate == 0 { zb0001Len-- zb0001Mask |= 0x100000 } - if (*z).unauthenticatedProposal.SeedProof.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.Round.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x200000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.Seed.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x400000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.TxnCounter == 0 { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x800000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.TimeStamp == 0 { + if (*z).unauthenticatedProposal.SeedProof.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x1000000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.Seed.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2000000 } - if (*z).unauthenticatedProposal.Block.Payset.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.TxnCounter == 0 { zb0001Len-- zb0001Mask |= 0x4000000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.TimeStamp == 0 { zb0001Len-- zb0001Mask |= 0x8000000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x10000000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeApprove == false { + if (*z).unauthenticatedProposal.Block.Payset.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x20000000 } + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x40000000 + } + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x80000000 + } + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeApprove == false { + zb0001Len-- + zb0001Mask |= 0x100000000 + } // variable map header, size zb0001Len o = msgp.AppendMapHeader(o, zb0001Len) if zb0001Len != 0 { if (zb0001Mask & 0x10) == 0 { // if not empty + // string "ccl" + o = append(o, 0xa3, 0x63, 0x63, 0x6c) + o, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertLastRound") + return + } + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // string "ccv" + o = append(o, 0xa3, 0x63, 0x63, 0x76) + o, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertVoters") + return + } + } + if (zb0001Mask & 0x40) == 0 { // if not empty + // string "ccvt" + o = append(o, 0xa4, 0x63, 0x63, 0x76, 0x74) + o, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertVotersTotal") + return + } + } + if (zb0001Mask & 0x80) == 0 { // if not empty // string "earn" o = append(o, 0xa4, 0x65, 0x61, 0x72, 0x6e) o = msgp.AppendUint64(o, (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel) } - if (zb0001Mask & 0x20) == 0 { // if not empty + if (zb0001Mask & 0x100) == 0 { // if not empty // string "fees" o = append(o, 0xa4, 0x66, 0x65, 0x65, 0x73) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.MarshalMsg(o) @@ -1537,17 +1576,17 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x40) == 0 { // if not empty + if (zb0001Mask & 0x200) == 0 { // if not empty // string "frac" o = append(o, 0xa4, 0x66, 0x72, 0x61, 0x63) o = msgp.AppendUint64(o, (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsResidue) } - if (zb0001Mask & 0x80) == 0 { // if not empty + if (zb0001Mask & 0x400) == 0 { // if not empty // string "gen" o = append(o, 0xa3, 0x67, 0x65, 0x6e) o = msgp.AppendString(o, (*z).unauthenticatedProposal.Block.BlockHeader.GenesisID) } - if (zb0001Mask & 0x100) == 0 { // if not empty + if (zb0001Mask & 0x800) == 0 { // if not empty // string "gh" o = append(o, 0xa2, 0x67, 0x68) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.MarshalMsg(o) @@ -1556,7 +1595,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x200) == 0 { // if not empty + if (zb0001Mask & 0x1000) == 0 { // if not empty // string "nextbefore" o = append(o, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MarshalMsg(o) @@ -1565,7 +1604,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x400) == 0 { // if not empty + if (zb0001Mask & 0x2000) == 0 { // if not empty // string "nextproto" o = append(o, 0xa9, 0x6e, 0x65, 0x78, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.MarshalMsg(o) @@ -1574,7 +1613,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x800) == 0 { // if not empty + if (zb0001Mask & 0x4000) == 0 { // if not empty // string "nextswitch" o = append(o, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x73, 0x77, 0x69, 0x74, 0x63, 0x68) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MarshalMsg(o) @@ -1583,17 +1622,17 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x1000) == 0 { // if not empty + if (zb0001Mask & 0x8000) == 0 { // if not empty // string "nextyes" o = append(o, 0xa7, 0x6e, 0x65, 0x78, 0x74, 0x79, 0x65, 0x73) o = msgp.AppendUint64(o, (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolApprovals) } - if (zb0001Mask & 0x2000) == 0 { // if not empty + if (zb0001Mask & 0x10000) == 0 { // if not empty // string "oper" o = append(o, 0xa4, 0x6f, 0x70, 0x65, 0x72) o = msgp.AppendUint64(o, uint64((*z).unauthenticatedProposal.OriginalPeriod)) } - if (zb0001Mask & 0x4000) == 0 { // if not empty + if (zb0001Mask & 0x20000) == 0 { // if not empty // string "oprop" o = append(o, 0xa5, 0x6f, 0x70, 0x72, 0x6f, 0x70) o, err = (*z).unauthenticatedProposal.OriginalProposer.MarshalMsg(o) @@ -1602,7 +1641,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x8000) == 0 { // if not empty + if (zb0001Mask & 0x40000) == 0 { // if not empty // string "prev" o = append(o, 0xa4, 0x70, 0x72, 0x65, 0x76) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.Branch.MarshalMsg(o) @@ -1611,7 +1650,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x10000) == 0 { // if not empty + if (zb0001Mask & 0x80000) == 0 { // if not empty // string "proto" o = append(o, 0xa5, 0x70, 0x72, 0x6f, 0x74, 0x6f) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.MarshalMsg(o) @@ -1620,12 +1659,12 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x20000) == 0 { // if not empty + if (zb0001Mask & 0x100000) == 0 { // if not empty // string "rate" o = append(o, 0xa4, 0x72, 0x61, 0x74, 0x65) o = msgp.AppendUint64(o, (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRate) } - if (zb0001Mask & 0x40000) == 0 { // if not empty + if (zb0001Mask & 0x200000) == 0 { // if not empty // string "rnd" o = append(o, 0xa3, 0x72, 0x6e, 0x64) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.Round.MarshalMsg(o) @@ -1634,7 +1673,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x80000) == 0 { // if not empty + if (zb0001Mask & 0x400000) == 0 { // if not empty // string "rwcalr" o = append(o, 0xa6, 0x72, 0x77, 0x63, 0x61, 0x6c, 0x72) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.MarshalMsg(o) @@ -1643,7 +1682,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x100000) == 0 { // if not empty + if (zb0001Mask & 0x800000) == 0 { // if not empty // string "rwd" o = append(o, 0xa3, 0x72, 0x77, 0x64) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.MarshalMsg(o) @@ -1652,7 +1691,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x200000) == 0 { // if not empty + if (zb0001Mask & 0x1000000) == 0 { // if not empty // string "sdpf" o = append(o, 0xa4, 0x73, 0x64, 0x70, 0x66) o, err = (*z).unauthenticatedProposal.SeedProof.MarshalMsg(o) @@ -1661,7 +1700,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x400000) == 0 { // if not empty + if (zb0001Mask & 0x2000000) == 0 { // if not empty // string "seed" o = append(o, 0xa4, 0x73, 0x65, 0x65, 0x64) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.Seed.MarshalMsg(o) @@ -1670,17 +1709,17 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x800000) == 0 { // if not empty + if (zb0001Mask & 0x4000000) == 0 { // if not empty // string "tc" o = append(o, 0xa2, 0x74, 0x63) o = msgp.AppendUint64(o, (*z).unauthenticatedProposal.Block.BlockHeader.TxnCounter) } - if (zb0001Mask & 0x1000000) == 0 { // if not empty + if (zb0001Mask & 0x8000000) == 0 { // if not empty // string "ts" o = append(o, 0xa2, 0x74, 0x73) o = msgp.AppendInt64(o, (*z).unauthenticatedProposal.Block.BlockHeader.TimeStamp) } - if (zb0001Mask & 0x2000000) == 0 { // if not empty + if (zb0001Mask & 0x10000000) == 0 { // if not empty // string "txn" o = append(o, 0xa3, 0x74, 0x78, 0x6e) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.MarshalMsg(o) @@ -1689,7 +1728,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x4000000) == 0 { // if not empty + if (zb0001Mask & 0x20000000) == 0 { // if not empty // string "txns" o = append(o, 0xa4, 0x74, 0x78, 0x6e, 0x73) o, err = (*z).unauthenticatedProposal.Block.Payset.MarshalMsg(o) @@ -1698,7 +1737,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x8000000) == 0 { // if not empty + if (zb0001Mask & 0x40000000) == 0 { // if not empty // string "upgradedelay" o = append(o, 0xac, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x64, 0x65, 0x6c, 0x61, 0x79) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.MarshalMsg(o) @@ -1707,7 +1746,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x10000000) == 0 { // if not empty + if (zb0001Mask & 0x80000000) == 0 { // if not empty // string "upgradeprop" o = append(o, 0xab, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x70, 0x72, 0x6f, 0x70) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.MarshalMsg(o) @@ -1716,7 +1755,7 @@ func (z *proposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x20000000) == 0 { // if not empty + if (zb0001Mask & 0x100000000) == 0 { // if not empty // string "upgradeyes" o = append(o, 0xaa, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x79, 0x65, 0x73) o = msgp.AppendBool(o, (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeApprove) @@ -1919,6 +1958,30 @@ func (z *proposal) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertVoters") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertVotersTotal") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertLastRound") + return + } + } if zb0001 > 0 { zb0001-- bts, err = (*z).unauthenticatedProposal.Block.Payset.UnmarshalMsg(bts) @@ -2110,6 +2173,24 @@ func (z *proposal) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "TxnCounter") return } + case "ccv": + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertVoters") + return + } + case "ccvt": + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertVotersTotal") + return + } + case "ccl": + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertLastRound") + return + } case "txns": bts, err = (*z).unauthenticatedProposal.Block.Payset.UnmarshalMsg(bts) if err != nil { @@ -2158,13 +2239,13 @@ func (_ *proposal) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *proposal) Msgsize() (s int) { - s = 3 + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.Round.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.Branch.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.Seed.Msgsize() + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len((*z).unauthenticatedProposal.Block.BlockHeader.GenesisID) + 3 + (*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.Msgsize() + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.Msgsize() + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 7 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.Msgsize() + 6 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.Msgsize() + 10 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.Msgsize() + 8 + msgp.Uint64Size + 11 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.Msgsize() + 11 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.Msgsize() + 12 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.Msgsize() + 13 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.Msgsize() + 11 + msgp.BoolSize + 3 + msgp.Uint64Size + 5 + (*z).unauthenticatedProposal.Block.Payset.Msgsize() + 5 + (*z).unauthenticatedProposal.SeedProof.Msgsize() + 5 + msgp.Uint64Size + 6 + (*z).unauthenticatedProposal.OriginalProposer.Msgsize() + s = 3 + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.Round.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.Branch.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.Seed.Msgsize() + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len((*z).unauthenticatedProposal.Block.BlockHeader.GenesisID) + 3 + (*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.Msgsize() + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.Msgsize() + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 7 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.Msgsize() + 6 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.Msgsize() + 10 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.Msgsize() + 8 + msgp.Uint64Size + 11 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.Msgsize() + 11 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.Msgsize() + 12 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.Msgsize() + 13 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.Msgsize() + 11 + msgp.BoolSize + 3 + msgp.Uint64Size + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.Msgsize() + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.Payset.Msgsize() + 5 + (*z).unauthenticatedProposal.SeedProof.Msgsize() + 5 + msgp.Uint64Size + 6 + (*z).unauthenticatedProposal.OriginalProposer.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *proposal) MsgIsZero() bool { - return ((*z).unauthenticatedProposal.Block.BlockHeader.Round.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.Branch.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.Seed.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.TimeStamp == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.GenesisID == "") && ((*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRate == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsResidue == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeApprove == false) && ((*z).unauthenticatedProposal.Block.BlockHeader.TxnCounter == 0) && ((*z).unauthenticatedProposal.Block.Payset.MsgIsZero()) && ((*z).unauthenticatedProposal.SeedProof.MsgIsZero()) && ((*z).unauthenticatedProposal.OriginalPeriod == 0) && ((*z).unauthenticatedProposal.OriginalProposer.MsgIsZero()) + return ((*z).unauthenticatedProposal.Block.BlockHeader.Round.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.Branch.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.Seed.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.TimeStamp == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.GenesisID == "") && ((*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRate == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsResidue == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeApprove == false) && ((*z).unauthenticatedProposal.Block.BlockHeader.TxnCounter == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.Payset.MsgIsZero()) && ((*z).unauthenticatedProposal.SeedProof.MsgIsZero()) && ((*z).unauthenticatedProposal.OriginalPeriod == 0) && ((*z).unauthenticatedProposal.OriginalProposer.MsgIsZero()) } // MarshalMsg implements msgp.Marshaler @@ -3091,125 +3172,164 @@ func (z step) MsgIsZero() bool { func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0001Len := uint32(27) - var zb0001Mask uint32 /* 32 bits */ - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel == 0 { + zb0001Len := uint32(30) + var zb0001Mask uint64 /* 35 bits */ + if (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x20 } - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x40 } - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsResidue == 0 { + if (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x80 } - if (*z).unauthenticatedProposal.Block.BlockHeader.GenesisID == "" { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel == 0 { zb0001Len-- zb0001Mask |= 0x100 } - if (*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x200 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsResidue == 0 { zb0001Len-- zb0001Mask |= 0x400 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.GenesisID == "" { zb0001Len-- zb0001Mask |= 0x800 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x1000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0 { + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2000 } - if (*z).unauthenticatedProposal.OriginalPeriod == 0 { + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x4000 } - if (*z).unauthenticatedProposal.OriginalProposer.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x8000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.Branch.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0 { zb0001Len-- zb0001Mask |= 0x10000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero() { + if (*z).unauthenticatedProposal.OriginalPeriod == 0 { zb0001Len-- zb0001Mask |= 0x20000 } - if (*z).PriorVote.MsgIsZero() { + if (*z).unauthenticatedProposal.OriginalProposer.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x40000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRate == 0 { + if (*z).unauthenticatedProposal.Block.BlockHeader.Branch.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x80000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.Round.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x100000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero() { + if (*z).PriorVote.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x200000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRate == 0 { zb0001Len-- zb0001Mask |= 0x400000 } - if (*z).unauthenticatedProposal.SeedProof.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.Round.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x800000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.Seed.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x1000000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.TxnCounter == 0 { + if (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2000000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.TimeStamp == 0 { + if (*z).unauthenticatedProposal.SeedProof.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x4000000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.Seed.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x8000000 } - if (*z).unauthenticatedProposal.Block.Payset.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.TxnCounter == 0 { zb0001Len-- zb0001Mask |= 0x10000000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.TimeStamp == 0 { zb0001Len-- zb0001Mask |= 0x20000000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero() { + if (*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x40000000 } - if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeApprove == false { + if (*z).unauthenticatedProposal.Block.Payset.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x80000000 } + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x100000000 + } + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x200000000 + } + if (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeApprove == false { + zb0001Len-- + zb0001Mask |= 0x400000000 + } // variable map header, size zb0001Len o = msgp.AppendMapHeader(o, zb0001Len) if zb0001Len != 0 { if (zb0001Mask & 0x20) == 0 { // if not empty + // string "ccl" + o = append(o, 0xa3, 0x63, 0x63, 0x6c) + o, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertLastRound") + return + } + } + if (zb0001Mask & 0x40) == 0 { // if not empty + // string "ccv" + o = append(o, 0xa3, 0x63, 0x63, 0x76) + o, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertVoters") + return + } + } + if (zb0001Mask & 0x80) == 0 { // if not empty + // string "ccvt" + o = append(o, 0xa4, 0x63, 0x63, 0x76, 0x74) + o, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertVotersTotal") + return + } + } + if (zb0001Mask & 0x100) == 0 { // if not empty // string "earn" o = append(o, 0xa4, 0x65, 0x61, 0x72, 0x6e) o = msgp.AppendUint64(o, (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel) } - if (zb0001Mask & 0x40) == 0 { // if not empty + if (zb0001Mask & 0x200) == 0 { // if not empty // string "fees" o = append(o, 0xa4, 0x66, 0x65, 0x65, 0x73) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.MarshalMsg(o) @@ -3218,17 +3338,17 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x80) == 0 { // if not empty + if (zb0001Mask & 0x400) == 0 { // if not empty // string "frac" o = append(o, 0xa4, 0x66, 0x72, 0x61, 0x63) o = msgp.AppendUint64(o, (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsResidue) } - if (zb0001Mask & 0x100) == 0 { // if not empty + if (zb0001Mask & 0x800) == 0 { // if not empty // string "gen" o = append(o, 0xa3, 0x67, 0x65, 0x6e) o = msgp.AppendString(o, (*z).unauthenticatedProposal.Block.BlockHeader.GenesisID) } - if (zb0001Mask & 0x200) == 0 { // if not empty + if (zb0001Mask & 0x1000) == 0 { // if not empty // string "gh" o = append(o, 0xa2, 0x67, 0x68) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.MarshalMsg(o) @@ -3237,7 +3357,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x400) == 0 { // if not empty + if (zb0001Mask & 0x2000) == 0 { // if not empty // string "nextbefore" o = append(o, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MarshalMsg(o) @@ -3246,7 +3366,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x800) == 0 { // if not empty + if (zb0001Mask & 0x4000) == 0 { // if not empty // string "nextproto" o = append(o, 0xa9, 0x6e, 0x65, 0x78, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.MarshalMsg(o) @@ -3255,7 +3375,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x1000) == 0 { // if not empty + if (zb0001Mask & 0x8000) == 0 { // if not empty // string "nextswitch" o = append(o, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x73, 0x77, 0x69, 0x74, 0x63, 0x68) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MarshalMsg(o) @@ -3264,17 +3384,17 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x2000) == 0 { // if not empty + if (zb0001Mask & 0x10000) == 0 { // if not empty // string "nextyes" o = append(o, 0xa7, 0x6e, 0x65, 0x78, 0x74, 0x79, 0x65, 0x73) o = msgp.AppendUint64(o, (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolApprovals) } - if (zb0001Mask & 0x4000) == 0 { // if not empty + if (zb0001Mask & 0x20000) == 0 { // if not empty // string "oper" o = append(o, 0xa4, 0x6f, 0x70, 0x65, 0x72) o = msgp.AppendUint64(o, uint64((*z).unauthenticatedProposal.OriginalPeriod)) } - if (zb0001Mask & 0x8000) == 0 { // if not empty + if (zb0001Mask & 0x40000) == 0 { // if not empty // string "oprop" o = append(o, 0xa5, 0x6f, 0x70, 0x72, 0x6f, 0x70) o, err = (*z).unauthenticatedProposal.OriginalProposer.MarshalMsg(o) @@ -3283,7 +3403,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x10000) == 0 { // if not empty + if (zb0001Mask & 0x80000) == 0 { // if not empty // string "prev" o = append(o, 0xa4, 0x70, 0x72, 0x65, 0x76) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.Branch.MarshalMsg(o) @@ -3292,7 +3412,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x20000) == 0 { // if not empty + if (zb0001Mask & 0x100000) == 0 { // if not empty // string "proto" o = append(o, 0xa5, 0x70, 0x72, 0x6f, 0x74, 0x6f) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.MarshalMsg(o) @@ -3301,7 +3421,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x40000) == 0 { // if not empty + if (zb0001Mask & 0x200000) == 0 { // if not empty // string "pv" o = append(o, 0xa2, 0x70, 0x76) o, err = (*z).PriorVote.MarshalMsg(o) @@ -3310,12 +3430,12 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x80000) == 0 { // if not empty + if (zb0001Mask & 0x400000) == 0 { // if not empty // string "rate" o = append(o, 0xa4, 0x72, 0x61, 0x74, 0x65) o = msgp.AppendUint64(o, (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRate) } - if (zb0001Mask & 0x100000) == 0 { // if not empty + if (zb0001Mask & 0x800000) == 0 { // if not empty // string "rnd" o = append(o, 0xa3, 0x72, 0x6e, 0x64) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.Round.MarshalMsg(o) @@ -3324,7 +3444,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x200000) == 0 { // if not empty + if (zb0001Mask & 0x1000000) == 0 { // if not empty // string "rwcalr" o = append(o, 0xa6, 0x72, 0x77, 0x63, 0x61, 0x6c, 0x72) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.MarshalMsg(o) @@ -3333,7 +3453,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x400000) == 0 { // if not empty + if (zb0001Mask & 0x2000000) == 0 { // if not empty // string "rwd" o = append(o, 0xa3, 0x72, 0x77, 0x64) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.MarshalMsg(o) @@ -3342,7 +3462,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x800000) == 0 { // if not empty + if (zb0001Mask & 0x4000000) == 0 { // if not empty // string "sdpf" o = append(o, 0xa4, 0x73, 0x64, 0x70, 0x66) o, err = (*z).unauthenticatedProposal.SeedProof.MarshalMsg(o) @@ -3351,7 +3471,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x1000000) == 0 { // if not empty + if (zb0001Mask & 0x8000000) == 0 { // if not empty // string "seed" o = append(o, 0xa4, 0x73, 0x65, 0x65, 0x64) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.Seed.MarshalMsg(o) @@ -3360,17 +3480,17 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x2000000) == 0 { // if not empty + if (zb0001Mask & 0x10000000) == 0 { // if not empty // string "tc" o = append(o, 0xa2, 0x74, 0x63) o = msgp.AppendUint64(o, (*z).unauthenticatedProposal.Block.BlockHeader.TxnCounter) } - if (zb0001Mask & 0x4000000) == 0 { // if not empty + if (zb0001Mask & 0x20000000) == 0 { // if not empty // string "ts" o = append(o, 0xa2, 0x74, 0x73) o = msgp.AppendInt64(o, (*z).unauthenticatedProposal.Block.BlockHeader.TimeStamp) } - if (zb0001Mask & 0x8000000) == 0 { // if not empty + if (zb0001Mask & 0x40000000) == 0 { // if not empty // string "txn" o = append(o, 0xa3, 0x74, 0x78, 0x6e) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.MarshalMsg(o) @@ -3379,7 +3499,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x10000000) == 0 { // if not empty + if (zb0001Mask & 0x80000000) == 0 { // if not empty // string "txns" o = append(o, 0xa4, 0x74, 0x78, 0x6e, 0x73) o, err = (*z).unauthenticatedProposal.Block.Payset.MarshalMsg(o) @@ -3388,7 +3508,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x20000000) == 0 { // if not empty + if (zb0001Mask & 0x100000000) == 0 { // if not empty // string "upgradedelay" o = append(o, 0xac, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x64, 0x65, 0x6c, 0x61, 0x79) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.MarshalMsg(o) @@ -3397,7 +3517,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x40000000) == 0 { // if not empty + if (zb0001Mask & 0x200000000) == 0 { // if not empty // string "upgradeprop" o = append(o, 0xab, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x70, 0x72, 0x6f, 0x70) o, err = (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.MarshalMsg(o) @@ -3406,7 +3526,7 @@ func (z *transmittedPayload) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x80000000) == 0 { // if not empty + if (zb0001Mask & 0x400000000) == 0 { // if not empty // string "upgradeyes" o = append(o, 0xaa, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x79, 0x65, 0x73) o = msgp.AppendBool(o, (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeApprove) @@ -3609,6 +3729,30 @@ func (z *transmittedPayload) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertVoters") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertVotersTotal") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertLastRound") + return + } + } if zb0001 > 0 { zb0001-- bts, err = (*z).unauthenticatedProposal.Block.Payset.UnmarshalMsg(bts) @@ -3808,6 +3952,24 @@ func (z *transmittedPayload) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "TxnCounter") return } + case "ccv": + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertVoters") + return + } + case "ccvt": + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertVotersTotal") + return + } + case "ccl": + bts, err = (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertLastRound") + return + } case "txns": bts, err = (*z).unauthenticatedProposal.Block.Payset.UnmarshalMsg(bts) if err != nil { @@ -3862,13 +4024,13 @@ func (_ *transmittedPayload) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *transmittedPayload) Msgsize() (s int) { - s = 3 + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.Round.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.Branch.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.Seed.Msgsize() + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len((*z).unauthenticatedProposal.Block.BlockHeader.GenesisID) + 3 + (*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.Msgsize() + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.Msgsize() + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 7 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.Msgsize() + 6 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.Msgsize() + 10 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.Msgsize() + 8 + msgp.Uint64Size + 11 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.Msgsize() + 11 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.Msgsize() + 12 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.Msgsize() + 13 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.Msgsize() + 11 + msgp.BoolSize + 3 + msgp.Uint64Size + 5 + (*z).unauthenticatedProposal.Block.Payset.Msgsize() + 5 + (*z).unauthenticatedProposal.SeedProof.Msgsize() + 5 + msgp.Uint64Size + 6 + (*z).unauthenticatedProposal.OriginalProposer.Msgsize() + 3 + (*z).PriorVote.Msgsize() + s = 3 + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.Round.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.Branch.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.Seed.Msgsize() + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len((*z).unauthenticatedProposal.Block.BlockHeader.GenesisID) + 3 + (*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.Msgsize() + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.Msgsize() + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 7 + (*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.Msgsize() + 6 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.Msgsize() + 10 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.Msgsize() + 8 + msgp.Uint64Size + 11 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.Msgsize() + 11 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.Msgsize() + 12 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.Msgsize() + 13 + (*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.Msgsize() + 11 + msgp.BoolSize + 3 + msgp.Uint64Size + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.Msgsize() + 4 + (*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.Msgsize() + 5 + (*z).unauthenticatedProposal.Block.Payset.Msgsize() + 5 + (*z).unauthenticatedProposal.SeedProof.Msgsize() + 5 + msgp.Uint64Size + 6 + (*z).unauthenticatedProposal.OriginalProposer.Msgsize() + 3 + (*z).PriorVote.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *transmittedPayload) MsgIsZero() bool { - return ((*z).unauthenticatedProposal.Block.BlockHeader.Round.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.Branch.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.Seed.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.TimeStamp == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.GenesisID == "") && ((*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRate == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsResidue == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeApprove == false) && ((*z).unauthenticatedProposal.Block.BlockHeader.TxnCounter == 0) && ((*z).unauthenticatedProposal.Block.Payset.MsgIsZero()) && ((*z).unauthenticatedProposal.SeedProof.MsgIsZero()) && ((*z).unauthenticatedProposal.OriginalPeriod == 0) && ((*z).unauthenticatedProposal.OriginalProposer.MsgIsZero()) && ((*z).PriorVote.MsgIsZero()) + return ((*z).unauthenticatedProposal.Block.BlockHeader.Round.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.Branch.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.Seed.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.TxnRoot.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.TimeStamp == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.GenesisID == "") && ((*z).unauthenticatedProposal.Block.BlockHeader.GenesisHash.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.FeeSink.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsLevel == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRate == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsResidue == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.UpgradeVote.UpgradeApprove == false) && ((*z).unauthenticatedProposal.Block.BlockHeader.TxnCounter == 0) && ((*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVoters.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.CompactCertVotersTotal.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.BlockHeader.CompactCertLastRound.MsgIsZero()) && ((*z).unauthenticatedProposal.Block.Payset.MsgIsZero()) && ((*z).unauthenticatedProposal.SeedProof.MsgIsZero()) && ((*z).unauthenticatedProposal.OriginalPeriod == 0) && ((*z).unauthenticatedProposal.OriginalProposer.MsgIsZero()) && ((*z).PriorVote.MsgIsZero()) } // MarshalMsg implements msgp.Marshaler @@ -4574,121 +4736,160 @@ func (z *unauthenticatedEquivocationVote) MsgIsZero() bool { func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0001Len := uint32(26) - var zb0001Mask uint32 /* 30 bits */ - if (*z).Block.BlockHeader.RewardsState.RewardsLevel == 0 { + zb0001Len := uint32(29) + var zb0001Mask uint64 /* 33 bits */ + if (*z).Block.BlockHeader.CompactCertLastRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x10 } - if (*z).Block.BlockHeader.RewardsState.FeeSink.MsgIsZero() { + if (*z).Block.BlockHeader.CompactCertVoters.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x20 } - if (*z).Block.BlockHeader.RewardsState.RewardsResidue == 0 { + if (*z).Block.BlockHeader.CompactCertVotersTotal.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x40 } - if (*z).Block.BlockHeader.GenesisID == "" { + if (*z).Block.BlockHeader.RewardsState.RewardsLevel == 0 { zb0001Len-- zb0001Mask |= 0x80 } - if (*z).Block.BlockHeader.GenesisHash.MsgIsZero() { + if (*z).Block.BlockHeader.RewardsState.FeeSink.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x100 } - if (*z).Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero() { + if (*z).Block.BlockHeader.RewardsState.RewardsResidue == 0 { zb0001Len-- zb0001Mask |= 0x200 } - if (*z).Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero() { + if (*z).Block.BlockHeader.GenesisID == "" { zb0001Len-- zb0001Mask |= 0x400 } - if (*z).Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero() { + if (*z).Block.BlockHeader.GenesisHash.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x800 } - if (*z).Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0 { + if (*z).Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x1000 } - if (*z).OriginalPeriod == 0 { + if (*z).Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2000 } - if (*z).OriginalProposer.MsgIsZero() { + if (*z).Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x4000 } - if (*z).Block.BlockHeader.Branch.MsgIsZero() { + if (*z).Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0 { zb0001Len-- zb0001Mask |= 0x8000 } - if (*z).Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero() { + if (*z).OriginalPeriod == 0 { zb0001Len-- zb0001Mask |= 0x10000 } - if (*z).Block.BlockHeader.RewardsState.RewardsRate == 0 { + if (*z).OriginalProposer.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x20000 } - if (*z).Block.BlockHeader.Round.MsgIsZero() { + if (*z).Block.BlockHeader.Branch.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x40000 } - if (*z).Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero() { + if (*z).Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x80000 } - if (*z).Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero() { + if (*z).Block.BlockHeader.RewardsState.RewardsRate == 0 { zb0001Len-- zb0001Mask |= 0x100000 } - if (*z).SeedProof.MsgIsZero() { + if (*z).Block.BlockHeader.Round.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x200000 } - if (*z).Block.BlockHeader.Seed.MsgIsZero() { + if (*z).Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x400000 } - if (*z).Block.BlockHeader.TxnCounter == 0 { + if (*z).Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x800000 } - if (*z).Block.BlockHeader.TimeStamp == 0 { + if (*z).SeedProof.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x1000000 } - if (*z).Block.BlockHeader.TxnRoot.MsgIsZero() { + if (*z).Block.BlockHeader.Seed.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2000000 } - if (*z).Block.Payset.MsgIsZero() { + if (*z).Block.BlockHeader.TxnCounter == 0 { zb0001Len-- zb0001Mask |= 0x4000000 } - if (*z).Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero() { + if (*z).Block.BlockHeader.TimeStamp == 0 { zb0001Len-- zb0001Mask |= 0x8000000 } - if (*z).Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero() { + if (*z).Block.BlockHeader.TxnRoot.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x10000000 } - if (*z).Block.BlockHeader.UpgradeVote.UpgradeApprove == false { + if (*z).Block.Payset.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x20000000 } + if (*z).Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x40000000 + } + if (*z).Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x80000000 + } + if (*z).Block.BlockHeader.UpgradeVote.UpgradeApprove == false { + zb0001Len-- + zb0001Mask |= 0x100000000 + } // variable map header, size zb0001Len o = msgp.AppendMapHeader(o, zb0001Len) if zb0001Len != 0 { if (zb0001Mask & 0x10) == 0 { // if not empty + // string "ccl" + o = append(o, 0xa3, 0x63, 0x63, 0x6c) + o, err = (*z).Block.BlockHeader.CompactCertLastRound.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertLastRound") + return + } + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // string "ccv" + o = append(o, 0xa3, 0x63, 0x63, 0x76) + o, err = (*z).Block.BlockHeader.CompactCertVoters.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertVoters") + return + } + } + if (zb0001Mask & 0x40) == 0 { // if not empty + // string "ccvt" + o = append(o, 0xa4, 0x63, 0x63, 0x76, 0x74) + o, err = (*z).Block.BlockHeader.CompactCertVotersTotal.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertVotersTotal") + return + } + } + if (zb0001Mask & 0x80) == 0 { // if not empty // string "earn" o = append(o, 0xa4, 0x65, 0x61, 0x72, 0x6e) o = msgp.AppendUint64(o, (*z).Block.BlockHeader.RewardsState.RewardsLevel) } - if (zb0001Mask & 0x20) == 0 { // if not empty + if (zb0001Mask & 0x100) == 0 { // if not empty // string "fees" o = append(o, 0xa4, 0x66, 0x65, 0x65, 0x73) o, err = (*z).Block.BlockHeader.RewardsState.FeeSink.MarshalMsg(o) @@ -4697,17 +4898,17 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x40) == 0 { // if not empty + if (zb0001Mask & 0x200) == 0 { // if not empty // string "frac" o = append(o, 0xa4, 0x66, 0x72, 0x61, 0x63) o = msgp.AppendUint64(o, (*z).Block.BlockHeader.RewardsState.RewardsResidue) } - if (zb0001Mask & 0x80) == 0 { // if not empty + if (zb0001Mask & 0x400) == 0 { // if not empty // string "gen" o = append(o, 0xa3, 0x67, 0x65, 0x6e) o = msgp.AppendString(o, (*z).Block.BlockHeader.GenesisID) } - if (zb0001Mask & 0x100) == 0 { // if not empty + if (zb0001Mask & 0x800) == 0 { // if not empty // string "gh" o = append(o, 0xa2, 0x67, 0x68) o, err = (*z).Block.BlockHeader.GenesisHash.MarshalMsg(o) @@ -4716,7 +4917,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x200) == 0 { // if not empty + if (zb0001Mask & 0x1000) == 0 { // if not empty // string "nextbefore" o = append(o, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65) o, err = (*z).Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MarshalMsg(o) @@ -4725,7 +4926,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x400) == 0 { // if not empty + if (zb0001Mask & 0x2000) == 0 { // if not empty // string "nextproto" o = append(o, 0xa9, 0x6e, 0x65, 0x78, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f) o, err = (*z).Block.BlockHeader.UpgradeState.NextProtocol.MarshalMsg(o) @@ -4734,7 +4935,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x800) == 0 { // if not empty + if (zb0001Mask & 0x4000) == 0 { // if not empty // string "nextswitch" o = append(o, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x73, 0x77, 0x69, 0x74, 0x63, 0x68) o, err = (*z).Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MarshalMsg(o) @@ -4743,17 +4944,17 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x1000) == 0 { // if not empty + if (zb0001Mask & 0x8000) == 0 { // if not empty // string "nextyes" o = append(o, 0xa7, 0x6e, 0x65, 0x78, 0x74, 0x79, 0x65, 0x73) o = msgp.AppendUint64(o, (*z).Block.BlockHeader.UpgradeState.NextProtocolApprovals) } - if (zb0001Mask & 0x2000) == 0 { // if not empty + if (zb0001Mask & 0x10000) == 0 { // if not empty // string "oper" o = append(o, 0xa4, 0x6f, 0x70, 0x65, 0x72) o = msgp.AppendUint64(o, uint64((*z).OriginalPeriod)) } - if (zb0001Mask & 0x4000) == 0 { // if not empty + if (zb0001Mask & 0x20000) == 0 { // if not empty // string "oprop" o = append(o, 0xa5, 0x6f, 0x70, 0x72, 0x6f, 0x70) o, err = (*z).OriginalProposer.MarshalMsg(o) @@ -4762,7 +4963,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x8000) == 0 { // if not empty + if (zb0001Mask & 0x40000) == 0 { // if not empty // string "prev" o = append(o, 0xa4, 0x70, 0x72, 0x65, 0x76) o, err = (*z).Block.BlockHeader.Branch.MarshalMsg(o) @@ -4771,7 +4972,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x10000) == 0 { // if not empty + if (zb0001Mask & 0x80000) == 0 { // if not empty // string "proto" o = append(o, 0xa5, 0x70, 0x72, 0x6f, 0x74, 0x6f) o, err = (*z).Block.BlockHeader.UpgradeState.CurrentProtocol.MarshalMsg(o) @@ -4780,12 +4981,12 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x20000) == 0 { // if not empty + if (zb0001Mask & 0x100000) == 0 { // if not empty // string "rate" o = append(o, 0xa4, 0x72, 0x61, 0x74, 0x65) o = msgp.AppendUint64(o, (*z).Block.BlockHeader.RewardsState.RewardsRate) } - if (zb0001Mask & 0x40000) == 0 { // if not empty + if (zb0001Mask & 0x200000) == 0 { // if not empty // string "rnd" o = append(o, 0xa3, 0x72, 0x6e, 0x64) o, err = (*z).Block.BlockHeader.Round.MarshalMsg(o) @@ -4794,7 +4995,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x80000) == 0 { // if not empty + if (zb0001Mask & 0x400000) == 0 { // if not empty // string "rwcalr" o = append(o, 0xa6, 0x72, 0x77, 0x63, 0x61, 0x6c, 0x72) o, err = (*z).Block.BlockHeader.RewardsState.RewardsRecalculationRound.MarshalMsg(o) @@ -4803,7 +5004,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x100000) == 0 { // if not empty + if (zb0001Mask & 0x800000) == 0 { // if not empty // string "rwd" o = append(o, 0xa3, 0x72, 0x77, 0x64) o, err = (*z).Block.BlockHeader.RewardsState.RewardsPool.MarshalMsg(o) @@ -4812,7 +5013,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x200000) == 0 { // if not empty + if (zb0001Mask & 0x1000000) == 0 { // if not empty // string "sdpf" o = append(o, 0xa4, 0x73, 0x64, 0x70, 0x66) o, err = (*z).SeedProof.MarshalMsg(o) @@ -4821,7 +5022,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x400000) == 0 { // if not empty + if (zb0001Mask & 0x2000000) == 0 { // if not empty // string "seed" o = append(o, 0xa4, 0x73, 0x65, 0x65, 0x64) o, err = (*z).Block.BlockHeader.Seed.MarshalMsg(o) @@ -4830,17 +5031,17 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x800000) == 0 { // if not empty + if (zb0001Mask & 0x4000000) == 0 { // if not empty // string "tc" o = append(o, 0xa2, 0x74, 0x63) o = msgp.AppendUint64(o, (*z).Block.BlockHeader.TxnCounter) } - if (zb0001Mask & 0x1000000) == 0 { // if not empty + if (zb0001Mask & 0x8000000) == 0 { // if not empty // string "ts" o = append(o, 0xa2, 0x74, 0x73) o = msgp.AppendInt64(o, (*z).Block.BlockHeader.TimeStamp) } - if (zb0001Mask & 0x2000000) == 0 { // if not empty + if (zb0001Mask & 0x10000000) == 0 { // if not empty // string "txn" o = append(o, 0xa3, 0x74, 0x78, 0x6e) o, err = (*z).Block.BlockHeader.TxnRoot.MarshalMsg(o) @@ -4849,7 +5050,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x4000000) == 0 { // if not empty + if (zb0001Mask & 0x20000000) == 0 { // if not empty // string "txns" o = append(o, 0xa4, 0x74, 0x78, 0x6e, 0x73) o, err = (*z).Block.Payset.MarshalMsg(o) @@ -4858,7 +5059,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x8000000) == 0 { // if not empty + if (zb0001Mask & 0x40000000) == 0 { // if not empty // string "upgradedelay" o = append(o, 0xac, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x64, 0x65, 0x6c, 0x61, 0x79) o, err = (*z).Block.BlockHeader.UpgradeVote.UpgradeDelay.MarshalMsg(o) @@ -4867,7 +5068,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x10000000) == 0 { // if not empty + if (zb0001Mask & 0x80000000) == 0 { // if not empty // string "upgradeprop" o = append(o, 0xab, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x70, 0x72, 0x6f, 0x70) o, err = (*z).Block.BlockHeader.UpgradeVote.UpgradePropose.MarshalMsg(o) @@ -4876,7 +5077,7 @@ func (z *unauthenticatedProposal) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x20000000) == 0 { // if not empty + if (zb0001Mask & 0x100000000) == 0 { // if not empty // string "upgradeyes" o = append(o, 0xaa, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x79, 0x65, 0x73) o = msgp.AppendBool(o, (*z).Block.BlockHeader.UpgradeVote.UpgradeApprove) @@ -5079,6 +5280,30 @@ func (z *unauthenticatedProposal) UnmarshalMsg(bts []byte) (o []byte, err error) return } } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Block.BlockHeader.CompactCertVoters.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertVoters") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Block.BlockHeader.CompactCertVotersTotal.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertVotersTotal") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Block.BlockHeader.CompactCertLastRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertLastRound") + return + } + } if zb0001 > 0 { zb0001-- bts, err = (*z).Block.Payset.UnmarshalMsg(bts) @@ -5270,6 +5495,24 @@ func (z *unauthenticatedProposal) UnmarshalMsg(bts []byte) (o []byte, err error) err = msgp.WrapError(err, "TxnCounter") return } + case "ccv": + bts, err = (*z).Block.BlockHeader.CompactCertVoters.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertVoters") + return + } + case "ccvt": + bts, err = (*z).Block.BlockHeader.CompactCertVotersTotal.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertVotersTotal") + return + } + case "ccl": + bts, err = (*z).Block.BlockHeader.CompactCertLastRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertLastRound") + return + } case "txns": bts, err = (*z).Block.Payset.UnmarshalMsg(bts) if err != nil { @@ -5318,13 +5561,13 @@ func (_ *unauthenticatedProposal) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *unauthenticatedProposal) Msgsize() (s int) { - s = 3 + 4 + (*z).Block.BlockHeader.Round.Msgsize() + 5 + (*z).Block.BlockHeader.Branch.Msgsize() + 5 + (*z).Block.BlockHeader.Seed.Msgsize() + 4 + (*z).Block.BlockHeader.TxnRoot.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len((*z).Block.BlockHeader.GenesisID) + 3 + (*z).Block.BlockHeader.GenesisHash.Msgsize() + 5 + (*z).Block.BlockHeader.RewardsState.FeeSink.Msgsize() + 4 + (*z).Block.BlockHeader.RewardsState.RewardsPool.Msgsize() + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 7 + (*z).Block.BlockHeader.RewardsState.RewardsRecalculationRound.Msgsize() + 6 + (*z).Block.BlockHeader.UpgradeState.CurrentProtocol.Msgsize() + 10 + (*z).Block.BlockHeader.UpgradeState.NextProtocol.Msgsize() + 8 + msgp.Uint64Size + 11 + (*z).Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.Msgsize() + 11 + (*z).Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.Msgsize() + 12 + (*z).Block.BlockHeader.UpgradeVote.UpgradePropose.Msgsize() + 13 + (*z).Block.BlockHeader.UpgradeVote.UpgradeDelay.Msgsize() + 11 + msgp.BoolSize + 3 + msgp.Uint64Size + 5 + (*z).Block.Payset.Msgsize() + 5 + (*z).SeedProof.Msgsize() + 5 + msgp.Uint64Size + 6 + (*z).OriginalProposer.Msgsize() + s = 3 + 4 + (*z).Block.BlockHeader.Round.Msgsize() + 5 + (*z).Block.BlockHeader.Branch.Msgsize() + 5 + (*z).Block.BlockHeader.Seed.Msgsize() + 4 + (*z).Block.BlockHeader.TxnRoot.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len((*z).Block.BlockHeader.GenesisID) + 3 + (*z).Block.BlockHeader.GenesisHash.Msgsize() + 5 + (*z).Block.BlockHeader.RewardsState.FeeSink.Msgsize() + 4 + (*z).Block.BlockHeader.RewardsState.RewardsPool.Msgsize() + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 7 + (*z).Block.BlockHeader.RewardsState.RewardsRecalculationRound.Msgsize() + 6 + (*z).Block.BlockHeader.UpgradeState.CurrentProtocol.Msgsize() + 10 + (*z).Block.BlockHeader.UpgradeState.NextProtocol.Msgsize() + 8 + msgp.Uint64Size + 11 + (*z).Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.Msgsize() + 11 + (*z).Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.Msgsize() + 12 + (*z).Block.BlockHeader.UpgradeVote.UpgradePropose.Msgsize() + 13 + (*z).Block.BlockHeader.UpgradeVote.UpgradeDelay.Msgsize() + 11 + msgp.BoolSize + 3 + msgp.Uint64Size + 4 + (*z).Block.BlockHeader.CompactCertVoters.Msgsize() + 5 + (*z).Block.BlockHeader.CompactCertVotersTotal.Msgsize() + 4 + (*z).Block.BlockHeader.CompactCertLastRound.Msgsize() + 5 + (*z).Block.Payset.Msgsize() + 5 + (*z).SeedProof.Msgsize() + 5 + msgp.Uint64Size + 6 + (*z).OriginalProposer.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *unauthenticatedProposal) MsgIsZero() bool { - return ((*z).Block.BlockHeader.Round.MsgIsZero()) && ((*z).Block.BlockHeader.Branch.MsgIsZero()) && ((*z).Block.BlockHeader.Seed.MsgIsZero()) && ((*z).Block.BlockHeader.TxnRoot.MsgIsZero()) && ((*z).Block.BlockHeader.TimeStamp == 0) && ((*z).Block.BlockHeader.GenesisID == "") && ((*z).Block.BlockHeader.GenesisHash.MsgIsZero()) && ((*z).Block.BlockHeader.RewardsState.FeeSink.MsgIsZero()) && ((*z).Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero()) && ((*z).Block.BlockHeader.RewardsState.RewardsLevel == 0) && ((*z).Block.BlockHeader.RewardsState.RewardsRate == 0) && ((*z).Block.BlockHeader.RewardsState.RewardsResidue == 0) && ((*z).Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0) && ((*z).Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeVote.UpgradeApprove == false) && ((*z).Block.BlockHeader.TxnCounter == 0) && ((*z).Block.Payset.MsgIsZero()) && ((*z).SeedProof.MsgIsZero()) && ((*z).OriginalPeriod == 0) && ((*z).OriginalProposer.MsgIsZero()) + return ((*z).Block.BlockHeader.Round.MsgIsZero()) && ((*z).Block.BlockHeader.Branch.MsgIsZero()) && ((*z).Block.BlockHeader.Seed.MsgIsZero()) && ((*z).Block.BlockHeader.TxnRoot.MsgIsZero()) && ((*z).Block.BlockHeader.TimeStamp == 0) && ((*z).Block.BlockHeader.GenesisID == "") && ((*z).Block.BlockHeader.GenesisHash.MsgIsZero()) && ((*z).Block.BlockHeader.RewardsState.FeeSink.MsgIsZero()) && ((*z).Block.BlockHeader.RewardsState.RewardsPool.MsgIsZero()) && ((*z).Block.BlockHeader.RewardsState.RewardsLevel == 0) && ((*z).Block.BlockHeader.RewardsState.RewardsRate == 0) && ((*z).Block.BlockHeader.RewardsState.RewardsResidue == 0) && ((*z).Block.BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeState.NextProtocol.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeState.NextProtocolApprovals == 0) && ((*z).Block.BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero()) && ((*z).Block.BlockHeader.UpgradeVote.UpgradeApprove == false) && ((*z).Block.BlockHeader.TxnCounter == 0) && ((*z).Block.BlockHeader.CompactCertVoters.MsgIsZero()) && ((*z).Block.BlockHeader.CompactCertVotersTotal.MsgIsZero()) && ((*z).Block.BlockHeader.CompactCertLastRound.MsgIsZero()) && ((*z).Block.Payset.MsgIsZero()) && ((*z).SeedProof.MsgIsZero()) && ((*z).OriginalPeriod == 0) && ((*z).OriginalProposer.MsgIsZero()) } // MarshalMsg implements msgp.Marshaler diff --git a/config/consensus.go b/config/consensus.go index 240fc4e715..7848ce6e86 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -295,6 +295,34 @@ type ConsensusParams struct { // maximum total minimum balance requirement for an account, used // to limit the maximum size of a single balance record MaximumMinimumBalance uint64 + + // CompactCertRounds defines the frequency with with compact + // certificates are generated. Every round that is a multiple + // of CompactCertRounds, the block header will include a Merkle + // commitment to the set of online accounts (that can vote after + // another CompactCertRounds rounds), and that block will be signed + // (forming a compact certificate) by the voters from the previous + // such Merkle tree commitment. A value of zero means no compact + // certificates. + CompactCertRounds uint64 + + // CompactCertTopVoters is a bound on how many online accounts get to + // participate in forming the compact certificate, by including the + // top CompactCertTopVoters accounts (by normalized balance) into the + // Merkle commitment. + CompactCertTopVoters uint64 + + // CompactCertVotersLookback is the number of blocks we skip before + // publishing a Merkle commitment to the online accounts. Namely, + // if block number N contains a Merkle commitment to the online + // accounts (which, incidentally, means N%CompactCertRounds=0), + // then the balances reflected in that commitment must come from + // block N-CompactCertVotersLookback. This gives each node some + // time (CompactCertVotersLookback blocks worth of time) to + // construct this Merkle tree, so as to avoid placing the + // construction of this Merkle tree (and obtaining the requisite + // accounts and balances) in the critical path. + CompactCertVotersLookback uint64 } // ConsensusProtocols defines a set of supported protocol versions and their diff --git a/data/basics/userBalance.go b/data/basics/userBalance.go index 7ec30b836a..2ccb5eeb40 100644 --- a/data/basics/userBalance.go +++ b/data/basics/userBalance.go @@ -381,6 +381,14 @@ func (u AccountData) Money(proto config.ConsensusParams, rewardsLevel uint64) (m return e.MicroAlgos, e.RewardedMicroAlgos } +// PendingRewards computes the amount of rewards (in microalgos) that +// have yet to be added to the account balance. +func PendingRewards(ot *OverflowTracker, proto config.ConsensusParams, microAlgos MicroAlgos, rewardsBase uint64, rewardsLevel uint64) MicroAlgos { + rewardsUnits := microAlgos.RewardUnits(proto) + rewardsDelta := ot.Sub(rewardsLevel, rewardsBase) + return MicroAlgos{Raw: ot.Mul(rewardsUnits, rewardsDelta)} +} + // WithUpdatedRewards returns an updated number of algos in an AccountData // to reflect rewards up to some rewards level. func (u AccountData) WithUpdatedRewards(proto config.ConsensusParams, rewardsLevel uint64) AccountData { diff --git a/data/bookkeeping/block.go b/data/bookkeeping/block.go index 54426066b6..8a4d79194e 100644 --- a/data/bookkeeping/block.go +++ b/data/bookkeeping/block.go @@ -121,6 +121,24 @@ type ( // transactions have ever been committed (since TxnCounter // started being supported). TxnCounter uint64 `codec:"tc"` + + // CompactCertVoters is the root of a Merkle tree containing the + // online accounts that will help sign a compact certificate. The + // Merkle root, and the compact certificate, happen on blocks that + // are a multiple of ConsensusParams.CompactCertRounds. For blocks + // that are not a multiple of ConsensusParams.CompactCertRounds, + // this value is zero. + CompactCertVoters crypto.Digest `codec:"ccv"` + + // CompactCertVotersTotal is the total number of microalgos held by + // the accounts in CompactCertVoters (or zero, if the merkle root is + // zero). This is intended for computing the threshold of votes to + // expect from CompactCertVoters. + CompactCertVotersTotal basics.MicroAlgos `codec:"ccvt"` + + // CompactCertLastRound is the last round for which we have committed + // a CompactCert transaction. + CompactCertLastRound basics.Round `codec:"ccl"` } // RewardsState represents the global parameters controlling the rate diff --git a/data/bookkeeping/msgp_gen.go b/data/bookkeeping/msgp_gen.go index a93dfe395e..76ed260a44 100644 --- a/data/bookkeeping/msgp_gen.go +++ b/data/bookkeeping/msgp_gen.go @@ -69,109 +69,148 @@ import ( func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0001Len := uint32(23) - var zb0001Mask uint32 /* 26 bits */ - if (*z).BlockHeader.RewardsState.RewardsLevel == 0 { + zb0001Len := uint32(26) + var zb0001Mask uint32 /* 29 bits */ + if (*z).BlockHeader.CompactCertLastRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x8 } - if (*z).BlockHeader.RewardsState.FeeSink.MsgIsZero() { + if (*z).BlockHeader.CompactCertVoters.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x10 } - if (*z).BlockHeader.RewardsState.RewardsResidue == 0 { + if (*z).BlockHeader.CompactCertVotersTotal.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x20 } - if (*z).BlockHeader.GenesisID == "" { + if (*z).BlockHeader.RewardsState.RewardsLevel == 0 { zb0001Len-- zb0001Mask |= 0x40 } - if (*z).BlockHeader.GenesisHash.MsgIsZero() { + if (*z).BlockHeader.RewardsState.FeeSink.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x80 } - if (*z).BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero() { + if (*z).BlockHeader.RewardsState.RewardsResidue == 0 { zb0001Len-- zb0001Mask |= 0x100 } - if (*z).BlockHeader.UpgradeState.NextProtocol.MsgIsZero() { + if (*z).BlockHeader.GenesisID == "" { zb0001Len-- zb0001Mask |= 0x200 } - if (*z).BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero() { + if (*z).BlockHeader.GenesisHash.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x400 } - if (*z).BlockHeader.UpgradeState.NextProtocolApprovals == 0 { + if (*z).BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x800 } - if (*z).BlockHeader.Branch.MsgIsZero() { + if (*z).BlockHeader.UpgradeState.NextProtocol.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x1000 } - if (*z).BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero() { + if (*z).BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2000 } - if (*z).BlockHeader.RewardsState.RewardsRate == 0 { + if (*z).BlockHeader.UpgradeState.NextProtocolApprovals == 0 { zb0001Len-- zb0001Mask |= 0x4000 } - if (*z).BlockHeader.Round.MsgIsZero() { + if (*z).BlockHeader.Branch.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x8000 } - if (*z).BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero() { + if (*z).BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x10000 } - if (*z).BlockHeader.RewardsState.RewardsPool.MsgIsZero() { + if (*z).BlockHeader.RewardsState.RewardsRate == 0 { zb0001Len-- zb0001Mask |= 0x20000 } - if (*z).BlockHeader.Seed.MsgIsZero() { + if (*z).BlockHeader.Round.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x40000 } - if (*z).BlockHeader.TxnCounter == 0 { + if (*z).BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x80000 } - if (*z).BlockHeader.TimeStamp == 0 { + if (*z).BlockHeader.RewardsState.RewardsPool.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x100000 } - if (*z).BlockHeader.TxnRoot.MsgIsZero() { + if (*z).BlockHeader.Seed.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x200000 } - if (*z).Payset.MsgIsZero() { + if (*z).BlockHeader.TxnCounter == 0 { zb0001Len-- zb0001Mask |= 0x400000 } - if (*z).BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero() { + if (*z).BlockHeader.TimeStamp == 0 { zb0001Len-- zb0001Mask |= 0x800000 } - if (*z).BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero() { + if (*z).BlockHeader.TxnRoot.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x1000000 } - if (*z).BlockHeader.UpgradeVote.UpgradeApprove == false { + if (*z).Payset.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2000000 } + if (*z).BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4000000 + } + if (*z).BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x8000000 + } + if (*z).BlockHeader.UpgradeVote.UpgradeApprove == false { + zb0001Len-- + zb0001Mask |= 0x10000000 + } // variable map header, size zb0001Len o = msgp.AppendMapHeader(o, zb0001Len) if zb0001Len != 0 { if (zb0001Mask & 0x8) == 0 { // if not empty + // string "ccl" + o = append(o, 0xa3, 0x63, 0x63, 0x6c) + o, err = (*z).BlockHeader.CompactCertLastRound.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertLastRound") + return + } + } + if (zb0001Mask & 0x10) == 0 { // if not empty + // string "ccv" + o = append(o, 0xa3, 0x63, 0x63, 0x76) + o, err = (*z).BlockHeader.CompactCertVoters.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertVoters") + return + } + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // string "ccvt" + o = append(o, 0xa4, 0x63, 0x63, 0x76, 0x74) + o, err = (*z).BlockHeader.CompactCertVotersTotal.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertVotersTotal") + return + } + } + if (zb0001Mask & 0x40) == 0 { // if not empty // string "earn" o = append(o, 0xa4, 0x65, 0x61, 0x72, 0x6e) o = msgp.AppendUint64(o, (*z).BlockHeader.RewardsState.RewardsLevel) } - if (zb0001Mask & 0x10) == 0 { // if not empty + if (zb0001Mask & 0x80) == 0 { // if not empty // string "fees" o = append(o, 0xa4, 0x66, 0x65, 0x65, 0x73) o, err = (*z).BlockHeader.RewardsState.FeeSink.MarshalMsg(o) @@ -180,17 +219,17 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x20) == 0 { // if not empty + if (zb0001Mask & 0x100) == 0 { // if not empty // string "frac" o = append(o, 0xa4, 0x66, 0x72, 0x61, 0x63) o = msgp.AppendUint64(o, (*z).BlockHeader.RewardsState.RewardsResidue) } - if (zb0001Mask & 0x40) == 0 { // if not empty + if (zb0001Mask & 0x200) == 0 { // if not empty // string "gen" o = append(o, 0xa3, 0x67, 0x65, 0x6e) o = msgp.AppendString(o, (*z).BlockHeader.GenesisID) } - if (zb0001Mask & 0x80) == 0 { // if not empty + if (zb0001Mask & 0x400) == 0 { // if not empty // string "gh" o = append(o, 0xa2, 0x67, 0x68) o, err = (*z).BlockHeader.GenesisHash.MarshalMsg(o) @@ -199,7 +238,7 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x100) == 0 { // if not empty + if (zb0001Mask & 0x800) == 0 { // if not empty // string "nextbefore" o = append(o, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65) o, err = (*z).BlockHeader.UpgradeState.NextProtocolVoteBefore.MarshalMsg(o) @@ -208,7 +247,7 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x200) == 0 { // if not empty + if (zb0001Mask & 0x1000) == 0 { // if not empty // string "nextproto" o = append(o, 0xa9, 0x6e, 0x65, 0x78, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f) o, err = (*z).BlockHeader.UpgradeState.NextProtocol.MarshalMsg(o) @@ -217,7 +256,7 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x400) == 0 { // if not empty + if (zb0001Mask & 0x2000) == 0 { // if not empty // string "nextswitch" o = append(o, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x73, 0x77, 0x69, 0x74, 0x63, 0x68) o, err = (*z).BlockHeader.UpgradeState.NextProtocolSwitchOn.MarshalMsg(o) @@ -226,12 +265,12 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x800) == 0 { // if not empty + if (zb0001Mask & 0x4000) == 0 { // if not empty // string "nextyes" o = append(o, 0xa7, 0x6e, 0x65, 0x78, 0x74, 0x79, 0x65, 0x73) o = msgp.AppendUint64(o, (*z).BlockHeader.UpgradeState.NextProtocolApprovals) } - if (zb0001Mask & 0x1000) == 0 { // if not empty + if (zb0001Mask & 0x8000) == 0 { // if not empty // string "prev" o = append(o, 0xa4, 0x70, 0x72, 0x65, 0x76) o, err = (*z).BlockHeader.Branch.MarshalMsg(o) @@ -240,7 +279,7 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x2000) == 0 { // if not empty + if (zb0001Mask & 0x10000) == 0 { // if not empty // string "proto" o = append(o, 0xa5, 0x70, 0x72, 0x6f, 0x74, 0x6f) o, err = (*z).BlockHeader.UpgradeState.CurrentProtocol.MarshalMsg(o) @@ -249,12 +288,12 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x4000) == 0 { // if not empty + if (zb0001Mask & 0x20000) == 0 { // if not empty // string "rate" o = append(o, 0xa4, 0x72, 0x61, 0x74, 0x65) o = msgp.AppendUint64(o, (*z).BlockHeader.RewardsState.RewardsRate) } - if (zb0001Mask & 0x8000) == 0 { // if not empty + if (zb0001Mask & 0x40000) == 0 { // if not empty // string "rnd" o = append(o, 0xa3, 0x72, 0x6e, 0x64) o, err = (*z).BlockHeader.Round.MarshalMsg(o) @@ -263,7 +302,7 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x10000) == 0 { // if not empty + if (zb0001Mask & 0x80000) == 0 { // if not empty // string "rwcalr" o = append(o, 0xa6, 0x72, 0x77, 0x63, 0x61, 0x6c, 0x72) o, err = (*z).BlockHeader.RewardsState.RewardsRecalculationRound.MarshalMsg(o) @@ -272,7 +311,7 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x20000) == 0 { // if not empty + if (zb0001Mask & 0x100000) == 0 { // if not empty // string "rwd" o = append(o, 0xa3, 0x72, 0x77, 0x64) o, err = (*z).BlockHeader.RewardsState.RewardsPool.MarshalMsg(o) @@ -281,7 +320,7 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x40000) == 0 { // if not empty + if (zb0001Mask & 0x200000) == 0 { // if not empty // string "seed" o = append(o, 0xa4, 0x73, 0x65, 0x65, 0x64) o, err = (*z).BlockHeader.Seed.MarshalMsg(o) @@ -290,17 +329,17 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x80000) == 0 { // if not empty + if (zb0001Mask & 0x400000) == 0 { // if not empty // string "tc" o = append(o, 0xa2, 0x74, 0x63) o = msgp.AppendUint64(o, (*z).BlockHeader.TxnCounter) } - if (zb0001Mask & 0x100000) == 0 { // if not empty + if (zb0001Mask & 0x800000) == 0 { // if not empty // string "ts" o = append(o, 0xa2, 0x74, 0x73) o = msgp.AppendInt64(o, (*z).BlockHeader.TimeStamp) } - if (zb0001Mask & 0x200000) == 0 { // if not empty + if (zb0001Mask & 0x1000000) == 0 { // if not empty // string "txn" o = append(o, 0xa3, 0x74, 0x78, 0x6e) o, err = (*z).BlockHeader.TxnRoot.MarshalMsg(o) @@ -309,7 +348,7 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x400000) == 0 { // if not empty + if (zb0001Mask & 0x2000000) == 0 { // if not empty // string "txns" o = append(o, 0xa4, 0x74, 0x78, 0x6e, 0x73) o, err = (*z).Payset.MarshalMsg(o) @@ -318,7 +357,7 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x800000) == 0 { // if not empty + if (zb0001Mask & 0x4000000) == 0 { // if not empty // string "upgradedelay" o = append(o, 0xac, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x64, 0x65, 0x6c, 0x61, 0x79) o, err = (*z).BlockHeader.UpgradeVote.UpgradeDelay.MarshalMsg(o) @@ -327,7 +366,7 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x1000000) == 0 { // if not empty + if (zb0001Mask & 0x8000000) == 0 { // if not empty // string "upgradeprop" o = append(o, 0xab, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x70, 0x72, 0x6f, 0x70) o, err = (*z).BlockHeader.UpgradeVote.UpgradePropose.MarshalMsg(o) @@ -336,7 +375,7 @@ func (z *Block) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x2000000) == 0 { // if not empty + if (zb0001Mask & 0x10000000) == 0 { // if not empty // string "upgradeyes" o = append(o, 0xaa, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x79, 0x65, 0x73) o = msgp.AppendBool(o, (*z).BlockHeader.UpgradeVote.UpgradeApprove) @@ -539,6 +578,30 @@ func (z *Block) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BlockHeader.CompactCertVoters.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertVoters") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BlockHeader.CompactCertVotersTotal.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertVotersTotal") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).BlockHeader.CompactCertLastRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertLastRound") + return + } + } if zb0001 > 0 { zb0001-- bts, err = (*z).Payset.UnmarshalMsg(bts) @@ -702,6 +765,24 @@ func (z *Block) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "TxnCounter") return } + case "ccv": + bts, err = (*z).BlockHeader.CompactCertVoters.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertVoters") + return + } + case "ccvt": + bts, err = (*z).BlockHeader.CompactCertVotersTotal.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertVotersTotal") + return + } + case "ccl": + bts, err = (*z).BlockHeader.CompactCertLastRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertLastRound") + return + } case "txns": bts, err = (*z).Payset.UnmarshalMsg(bts) if err != nil { @@ -728,13 +809,13 @@ func (_ *Block) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *Block) Msgsize() (s int) { - s = 3 + 4 + (*z).BlockHeader.Round.Msgsize() + 5 + (*z).BlockHeader.Branch.Msgsize() + 5 + (*z).BlockHeader.Seed.Msgsize() + 4 + (*z).BlockHeader.TxnRoot.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len((*z).BlockHeader.GenesisID) + 3 + (*z).BlockHeader.GenesisHash.Msgsize() + 5 + (*z).BlockHeader.RewardsState.FeeSink.Msgsize() + 4 + (*z).BlockHeader.RewardsState.RewardsPool.Msgsize() + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 7 + (*z).BlockHeader.RewardsState.RewardsRecalculationRound.Msgsize() + 6 + (*z).BlockHeader.UpgradeState.CurrentProtocol.Msgsize() + 10 + (*z).BlockHeader.UpgradeState.NextProtocol.Msgsize() + 8 + msgp.Uint64Size + 11 + (*z).BlockHeader.UpgradeState.NextProtocolVoteBefore.Msgsize() + 11 + (*z).BlockHeader.UpgradeState.NextProtocolSwitchOn.Msgsize() + 12 + (*z).BlockHeader.UpgradeVote.UpgradePropose.Msgsize() + 13 + (*z).BlockHeader.UpgradeVote.UpgradeDelay.Msgsize() + 11 + msgp.BoolSize + 3 + msgp.Uint64Size + 5 + (*z).Payset.Msgsize() + s = 3 + 4 + (*z).BlockHeader.Round.Msgsize() + 5 + (*z).BlockHeader.Branch.Msgsize() + 5 + (*z).BlockHeader.Seed.Msgsize() + 4 + (*z).BlockHeader.TxnRoot.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len((*z).BlockHeader.GenesisID) + 3 + (*z).BlockHeader.GenesisHash.Msgsize() + 5 + (*z).BlockHeader.RewardsState.FeeSink.Msgsize() + 4 + (*z).BlockHeader.RewardsState.RewardsPool.Msgsize() + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 7 + (*z).BlockHeader.RewardsState.RewardsRecalculationRound.Msgsize() + 6 + (*z).BlockHeader.UpgradeState.CurrentProtocol.Msgsize() + 10 + (*z).BlockHeader.UpgradeState.NextProtocol.Msgsize() + 8 + msgp.Uint64Size + 11 + (*z).BlockHeader.UpgradeState.NextProtocolVoteBefore.Msgsize() + 11 + (*z).BlockHeader.UpgradeState.NextProtocolSwitchOn.Msgsize() + 12 + (*z).BlockHeader.UpgradeVote.UpgradePropose.Msgsize() + 13 + (*z).BlockHeader.UpgradeVote.UpgradeDelay.Msgsize() + 11 + msgp.BoolSize + 3 + msgp.Uint64Size + 4 + (*z).BlockHeader.CompactCertVoters.Msgsize() + 5 + (*z).BlockHeader.CompactCertVotersTotal.Msgsize() + 4 + (*z).BlockHeader.CompactCertLastRound.Msgsize() + 5 + (*z).Payset.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *Block) MsgIsZero() bool { - return ((*z).BlockHeader.Round.MsgIsZero()) && ((*z).BlockHeader.Branch.MsgIsZero()) && ((*z).BlockHeader.Seed.MsgIsZero()) && ((*z).BlockHeader.TxnRoot.MsgIsZero()) && ((*z).BlockHeader.TimeStamp == 0) && ((*z).BlockHeader.GenesisID == "") && ((*z).BlockHeader.GenesisHash.MsgIsZero()) && ((*z).BlockHeader.RewardsState.FeeSink.MsgIsZero()) && ((*z).BlockHeader.RewardsState.RewardsPool.MsgIsZero()) && ((*z).BlockHeader.RewardsState.RewardsLevel == 0) && ((*z).BlockHeader.RewardsState.RewardsRate == 0) && ((*z).BlockHeader.RewardsState.RewardsResidue == 0) && ((*z).BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero()) && ((*z).BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero()) && ((*z).BlockHeader.UpgradeState.NextProtocol.MsgIsZero()) && ((*z).BlockHeader.UpgradeState.NextProtocolApprovals == 0) && ((*z).BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero()) && ((*z).BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero()) && ((*z).BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero()) && ((*z).BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero()) && ((*z).BlockHeader.UpgradeVote.UpgradeApprove == false) && ((*z).BlockHeader.TxnCounter == 0) && ((*z).Payset.MsgIsZero()) + return ((*z).BlockHeader.Round.MsgIsZero()) && ((*z).BlockHeader.Branch.MsgIsZero()) && ((*z).BlockHeader.Seed.MsgIsZero()) && ((*z).BlockHeader.TxnRoot.MsgIsZero()) && ((*z).BlockHeader.TimeStamp == 0) && ((*z).BlockHeader.GenesisID == "") && ((*z).BlockHeader.GenesisHash.MsgIsZero()) && ((*z).BlockHeader.RewardsState.FeeSink.MsgIsZero()) && ((*z).BlockHeader.RewardsState.RewardsPool.MsgIsZero()) && ((*z).BlockHeader.RewardsState.RewardsLevel == 0) && ((*z).BlockHeader.RewardsState.RewardsRate == 0) && ((*z).BlockHeader.RewardsState.RewardsResidue == 0) && ((*z).BlockHeader.RewardsState.RewardsRecalculationRound.MsgIsZero()) && ((*z).BlockHeader.UpgradeState.CurrentProtocol.MsgIsZero()) && ((*z).BlockHeader.UpgradeState.NextProtocol.MsgIsZero()) && ((*z).BlockHeader.UpgradeState.NextProtocolApprovals == 0) && ((*z).BlockHeader.UpgradeState.NextProtocolVoteBefore.MsgIsZero()) && ((*z).BlockHeader.UpgradeState.NextProtocolSwitchOn.MsgIsZero()) && ((*z).BlockHeader.UpgradeVote.UpgradePropose.MsgIsZero()) && ((*z).BlockHeader.UpgradeVote.UpgradeDelay.MsgIsZero()) && ((*z).BlockHeader.UpgradeVote.UpgradeApprove == false) && ((*z).BlockHeader.TxnCounter == 0) && ((*z).BlockHeader.CompactCertVoters.MsgIsZero()) && ((*z).BlockHeader.CompactCertVotersTotal.MsgIsZero()) && ((*z).BlockHeader.CompactCertLastRound.MsgIsZero()) && ((*z).Payset.MsgIsZero()) } // MarshalMsg implements msgp.Marshaler @@ -769,105 +850,144 @@ func (z *BlockHash) MsgIsZero() bool { func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0001Len := uint32(22) - var zb0001Mask uint32 /* 25 bits */ - if (*z).RewardsState.RewardsLevel == 0 { + zb0001Len := uint32(25) + var zb0001Mask uint32 /* 28 bits */ + if (*z).CompactCertLastRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x8 } - if (*z).RewardsState.FeeSink.MsgIsZero() { + if (*z).CompactCertVoters.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x10 } - if (*z).RewardsState.RewardsResidue == 0 { + if (*z).CompactCertVotersTotal.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x20 } - if (*z).GenesisID == "" { + if (*z).RewardsState.RewardsLevel == 0 { zb0001Len-- zb0001Mask |= 0x40 } - if (*z).GenesisHash.MsgIsZero() { + if (*z).RewardsState.FeeSink.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x80 } - if (*z).UpgradeState.NextProtocolVoteBefore.MsgIsZero() { + if (*z).RewardsState.RewardsResidue == 0 { zb0001Len-- zb0001Mask |= 0x100 } - if (*z).UpgradeState.NextProtocol.MsgIsZero() { + if (*z).GenesisID == "" { zb0001Len-- zb0001Mask |= 0x200 } - if (*z).UpgradeState.NextProtocolSwitchOn.MsgIsZero() { + if (*z).GenesisHash.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x400 } - if (*z).UpgradeState.NextProtocolApprovals == 0 { + if (*z).UpgradeState.NextProtocolVoteBefore.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x800 } - if (*z).Branch.MsgIsZero() { + if (*z).UpgradeState.NextProtocol.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x1000 } - if (*z).UpgradeState.CurrentProtocol.MsgIsZero() { + if (*z).UpgradeState.NextProtocolSwitchOn.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x2000 } - if (*z).RewardsState.RewardsRate == 0 { + if (*z).UpgradeState.NextProtocolApprovals == 0 { zb0001Len-- zb0001Mask |= 0x4000 } - if (*z).Round.MsgIsZero() { + if (*z).Branch.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x8000 } - if (*z).RewardsState.RewardsRecalculationRound.MsgIsZero() { + if (*z).UpgradeState.CurrentProtocol.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x10000 } - if (*z).RewardsState.RewardsPool.MsgIsZero() { + if (*z).RewardsState.RewardsRate == 0 { zb0001Len-- zb0001Mask |= 0x20000 } - if (*z).Seed.MsgIsZero() { + if (*z).Round.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x40000 } - if (*z).TxnCounter == 0 { + if (*z).RewardsState.RewardsRecalculationRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x80000 } - if (*z).TimeStamp == 0 { + if (*z).RewardsState.RewardsPool.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x100000 } - if (*z).TxnRoot.MsgIsZero() { + if (*z).Seed.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x200000 } - if (*z).UpgradeVote.UpgradeDelay.MsgIsZero() { + if (*z).TxnCounter == 0 { zb0001Len-- zb0001Mask |= 0x400000 } - if (*z).UpgradeVote.UpgradePropose.MsgIsZero() { + if (*z).TimeStamp == 0 { zb0001Len-- zb0001Mask |= 0x800000 } - if (*z).UpgradeVote.UpgradeApprove == false { + if (*z).TxnRoot.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x1000000 } + if (*z).UpgradeVote.UpgradeDelay.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2000000 + } + if (*z).UpgradeVote.UpgradePropose.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4000000 + } + if (*z).UpgradeVote.UpgradeApprove == false { + zb0001Len-- + zb0001Mask |= 0x8000000 + } // variable map header, size zb0001Len o = msgp.AppendMapHeader(o, zb0001Len) if zb0001Len != 0 { if (zb0001Mask & 0x8) == 0 { // if not empty + // string "ccl" + o = append(o, 0xa3, 0x63, 0x63, 0x6c) + o, err = (*z).CompactCertLastRound.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertLastRound") + return + } + } + if (zb0001Mask & 0x10) == 0 { // if not empty + // string "ccv" + o = append(o, 0xa3, 0x63, 0x63, 0x76) + o, err = (*z).CompactCertVoters.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertVoters") + return + } + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // string "ccvt" + o = append(o, 0xa4, 0x63, 0x63, 0x76, 0x74) + o, err = (*z).CompactCertVotersTotal.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CompactCertVotersTotal") + return + } + } + if (zb0001Mask & 0x40) == 0 { // if not empty // string "earn" o = append(o, 0xa4, 0x65, 0x61, 0x72, 0x6e) o = msgp.AppendUint64(o, (*z).RewardsState.RewardsLevel) } - if (zb0001Mask & 0x10) == 0 { // if not empty + if (zb0001Mask & 0x80) == 0 { // if not empty // string "fees" o = append(o, 0xa4, 0x66, 0x65, 0x65, 0x73) o, err = (*z).RewardsState.FeeSink.MarshalMsg(o) @@ -876,17 +996,17 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x20) == 0 { // if not empty + if (zb0001Mask & 0x100) == 0 { // if not empty // string "frac" o = append(o, 0xa4, 0x66, 0x72, 0x61, 0x63) o = msgp.AppendUint64(o, (*z).RewardsState.RewardsResidue) } - if (zb0001Mask & 0x40) == 0 { // if not empty + if (zb0001Mask & 0x200) == 0 { // if not empty // string "gen" o = append(o, 0xa3, 0x67, 0x65, 0x6e) o = msgp.AppendString(o, (*z).GenesisID) } - if (zb0001Mask & 0x80) == 0 { // if not empty + if (zb0001Mask & 0x400) == 0 { // if not empty // string "gh" o = append(o, 0xa2, 0x67, 0x68) o, err = (*z).GenesisHash.MarshalMsg(o) @@ -895,7 +1015,7 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x100) == 0 { // if not empty + if (zb0001Mask & 0x800) == 0 { // if not empty // string "nextbefore" o = append(o, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65) o, err = (*z).UpgradeState.NextProtocolVoteBefore.MarshalMsg(o) @@ -904,7 +1024,7 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x200) == 0 { // if not empty + if (zb0001Mask & 0x1000) == 0 { // if not empty // string "nextproto" o = append(o, 0xa9, 0x6e, 0x65, 0x78, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f) o, err = (*z).UpgradeState.NextProtocol.MarshalMsg(o) @@ -913,7 +1033,7 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x400) == 0 { // if not empty + if (zb0001Mask & 0x2000) == 0 { // if not empty // string "nextswitch" o = append(o, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x73, 0x77, 0x69, 0x74, 0x63, 0x68) o, err = (*z).UpgradeState.NextProtocolSwitchOn.MarshalMsg(o) @@ -922,12 +1042,12 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x800) == 0 { // if not empty + if (zb0001Mask & 0x4000) == 0 { // if not empty // string "nextyes" o = append(o, 0xa7, 0x6e, 0x65, 0x78, 0x74, 0x79, 0x65, 0x73) o = msgp.AppendUint64(o, (*z).UpgradeState.NextProtocolApprovals) } - if (zb0001Mask & 0x1000) == 0 { // if not empty + if (zb0001Mask & 0x8000) == 0 { // if not empty // string "prev" o = append(o, 0xa4, 0x70, 0x72, 0x65, 0x76) o, err = (*z).Branch.MarshalMsg(o) @@ -936,7 +1056,7 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x2000) == 0 { // if not empty + if (zb0001Mask & 0x10000) == 0 { // if not empty // string "proto" o = append(o, 0xa5, 0x70, 0x72, 0x6f, 0x74, 0x6f) o, err = (*z).UpgradeState.CurrentProtocol.MarshalMsg(o) @@ -945,12 +1065,12 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x4000) == 0 { // if not empty + if (zb0001Mask & 0x20000) == 0 { // if not empty // string "rate" o = append(o, 0xa4, 0x72, 0x61, 0x74, 0x65) o = msgp.AppendUint64(o, (*z).RewardsState.RewardsRate) } - if (zb0001Mask & 0x8000) == 0 { // if not empty + if (zb0001Mask & 0x40000) == 0 { // if not empty // string "rnd" o = append(o, 0xa3, 0x72, 0x6e, 0x64) o, err = (*z).Round.MarshalMsg(o) @@ -959,7 +1079,7 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x10000) == 0 { // if not empty + if (zb0001Mask & 0x80000) == 0 { // if not empty // string "rwcalr" o = append(o, 0xa6, 0x72, 0x77, 0x63, 0x61, 0x6c, 0x72) o, err = (*z).RewardsState.RewardsRecalculationRound.MarshalMsg(o) @@ -968,7 +1088,7 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x20000) == 0 { // if not empty + if (zb0001Mask & 0x100000) == 0 { // if not empty // string "rwd" o = append(o, 0xa3, 0x72, 0x77, 0x64) o, err = (*z).RewardsState.RewardsPool.MarshalMsg(o) @@ -977,7 +1097,7 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x40000) == 0 { // if not empty + if (zb0001Mask & 0x200000) == 0 { // if not empty // string "seed" o = append(o, 0xa4, 0x73, 0x65, 0x65, 0x64) o, err = (*z).Seed.MarshalMsg(o) @@ -986,17 +1106,17 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x80000) == 0 { // if not empty + if (zb0001Mask & 0x400000) == 0 { // if not empty // string "tc" o = append(o, 0xa2, 0x74, 0x63) o = msgp.AppendUint64(o, (*z).TxnCounter) } - if (zb0001Mask & 0x100000) == 0 { // if not empty + if (zb0001Mask & 0x800000) == 0 { // if not empty // string "ts" o = append(o, 0xa2, 0x74, 0x73) o = msgp.AppendInt64(o, (*z).TimeStamp) } - if (zb0001Mask & 0x200000) == 0 { // if not empty + if (zb0001Mask & 0x1000000) == 0 { // if not empty // string "txn" o = append(o, 0xa3, 0x74, 0x78, 0x6e) o, err = (*z).TxnRoot.MarshalMsg(o) @@ -1005,7 +1125,7 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x400000) == 0 { // if not empty + if (zb0001Mask & 0x2000000) == 0 { // if not empty // string "upgradedelay" o = append(o, 0xac, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x64, 0x65, 0x6c, 0x61, 0x79) o, err = (*z).UpgradeVote.UpgradeDelay.MarshalMsg(o) @@ -1014,7 +1134,7 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x800000) == 0 { // if not empty + if (zb0001Mask & 0x4000000) == 0 { // if not empty // string "upgradeprop" o = append(o, 0xab, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x70, 0x72, 0x6f, 0x70) o, err = (*z).UpgradeVote.UpgradePropose.MarshalMsg(o) @@ -1023,7 +1143,7 @@ func (z *BlockHeader) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0001Mask & 0x1000000) == 0 { // if not empty + if (zb0001Mask & 0x8000000) == 0 { // if not empty // string "upgradeyes" o = append(o, 0xaa, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x79, 0x65, 0x73) o = msgp.AppendBool(o, (*z).UpgradeVote.UpgradeApprove) @@ -1226,6 +1346,30 @@ func (z *BlockHeader) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).CompactCertVoters.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertVoters") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).CompactCertVotersTotal.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertVotersTotal") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).CompactCertLastRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CompactCertLastRound") + return + } + } if zb0001 > 0 { err = msgp.ErrTooManyArrayFields(zb0001) if err != nil { @@ -1381,6 +1525,24 @@ func (z *BlockHeader) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "TxnCounter") return } + case "ccv": + bts, err = (*z).CompactCertVoters.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertVoters") + return + } + case "ccvt": + bts, err = (*z).CompactCertVotersTotal.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertVotersTotal") + return + } + case "ccl": + bts, err = (*z).CompactCertLastRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CompactCertLastRound") + return + } default: err = msgp.ErrNoField(string(field)) if err != nil { @@ -1401,13 +1563,13 @@ func (_ *BlockHeader) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *BlockHeader) Msgsize() (s int) { - s = 3 + 4 + (*z).Round.Msgsize() + 5 + (*z).Branch.Msgsize() + 5 + (*z).Seed.Msgsize() + 4 + (*z).TxnRoot.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len((*z).GenesisID) + 3 + (*z).GenesisHash.Msgsize() + 5 + (*z).RewardsState.FeeSink.Msgsize() + 4 + (*z).RewardsState.RewardsPool.Msgsize() + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 7 + (*z).RewardsState.RewardsRecalculationRound.Msgsize() + 6 + (*z).UpgradeState.CurrentProtocol.Msgsize() + 10 + (*z).UpgradeState.NextProtocol.Msgsize() + 8 + msgp.Uint64Size + 11 + (*z).UpgradeState.NextProtocolVoteBefore.Msgsize() + 11 + (*z).UpgradeState.NextProtocolSwitchOn.Msgsize() + 12 + (*z).UpgradeVote.UpgradePropose.Msgsize() + 13 + (*z).UpgradeVote.UpgradeDelay.Msgsize() + 11 + msgp.BoolSize + 3 + msgp.Uint64Size + s = 3 + 4 + (*z).Round.Msgsize() + 5 + (*z).Branch.Msgsize() + 5 + (*z).Seed.Msgsize() + 4 + (*z).TxnRoot.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len((*z).GenesisID) + 3 + (*z).GenesisHash.Msgsize() + 5 + (*z).RewardsState.FeeSink.Msgsize() + 4 + (*z).RewardsState.RewardsPool.Msgsize() + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 5 + msgp.Uint64Size + 7 + (*z).RewardsState.RewardsRecalculationRound.Msgsize() + 6 + (*z).UpgradeState.CurrentProtocol.Msgsize() + 10 + (*z).UpgradeState.NextProtocol.Msgsize() + 8 + msgp.Uint64Size + 11 + (*z).UpgradeState.NextProtocolVoteBefore.Msgsize() + 11 + (*z).UpgradeState.NextProtocolSwitchOn.Msgsize() + 12 + (*z).UpgradeVote.UpgradePropose.Msgsize() + 13 + (*z).UpgradeVote.UpgradeDelay.Msgsize() + 11 + msgp.BoolSize + 3 + msgp.Uint64Size + 4 + (*z).CompactCertVoters.Msgsize() + 5 + (*z).CompactCertVotersTotal.Msgsize() + 4 + (*z).CompactCertLastRound.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *BlockHeader) MsgIsZero() bool { - return ((*z).Round.MsgIsZero()) && ((*z).Branch.MsgIsZero()) && ((*z).Seed.MsgIsZero()) && ((*z).TxnRoot.MsgIsZero()) && ((*z).TimeStamp == 0) && ((*z).GenesisID == "") && ((*z).GenesisHash.MsgIsZero()) && ((*z).RewardsState.FeeSink.MsgIsZero()) && ((*z).RewardsState.RewardsPool.MsgIsZero()) && ((*z).RewardsState.RewardsLevel == 0) && ((*z).RewardsState.RewardsRate == 0) && ((*z).RewardsState.RewardsResidue == 0) && ((*z).RewardsState.RewardsRecalculationRound.MsgIsZero()) && ((*z).UpgradeState.CurrentProtocol.MsgIsZero()) && ((*z).UpgradeState.NextProtocol.MsgIsZero()) && ((*z).UpgradeState.NextProtocolApprovals == 0) && ((*z).UpgradeState.NextProtocolVoteBefore.MsgIsZero()) && ((*z).UpgradeState.NextProtocolSwitchOn.MsgIsZero()) && ((*z).UpgradeVote.UpgradePropose.MsgIsZero()) && ((*z).UpgradeVote.UpgradeDelay.MsgIsZero()) && ((*z).UpgradeVote.UpgradeApprove == false) && ((*z).TxnCounter == 0) + return ((*z).Round.MsgIsZero()) && ((*z).Branch.MsgIsZero()) && ((*z).Seed.MsgIsZero()) && ((*z).TxnRoot.MsgIsZero()) && ((*z).TimeStamp == 0) && ((*z).GenesisID == "") && ((*z).GenesisHash.MsgIsZero()) && ((*z).RewardsState.FeeSink.MsgIsZero()) && ((*z).RewardsState.RewardsPool.MsgIsZero()) && ((*z).RewardsState.RewardsLevel == 0) && ((*z).RewardsState.RewardsRate == 0) && ((*z).RewardsState.RewardsResidue == 0) && ((*z).RewardsState.RewardsRecalculationRound.MsgIsZero()) && ((*z).UpgradeState.CurrentProtocol.MsgIsZero()) && ((*z).UpgradeState.NextProtocol.MsgIsZero()) && ((*z).UpgradeState.NextProtocolApprovals == 0) && ((*z).UpgradeState.NextProtocolVoteBefore.MsgIsZero()) && ((*z).UpgradeState.NextProtocolSwitchOn.MsgIsZero()) && ((*z).UpgradeVote.UpgradePropose.MsgIsZero()) && ((*z).UpgradeVote.UpgradeDelay.MsgIsZero()) && ((*z).UpgradeVote.UpgradeApprove == false) && ((*z).TxnCounter == 0) && ((*z).CompactCertVoters.MsgIsZero()) && ((*z).CompactCertVotersTotal.MsgIsZero()) && ((*z).CompactCertLastRound.MsgIsZero()) } // MarshalMsg implements msgp.Marshaler diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index 78d05fe8b6..e25e979d65 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -205,6 +205,9 @@ type accountUpdates struct { // commitSyncerClosed is the blocking channel for syncronizing closing the commitSyncer goroutine. Once it's closed, the // commitSyncer can be assumed to have aborted. commitSyncerClosed chan struct{} + + // voters keeps track of Merkle trees of online accounts, used for compact certificates. + voters *votersTracker } type deferedCommit struct { @@ -274,6 +277,12 @@ func (au *accountUpdates) loadFromDisk(l ledgerForTracker) error { au.generateCatchpoint(basics.Round(writingCatchpointRound), au.lastCatchpointLabel, writingCatchpointDigest, time.Duration(0)) } + au.voters = &votersTracker{} + err = au.voters.loadFromDisk(l, au) + if err != nil { + return err + } + return nil } @@ -594,6 +603,8 @@ func (au *accountUpdates) committedUpTo(committedRound basics.Round) (retRound b return } + newBase = au.voters.lowestRound(newBase) + offset = uint64(newBase - au.dbRound) // check to see if this is a catchpoint round @@ -718,6 +729,11 @@ func (aul *accountUpdatesLedgerEvaluator) GenesisHash() crypto.Digest { return aul.au.ledger.GenesisHash() } +// CompactCertVoters returns the top online accounts at round rnd. +func (aul *accountUpdatesLedgerEvaluator) CompactCertVoters(rnd basics.Round) (voters *VotersForRound, err error) { + return aul.au.voters.getVoters(rnd) +} + // BlockHdr returns the header of the given round. When the evaluator is running, it's only referring to the previous header, which is what we // are providing here. Any attempt to access a different header would get denied. func (aul *accountUpdatesLedgerEvaluator) BlockHdr(r basics.Round) (bookkeeping.BlockHeader, error) { @@ -1336,6 +1352,8 @@ func (au *accountUpdates) newBlockImpl(blk bookkeeping.Block, delta StateDelta) } au.roundTotals = append(au.roundTotals, newTotals) + + au.voters.newBlock(blk.BlockHeader) } // lookupImpl returns the accound data for a given address at a given round. The withRewards indicates whether the diff --git a/ledger/eval.go b/ledger/eval.go index d898b9951a..092e9b3646 100644 --- a/ledger/eval.go +++ b/ledger/eval.go @@ -188,6 +188,7 @@ type ledgerForEvaluator interface { isDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, txlease) (bool, error) LookupWithoutRewards(basics.Round, basics.Address) (basics.AccountData, error) GetCreatorForRound(basics.Round, basics.CreatableIndex, basics.CreatableType) (basics.Address, bool, error) + CompactCertVoters(basics.Round) (*VotersForRound, error) } // StartEvaluator creates a BlockEvaluator, given a ledger and a block header @@ -753,6 +754,31 @@ func applyTransaction(tx transactions.Transaction, balances apply.Balances, stev return } +// compactCertVotersAndTotal returns the expected values of CompactCertVoters +// and CompactCertVotersTotal for a block. +func (eval *BlockEvaluator) compactCertVotersAndTotal() (root crypto.Digest, total basics.MicroAlgos, err error) { + if eval.proto.CompactCertRounds == 0 { + return + } + + if eval.block.Round()%basics.Round(eval.proto.CompactCertRounds) != 0 { + return + } + + lookback := eval.block.Round().SubSaturate(basics.Round(eval.proto.CompactCertVotersLookback)) + voters, err := eval.l.CompactCertVoters(lookback) + if err != nil { + return + } + + if voters != nil { + root = voters.Tree.Root() + total = voters.TotalWeight + } + + return +} + // Call "endOfBlock" after all the block's rewards and transactions are processed. func (eval *BlockEvaluator) endOfBlock() error { if eval.generate { @@ -762,6 +788,12 @@ func (eval *BlockEvaluator) endOfBlock() error { } else { eval.block.TxnCounter = 0 } + + var err error + eval.block.CompactCertVoters, eval.block.CompactCertVotersTotal, err = eval.compactCertVotersAndTotal() + if err != nil { + return err + } } return nil @@ -783,6 +815,20 @@ func (eval *BlockEvaluator) finalValidation() error { if eval.block.TxnCounter != expectedTxnCount { return fmt.Errorf("txn count wrong: %d != %d", eval.block.TxnCounter, expectedTxnCount) } + + expectedVoters, expectedVotersWeight, err := eval.compactCertVotersAndTotal() + if err != nil { + return err + } + if eval.block.CompactCertVoters != expectedVoters { + return fmt.Errorf("CompactCertVoters wrong: %v != %v", eval.block.CompactCertVoters, expectedVoters) + } + if eval.block.CompactCertVotersTotal != expectedVotersWeight { + return fmt.Errorf("CompactCertVotersTotal wrong: %v != %v", eval.block.CompactCertVotersTotal, expectedVotersWeight) + } + if eval.block.CompactCertLastRound != 0 { + return fmt.Errorf("CompactCertLastRound wrong: %v != %v", eval.block.CompactCertLastRound, 0) + } } return nil diff --git a/ledger/ledger.go b/ledger/ledger.go index f7d6c82bb4..0464f188d9 100644 --- a/ledger/ledger.go +++ b/ledger/ledger.go @@ -367,6 +367,15 @@ func (l *Ledger) GetCreator(cidx basics.CreatableIndex, ctype basics.CreatableTy return l.accts.GetCreatorForRound(l.blockQ.latest(), cidx, ctype) } +// CompactCertVoters returns the top online accounts at round rnd. +// The result might be nil, even with err=nil, if there are no voters +// for that round because compact certs were not enabled. +func (l *Ledger) CompactCertVoters(rnd basics.Round) (voters *VotersForRound, err error) { + l.trackerMu.RLock() + defer l.trackerMu.RUnlock() + return l.accts.voters.getVoters(rnd) +} + // ListAssets takes a maximum asset index and maximum result length, and // returns up to that many CreatableLocators from the database where app idx is // less than or equal to the maximum. diff --git a/ledger/voters.go b/ledger/voters.go new file mode 100644 index 0000000000..089c782da3 --- /dev/null +++ b/ledger/voters.go @@ -0,0 +1,351 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package ledger + +import ( + "fmt" + "sync" + + "github.com/algorand/go-deadlock" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/compactcert" + "github.com/algorand/go-algorand/crypto/merklearray" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" +) + +// The votersTracker maintains the Merkle tree for the most recent +// commitments to online accounts for compact certificates. +// +// We maintain multiple Merkle trees: we might commit to a new Merkle tree in +// block X, but we need the Merkle tree from block X-params.CompactCertBlocks +// to build the compact certificate for block X. +// +// votersTracker is kind-of like a tracker, but hangs off the acctupdates +// rather than a direct ledger tracker. We don't have an explicit interface +// for such an "accounts tracker" yet, however. +type votersTracker struct { + // round contains the top online accounts in a given round. + // + // To avoid increasing block latency, we include a Merkle commitment + // to the top online accounts as of block X in the block header of + // block X+CompactCertVotersLookback. This gives each node some time + // to construct this Merkle tree, before its root is needed in a block. + // + // This round map is indexed by the block X, using the terminology from + // the above example, to be used in X+CompactCertVotersLookback. + // + // We maintain round entries for two reasons: + // + // The first is to maintain the tree for an upcoming block -- that is, + // if X+Loookback= minRound { + minRound = hdr.CompactCertLastRound + } + for rPlusLookback >= minRound { + r := rPlusLookback.SubSaturate(basics.Round(proto.CompactCertVotersLookback)) + hdr, err = l.BlockHdr(r) + if err != nil { + switch err.(type) { + case ErrNoEntry: + // If we cannot retrieve a block to construct the tree + // then we must have already evicted that block, which + // must have been because compact certs weren't enabled + // when that block was being considered for eviction. + // OK to stop. + return nil + default: + return err + } + } + + if config.Consensus[hdr.CurrentProtocol].CompactCertRounds == 0 { + // Walked backwards past a protocol upgrade, no more compact certs. + return nil + } + + err = vt.loadTree(hdr) + if err != nil { + return err + } + + rPlusLookback = rPlusLookback.SubSaturate(basics.Round(proto.CompactCertRounds)) + } + + return nil +} + +func (vt *votersTracker) loadTree(hdr bookkeeping.BlockHeader) error { + r := hdr.Round + + _, ok := vt.round[r] + if ok { + // Already loaded. + return nil + } + + proto := config.Consensus[hdr.CurrentProtocol] + if proto.CompactCertRounds == 0 { + // No compact certs. + return nil + } + + tr := &VotersForRound{ + Proto: proto, + } + tr.cond = sync.NewCond(&tr.mu) + vt.round[r] = tr + + go func() { + err := tr.loadTree(vt.l, vt.au, hdr) + if err != nil { + vt.au.log.Warnf("votersTracker.loadTree(%d): %v", hdr.Round, err) + + tr.mu.Lock() + tr.loadTreeError = err + tr.cond.Broadcast() + tr.mu.Unlock() + } + }() + return nil +} + +func (tr *VotersForRound) loadTree(l ledgerForTracker, au *accountUpdates, hdr bookkeeping.BlockHeader) error { + r := hdr.Round + + // certRound is the block that we expect to form a compact certificate for, + // using the balances from round r. + certRound := r + basics.Round(tr.Proto.CompactCertVotersLookback+tr.Proto.CompactCertRounds) + + // sigKeyRound is the ephemeral key ID that we expect to be used for signing + // the block from certRound. It is one higher because the keys for certRound + // might be deleted by the time consensus is reached on the block and we try + // to sign the compact cert for block certRound. + sigKeyRound := certRound + 1 + + top, err := au.onlineTop(r, sigKeyRound, tr.Proto.CompactCertTopVoters) + if err != nil { + return err + } + + participants := make(participantsArray, len(top)) + addrToPos := make(map[basics.Address]uint64) + var totalWeight basics.MicroAlgos + + for i, acct := range top { + var ot basics.OverflowTracker + rewards := basics.PendingRewards(&ot, tr.Proto, acct.MicroAlgos, acct.RewardsBase, hdr.RewardsLevel) + money := ot.AddA(acct.MicroAlgos, rewards) + if ot.Overflowed { + return fmt.Errorf("votersTracker.loadTree: overflow adding rewards %d + %d", acct.MicroAlgos, rewards) + } + + totalWeight = ot.AddA(totalWeight, money) + if ot.Overflowed { + return fmt.Errorf("votersTracker.loadTree: overflow computing totalWeight %d + %d", totalWeight.ToUint64(), money.ToUint64()) + } + + keyDilution := acct.VoteKeyDilution + if keyDilution == 0 { + keyDilution = tr.Proto.DefaultKeyDilution + } + + participants[i] = compactcert.Participant{ + PK: acct.VoteID, + Weight: money.ToUint64(), + KeyDilution: keyDilution, + } + addrToPos[acct.Address] = uint64(i) + } + + tree, err := merklearray.Build(participants) + if err != nil { + return err + } + + tr.mu.Lock() + tr.AddrToPos = addrToPos + tr.Participants = participants + tr.TotalWeight = totalWeight + tr.Tree = tree + tr.cond.Broadcast() + tr.mu.Unlock() + + return nil +} + +func (vt *votersTracker) newBlock(hdr bookkeeping.BlockHeader) { + proto := config.Consensus[hdr.CurrentProtocol] + if proto.CompactCertRounds == 0 { + // No compact certs. + return + } + + // Check if any blocks can be forgotten because the compact cert is available. + for r, tr := range vt.round { + commitRound := r + basics.Round(tr.Proto.CompactCertVotersLookback) + certRound := commitRound + basics.Round(tr.Proto.CompactCertRounds) + if certRound <= hdr.CompactCertLastRound { + delete(vt.round, r) + } + } + + // This might be a block where we snapshot the online participants, + // to eventually construct a merkle tree for commitment in a later + // block. + r := uint64(hdr.Round) + if (r+proto.CompactCertVotersLookback)%proto.CompactCertRounds == 0 { + _, ok := vt.round[basics.Round(r)] + if ok { + vt.au.log.Errorf("votersTracker.newBlock: round %d already present", r) + } else { + err := vt.loadTree(hdr) + if err != nil { + vt.au.log.Warnf("votersTracker.newBlock: loadTree: %v", err) + } + } + } +} + +// lowestRound() returns the lowest round state (blocks and accounts) needed by +// the votersTracker in case of a restart. The accountUpdates tracker will +// not delete account state before this round, so that after a restart, it's +// possible to reconstruct the votersTracker. If votersTracker does +// not need any blocks, it returns base. +func (vt *votersTracker) lowestRound(base basics.Round) basics.Round { + minRound := base + for r := range vt.round { + if r < minRound { + minRound = r + } + } + return minRound +} + +// getVoters() returns the top online participants from round r. +func (vt *votersTracker) getVoters(r basics.Round) (*VotersForRound, error) { + tr, ok := vt.round[r] + if !ok { + // Not tracked: compact certs not enabled. + return nil, nil + } + + // Wait for the Merkle tree to be constructed. + tr.mu.Lock() + defer tr.mu.Unlock() + for tr.Tree == nil { + if tr.loadTreeError != nil { + return nil, tr.loadTreeError + } + + tr.cond.Wait() + } + + return tr, nil +} + +//msgp:ignore participantsArray +// participantsArray implements merklearray.Array and is used to commit +// to a Merkle tree of online accounts. +type participantsArray []compactcert.Participant + +func (a participantsArray) Length() uint64 { + return uint64(len(a)) +} + +func (a participantsArray) Get(pos uint64) (crypto.Hashable, error) { + if pos >= uint64(len(a)) { + return nil, fmt.Errorf("participantsArray.Get(%d) out of bounds %d", pos, len(a)) + } + + return a[pos], nil +} From ba31225b53c21816435036860dbac606101b7ff4 Mon Sep 17 00:00:00 2001 From: "Ryan R. Fox" Date: Mon, 14 Sep 2020 17:40:04 -0400 Subject: [PATCH 035/136] fixup docs (#1498) fix a minor formatting issue in algod.oas2.json to allow better auto-generated documentation. --- daemon/algod/api/algod.oas2.json | 2 +- daemon/algod/api/algod.oas3.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/algod/api/algod.oas2.json b/daemon/algod/api/algod.oas2.json index 3bf9c81493..9612ca71c7 100644 --- a/daemon/algod/api/algod.oas2.json +++ b/daemon/algod/api/algod.oas2.json @@ -708,7 +708,7 @@ }, "/v2/transactions/pending/{txid}": { "get": { - "description": "Given a transaction id of a recently submitted transaction, it returns information about it. There are several cases when this might succeed:\n- transaction committed (committed round \u003e 0) - transaction still in the pool (committed round = 0, pool error = \"\") - transaction removed from pool due to error (committed round = 0, pool error != \"\")\nOr the transaction may have happened sufficiently long ago that the node no longer remembers it, and this will return an error.\n", + "description": "Given a transaction id of a recently submitted transaction, it returns information about it. There are several cases when this might succeed:\n- transaction committed (committed round \u003e 0)\n- transaction still in the pool (committed round = 0, pool error = \"\")\n- transaction removed from pool due to error (committed round = 0, pool error != \"\")\nOr the transaction may have happened sufficiently long ago that the node no longer remembers it, and this will return an error.\n", "produces": [ "application/json", "application/msgpack" diff --git a/daemon/algod/api/algod.oas3.yml b/daemon/algod/api/algod.oas3.yml index 9385b43722..2f286218c4 100644 --- a/daemon/algod/api/algod.oas3.yml +++ b/daemon/algod/api/algod.oas3.yml @@ -3072,7 +3072,7 @@ }, "/v2/transactions/pending/{txid}": { "get": { - "description": "Given a transaction id of a recently submitted transaction, it returns information about it. There are several cases when this might succeed:\n- transaction committed (committed round > 0) - transaction still in the pool (committed round = 0, pool error = \"\") - transaction removed from pool due to error (committed round = 0, pool error != \"\")\nOr the transaction may have happened sufficiently long ago that the node no longer remembers it, and this will return an error.\n", + "description": "Given a transaction id of a recently submitted transaction, it returns information about it. There are several cases when this might succeed:\n- transaction committed (committed round > 0)\n- transaction still in the pool (committed round = 0, pool error = \"\")\n- transaction removed from pool due to error (committed round = 0, pool error != \"\")\nOr the transaction may have happened sufficiently long ago that the node no longer remembers it, and this will return an error.\n", "operationId": "PendingTransactionInformation", "parameters": [ { From 09e0ff8ebbf445edc4cc107e28f3e4c0e82a00cd Mon Sep 17 00:00:00 2001 From: Will Winder Date: Mon, 14 Sep 2020 17:41:00 -0400 Subject: [PATCH 036/136] REST docs update: /v1/account/{addr}/transactions (#1503) Clarify documentation for v1 transactions by account endpoint. Closes #1127 --- daemon/algod/api/server/v1/handlers/handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/algod/api/server/v1/handlers/handlers.go b/daemon/algod/api/server/v1/handlers/handlers.go index 163bf056a2..377c35d559 100644 --- a/daemon/algod/api/server/v1/handlers/handlers.go +++ b/daemon/algod/api/server/v1/handlers/handlers.go @@ -1615,7 +1615,7 @@ func Transactions(ctx lib.ReqContext, context echo.Context) { // swagger:operation GET /v1/account/{address}/transactions Transactions // --- // Summary: Get a list of confirmed transactions. - // Description: Returns the list of confirmed transactions between within a date range. This call is available only when the indexer is running. + // Description: Returns the list of confirmed transactions between within a date range. When indexer is disabled this call requires firstRound and lastRound and returns an error if firstRound is not available to the node. The transaction results start from the oldest round. // Produces: // - application/json // Schemes: From c2fada3d29d412c82f40a2f688a2a212a284811f Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Mon, 14 Sep 2020 22:18:50 -0400 Subject: [PATCH 037/136] Change database synchronous mode during fast catchup (#1501) Use normal synchronous mode during fast catchup --- config/config.go | 15 ++++- config/local_defaults.go | 4 +- installer/config.json.example | 4 +- ledger/catchupaccessor.go | 6 ++ ledger/ledger.go | 41 ++++++++++++-- test/testdata/configs/config-v12.json | 79 +++++++++++++++++++++++++++ util/db/dbutil.go | 67 +++++++++++++++++------ util/db/dbutil_test.go | 33 +++++++++++ util/db/fullfsync_darwin.go | 2 +- 9 files changed, 226 insertions(+), 25 deletions(-) create mode 100644 test/testdata/configs/config-v12.json diff --git a/config/config.go b/config/config.go index 4ade543614..9c3fae01bb 100644 --- a/config/config.go +++ b/config/config.go @@ -63,7 +63,7 @@ type Local struct { // Version tracks the current version of the defaults so we can migrate old -> new // This is specifically important whenever we decide to change the default value // for an existing parameter. This field tag must be updated any time we add a new version. - Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11"` + Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12"` // environmental (may be overridden) // When enabled, stores blocks indefinitally, otherwise, only the most recents blocks @@ -336,6 +336,19 @@ type Local struct { // A value of 0 means automatic, which is the default value. In this mode, a non archival node would not track the catchpoints, and an archival node would track the catchpoints as long as CatchpointInterval > 0. // Other values of CatchpointTracking would give a warning in the log file, and would behave as if the default value was provided. CatchpointTracking int64 `version[11]:"0"` + + // LedgerSynchronousMode defines the synchronous mode used by the ledger database. The supported options are: + // 0 - SQLite continues without syncing as soon as it has handed data off to the operating system. + // 1 - SQLite database engine will still sync at the most critical moments, but less often than in FULL mode. + // 2 - SQLite database engine will use the xSync method of the VFS to ensure that all content is safely written to the disk surface prior to continuing. On Mac OS, the data is additionally syncronized via fullfsync. + // 3 - In addition to what being done in 2, it provides additional durability if the commit is followed closely by a power loss. + // for further information see the description of SynchronousMode in dbutil.go + LedgerSynchronousMode int `version[12]:"2"` + + // AccountsRebuildSynchronousMode defines the synchronous mode used by the ledger database while the account database is being rebuilt. This is not a typical operational usecase, + // and is expected to happen only on either startup ( after enabling the catchpoint interval, or on certain database upgrades ) or during fast catchup. The values specified here + // and their meanings are identical to the ones in LedgerSynchronousMode. + AccountsRebuildSynchronousMode int `version[12]:"1"` } // Filenames of config files within the configdir (e.g. ~/.algorand) diff --git a/config/local_defaults.go b/config/local_defaults.go index 28743e4c82..33f6fd1c2c 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -20,7 +20,8 @@ package config var defaultLocal = Local{ - Version: 11, + Version: 12, + AccountsRebuildSynchronousMode: 1, AnnounceParticipationKey: true, Archival: false, BaseLoggerDebugLevel: 4, @@ -64,6 +65,7 @@ var defaultLocal = Local{ IncomingMessageFilterBucketCount: 5, IncomingMessageFilterBucketSize: 512, IsIndexerActive: false, + LedgerSynchronousMode: 2, LogArchiveMaxAge: "", LogArchiveName: "node.archive.log", LogSizeLimit: 1073741824, diff --git a/installer/config.json.example b/installer/config.json.example index c29679a17f..9df7ffc657 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -1,5 +1,6 @@ { - "Version": 11, + "Version": 12, + "AccountsRebuildSynchronousMode": 1, "AnnounceParticipationKey": true, "Archival": false, "BaseLoggerDebugLevel": 4, @@ -43,6 +44,7 @@ "IncomingMessageFilterBucketCount": 5, "IncomingMessageFilterBucketSize": 512, "IsIndexerActive": false, + "LedgerSynchronousMode": 2, "LogArchiveMaxAge": "", "LogArchiveName": "node.archive.log", "LogSizeLimit": 1073741824, diff --git a/ledger/catchupaccessor.go b/ledger/catchupaccessor.go index 7c5e2be602..c6dec55973 100644 --- a/ledger/catchupaccessor.go +++ b/ledger/catchupaccessor.go @@ -174,6 +174,9 @@ func (c *CatchpointCatchupAccessorImpl) SetLabel(ctx context.Context, label stri // ResetStagingBalances resets the current staging balances, preparing for a new set of balances to be added func (c *CatchpointCatchupAccessorImpl) ResetStagingBalances(ctx context.Context, newCatchup bool) (err error) { wdb := c.ledger.trackerDB().wdb + if !newCatchup { + c.ledger.setSynchronousMode(ctx, c.ledger.synchronousMode) + } err = wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { err = resetCatchpointStagingBalances(ctx, tx, newCatchup) if err != nil { @@ -270,6 +273,7 @@ func (c *CatchpointCatchupAccessorImpl) processStagingContent(ctx context.Contex progress.SeenHeader = true progress.TotalAccounts = fileHeader.TotalAccounts progress.TotalChunks = fileHeader.TotalChunks + c.ledger.setSynchronousMode(ctx, c.ledger.accountsRebuildSynchronousMode) } return err } @@ -362,6 +366,8 @@ func (c *CatchpointCatchupAccessorImpl) processStagingBalances(ctx context.Conte // not strictly required, but clean up the pointer in case of either a failuire or when we're done. if err != nil || progress.ProcessedAccounts == progress.TotalAccounts { progress.cachedTrie = nil + // restore "normal" syncronous mode + c.ledger.setSynchronousMode(ctx, c.ledger.synchronousMode) } return err } diff --git a/ledger/ledger.go b/ledger/ledger.go index 0464f188d9..7cfe34a4c5 100644 --- a/ledger/ledger.go +++ b/ledger/ledger.go @@ -32,6 +32,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/util/db" ) // Ledger is a database storing the contents of the ledger. @@ -52,6 +53,12 @@ type Ledger struct { // (archival mode) or trims older blocks to save space (non-archival). archival bool + // the synchronous mode that would be used for the ledger databases. + synchronousMode db.SynchronousMode + + // the synchronous mode that would be used while the accounts database is being rebuilt. + accountsRebuildSynchronousMode db.SynchronousMode + // genesisHash stores the genesis hash for this ledger. genesisHash crypto.Digest @@ -89,11 +96,13 @@ func OpenLedger( ) (*Ledger, error) { var err error l := &Ledger{ - log: log, - archival: cfg.Archival, - genesisHash: genesisInitState.GenesisHash, - genesisAccounts: genesisInitState.Accounts, - genesisProto: config.Consensus[genesisInitState.Block.CurrentProtocol], + log: log, + archival: cfg.Archival, + genesisHash: genesisInitState.GenesisHash, + genesisAccounts: genesisInitState.Accounts, + genesisProto: config.Consensus[genesisInitState.Block.CurrentProtocol], + synchronousMode: db.SynchronousMode(cfg.LedgerSynchronousMode), + accountsRebuildSynchronousMode: db.SynchronousMode(cfg.AccountsRebuildSynchronousMode), } l.headerCache.maxEntries = 10 @@ -114,6 +123,8 @@ func OpenLedger( l.blockDBs.rdb.SetLogger(log) l.blockDBs.wdb.SetLogger(log) + l.setSynchronousMode(context.Background(), l.synchronousMode) + err = l.blockDBs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { return initBlocksDB(tx, l, []bookkeeping.Block{genesisInitState.Block}, cfg.Archival) }) @@ -240,6 +251,26 @@ func openLedgerDB(dbPathPrefix string, dbMem bool) (trackerDBs dbPair, blockDBs return } +// setSynchronousMode sets the writing database connections syncronous mode to the specified mode +func (l *Ledger) setSynchronousMode(ctx context.Context, synchronousMode db.SynchronousMode) { + if synchronousMode < db.SynchronousModeOff || synchronousMode > db.SynchronousModeExtra { + l.log.Warnf("ledger.setSynchronousMode unable to set syncronous mode : requested value %d is invalid", synchronousMode) + return + } + + err := l.blockDBs.wdb.SetSynchronousMode(ctx, synchronousMode, synchronousMode >= db.SynchronousModeFull) + if err != nil { + l.log.Warnf("ledger.setSynchronousMode unable to set syncronous mode on blocks db: %v", err) + return + } + + err = l.trackerDBs.wdb.SetSynchronousMode(ctx, synchronousMode, synchronousMode >= db.SynchronousModeFull) + if err != nil { + l.log.Warnf("ledger.setSynchronousMode unable to set syncronous mode on trackers db: %v", err) + return + } +} + // initBlocksDB performs DB initialization: // - creates and populates it with genesis blocks // - ensures DB is in good shape for archival mode and resets it if not diff --git a/test/testdata/configs/config-v12.json b/test/testdata/configs/config-v12.json new file mode 100644 index 0000000000..9df7ffc657 --- /dev/null +++ b/test/testdata/configs/config-v12.json @@ -0,0 +1,79 @@ +{ + "Version": 12, + "AccountsRebuildSynchronousMode": 1, + "AnnounceParticipationKey": true, + "Archival": false, + "BaseLoggerDebugLevel": 4, + "BroadcastConnectionsLimit": -1, + "CadaverSizeTarget": 1073741824, + "CatchpointFileHistoryLength": 365, + "CatchpointInterval": 10000, + "CatchpointTracking": 0, + "CatchupBlockDownloadRetryAttempts": 1000, + "CatchupFailurePeerRefreshRate": 10, + "CatchupGossipBlockFetchTimeoutSec": 4, + "CatchupHTTPBlockFetchTimeoutSec": 4, + "CatchupLedgerDownloadRetryAttempts": 50, + "CatchupParallelBlocks": 16, + "ConnectionsRateLimitingCount": 60, + "ConnectionsRateLimitingWindowSeconds": 1, + "DNSBootstrapID": ".algorand.network", + "DNSSecurityFlags": 1, + "DeadlockDetection": 0, + "DisableOutgoingConnectionThrottling": false, + "EnableAgreementReporting": false, + "EnableAgreementTimeMetrics": false, + "EnableAssembleStats": false, + "EnableBlockService": false, + "EnableDeveloperAPI": false, + "EnableGossipBlockService": true, + "EnableIncomingMessageFilter": false, + "EnableLedgerService": false, + "EnableMetricReporting": false, + "EnableOutgoingNetworkMessageFiltering": true, + "EnablePingHandler": true, + "EnableProcessBlockStats": false, + "EnableProfiler": false, + "EnableRequestLogger": false, + "EnableTopAccountsReporting": false, + "EndpointAddress": "127.0.0.1:0", + "FallbackDNSResolverAddress": "", + "ForceRelayMessages": false, + "GossipFanout": 4, + "IncomingConnectionsLimit": 10000, + "IncomingMessageFilterBucketCount": 5, + "IncomingMessageFilterBucketSize": 512, + "IsIndexerActive": false, + "LedgerSynchronousMode": 2, + "LogArchiveMaxAge": "", + "LogArchiveName": "node.archive.log", + "LogSizeLimit": 1073741824, + "MaxConnectionsPerIP": 30, + "NetAddress": "", + "NetworkProtocolVersion": "", + "NodeExporterListenAddress": ":9100", + "NodeExporterPath": "./node_exporter", + "OptimizeAccountsDatabaseOnStartup": false, + "OutgoingMessageFilterBucketCount": 3, + "OutgoingMessageFilterBucketSize": 128, + "PeerConnectionsUpdateInterval": 3600, + "PeerPingPeriodSeconds": 0, + "PriorityPeers": {}, + "PublicAddress": "", + "ReconnectTime": 60000000000, + "ReservedFDs": 256, + "RestReadTimeoutSeconds": 15, + "RestWriteTimeoutSeconds": 120, + "RunHosted": false, + "SuggestedFeeBlockHistory": 3, + "SuggestedFeeSlidingWindowSize": 50, + "TLSCertFile": "", + "TLSKeyFile": "", + "TelemetryToLog": true, + "TxPoolExponentialIncreaseFactor": 2, + "TxPoolSize": 15000, + "TxSyncIntervalSeconds": 60, + "TxSyncServeResponseSize": 1000000, + "TxSyncTimeoutSeconds": 30, + "UseXForwardedForAddressField": "" +} diff --git a/util/db/dbutil.go b/util/db/dbutil.go index a5ce68bfe2..d970c8c4fe 100644 --- a/util/db/dbutil.go +++ b/util/db/dbutil.go @@ -46,10 +46,9 @@ import ( // go-sqlite3 with the "sqlite_unlock_notify" Go build tag. const busy = 1000 -// initStatements is a list of statements we execute after opening a database -// connection. For now, it's just an optional "PRAGMA fullfsync=true" on -// MacOSX. -var initStatements []string +// enableFullfsyncStatements is a list of statements we execute to enable a fullfsync. +// Currently, it's only supported by MacOSX. +var enableFullfsyncStatements []string var sqliteInitOnce sync.Once // An Accessor manages a sqlite database handle and any outstanding batching operations. @@ -124,23 +123,13 @@ func makeAccessorImpl(dbfilename string, readOnly bool, inMemory bool, params [] // init failed, db closed and err is set return db, err } - err = db.runInitStatements() - } - - return db, err -} - -// runInitStatements executes initialization statements. -func (db *Accessor) runInitStatements() error { - for _, stmt := range initStatements { - _, err := db.Handle.Exec(stmt) + err = db.SetSynchronousMode(context.Background(), SynchronousModeFull, true) if err != nil { db.Close() - return err } } - return nil + return db, err } // SetLogger sets the Logger, mainly for unit test quietness @@ -453,3 +442,49 @@ type idemFn func(ctx context.Context, tx *sql.Tx) error const infoTxRetries = 5 const warnTxRetriesInterval = 1 + +// SynchronousMode is the syncronious modes supported by sqlite database. +type SynchronousMode int + +const ( + // SynchronousModeOff (0), SQLite continues without syncing as soon as it has handed data off to the operating system. If the application running SQLite crashes, + // the data will be safe, but the database might become corrupted if the operating system crashes or the computer loses power before that data has been written to the + // disk surface. On the other hand, commits can be orders of magnitude faster with synchronous OFF. + SynchronousModeOff SynchronousMode = 0 + // SynchronousModeNormal (1), the SQLite database engine will still sync at the most critical moments, but less often than in FULL mode. There is a very small + // (though non-zero) chance that a power failure at just the wrong time could corrupt the database in journal_mode=DELETE on an older filesystem. + // WAL mode is safe from corruption with synchronous=NORMAL, and probably DELETE mode is safe too on modern filesystems. WAL mode is always consistent with synchronous=NORMAL, + // but WAL mode does lose durability. A transaction committed in WAL mode with synchronous=NORMAL might roll back following a power loss or system crash. + // Transactions are durable across application crashes regardless of the synchronous setting or journal mode. + // The synchronous=NORMAL setting is a good choice for most applications running in WAL mode. + SynchronousModeNormal SynchronousMode = 1 + // SynchronousModeFull (2), the SQLite database engine will use the xSync method of the VFS to ensure that all content is safely written to the disk surface prior to continuing. + // This ensures that an operating system crash or power failure will not corrupt the database. FULL synchronous is very safe, but it is also slower. + // FULL is the most commonly used synchronous setting when not in WAL mode. + SynchronousModeFull SynchronousMode = 2 + // SynchronousModeExtra synchronous is like FULL with the addition that the directory containing a rollback journal is synced after that journal is unlinked to commit a + // transaction in DELETE mode. EXTRA provides additional durability if the commit is followed closely by a power loss. + SynchronousModeExtra SynchronousMode = 3 +) + +// SetSynchronousMode updates the syncronous mode of the connection +func (db *Accessor) SetSynchronousMode(ctx context.Context, mode SynchronousMode, fullfsync bool) (err error) { + if mode < SynchronousModeOff || mode > SynchronousModeExtra { + return fmt.Errorf("invalid value(%d) was provided to mode", mode) + } + _, err = db.Handle.ExecContext(ctx, fmt.Sprintf("PRAGMA synchronous=%d", mode)) + if err != nil { + return err + } + if fullfsync { + for _, stmt := range enableFullfsyncStatements { + _, err = db.Handle.ExecContext(ctx, stmt) + if err != nil { + break + } + } + } else { + _, err = db.Handle.ExecContext(ctx, "PRAGMA fullfsync=false") + } + return +} diff --git a/util/db/dbutil_test.go b/util/db/dbutil_test.go index ee043cecf0..0cb7af8a56 100644 --- a/util/db/dbutil_test.go +++ b/util/db/dbutil_test.go @@ -385,3 +385,36 @@ func TestResettingTransactionWarnDeadline(t *testing.T) { require.Equal(t, 0, logger.warningsCounter) }) } + +// Test the SetSynchronousMode function +func TestSetSynchronousMode(t *testing.T) { + setSynchrounousModeHelper := func(mem bool, ctx context.Context, mode SynchronousMode, fullfsync bool) error { + acc, err := MakeAccessor("fn.db", false, mem) + require.NoError(t, err) + if !mem { + defer os.Remove("fn.db") + defer os.Remove("fn.db-shm") + defer os.Remove("fn.db-wal") + } + defer acc.Close() + return acc.SetSynchronousMode(ctx, mode, fullfsync) + } + // check with canceled context. + ctx, cancelFunc := context.WithCancel(context.Background()) + cancelFunc() + + require.Error(t, context.Canceled, setSynchrounousModeHelper(true, ctx, SynchronousModeOff, false)) + require.Error(t, context.Canceled, setSynchrounousModeHelper(false, ctx, SynchronousModeOff, false)) + + require.Contains(t, setSynchrounousModeHelper(false, context.Background(), SynchronousModeOff-1, false).Error(), "invalid value") + require.Contains(t, setSynchrounousModeHelper(false, context.Background(), SynchronousModeExtra+1, false).Error(), "invalid value") + + // try all success permutations - + for _, mode := range []SynchronousMode{SynchronousModeOff, SynchronousModeNormal, SynchronousModeFull, SynchronousModeExtra} { + for _, disk := range []bool{true, false} { + for _, fullfsync := range []bool{true, false} { + require.NoError(t, setSynchrounousModeHelper(disk, context.Background(), mode, fullfsync)) + } + } + } +} diff --git a/util/db/fullfsync_darwin.go b/util/db/fullfsync_darwin.go index 6059b55662..14e2155bc3 100644 --- a/util/db/fullfsync_darwin.go +++ b/util/db/fullfsync_darwin.go @@ -18,5 +18,5 @@ package db func init() { // See https://www.sqlite.org/pragma.html#pragma_fullfsync - initStatements = append(initStatements, "PRAGMA fullfsync=true") + enableFullfsyncStatements = append(enableFullfsyncStatements, "PRAGMA fullfsync=true") } From 9c41ddf7729f6abc016f07f8da1081e4d412aac3 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Tue, 15 Sep 2020 13:58:23 -0400 Subject: [PATCH 038/136] Don't change parameter name. --- cmd/goal/asset.go | 10 +++++----- test/scripts/e2e_subs/e2e-app-real-assets-round.sh | 2 +- test/scripts/e2e_subs/limit-swap-test.sh | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/goal/asset.go b/cmd/goal/asset.go index d6ce6d9414..529bd2b617 100644 --- a/cmd/goal/asset.go +++ b/cmd/goal/asset.go @@ -110,7 +110,7 @@ func init() { addTxnFlags(freezeAssetCmd) infoAssetCmd.Flags().Uint64Var(&assetID, "assetid", 0, "ID of the asset to look up") - infoAssetCmd.Flags().StringVar(&assetUnitName, "unit", "", "Unit name of the asset to look up") + infoAssetCmd.Flags().StringVar(&assetUnitName, "asset", "", "Unit name of the asset to look up") infoAssetCmd.Flags().StringVar(&assetCreator, "creator", "", "Account address of the asset creator") } @@ -125,16 +125,16 @@ var assetCmd = &cobra.Command{ } func lookupAssetID(cmd *cobra.Command, creator string, client libgoal.Client) { - if cmd.Flags().Changed("assetid") && cmd.Flags().Changed("unit") { - reportErrorf("Only one of [--assetid] or [--unit and --creator] can be specified") + if cmd.Flags().Changed("assetid") && cmd.Flags().Changed("asset") { + reportErrorf("Only one of [--assetid] or [--asset and --creator] can be specified") } if cmd.Flags().Changed("assetid") { return } - if !cmd.Flags().Changed("unit") { - reportErrorf("Either [--assetid] or [--unit and --creator] must be specified") + if !cmd.Flags().Changed("asset") { + reportErrorf("Either [--assetid] or [--asset and --creator] must be specified") } if !cmd.Flags().Changed("creator") { diff --git a/test/scripts/e2e_subs/e2e-app-real-assets-round.sh b/test/scripts/e2e_subs/e2e-app-real-assets-round.sh index 6401042c5e..6793d6bb53 100755 --- a/test/scripts/e2e_subs/e2e-app-real-assets-round.sh +++ b/test/scripts/e2e_subs/e2e-app-real-assets-round.sh @@ -18,7 +18,7 @@ ACCOUNT=$(${gcmd} account list|awk '{ print $3 }') # Create an ASA in account ${gcmd} asset create --creator ${ACCOUNT} --name bogocoin --unitname bogo --total 1337 -ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --unit bogo|grep 'Asset ID'|awk '{ print $3 }') +ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --asset bogo|grep 'Asset ID'|awk '{ print $3 }') # Create app that reads asset balance and checks asset details and checks round ROUND=$(goal node status | grep 'Last committed' | awk '{ print $4 }') diff --git a/test/scripts/e2e_subs/limit-swap-test.sh b/test/scripts/e2e_subs/limit-swap-test.sh index 463128fb8e..52907b47ff 100755 --- a/test/scripts/e2e_subs/limit-swap-test.sh +++ b/test/scripts/e2e_subs/limit-swap-test.sh @@ -16,7 +16,7 @@ ZERO_ADDRESS=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ ${gcmd} asset create --creator ${ACCOUNT} --name bogocoin --unitname bogo --total 1000000000000 -ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --unit bogo|grep 'Asset ID'|awk '{ print $3 }') +ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --asset bogo|grep 'Asset ID'|awk '{ print $3 }') # Asset ID: 5 From c9417bad3167d82099e8ca8b70805ce5459d01e1 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Tue, 15 Sep 2020 16:44:42 -0400 Subject: [PATCH 039/136] calculate normalized balance using genesis proto (#1509) Calculate normalized balance using genesis proto --- ledger/acctupdates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index e25e979d65..1dedc643d4 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -421,7 +421,7 @@ func (au *accountUpdates) onlineTop(rnd basics.Round, voteRnd basics.Round, n ui return nil, err } - proto := au.protos[offset] + proto := au.ledger.GenesisProto() // Determine how many accounts have been modified in-memory, // so that we obtain enough top accounts from disk (accountdb). From fbf88ed5e5a9001637c7ce0fec8d4029f213fcbb Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Wed, 16 Sep 2020 00:47:47 -0400 Subject: [PATCH 040/136] Exit with a non-zero in case of an error (#1511) Exit algod with a non-zero in case of a startup error --- cmd/algod/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/algod/main.go b/cmd/algod/main.go index fb22c66a89..c20f4b7238 100644 --- a/cmd/algod/main.go +++ b/cmd/algod/main.go @@ -293,6 +293,7 @@ func main() { if err != nil { fmt.Fprintln(os.Stderr, err) log.Error(err) + os.Exit(1) return } From 18cdaed379d68b9442bf44befdc944137a901270 Mon Sep 17 00:00:00 2001 From: algonautshant <55754073+algonautshant@users.noreply.github.com> Date: Wed, 16 Sep 2020 08:25:43 -0400 Subject: [PATCH 041/136] Test for process return in expect tests (#1502) Test for process return in expect tests --- .../goal/expect/goalAppAccountAddressTest.exp | 15 ++- .../cli/goal/expect/goalExpectCommon.exp | 37 ++++++ test/e2e-go/cli/goal/expect/testInfraTest.exp | 106 ++++++++++++++++++ 3 files changed, 153 insertions(+), 5 deletions(-) create mode 100644 test/e2e-go/cli/goal/expect/testInfraTest.exp diff --git a/test/e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp b/test/e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp index 1f912fa51c..8ff601d434 100644 --- a/test/e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp +++ b/test/e2e-go/cli/goal/expect/goalAppAccountAddressTest.exp @@ -109,6 +109,7 @@ proc goalAppAccountAddress { TEST_ALGO_DIR TEST_DATA_DIR} { "*committed in round*" {puts "app call successful"; close} eof {::AlgorandGoal::Abort "app call failed" } } + ::AlgorandGoal::CheckProcessReturnedCode 1 puts "Checking the results" set EXPECTED_OUTPUT "Account0*$PRIMARY_ACCOUNT_ADDRESS" @@ -124,8 +125,9 @@ proc goalAppAccountAddress { TEST_ALGO_DIR TEST_DATA_DIR} { expect { timeout { puts timeout; ::AlgorandGoal::Abort "\n Failed to see expected output" } "*$EXPECTED_OUTPUT*" {puts "Local state read correctly"; close} - eof {::AlgorandGoal::Abort "app read failed" } + eof {::AlgorandGoal::Abort "App read failed. Expected output includes: $EXPECTED_OUTPUT" } } + ::AlgorandGoal::CheckProcessReturnedCode 1 # check the local state of account 2 spawn goal app read --app-id $APP_ID --local --guess-format \ @@ -134,9 +136,10 @@ proc goalAppAccountAddress { TEST_ALGO_DIR TEST_DATA_DIR} { timeout { puts timeout; ::AlgorandGoal::Abort "\n Failed to see expected output" } "Please enter the password for wallet '$WALLET_1_NAME':" {send "$WALLET_1_PASSWORD\r" ; exp_continue} "*$EXPECTED_OUTPUT*" {puts "Local state read correctly"; close} - eof {::AlgorandGoal::Abort "app read failed" } + eof {::AlgorandGoal::Abort "App read failed. Expected output includes: $EXPECTED_OUTPUT" } } - + ::AlgorandGoal::CheckProcessReturnedCode 1 + # call the app with a missing app-account. It should fail puts "Calling goal app call to get the local state params" spawn goal app call --app-id $APP_ID --from $PRIMARY_ACCOUNT_ADDRESS -w $PRIMARY_WALLET_NAME -d $TEST_PRIMARY_NODE_DIR \ @@ -149,6 +152,10 @@ proc goalAppAccountAddress { TEST_ALGO_DIR TEST_DATA_DIR} { {puts "Error received successfully "; close} eof {::AlgorandGoal::Abort "failed to get the expected error" } } + lassign [::AlgorandGoal::CheckProcessReturnedCode 0] response OS_CODE ERR_CODE KILLED KILL_SIGNAL EXP + if {$response != 1 || $OS_CODE != 0 || $ERR_CODE != 1} { + ::AlgorandGoal::Abort "failed to get the expected error. Expected ERR_CODE = 1 got ERR_CODE = $ERR_CODE" + } # Shutdown the network ::AlgorandGoal::StopNetwork $NETWORK_NAME $TEST_ALGO_DIR $TEST_ROOT_DIR @@ -170,8 +177,6 @@ if { [catch { goalAppAccountAddress $TEST_ALGO_DIR $TEST_DATA_DIR - exit 0 - } EXCEPTION ] } { ::AlgorandGoal::Abort "ERROR in goalAppAccountAddressTest: $EXCEPTION" } diff --git a/test/e2e-go/cli/goal/expect/goalExpectCommon.exp b/test/e2e-go/cli/goal/expect/goalExpectCommon.exp index 91272ffba9..6d8e79c6c7 100755 --- a/test/e2e-go/cli/goal/expect/goalExpectCommon.exp +++ b/test/e2e-go/cli/goal/expect/goalExpectCommon.exp @@ -56,6 +56,43 @@ proc ::AlgorandGoal::Abort { ERROR } { exit 1 } +# Utility method to test the process returned value +# Returns 0 when no error code is detected +# When an error code is detected: +# If ABORT = 1 Calls AlgorandGoal::Abort +# if ABORT = 0 Returns 1 OS_CODE ERR_CODE KILLED KILL_SIGNAL EXP +# If SIGHUP is detected, it ignores it. +proc ::AlgorandGoal::CheckProcessReturnedCode {ABORT} { + upvar spawn_id spawn_id + lassign [wait -i $spawn_id] PID SPAWNID OS_CODE ERR_CODE KILLED KILL_SIGNAL EXP + + if {$KILLED == "CHILDKILLED"} { + if {$KILL_SIGNAL == "SIGHUP" && $EXP == "hangup"} { + # this is caused by expect close. Will ignore. + return 0 + } + if {$ABORT} { + ::AlgorandGoal::Abort "process killed: $KILL_SIGNAL $EXP" + } + return [list 1 $OS_CODE $ERR_CODE $KILLED $KILL_SIGNAL $EXP] + } + + if {$OS_CODE == -1} { + if {$ABORT} { + ::AlgorandGoal::Abort "OS error code: $ERR_CODE" + } + return [list 1 $OS_CODE $ERR_CODE $KILLED $KILL_SIGNAL $EXP] + } else { + if {$ERR_CODE != 0} { + if {$ABORT} { + ::AlgorandGoal::Abort "porcess returned non-zero value: $ERR_CODE" + } + return [list 1 $OS_CODE $ERR_CODE $KILLED $KILL_SIGNAL $EXP] + } + } + return 0 +} + # Start the node proc ::AlgorandGoal::StartNode { TEST_ALGO_DIR {SYSTEMD_MANAGED "False"} {PEER_ADDRESS ""} } { set ::GLOBAL_TEST_ALGO_DIR $TEST_ALGO_DIR diff --git a/test/e2e-go/cli/goal/expect/testInfraTest.exp b/test/e2e-go/cli/goal/expect/testInfraTest.exp new file mode 100644 index 0000000000..308ac59166 --- /dev/null +++ b/test/e2e-go/cli/goal/expect/testInfraTest.exp @@ -0,0 +1,106 @@ +#!/usr/bin/expect -f +#exp_internal 1 +set err 0 +log_user 1 + +source goalExpectCommon.exp + +# This test tests the testing procedure CheckProcessReturnedCode +# When a process crashes, CheckProcessReturnedCode should return 1 +proc checkProcessReturnedCodeTest {} { + # Test the process killed branch + spawn /bin/bash -c "kill -11 $$" + lassign [::AlgorandGoal::CheckProcessReturnedCode 0] response OS_CODE ERR_CODE KILLED KILL_SIGNAL EXP + if {$response == 0} { + puts "expected failure code 1 not 0" + exit 1 + } + if {$KILLED != "CHILDKILLED" || $KILL_SIGNAL != "SIGSEGV" || $EXP != "segmentation violation"} { + puts "expected CHILDKILLED SIGSEGV segmentation violation" + puts "got: $KILLED $KILL_SIGNAL $EXP" + exit 1 + } + + # Test the sighup branch + spawn /bin/bash -c "kill -1 $$" + lassign [::AlgorandGoal::CheckProcessReturnedCode 0] response OS_CODE ERR_CODE KILLED KILL_SIGNAL EXP + if {$response != 0} { + puts "expected 0" + puts "got: $KILLED $KILL_SIGNAL $EXP" + exit 1 + } + + # TODO: test OS_CODE == -1 branch + + # test ERR_CODE != 0 branch + spawn /bin/bash -c "exit 33" + lassign [::AlgorandGoal::CheckProcessReturnedCode 0] response OS_CODE ERR_CODE KILLED KILL_SIGNAL EXP + if {$response == 0} { + puts "expected failure code 1 not 0" + exit 1 + } + if {$ERR_CODE != 33} { + puts "expected ERR_CODE 33 got: $ERR_CODE" + exit 1 + } + + # test ERR_CODE == 0 branch + spawn /bin/bash -c "exit 0" + lassign [::AlgorandGoal::CheckProcessReturnedCode 0] response OS_CODE ERR_CODE KILLED KILL_SIGNAL EXP + if {$response != 0} { + puts "expected failure code 0 not $response" + exit 1 + } + + # test close sending sighup + spawn /bin/bash -c "echo 44; sleep 2s; kill -11 $$" + expect { + 44 { + close + } + } + lassign [::AlgorandGoal::CheckProcessReturnedCode 0] response OS_CODE ERR_CODE KILLED KILL_SIGNAL EXP + if {$response != 0} { + puts "expected 0" + puts "got: $KILLED $KILL_SIGNAL $EXP" + exit 1 + } + + # same, without close. should get to segv + spawn /bin/bash -c "echo 44; sleep 2s; kill -11 $$" + expect { + 44 { + puts "not closing" + } + } + lassign [::AlgorandGoal::CheckProcessReturnedCode 0] response OS_CODE ERR_CODE KILLED KILL_SIGNAL EXP + if {$KILLED != "CHILDKILLED" || $KILL_SIGNAL != "SIGSEGV" || $EXP != "segmentation violation"} { + puts "expected CHILDKILLED SIGSEGV segmentation violation" + puts "got: $KILLED $KILL_SIGNAL $EXP" + exit 1 + } +} + +# When eof is expected, the spawn_id is no longer open +# This test confirms this behavior +proc closeOnEofTest {} { + spawn /bin/bash -c "echo this is some command" + expect { + eof { + if {[catch { + close + } EXCEPTION] } { + if {![string match {spawn_id: spawn id * not open} $EXCEPTION]} { + puts "expected: spawn_id: spawn id expID not open" + puts "got: $EXCEPTION" + exit 1 + } + } + } + } + +} + + +checkProcessReturnedCodeTest +closeOnEofTest From c1b79d8912e6e7629c584b31ec44d21c9eb0c997 Mon Sep 17 00:00:00 2001 From: btoll Date: Wed, 16 Sep 2020 17:05:41 -0400 Subject: [PATCH 042/136] Create new `algorand-devtools` package (#1492) --- installer/debian/algorand-devtools/conffiles | 2 + installer/debian/algorand-devtools/control | 10 ++ installer/debian/{ => algorand}/conffiles | 0 installer/debian/{ => algorand}/control | 0 installer/debian/{ => algorand}/postinst | 0 installer/debian/{ => algorand}/postrm | 0 installer/debian/{ => algorand}/preinst | 4 +- installer/debian/{ => algorand}/prerm | 0 .../algorand-devtools/algorand-devtools.repo | 6 + .../algorand-devtools/algorand-devtools.spec | 46 +++++++ .../{ => algorand}/0yum-algorand-hourly.cron | 0 installer/rpm/{ => algorand}/algorand.repo | 0 installer/rpm/{ => algorand}/algorand.spec | 6 +- .../rpm/{ => algorand}/yum-cron-algorand.conf | 0 package.yaml | 2 +- scripts/build_deb.sh | 6 +- scripts/build_package.sh | 2 +- scripts/compute_package_name.sh | 20 ++- scripts/release/build/deb/build_deb.sh | 127 ++++++++++++++++++ scripts/release/build/deb/package.sh | 21 +-- scripts/release/build/rpm/package.sh | 39 +++--- scripts/release/build/rpm/sign.sh | 30 ++--- scripts/release/build/stage/package/task.sh | 4 +- scripts/release/common/setup.sh | 8 +- scripts/release/forward_gpg_agent.sh | 6 +- scripts/release/mule/Makefile.mule | 6 +- scripts/release/mule/package/deb/package.sh | 110 ++++++++------- scripts/release/mule/package/rpm/package.sh | 14 +- scripts/release/mule/test/test.sh | 20 ++- scripts/release/mule/test/tests/run_tests | 12 +- .../mule/test/tests/verify_pkg_string.sh | 3 +- scripts/release/test/deb/test_algorand.sh | 62 ++++----- scripts/release/test/rpm/test_algorand.sh | 60 ++++----- scripts/release/test/stage/setup/run.sh | 2 +- 34 files changed, 446 insertions(+), 182 deletions(-) create mode 100644 installer/debian/algorand-devtools/conffiles create mode 100644 installer/debian/algorand-devtools/control rename installer/debian/{ => algorand}/conffiles (100%) rename installer/debian/{ => algorand}/control (100%) rename installer/debian/{ => algorand}/postinst (100%) rename installer/debian/{ => algorand}/postrm (100%) rename installer/debian/{ => algorand}/preinst (70%) rename installer/debian/{ => algorand}/prerm (100%) create mode 100644 installer/rpm/algorand-devtools/algorand-devtools.repo create mode 100644 installer/rpm/algorand-devtools/algorand-devtools.spec rename installer/rpm/{ => algorand}/0yum-algorand-hourly.cron (100%) rename installer/rpm/{ => algorand}/algorand.repo (100%) rename installer/rpm/{ => algorand}/algorand.spec (91%) rename installer/rpm/{ => algorand}/yum-cron-algorand.conf (100%) create mode 100755 scripts/release/build/deb/build_deb.sh diff --git a/installer/debian/algorand-devtools/conffiles b/installer/debian/algorand-devtools/conffiles new file mode 100644 index 0000000000..382fb0fdf5 --- /dev/null +++ b/installer/debian/algorand-devtools/conffiles @@ -0,0 +1,2 @@ +/etc/apt/apt.conf.d/53algorand-devtools-upgrades + diff --git a/installer/debian/algorand-devtools/control b/installer/debian/algorand-devtools/control new file mode 100644 index 0000000000..c728b1b85c --- /dev/null +++ b/installer/debian/algorand-devtools/control @@ -0,0 +1,10 @@ +Package: @PKG_NAME@ +Homepage: https://www.algorand.com/ +Maintainer: Algorand developers +Version: @VER@ +Architecture: @ARCH@ +Pre-Depends: algorand (>= @VER@) +Recommends: unattended-upgrades +Description: Algorand tools software + This package provides development tools for the Algorand blockchain. + diff --git a/installer/debian/conffiles b/installer/debian/algorand/conffiles similarity index 100% rename from installer/debian/conffiles rename to installer/debian/algorand/conffiles diff --git a/installer/debian/control b/installer/debian/algorand/control similarity index 100% rename from installer/debian/control rename to installer/debian/algorand/control diff --git a/installer/debian/postinst b/installer/debian/algorand/postinst similarity index 100% rename from installer/debian/postinst rename to installer/debian/algorand/postinst diff --git a/installer/debian/postrm b/installer/debian/algorand/postrm similarity index 100% rename from installer/debian/postrm rename to installer/debian/algorand/postrm diff --git a/installer/debian/preinst b/installer/debian/algorand/preinst similarity index 70% rename from installer/debian/preinst rename to installer/debian/algorand/preinst index 6817072ee5..6608010ba9 100755 --- a/installer/debian/preinst +++ b/installer/debian/algorand/preinst @@ -13,7 +13,9 @@ if dpkg-query --list 'algorand*' &> /dev/null then if PKG_INFO=$(dpkg-query --show --showformat='${Package} ${Status}\n' 'algorand*' | grep "install ok installed") then - INSTALLED_PKG=$(grep -v algorand-indexer <<< "$PKG_INFO" | awk '{print $1}') + # Filter out `algorand-indexer` and `algorand-devtools` packages, they are allowed to be + # installed alongside other `algorand` packages. + INSTALLED_PKG=$(grep -v -e algorand-indexer -e algorand-devtools <<< "$PKG_INFO" | awk '{print $1}') if [ -n "$INSTALLED_PKG" ] then diff --git a/installer/debian/prerm b/installer/debian/algorand/prerm similarity index 100% rename from installer/debian/prerm rename to installer/debian/algorand/prerm diff --git a/installer/rpm/algorand-devtools/algorand-devtools.repo b/installer/rpm/algorand-devtools/algorand-devtools.repo new file mode 100644 index 0000000000..60b1d8cbaf --- /dev/null +++ b/installer/rpm/algorand-devtools/algorand-devtools.repo @@ -0,0 +1,6 @@ +[algorand] +name=Algorand-devtools +baseurl=https://releases.algorand.com/rpm/stable/ +enabled=1 +gpgcheck=1 +gpgkey=https://releases.algorand.com/rpm/rpm_algorand.pub diff --git a/installer/rpm/algorand-devtools/algorand-devtools.spec b/installer/rpm/algorand-devtools/algorand-devtools.spec new file mode 100644 index 0000000000..5bd441b03a --- /dev/null +++ b/installer/rpm/algorand-devtools/algorand-devtools.spec @@ -0,0 +1,46 @@ +Name: @PKG_NAME@ +Version: @VER@ +Release: 1 +Summary: Algorand tools software +URL: https://www.algorand.com +License: AGPL-3+ +Requires: algorand >= @VER@ + +%define SRCDIR go-algorand-rpmbuild +%define _buildshell /bin/bash + +%description +This package provides development tools for the Algorand blockchain. + +%license +%include %{LICENSE_FILE} + +%prep +## Nothing to prep; intended to be built using scripts/release/mule/package/{OS_TYPE}/{ARCH}/rpm/package.sh + +%build +## Nothing to prep; intended to be built using scripts/release/mule/package/{OS_TYPE}/{ARCH}/rpm/package.sh + +%install +mkdir -p %{buildroot}/usr/bin +# NOTE: keep in sync with scripts/build_deb.sh bin_files +# NOTE: keep in sync with %files section below +for f in carpenter catchupsrv msgpacktool tealcut tealdbg; do + install -m 755 ${ALGO_BIN}/${f} %{buildroot}/usr/bin/${f} +done + +mkdir -p %{buildroot}/etc/pki/rpm-gpg +install -m 644 ${REPO_DIR}/installer/rpm/RPM-GPG-KEY-Algorand %{buildroot}/etc/pki/rpm-gpg/RPM-GPG-KEY-Algorand + +mkdir -p %{buildroot}/usr/lib/algorand/yum.repos.d +install -m 644 ${REPO_DIR}/installer/rpm/algorand-devtools/algorand-devtools.repo %{buildroot}/usr/lib/algorand/yum.repos.d/algorand-devtools.repo + +%files +/usr/bin/carpenter +/usr/bin/catchupsrv +/usr/bin/msgpacktool +/usr/bin/tealcut +/usr/bin/tealdbg +/etc/pki/rpm-gpg/RPM-GPG-KEY-Algorand +/usr/lib/algorand/yum.repos.d/algorand-devtools.repo + diff --git a/installer/rpm/0yum-algorand-hourly.cron b/installer/rpm/algorand/0yum-algorand-hourly.cron similarity index 100% rename from installer/rpm/0yum-algorand-hourly.cron rename to installer/rpm/algorand/0yum-algorand-hourly.cron diff --git a/installer/rpm/algorand.repo b/installer/rpm/algorand/algorand.repo similarity index 100% rename from installer/rpm/algorand.repo rename to installer/rpm/algorand/algorand.repo diff --git a/installer/rpm/algorand.spec b/installer/rpm/algorand/algorand.spec similarity index 91% rename from installer/rpm/algorand.spec rename to installer/rpm/algorand/algorand.spec index 57b7e6a6c1..edcdba68d0 100644 --- a/installer/rpm/algorand.spec +++ b/installer/rpm/algorand/algorand.spec @@ -42,16 +42,16 @@ install -m 644 ${REPO_DIR}/installer/algorand.service %{buildroot}/lib/systemd/s install -m 644 ${REPO_DIR}/installer/algorand@.service %{buildroot}/lib/systemd/system/algorand@.service mkdir -p %{buildroot}/etc/cron.hourly -install -m 755 ${REPO_DIR}/installer/rpm/0yum-algorand-hourly.cron %{buildroot}/etc/cron.hourly/0yum-algorand-hourly.cron +install -m 755 ${REPO_DIR}/installer/rpm/algorand/0yum-algorand-hourly.cron %{buildroot}/etc/cron.hourly/0yum-algorand-hourly.cron mkdir -p %{buildroot}/etc/yum -install -m 644 ${REPO_DIR}/installer/rpm/yum-cron-algorand.conf %{buildroot}/etc/yum/yum-cron-algorand.conf +install -m 644 ${REPO_DIR}/installer/rpm/algorand/yum-cron-algorand.conf %{buildroot}/etc/yum/yum-cron-algorand.conf mkdir -p %{buildroot}/etc/pki/rpm-gpg install -m 644 ${REPO_DIR}/installer/rpm/RPM-GPG-KEY-Algorand %{buildroot}/etc/pki/rpm-gpg/RPM-GPG-KEY-Algorand mkdir -p %{buildroot}/usr/lib/algorand/yum.repos.d -install -m 644 ${REPO_DIR}/installer/rpm/algorand.repo %{buildroot}/usr/lib/algorand/yum.repos.d/algorand.repo +install -m 644 ${REPO_DIR}/installer/rpm/algorand/algorand.repo %{buildroot}/usr/lib/algorand/yum.repos.d/algorand.repo mkdir -p %{buildroot}/var/lib/algorand/genesis if [ "%{RELEASE_GENESIS_PROCESS}" != "x" ]; then diff --git a/installer/rpm/yum-cron-algorand.conf b/installer/rpm/algorand/yum-cron-algorand.conf similarity index 100% rename from installer/rpm/yum-cron-algorand.conf rename to installer/rpm/algorand/yum-cron-algorand.conf diff --git a/package.yaml b/package.yaml index 7b3778f33c..37cc922e30 100644 --- a/package.yaml +++ b/package.yaml @@ -44,8 +44,8 @@ tasks: jobs: package: tasks: - - docker.Make.rpm - docker.Make.deb + - docker.Make.rpm package-docker: tasks: diff --git a/scripts/build_deb.sh b/scripts/build_deb.sh index d025a18252..0cd7c154ae 100755 --- a/scripts/build_deb.sh +++ b/scripts/build_deb.sh @@ -52,7 +52,7 @@ mkdir -p "${PKG_ROOT}/usr/bin" if [ "${VARIATION}" = "" ]; then # NOTE: keep in sync with installer/rpm/algorand.spec - bin_files=("algocfg" "algod" "algoh" "algokey" "carpenter" "catchupsrv" "ddconfig.sh" "diagcfg" "goal" "kmd" "msgpacktool" "node_exporter" "tealdbg") + bin_files=("algocfg" "algod" "algoh" "algokey" "carpenter" "catchupsrv" "ddconfig.sh" "diagcfg" "goal" "kmd" "msgpacktool" "node_exporter" "tealcut" "tealdbg") fi for bin in "${bin_files[@]}"; do @@ -118,8 +118,8 @@ mkdir -p "${PKG_ROOT}/DEBIAN" debian_files=("control" "preinst" "postinst" "prerm" "postrm" "conffiles") for ctl in "${debian_files[@]}"; do # Copy first, to preserve permissions, then overwrite to fill in template. - cp -a "installer/debian/${ctl}" "${PKG_ROOT}/DEBIAN/${ctl}" - < "installer/debian/${ctl}" \ + cp -a "installer/debian/algorand/${ctl}" "${PKG_ROOT}/DEBIAN/${ctl}" + < "installer/debian/algorand/${ctl}" \ sed -e "s,@ARCH@,${ARCH}," \ -e "s,@VER@,${VER}," \ -e "s,@PKG_NAME@,${PKG_NAME}," \ diff --git a/scripts/build_package.sh b/scripts/build_package.sh index b977ef467b..e43e419b77 100755 --- a/scripts/build_package.sh +++ b/scripts/build_package.sh @@ -58,7 +58,7 @@ DEFAULT_RELEASE_NETWORK=$(./scripts/compute_branch_release_network.sh "${DEFAULT mkdir ${PKG_ROOT}/bin # If you modify this list, also update this list in ./cmd/updater/update.sh backup_binaries() -bin_files=("algocfg" "algod" "algoh" "algokey" "carpenter" "catchupsrv" "ddconfig.sh" "diagcfg" "find-nodes.sh" "goal" "kmd" "msgpacktool" "node_exporter" "tealdbg" "update.sh" "updater" "COPYING") +bin_files=("algocfg" "algod" "algoh" "algokey" "carpenter" "catchupsrv" "ddconfig.sh" "diagcfg" "find-nodes.sh" "goal" "kmd" "msgpacktool" "node_exporter" "tealcut" "tealdbg" "update.sh" "updater" "COPYING") for bin in "${bin_files[@]}"; do cp ${GOPATHBIN}/${bin} ${PKG_ROOT}/bin if [ $? -ne 0 ]; then exit 1; fi diff --git a/scripts/compute_package_name.sh b/scripts/compute_package_name.sh index 9e16b4e8e9..d3bfb28c50 100755 --- a/scripts/compute_package_name.sh +++ b/scripts/compute_package_name.sh @@ -1,12 +1,20 @@ #!/usr/bin/env bash -# ./compute_package_name +# ./compute_package_name +# +# Examples: +# ./compute_package_name stable -> algorand +# ./compute_package_name beta -> algorand-beta +# ./compute_package_name nightly algorand-devtools -> algorand-devtools-nightly -if [ "$1" = "nightly" ]; then - echo "algorand-nightly" -elif [ "$1" = "beta" ]; then - echo "algorand-beta" +CHANNEL=${1:-stable} +NAME=${2:-algorand} + +if [ "$CHANNEL" = beta ]; then + echo "$NAME-beta" +elif [ "$CHANNEL" = nightly ]; then + echo "$NAME-nightly" else - echo "algorand" + echo "$NAME" fi diff --git a/scripts/release/build/deb/build_deb.sh b/scripts/release/build/deb/build_deb.sh new file mode 100755 index 0000000000..05b318ca6f --- /dev/null +++ b/scripts/release/build/deb/build_deb.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# shellcheck disable=2038,2045,2064 + +set -ex + +echo +date "+build_release begin PACKAGE DEB stage %Y%m%d_%H%M%S" +echo + +ARCH="$1" +OUTDIR="$2" +CHANNEL="$3" +PKG_NAME="$4" +OS_TYPE=$(./scripts/ostype.sh) +mkdir -p "$OUTDIR/bin" +VER=${VERSION:-$(./scripts/compute_build_number.sh -f)} + +echo "Building debian package $PKG_NAME ($CHANNEL) for '$OS - $ARCH'" + +DEFAULTNETWORK=$("./scripts/compute_branch_network.sh") +DEFAULT_RELEASE_NETWORK=$("./scripts/compute_branch_release_network.sh" "$DEFAULTNETWORK") +export DEFAULT_RELEASE_NETWORK + +PKG_ROOT=$(mktemp -d) +trap "rm -rf $PKG_ROOT" 0 + +mkdir -p "$PKG_ROOT/usr/bin" + +# NOTE: keep in sync with `./installer/rpm/algorand.spec`. +if [ "$PKG_NAME" = "algorand-devtools" ]; then + BIN_FILES=("carpenter" "catchupsrv" "msgpacktool" "tealcut" "tealdbg") + UNATTENDED_UPGRADES_FILE="53algorand-devtools-upgrades" + OUTPUT_DEB="$OUTDIR/algorand_devtools_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" +else + BIN_FILES=("algocfg" "algod" "algoh" "algokey" "ddconfig.sh" "diagcfg" "goal" "kmd" "node_exporter") + UNATTENDED_UPGRADES_FILE="51algorand-upgrades" + OUTPUT_DEB="$OUTDIR/algorand_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" +fi + +for binary in "${BIN_FILES[@]}"; do + cp "$GOPATH/bin/$binary" "$PKG_ROOT/usr/bin" + chmod 755 "$PKG_ROOT/usr/bin/$binary" +done + +if [ "$PKG_NAME" != "algorand-devtools" ]; then + mkdir -p "${PKG_ROOT}/usr/lib/algorand" + lib_files=("updater" "find-nodes.sh") + for lib in "${lib_files[@]}"; do + cp "$GOPATH/bin/$lib" "$PKG_ROOT/usr/lib/algorand" + chmod g-w "$PKG_ROOT/usr/lib/algorand/$lib" + done + + data_files=("config.json.example" "system.json") + mkdir -p "$PKG_ROOT/var/lib/algorand" + for data in "${data_files[@]}"; do + cp "./installer/$data" "$PKG_ROOT/var/lib/algorand" + done + + genesis_dirs=("devnet" "testnet" "mainnet" "betanet") + for dir in "${genesis_dirs[@]}"; do + mkdir -p "$PKG_ROOT/var/lib/algorand/genesis/$dir" + cp "./installer/genesis/$dir/genesis.json" "$PKG_ROOT/var/lib/algorand/genesis/$dir/genesis.json" + done + cp "./installer/genesis/$DEFAULTNETWORK/genesis.json" "$PKG_ROOT/var/lib/algorand/genesis.json" + + # files should not be group writable but directories should be + chmod -R g-w "$PKG_ROOT/var/lib/algorand" + find "$PKG_ROOT/var/lib/algorand" -type d | xargs chmod g+w + + SYSTEMD_FILES=("algorand.service" "algorand@.service") + mkdir -p "$PKG_ROOT/lib/systemd/system" + for svc in "${SYSTEMD_FILES[@]}"; do + cp "./installer/${svc}" "${PKG_ROOT}/lib/systemd/system" + chmod 644 "${PKG_ROOT}/lib/systemd/system/${svc}" + done +fi + +mkdir -p "$PKG_ROOT/etc/apt/apt.conf.d" +cat < "$PKG_ROOT/etc/apt/apt.conf.d/$UNATTENDED_UPGRADES_FILE" +## This file is provided by the Algorand package to configure +## unattended upgrades for the Algorand node software. + +Unattended-Upgrade::Allowed-Origins { + "Algorand:$CHANNEL"; +}; + +Dpkg::Options { + "--force-confdef"; + "--force-confold"; +}; +EOF + +mkdir -p "$PKG_ROOT/DEBIAN" +# Can contain `control`, `preinst`, `postinst`, `prerm`, `postrm`, `conffiles`. +CTL_FILES_DIR="./installer/debian/$PKG_NAME" +for ctl_file in $(ls "$CTL_FILES_DIR"); do + # Copy first, to preserve permissions, then overwrite to fill in template. + cp -a "$CTL_FILES_DIR/$ctl_file" "$PKG_ROOT/DEBIAN/$ctl_file" + < "$CTL_FILES_DIR/$ctl_file" \ + sed -e "s,@ARCH@,$ARCH," \ + -e "s,@VER@,$VER," \ + -e "s,@PKG_NAME@,$PKG_NAME," \ + > "$PKG_ROOT/DEBIAN/$ctl_file" +done + +# TODO: make `Files:` segments for vendor/... and crypto/libsodium-fork, but reasonably this should be understood to cover all _our_ files and copied in packages continue to be licenced under their own terms +cat < "$PKG_ROOT/DEBIAN/copyright" +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: Algorand +Upstream-Contact: Algorand developers +Source: https://github.com/algorand/go-algorand + +Files: * +Copyright: Algorand developers +License: AGPL-3+ +EOF + +sed 's/^$/./g' < COPYING | sed 's/^/ /g' >> "$PKG_ROOT/DEBIAN/copyright" +mkdir -p "$PKG_ROOT/usr/share/doc/$PKG_NAME" +cp -p "$PKG_ROOT/DEBIAN/copyright" "$PKG_ROOT/usr/share/doc/$PKG_NAME/copyright" + +dpkg-deb --build "$PKG_ROOT" "$OUTPUT_DEB" + +echo +date "+build_release end PACKAGE DEB stage %Y%m%d_%H%M%S" +echo + diff --git a/scripts/release/build/deb/package.sh b/scripts/release/build/deb/package.sh index 3871dfbc11..51ecffd6c0 100755 --- a/scripts/release/build/deb/package.sh +++ b/scripts/release/build/deb/package.sh @@ -16,19 +16,20 @@ export PATH="${GOPATH}":/usr/local/go/bin:"${PATH}" pushd "${REPO_ROOT}" ./scripts/build_packages.sh "${PLATFORM}" -DEBTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "debtmp") -trap "rm -rf ${DEBTMP}" 0 +PKG_NAMES=("$ALGORAND_PACKAGE_NAME" "$DEVTOOLS_PACKAGE_NAME") +for pkg_name in "${PKG_NAMES[@]}"; do + DEBTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "debtmp") + trap "rm -rf ${DEBTMP}" 0 -if ! ./scripts/build_deb.sh "${ARCH}" "${DEBTMP}" "${CHANNEL}" -then - echo "Error building debian package for ${PLATFORM}. Aborting..." - exit 1 -fi + if ! ./scripts/release/build/deb/build_deb.sh "${ARCH}" "${DEBTMP}" "${CHANNEL}" "${pkg_name}" + then + echo "Error building debian package ${pkg_name} for ${PLATFORM}. Aborting..." + exit 1 + fi -BRANCH=$("./scripts/compute_branch.sh") -CHANNEL=$("./scripts/compute_branch_channel.sh" "$BRANCH") + cp -p "${DEBTMP}"/*.deb "${PKG_ROOT}/${pkg_name}_${CHANNEL}_${OS}-${ARCH}_${FULLVERSION}.deb" +done -cp -p "${DEBTMP}"/*.deb "${PKG_ROOT}/algorand_${CHANNEL}_${OS}-${ARCH}_${FULLVERSION}.deb" popd # build docker release package diff --git a/scripts/release/build/rpm/package.sh b/scripts/release/build/rpm/package.sh index 4a9211cca5..2e099f01d0 100755 --- a/scripts/release/build/rpm/package.sh +++ b/scripts/release/build/rpm/package.sh @@ -6,31 +6,40 @@ echo "Building RPM package" cd "$(dirname "$0")"/.. -REPO_DIR=${HOME}/subhome/go/src/github.com/algorand/go-algorand +REPO_DIR="$HOME/subhome/go/src/github.com/algorand/go-algorand" export REPO_DIR -DEFAULT_RELEASE_NETWORK=$("$REPO_DIR/scripts/compute_branch_release_network.sh" "${DEFAULTNETWORK}") +DEFAULT_RELEASE_NETWORK=$("$REPO_DIR/scripts/compute_branch_release_network.sh" "$DEFAULTNETWORK") export DEFAULT_RELEASE_NETWORK DEFAULTNETWORK=devnet export DEFAULT_NETWORK ALGO_BIN="$HOME/subhome/go/bin" export ALGO_BIN -RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp") -trap 'rm -rf ${RPMTMP}' 0 - BRANCH=$("$REPO_DIR/scripts/compute_branch.sh") CHANNEL=$("$REPO_DIR/scripts/compute_branch_channel.sh" "$BRANCH") -PKG_NAME=$("$REPO_DIR/scripts/compute_package_name.sh" "${CHANNEL:-stable}") +ALGORAND_PACKAGE_NAME=$("$REPO_DIR/scripts/compute_package_name.sh" "${CHANNEL:-stable}") +DEVTOOLS_PACKAGE_NAME=$("$REPO_DIR/scripts/compute_package_name.sh" "${CHANNEL:-stable}" algorand-devtools) + +PKG_NAMES=("$ALGORAND_PACKAGE_NAME" "$DEVTOOLS_PACKAGE_NAME") +for pkg_name in "${PKG_NAMES[@]}"; do + RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp") + trap 'rm -rf $RPMTMP' 0 + + TEMPDIR=$(mktemp -d) + trap 'rm -rf $TEMPDIR' 0 + + mkdir "$TEMPDIR/$pkg_name" + + echo "Building rpm package $pkg_name ($CHANNEL)" -TEMPDIR=$(mktemp -d) -trap 'rm -rf $TEMPDIR' 0 -< "$REPO_DIR/installer/rpm/algorand.spec" \ - sed -e "s,@PKG_NAME@,${PKG_NAME:-algorand}," \ - -e "s,@VER@,$FULLVERSION," \ - > "$TEMPDIR/algorand.spec" + < "$REPO_DIR/installer/rpm/$pkg_name/$pkg_name.spec" \ + sed -e "s,@PKG_NAME@,$pkg_name," \ + -e "s,@VER@,$FULLVERSION," \ + > "$TEMPDIR/$pkg_name/$pkg_name.spec" -rpmbuild --define "_rpmdir ${RPMTMP}" --define "RELEASE_GENESIS_PROCESS x${RELEASE_GENESIS_PROCESS}" --define "LICENSE_FILE $REPO_DIR/COPYING" -bb "${TEMPDIR}/algorand.spec" + rpmbuild --define "_rpmdir $RPMTMP" --define "RELEASE_GENESIS_PROCESS x${RELEASE_GENESIS_PROCESS}" --define "LICENSE_FILE $REPO_DIR/COPYING" -bb "$TEMPDIR/$pkg_name/$pkg_name.spec" -mkdir -p /root/subhome/node_pkg -cp -p "${RPMTMP}"/*/*.rpm /root/subhome/node_pkg + mkdir -p /root/subhome/node_pkg + cp -p "$RPMTMP"/*/*.rpm /root/subhome/node_pkg +done diff --git a/scripts/release/build/rpm/sign.sh b/scripts/release/build/rpm/sign.sh index 756d914f00..a120d064cd 100755 --- a/scripts/release/build/rpm/sign.sh +++ b/scripts/release/build/rpm/sign.sh @@ -5,44 +5,44 @@ set -ex export HOME=/root -export GOPATH=${HOME}/go -export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} +export GOPATH="$HOME/go" +export PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" -cd ${HOME} && tar jxf /root/subhome/gnupg*.tar.bz2 +cd "$HOME" && tar jxf /root/subhome/gnupg*.tar.bz2 -export PATH="${HOME}/gnupg2/bin:${PATH}" -export LD_LIBRARY_PATH=${HOME}/gnupg2/lib +export PATH="$HOME/gnupg2/bin:$PATH" +export LD_LIBRARY_PATH="$HOME/gnupg2/lib" umask 0077 mkdir -p ~/.gnupg umask 0022 -touch "${HOME}/.gnupg/gpg.conf" -if grep -q no-autostart "${HOME}/.gnupg/gpg.conf"; then - echo "" +touch "$HOME/.gnupg/gpg.conf" +if grep -q no-autostart "$HOME/.gnupg/gpg.conf"; then + echo else - echo "no-autostart" >> "${HOME}/.gnupg/gpg.conf" + echo "no-autostart" >> "$HOME/.gnupg/gpg.conf" fi -rm -f ${HOME}/.gnupg/S.gpg-agent +rm -f $HOME/.gnupg/S.gpg-agent (cd ~/.gnupg && ln -s /root/S.gpg-agent S.gpg-agent) gpg --import /root/keys/dev.pub gpg --import /root/keys/rpm.pub rpmkeys --import /root/keys/rpm.pub -echo "wat" | gpg -u rpm@algorand.com --clearsign +echo wat | gpg -u rpm@algorand.com --clearsign -cat <"${HOME}/.rpmmacros" +cat <"$HOME/.rpmmacros" %_gpg_name Algorand RPM -%__gpg ${HOME}/gnupg2/bin/gpg +%__gpg $HOME/gnupg2/bin/gpg %__gpg_check_password_cmd true EOF -cat <"${HOME}/rpmsign.py" +cat <"$HOME/rpmsign.py" import rpm import sys rpm.addSign(sys.argv[1], '') EOF NEWEST_RPM=$(ls -t /root/subhome/node_pkg/*rpm | head -1) -python2 "${HOME}/rpmsign.py" "${NEWEST_RPM}" +python2 "$HOME/rpmsign.py" "$NEWEST_RPM" diff --git a/scripts/release/build/stage/package/task.sh b/scripts/release/build/stage/package/task.sh index eebe44f92b..9685fca5ee 100755 --- a/scripts/release/build/stage/package/task.sh +++ b/scripts/release/build/stage/package/task.sh @@ -7,12 +7,12 @@ echo date "+build_release begin PACKAGE stage %Y%m%d_%H%M%S" echo -. "${HOME}/build_env" +. "$HOME/build_env" # Order matters (deb -> rpm)! "${HOME}"/go/src/github.com/algorand/go-algorand/scripts/release/build/deb/package.sh -sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${HOME},dst=/root/subhome algocentosbuild /root/subhome/go/src/github.com/algorand/go-algorand/scripts/release/build/rpm/package.sh" +sg docker "docker run --rm --env-file $HOME/build_env_docker --mount type=bind,src=$HOME,dst=/root/subhome algocentosbuild /root/subhome/go/src/github.com/algorand/go-algorand/scripts/release/build/rpm/package.sh" echo date "+build_release end PACKAGE stage %Y%m%d_%H%M%S" diff --git a/scripts/release/common/setup.sh b/scripts/release/common/setup.sh index 0426a73d51..80d3c22f4d 100755 --- a/scripts/release/common/setup.sh +++ b/scripts/release/common/setup.sh @@ -56,7 +56,7 @@ then exit 1 fi cd "${HOME}" -if ! curl -O https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz +if ! curl -O "https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz" then echo Golang could not be installed! exit 1 @@ -134,10 +134,12 @@ REPO_ROOT="${GOPATH}"/src/github.com/algorand/go-algorand PLATFORM=$("${REPO_ROOT}"/scripts/osarchtype.sh) PLATFORM_SPLIT=(${PLATFORM//\// }) +CHANNEL=${CHANNEL:-$("${GOPATH}"/src/github.com/algorand/go-algorand/scripts/compute_branch_channel.sh "$BRANCH")} + # a bash user might `source build_env` to manually continue a broken build cat << EOF > "${HOME}"/build_env export BRANCH=${BRANCH} -export CHANNEL=${CHANNEL:-$("${GOPATH}"/src/github.com/algorand/go-algorand/scripts/compute_branch_channel.sh "$BRANCH")} +export CHANNEL=${CHANNEL} export COMMIT_HASH=${COMMIT_HASH} export DEFAULTNETWORK=$(PATH=${PATH} "${REPO_ROOT}"/scripts/compute_branch_network.sh) export DC_IP=$(curl --silent http://169.254.169.254/latest/meta-data/local-ipv4) @@ -149,6 +151,8 @@ export ARCH=${PLATFORM_SPLIT[1]} export REPO_ROOT=${REPO_ROOT} export RELEASE_GENESIS_PROCESS=true export VARIATIONS=base +export ALGORAND_PACKAGE_NAME=$("${GOPATH}"/src/github.com/algorand/go-algorand/scripts/compute_package_name.sh "${CHANNEL:-stable}") +export DEVTOOLS_PACKAGE_NAME=$("${GOPATH}"/src/github.com/algorand/go-algorand/scripts/compute_package_name.sh "${CHANNEL:-stable}" algorand-devtools) EOF # strip leading 'export ' for docker --env-file diff --git a/scripts/release/forward_gpg_agent.sh b/scripts/release/forward_gpg_agent.sh index 458c72510f..38161ec034 100755 --- a/scripts/release/forward_gpg_agent.sh +++ b/scripts/release/forward_gpg_agent.sh @@ -9,7 +9,7 @@ then exit 1 fi -BRANCH=build-packages +BRANCH=${1:-build-packages} EC2_INSTANCE_KEY=ReleaseBuildInstanceKey.pem # Get the ec2 instance name and the ephemeral private key from the Jenkins server. @@ -24,12 +24,12 @@ scp -i "$JENKINS_KEY" "$JENKINS":~/"$EC2_INSTANCE_KEY" . ssh -i "$JENKINS_KEY" "$JENKINS" 'rm ~/"$EC2_INSTANCE_KEY"' # Second, get the ec2 instance name. -INSTANCE=$(ssh -i "$JENKINS_KEY" "$JENKINS" sudo cat /opt/jenkins/workspace/$BRANCH/scripts/release/common/ec2/tmp/instance) +INSTANCE=$(ssh -i "$JENKINS_KEY" "$JENKINS" sudo cat /opt/jenkins/workspace/"$BRANCH"/scripts/release/common/ec2/tmp/instance) gpgp=$(find /usr/lib/gnupg{2,,1} -type f -name gpg-preset-passphrase 2> /dev/null) # Here we need to grab the signing subkey, hence `tail -1`. -KEYGRIP=$(gpg -K --with-keygrip --textmode dev@algorand.com | egrep -A 1 '^ssb[^#]' | grep Keygrip | awk '{ print $3 }') +KEYGRIP=$(gpg -K --with-keygrip --textmode dev@algorand.com | grep -AE 1 '^ssb[^#]' | grep Keygrip | awk '{ print $3 }') echo "enter dev@ password" $gpgp --verbose --preset "$KEYGRIP" diff --git a/scripts/release/mule/Makefile.mule b/scripts/release/mule/Makefile.mule index 0f5584eaf9..eb0465423f 100644 --- a/scripts/release/mule/Makefile.mule +++ b/scripts/release/mule/Makefile.mule @@ -46,9 +46,13 @@ mule-deploy-%: PKG_TYPE=$* mule-deploy-%: scripts/release/mule/deploy/$(PKG_TYPE)/deploy.sh +mule-package-deb: ci-build mule-package-%: PKG_TYPE=$* -mule-package-%: ci-build +mule-package-%: + echo Building algorand package... scripts/release/mule/package/$(PKG_TYPE)/package.sh + echo Building algorand-devtools package... + scripts/release/mule/package/$(PKG_TYPE)/package.sh algorand-devtools mule-sign-%: PKG_TYPE=$* mule-sign-%: diff --git a/scripts/release/mule/package/deb/package.sh b/scripts/release/mule/package/deb/package.sh index 41d7e5ebf5..54bcd910c8 100755 --- a/scripts/release/mule/package/deb/package.sh +++ b/scripts/release/mule/package/deb/package.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# shellcheck disable=2038,2064 +# shellcheck disable=2038,2045,2064 set -ex @@ -14,7 +14,8 @@ CHANNEL=${CHANNEL:-$(./scripts/compute_branch_channel.sh "$BRANCH")} OUTDIR="./tmp/node_pkgs/$OS_TYPE/$ARCH" mkdir -p "$OUTDIR/bin" ALGO_BIN="./tmp/node_pkgs/$OS_TYPE/$ARCH/$CHANNEL/$OS_TYPE-$ARCH/bin" -PKG_NAME=$(./scripts/compute_package_name.sh "${CHANNEL:-stable}") +# A make target in Makefile.mule may pass the name as an argument. +PKG_NAME=${1:-$(./scripts/compute_package_name.sh "${CHANNEL:-stable}")} VER=${VERSION:-$(./scripts/compute_build_number.sh -f)} echo "Building debian package for '${OS} - ${ARCH}'" @@ -28,59 +29,78 @@ trap "rm -rf $PKG_ROOT" 0 mkdir -p "${PKG_ROOT}/usr/bin" -# NOTE: keep in sync with installer/rpm/algorand.spec -bin_files=("algocfg" "algod" "algoh" "algokey" "carpenter" "catchupsrv" "ddconfig.sh" "diagcfg" "goal" "kmd" "msgpacktool" "node_exporter" "tealdbg") +# NOTE: keep in sync with `./installer/rpm/algorand.spec`. +if [ "$PKG_NAME" = "algorand-devtools" ]; then + BIN_FILES=("carpenter" "catchupsrv" "msgpacktool" "tealcut" "tealdbg") + UNATTENDED_UPGRADES_FILE="53algorand-devtools-upgrades" +else + BIN_FILES=("algocfg" "algod" "algoh" "algokey" "ddconfig.sh" "diagcfg" "goal" "kmd" "node_exporter") + UNATTENDED_UPGRADES_FILE="51algorand-upgrades" +fi -for binary in "${bin_files[@]}"; do +OUTPUT_DEB="$OUTDIR/${PKG_NAME}_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" + +for binary in "${BIN_FILES[@]}"; do cp "${ALGO_BIN}/${binary}" "${PKG_ROOT}"/usr/bin chmod 755 "${PKG_ROOT}/usr/bin/${binary}" done -mkdir -p "${PKG_ROOT}/usr/lib/algorand" -lib_files=("updater" "find-nodes.sh") -for lib in "${lib_files[@]}"; do - cp "${ALGO_BIN}/${lib}" "${PKG_ROOT}/usr/lib/algorand" - chmod g-w "${PKG_ROOT}/usr/lib/algorand/${lib}" -done - -data_files=("config.json.example" "system.json") -mkdir -p "${PKG_ROOT}/var/lib/algorand" -for data in "${data_files[@]}"; do - cp "installer/${data}" "${PKG_ROOT}/var/lib/algorand" -done - -cp "./installer/genesis/${DEFAULTNETWORK}/genesis.json" "${PKG_ROOT}/var/lib/algorand/genesis.json" - -systemd_files=("algorand.service" "algorand@.service") -mkdir -p "${PKG_ROOT}/lib/systemd/system" -for svc in "${systemd_files[@]}"; do - cp "installer/${svc}" "${PKG_ROOT}/lib/systemd/system" - chmod 644 "${PKG_ROOT}/lib/systemd/system/${svc}" -done +if [ "$PKG_NAME" != "algorand-devtools" ]; then + mkdir -p "${PKG_ROOT}/usr/lib/algorand" + lib_files=("updater" "find-nodes.sh") + for lib in "${lib_files[@]}"; do + cp "${ALGO_BIN}/${lib}" "${PKG_ROOT}/usr/lib/algorand" + chmod g-w "${PKG_ROOT}/usr/lib/algorand/${lib}" + done + + data_files=("config.json.example" "system.json") + mkdir -p "${PKG_ROOT}/var/lib/algorand" + for data in "${data_files[@]}"; do + cp "installer/${data}" "${PKG_ROOT}/var/lib/algorand" + done + + cp "./installer/genesis/${DEFAULTNETWORK}/genesis.json" "${PKG_ROOT}/var/lib/algorand/genesis.json" + + # files should not be group writable but directories should be + chmod -R g-w "${PKG_ROOT}/var/lib/algorand" + find "${PKG_ROOT}/var/lib/algorand" -type d | xargs chmod g+w + + SYSTEMD_FILES=("algorand.service" "algorand@.service") + mkdir -p "${PKG_ROOT}/lib/systemd/system" + for svc in "${SYSTEMD_FILES[@]}"; do + cp "installer/${svc}" "${PKG_ROOT}/lib/systemd/system" + chmod 644 "${PKG_ROOT}/lib/systemd/system/${svc}" + done +fi -unattended_upgrades_files=("51algorand-upgrades") mkdir -p "${PKG_ROOT}/etc/apt/apt.conf.d" -for f in "${unattended_upgrades_files[@]}"; do - < "installer/${f}" \ - sed -e "s,@CHANNEL@,${CHANNEL}," \ - > "${PKG_ROOT}/etc/apt/apt.conf.d/${f}" -done - -# files should not be group writable but directories should be -chmod -R g-w "${PKG_ROOT}/var/lib/algorand" -find "${PKG_ROOT}/var/lib/algorand" -type d | xargs chmod g+w +cat < "${PKG_ROOT}/etc/apt/apt.conf.d/${UNATTENDED_UPGRADES_FILE}" +## This file is provided by the Algorand package to configure +## unattended upgrades for the Algorand node software. + +Unattended-Upgrade::Allowed-Origins { + "Algorand:${CHANNEL}"; +}; + +Dpkg::Options { + "--force-confdef"; + "--force-confold"; +}; +EOF mkdir -p "${PKG_ROOT}/DEBIAN" -debian_files=("control" "preinst" "postinst" "prerm" "postrm" "conffiles") -for ctl in "${debian_files[@]}"; do +# Can contain `control`, `preinst`, `postinst`, `prerm`, `postrm`, `conffiles`. +CTL_FILES_DIR="installer/debian/${PKG_NAME}" +for ctl_file in $(ls "${CTL_FILES_DIR}"); do # Copy first, to preserve permissions, then overwrite to fill in template. - cp -a "installer/debian/${ctl}" "${PKG_ROOT}/DEBIAN/${ctl}" - < "installer/debian/${ctl}" \ + cp -a "${CTL_FILES_DIR}/${ctl_file}" "${PKG_ROOT}/DEBIAN/${ctl_file}" + < "${CTL_FILES_DIR}/${ctl_file}" \ sed -e "s,@ARCH@,${ARCH}," \ -e "s,@VER@,${VER}," \ -e "s,@PKG_NAME@,${PKG_NAME}," \ - > "${PKG_ROOT}/DEBIAN/${ctl}" + > "${PKG_ROOT}/DEBIAN/${ctl_file}" done + # TODO: make `Files:` segments for vendor/... and crypto/libsodium-fork, but reasonably this should be understood to cover all _our_ files and copied in packages continue to be licenced under their own terms cat < "${PKG_ROOT}/DEBIAN/copyright" Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ @@ -92,12 +112,12 @@ Files: * Copyright: Algorand developers License: AGPL-3+ EOF + sed 's/^$/./g' < COPYING | sed 's/^/ /g' >> "${PKG_ROOT}/DEBIAN/copyright" -mkdir -p "${PKG_ROOT}/usr/share/doc/algorand" -cp -p "${PKG_ROOT}/DEBIAN/copyright" "${PKG_ROOT}/usr/share/doc/algorand/copyright" +mkdir -p "${PKG_ROOT}/usr/share/doc/${PKG_NAME}" +cp -p "${PKG_ROOT}/DEBIAN/copyright" "${PKG_ROOT}/usr/share/doc/${PKG_NAME}/copyright" -OUTPUT="$OUTDIR/algorand_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" -dpkg-deb --build "${PKG_ROOT}" "${OUTPUT}" +dpkg-deb --build "${PKG_ROOT}" "${OUTPUT_DEB}" echo date "+build_release end PACKAGE DEB stage %Y%m%d_%H%M%S" diff --git a/scripts/release/mule/package/rpm/package.sh b/scripts/release/mule/package/rpm/package.sh index 5bf71facd8..24a8b1e887 100755 --- a/scripts/release/mule/package/rpm/package.sh +++ b/scripts/release/mule/package/rpm/package.sh @@ -14,9 +14,10 @@ ALGO_BIN="$REPO_DIR/tmp/node_pkgs/$OS_TYPE/$ARCH/$CHANNEL/$OS_TYPE-$ARCH/bin" # TODO: Should there be a default network? DEFAULTNETWORK=devnet DEFAULT_RELEASE_NETWORK=$(./scripts/compute_branch_release_network.sh "$DEFAULTNETWORK") -PKG_NAME=$(./scripts/compute_package_name.sh "${CHANNEL:-stable}") +# A make target in Makefile.mule may pass the name as an argument. +PKG_NAME=${1:-$(./scripts/compute_package_name.sh "${CHANNEL:-stable}")} -# The following need to be exported for use in ./go-algorand/installer/rpm/algorand.spec. +# The following need to be exported for use in ./go-algorand/installer/rpm/$PKG_NAME/$PKG_NAME.spec. export DEFAULT_NETWORK export DEFAULT_RELEASE_NETWORK export REPO_DIR @@ -27,12 +28,13 @@ trap 'rm -rf $RPMTMP' 0 TEMPDIR=$(mktemp -d) trap 'rm -rf $TEMPDIR' 0 -< "./installer/rpm/algorand.spec" \ - sed -e "s,@PKG_NAME@,$PKG_NAME," \ +< "./installer/rpm/$PKG_NAME/$PKG_NAME.spec" \ + sed -e "s,@PKG_NAME@,${PKG_NAME}," \ -e "s,@VER@,$FULLVERSION," \ - > "$TEMPDIR/algorand.spec" + -e "s,@ARCH@,$ARCH," \ + > "$TEMPDIR/$PKG_NAME.spec" -rpmbuild --buildroot "$HOME/foo" --define "_rpmdir $RPMTMP" --define "RELEASE_GENESIS_PROCESS x$RELEASE_GENESIS_PROCESS" --define "LICENSE_FILE ./COPYING" -bb "$TEMPDIR/algorand.spec" +rpmbuild --buildroot "$HOME/foo" --define "_rpmdir $RPMTMP" --define "RELEASE_GENESIS_PROCESS x$RELEASE_GENESIS_PROCESS" --define "LICENSE_FILE ./COPYING" -bb "$TEMPDIR/$PKG_NAME.spec" cp -p "$RPMTMP"/*/*.rpm "./tmp/node_pkgs/$OS_TYPE/$ARCH" diff --git a/scripts/release/mule/test/test.sh b/scripts/release/mule/test/test.sh index 452ce6c12e..831220ecc3 100755 --- a/scripts/release/mule/test/test.sh +++ b/scripts/release/mule/test/test.sh @@ -9,13 +9,25 @@ ARCH_TYPE=$(./scripts/archtype.sh) export ARCH_TYPE OS_TYPE=$(./scripts/ostype.sh) export OS_TYPE -VERSION=${VERSION:-$(./scripts/compute_build_number.sh -f)} + +if [ -z "$VERSION" ]; then + VERSION=${VERSION:-$(./scripts/compute_build_number.sh -f)} +fi export VERSION -BRANCH=${BRANCH:-$(git rev-parse --abbrev-ref HEAD)} + +if [ -z "$BRANCH" ]; then + BRANCH=${BRANCH:-$(git rev-parse --abbrev-ref HEAD)} +fi export BRANCH -CHANNEL=${CHANNEL:-$(./scripts/compute_branch_channel.sh "$BRANCH")} + +if [ -z "$CHANNEL" ]; then + CHANNEL=${CHANNEL:-$(./scripts/compute_branch_channel.sh "$BRANCH")} +fi export CHANNEL -SHA=${SHA:-$(git rev-parse HEAD)} + +if [ -z "$SHA" ]; then + SHA=${SHA:-$(git rev-parse HEAD)} +fi export SHA if ! $USE_CACHE diff --git a/scripts/release/mule/test/tests/run_tests b/scripts/release/mule/test/tests/run_tests index 6dd3146842..c7ba6e23a9 100755 --- a/scripts/release/mule/test/tests/run_tests +++ b/scripts/release/mule/test/tests/run_tests @@ -45,9 +45,17 @@ fi if [ "$PKG_TYPE" == "deb" ] then - dpkg -i "$WORKDIR/tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE"/*"$FULLVERSION"*.deb + for deb in $(ls "$WORKDIR/tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE"/*"$FULLVERSION"*.deb); do + if [[ ! "$deb" =~ devtools ]]; then + dpkg -i "$deb" + fi + done else - yum install "$WORKDIR/tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE"/*"$FULLVERSION"*.rpm -y + for rpm in $(ls "$WORKDIR/tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE"/*"$FULLVERSION"*.rpm); do + if [[ ! "$rpm" =~ devtools ]]; then + yum install "$rpm" -y + fi + done fi export BRANCH diff --git a/scripts/release/mule/test/tests/verify_pkg_string.sh b/scripts/release/mule/test/tests/verify_pkg_string.sh index e9b602a1e9..a2850a54d4 100755 --- a/scripts/release/mule/test/tests/verify_pkg_string.sh +++ b/scripts/release/mule/test/tests/verify_pkg_string.sh @@ -13,7 +13,8 @@ SHORT_HASH=${COMMIT_HASH:0:8} # # Since we're passing in the full hash, we won't using the closing paren. # Use a regex over the multi-line string. -if [[ "$STR" =~ .*"$FULLVERSION.$CHANNEL [$BRANCH] (commit #$SHORT_HASH".* ]] +if [[ "$STR" =~ .*"$FULLVERSION.$CHANNEL [$BRANCH] (commit #$SHORT_HASH".* ]] || + [[ "$STR" =~ .*"$FULLVERSION. [$BRANCH] (commit #$SHORT_HASH".* ]] then echo -e "[$0] The result of \`algod -v\` is a correct match.\n$STR" exit 0 diff --git a/scripts/release/test/deb/test_algorand.sh b/scripts/release/test/deb/test_algorand.sh index 3c3d7bb87a..1d165e4af8 100755 --- a/scripts/release/test/deb/test_algorand.sh +++ b/scripts/release/test/deb/test_algorand.sh @@ -1,45 +1,47 @@ #!/usr/bin/env bash +# shellcheck disable=2045 set -ex -export GOPATH=${HOME}/go -export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} +export GOPATH="$HOME/go" +export PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" PKG_NAME=algorand -if [ "${CHANNEL}" = beta ]; then +if [ "$CHANNEL" = beta ]; then PKG_NAME=algorand-beta fi apt-get update apt-get install -y gnupg2 curl software-properties-common python3 -apt-get install -y /root/subhome/node_pkg/*.deb -algod -v - -mkdir -p /root/testnode -cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode - -goal node start -d /root/testnode -goal node wait -d /root/testnode -w 120 -goal node stop -d /root/testnode - -apt-key add /root/keys/dev.pub -apt-key add /root/keys/rpm.pub -add-apt-repository "deb http://${DC_IP}:8111/ stable main" -apt-get update -apt-get install -y "${PKG_NAME}" -algod -v -# check that the installed version is now the current version -algod -v | grep -q "${FULLVERSION}.${CHANNEL}" - -if [ ! -d /root/testnode ]; then - mkdir -p /root/testnode - cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode -fi - -goal node start -d /root/testnode -goal node wait -d /root/testnode -w 120 -goal node stop -d /root/testnode +for deb in $(ls /root/subhome/node_pkg/*.deb); do + if [[ ! "$deb" =~ devtools ]]; then + apt-get install -y "$deb" + algod -v + + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode + + goal node start -d /root/testnode + goal node wait -d /root/testnode -w 120 + goal node stop -d /root/testnode + + apt-key add /root/keys/dev.pub + apt-key add /root/keys/rpm.pub + add-apt-repository "deb http://${DC_IP}:8111/ stable main" + apt-get update + apt-get install -y "$PKG_NAME" + + if [ ! -d /root/testnode ]; then + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode + fi + + goal node start -d /root/testnode + goal node wait -d /root/testnode -w 120 + goal node stop -d /root/testnode + fi +done echo UBUNTU_DOCKER_TEST_OK diff --git a/scripts/release/test/rpm/test_algorand.sh b/scripts/release/test/rpm/test_algorand.sh index 57c774a124..a3dbd3fc39 100755 --- a/scripts/release/test/rpm/test_algorand.sh +++ b/scripts/release/test/rpm/test_algorand.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# shellcheck disable=1090,2012 +# shellcheck disable=2012,2045 set -ex @@ -27,7 +27,7 @@ cd "${HOME}"/.gnupg && ln -s "${HOME}"/S.gpg-agent S.gpg-agent gpg --import /root/keys/dev.pub gpg --import /root/keys/rpm.pub rpmkeys --import /root/keys/rpm.pub -echo "wat" | gpg -u rpm@algorand.com --clearsign +echo wat | gpg -u rpm@algorand.com --clearsign cat <"${HOME}/.rpmmacros" %_gpg_name Algorand RPM @@ -41,42 +41,42 @@ import sys rpm.addSign(sys.argv[1], '') EOF -NEWEST_RPM=$(ls -t /root/subhome/node_pkg/*.rpm | head -1) -python2 "${HOME}/rpmsign.py" "${NEWEST_RPM}" +for rpm in $(ls /root/subhome/node_pkg/*.rpm); do + if [[ ! "$rpm" =~ devtools ]]; then + python2 "${HOME}/rpmsign.py" "$rpm" -cp -p "${NEWEST_RPM}" /root/dummyrepo -createrepo --database /root/dummyrepo -rm -f /root/dummyrepo/repodata/repomd.xml.asc -gpg -u rpm@algorand.com --detach-sign --armor /root/dummyrepo/repodata/repomd.xml + cp -p "$rpm" /root/dummyrepo + createrepo --database /root/dummyrepo + rm -f /root/dummyrepo/repodata/repomd.xml.asc + gpg -u rpm@algorand.com --detach-sign --armor /root/dummyrepo/repodata/repomd.xml -OLDRPM=$(ls -t /root/subhome/node_pkg/*.rpm | head -1) -if [ -f "${OLDRPM}" ]; then - yum install -y "${OLDRPM}" - algod -v + OLDRPM=$(ls -t /root/subhome/node_pkg/*.rpm | head -1) + if [ -f "${OLDRPM}" ]; then + yum install -y "${OLDRPM}" + algod -v - mkdir -p /root/testnode - cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode - goal node start -d /root/testnode - goal node wait -d /root/testnode -w 120 - goal node stop -d /root/testnode -fi + goal node start -d /root/testnode + goal node wait -d /root/testnode -w 120 + goal node stop -d /root/testnode + fi -yum-config-manager --add-repo "http://${DC_IP}:8111/algodummy.repo" + yum-config-manager --add-repo "http://${DC_IP}:8111/algodummy.repo" -yum install -y algorand -algod -v -# check that the installed version is now the current version -algod -v | grep -q "${FULLVERSION}.${CHANNEL}" + yum install -y algorand -if [ ! -d /root/testnode ]; then - mkdir -p /root/testnode - cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode -fi + if [ ! -d /root/testnode ]; then + mkdir -p /root/testnode + cp -p /var/lib/algorand/genesis/testnet/genesis.json /root/testnode + fi -goal node start -d /root/testnode -goal node wait -d /root/testnode -w 120 -goal node stop -d /root/testnode + goal node start -d /root/testnode + goal node wait -d /root/testnode -w 120 + goal node stop -d /root/testnode + fi +done echo CENTOS_DOCKER_TEST_OK diff --git a/scripts/release/test/stage/setup/run.sh b/scripts/release/test/stage/setup/run.sh index 4aef7cce39..ae94b760b3 100755 --- a/scripts/release/test/stage/setup/run.sh +++ b/scripts/release/test/stage/setup/run.sh @@ -19,7 +19,7 @@ BUILD_ENV=$(ssh -i ReleaseBuildInstanceKey.pem -A ubuntu@"$INSTANCE" cat build_e CHANNEL=$(sed -n 's/.*CHANNEL=\(.*\)/\1/p' <<< "$BUILD_ENV") RELEASE=$(sed -n 's/.*FULLVERSION=\(.*\)/\1/p' <<< "$BUILD_ENV") -rm -rf pkg/* && mkdir -p pkg/"$FULLVERSION" +rm -rf pkg/* && mkdir -p pkg/"$RELEASE" aws s3 sync "s3://algorand-staging/releases/$CHANNEL/$RELEASE/" pkg/ --exclude "*" --include "*.deb" --include "*.rpm" From f5b760a3752a578ece8a5cee73d4c80c3f8eee43 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Wed, 16 Sep 2020 17:10:52 -0400 Subject: [PATCH 043/136] Return 404 if v1/transaction/{txid} doesn't find the txn. (#1504) Return 404 instead of 500 if the transaction isn't found. --- daemon/algod/api/server/v1/handlers/handlers.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/daemon/algod/api/server/v1/handlers/handlers.go b/daemon/algod/api/server/v1/handlers/handlers.go index 377c35d559..03cc5083bb 100644 --- a/daemon/algod/api/server/v1/handlers/handlers.go +++ b/daemon/algod/api/server/v1/handlers/handlers.go @@ -17,6 +17,7 @@ package handlers import ( + "database/sql" "encoding/base64" "errors" "fmt" @@ -1846,6 +1847,10 @@ func GetTransactionByID(ctx lib.ReqContext, context echo.Context) { } rnd, err := indexer.GetRoundByTXID(queryTxID) + if err == sql.ErrNoRows { + lib.ErrorResponse(w, http.StatusNotFound, err, errTransactionNotFound, ctx.Log) + return + } if err != nil { lib.ErrorResponse(w, http.StatusInternalServerError, err, errFailedGettingInformationFromIndexer, ctx.Log) return From d8d60e9a243c802725951ab5834509b9d6618969 Mon Sep 17 00:00:00 2001 From: btoll Date: Wed, 16 Sep 2020 17:11:53 -0400 Subject: [PATCH 044/136] Remove commas from mule yaml files (#1469) Remove commas from mule yaml files. Their presence is causing errors. --- package-deploy.yaml | 8 ++++---- package-sign.yaml | 4 ++-- package-test.yaml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package-deploy.yaml b/package-deploy.yaml index 2206ed4a78..8ace1b87ab 100644 --- a/package-deploy.yaml +++ b/package-deploy.yaml @@ -6,7 +6,7 @@ agents: buildArgs: - GOLANG_VERSION=`./scripts/get_golang_version.sh` env: - - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID, + - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY volumes: - /var/run/docker.sock:/var/run/docker.sock @@ -19,11 +19,11 @@ agents: buildArgs: - GOLANG_VERSION=`./scripts/get_golang_version.sh` env: - - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID, + - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY volumes: - - $HOME/packages:/root/packages, - - $XDG_RUNTIME_DIR/gnupg/S.gpg-agent:/root/.gnupg/S.gpg-agent, + - $HOME/packages:/root/packages + - $XDG_RUNTIME_DIR/gnupg/S.gpg-agent:/root/.gnupg/S.gpg-agent - $HOME/.gnupg/pubring.kbx:/root/.gnupg/pubring.kbx workDir: $HOME/projects/go-algorand diff --git a/package-sign.yaml b/package-sign.yaml index bb32340018..63d301e18d 100644 --- a/package-sign.yaml +++ b/package-sign.yaml @@ -6,10 +6,10 @@ agents: buildArgs: - GOLANG_VERSION=`./scripts/get_golang_version.sh` env: - - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID, + - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY volumes: - - $XDG_RUNTIME_DIR/gnupg/S.gpg-agent:/root/.gnupg/S.gpg-agent, + - $XDG_RUNTIME_DIR/gnupg/S.gpg-agent:/root/.gnupg/S.gpg-agent - $HOME/.gnupg/pubring.kbx:/root/.gnupg/pubring.kbx workDir: $HOME/projects/go-algorand diff --git a/package-test.yaml b/package-test.yaml index 511974c777..7f51a47cc9 100644 --- a/package-test.yaml +++ b/package-test.yaml @@ -6,7 +6,7 @@ agents: buildArgs: - GOLANG_VERSION=`./scripts/get_golang_version.sh` env: - - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID, + - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY volumes: - /var/run/docker.sock:/var/run/docker.sock From 967240a1f9c3aaa2e4e20f8cd8aae55aaac5aa6d Mon Sep 17 00:00:00 2001 From: btoll Date: Wed, 16 Sep 2020 17:51:14 -0400 Subject: [PATCH 045/136] Remove devtools binaries from algorand package (rpm) (#1515) --- installer/rpm/algorand/algorand.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/rpm/algorand/algorand.spec b/installer/rpm/algorand/algorand.spec index edcdba68d0..2d81d0917b 100644 --- a/installer/rpm/algorand/algorand.spec +++ b/installer/rpm/algorand/algorand.spec @@ -27,7 +27,7 @@ This package provides an implementation of the Algorand protocol. mkdir -p %{buildroot}/usr/bin # NOTE: keep in sync with scripts/build_deb.sh bin_files # NOTE: keep in sync with %files section below -for f in algocfg algod algoh algokey carpenter catchupsrv ddconfig.sh diagcfg goal kmd msgpacktool node_exporter tealdbg; do +for f in algocfg algod algoh algokey ddconfig.sh diagcfg goal kmd node_exporter; do install -m 755 ${ALGO_BIN}/${f} %{buildroot}/usr/bin/${f} done From 34904e2c0e25cee4fdd01a233557bdb4866aab03 Mon Sep 17 00:00:00 2001 From: btoll Date: Wed, 16 Sep 2020 22:03:43 -0400 Subject: [PATCH 046/136] No hardcoded dependency package names (#1516) Remove the hard-coded package names from the installer and control package scripts and pass in the algorand package name that is currently being built. --- installer/debian/algorand-devtools/control | 2 +- installer/rpm/algorand/algorand.spec | 4 ---- scripts/release/build/deb/build_deb.sh | 2 ++ scripts/release/build/rpm/package.sh | 1 + scripts/release/mule/package/deb/package.sh | 19 ++++++++++--------- scripts/release/mule/package/rpm/package.sh | 18 ++++++++++++------ 6 files changed, 26 insertions(+), 20 deletions(-) diff --git a/installer/debian/algorand-devtools/control b/installer/debian/algorand-devtools/control index c728b1b85c..917ed0b7d4 100644 --- a/installer/debian/algorand-devtools/control +++ b/installer/debian/algorand-devtools/control @@ -3,7 +3,7 @@ Homepage: https://www.algorand.com/ Maintainer: Algorand developers Version: @VER@ Architecture: @ARCH@ -Pre-Depends: algorand (>= @VER@) +Pre-Depends: @REQUIRED_ALGORAND_PKG@ (>= @VER@) Recommends: unattended-upgrades Description: Algorand tools software This package provides development tools for the Algorand blockchain. diff --git a/installer/rpm/algorand/algorand.spec b/installer/rpm/algorand/algorand.spec index 2d81d0917b..2c59cb1b4c 100644 --- a/installer/rpm/algorand/algorand.spec +++ b/installer/rpm/algorand/algorand.spec @@ -72,15 +72,11 @@ fi /usr/bin/algod /usr/bin/algoh /usr/bin/algokey -/usr/bin/carpenter -/usr/bin/catchupsrv /usr/bin/ddconfig.sh /usr/bin/diagcfg /usr/bin/goal /usr/bin/kmd -/usr/bin/msgpacktool /usr/bin/node_exporter -/usr/bin/tealdbg /var/lib/algorand/config.json.example %config(noreplace) /var/lib/algorand/system.json %config(noreplace) /var/lib/algorand/genesis.json diff --git a/scripts/release/build/deb/build_deb.sh b/scripts/release/build/deb/build_deb.sh index 05b318ca6f..d5c0446f5b 100755 --- a/scripts/release/build/deb/build_deb.sh +++ b/scripts/release/build/deb/build_deb.sh @@ -31,6 +31,7 @@ if [ "$PKG_NAME" = "algorand-devtools" ]; then BIN_FILES=("carpenter" "catchupsrv" "msgpacktool" "tealcut" "tealdbg") UNATTENDED_UPGRADES_FILE="53algorand-devtools-upgrades" OUTPUT_DEB="$OUTDIR/algorand_devtools_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" + REQUIRED_ALGORAND_PKG=$("./scripts/compute_package_name.sh" "$CHANNEL") else BIN_FILES=("algocfg" "algod" "algoh" "algokey" "ddconfig.sh" "diagcfg" "goal" "kmd" "node_exporter") UNATTENDED_UPGRADES_FILE="51algorand-upgrades" @@ -100,6 +101,7 @@ for ctl_file in $(ls "$CTL_FILES_DIR"); do sed -e "s,@ARCH@,$ARCH," \ -e "s,@VER@,$VER," \ -e "s,@PKG_NAME@,$PKG_NAME," \ + -e "s,@REQUIRED_ALGORAND_PKG@,$REQUIRED_ALGORAND_PKG," \ > "$PKG_ROOT/DEBIAN/$ctl_file" done diff --git a/scripts/release/build/rpm/package.sh b/scripts/release/build/rpm/package.sh index 2e099f01d0..4018a43b73 100755 --- a/scripts/release/build/rpm/package.sh +++ b/scripts/release/build/rpm/package.sh @@ -35,6 +35,7 @@ for pkg_name in "${PKG_NAMES[@]}"; do < "$REPO_DIR/installer/rpm/$pkg_name/$pkg_name.spec" \ sed -e "s,@PKG_NAME@,$pkg_name," \ -e "s,@VER@,$FULLVERSION," \ + -e "s,@REQUIRED_ALGORAND_PKG@,$ALGORAND_PACKAGE_NAME," \ > "$TEMPDIR/$pkg_name/$pkg_name.spec" rpmbuild --define "_rpmdir $RPMTMP" --define "RELEASE_GENESIS_PROCESS x${RELEASE_GENESIS_PROCESS}" --define "LICENSE_FILE $REPO_DIR/COPYING" -bb "$TEMPDIR/$pkg_name/$pkg_name.spec" diff --git a/scripts/release/mule/package/deb/package.sh b/scripts/release/mule/package/deb/package.sh index 54bcd910c8..d73f5af8d0 100755 --- a/scripts/release/mule/package/deb/package.sh +++ b/scripts/release/mule/package/deb/package.sh @@ -14,9 +14,9 @@ CHANNEL=${CHANNEL:-$(./scripts/compute_branch_channel.sh "$BRANCH")} OUTDIR="./tmp/node_pkgs/$OS_TYPE/$ARCH" mkdir -p "$OUTDIR/bin" ALGO_BIN="./tmp/node_pkgs/$OS_TYPE/$ARCH/$CHANNEL/$OS_TYPE-$ARCH/bin" -# A make target in Makefile.mule may pass the name as an argument. -PKG_NAME=${1:-$(./scripts/compute_package_name.sh "${CHANNEL:-stable}")} VER=${VERSION:-$(./scripts/compute_build_number.sh -f)} +# A make target in Makefile.mule may pass the name as an argument. +ALGORAND_PACKAGE_NAME=${1:-$(./scripts/compute_package_name.sh "$CHANNEL")} echo "Building debian package for '${OS} - ${ARCH}'" @@ -30,22 +30,23 @@ trap "rm -rf $PKG_ROOT" 0 mkdir -p "${PKG_ROOT}/usr/bin" # NOTE: keep in sync with `./installer/rpm/algorand.spec`. -if [ "$PKG_NAME" = "algorand-devtools" ]; then +if [ "$ALGORAND_PACKAGE_NAME" = "algorand-devtools" ]; then BIN_FILES=("carpenter" "catchupsrv" "msgpacktool" "tealcut" "tealdbg") UNATTENDED_UPGRADES_FILE="53algorand-devtools-upgrades" + REQUIRED_ALGORAND_PKG=$("./scripts/compute_package_name.sh" "$CHANNEL") else BIN_FILES=("algocfg" "algod" "algoh" "algokey" "ddconfig.sh" "diagcfg" "goal" "kmd" "node_exporter") UNATTENDED_UPGRADES_FILE="51algorand-upgrades" fi -OUTPUT_DEB="$OUTDIR/${PKG_NAME}_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" +OUTPUT_DEB="$OUTDIR/${ALGORAND_PACKAGE_NAME}_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" for binary in "${BIN_FILES[@]}"; do cp "${ALGO_BIN}/${binary}" "${PKG_ROOT}"/usr/bin chmod 755 "${PKG_ROOT}/usr/bin/${binary}" done -if [ "$PKG_NAME" != "algorand-devtools" ]; then +if [ "$ALGORAND_PACKAGE_NAME" != "algorand-devtools" ]; then mkdir -p "${PKG_ROOT}/usr/lib/algorand" lib_files=("updater" "find-nodes.sh") for lib in "${lib_files[@]}"; do @@ -90,14 +91,14 @@ EOF mkdir -p "${PKG_ROOT}/DEBIAN" # Can contain `control`, `preinst`, `postinst`, `prerm`, `postrm`, `conffiles`. -CTL_FILES_DIR="installer/debian/${PKG_NAME}" +CTL_FILES_DIR="installer/debian/${ALGORAND_PACKAGE_NAME}" for ctl_file in $(ls "${CTL_FILES_DIR}"); do # Copy first, to preserve permissions, then overwrite to fill in template. cp -a "${CTL_FILES_DIR}/${ctl_file}" "${PKG_ROOT}/DEBIAN/${ctl_file}" < "${CTL_FILES_DIR}/${ctl_file}" \ sed -e "s,@ARCH@,${ARCH}," \ -e "s,@VER@,${VER}," \ - -e "s,@PKG_NAME@,${PKG_NAME}," \ + -e "s,@REQUIRED_ALGORAND_PKG@,$REQUIRED_ALGORAND_PKG," \ > "${PKG_ROOT}/DEBIAN/${ctl_file}" done @@ -114,8 +115,8 @@ License: AGPL-3+ EOF sed 's/^$/./g' < COPYING | sed 's/^/ /g' >> "${PKG_ROOT}/DEBIAN/copyright" -mkdir -p "${PKG_ROOT}/usr/share/doc/${PKG_NAME}" -cp -p "${PKG_ROOT}/DEBIAN/copyright" "${PKG_ROOT}/usr/share/doc/${PKG_NAME}/copyright" +mkdir -p "${PKG_ROOT}/usr/share/doc/${ALGORAND_PACKAGE_NAME}" +cp -p "${PKG_ROOT}/DEBIAN/copyright" "${PKG_ROOT}/usr/share/doc/${ALGORAND_PACKAGE_NAME}/copyright" dpkg-deb --build "${PKG_ROOT}" "${OUTPUT_DEB}" diff --git a/scripts/release/mule/package/rpm/package.sh b/scripts/release/mule/package/rpm/package.sh index 24a8b1e887..abf3d91814 100755 --- a/scripts/release/mule/package/rpm/package.sh +++ b/scripts/release/mule/package/rpm/package.sh @@ -14,10 +14,15 @@ ALGO_BIN="$REPO_DIR/tmp/node_pkgs/$OS_TYPE/$ARCH/$CHANNEL/$OS_TYPE-$ARCH/bin" # TODO: Should there be a default network? DEFAULTNETWORK=devnet DEFAULT_RELEASE_NETWORK=$(./scripts/compute_branch_release_network.sh "$DEFAULTNETWORK") + # A make target in Makefile.mule may pass the name as an argument. -PKG_NAME=${1:-$(./scripts/compute_package_name.sh "${CHANNEL:-stable}")} +ALGORAND_PACKAGE_NAME=${1:-$(./scripts/compute_package_name.sh "$CHANNEL")} + +if [[ "$ALGORAND_PACKAGE_NAME" =~ devtools ]]; then + REQUIRED_ALGORAND_PACKAGE=$(./scripts/compute_package_name.sh "$CHANNEL") +fi -# The following need to be exported for use in ./go-algorand/installer/rpm/$PKG_NAME/$PKG_NAME.spec. +# The following need to be exported for use in ./go-algorand/installer/rpm/$ALGORAND_PACKAGE_NAME/$ALGORAND_PACKAGE_NAME.spec. export DEFAULT_NETWORK export DEFAULT_RELEASE_NETWORK export REPO_DIR @@ -28,13 +33,14 @@ trap 'rm -rf $RPMTMP' 0 TEMPDIR=$(mktemp -d) trap 'rm -rf $TEMPDIR' 0 -< "./installer/rpm/$PKG_NAME/$PKG_NAME.spec" \ - sed -e "s,@PKG_NAME@,${PKG_NAME}," \ +< "./installer/rpm/$ALGORAND_PACKAGE_NAME/$ALGORAND_PACKAGE_NAME.spec" \ + sed -e "s,@ALGORAND_PACKAGE_NAME@,$REQUIRED_ALGORAND_PACKAGE," \ -e "s,@VER@,$FULLVERSION," \ -e "s,@ARCH@,$ARCH," \ - > "$TEMPDIR/$PKG_NAME.spec" + -e "s,@REQUIRED_ALGORAND_PKG@,$ALGORAND_PACKAGE_NAME," \ + > "$TEMPDIR/$ALGORAND_PACKAGE_NAME.spec" -rpmbuild --buildroot "$HOME/foo" --define "_rpmdir $RPMTMP" --define "RELEASE_GENESIS_PROCESS x$RELEASE_GENESIS_PROCESS" --define "LICENSE_FILE ./COPYING" -bb "$TEMPDIR/$PKG_NAME.spec" +rpmbuild --buildroot "$HOME/foo" --define "_rpmdir $RPMTMP" --define "RELEASE_GENESIS_PROCESS x$RELEASE_GENESIS_PROCESS" --define "LICENSE_FILE ./COPYING" -bb "$TEMPDIR/$ALGORAND_PACKAGE_NAME.spec" cp -p "$RPMTMP"/*/*.rpm "./tmp/node_pkgs/$OS_TYPE/$ARCH" From de964b3df84c119530bcd17e04eca635b52aba0b Mon Sep 17 00:00:00 2001 From: egieseke Date: Thu, 17 Sep 2020 09:34:40 -0400 Subject: [PATCH 047/136] Updates to support testing TEAL application size (#1506) Added support for testing TEAL app size in addition to operations. --- cmd/pingpong/runCmd.go | 12 ++ shared/pingpong/accounts.go | 113 +++++++++-------- shared/pingpong/config.go | 6 + shared/pingpong/pingpong.go | 119 +++++++++++++----- .../cli/goal/expect/goalExpectCommon.exp | 28 +++++ test/e2e-go/cli/goal/expect/pingpongTest.exp | 106 ++++++++++++++++ 6 files changed, 301 insertions(+), 83 deletions(-) create mode 100644 test/e2e-go/cli/goal/expect/pingpongTest.exp diff --git a/cmd/pingpong/runCmd.go b/cmd/pingpong/runCmd.go index 8755063c2f..5b33ccd591 100644 --- a/cmd/pingpong/runCmd.go +++ b/cmd/pingpong/runCmd.go @@ -59,6 +59,9 @@ var groupSize uint32 var numAsset uint32 var numApp uint32 var appProgOps uint32 +var appProgHashs uint32 +var appProgHashSize string +var duration uint32 var rekey bool func init() { @@ -91,8 +94,12 @@ func init() { runCmd.Flags().Uint32Var(&numAsset, "numasset", 0, "The number of assets each account holds") runCmd.Flags().Uint32Var(&numApp, "numapp", 0, "The number of apps each account opts in to") runCmd.Flags().Uint32Var(&appProgOps, "appprogops", 0, "The approximate number of TEAL operations to perform in each ApplicationCall transaction") + runCmd.Flags().Uint32Var(&appProgHashs, "appproghashes", 0, "The number of hashes to include in the Application") + runCmd.Flags().StringVar(&appProgHashSize, "appproghashsize", "sha256", "The size of hashes to include in the Application") runCmd.Flags().BoolVar(&randomLease, "randomlease", false, "set the lease to contain a random value") runCmd.Flags().BoolVar(&rekey, "rekey", false, "Create RekeyTo transactions. Requires groupsize=2 and any of random flags exc random dst") + runCmd.Flags().Uint32Var(&duration, "duration", 0, "The number of seconds to run the pingpong test, forever if 0") + } var runCmd = &cobra.Command{ @@ -200,6 +207,9 @@ var runCmd = &cobra.Command{ } cfg.RefreshTime = time.Duration(uint32(val)) * time.Second } + if duration > 0 { + cfg.MaxRuntime = time.Duration(uint32(duration)) * time.Second + } if randomNote { cfg.RandomNote = true } @@ -248,6 +258,8 @@ var runCmd = &cobra.Command{ } cfg.AppProgOps = appProgOps + cfg.AppProgHashs = appProgHashs + cfg.AppProgHashSize = appProgHashSize if numApp <= 1000 { cfg.NumApp = numApp diff --git a/shared/pingpong/accounts.go b/shared/pingpong/accounts.go index efedb51254..ce1083e62b 100644 --- a/shared/pingpong/accounts.go +++ b/shared/pingpong/accounts.go @@ -89,9 +89,10 @@ func ensureAccounts(ac libgoal.Client, initCfg PpConfig) (accounts map[string]ui fmt.Printf("Located Source Account: %s -> %v\n", cfg.SrcAccount, accounts[cfg.SrcAccount]) } - // Only reuse existing accounts for non asset testing. + // Only reuse existing accounts for non asset testing and non app testing. // For asset testing, new participant accounts will be created since accounts are limited to 1000 assets. - if cfg.NumAsset == 0 { + // For app testing, new participant accounts will be created since accounts are limited to 10 aps. + if cfg.NumAsset == 0 && cfg.NumApp == 0 { // If we have more accounts than requested, pick the top N (not including src) if len(accounts) > int(cfg.NumPartAccounts+1) { fmt.Printf("Finding the richest %d accounts to use for transacting\n", cfg.NumPartAccounts) @@ -158,7 +159,9 @@ func prepareAssets(accounts map[string]uint64, client libgoal.Client, cfg PpConf return } assetName := fmt.Sprintf("pong%d", i) - fmt.Printf("Creating asset %s\n", assetName) + if !cfg.Quiet { + fmt.Printf("Creating asset %s\n", assetName) + } tx, createErr := client.MakeUnsignedAssetCreateTx(totalSupply, false, addr, addr, addr, addr, "ping", assetName, "", meta, 0) if createErr != nil { fmt.Printf("Cannot make asset create txn with meta %v\n", meta) @@ -196,7 +199,9 @@ func prepareAssets(accounts map[string]uint64, client libgoal.Client, cfg PpConf // 2) For each participant account, opt-in to assets of all other participant accounts for addr := range accounts { - fmt.Printf("Opting in assets from account %v\n", addr) + if !cfg.Quiet { + fmt.Printf("Opting in assets from account %v\n", addr) + } addrAccount, addrErr := client.AccountInformation(addr) if addrErr != nil { fmt.Printf("Cannot lookup source account\n") @@ -205,15 +210,21 @@ func prepareAssets(accounts map[string]uint64, client libgoal.Client, cfg PpConf } assetParams := addrAccount.AssetParams - fmt.Printf("Optining in %d assets %+v\n", len(assetParams), assetParams) + if !cfg.Quiet { + fmt.Printf("Optining in %d assets %+v\n", len(assetParams), assetParams) + } // Opt-in Accounts for each asset for k := range assetParams { - fmt.Printf("optin asset %+v\n", k) + if !cfg.Quiet { + fmt.Printf("optin asset %+v\n", k) + } for addr2 := range accounts { if addr != addr2 { - fmt.Printf("Opting in assets to account %v \n", addr2) + if !cfg.Quiet { + fmt.Printf("Opting in assets to account %v \n", addr2) + } _, addrErr2 := client.AccountInformation(addr2) if addrErr2 != nil { fmt.Printf("Cannot lookup optin account\n") @@ -251,7 +262,9 @@ func prepareAssets(accounts map[string]uint64, client libgoal.Client, cfg PpConf // Step 3) Evenly distribute the assets across all participant accounts for addr := range accounts { - fmt.Printf("Distributing assets from account %v\n", addr) + if !cfg.Quiet { + fmt.Printf("Distributing assets from account %v\n", addr) + } addrAccount, addrErr := client.AccountInformation(addr) if addrErr != nil { fmt.Printf("Cannot lookup source account\n") @@ -260,18 +273,22 @@ func prepareAssets(accounts map[string]uint64, client libgoal.Client, cfg PpConf } assetParams := addrAccount.AssetParams - fmt.Printf("Distributing %d assets\n", len(assetParams)) + if !cfg.Quiet { + fmt.Printf("Distributing %d assets\n", len(assetParams)) + } // Distribute assets to each account for k := range assetParams { - - fmt.Printf("Distributing asset %v \n", k) + if !cfg.Quiet { + fmt.Printf("Distributing asset %v \n", k) + } assetAmt := assetParams[k].Total / uint64(len(accounts)) for addr2 := range accounts { if addr != addr2 { - - fmt.Printf("Distributing assets from %v to %v \n", addr, addr2) + if !cfg.Quiet { + fmt.Printf("Distributing assets from %v to %v \n", addr, addr2) + } tx, sendErr := constructTxn(addr, addr2, cfg.MaxFee, assetAmt, k, client, cfg) if sendErr != nil { @@ -318,31 +335,15 @@ func signAndBroadcastTransaction(accounts map[string]uint64, sender string, tx t return } -func genBigNoOp(numOps uint32) []byte { - var progParts []string - progParts = append(progParts, `#pragma version 2`) - for i := uint32(0); i < numOps/2; i++ { - progParts = append(progParts, `int 1`) - progParts = append(progParts, `pop`) - } - progParts = append(progParts, `int 1`) - progParts = append(progParts, `return`) - progAsm := strings.Join(progParts, "\n") - progBytes, err := logic.AssembleString(progAsm) - if err != nil { - panic(err) - } - return progBytes -} - -func genBigHashes(numHashes int, numPad int, hash string) []byte { +func genBigNoOpAndBigHashes(numOps uint32, numHashes uint32, hashSize string) []byte { var progParts []string progParts = append(progParts, `#pragma version 2`) progParts = append(progParts, `byte base64 AA==`) - for i := 0; i < numHashes; i++ { - progParts = append(progParts, hash) + + for i := uint32(0); i < numHashes; i++ { + progParts = append(progParts, hashSize) } - for i := 0; i < numPad/2; i++ { + for i := uint32(0); i < numOps/2; i++ { progParts = append(progParts, `int 1`) progParts = append(progParts, `pop`) } @@ -425,12 +426,21 @@ func genMaxClone(numKeys int) []byte { } func prepareApps(accounts map[string]uint64, client libgoal.Client, cfg PpConfig) (appParams map[uint64]v1.AppParams, err error) { - // get existing apps - account, accountErr := client.AccountInformation(cfg.SrcAccount) - if accountErr != nil { - fmt.Printf("Cannot lookup source account") - err = accountErr - return + + var appAccount v1.Account + for tempAccount := range accounts { + if tempAccount != cfg.SrcAccount { + appAccount, err = client.AccountInformation(tempAccount) + if err != nil { + fmt.Printf("Warning, cannot lookup tempAccount account %s", tempAccount) + return + } + break + } + } + + if !cfg.Quiet { + fmt.Printf("Selected temp account: %s\n", appAccount.Address) } // Get wallet handle token @@ -440,27 +450,30 @@ func prepareApps(accounts map[string]uint64, client libgoal.Client, cfg PpConfig return } - toCreate := int(cfg.NumApp) - len(account.AppParams) + toCreate := int(cfg.NumApp) // create apps in srcAccount for i := 0; i < toCreate; i++ { var tx transactions.Transaction // generate app program with roughly some number of operations - prog := genBigNoOp(cfg.AppProgOps) + prog := genBigNoOpAndBigHashes(cfg.AppProgOps, cfg.AppProgHashs, cfg.AppProgHashSize) + if !cfg.Quiet { + fmt.Printf("generated program: \n%s\n", prog) + } globSchema := basics.StateSchema{NumByteSlice: 64} locSchema := basics.StateSchema{} tx, err = client.MakeUnsignedAppCreateTx(transactions.NoOpOC, prog, prog, globSchema, locSchema, nil, nil, nil, nil) if err != nil { fmt.Printf("Cannot create app txn\n") - return + panic(err) } - tx, err = client.FillUnsignedTxTemplate(cfg.SrcAccount, 0, 0, cfg.MaxFee, tx) + tx, err = client.FillUnsignedTxTemplate(appAccount.Address, 0, 0, cfg.MaxFee, tx) if err != nil { fmt.Printf("Cannot fill app creation txn\n") - return + panic(err) } // Ensure different txids @@ -486,15 +499,15 @@ func prepareApps(accounts map[string]uint64, client libgoal.Client, cfg PpConfig fmt.Printf("Create a new app: txid=%s\n", txid) } - accounts[cfg.SrcAccount] -= tx.Fee.Raw + accounts[appAccount.Address] -= tx.Fee.Raw } + var account v1.Account // get these apps for { - account, accountErr = client.AccountInformation(cfg.SrcAccount) - if accountErr != nil { - fmt.Printf("Cannot lookup source account") - err = accountErr + account, err = client.AccountInformation(appAccount.Address) + if err != nil { + fmt.Printf("Warning, cannot lookup source account") return } if len(account.AppParams) >= int(cfg.NumApp) { diff --git a/shared/pingpong/config.go b/shared/pingpong/config.go index 7941f7d278..40b61a0776 100644 --- a/shared/pingpong/config.go +++ b/shared/pingpong/config.go @@ -54,7 +54,10 @@ type PpConfig struct { MinAccountAsset uint64 NumApp uint32 AppProgOps uint32 + AppProgHashs uint32 + AppProgHashSize string Rekey bool + MaxRuntime time.Duration } // DefaultConfig object for Ping Pong @@ -78,7 +81,10 @@ var DefaultConfig = PpConfig{ MinAccountAsset: 10000000, NumApp: 0, AppProgOps: 0, + AppProgHashs: 0, + AppProgHashSize: "sha256", Rekey: false, + MaxRuntime: 0, } // LoadConfigFromFile reads and loads Ping Pong configuration diff --git a/shared/pingpong/pingpong.go b/shared/pingpong/pingpong.go index b266e6a5f0..266d87a4e6 100644 --- a/shared/pingpong/pingpong.go +++ b/shared/pingpong/pingpong.go @@ -25,7 +25,7 @@ import ( "time" "github.com/algorand/go-algorand/crypto" - v1 "github.com/algorand/go-algorand/daemon/algod/api/spec/v1" + "github.com/algorand/go-algorand/daemon/algod/api/spec/v1" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/libgoal" @@ -40,61 +40,77 @@ func PrepareAccounts(ac libgoal.Client, initCfg PpConfig) (accounts map[string]u return } + wallet, walletErr := ac.GetUnencryptedWalletHandle() + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "unable to access wallet %v\n", walletErr) + err = walletErr + return + } if cfg.NumAsset > 0 { // zero out max amount for asset transactions cfg.MaxAmt = 0 - wallet, walletErr := ac.GetUnencryptedWalletHandle() + var assetAccounts map[string]uint64 + assetAccounts, err = prepareNewAccounts(ac, cfg, wallet, accounts) if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "unable to access wallet %v\n", walletErr) - err = walletErr + _, _ = fmt.Fprintf(os.Stderr, "prepare new accounts failed: %v\n", err) return } - fmt.Printf("Generating %v new accounts for asset transfer test\n", cfg.NumPartAccounts) - // remove existing accounts except for src account - for k := range accounts { - if k != cfg.SrcAccount { - delete(accounts, k) - } - } - // create new accounts for asset testing - assetAccounts := make(map[string]uint64) - assetAccounts, err = generateAccounts(ac, assetAccounts, cfg.NumPartAccounts-1, wallet) - for addr := range assetAccounts { - fmt.Printf("generated account %v\n", addr) + assetParams, err = prepareAssets(assetAccounts, ac, cfg) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "prepare assets failed %v\n", err) + return } - for k := range assetAccounts { - accounts[k] = assetAccounts[k] + if !cfg.Quiet { + for addr := range accounts { + fmt.Printf("final prepareAccounts, account addr: %s, balance: %d\n", addr, accounts[addr]) + } } - err = fundAccounts(accounts, ac, cfg) + } else if cfg.NumApp > 0 { + + var appAccounts map[string]uint64 + appAccounts, err = prepareNewAccounts(ac, cfg, wallet, accounts) if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "fund accounts failed %v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "prepare new accounts failed: %v\n", err) return } - - assetParams, err = prepareAssets(assetAccounts, ac, cfg) + appParams, err = prepareApps(appAccounts, ac, cfg) if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "prepare assets failed %v\n", err) return } - - for k := range assetAccounts { - accounts[k] = assetAccounts[k] + if !cfg.Quiet { + for addr := range accounts { + fmt.Printf("final prepareAccounts, account addr: %s, balance: %d\n", addr, accounts[addr]) + } } - } else if cfg.NumApp > 0 { - appParams, err = prepareApps(accounts, ac, cfg) + } else { + err = fundAccounts(accounts, ac, cfg) if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "fund accounts failed %v\n", err) return } } - for addr := range accounts { - fmt.Printf("**** participant account %v\n", addr) + return +} + +func prepareNewAccounts(client libgoal.Client, cfg PpConfig, wallet []byte, accounts map[string]uint64) (newAccounts map[string]uint64, err error) { + // remove existing accounts except for src account + for k := range accounts { + if k != cfg.SrcAccount { + delete(accounts, k) + } } + // create new accounts for testing + newAccounts = make(map[string]uint64) + newAccounts, err = generateAccounts(client, newAccounts, cfg.NumPartAccounts-1, wallet) - err = fundAccounts(accounts, ac, cfg) + for k := range newAccounts { + accounts[k] = newAccounts[k] + } + err = fundAccounts(accounts, client, cfg) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "fund accounts failed %v\n", err) return @@ -107,6 +123,11 @@ func PrepareAccounts(ac libgoal.Client, initCfg PpConfig) (accounts map[string]u func computeAccountMinBalance(cfg PpConfig) (requiredBalance uint64) { const minActiveAccountBalance uint64 = 100000 // min balance for any active account + if cfg.NumApp > 0 { + requiredBalance = (cfg.MinAccountFunds + (cfg.MaxAmt+cfg.MaxFee)*10) * 2 + fmt.Printf("required min balance for app accounts: %d\n", requiredBalance) + return + } var fee uint64 = 1000 if cfg.MinFee > fee { fee = cfg.MinFee @@ -148,18 +169,26 @@ func fundAccounts(accounts map[string]uint64, client libgoal.Client, cfg PpConfi fmt.Printf("adjusting account balance to %d\n", minFund) for addr, balance := range accounts { - fmt.Printf("adjusting balance of account %v\n", addr) + if !cfg.Quiet { + fmt.Printf("adjusting balance of account %v\n", addr) + } if balance < minFund { toSend := minFund - balance if srcFunds <= toSend { return fmt.Errorf("source account %s has insufficient funds %d - needs %d", cfg.SrcAccount, srcFunds, toSend) } srcFunds -= toSend - _, err := client.SendPaymentFromUnencryptedWallet(cfg.SrcAccount, addr, fee, toSend, nil) + if !cfg.Quiet { + fmt.Printf("adjusting balance of account %v by %d\n ", addr, toSend) + } + _, err := sendPaymentFromUnencryptedWallet(client, cfg.SrcAccount, addr, fee, toSend, nil) if err != nil { return err } accounts[addr] = minFund + if !cfg.Quiet { + fmt.Printf("account balance for key %s is %d\n", addr, accounts[addr]) + } totalSent++ throttleTransactionRate(startTime, cfg, totalSent) @@ -168,6 +197,18 @@ func fundAccounts(accounts map[string]uint64, client libgoal.Client, cfg PpConfi return nil } +func sendPaymentFromUnencryptedWallet(client libgoal.Client, from, to string, fee, amount uint64, note []byte) (transactions.Transaction, error) { + wh, err := client.GetUnencryptedWalletHandle() + if err != nil { + return transactions.Transaction{}, err + } + // generate a random lease to avoid duplicate transaction failures + var lease [32]byte + crypto.RandBytes(lease[:]) + + return client.SendPaymentFromWalletWithLease(wh, nil, from, to, fee, amount, note, "", lease, 0, 0) +} + func refreshAccounts(accounts map[string]uint64, client libgoal.Client, cfg PpConfig) error { for addr := range accounts { amount, err := client.GetBalance(addr) @@ -221,6 +262,10 @@ func RunPingPong(ctx context.Context, ac libgoal.Client, accounts map[string]uin } else { runTime = 10000 * time.Hour // Effectively 'forever' } + var endTime time.Time + if cfg.MaxRuntime > 0 { + endTime = time.Now().Add(cfg.MaxRuntime) + } restTime := cfg.RestTime refreshTime := time.Now().Add(cfg.RefreshTime) @@ -234,6 +279,11 @@ func RunPingPong(ctx context.Context, ac libgoal.Client, accounts map[string]uin var totalSent, totalSucceeded uint64 for !time.Now().After(stopTime) { + if cfg.MaxRuntime > 0 && time.Now().After(endTime) { + fmt.Printf("Terminating after max run time of %.f seconds\n", cfg.MaxRuntime.Seconds()) + return + } + minimumAmount := cfg.MinAccountFunds + (cfg.MaxAmt+cfg.MaxFee)*2 fromList := listSufficientAccounts(accounts, minimumAmount, cfg.SrcAccount) // in group tests txns are sent back and forth, so both parties need funds @@ -358,6 +408,9 @@ func sendFromTo( sentCount++ _, sendErr = client.BroadcastTransaction(stxn) + if sendErr != nil { + fmt.Printf("Warning, cannot broadcast txn, %s\n", sendErr) + } } else { // Generate txn group diff --git a/test/e2e-go/cli/goal/expect/goalExpectCommon.exp b/test/e2e-go/cli/goal/expect/goalExpectCommon.exp index 6d8e79c6c7..619cfe654a 100755 --- a/test/e2e-go/cli/goal/expect/goalExpectCommon.exp +++ b/test/e2e-go/cli/goal/expect/goalExpectCommon.exp @@ -200,6 +200,7 @@ proc ::AlgorandGoal::CreateNetwork { NETWORK_NAME NETWORK_TEMPLATE TEST_ALGO_DIR # Start the network proc ::AlgorandGoal::StartNetwork { NETWORK_NAME NETWORK_TEMPLATE TEST_ALGO_DIR TEST_ROOT_DIR } { + set timeout 120 set ::GLOBAL_TEST_ALGO_DIR $TEST_ALGO_DIR set ::GLOBAL_TEST_ROOT_DIR $TEST_ROOT_DIR set ::GLOBAL_NETWORK_NAME $NETWORK_NAME @@ -1144,3 +1145,30 @@ proc ::AlgorandGoal::InspectTransactionFile { TRX_FILE } { eof } } + +# Run pingpong test +proc ::AlgorandGoal::RunPingpong {DURATION PINGPONG_OPTIONS TEST_PRIMARY_NODE_DIR} { + set timeout [expr $DURATION + 60] + if { [ catch { + set pingpong_base "pingpong run --duration $DURATION -d $TEST_PRIMARY_NODE_DIR --quiet " + set pingpong_command [concat $pingpong_base $PINGPONG_OPTIONS] + puts "starting pingpong test with command: $pingpong_command" + eval spawn $pingpong_command + expect { + timeout { puts "pingpong test interrupted by timeout, terminating after $timeout seconds" } + -re {Sent (\d+) transactions \((\d+) attempted\).} { + set actual $expect_out(1,string) ; + set attempted $expect_out(2,string) ; + puts "actual: $actual, attempted: $attempted"; + if { $actual != $attempted } then { ::AlgorandGoal::Abort "Pingpong attempted to send $attempted transactions, but actual was $actual"; break;} + exp_continue + } + "Terminating after max run time of" {puts "end of ping pong test"} + eof {::AlgorandGoal::Abort "pingpong terminated unexpectedly: $expect_out(buffer)"} + "Error" {::AlgorandGoal::Abort "error running pingpong: $expect_out(buffer)"} + } + } EXCEPTION ] } { + ::AlgorandGoal::Abort "ERROR in RunPingpong: $EXCEPTION" + } +} + diff --git a/test/e2e-go/cli/goal/expect/pingpongTest.exp b/test/e2e-go/cli/goal/expect/pingpongTest.exp new file mode 100644 index 0000000000..72b212f418 --- /dev/null +++ b/test/e2e-go/cli/goal/expect/pingpongTest.exp @@ -0,0 +1,106 @@ +#!/usr/bin/expect -f +#exp_internal 1 +set err 0 +log_user 1 + +source goalExpectCommon.exp + +set TEST_ALGO_DIR [lindex $argv 0] +set TEST_DATA_DIR [lindex $argv 1] + +proc pingpongTest { TEST_ALGO_DIR TEST_DATA_DIR} { + + set timeout 60 + set TIME_STAMP [clock seconds] + + set TEST_ROOT_DIR $TEST_ALGO_DIR/root_$TIME_STAMP + set TEST_PRIMARY_NODE_DIR $TEST_ROOT_DIR/Primary/ + set NETWORK_NAME test_net_expect_$TIME_STAMP + set NETWORK_TEMPLATE "$TEST_DATA_DIR/nettemplates/TwoNodes50EachFuture.json" + + exec cp $TEST_DATA_DIR/../../gen/devnet/genesis.json $TEST_ALGO_DIR + + # Create network + ::AlgorandGoal::CreateNetwork $NETWORK_NAME $NETWORK_TEMPLATE $TEST_ALGO_DIR $TEST_ROOT_DIR + + # Start network + ::AlgorandGoal::StartNetwork $NETWORK_NAME $NETWORK_TEMPLATE $TEST_ALGO_DIR $TEST_ROOT_DIR + + set PRIMARY_NODE_ADDRESS [ ::AlgorandGoal::GetAlgodNetworkAddress $TEST_PRIMARY_NODE_DIR ] + puts "Primary Node Address: $PRIMARY_NODE_ADDRESS" + + set PRIMARY_WALLET_NAME unencrypted-default-wallet + + # Determine primary account + set PRIMARY_ACCOUNT_ADDRESS [::AlgorandGoal::GetHighestFundedAccountForWallet $PRIMARY_WALLET_NAME $TEST_PRIMARY_NODE_DIR] + + # Check the balance of the primary account + set PRIMARY_ACCOUNT_BALANCE [::AlgorandGoal::GetAccountBalance $PRIMARY_WALLET_NAME $PRIMARY_ACCOUNT_ADDRESS $TEST_PRIMARY_NODE_DIR] + puts "Primary Account Balance: $PRIMARY_ACCOUNT_BALANCE" + + ::AlgorandGoal::WaitForRound 1 $TEST_PRIMARY_NODE_DIR + + set TEAL_PROGS_DIR "$TEST_DATA_DIR/../scripts/e2e_subs/tealprogs" + + # Network Setup complete + #---------------------- + + # Run pingpong tests + #---------------------- + + + set pingpong_duration 5 + + set pingpongArray(1_smallops_smallhash) "--appprogops 2 --appproghashes 2 --appproghashsize sha512_256 --numapp 10 --tps 200 --rest 0 --refresh 10 --numaccounts 5 --minaccount 100000000" + set pingpongArray(2_smallops_mediumhash) "--appprogops 2 --appproghashes 5 --appproghashsize sha512_256 --numapp 10 --tps 200 --rest 0 --refresh 10 --numaccounts 50 --minaccount 100000000" + set pingpongArray(3_smallops_bighash) "--appprogops 2 --appproghashes 10 --appproghashsize sha512_256 --numapp 10 --tps 200 --rest 0 --refresh 10 --numaccounts 50 --minaccount 100000000" + set pingpongArray(4_mediumops_smallhash) "--appprogops 200 --appproghashes 2 --appproghashsize sha512_256 --numapp 10 --tps 200 --rest 0 --refresh 10 --numaccounts 50 --minaccount 100000000" + set pingpongArray(5_mediumops_mediumhash) "--appprogops 200 --appproghashes 5 --appproghashsize sha512_256 --numapp 10 --tps 200 --rest 0 --refresh 10 --numaccounts 50 --minaccount 100000000" + set pingpongArray(6_mediumops_bighash) "--appprogops 200 --appproghashes 10 --appproghashsize sha512_256 --numapp 10 --tps 200 --rest 0 --refresh 10 --numaccounts 50 --minaccount 100000000" + set pingpongArray(7_bigops_smallhash) "--appprogops 500 --appproghashes 2 --appproghashsize sha512_256 --numapp 10 --tps 200 --rest 0 --refresh 10 --numaccounts 50 --minaccount 100000000" + set pingpongArray(8_bigops_mediumhash) "--appprogops 300 --appproghashes 5 --appproghashsize sha512_256 --numapp 10 --tps 200 --rest 0 --refresh 10 --numaccounts 50 --minaccount 100000000" + set pingpongArray(9_bigops_bighash) "--appprogops 220 --appproghashes 10 --appproghashsize sha512_256 --numapp 10 --tps 200 --rest 0 --refresh 10 --numaccounts 50 --minaccount 100000000" + + set pingpongArray(10_payment_transaction) "--tps 200 --rest 0 --refresh 10 --numaccounts 50" + set pingpongArray(11_teal_light_transaction) "--teal=light --tps 200 --rest 0 --refresh 10 --numaccounts 50" + set pingpongArray(10_teal_normal_transaction) "--teal=normal --tps 200 --rest 0 --refresh 10 --numaccounts 50" + set pingpongArray(12_teal_heavy_transaction) "--teal=heavy --tps 200 --rest 0 --refresh 10 --numaccounts 50" + set pingpongArray(13_atomic_transfer_small_transaction) "--groupsize=5 --tps 200 --rest 0 --refresh 10 --numaccounts 50" + set pingpongArray(14_atomic_transfer_large_transaction) "--groupsize=12 --tps 200 --rest 0 --refresh 10 --numaccounts 50" + set pingpongArray(15_asset_transfer_small_transaction) "--tps 200 --numasset=5 --mf 0 --rest 0 --numaccounts 10 --refresh 10 --mf=1000" + set pingpongArray(16_asset_transfer_large_transaction) "--tps 200 --numasset=10 --mf 0 --rest 0 --numaccounts 10 --refresh 10 --mf=1000" + set pingpongArray(17_stateful_teal_small_transaction) "--numapp 10 --appprogops 10 --tps 200 --rest 0 --refresh 10 --numaccounts 50 --minaccount 100000000" + set pingpongArray(18_stateful_teal_medium_transaction) "--numapp 10 --appprogops 200 --tps 200 --rest 0 --refresh 10 --numaccounts 50 --minaccount 100000000" + set pingpongArray(19_stateful_teal_large_transaction) "--numapp 10 --appprogops 600 --tps 200 --rest 0 --refresh 10 --numaccounts 50 --minaccount 100000000" + set pingpongArray(20_rekey_payment_transaction) "--rekey=true --groupsize=2 --randomnote=true --tps 200 --rest 0 --refresh 10 --numaccounts 50" + + + foreach index [array names pingpongArray] { + puts "pingpongArray($index): $pingpongArray($index)" + ::AlgorandGoal::RunPingpong $pingpong_duration $pingpongArray($index) $TEST_PRIMARY_NODE_DIR + } + + # Shutdown the network + #---------------------- + ::AlgorandGoal::StopNetwork $NETWORK_NAME $TEST_ALGO_DIR $TEST_ROOT_DIR + + puts "Pinpong Test Successful" + +} + + +if { [catch { + source goalExpectCommon.exp + + puts "starting pinpongTest" + + puts "TEST_ALGO_DIR: $TEST_ALGO_DIR" + puts "TEST_DATA_DIR: $TEST_DATA_DIR" + + pingpongTest $TEST_ALGO_DIR $TEST_DATA_DIR + + exit 0 + +} EXCEPTION ] } { + ::AlgorandGoal::Abort "ERROR in pinpongTest: $EXCEPTION" +} From 58d398bf6ff46fdb81a1f49ef92a1e7935447bd3 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Thu, 17 Sep 2020 12:47:16 -0400 Subject: [PATCH 048/136] Revert "Don't change parameter name." This reverts commit 9c41ddf7729f6abc016f07f8da1081e4d412aac3. --- cmd/goal/asset.go | 10 +++++----- test/scripts/e2e_subs/e2e-app-real-assets-round.sh | 2 +- test/scripts/e2e_subs/limit-swap-test.sh | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/goal/asset.go b/cmd/goal/asset.go index 529bd2b617..d6ce6d9414 100644 --- a/cmd/goal/asset.go +++ b/cmd/goal/asset.go @@ -110,7 +110,7 @@ func init() { addTxnFlags(freezeAssetCmd) infoAssetCmd.Flags().Uint64Var(&assetID, "assetid", 0, "ID of the asset to look up") - infoAssetCmd.Flags().StringVar(&assetUnitName, "asset", "", "Unit name of the asset to look up") + infoAssetCmd.Flags().StringVar(&assetUnitName, "unit", "", "Unit name of the asset to look up") infoAssetCmd.Flags().StringVar(&assetCreator, "creator", "", "Account address of the asset creator") } @@ -125,16 +125,16 @@ var assetCmd = &cobra.Command{ } func lookupAssetID(cmd *cobra.Command, creator string, client libgoal.Client) { - if cmd.Flags().Changed("assetid") && cmd.Flags().Changed("asset") { - reportErrorf("Only one of [--assetid] or [--asset and --creator] can be specified") + if cmd.Flags().Changed("assetid") && cmd.Flags().Changed("unit") { + reportErrorf("Only one of [--assetid] or [--unit and --creator] can be specified") } if cmd.Flags().Changed("assetid") { return } - if !cmd.Flags().Changed("asset") { - reportErrorf("Either [--assetid] or [--asset and --creator] must be specified") + if !cmd.Flags().Changed("unit") { + reportErrorf("Either [--assetid] or [--unit and --creator] must be specified") } if !cmd.Flags().Changed("creator") { diff --git a/test/scripts/e2e_subs/e2e-app-real-assets-round.sh b/test/scripts/e2e_subs/e2e-app-real-assets-round.sh index 6793d6bb53..6401042c5e 100755 --- a/test/scripts/e2e_subs/e2e-app-real-assets-round.sh +++ b/test/scripts/e2e_subs/e2e-app-real-assets-round.sh @@ -18,7 +18,7 @@ ACCOUNT=$(${gcmd} account list|awk '{ print $3 }') # Create an ASA in account ${gcmd} asset create --creator ${ACCOUNT} --name bogocoin --unitname bogo --total 1337 -ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --asset bogo|grep 'Asset ID'|awk '{ print $3 }') +ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --unit bogo|grep 'Asset ID'|awk '{ print $3 }') # Create app that reads asset balance and checks asset details and checks round ROUND=$(goal node status | grep 'Last committed' | awk '{ print $4 }') diff --git a/test/scripts/e2e_subs/limit-swap-test.sh b/test/scripts/e2e_subs/limit-swap-test.sh index 52907b47ff..463128fb8e 100755 --- a/test/scripts/e2e_subs/limit-swap-test.sh +++ b/test/scripts/e2e_subs/limit-swap-test.sh @@ -16,7 +16,7 @@ ZERO_ADDRESS=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ ${gcmd} asset create --creator ${ACCOUNT} --name bogocoin --unitname bogo --total 1000000000000 -ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --asset bogo|grep 'Asset ID'|awk '{ print $3 }') +ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --unit bogo|grep 'Asset ID'|awk '{ print $3 }') # Asset ID: 5 From 85eb20e8a67298b639b91bdf3e6c9c5c4b6f883d Mon Sep 17 00:00:00 2001 From: Will Winder Date: Thu, 17 Sep 2020 14:21:48 -0400 Subject: [PATCH 049/136] Add 'asset' back in and mark it deprecated. --- cmd/goal/asset.go | 17 ++++++++++++++--- .../e2e-go/cli/goal/expect/goalExpectCommon.exp | 2 +- tools/teal/examples/limitorder.sh | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cmd/goal/asset.go b/cmd/goal/asset.go index d6ce6d9414..8716c22e1b 100644 --- a/cmd/goal/asset.go +++ b/cmd/goal/asset.go @@ -110,6 +110,7 @@ func init() { addTxnFlags(freezeAssetCmd) infoAssetCmd.Flags().Uint64Var(&assetID, "assetid", 0, "ID of the asset to look up") + infoAssetCmd.Flags().StringVar(&assetUnitName, "asset", "", "DEPRECATED! Unit name of the asset to look up") infoAssetCmd.Flags().StringVar(&assetUnitName, "unit", "", "Unit name of the asset to look up") infoAssetCmd.Flags().StringVar(&assetCreator, "creator", "", "Account address of the asset creator") } @@ -125,15 +126,25 @@ var assetCmd = &cobra.Command{ } func lookupAssetID(cmd *cobra.Command, creator string, client libgoal.Client) { - if cmd.Flags().Changed("assetid") && cmd.Flags().Changed("unit") { - reportErrorf("Only one of [--assetid] or [--unit and --creator] can be specified") + if cmd.Flags().Changed("asset") { + reportWarnln("The [--asset] flag is deprecated and will be removed in a future release, use [--unit] instead.") + } + + if cmd.Flags().Changed("asset") && cmd.Flags().Changed("unit"){ + reportErrorf("The [--asset] flag has been replaced by [--unit], do not provide both flags.") + } + + assetOrUnit := cmd.Flags().Changed("asset") || cmd.Flags().Changed("unit") + + if cmd.Flags().Changed("assetid") && assetOrUnit { + reportErrorf("Only one of [--assetid] or [--unit and --creator] should be specified") } if cmd.Flags().Changed("assetid") { return } - if !cmd.Flags().Changed("unit") { + if !assetOrUnit { reportErrorf("Either [--assetid] or [--unit and --creator] must be specified") } diff --git a/test/e2e-go/cli/goal/expect/goalExpectCommon.exp b/test/e2e-go/cli/goal/expect/goalExpectCommon.exp index 91272ffba9..d86e3696bb 100755 --- a/test/e2e-go/cli/goal/expect/goalExpectCommon.exp +++ b/test/e2e-go/cli/goal/expect/goalExpectCommon.exp @@ -471,7 +471,7 @@ proc ::AlgorandGoal::AssetLookup { CREATOR UNIT_NAME TEST_PRIMARY_NODE_DIR } { set timeout 10 if { [ catch { set ASSET_ID "NOT SET" - spawn goal asset info -d $TEST_PRIMARY_NODE_DIR --creator $CREATOR --asset $UNIT_NAME + spawn goal asset info -d $TEST_PRIMARY_NODE_DIR --creator $CREATOR --unit $UNIT_NAME expect { timeout { ::AlgorandGoal::Abort "Timed out asset lookup" } -re {Asset ID:\s+([0-9]+)} {set ASSET_ID $expect_out(1,string); close } diff --git a/tools/teal/examples/limitorder.sh b/tools/teal/examples/limitorder.sh index 09e0a5360a..4d87aca315 100755 --- a/tools/teal/examples/limitorder.sh +++ b/tools/teal/examples/limitorder.sh @@ -5,7 +5,7 @@ goal asset create -d . --creator G5PM2K5RIEHHO7ZKR2ZTQDYY6DVBYOMGOFZMMNGJCW4BYNM # > Issued transaction from account G5PM2K5RIEHHO7ZKR2ZTQDYY6DVBYOMGOFZMMNGJCW4BYNMT7HC4HTZIDU, txid JH7M5L43YLQ5DTRIVVBUUB2E4BFE7TPVAPPEGCUVNYSFRLT55Z3Q (fee 1000) # > Transaction JH7M5L43YLQ5DTRIVVBUUB2E4BFE7TPVAPPEGCUVNYSFRLT55Z3Q still pending as of round 148369 # > Transaction JH7M5L43YLQ5DTRIVVBUUB2E4BFE7TPVAPPEGCUVNYSFRLT55Z3Q committed in round 148371 -goal asset info --creator G5PM2K5RIEHHO7ZKR2ZTQDYY6DVBYOMGOFZMMNGJCW4BYNMT7HC4HTZIDU -d . --asset e.g.Coin +goal asset info --creator G5PM2K5RIEHHO7ZKR2ZTQDYY6DVBYOMGOFZMMNGJCW4BYNMT7HC4HTZIDU -d . --unit e.g.Coin # > Asset ID: 39 # > Creator: G5PM2K5RIEHHO7ZKR2ZTQDYY6DVBYOMGOFZMMNGJCW4BYNMT7HC4HTZIDU # > Asset name: From 60d4ec68a7d5bd8f84301d141b649b5bcb155cdf Mon Sep 17 00:00:00 2001 From: btoll Date: Thu, 17 Sep 2020 18:38:32 -0400 Subject: [PATCH 050/136] Fix pathing issue to installer directory (#1517) Regardless of the channel, the installer scripts are either located at algorand or algorand-devtools and do not include the channel name (hence, the package name shouldn't be used to construct the path). --- .../algorand-devtools/algorand-devtools.spec | 2 +- scripts/release/build/deb/build_deb.sh | 17 ++++++++++++----- scripts/release/build/deb/package.sh | 8 +++++++- scripts/release/build/rpm/package.sh | 8 +++++++- scripts/release/mule/package/deb/package.sh | 15 ++++++++++----- scripts/release/mule/package/rpm/package.sh | 7 ++++++- 6 files changed, 43 insertions(+), 14 deletions(-) diff --git a/installer/rpm/algorand-devtools/algorand-devtools.spec b/installer/rpm/algorand-devtools/algorand-devtools.spec index 5bd441b03a..ad5b5a1cd5 100644 --- a/installer/rpm/algorand-devtools/algorand-devtools.spec +++ b/installer/rpm/algorand-devtools/algorand-devtools.spec @@ -4,7 +4,7 @@ Release: 1 Summary: Algorand tools software URL: https://www.algorand.com License: AGPL-3+ -Requires: algorand >= @VER@ +Requires: @REQUIRED_ALGORAND_PKG@ >= @VER@ %define SRCDIR go-algorand-rpmbuild %define _buildshell /bin/bash diff --git a/scripts/release/build/deb/build_deb.sh b/scripts/release/build/deb/build_deb.sh index d5c0446f5b..ec1d34ab9b 100755 --- a/scripts/release/build/deb/build_deb.sh +++ b/scripts/release/build/deb/build_deb.sh @@ -27,15 +27,15 @@ trap "rm -rf $PKG_ROOT" 0 mkdir -p "$PKG_ROOT/usr/bin" # NOTE: keep in sync with `./installer/rpm/algorand.spec`. -if [ "$PKG_NAME" = "algorand-devtools" ]; then +if [[ "$PKG_NAME" =~ devtools ]]; then BIN_FILES=("carpenter" "catchupsrv" "msgpacktool" "tealcut" "tealdbg") UNATTENDED_UPGRADES_FILE="53algorand-devtools-upgrades" - OUTPUT_DEB="$OUTDIR/algorand_devtools_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" + OUTPUT_DEB="$OUTDIR/algorand-devtools_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" REQUIRED_ALGORAND_PKG=$("./scripts/compute_package_name.sh" "$CHANNEL") else BIN_FILES=("algocfg" "algod" "algoh" "algokey" "ddconfig.sh" "diagcfg" "goal" "kmd" "node_exporter") - UNATTENDED_UPGRADES_FILE="51algorand-upgrades" OUTPUT_DEB="$OUTDIR/algorand_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" + UNATTENDED_UPGRADES_FILE="51algorand-upgrades" fi for binary in "${BIN_FILES[@]}"; do @@ -43,7 +43,7 @@ for binary in "${BIN_FILES[@]}"; do chmod 755 "$PKG_ROOT/usr/bin/$binary" done -if [ "$PKG_NAME" != "algorand-devtools" ]; then +if [[ ! "$PKG_NAME" =~ devtools ]]; then mkdir -p "${PKG_ROOT}/usr/lib/algorand" lib_files=("updater" "find-nodes.sh") for lib in "${lib_files[@]}"; do @@ -92,8 +92,15 @@ Dpkg::Options { EOF mkdir -p "$PKG_ROOT/DEBIAN" + +if [[ "$PKG_NAME" =~ devtools ]]; then + INSTALLER_DIR="algorand-devtools" +else + INSTALLER_DIR=algorand +fi + # Can contain `control`, `preinst`, `postinst`, `prerm`, `postrm`, `conffiles`. -CTL_FILES_DIR="./installer/debian/$PKG_NAME" +CTL_FILES_DIR="./installer/debian/$INSTALLER_DIR" for ctl_file in $(ls "$CTL_FILES_DIR"); do # Copy first, to preserve permissions, then overwrite to fill in template. cp -a "$CTL_FILES_DIR/$ctl_file" "$PKG_ROOT/DEBIAN/$ctl_file" diff --git a/scripts/release/build/deb/package.sh b/scripts/release/build/deb/package.sh index 51ecffd6c0..b4688c9ebd 100755 --- a/scripts/release/build/deb/package.sh +++ b/scripts/release/build/deb/package.sh @@ -27,7 +27,13 @@ for pkg_name in "${PKG_NAMES[@]}"; do exit 1 fi - cp -p "${DEBTMP}"/*.deb "${PKG_ROOT}/${pkg_name}_${CHANNEL}_${OS}-${ARCH}_${FULLVERSION}.deb" + if [[ "$pkg_name" =~ devtools ]]; then + BASE_NAME="algorand-devtools" + else + BASE_NAME=algorand + fi + + cp -p "${DEBTMP}"/*.deb "${PKG_ROOT}/${BASE_NAME}_${CHANNEL}_${OS}-${ARCH}_${FULLVERSION}.deb" done popd diff --git a/scripts/release/build/rpm/package.sh b/scripts/release/build/rpm/package.sh index 4018a43b73..45c756fa5d 100755 --- a/scripts/release/build/rpm/package.sh +++ b/scripts/release/build/rpm/package.sh @@ -30,9 +30,15 @@ for pkg_name in "${PKG_NAMES[@]}"; do mkdir "$TEMPDIR/$pkg_name" + if [[ "$pkg_name" =~ devtools ]]; then + INSTALLER_DIR="algorand-devtools" + else + INSTALLER_DIR=algorand + fi + echo "Building rpm package $pkg_name ($CHANNEL)" - < "$REPO_DIR/installer/rpm/$pkg_name/$pkg_name.spec" \ + < "$REPO_DIR/installer/rpm/$INSTALLER_DIR/$INSTALLER_DIR.spec" \ sed -e "s,@PKG_NAME@,$pkg_name," \ -e "s,@VER@,$FULLVERSION," \ -e "s,@REQUIRED_ALGORAND_PKG@,$ALGORAND_PACKAGE_NAME," \ diff --git a/scripts/release/mule/package/deb/package.sh b/scripts/release/mule/package/deb/package.sh index d73f5af8d0..5662269b39 100755 --- a/scripts/release/mule/package/deb/package.sh +++ b/scripts/release/mule/package/deb/package.sh @@ -30,23 +30,23 @@ trap "rm -rf $PKG_ROOT" 0 mkdir -p "${PKG_ROOT}/usr/bin" # NOTE: keep in sync with `./installer/rpm/algorand.spec`. -if [ "$ALGORAND_PACKAGE_NAME" = "algorand-devtools" ]; then +if [[ "$ALGORAND_PACKAGE_NAME" =~ devtools ]]; then BIN_FILES=("carpenter" "catchupsrv" "msgpacktool" "tealcut" "tealdbg") UNATTENDED_UPGRADES_FILE="53algorand-devtools-upgrades" + OUTPUT_DEB="$OUTDIR/algorand-devtools_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" REQUIRED_ALGORAND_PKG=$("./scripts/compute_package_name.sh" "$CHANNEL") else BIN_FILES=("algocfg" "algod" "algoh" "algokey" "ddconfig.sh" "diagcfg" "goal" "kmd" "node_exporter") UNATTENDED_UPGRADES_FILE="51algorand-upgrades" + OUTPUT_DEB="$OUTDIR/algorand_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" fi -OUTPUT_DEB="$OUTDIR/${ALGORAND_PACKAGE_NAME}_${CHANNEL}_${OS_TYPE}-${ARCH}_${VER}.deb" - for binary in "${BIN_FILES[@]}"; do cp "${ALGO_BIN}/${binary}" "${PKG_ROOT}"/usr/bin chmod 755 "${PKG_ROOT}/usr/bin/${binary}" done -if [ "$ALGORAND_PACKAGE_NAME" != "algorand-devtools" ]; then +if [[ ! "$ALGORAND_PACKAGE_NAME" =~ devtools ]]; then mkdir -p "${PKG_ROOT}/usr/lib/algorand" lib_files=("updater" "find-nodes.sh") for lib in "${lib_files[@]}"; do @@ -90,8 +90,13 @@ Dpkg::Options { EOF mkdir -p "${PKG_ROOT}/DEBIAN" +if [[ "$PKG_NAME" =~ devtools ]]; then + INSTALLER_DIR="algorand-devtools" +else + INSTALLER_DIR=algorand +fi # Can contain `control`, `preinst`, `postinst`, `prerm`, `postrm`, `conffiles`. -CTL_FILES_DIR="installer/debian/${ALGORAND_PACKAGE_NAME}" +CTL_FILES_DIR="installer/debian/${INSTALLER_DIR}" for ctl_file in $(ls "${CTL_FILES_DIR}"); do # Copy first, to preserve permissions, then overwrite to fill in template. cp -a "${CTL_FILES_DIR}/${ctl_file}" "${PKG_ROOT}/DEBIAN/${ctl_file}" diff --git a/scripts/release/mule/package/rpm/package.sh b/scripts/release/mule/package/rpm/package.sh index abf3d91814..3d60cb9d29 100755 --- a/scripts/release/mule/package/rpm/package.sh +++ b/scripts/release/mule/package/rpm/package.sh @@ -32,8 +32,13 @@ RPMTMP=$(mktemp -d 2>/dev/null || mktemp -d -t "rpmtmp") trap 'rm -rf $RPMTMP' 0 TEMPDIR=$(mktemp -d) +if [[ "$ALGORAND_PACKAGE_NAME" =~ devtools ]]; then + INSTALLER_DIR="algorand-devtools" +else + INSTALLER_DIR=algorand +fi trap 'rm -rf $TEMPDIR' 0 -< "./installer/rpm/$ALGORAND_PACKAGE_NAME/$ALGORAND_PACKAGE_NAME.spec" \ +< "./installer/rpm/$INSTALLER_DIR/$INSTALLER_DIR.spec" \ sed -e "s,@ALGORAND_PACKAGE_NAME@,$REQUIRED_ALGORAND_PACKAGE," \ -e "s,@VER@,$FULLVERSION," \ -e "s,@ARCH@,$ARCH," \ From 0a3594746ea182fa3700e7e272037f4efc964df3 Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Fri, 18 Sep 2020 10:33:45 -0400 Subject: [PATCH 051/136] Remove the Lookup from the ledgerForEvaluator interface (#1521) Refactor: Remove the Lookup from the ledgerForEvaluator interface --- ledger/acctupdates.go | 5 ----- ledger/eval.go | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index 1dedc643d4..45fd5324aa 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -743,11 +743,6 @@ func (aul *accountUpdatesLedgerEvaluator) BlockHdr(r basics.Round) (bookkeeping. return bookkeeping.BlockHeader{}, ErrNoEntry{} } -// Lookup returns the account balance for a given address at a given round -func (aul *accountUpdatesLedgerEvaluator) Lookup(rnd basics.Round, addr basics.Address) (basics.AccountData, error) { - return aul.au.lookupImpl(rnd, addr, true) -} - // Totals returns the totals for a given round func (aul *accountUpdatesLedgerEvaluator) Totals(rnd basics.Round) (AccountTotals, error) { return aul.au.totalsImpl(rnd) diff --git a/ledger/eval.go b/ledger/eval.go index 092e9b3646..6266f96537 100644 --- a/ledger/eval.go +++ b/ledger/eval.go @@ -183,7 +183,6 @@ type BlockEvaluator struct { type ledgerForEvaluator interface { GenesisHash() crypto.Digest BlockHdr(basics.Round) (bookkeeping.BlockHeader, error) - Lookup(basics.Round, basics.Address) (basics.AccountData, error) Totals(basics.Round) (AccountTotals, error) isDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, txlease) (bool, error) LookupWithoutRewards(basics.Round, basics.Address) (basics.AccountData, error) @@ -233,6 +232,8 @@ func startEvaluator(l ledgerForEvaluator, hdr bookkeeping.BlockHeader, paysetHin eval.block.Payset = make([]transactions.SignedTxnInBlock, 0, paysetHint) } + prevProto := proto + if hdr.Round > 0 { var err error eval.prevHeader, err = l.BlockHdr(base.rnd) @@ -241,6 +242,11 @@ func startEvaluator(l ledgerForEvaluator, hdr bookkeeping.BlockHeader, paysetHin } base.txnCount = eval.prevHeader.TxnCounter + + prevProto, ok = config.Consensus[eval.prevHeader.CurrentProtocol] + if !ok { + return nil, protocol.Error(eval.prevHeader.CurrentProtocol) + } } prevTotals, err := l.Totals(eval.prevHeader.Round) @@ -249,11 +255,15 @@ func startEvaluator(l ledgerForEvaluator, hdr bookkeeping.BlockHeader, paysetHin } poolAddr := eval.prevHeader.RewardsPool - incentivePoolData, err := l.Lookup(eval.prevHeader.Round, poolAddr) + // get the reward pool account data without any rewards + incentivePoolData, err := l.LookupWithoutRewards(eval.prevHeader.Round, poolAddr) if err != nil { return nil, err } + // this is expected to be a no-op, but update the rewards on the rewards pool if it was configured to receive rewards ( unlike mainnet ). + incentivePoolData = incentivePoolData.WithUpdatedRewards(prevProto, eval.prevHeader.RewardsLevel) + if generate { if eval.proto.SupportGenesisHash { eval.block.BlockHeader.GenesisHash = eval.genesisHash From 3573f43ba4058368de86dc95596e129511b5c6c9 Mon Sep 17 00:00:00 2001 From: John Lee Date: Fri, 18 Sep 2020 15:09:13 -0400 Subject: [PATCH 052/136] Bug fix - revert change to grep recommended by shellcheck (#1523) This script was modified to implement a suggestion from shellcheck. However, changing egrep -A to grep -AE doesn't work in Ubuntu 18.04. --- scripts/release/forward_gpg_agent.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/release/forward_gpg_agent.sh b/scripts/release/forward_gpg_agent.sh index 38161ec034..ef9665fd45 100755 --- a/scripts/release/forward_gpg_agent.sh +++ b/scripts/release/forward_gpg_agent.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# shellcheck disable=2196 + # JENKINS=username@ip # JENKINS_KEY=location/to/jenkins/private_key.pem @@ -29,7 +31,7 @@ INSTANCE=$(ssh -i "$JENKINS_KEY" "$JENKINS" sudo cat /opt/jenkins/workspace/"$BR gpgp=$(find /usr/lib/gnupg{2,,1} -type f -name gpg-preset-passphrase 2> /dev/null) # Here we need to grab the signing subkey, hence `tail -1`. -KEYGRIP=$(gpg -K --with-keygrip --textmode dev@algorand.com | grep -AE 1 '^ssb[^#]' | grep Keygrip | awk '{ print $3 }') +KEYGRIP=$(gpg -K --with-keygrip --textmode dev@algorand.com | egrep -A 1 '^ssb[^#]' | grep Keygrip | awk '{ print $3 }') echo "enter dev@ password" $gpgp --verbose --preset "$KEYGRIP" From f2c1347ad881f9250ff13324a725ca402d66ef67 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Fri, 18 Sep 2020 18:45:29 -0400 Subject: [PATCH 053/136] Add metrics for the various ledger components (#1510) Measure count of various database options and the sum of microseconds spent on those operations. --- ledger/acctupdates.go | 32 ++++++++++++++++++++++- ledger/blockqueue.go | 38 ++++++++++++++++++++++++++++ ledger/catchupaccessor.go | 53 +++++++++++++++++++++++++++++++++++++++ ledger/ledger.go | 13 ++++++++++ util/metrics/counter.go | 12 +++++++++ 5 files changed, 147 insertions(+), 1 deletion(-) diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index 45fd5324aa..0eb5fc8842 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -42,6 +42,7 @@ import ( "github.com/algorand/go-algorand/logging/telemetryspec" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/util/db" + "github.com/algorand/go-algorand/util/metrics" ) const ( @@ -460,10 +461,13 @@ func (au *accountUpdates) onlineTop(rnd basics.Round, voteRnd basics.Round, n ui batchSize := uint64(1024) for uint64(len(candidates)) < n+uint64(len(modifiedAccounts)) { var accts map[basics.Address]*onlineAccount + start := time.Now() + ledgerAccountsonlinetopCount.Inc(nil) err = au.dbs.rdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { accts, err = accountsOnlineTop(tx, batchOffset, batchSize, proto) return }) + ledgerAccountsonlinetopMicros.AddMicrosecondsSince(start, nil) if err != nil { return nil, err } @@ -656,10 +660,13 @@ func (au *accountUpdates) Totals(rnd basics.Round) (totals AccountTotals, err er // GetCatchpointStream returns an io.Reader to the catchpoint file associated with the provided round func (au *accountUpdates) GetCatchpointStream(round basics.Round) (io.ReadCloser, error) { dbFileName := "" + start := time.Now() + ledgerGetcatchpointCount.Inc(nil) err := au.dbs.rdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { dbFileName, _, _, err = getCatchpoint(tx, round) return }) + ledgerGetcatchpointMicros.AddMicrosecondsSince(start, nil) if err != nil && err != sql.ErrNoRows { // we had some sql error. return nil, fmt.Errorf("accountUpdates: getCatchpointStream: unable to lookup catchpoint %d: %v", round, err) @@ -828,6 +835,8 @@ func (au *accountUpdates) initializeFromDisk(l ledgerForTracker) (lastBalancesRo } lastestBlockRound = l.Latest() + start := time.Now() + ledgerAccountsinitCount.Inc(nil) err = au.dbs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { var err0 error au.dbRound, err0 = au.accountsInitialize(ctx, tx) @@ -855,6 +864,7 @@ func (au *accountUpdates) initializeFromDisk(l ledgerForTracker) (lastBalancesRo au.roundTotals = []AccountTotals{totals} return nil }) + ledgerAccountsinitMicros.AddMicrosecondsSince(start, nil) if err != nil { return } @@ -1559,6 +1569,8 @@ func (au *accountUpdates) commitRound(offset uint64, dbRound basics.Round, lookb genesisProto := au.ledger.GenesisProto() + start := time.Now() + ledgerCommitroundCount.Inc(nil) err := au.dbs.wdb.AtomicCommitWriteLock(func(ctx context.Context, tx *sql.Tx) (err error) { treeTargetRound := basics.Round(0) if au.catchpointInterval > 0 { @@ -1607,7 +1619,7 @@ func (au *accountUpdates) commitRound(offset uint64, dbRound basics.Round, lookb } return nil }, &au.accountsMu) - + ledgerCommitroundMicros.AddMicrosecondsSince(start, nil) if err != nil { au.balancesTrie = nil au.log.Warnf("unable to advance account snapshot: %v", err) @@ -1736,6 +1748,8 @@ func (au *accountUpdates) generateCatchpoint(committedRound basics.Round, label } var catchpointWriter *catchpointWriter + start := time.Now() + ledgerGeneratecatchpointCount.Inc(nil) err = au.dbs.rdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { catchpointWriter = makeCatchpointWriter(au.ctx, absCatchpointFileName, tx, committedRound, committedRoundDigest, label) for more { @@ -1780,6 +1794,7 @@ func (au *accountUpdates) generateCatchpoint(committedRound basics.Round, label } return }) + ledgerGeneratecatchpointMicros.AddMicrosecondsSince(start, nil) if err != nil { au.log.Warnf("accountUpdates: generateCatchpoint: %v", err) @@ -1890,6 +1905,7 @@ func (au *accountUpdates) vacuumDatabase(ctx context.Context) (err error) { } }() + ledgerVacuumCount.Inc(nil) vacuumStats, err := au.dbs.wdb.Vacuum(ctx) close(vacuumExitCh) vacuumLoggingAbort.Wait() @@ -1899,6 +1915,7 @@ func (au *accountUpdates) vacuumDatabase(ctx context.Context) (err error) { return err } vacuumElapsedTime := time.Now().Sub(startTime) + ledgerVacuumMicros.AddUint64(uint64(vacuumElapsedTime.Microseconds()), nil) au.log.Infof("Vacuuming accounts database completed within %v, reducing number of pages from %d to %d and size from %d to %d", vacuumElapsedTime, vacuumStats.PagesBefore, vacuumStats.PagesAfter, vacuumStats.SizeBefore, vacuumStats.SizeAfter) @@ -1913,3 +1930,16 @@ func (au *accountUpdates) vacuumDatabase(ctx context.Context) (err error) { au.log.EventWithDetails(telemetryspec.Accounts, telemetryspec.BalancesAccountVacuumEvent, vacuumTelemetryStats) return } + +var ledgerAccountsonlinetopCount = metrics.NewCounter("ledger_accountsonlinetop_count", "calls") +var ledgerAccountsonlinetopMicros = metrics.NewCounter("ledger_accountsonlinetop_micros", "µs spent") +var ledgerGetcatchpointCount = metrics.NewCounter("ledger_getcatchpoint_count", "calls") +var ledgerGetcatchpointMicros = metrics.NewCounter("ledger_getcatchpoint_micros", "µs spent") +var ledgerAccountsinitCount = metrics.NewCounter("ledger_accountsinit_count", "calls") +var ledgerAccountsinitMicros = metrics.NewCounter("ledger_accountsinit_micros", "µs spent") +var ledgerCommitroundCount = metrics.NewCounter("ledger_commitround_count", "calls") +var ledgerCommitroundMicros = metrics.NewCounter("ledger_commitround_micros", "µs spent") +var ledgerGeneratecatchpointCount = metrics.NewCounter("ledger_generatecatchpoint_count", "calls") +var ledgerGeneratecatchpointMicros = metrics.NewCounter("ledger_generatecatchpoint_micros", "µs spent") +var ledgerVacuumCount = metrics.NewCounter("ledger_vacuum_count", "calls") +var ledgerVacuumMicros = metrics.NewCounter("ledger_vacuum_micros", "µs spent") diff --git a/ledger/blockqueue.go b/ledger/blockqueue.go index 6b889ae7c9..2fc9f3f511 100644 --- a/ledger/blockqueue.go +++ b/ledger/blockqueue.go @@ -21,6 +21,7 @@ import ( "database/sql" "fmt" "sync" + "time" "github.com/algorand/go-deadlock" @@ -29,6 +30,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/metrics" ) type blockEntry struct { @@ -54,11 +56,14 @@ func bqInit(l *Ledger) (*blockQueue, error) { bq.l = l bq.running = true bq.closed = make(chan struct{}) + ledgerBlockqInitCount.Inc(nil) + start := time.Now() err := bq.l.blockDBs.rdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { var err0 error bq.lastCommitted, err0 = blockLatest(tx) return err0 }) + ledgerBlockqInitMicros.AddMicrosecondsSince(start, nil) if err != nil { return nil, err } @@ -101,6 +106,8 @@ func (bq *blockQueue) syncer() { workQ := bq.q bq.mu.Unlock() + start := time.Now() + ledgerSyncBlockputCount.Inc(nil) err := bq.l.blockDBs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { for _, e := range workQ { err0 := blockPut(tx, e.block, e.cert) @@ -110,6 +117,7 @@ func (bq *blockQueue) syncer() { } return nil }) + ledgerSyncBlockputMicros.AddMicrosecondsSince(start, nil) bq.mu.Lock() @@ -134,9 +142,12 @@ func (bq *blockQueue) syncer() { bq.mu.Unlock() minToSave := bq.l.notifyCommit(committed) + bfstart := time.Now() + ledgerSyncBlockforgetCount.Inc(nil) err = bq.l.blockDBs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { return blockForgetBefore(tx, minToSave) }) + ledgerSyncBlockforgetMicros.AddMicrosecondsSince(bfstart, nil) if err != nil { bq.l.log.Warnf("blockQueue.syncer: blockForgetBefore(%d): %v", minToSave, err) } @@ -245,11 +256,14 @@ func (bq *blockQueue) getBlock(r basics.Round) (blk bookkeeping.Block, err error return } + start := time.Now() + ledgerGetblockCount.Inc(nil) err = bq.l.blockDBs.rdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { var err0 error blk, err0 = blockGet(tx, r) return err0 }) + ledgerGetblockMicros.AddMicrosecondsSince(start, nil) err = updateErrNoEntry(err, lastCommitted, latest) return } @@ -264,11 +278,14 @@ func (bq *blockQueue) getBlockHdr(r basics.Round) (hdr bookkeeping.BlockHeader, return } + start := time.Now() + ledgerGetblockhdrCount.Inc(nil) err = bq.l.blockDBs.rdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { var err0 error hdr, err0 = blockGetHdr(tx, r) return err0 }) + ledgerGetblockhdrMicros.AddMicrosecondsSince(start, nil) err = updateErrNoEntry(err, lastCommitted, latest) return } @@ -287,11 +304,14 @@ func (bq *blockQueue) getEncodedBlockCert(r basics.Round) (blk []byte, cert []by return } + start := time.Now() + ledgerGeteblockcertCount.Inc(nil) err = bq.l.blockDBs.rdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { var err0 error blk, cert, err0 = blockGetEncodedCert(tx, r) return err0 }) + ledgerGeteblockcertMicros.AddMicrosecondsSince(start, nil) err = updateErrNoEntry(err, lastCommitted, latest) return } @@ -306,11 +326,29 @@ func (bq *blockQueue) getBlockCert(r basics.Round) (blk bookkeeping.Block, cert return } + start := time.Now() + ledgerGetblockcertCount.Inc(nil) err = bq.l.blockDBs.rdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { var err0 error blk, cert, err0 = blockGetCert(tx, r) return err0 }) + ledgerGetblockcertMicros.AddMicrosecondsSince(start, nil) err = updateErrNoEntry(err, lastCommitted, latest) return } + +var ledgerBlockqInitCount = metrics.NewCounter("ledger_blockq_init_count", "calls to init block queue") +var ledgerBlockqInitMicros = metrics.NewCounter("ledger_blockq_init_micros", "µs spent to init block queue") +var ledgerSyncBlockputCount = metrics.NewCounter("ledger_blockq_sync_put_count", "calls to sync block queue") +var ledgerSyncBlockputMicros = metrics.NewCounter("ledger_blockq_sync_put_micros", "µs spent to sync block queue") +var ledgerSyncBlockforgetCount = metrics.NewCounter("ledger_blockq_sync_forget_count", "calls") +var ledgerSyncBlockforgetMicros = metrics.NewCounter("ledger_blockq_sync_forget_micros", "µs spent") +var ledgerGetblockCount = metrics.NewCounter("ledger_blockq_getblock_count", "calls") +var ledgerGetblockMicros = metrics.NewCounter("ledger_blockq_getblock_micros", "µs spent") +var ledgerGetblockhdrCount = metrics.NewCounter("ledger_blockq_getblockhdr_count", "calls") +var ledgerGetblockhdrMicros = metrics.NewCounter("ledger_blockq_getblockhdr_micros", "µs spent") +var ledgerGeteblockcertCount = metrics.NewCounter("ledger_blockq_geteblockcert_count", "calls") +var ledgerGeteblockcertMicros = metrics.NewCounter("ledger_blockq_geteblockcert_micros", "µs spent") +var ledgerGetblockcertCount = metrics.NewCounter("ledger_blockq_getblockcert_count", "calls") +var ledgerGetblockcertMicros = metrics.NewCounter("ledger_blockq_getblockcert_micros", "µs spent") diff --git a/ledger/catchupaccessor.go b/ledger/catchupaccessor.go index c6dec55973..5621d5f6ec 100644 --- a/ledger/catchupaccessor.go +++ b/ledger/catchupaccessor.go @@ -22,6 +22,7 @@ import ( "encoding/hex" "fmt" "strings" + "time" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" @@ -30,6 +31,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/metrics" ) // CatchpointCatchupAccessor is an interface for the accessor wrapping the database storage for the catchpoint catchup functionality. @@ -177,6 +179,8 @@ func (c *CatchpointCatchupAccessorImpl) ResetStagingBalances(ctx context.Context if !newCatchup { c.ledger.setSynchronousMode(ctx, c.ledger.synchronousMode) } + start := time.Now() + ledgerResetstagingbalancesCount.Inc(nil) err = wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { err = resetCatchpointStagingBalances(ctx, tx, newCatchup) if err != nil { @@ -209,6 +213,7 @@ func (c *CatchpointCatchupAccessorImpl) ResetStagingBalances(ctx context.Context } return }) + ledgerResetstagingbalancesMicros.AddMicrosecondsSince(start, nil) return } @@ -256,6 +261,8 @@ func (c *CatchpointCatchupAccessorImpl) processStagingContent(ctx context.Contex // later on: // TotalAccounts, TotalAccounts, Catchpoint, BlockHeaderDigest, BalancesRound wdb := c.ledger.trackerDB().wdb + start := time.Now() + ledgerProcessstagingcontentCount.Inc(nil) err = wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { sq, err := accountsDbInit(tx, tx) if err != nil { @@ -269,6 +276,7 @@ func (c *CatchpointCatchupAccessorImpl) processStagingContent(ctx context.Contex err = accountsPutTotals(tx, fileHeader.Totals, true) return }) + ledgerProcessstagingcontentMicros.AddMicrosecondsSince(start, nil) if err == nil { progress.SeenHeader = true progress.TotalAccounts = fileHeader.TotalAccounts @@ -296,6 +304,8 @@ func (c *CatchpointCatchupAccessorImpl) processStagingBalances(ctx context.Conte proto := c.ledger.GenesisProto() wdb := c.ledger.trackerDB().wdb + start := time.Now() + ledgerProcessstagingbalancesCount.Inc(nil) err = wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { // create the merkle trie for the balances var mc *merkleCommitter @@ -359,6 +369,7 @@ func (c *CatchpointCatchupAccessorImpl) processStagingBalances(ctx context.Conte err = progress.EvictAsNeeded(uint64(len(balances.Balances))) return }) + ledgerProcessstagingbalancesMicros.AddMicrosecondsSince(start, nil) if err == nil { progress.ProcessedAccounts += uint64(len(balances.Balances)) progress.ProcessedBytes += uint64(len(bytes)) @@ -415,6 +426,8 @@ func (c *CatchpointCatchupAccessorImpl) VerifyCatchpoint(ctx context.Context, bl } blockRound = basics.Round(iRound) + start := time.Now() + ledgerVerifycatchpointCount.Inc(nil) err = rdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { // create the merkle trie for the balances mc, err0 := makeMerkleCommitter(tx, true) @@ -438,6 +451,7 @@ func (c *CatchpointCatchupAccessorImpl) VerifyCatchpoint(ctx context.Context, bl } return }) + ledgerVerifycatchpointMicros.AddMicrosecondsSince(start, nil) if err != nil { return err } @@ -460,6 +474,8 @@ func (c *CatchpointCatchupAccessorImpl) StoreBalancesRound(ctx context.Context, // trust the one in the catchpoint file header, so we'll calculate it ourselves. balancesRound := blk.Round() - basics.Round(config.Consensus[blk.CurrentProtocol].MaxBalLookback) wdb := c.ledger.trackerDB().wdb + start := time.Now() + ledgerStorebalancesroundCount.Inc(nil) err = wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { sq, err := accountsDbInit(tx, tx) if err != nil { @@ -472,15 +488,19 @@ func (c *CatchpointCatchupAccessorImpl) StoreBalancesRound(ctx context.Context, } return }) + ledgerStorebalancesroundMicros.AddMicrosecondsSince(start, nil) return } // StoreFirstBlock stores a single block to the blocks database. func (c *CatchpointCatchupAccessorImpl) StoreFirstBlock(ctx context.Context, blk *bookkeeping.Block) (err error) { blockDbs := c.ledger.blockDB() + start := time.Now() + ledgerStorefirstblockCount.Inc(nil) err = blockDbs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { return blockStartCatchupStaging(tx, *blk) }) + ledgerStorefirstblockMicros.AddMicrosecondsSince(start, nil) if err != nil { return err } @@ -490,9 +510,12 @@ func (c *CatchpointCatchupAccessorImpl) StoreFirstBlock(ctx context.Context, blk // StoreBlock stores a single block to the blocks database. func (c *CatchpointCatchupAccessorImpl) StoreBlock(ctx context.Context, blk *bookkeeping.Block) (err error) { blockDbs := c.ledger.blockDB() + start := time.Now() + ledgerCatchpointStoreblockCount.Inc(nil) err = blockDbs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { return blockPutStaging(tx, *blk) }) + ledgerCatchpointStoreblockMicros.AddMicrosecondsSince(start, nil) if err != nil { return err } @@ -502,12 +525,15 @@ func (c *CatchpointCatchupAccessorImpl) StoreBlock(ctx context.Context, blk *boo // FinishBlocks concludes the catchup of the blocks database. func (c *CatchpointCatchupAccessorImpl) FinishBlocks(ctx context.Context, applyChanges bool) (err error) { blockDbs := c.ledger.blockDB() + start := time.Now() + ledgerCatchpointFinishblocksCount.Inc(nil) err = blockDbs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { if applyChanges { return blockCompleteCatchup(tx) } return blockAbortCatchup(tx) }) + ledgerCatchpointFinishblocksMicros.AddMicrosecondsSince(start, nil) if err != nil { return err } @@ -517,10 +543,13 @@ func (c *CatchpointCatchupAccessorImpl) FinishBlocks(ctx context.Context, applyC // EnsureFirstBlock ensure that we have a single block in the staging block table, and returns that block func (c *CatchpointCatchupAccessorImpl) EnsureFirstBlock(ctx context.Context) (blk bookkeeping.Block, err error) { blockDbs := c.ledger.blockDB() + start := time.Now() + ledgerCatchpointEnsureblock1Count.Inc(nil) err = blockDbs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { blk, err = blockEnsureSingleBlock(tx) return }) + ledgerCatchpointEnsureblock1Micros.AddMicrosecondsSince(start, nil) if err != nil { return blk, err } @@ -545,6 +574,8 @@ func (c *CatchpointCatchupAccessorImpl) CompleteCatchup(ctx context.Context) (er // finishBalances concludes the catchup of the balances(tracker) database. func (c *CatchpointCatchupAccessorImpl) finishBalances(ctx context.Context) (err error) { wdb := c.ledger.trackerDB().wdb + start := time.Now() + ledgerCatchpointFinishBalsCount.Inc(nil) err = wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { var balancesRound uint64 var totals AccountTotals @@ -602,5 +633,27 @@ func (c *CatchpointCatchupAccessorImpl) finishBalances(ctx context.Context) (err return }) + ledgerCatchpointFinishBalsMicros.AddMicrosecondsSince(start, nil) return err } + +var ledgerResetstagingbalancesCount = metrics.NewCounter("ledger_catchup_resetstagingbalances_count", "calls") +var ledgerResetstagingbalancesMicros = metrics.NewCounter("ledger_catchup_resetstagingbalances_micros", "µs spent") +var ledgerProcessstagingcontentCount = metrics.NewCounter("ledger_catchup_processstagingcontent_count", "calls") +var ledgerProcessstagingcontentMicros = metrics.NewCounter("ledger_catchup_processstagingcontent_micros", "µs spent") +var ledgerProcessstagingbalancesCount = metrics.NewCounter("ledger_catchup_processstagingbalances_count", "calls") +var ledgerProcessstagingbalancesMicros = metrics.NewCounter("ledger_catchup_processstagingbalances_micros", "µs spent") +var ledgerVerifycatchpointCount = metrics.NewCounter("ledger_catchup_verifycatchpoint_count", "calls") +var ledgerVerifycatchpointMicros = metrics.NewCounter("ledger_catchup_verifycatchpoint_micros", "µs spent") +var ledgerStorebalancesroundCount = metrics.NewCounter("ledger_catchup_storebalancesround_count", "calls") +var ledgerStorebalancesroundMicros = metrics.NewCounter("ledger_catchup_storebalancesround_micros", "µs spent") +var ledgerStorefirstblockCount = metrics.NewCounter("ledger_catchup_storefirstblock_count", "calls") +var ledgerStorefirstblockMicros = metrics.NewCounter("ledger_catchup_storefirstblock_micros", "µs spent") +var ledgerCatchpointStoreblockCount = metrics.NewCounter("ledger_catchup_catchpoint_storeblock_count", "calls") +var ledgerCatchpointStoreblockMicros = metrics.NewCounter("ledger_catchup_catchpoint_storeblock_micros", "µs spent") +var ledgerCatchpointFinishblocksCount = metrics.NewCounter("ledger_catchup_catchpoint_finishblocks_count", "calls") +var ledgerCatchpointFinishblocksMicros = metrics.NewCounter("ledger_catchup_catchpoint_finishblocks_micros", "µs spent") +var ledgerCatchpointEnsureblock1Count = metrics.NewCounter("ledger_catchup_catchpoint_ensureblock1_count", "calls") +var ledgerCatchpointEnsureblock1Micros = metrics.NewCounter("ledger_catchup_catchpoint_ensureblock1_micros", "µs spent") +var ledgerCatchpointFinishBalsCount = metrics.NewCounter("ledger_catchup_catchpoint_finish_bals_count", "calls") +var ledgerCatchpointFinishBalsMicros = metrics.NewCounter("ledger_catchup_catchpoint_finish_bals_micros", "µs spent") diff --git a/ledger/ledger.go b/ledger/ledger.go index 7cfe34a4c5..c61a681fd2 100644 --- a/ledger/ledger.go +++ b/ledger/ledger.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "os" + "time" "github.com/algorand/go-deadlock" @@ -33,6 +34,7 @@ import ( "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/util/db" + "github.com/algorand/go-algorand/util/metrics" ) // Ledger is a database storing the contents of the ledger. @@ -125,9 +127,12 @@ func OpenLedger( l.setSynchronousMode(context.Background(), l.synchronousMode) + start := time.Now() + ledgerInitblocksdbCount.Inc(nil) err = l.blockDBs.wdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { return initBlocksDB(tx, l, []bookkeeping.Block{genesisInitState.Block}, cfg.Archival) }) + ledgerInitblocksdbMicros.AddMicrosecondsSince(start, nil) if err != nil { err = fmt.Errorf("OpenLedger.initBlocksDB %v", err) return nil, err @@ -195,6 +200,8 @@ func (l *Ledger) reloadLedger() error { // verifyMatchingGenesisHash tests to see that the latest block header pointing to the same genesis hash provided in genesisHash. func (l *Ledger) verifyMatchingGenesisHash() (err error) { // Check that the genesis hash, if present, matches. + start := time.Now() + ledgerVerifygenhashCount.Inc(nil) err = l.blockDBs.rdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { latest, err := blockLatest(tx) if err != nil { @@ -215,6 +222,7 @@ func (l *Ledger) verifyMatchingGenesisHash() (err error) { } return nil }) + ledgerVerifygenhashMicros.AddMicrosecondsSince(start, nil) return } @@ -643,3 +651,8 @@ type txlease struct { sender basics.Address lease [32]byte } + +var ledgerInitblocksdbCount = metrics.NewCounter("ledger_initblocksdb_count", "calls") +var ledgerInitblocksdbMicros = metrics.NewCounter("ledger_initblocksdb_micros", "µs spent") +var ledgerVerifygenhashCount = metrics.NewCounter("ledger_verifygenhash_count", "calls") +var ledgerVerifygenhashMicros = metrics.NewCounter("ledger_verifygenhash_micros", "µs spent") diff --git a/util/metrics/counter.go b/util/metrics/counter.go index 8641b1f4be..2cf614d491 100644 --- a/util/metrics/counter.go +++ b/util/metrics/counter.go @@ -21,6 +21,7 @@ import ( "strconv" "strings" "sync/atomic" + "time" ) // MakeCounter create a new counter with the provided name and description. @@ -36,6 +37,11 @@ func MakeCounter(metric MetricName) *Counter { return c } +// NewCounter is a shortcut to MakeCounter in one shorter line. +func NewCounter(name, desc string) *Counter { + return MakeCounter(MetricName{Name: name, Description: desc}) +} + // Register registers the counter with the default/specific registry func (counter *Counter) Register(reg *Registry) { if reg == nil { @@ -99,6 +105,12 @@ func (counter *Counter) AddUint64(x uint64, labels map[string]string) { } } +// AddMicrosecondsSince increases counter by microseconds between Time t and now. +// Fastest if labels is nil +func (counter *Counter) AddMicrosecondsSince(t time.Time, labels map[string]string) { + counter.AddUint64(uint64(time.Now().Sub(t).Microseconds()), labels) +} + func (counter *Counter) fastAddUint64(x uint64) { if atomic.AddUint64(&counter.intValue, x) == x { // What we just added is the whole value, this From 52b68983b8f565dd7d4281ce4050ab7fc15d65e3 Mon Sep 17 00:00:00 2001 From: nicholasguoalgorand <67928479+nicholasguoalgorand@users.noreply.github.com> Date: Sat, 19 Sep 2020 11:01:46 -0700 Subject: [PATCH 054/136] Vote validation error refactor (#1508) Agreement : avoid classifying an old vote as malicious, and use Info message instead. --- agreement/abstractions.go | 15 +++++++++++++ agreement/asyncVoteVerifier.go | 13 +++++++++-- agreement/common_test.go | 32 +++++++++++++++++++++++++++ agreement/selector.go | 2 +- agreement/vote.go | 6 +++--- agreement/voteAggregator_test.go | 37 ++++++++++++++++++++++++++++++++ ledger/acctupdates.go | 15 ++++++++++++- node/impls.go | 15 +++++++++++++ 8 files changed, 128 insertions(+), 7 deletions(-) diff --git a/agreement/abstractions.go b/agreement/abstractions.go index af1f98bf24..e8341708c2 100644 --- a/agreement/abstractions.go +++ b/agreement/abstractions.go @@ -307,3 +307,18 @@ type Message struct { type EventsProcessingMonitor interface { UpdateEventsQueue(queueName string, queueLength int) } + +// LedgerDroppedRoundError is a wrapper error for when the ledger cannot return a Lookup query because +// the entry is old and was dropped from the ledger. The purpose of this wrapper is to help the +// agreement differentiate between a malicious vote and a vote that it cannot verify +type LedgerDroppedRoundError struct { + Err error +} + +func (e *LedgerDroppedRoundError) Error() string { + return e.Err.Error() +} + +func (e *LedgerDroppedRoundError) Unwrap() error { + return e.Err +} diff --git a/agreement/asyncVoteVerifier.go b/agreement/asyncVoteVerifier.go index 85d890d8f6..ca6fae0cd4 100644 --- a/agreement/asyncVoteVerifier.go +++ b/agreement/asyncVoteVerifier.go @@ -18,6 +18,7 @@ package agreement import ( "context" + "errors" "sync" "github.com/algorand/go-algorand/util/execpool" @@ -104,7 +105,11 @@ func (avv *AsyncVoteVerifier) executeVoteVerification(task interface{}) interfac // request was not cancelled, so we verify it here and return the result on the channel v, err := req.uv.verify(req.l) req.message.Vote = v - return &asyncVerifyVoteResponse{v: v, index: req.index, message: req.message, err: err, req: &req} + + var e *LedgerDroppedRoundError + cancelled := errors.As(err, &e) + + return &asyncVerifyVoteResponse{v: v, index: req.index, message: req.message, err: err, cancelled: cancelled, req: &req} } } @@ -118,7 +123,11 @@ func (avv *AsyncVoteVerifier) executeEqVoteVerification(task interface{}) interf default: // request was not cancelled, so we verify it here and return the result on the channel ev, err := req.uev.verify(req.l) - return &asyncVerifyVoteResponse{ev: ev, index: req.index, message: req.message, err: err, req: &req} + + var e *LedgerDroppedRoundError + cancelled := errors.As(err, &e) + + return &asyncVerifyVoteResponse{ev: ev, index: req.index, message: req.message, err: err, cancelled: cancelled, req: &req} } } diff --git a/agreement/common_test.go b/agreement/common_test.go index c58c4c19ab..de9ee98c29 100644 --- a/agreement/common_test.go +++ b/agreement/common_test.go @@ -192,6 +192,8 @@ type testLedger struct { certs map[basics.Round]Certificate nextRound basics.Round + maxNumBlocks uint64 + // constant state map[basics.Address]basics.AccountData @@ -236,6 +238,27 @@ func makeTestLedgerWithConsensusVersion(state map[basics.Address]basics.AccountD return l } +func makeTestLedgerMaxBlocks(state map[basics.Address]basics.AccountData, maxNumBlocks uint64) Ledger { + l := new(testLedger) + l.entries = make(map[basics.Round]bookkeeping.Block) + l.certs = make(map[basics.Round]Certificate) + l.nextRound = 1 + + l.maxNumBlocks = maxNumBlocks + + l.state = make(map[basics.Address]basics.AccountData) + for k, v := range state { + l.state[k] = v + } + + l.notifications = make(map[basics.Round]signal) + + l.consensusVersion = func(r basics.Round) (protocol.ConsensusVersion, error) { + return protocol.ConsensusCurrentVersion, nil + } + return l +} + func (l *testLedger) NextRound() basics.Round { l.mu.Lock() defer l.mu.Unlock() @@ -290,6 +313,10 @@ func (l *testLedger) LookupDigest(r basics.Round) (crypto.Digest, error) { panic(err) } + if l.maxNumBlocks != 0 && r+round(l.maxNumBlocks) < l.nextRound { + return crypto.Digest{}, &LedgerDroppedRoundError{} + } + return l.entries[r].Digest(), nil } @@ -301,6 +328,11 @@ func (l *testLedger) Lookup(r basics.Round, a basics.Address) (basics.AccountDat err := fmt.Errorf("Lookup called on future round: %v >= %v! (this is probably a bug)", r, l.nextRound) panic(err) } + + if l.maxNumBlocks != 0 && r+round(l.maxNumBlocks) < l.nextRound { + return basics.AccountData{}, &LedgerDroppedRoundError{} + } + return l.state[a], nil } diff --git a/agreement/selector.go b/agreement/selector.go index 3a40728265..d8dec8746b 100644 --- a/agreement/selector.go +++ b/agreement/selector.go @@ -66,7 +66,7 @@ func membership(l LedgerReader, addr basics.Address, r basics.Round, p period, s record, err := l.Lookup(balanceRound, addr) if err != nil { - err = fmt.Errorf("Service.initializeVote (r=%d): Failed to obtain balance record for address %v in round %d: %v", r, addr, balanceRound, err) + err = fmt.Errorf("Service.initializeVote (r=%d): Failed to obtain balance record for address %v in round %d: %w", r, addr, balanceRound, err) return } diff --git a/agreement/vote.go b/agreement/vote.go index e38c588e98..758cb4d795 100644 --- a/agreement/vote.go +++ b/agreement/vote.go @@ -90,7 +90,7 @@ func (uv unauthenticatedVote) verify(l LedgerReader) (vote, error) { rv := uv.R m, err := membership(l, rv.Sender, rv.Round, rv.Period, rv.Step) if err != nil { - return vote{}, fmt.Errorf("unauthenticatedVote.verify: could not get membership parameters: %v", err) + return vote{}, fmt.Errorf("unauthenticatedVote.verify: could not get membership parameters: %w", err) } switch rv.Step { @@ -204,12 +204,12 @@ func (pair unauthenticatedEquivocationVote) verify(l LedgerReader) (equivocation v0, err := uv0.verify(l) if err != nil { - return equivocationVote{}, fmt.Errorf("unauthenticatedEquivocationVote.verify: failed to verify pair 0: %v", err) + return equivocationVote{}, fmt.Errorf("unauthenticatedEquivocationVote.verify: failed to verify pair 0: %w", err) } _, err = uv1.verify(l) if err != nil { - return equivocationVote{}, fmt.Errorf("unauthenticatedEquivocationVote.verify: failed to verify pair 1: %v", err) + return equivocationVote{}, fmt.Errorf("unauthenticatedEquivocationVote.verify: failed to verify pair 1: %w", err) } return equivocationVote{ diff --git a/agreement/voteAggregator_test.go b/agreement/voteAggregator_test.go index 8282fbf5f2..14bd98d5af 100644 --- a/agreement/voteAggregator_test.go +++ b/agreement/voteAggregator_test.go @@ -839,3 +839,40 @@ func TestVoteAggregatorFiltersVoteNextRound(t *testing.T) { require.NoError(t, err) require.NoErrorf(t, res, "Votes from next round not correctly filtered") } + +func TestVoteAggregatorOldVote(t *testing.T) { + cparams := config.Consensus[protocol.ConsensusCurrentVersion] + maxNumBlocks := 2 * cparams.SeedRefreshInterval * cparams.SeedLookback + ledger := makeTestLedgerMaxBlocks(readOnlyGenesis100, maxNumBlocks) + addresses, vrfSecrets, otSecrets := readOnlyAddrs100, readOnlyVRF100, readOnlyOT100 + round := ledger.NextRound() + period := period(0) + + var proposal proposalValue + proposal.BlockDigest = randomBlockHash() + + var uvs []unauthenticatedVote + for i := range addresses { + address := addresses[i] + step := step(1) + rv := rawVote{Sender: address, Round: round, Period: period, Step: step, Proposal: proposal} + uv, err := makeVote(rv, otSecrets[i], vrfSecrets[i], ledger) + assert.NoError(t, err) + uvs = append(uvs, uv) + } + + for r := 1; r < 1000; r++ { + ledger.EnsureBlock(makeRandomBlock(ledger.NextRound()), Certificate{}) + } + + avv := MakeAsyncVoteVerifier(nil) + defer avv.Quit() + + results := make(chan asyncVerifyVoteResponse, len(uvs)) + + for i, uv := range uvs { + avv.verifyVote(context.Background(), ledger, uv, i, message{}, results) + result := <-results + require.True(t, result.cancelled) + } +} diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index 0eb5fc8842..765020c54b 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -217,6 +217,16 @@ type deferedCommit struct { lookback basics.Round } +// RoundOffsetError is an error for when requested round is behind earliest stored db entry +type RoundOffsetError struct { + round basics.Round + dbRound basics.Round +} + +func (e *RoundOffsetError) Error() string { + return fmt.Sprintf("round %d before dbRound %d", e.round, e.dbRound) +} + // initialize initializes the accountUpdates structure func (au *accountUpdates) initialize(cfg config.Local, dbPathPrefix string, genesisProto config.ConsensusParams, genesisAccounts map[basics.Address]basics.AccountData) { au.initProto = genesisProto @@ -1453,7 +1463,10 @@ func (au *accountUpdates) accountsCreateCatchpointLabel(committedRound basics.Ro // roundOffset calculates the offset of the given round compared to the current dbRound. Requires that the lock would be taken. func (au *accountUpdates) roundOffset(rnd basics.Round) (offset uint64, err error) { if rnd < au.dbRound { - err = fmt.Errorf("round %d before dbRound %d", rnd, au.dbRound) + err = &RoundOffsetError{ + round: rnd, + dbRound: au.dbRound, + } return } diff --git a/node/impls.go b/node/impls.go index 2fc65e6619..ad00f979be 100644 --- a/node/impls.go +++ b/node/impls.go @@ -18,12 +18,15 @@ package node import ( "context" + "errors" "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/catchup" "github.com/algorand/go-algorand/data" + "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/pools" + "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/util/execpool" @@ -111,3 +114,15 @@ func (l agreementLedger) EnsureDigest(cert agreement.Certificate, verifier *agre // 4. no other senders to this channel exists l.UnmatchedPendingCertificates <- catchup.PendingUnmatchedCertificate{Cert: cert, VoteVerifier: verifier} } + +// Wrapping error with a LedgerDroppedRoundError when an old round is requested but the ledger has already dropped the entry +func (l agreementLedger) Lookup(rnd basics.Round, addr basics.Address) (basics.AccountData, error) { + record, err := l.Ledger.Lookup(rnd, addr) + var e *ledger.RoundOffsetError + if errors.As(err, &e) { + err = &agreement.LedgerDroppedRoundError{ + Err: err, + } + } + return record, err +} From 76c1fc34660f8cac5c89fc15935cfe4b9e1fb91e Mon Sep 17 00:00:00 2001 From: Will Winder Date: Mon, 21 Sep 2020 09:00:01 -0400 Subject: [PATCH 055/136] Switch unit to unitname --- cmd/goal/asset.go | 14 +++++++------- test/e2e-go/cli/goal/expect/goalExpectCommon.exp | 2 +- test/scripts/e2e_subs/e2e-app-real-assets-round.sh | 2 +- test/scripts/e2e_subs/limit-swap-test.sh | 2 +- tools/teal/examples/limitorder.sh | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/goal/asset.go b/cmd/goal/asset.go index 8716c22e1b..4ca454c80f 100644 --- a/cmd/goal/asset.go +++ b/cmd/goal/asset.go @@ -111,7 +111,7 @@ func init() { infoAssetCmd.Flags().Uint64Var(&assetID, "assetid", 0, "ID of the asset to look up") infoAssetCmd.Flags().StringVar(&assetUnitName, "asset", "", "DEPRECATED! Unit name of the asset to look up") - infoAssetCmd.Flags().StringVar(&assetUnitName, "unit", "", "Unit name of the asset to look up") + infoAssetCmd.Flags().StringVar(&assetUnitName, "unitname", "", "Unit name of the asset to look up") infoAssetCmd.Flags().StringVar(&assetCreator, "creator", "", "Account address of the asset creator") } @@ -127,17 +127,17 @@ var assetCmd = &cobra.Command{ func lookupAssetID(cmd *cobra.Command, creator string, client libgoal.Client) { if cmd.Flags().Changed("asset") { - reportWarnln("The [--asset] flag is deprecated and will be removed in a future release, use [--unit] instead.") + reportWarnln("The [--asset] flag is deprecated and will be removed in a future release, use [--unitname] instead.") } - if cmd.Flags().Changed("asset") && cmd.Flags().Changed("unit"){ - reportErrorf("The [--asset] flag has been replaced by [--unit], do not provide both flags.") + if cmd.Flags().Changed("asset") && cmd.Flags().Changed("unitname") { + reportErrorf("The [--asset] flag has been replaced by [--unitname], do not provide both flags.") } - assetOrUnit := cmd.Flags().Changed("asset") || cmd.Flags().Changed("unit") + assetOrUnit := cmd.Flags().Changed("asset") || cmd.Flags().Changed("unitname") if cmd.Flags().Changed("assetid") && assetOrUnit { - reportErrorf("Only one of [--assetid] or [--unit and --creator] should be specified") + reportErrorf("Only one of [--assetid] or [--unitname and --creator] should be specified") } if cmd.Flags().Changed("assetid") { @@ -145,7 +145,7 @@ func lookupAssetID(cmd *cobra.Command, creator string, client libgoal.Client) { } if !assetOrUnit { - reportErrorf("Either [--assetid] or [--unit and --creator] must be specified") + reportErrorf("Either [--assetid] or [--unitname and --creator] must be specified") } if !cmd.Flags().Changed("creator") { diff --git a/test/e2e-go/cli/goal/expect/goalExpectCommon.exp b/test/e2e-go/cli/goal/expect/goalExpectCommon.exp index d86e3696bb..4fd94671c7 100755 --- a/test/e2e-go/cli/goal/expect/goalExpectCommon.exp +++ b/test/e2e-go/cli/goal/expect/goalExpectCommon.exp @@ -471,7 +471,7 @@ proc ::AlgorandGoal::AssetLookup { CREATOR UNIT_NAME TEST_PRIMARY_NODE_DIR } { set timeout 10 if { [ catch { set ASSET_ID "NOT SET" - spawn goal asset info -d $TEST_PRIMARY_NODE_DIR --creator $CREATOR --unit $UNIT_NAME + spawn goal asset info -d $TEST_PRIMARY_NODE_DIR --creator $CREATOR --unitname $UNIT_NAME expect { timeout { ::AlgorandGoal::Abort "Timed out asset lookup" } -re {Asset ID:\s+([0-9]+)} {set ASSET_ID $expect_out(1,string); close } diff --git a/test/scripts/e2e_subs/e2e-app-real-assets-round.sh b/test/scripts/e2e_subs/e2e-app-real-assets-round.sh index 6401042c5e..19314fa8d4 100755 --- a/test/scripts/e2e_subs/e2e-app-real-assets-round.sh +++ b/test/scripts/e2e_subs/e2e-app-real-assets-round.sh @@ -18,7 +18,7 @@ ACCOUNT=$(${gcmd} account list|awk '{ print $3 }') # Create an ASA in account ${gcmd} asset create --creator ${ACCOUNT} --name bogocoin --unitname bogo --total 1337 -ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --unit bogo|grep 'Asset ID'|awk '{ print $3 }') +ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --unitname bogo|grep 'Asset ID'|awk '{ print $3 }') # Create app that reads asset balance and checks asset details and checks round ROUND=$(goal node status | grep 'Last committed' | awk '{ print $4 }') diff --git a/test/scripts/e2e_subs/limit-swap-test.sh b/test/scripts/e2e_subs/limit-swap-test.sh index 463128fb8e..dd3d6fef90 100755 --- a/test/scripts/e2e_subs/limit-swap-test.sh +++ b/test/scripts/e2e_subs/limit-swap-test.sh @@ -16,7 +16,7 @@ ZERO_ADDRESS=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY5HFKQ ${gcmd} asset create --creator ${ACCOUNT} --name bogocoin --unitname bogo --total 1000000000000 -ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --unit bogo|grep 'Asset ID'|awk '{ print $3 }') +ASSET_ID=$(${gcmd} asset info --creator $ACCOUNT --unitname bogo|grep 'Asset ID'|awk '{ print $3 }') # Asset ID: 5 diff --git a/tools/teal/examples/limitorder.sh b/tools/teal/examples/limitorder.sh index 4d87aca315..5d6b7bb95b 100755 --- a/tools/teal/examples/limitorder.sh +++ b/tools/teal/examples/limitorder.sh @@ -5,7 +5,7 @@ goal asset create -d . --creator G5PM2K5RIEHHO7ZKR2ZTQDYY6DVBYOMGOFZMMNGJCW4BYNM # > Issued transaction from account G5PM2K5RIEHHO7ZKR2ZTQDYY6DVBYOMGOFZMMNGJCW4BYNMT7HC4HTZIDU, txid JH7M5L43YLQ5DTRIVVBUUB2E4BFE7TPVAPPEGCUVNYSFRLT55Z3Q (fee 1000) # > Transaction JH7M5L43YLQ5DTRIVVBUUB2E4BFE7TPVAPPEGCUVNYSFRLT55Z3Q still pending as of round 148369 # > Transaction JH7M5L43YLQ5DTRIVVBUUB2E4BFE7TPVAPPEGCUVNYSFRLT55Z3Q committed in round 148371 -goal asset info --creator G5PM2K5RIEHHO7ZKR2ZTQDYY6DVBYOMGOFZMMNGJCW4BYNMT7HC4HTZIDU -d . --unit e.g.Coin +goal asset info --creator G5PM2K5RIEHHO7ZKR2ZTQDYY6DVBYOMGOFZMMNGJCW4BYNMT7HC4HTZIDU -d . --unitname e.g.Coin # > Asset ID: 39 # > Creator: G5PM2K5RIEHHO7ZKR2ZTQDYY6DVBYOMGOFZMMNGJCW4BYNMT7HC4HTZIDU # > Asset name: From 79e639de1b9a2a36c7f399fac1ca5c1049f13c3f Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Mon, 21 Sep 2020 21:53:41 -0400 Subject: [PATCH 056/136] Add a transaction type to store compact certificates (#1391) Add a transaction type to store compact certificates --- config/consensus.go | 15 ++ data/pools/transactionPool.go | 10 + data/transactions/compactcert.go | 66 +++++ data/transactions/msgp_gen.go | 367 +++++++++++++++++++++------ data/transactions/msgp_gen_test.go | 62 +++++ data/transactions/transaction.go | 40 ++- data/transactions/verify/txn.go | 9 + data/transactions/verify/txn_test.go | 74 ++++++ ledger/compactcert.go | 159 ++++++++++++ ledger/cow.go | 22 ++ ledger/cow_test.go | 8 + ledger/eval.go | 57 ++++- protocol/hash.go | 1 + protocol/txntype.go | 3 + 14 files changed, 800 insertions(+), 93 deletions(-) create mode 100644 data/transactions/compactcert.go create mode 100644 ledger/compactcert.go diff --git a/config/consensus.go b/config/consensus.go index 7848ce6e86..4c4fdda5ba 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -323,6 +323,21 @@ type ConsensusParams struct { // construction of this Merkle tree (and obtaining the requisite // accounts and balances) in the critical path. CompactCertVotersLookback uint64 + + // CompactCertWeightThreshold is the percentage of top voters weight + // that must sign the message (block header) for security. The compact + // certificate ensures this threshold holds; however, forming a valid + // compact certificate requires a somewhat higher number of signatures, + // and the more signatures are collected, the smaller the compact cert + // can be. + // + // This threshold can be thought of as the maximum percentage of + // malicious weight that compact certificates defend against. + CompactCertWeightThreshold uint64 + + // CompactCertSecKQ is the security parameter (k+q) for the compact + // certificate scheme. + CompactCertSecKQ uint64 } // ConsensusProtocols defines a set of supported protocol versions and their diff --git a/data/pools/transactionPool.go b/data/pools/transactionPool.go index 0b972d9b18..67f78f7109 100644 --- a/data/pools/transactionPool.go +++ b/data/pools/transactionPool.go @@ -293,6 +293,16 @@ func (pool *TransactionPool) computeFeePerByte() uint64 { // checkSufficientFee take a set of signed transactions and verifies that each transaction has // sufficient fee to get into the transaction pool func (pool *TransactionPool) checkSufficientFee(txgroup []transactions.SignedTxn) error { + // Special case: the compact cert transaction, if issued from the + // special compact-cert-sender address, in a singleton group, pays + // no fee. + if len(txgroup) == 1 { + t := txgroup[0].Txn + if t.Type == protocol.CompactCertTx && t.Sender == transactions.CompactCertSender && t.Fee.IsZero() { + return nil + } + } + // get the current fee per byte feePerByte := pool.computeFeePerByte() diff --git a/data/transactions/compactcert.go b/data/transactions/compactcert.go new file mode 100644 index 0000000000..f17a20d314 --- /dev/null +++ b/data/transactions/compactcert.go @@ -0,0 +1,66 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package transactions + +import ( + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/compactcert" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/protocol" +) + +// CompactCertTxnFields captures the fields used for compact cert transactions. +type CompactCertTxnFields struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + CertRound basics.Round `codec:"certrnd"` + Cert compactcert.Cert `codec:"cert"` +} + +// Empty returns whether the CompactCertTxnFields are all zero, +// in the sense of being omitted in a msgpack encoding. +func (cc CompactCertTxnFields) Empty() bool { + if cc.CertRound != 0 { + return false + } + if !cc.Cert.SigCommit.IsZero() || cc.Cert.SignedWeight != 0 { + return false + } + if len(cc.Cert.SigProofs) != 0 || len(cc.Cert.PartProofs) != 0 { + return false + } + if len(cc.Cert.Reveals) != 0 { + return false + } + return true +} + +//msgp:ignore specialAddr +// specialAddr is used to form a unique address that will send out compact certs. +type specialAddr string + +// ToBeHashed implements the crypto.Hashable interface +func (a specialAddr) ToBeHashed() (protocol.HashID, []byte) { + return protocol.SpecialAddr, []byte(a) +} + +// CompactCertSender is the computed address for sending out compact certs. +var CompactCertSender basics.Address + +func init() { + CompactCertSender = basics.Address(crypto.HashObj(specialAddr("CompactCertSender"))) +} diff --git a/data/transactions/msgp_gen.go b/data/transactions/msgp_gen.go index beeeaa6008..77cc22765c 100644 --- a/data/transactions/msgp_gen.go +++ b/data/transactions/msgp_gen.go @@ -50,6 +50,14 @@ import ( // |-----> (*) Msgsize // |-----> (*) MsgIsZero // +// CompactCertTxnFields +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// // Header // |-----> (*) MarshalMsg // |-----> (*) CanMarshalMsg @@ -1481,6 +1489,143 @@ func (z *AssetTransferTxnFields) MsgIsZero() bool { return ((*z).XferAsset.MsgIsZero()) && ((*z).AssetAmount == 0) && ((*z).AssetSender.MsgIsZero()) && ((*z).AssetReceiver.MsgIsZero()) && ((*z).AssetCloseTo.MsgIsZero()) } +// MarshalMsg implements msgp.Marshaler +func (z *CompactCertTxnFields) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 3 bits */ + if (*z).Cert.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).CertRound.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "cert" + o = append(o, 0xa4, 0x63, 0x65, 0x72, 0x74) + o, err = (*z).Cert.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Cert") + return + } + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "certrnd" + o = append(o, 0xa7, 0x63, 0x65, 0x72, 0x74, 0x72, 0x6e, 0x64) + o, err = (*z).CertRound.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CertRound") + return + } + } + } + return +} + +func (_ *CompactCertTxnFields) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*CompactCertTxnFields) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *CompactCertTxnFields) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).CertRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CertRound") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Cert.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Cert") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = CompactCertTxnFields{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "certrnd": + bts, err = (*z).CertRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CertRound") + return + } + case "cert": + bts, err = (*z).Cert.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cert") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *CompactCertTxnFields) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*CompactCertTxnFields) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *CompactCertTxnFields) Msgsize() (s int) { + s = 1 + 8 + (*z).CertRound.Msgsize() + 5 + (*z).Cert.Msgsize() + return +} + +// MsgIsZero returns whether this is a zero value +func (z *CompactCertTxnFields) MsgIsZero() bool { + return ((*z).CertRound.MsgIsZero()) && ((*z).Cert.MsgIsZero()) +} + // MarshalMsg implements msgp.Marshaler func (z *Header) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) @@ -3661,177 +3806,185 @@ func (z *SignedTxnWithAD) MsgIsZero() bool { func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // omitempty: check for empty values - zb0006Len := uint32(40) - var zb0006Mask uint64 /* 48 bits */ + zb0006Len := uint32(42) + var zb0006Mask uint64 /* 51 bits */ if (*z).AssetTransferTxnFields.AssetAmount == 0 { zb0006Len-- - zb0006Mask |= 0x100 + zb0006Mask |= 0x200 } if (*z).AssetTransferTxnFields.AssetCloseTo.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x200 + zb0006Mask |= 0x400 } if (*z).AssetFreezeTxnFields.AssetFrozen == false { zb0006Len-- - zb0006Mask |= 0x400 + zb0006Mask |= 0x800 } if (*z).PaymentTxnFields.Amount.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x800 + zb0006Mask |= 0x1000 } if len((*z).ApplicationCallTxnFields.ApplicationArgs) == 0 { zb0006Len-- - zb0006Mask |= 0x1000 + zb0006Mask |= 0x2000 } if (*z).ApplicationCallTxnFields.OnCompletion == 0 { zb0006Len-- - zb0006Mask |= 0x2000 + zb0006Mask |= 0x4000 } if len((*z).ApplicationCallTxnFields.ApprovalProgram) == 0 { zb0006Len-- - zb0006Mask |= 0x4000 + zb0006Mask |= 0x8000 } if (*z).AssetConfigTxnFields.AssetParams.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x8000 + zb0006Mask |= 0x10000 } if len((*z).ApplicationCallTxnFields.ForeignAssets) == 0 { zb0006Len-- - zb0006Mask |= 0x10000 + zb0006Mask |= 0x20000 } if len((*z).ApplicationCallTxnFields.Accounts) == 0 { zb0006Len-- - zb0006Mask |= 0x20000 + zb0006Mask |= 0x40000 } if len((*z).ApplicationCallTxnFields.ForeignApps) == 0 { zb0006Len-- - zb0006Mask |= 0x40000 + zb0006Mask |= 0x80000 } if (*z).ApplicationCallTxnFields.GlobalStateSchema.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x80000 + zb0006Mask |= 0x100000 } if (*z).ApplicationCallTxnFields.ApplicationID.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x100000 + zb0006Mask |= 0x200000 } if (*z).ApplicationCallTxnFields.LocalStateSchema.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x200000 + zb0006Mask |= 0x400000 } if len((*z).ApplicationCallTxnFields.ClearStateProgram) == 0 { zb0006Len-- - zb0006Mask |= 0x400000 + zb0006Mask |= 0x800000 } if (*z).AssetTransferTxnFields.AssetReceiver.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x800000 + zb0006Mask |= 0x1000000 } if (*z).AssetTransferTxnFields.AssetSender.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x1000000 + zb0006Mask |= 0x2000000 } if (*z).AssetConfigTxnFields.ConfigAsset.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x2000000 + zb0006Mask |= 0x4000000 + } + if (*z).CompactCertTxnFields.Cert.MsgIsZero() { + zb0006Len-- + zb0006Mask |= 0x8000000 + } + if (*z).CompactCertTxnFields.CertRound.MsgIsZero() { + zb0006Len-- + zb0006Mask |= 0x10000000 } if (*z).PaymentTxnFields.CloseRemainderTo.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x4000000 + zb0006Mask |= 0x20000000 } if (*z).AssetFreezeTxnFields.FreezeAccount.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x8000000 + zb0006Mask |= 0x40000000 } if (*z).AssetFreezeTxnFields.FreezeAsset.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x10000000 + zb0006Mask |= 0x80000000 } if (*z).Header.Fee.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x20000000 + zb0006Mask |= 0x100000000 } if (*z).Header.FirstValid.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x40000000 + zb0006Mask |= 0x200000000 } if (*z).Header.GenesisID == "" { zb0006Len-- - zb0006Mask |= 0x80000000 + zb0006Mask |= 0x400000000 } if (*z).Header.GenesisHash.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x100000000 + zb0006Mask |= 0x800000000 } if (*z).Header.Group.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x200000000 + zb0006Mask |= 0x1000000000 } if (*z).Header.LastValid.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x400000000 + zb0006Mask |= 0x2000000000 } if (*z).Header.Lease == ([32]byte{}) { zb0006Len-- - zb0006Mask |= 0x800000000 + zb0006Mask |= 0x4000000000 } if (*z).KeyregTxnFields.Nonparticipation == false { zb0006Len-- - zb0006Mask |= 0x1000000000 + zb0006Mask |= 0x8000000000 } if len((*z).Header.Note) == 0 { zb0006Len-- - zb0006Mask |= 0x2000000000 + zb0006Mask |= 0x10000000000 } if (*z).PaymentTxnFields.Receiver.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x4000000000 + zb0006Mask |= 0x20000000000 } if (*z).Header.RekeyTo.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x8000000000 + zb0006Mask |= 0x40000000000 } if (*z).KeyregTxnFields.SelectionPK.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x10000000000 + zb0006Mask |= 0x80000000000 } if (*z).Header.Sender.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x20000000000 + zb0006Mask |= 0x100000000000 } if (*z).Type.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x40000000000 + zb0006Mask |= 0x200000000000 } if (*z).KeyregTxnFields.VoteFirst.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x80000000000 + zb0006Mask |= 0x400000000000 } if (*z).KeyregTxnFields.VoteKeyDilution == 0 { zb0006Len-- - zb0006Mask |= 0x100000000000 + zb0006Mask |= 0x800000000000 } if (*z).KeyregTxnFields.VotePK.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x200000000000 + zb0006Mask |= 0x1000000000000 } if (*z).KeyregTxnFields.VoteLast.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x400000000000 + zb0006Mask |= 0x2000000000000 } if (*z).AssetTransferTxnFields.XferAsset.MsgIsZero() { zb0006Len-- - zb0006Mask |= 0x800000000000 + zb0006Mask |= 0x4000000000000 } // variable map header, size zb0006Len o = msgp.AppendMapHeader(o, zb0006Len) if zb0006Len != 0 { - if (zb0006Mask & 0x100) == 0 { // if not empty + if (zb0006Mask & 0x200) == 0 { // if not empty // string "aamt" o = append(o, 0xa4, 0x61, 0x61, 0x6d, 0x74) o = msgp.AppendUint64(o, (*z).AssetTransferTxnFields.AssetAmount) } - if (zb0006Mask & 0x200) == 0 { // if not empty + if (zb0006Mask & 0x400) == 0 { // if not empty // string "aclose" o = append(o, 0xa6, 0x61, 0x63, 0x6c, 0x6f, 0x73, 0x65) o, err = (*z).AssetTransferTxnFields.AssetCloseTo.MarshalMsg(o) @@ -3840,12 +3993,12 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x400) == 0 { // if not empty + if (zb0006Mask & 0x800) == 0 { // if not empty // string "afrz" o = append(o, 0xa4, 0x61, 0x66, 0x72, 0x7a) o = msgp.AppendBool(o, (*z).AssetFreezeTxnFields.AssetFrozen) } - if (zb0006Mask & 0x800) == 0 { // if not empty + if (zb0006Mask & 0x1000) == 0 { // if not empty // string "amt" o = append(o, 0xa3, 0x61, 0x6d, 0x74) o, err = (*z).PaymentTxnFields.Amount.MarshalMsg(o) @@ -3854,7 +4007,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x1000) == 0 { // if not empty + if (zb0006Mask & 0x2000) == 0 { // if not empty // string "apaa" o = append(o, 0xa4, 0x61, 0x70, 0x61, 0x61) if (*z).ApplicationCallTxnFields.ApplicationArgs == nil { @@ -3866,17 +4019,17 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.AppendBytes(o, (*z).ApplicationCallTxnFields.ApplicationArgs[zb0002]) } } - if (zb0006Mask & 0x2000) == 0 { // if not empty + if (zb0006Mask & 0x4000) == 0 { // if not empty // string "apan" o = append(o, 0xa4, 0x61, 0x70, 0x61, 0x6e) o = msgp.AppendUint64(o, uint64((*z).ApplicationCallTxnFields.OnCompletion)) } - if (zb0006Mask & 0x4000) == 0 { // if not empty + if (zb0006Mask & 0x8000) == 0 { // if not empty // string "apap" o = append(o, 0xa4, 0x61, 0x70, 0x61, 0x70) o = msgp.AppendBytes(o, (*z).ApplicationCallTxnFields.ApprovalProgram) } - if (zb0006Mask & 0x8000) == 0 { // if not empty + if (zb0006Mask & 0x10000) == 0 { // if not empty // string "apar" o = append(o, 0xa4, 0x61, 0x70, 0x61, 0x72) o, err = (*z).AssetConfigTxnFields.AssetParams.MarshalMsg(o) @@ -3885,7 +4038,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x10000) == 0 { // if not empty + if (zb0006Mask & 0x20000) == 0 { // if not empty // string "apas" o = append(o, 0xa4, 0x61, 0x70, 0x61, 0x73) if (*z).ApplicationCallTxnFields.ForeignAssets == nil { @@ -3901,7 +4054,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { } } } - if (zb0006Mask & 0x20000) == 0 { // if not empty + if (zb0006Mask & 0x40000) == 0 { // if not empty // string "apat" o = append(o, 0xa4, 0x61, 0x70, 0x61, 0x74) if (*z).ApplicationCallTxnFields.Accounts == nil { @@ -3917,7 +4070,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { } } } - if (zb0006Mask & 0x40000) == 0 { // if not empty + if (zb0006Mask & 0x80000) == 0 { // if not empty // string "apfa" o = append(o, 0xa4, 0x61, 0x70, 0x66, 0x61) if (*z).ApplicationCallTxnFields.ForeignApps == nil { @@ -3933,7 +4086,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { } } } - if (zb0006Mask & 0x80000) == 0 { // if not empty + if (zb0006Mask & 0x100000) == 0 { // if not empty // string "apgs" o = append(o, 0xa4, 0x61, 0x70, 0x67, 0x73) o, err = (*z).ApplicationCallTxnFields.GlobalStateSchema.MarshalMsg(o) @@ -3942,7 +4095,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x100000) == 0 { // if not empty + if (zb0006Mask & 0x200000) == 0 { // if not empty // string "apid" o = append(o, 0xa4, 0x61, 0x70, 0x69, 0x64) o, err = (*z).ApplicationCallTxnFields.ApplicationID.MarshalMsg(o) @@ -3951,7 +4104,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x200000) == 0 { // if not empty + if (zb0006Mask & 0x400000) == 0 { // if not empty // string "apls" o = append(o, 0xa4, 0x61, 0x70, 0x6c, 0x73) o, err = (*z).ApplicationCallTxnFields.LocalStateSchema.MarshalMsg(o) @@ -3960,12 +4113,12 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x400000) == 0 { // if not empty + if (zb0006Mask & 0x800000) == 0 { // if not empty // string "apsu" o = append(o, 0xa4, 0x61, 0x70, 0x73, 0x75) o = msgp.AppendBytes(o, (*z).ApplicationCallTxnFields.ClearStateProgram) } - if (zb0006Mask & 0x800000) == 0 { // if not empty + if (zb0006Mask & 0x1000000) == 0 { // if not empty // string "arcv" o = append(o, 0xa4, 0x61, 0x72, 0x63, 0x76) o, err = (*z).AssetTransferTxnFields.AssetReceiver.MarshalMsg(o) @@ -3974,7 +4127,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x1000000) == 0 { // if not empty + if (zb0006Mask & 0x2000000) == 0 { // if not empty // string "asnd" o = append(o, 0xa4, 0x61, 0x73, 0x6e, 0x64) o, err = (*z).AssetTransferTxnFields.AssetSender.MarshalMsg(o) @@ -3983,7 +4136,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x2000000) == 0 { // if not empty + if (zb0006Mask & 0x4000000) == 0 { // if not empty // string "caid" o = append(o, 0xa4, 0x63, 0x61, 0x69, 0x64) o, err = (*z).AssetConfigTxnFields.ConfigAsset.MarshalMsg(o) @@ -3992,7 +4145,25 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x4000000) == 0 { // if not empty + if (zb0006Mask & 0x8000000) == 0 { // if not empty + // string "cert" + o = append(o, 0xa4, 0x63, 0x65, 0x72, 0x74) + o, err = (*z).CompactCertTxnFields.Cert.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Cert") + return + } + } + if (zb0006Mask & 0x10000000) == 0 { // if not empty + // string "certrnd" + o = append(o, 0xa7, 0x63, 0x65, 0x72, 0x74, 0x72, 0x6e, 0x64) + o, err = (*z).CompactCertTxnFields.CertRound.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "CertRound") + return + } + } + if (zb0006Mask & 0x20000000) == 0 { // if not empty // string "close" o = append(o, 0xa5, 0x63, 0x6c, 0x6f, 0x73, 0x65) o, err = (*z).PaymentTxnFields.CloseRemainderTo.MarshalMsg(o) @@ -4001,7 +4172,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x8000000) == 0 { // if not empty + if (zb0006Mask & 0x40000000) == 0 { // if not empty // string "fadd" o = append(o, 0xa4, 0x66, 0x61, 0x64, 0x64) o, err = (*z).AssetFreezeTxnFields.FreezeAccount.MarshalMsg(o) @@ -4010,7 +4181,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x10000000) == 0 { // if not empty + if (zb0006Mask & 0x80000000) == 0 { // if not empty // string "faid" o = append(o, 0xa4, 0x66, 0x61, 0x69, 0x64) o, err = (*z).AssetFreezeTxnFields.FreezeAsset.MarshalMsg(o) @@ -4019,7 +4190,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x20000000) == 0 { // if not empty + if (zb0006Mask & 0x100000000) == 0 { // if not empty // string "fee" o = append(o, 0xa3, 0x66, 0x65, 0x65) o, err = (*z).Header.Fee.MarshalMsg(o) @@ -4028,7 +4199,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x40000000) == 0 { // if not empty + if (zb0006Mask & 0x200000000) == 0 { // if not empty // string "fv" o = append(o, 0xa2, 0x66, 0x76) o, err = (*z).Header.FirstValid.MarshalMsg(o) @@ -4037,12 +4208,12 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x80000000) == 0 { // if not empty + if (zb0006Mask & 0x400000000) == 0 { // if not empty // string "gen" o = append(o, 0xa3, 0x67, 0x65, 0x6e) o = msgp.AppendString(o, (*z).Header.GenesisID) } - if (zb0006Mask & 0x100000000) == 0 { // if not empty + if (zb0006Mask & 0x800000000) == 0 { // if not empty // string "gh" o = append(o, 0xa2, 0x67, 0x68) o, err = (*z).Header.GenesisHash.MarshalMsg(o) @@ -4051,7 +4222,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x200000000) == 0 { // if not empty + if (zb0006Mask & 0x1000000000) == 0 { // if not empty // string "grp" o = append(o, 0xa3, 0x67, 0x72, 0x70) o, err = (*z).Header.Group.MarshalMsg(o) @@ -4060,7 +4231,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x400000000) == 0 { // if not empty + if (zb0006Mask & 0x2000000000) == 0 { // if not empty // string "lv" o = append(o, 0xa2, 0x6c, 0x76) o, err = (*z).Header.LastValid.MarshalMsg(o) @@ -4069,22 +4240,22 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x800000000) == 0 { // if not empty + if (zb0006Mask & 0x4000000000) == 0 { // if not empty // string "lx" o = append(o, 0xa2, 0x6c, 0x78) o = msgp.AppendBytes(o, ((*z).Header.Lease)[:]) } - if (zb0006Mask & 0x1000000000) == 0 { // if not empty + if (zb0006Mask & 0x8000000000) == 0 { // if not empty // string "nonpart" o = append(o, 0xa7, 0x6e, 0x6f, 0x6e, 0x70, 0x61, 0x72, 0x74) o = msgp.AppendBool(o, (*z).KeyregTxnFields.Nonparticipation) } - if (zb0006Mask & 0x2000000000) == 0 { // if not empty + if (zb0006Mask & 0x10000000000) == 0 { // if not empty // string "note" o = append(o, 0xa4, 0x6e, 0x6f, 0x74, 0x65) o = msgp.AppendBytes(o, (*z).Header.Note) } - if (zb0006Mask & 0x4000000000) == 0 { // if not empty + if (zb0006Mask & 0x20000000000) == 0 { // if not empty // string "rcv" o = append(o, 0xa3, 0x72, 0x63, 0x76) o, err = (*z).PaymentTxnFields.Receiver.MarshalMsg(o) @@ -4093,7 +4264,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x8000000000) == 0 { // if not empty + if (zb0006Mask & 0x40000000000) == 0 { // if not empty // string "rekey" o = append(o, 0xa5, 0x72, 0x65, 0x6b, 0x65, 0x79) o, err = (*z).Header.RekeyTo.MarshalMsg(o) @@ -4102,7 +4273,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x10000000000) == 0 { // if not empty + if (zb0006Mask & 0x80000000000) == 0 { // if not empty // string "selkey" o = append(o, 0xa6, 0x73, 0x65, 0x6c, 0x6b, 0x65, 0x79) o, err = (*z).KeyregTxnFields.SelectionPK.MarshalMsg(o) @@ -4111,7 +4282,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x20000000000) == 0 { // if not empty + if (zb0006Mask & 0x100000000000) == 0 { // if not empty // string "snd" o = append(o, 0xa3, 0x73, 0x6e, 0x64) o, err = (*z).Header.Sender.MarshalMsg(o) @@ -4120,7 +4291,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x40000000000) == 0 { // if not empty + if (zb0006Mask & 0x200000000000) == 0 { // if not empty // string "type" o = append(o, 0xa4, 0x74, 0x79, 0x70, 0x65) o, err = (*z).Type.MarshalMsg(o) @@ -4129,7 +4300,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x80000000000) == 0 { // if not empty + if (zb0006Mask & 0x400000000000) == 0 { // if not empty // string "votefst" o = append(o, 0xa7, 0x76, 0x6f, 0x74, 0x65, 0x66, 0x73, 0x74) o, err = (*z).KeyregTxnFields.VoteFirst.MarshalMsg(o) @@ -4138,12 +4309,12 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x100000000000) == 0 { // if not empty + if (zb0006Mask & 0x800000000000) == 0 { // if not empty // string "votekd" o = append(o, 0xa6, 0x76, 0x6f, 0x74, 0x65, 0x6b, 0x64) o = msgp.AppendUint64(o, (*z).KeyregTxnFields.VoteKeyDilution) } - if (zb0006Mask & 0x200000000000) == 0 { // if not empty + if (zb0006Mask & 0x1000000000000) == 0 { // if not empty // string "votekey" o = append(o, 0xa7, 0x76, 0x6f, 0x74, 0x65, 0x6b, 0x65, 0x79) o, err = (*z).KeyregTxnFields.VotePK.MarshalMsg(o) @@ -4152,7 +4323,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x400000000000) == 0 { // if not empty + if (zb0006Mask & 0x2000000000000) == 0 { // if not empty // string "votelst" o = append(o, 0xa7, 0x76, 0x6f, 0x74, 0x65, 0x6c, 0x73, 0x74) o, err = (*z).KeyregTxnFields.VoteLast.MarshalMsg(o) @@ -4161,7 +4332,7 @@ func (z *Transaction) MarshalMsg(b []byte) (o []byte, err error) { return } } - if (zb0006Mask & 0x800000000000) == 0 { // if not empty + if (zb0006Mask & 0x4000000000000) == 0 { // if not empty // string "xaid" o = append(o, 0xa4, 0x78, 0x61, 0x69, 0x64) o, err = (*z).AssetTransferTxnFields.XferAsset.MarshalMsg(o) @@ -4630,6 +4801,22 @@ func (z *Transaction) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + if zb0006 > 0 { + zb0006-- + bts, err = (*z).CompactCertTxnFields.CertRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CertRound") + return + } + } + if zb0006 > 0 { + zb0006-- + bts, err = (*z).CompactCertTxnFields.Cert.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Cert") + return + } + } if zb0006 > 0 { err = msgp.ErrTooManyArrayFields(zb0006) if err != nil { @@ -5011,6 +5198,18 @@ func (z *Transaction) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "ClearStateProgram") return } + case "certrnd": + bts, err = (*z).CompactCertTxnFields.CertRound.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "CertRound") + return + } + case "cert": + bts, err = (*z).CompactCertTxnFields.Cert.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cert") + return + } default: err = msgp.ErrNoField(string(field)) if err != nil { @@ -5047,13 +5246,13 @@ func (z *Transaction) Msgsize() (s int) { for zb0005 := range (*z).ApplicationCallTxnFields.ForeignAssets { s += (*z).ApplicationCallTxnFields.ForeignAssets[zb0005].Msgsize() } - s += 5 + (*z).ApplicationCallTxnFields.LocalStateSchema.Msgsize() + 5 + (*z).ApplicationCallTxnFields.GlobalStateSchema.Msgsize() + 5 + msgp.BytesPrefixSize + len((*z).ApplicationCallTxnFields.ApprovalProgram) + 5 + msgp.BytesPrefixSize + len((*z).ApplicationCallTxnFields.ClearStateProgram) + s += 5 + (*z).ApplicationCallTxnFields.LocalStateSchema.Msgsize() + 5 + (*z).ApplicationCallTxnFields.GlobalStateSchema.Msgsize() + 5 + msgp.BytesPrefixSize + len((*z).ApplicationCallTxnFields.ApprovalProgram) + 5 + msgp.BytesPrefixSize + len((*z).ApplicationCallTxnFields.ClearStateProgram) + 8 + (*z).CompactCertTxnFields.CertRound.Msgsize() + 5 + (*z).CompactCertTxnFields.Cert.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *Transaction) MsgIsZero() bool { - return ((*z).Type.MsgIsZero()) && ((*z).Header.Sender.MsgIsZero()) && ((*z).Header.Fee.MsgIsZero()) && ((*z).Header.FirstValid.MsgIsZero()) && ((*z).Header.LastValid.MsgIsZero()) && (len((*z).Header.Note) == 0) && ((*z).Header.GenesisID == "") && ((*z).Header.GenesisHash.MsgIsZero()) && ((*z).Header.Group.MsgIsZero()) && ((*z).Header.Lease == ([32]byte{})) && ((*z).Header.RekeyTo.MsgIsZero()) && ((*z).KeyregTxnFields.VotePK.MsgIsZero()) && ((*z).KeyregTxnFields.SelectionPK.MsgIsZero()) && ((*z).KeyregTxnFields.VoteFirst.MsgIsZero()) && ((*z).KeyregTxnFields.VoteLast.MsgIsZero()) && ((*z).KeyregTxnFields.VoteKeyDilution == 0) && ((*z).KeyregTxnFields.Nonparticipation == false) && ((*z).PaymentTxnFields.Receiver.MsgIsZero()) && ((*z).PaymentTxnFields.Amount.MsgIsZero()) && ((*z).PaymentTxnFields.CloseRemainderTo.MsgIsZero()) && ((*z).AssetConfigTxnFields.ConfigAsset.MsgIsZero()) && ((*z).AssetConfigTxnFields.AssetParams.MsgIsZero()) && ((*z).AssetTransferTxnFields.XferAsset.MsgIsZero()) && ((*z).AssetTransferTxnFields.AssetAmount == 0) && ((*z).AssetTransferTxnFields.AssetSender.MsgIsZero()) && ((*z).AssetTransferTxnFields.AssetReceiver.MsgIsZero()) && ((*z).AssetTransferTxnFields.AssetCloseTo.MsgIsZero()) && ((*z).AssetFreezeTxnFields.FreezeAccount.MsgIsZero()) && ((*z).AssetFreezeTxnFields.FreezeAsset.MsgIsZero()) && ((*z).AssetFreezeTxnFields.AssetFrozen == false) && ((*z).ApplicationCallTxnFields.ApplicationID.MsgIsZero()) && ((*z).ApplicationCallTxnFields.OnCompletion == 0) && (len((*z).ApplicationCallTxnFields.ApplicationArgs) == 0) && (len((*z).ApplicationCallTxnFields.Accounts) == 0) && (len((*z).ApplicationCallTxnFields.ForeignApps) == 0) && (len((*z).ApplicationCallTxnFields.ForeignAssets) == 0) && ((*z).ApplicationCallTxnFields.LocalStateSchema.MsgIsZero()) && ((*z).ApplicationCallTxnFields.GlobalStateSchema.MsgIsZero()) && (len((*z).ApplicationCallTxnFields.ApprovalProgram) == 0) && (len((*z).ApplicationCallTxnFields.ClearStateProgram) == 0) + return ((*z).Type.MsgIsZero()) && ((*z).Header.Sender.MsgIsZero()) && ((*z).Header.Fee.MsgIsZero()) && ((*z).Header.FirstValid.MsgIsZero()) && ((*z).Header.LastValid.MsgIsZero()) && (len((*z).Header.Note) == 0) && ((*z).Header.GenesisID == "") && ((*z).Header.GenesisHash.MsgIsZero()) && ((*z).Header.Group.MsgIsZero()) && ((*z).Header.Lease == ([32]byte{})) && ((*z).Header.RekeyTo.MsgIsZero()) && ((*z).KeyregTxnFields.VotePK.MsgIsZero()) && ((*z).KeyregTxnFields.SelectionPK.MsgIsZero()) && ((*z).KeyregTxnFields.VoteFirst.MsgIsZero()) && ((*z).KeyregTxnFields.VoteLast.MsgIsZero()) && ((*z).KeyregTxnFields.VoteKeyDilution == 0) && ((*z).KeyregTxnFields.Nonparticipation == false) && ((*z).PaymentTxnFields.Receiver.MsgIsZero()) && ((*z).PaymentTxnFields.Amount.MsgIsZero()) && ((*z).PaymentTxnFields.CloseRemainderTo.MsgIsZero()) && ((*z).AssetConfigTxnFields.ConfigAsset.MsgIsZero()) && ((*z).AssetConfigTxnFields.AssetParams.MsgIsZero()) && ((*z).AssetTransferTxnFields.XferAsset.MsgIsZero()) && ((*z).AssetTransferTxnFields.AssetAmount == 0) && ((*z).AssetTransferTxnFields.AssetSender.MsgIsZero()) && ((*z).AssetTransferTxnFields.AssetReceiver.MsgIsZero()) && ((*z).AssetTransferTxnFields.AssetCloseTo.MsgIsZero()) && ((*z).AssetFreezeTxnFields.FreezeAccount.MsgIsZero()) && ((*z).AssetFreezeTxnFields.FreezeAsset.MsgIsZero()) && ((*z).AssetFreezeTxnFields.AssetFrozen == false) && ((*z).ApplicationCallTxnFields.ApplicationID.MsgIsZero()) && ((*z).ApplicationCallTxnFields.OnCompletion == 0) && (len((*z).ApplicationCallTxnFields.ApplicationArgs) == 0) && (len((*z).ApplicationCallTxnFields.Accounts) == 0) && (len((*z).ApplicationCallTxnFields.ForeignApps) == 0) && (len((*z).ApplicationCallTxnFields.ForeignAssets) == 0) && ((*z).ApplicationCallTxnFields.LocalStateSchema.MsgIsZero()) && ((*z).ApplicationCallTxnFields.GlobalStateSchema.MsgIsZero()) && (len((*z).ApplicationCallTxnFields.ApprovalProgram) == 0) && (len((*z).ApplicationCallTxnFields.ClearStateProgram) == 0) && ((*z).CompactCertTxnFields.CertRound.MsgIsZero()) && ((*z).CompactCertTxnFields.Cert.MsgIsZero()) } // MarshalMsg implements msgp.Marshaler diff --git a/data/transactions/msgp_gen_test.go b/data/transactions/msgp_gen_test.go index 380695b1ff..cfd2193e5c 100644 --- a/data/transactions/msgp_gen_test.go +++ b/data/transactions/msgp_gen_test.go @@ -321,6 +321,68 @@ func BenchmarkUnmarshalAssetTransferTxnFields(b *testing.B) { } } +func TestMarshalUnmarshalCompactCertTxnFields(t *testing.T) { + v := CompactCertTxnFields{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingCompactCertTxnFields(t *testing.T) { + protocol.RunEncodingTest(t, &CompactCertTxnFields{}) +} + +func BenchmarkMarshalMsgCompactCertTxnFields(b *testing.B) { + v := CompactCertTxnFields{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgCompactCertTxnFields(b *testing.B) { + v := CompactCertTxnFields{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalCompactCertTxnFields(b *testing.B) { + v := CompactCertTxnFields{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + func TestMarshalUnmarshalHeader(t *testing.T) { v := Header{} bts, err := v.MarshalMsg(nil) diff --git a/data/transactions/transaction.go b/data/transactions/transaction.go index 2ed0ae5d19..67a4d45fc9 100644 --- a/data/transactions/transaction.go +++ b/data/transactions/transaction.go @@ -94,6 +94,7 @@ type Transaction struct { AssetTransferTxnFields AssetFreezeTxnFields ApplicationCallTxnFields + CompactCertTxnFields } // ApplyData contains information about the transaction's execution. @@ -349,6 +350,35 @@ func (tx Transaction) WellFormed(spec SpecialAddresses, proto config.ConsensusPa if tx.GlobalStateSchema.NumEntries() > proto.MaxGlobalSchemaEntries { return fmt.Errorf("tx.GlobalStateSchema too large, max number of keys is %d", proto.MaxGlobalSchemaEntries) } + + case protocol.CompactCertTx: + if proto.CompactCertRounds == 0 { + return fmt.Errorf("compact certs not supported") + } + + // This is a placeholder transaction used to store compact certs + // on the ledger, and ensure they are broadly available. Most of + // the fields must be empty. It must be issued from a special + // sender address. + if tx.Sender != CompactCertSender { + return fmt.Errorf("sender must be the compact-cert sender") + } + if !tx.Fee.IsZero() { + return fmt.Errorf("fee must be zero") + } + if len(tx.Note) != 0 { + return fmt.Errorf("note must be empty") + } + if !tx.Group.IsZero() { + return fmt.Errorf("group must be zero") + } + if !tx.RekeyTo.IsZero() { + return fmt.Errorf("rekey must be zero") + } + if tx.Lease != [32]byte{} { + return fmt.Errorf("lease must be zero") + } + default: return fmt.Errorf("unknown tx type %v", tx.Type) } @@ -378,6 +408,10 @@ func (tx Transaction) WellFormed(spec SpecialAddresses, proto config.ConsensusPa nonZeroFields[protocol.ApplicationCallTx] = true } + if !tx.CompactCertTxnFields.Empty() { + nonZeroFields[protocol.CompactCertTx] = true + } + for t, nonZero := range nonZeroFields { if nonZero && t != tx.Type { return fmt.Errorf("transaction of type %v has non-zero fields for type %v", tx.Type, t) @@ -385,7 +419,11 @@ func (tx Transaction) WellFormed(spec SpecialAddresses, proto config.ConsensusPa } if tx.Fee.LessThan(basics.MicroAlgos{Raw: proto.MinTxnFee}) { - return makeMinFeeErrorf("transaction had fee %v, which is less than the minimum %v", tx.Fee, proto.MinTxnFee) + if tx.Type == protocol.CompactCertTx { + // Zero fee allowed for compact cert txn. + } else { + return makeMinFeeErrorf("transaction had fee %v, which is less than the minimum %v", tx.Fee, proto.MinTxnFee) + } } if tx.LastValid < tx.FirstValid { return fmt.Errorf("transaction invalid range (%v--%v)", tx.FirstValid, tx.LastValid) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index eb433a089d..d0ea139426 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -173,6 +173,15 @@ func stxnVerifyCore(s *transactions.SignedTxn, ctx *Context) error { hasLogicSig = true } if numSigs == 0 { + // Special case: special sender address can issue special transaction + // types (compact cert txn) without any signature. The well-formed + // check ensures that this transaction cannot pay any fee, and + // cannot have any other interesting fields, except for the compact + // cert payload. + if s.Txn.Sender == transactions.CompactCertSender && s.Txn.Type == protocol.CompactCertTx { + return nil + } + return errors.New("signedtxn has no sig") } if numSigs > 1 { diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index cac466aab6..1b07f1e4ff 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -121,6 +121,80 @@ func TestTxnValidationEncodeDecode(t *testing.T) { } } +func TestTxnValidationEmptySig(t *testing.T) { + _, signed, _, _ := generateTestObjects(100, 50) + + for _, txn := range signed { + if Txn(&txn, Context{Params: Params{CurrSpecAddrs: spec, CurrProto: protocol.ConsensusCurrentVersion}}) != nil { + t.Errorf("signed transaction %#v did not verify", txn) + } + + txn.Sig = crypto.Signature{} + txn.Msig = crypto.MultisigSig{} + txn.Lsig = transactions.LogicSig{} + if Txn(&txn, Context{Params: Params{CurrSpecAddrs: spec, CurrProto: protocol.ConsensusCurrentVersion}}) == nil { + t.Errorf("transaction %#v verified without sig", txn) + } + } +} + +const ccProto = protocol.ConsensusVersion("test-compact-cert-enabled") + +func TestTxnValidationCompactCert(t *testing.T) { + proto := config.Consensus[protocol.ConsensusCurrentVersion] + proto.CompactCertRounds = 128 + config.Consensus[ccProto] = proto + + stxn := transactions.SignedTxn{ + Txn: transactions.Transaction{ + Type: protocol.CompactCertTx, + Header: transactions.Header{ + Sender: transactions.CompactCertSender, + FirstValid: 0, + LastValid: 10, + }, + }, + } + + err := Txn(&stxn, Context{Params: Params{CurrSpecAddrs: spec, CurrProto: ccProto}}) + require.NoError(t, err, "compact cert txn %#v did not verify", stxn) + + stxn2 := stxn + stxn2.Txn.Type = protocol.PaymentTx + stxn2.Txn.Header.Fee = basics.MicroAlgos{Raw: proto.MinTxnFee} + err = Txn(&stxn2, Context{Params: Params{CurrSpecAddrs: spec, CurrProto: ccProto}}) + require.Error(t, err, "payment txn %#v verified from CompactCertSender", stxn2) + + secret := keypair() + stxn2 = stxn + stxn2.Txn.Header.Sender = basics.Address(secret.SignatureVerifier) + stxn2.Txn.Header.Fee = basics.MicroAlgos{Raw: proto.MinTxnFee} + stxn2 = stxn2.Txn.Sign(secret) + err = Txn(&stxn2, Context{Params: Params{CurrSpecAddrs: spec, CurrProto: ccProto}}) + require.Error(t, err, "compact cert txn %#v verified from non-CompactCertSender", stxn2) + + // Compact cert txns are not allowed to have non-zero values for many fields + stxn2 = stxn + stxn2.Txn.Header.Fee = basics.MicroAlgos{Raw: proto.MinTxnFee} + err = Txn(&stxn2, Context{Params: Params{CurrSpecAddrs: spec, CurrProto: ccProto}}) + require.Error(t, err, "compact cert txn %#v verified", stxn2) + + stxn2 = stxn + stxn2.Txn.Header.Note = []byte{'A'} + err = Txn(&stxn2, Context{Params: Params{CurrSpecAddrs: spec, CurrProto: ccProto}}) + require.Error(t, err, "compact cert txn %#v verified", stxn2) + + stxn2 = stxn + stxn2.Txn.Lease[0] = 1 + err = Txn(&stxn2, Context{Params: Params{CurrSpecAddrs: spec, CurrProto: ccProto}}) + require.Error(t, err, "compact cert txn %#v verified", stxn2) + + stxn2 = stxn + stxn2.Txn.RekeyTo = basics.Address(secret.SignatureVerifier) + err = Txn(&stxn2, Context{Params: Params{CurrSpecAddrs: spec, CurrProto: ccProto}}) + require.Error(t, err, "compact cert txn %#v verified", stxn2) +} + func TestDecodeNil(t *testing.T) { // This is a regression test for improper decoding of a nil SignedTxn. // This is a subtle case because decoding a msgpack nil does not run diff --git a/ledger/compactcert.go b/ledger/compactcert.go new file mode 100644 index 0000000000..e552d50dc9 --- /dev/null +++ b/ledger/compactcert.go @@ -0,0 +1,159 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package ledger + +import ( + "fmt" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto/compactcert" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/logging" +) + +// AcceptableCompactCertWeight computes the acceptable signed weight +// of a compact cert if it were to appear in a transaction with a +// particular firstValid round. Earlier rounds require a smaller cert. +// votersHdr specifies the block that contains the Merkle commitment of +// the voters for this compact cert (and thus the compact cert is for +// votersHdr.Round() + CompactCertRounds). +func AcceptableCompactCertWeight(votersHdr bookkeeping.BlockHeader, firstValid basics.Round) uint64 { + proto := config.Consensus[votersHdr.CurrentProtocol] + certRound := votersHdr.Round + basics.Round(proto.CompactCertRounds) + total := votersHdr.CompactCertVotersTotal + + // The acceptable weight depends on the elapsed time (in rounds) + // from the block we are trying to construct a certificate for. + // Start by subtracting the round number of the block being certified. + // If that round hasn't even passed yet, require 100% votes in cert. + offset := firstValid.SubSaturate(certRound) + if offset == 0 { + return total.ToUint64() + } + + // During the first proto.CompactCertRound/2 + 1 + 1 blocks, the + // signatures are still being broadcast, so, continue requiring + // 100% votes. + // + // The first +1 comes from CompactCertWorker.broadcastSigs: it only + // broadcasts signatures for round R starting with round R+1, to + // ensure nodes have the block for round R already in their ledger, + // to check the sig. + // + // The second +1 comes from the fact that, if we are checking this + // acceptable weight to decide whether to allow this transaction in + // a block, the transaction was sent out one round ago. + offset = offset.SubSaturate(basics.Round(proto.CompactCertRounds/2 + 2)) + if offset == 0 { + return total.ToUint64() + } + + // In the next proto.CompactCertRounds/2 blocks, linearly scale + // the acceptable weight from 100% to 0%. If we are outside of + // that window, accept any weight. + if offset >= basics.Round(proto.CompactCertRounds/2) { + return 0 + } + + w, overflowed := basics.Muldiv(total.ToUint64(), proto.CompactCertRounds/2-uint64(offset), proto.CompactCertRounds/2) + if overflowed { + // Shouldn't happen, but a safe fallback is to accept a larger cert. + logging.Base().Warnf("AcceptableCompactCertWeight(%d, %d, %d, %d) overflow", + total, proto.CompactCertRounds, certRound, firstValid) + return 0 + } + + return w +} + +// CompactCertParams computes the parameters for building or verifying +// a compact cert for block hdr, using voters from block votersHdr. +func CompactCertParams(votersHdr bookkeeping.BlockHeader, hdr bookkeeping.BlockHeader) (res compactcert.Params, err error) { + proto := config.Consensus[votersHdr.CurrentProtocol] + + if proto.CompactCertRounds == 0 { + err = fmt.Errorf("compact certs not enabled") + return + } + + if votersHdr.Round%basics.Round(proto.CompactCertRounds) != 0 { + err = fmt.Errorf("votersHdr %d not a multiple of %d", + votersHdr.Round, proto.CompactCertRounds) + return + } + + if hdr.Round != votersHdr.Round+basics.Round(proto.CompactCertRounds) { + err = fmt.Errorf("certifying block %d not %d ahead of voters %d", + hdr.Round, proto.CompactCertRounds, votersHdr.Round) + return + } + + totalWeight := votersHdr.CompactCertVotersTotal.ToUint64() + provenWeight, overflowed := basics.Muldiv(totalWeight, proto.CompactCertWeightThreshold, 100) + if overflowed { + err = fmt.Errorf("overflow computing provenWeight[%d]: %d * %d / 100", + hdr.Round, totalWeight, proto.CompactCertWeightThreshold) + return + } + + res = compactcert.Params{ + Msg: hdr, + ProvenWeight: provenWeight, + SigRound: hdr.Round + 1, + SecKQ: proto.CompactCertSecKQ, + } + return +} + +// validateCompactCert checks that a compact cert is valid. +func validateCompactCert(certHdr bookkeeping.BlockHeader, cert compactcert.Cert, votersHdr bookkeeping.BlockHeader, lastCertRnd basics.Round, atRound basics.Round) error { + proto := config.Consensus[certHdr.CurrentProtocol] + + if proto.CompactCertRounds == 0 { + return fmt.Errorf("compact certs not enabled: rounds = %d", proto.CompactCertRounds) + } + + if certHdr.Round%basics.Round(proto.CompactCertRounds) != 0 { + return fmt.Errorf("cert at %d for non-multiple of %d", certHdr.Round, proto.CompactCertRounds) + } + + votersRound := certHdr.Round.SubSaturate(basics.Round(proto.CompactCertRounds)) + if votersRound != votersHdr.Round { + return fmt.Errorf("new cert is for %d (voters %d), but votersHdr from %d", + certHdr.Round, votersRound, votersHdr.Round) + } + + if lastCertRnd != 0 && lastCertRnd != votersRound { + return fmt.Errorf("last cert from %d, but new cert is for %d (voters %d)", + lastCertRnd, certHdr.Round, votersRound) + } + + acceptableWeight := AcceptableCompactCertWeight(votersHdr, atRound) + if cert.SignedWeight < acceptableWeight { + return fmt.Errorf("insufficient weight at %d: %d < %d", + atRound, cert.SignedWeight, acceptableWeight) + } + + ccParams, err := CompactCertParams(votersHdr, certHdr) + if err != nil { + return err + } + + verif := compactcert.MkVerifier(ccParams, votersHdr.CompactCertVoters) + return verif.Verify(&cert) +} diff --git a/ledger/cow.go b/ledger/cow.go index 6d1e7bd661..10b4c69166 100644 --- a/ledger/cow.go +++ b/ledger/cow.go @@ -37,6 +37,8 @@ type roundCowParent interface { isDup(basics.Round, basics.Round, transactions.Txid, txlease) (bool, error) txnCounter() uint64 getCreator(cidx basics.CreatableIndex, ctype basics.CreatableType) (basics.Address, bool, error) + compactCertLast() basics.Round + blockHdr(rnd basics.Round) (bookkeeping.BlockHeader, error) } type roundCowState struct { @@ -62,6 +64,10 @@ type StateDelta struct { // new block header; read-only hdr *bookkeeping.BlockHeader + + // last round for which we have seen a compact cert. + // zero if no compact cert seen. + compactCertSeen basics.Round } func makeRoundCowState(b roundCowParent, hdr bookkeeping.BlockHeader) *roundCowState { @@ -123,6 +129,17 @@ func (cb *roundCowState) txnCounter() uint64 { return cb.lookupParent.txnCounter() + uint64(len(cb.mods.Txids)) } +func (cb *roundCowState) compactCertLast() basics.Round { + if cb.mods.compactCertSeen != 0 { + return cb.mods.compactCertSeen + } + return cb.lookupParent.compactCertLast() +} + +func (cb *roundCowState) blockHdr(r basics.Round) (bookkeeping.BlockHeader, error) { + return cb.lookupParent.blockHdr(r) +} + func (cb *roundCowState) put(addr basics.Address, old basics.AccountData, new basics.AccountData, newCreatable *basics.CreatableLocator, deletedCreatable *basics.CreatableLocator) { prev, present := cb.mods.accts[addr] if present { @@ -153,6 +170,10 @@ func (cb *roundCowState) addTx(txn transactions.Transaction, txid transactions.T cb.mods.txleases[txlease{sender: txn.Sender, lease: txn.Lease}] = txn.LastValid } +func (cb *roundCowState) sawCompactCert(rnd basics.Round) { + cb.mods.compactCertSeen = rnd +} + func (cb *roundCowState) child() *roundCowState { return &roundCowState{ lookupParent: cb, @@ -190,6 +211,7 @@ func (cb *roundCowState) commitToParent() { for cidx, delta := range cb.mods.creatables { cb.commitParent.mods.creatables[cidx] = delta } + cb.commitParent.mods.compactCertSeen = cb.mods.compactCertSeen } func (cb *roundCowState) modifiedAccounts() []basics.Address { diff --git a/ledger/cow_test.go b/ledger/cow_test.go index 6b871e45a6..4bb21c545f 100644 --- a/ledger/cow_test.go +++ b/ledger/cow_test.go @@ -54,6 +54,14 @@ func (ml *mockLedger) txnCounter() uint64 { return 0 } +func (ml *mockLedger) compactCertLast() basics.Round { + return 0 +} + +func (ml *mockLedger) blockHdr(_ basics.Round) (bookkeeping.BlockHeader, error) { + return bookkeeping.BlockHeader{}, nil +} + func checkCow(t *testing.T, cow *roundCowState, accts map[basics.Address]basics.AccountData) { for addr, data := range accts { d, err := cow.lookup(addr) diff --git a/ledger/eval.go b/ledger/eval.go index 6266f96537..2d01e36066 100644 --- a/ledger/eval.go +++ b/ledger/eval.go @@ -23,6 +23,7 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/compactcert" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/committee" @@ -59,6 +60,9 @@ type roundCowBase struct { // TxnCounter from previous block header. txnCount uint64 + // CompactCertLastRound from previous block header. + compactCertSeen basics.Round + // The current protocol consensus params. proto config.ConsensusParams } @@ -79,6 +83,14 @@ func (x *roundCowBase) txnCounter() uint64 { return x.txnCount } +func (x *roundCowBase) compactCertLast() basics.Round { + return x.compactCertSeen +} + +func (x *roundCowBase) blockHdr(r basics.Round) (bookkeeping.BlockHeader, error) { + return x.l.BlockHdr(r) +} + // wrappers for roundCowState to satisfy the (current) apply.Balances interface func (cs *roundCowState) Get(addr basics.Address, withPendingRewards bool) (basics.BalanceRecord, error) { acctdata, err := cs.lookup(addr) @@ -161,6 +173,30 @@ func (cs *roundCowState) ConsensusParams() config.ConsensusParams { return cs.proto } +func (cs *roundCowState) compactCert(certRnd basics.Round, cert compactcert.Cert, atRound basics.Round) error { + lastCertRnd := cs.compactCertLast() + + certHdr, err := cs.blockHdr(certRnd) + if err != nil { + return err + } + + proto := config.Consensus[certHdr.CurrentProtocol] + votersRnd := certRnd.SubSaturate(basics.Round(proto.CompactCertRounds)) + votersHdr, err := cs.blockHdr(votersRnd) + if err != nil { + return err + } + + err = validateCompactCert(certHdr, cert, votersHdr, lastCertRnd, atRound) + if err != nil { + return err + } + + cs.sawCompactCert(certRnd) + return nil +} + // BlockEvaluator represents an in-progress evaluation of a block // against the ledger. type BlockEvaluator struct { @@ -242,7 +278,7 @@ func startEvaluator(l ledgerForEvaluator, hdr bookkeeping.BlockHeader, paysetHin } base.txnCount = eval.prevHeader.TxnCounter - + base.compactCertSeen = eval.prevHeader.CompactCertLastRound prevProto, ok = config.Consensus[eval.prevHeader.CurrentProtocol] if !ok { return nil, protocol.Error(eval.prevHeader.CurrentProtocol) @@ -658,11 +694,11 @@ func (eval *BlockEvaluator) transaction(txn transactions.SignedTxn, appEval *app // completely zero, which means the account will be deleted.) rewardlvl := cow.rewardsLevel() for _, addr := range cow.modifiedAccounts() { - // Skip FeeSink and RewardsPool MinBalance checks here. - // There's only two accounts, so space isn't an issue, and we don't + // Skip FeeSink, RewardsPool, and CompactCertSender MinBalance checks here. + // There's only a few accounts, so space isn't an issue, and we don't // expect them to have low balances, but if they do, it may cause - // surprises for every transaction. - if addr == spec.FeeSink || addr == spec.RewardsPool { + // surprises. + if addr == spec.FeeSink || addr == spec.RewardsPool || addr == transactions.CompactCertSender { continue } @@ -700,7 +736,7 @@ func (eval *BlockEvaluator) transaction(txn transactions.SignedTxn, appEval *app } // applyTransaction changes the balances according to this transaction. -func applyTransaction(tx transactions.Transaction, balances apply.Balances, steva apply.StateEvaluator, spec transactions.SpecialAddresses, ctr uint64) (ad transactions.ApplyData, err error) { +func applyTransaction(tx transactions.Transaction, balances *roundCowState, steva apply.StateEvaluator, spec transactions.SpecialAddresses, ctr uint64) (ad transactions.ApplyData, err error) { params := balances.ConsensusParams() // move fee to pool @@ -749,6 +785,9 @@ func applyTransaction(tx transactions.Transaction, balances apply.Balances, stev case protocol.ApplicationCallTx: err = apply.ApplicationCall(tx.ApplicationCallTxnFields, tx.Header, balances, &ad, ctr, steva) + case protocol.CompactCertTx: + err = balances.compactCert(tx.CertRound, tx.Cert, tx.Header.FirstValid) + default: err = fmt.Errorf("Unknown transaction type %v", tx.Type) } @@ -804,6 +843,8 @@ func (eval *BlockEvaluator) endOfBlock() error { if err != nil { return err } + + eval.block.CompactCertLastRound = eval.state.compactCertLast() } return nil @@ -836,8 +877,8 @@ func (eval *BlockEvaluator) finalValidation() error { if eval.block.CompactCertVotersTotal != expectedVotersWeight { return fmt.Errorf("CompactCertVotersTotal wrong: %v != %v", eval.block.CompactCertVotersTotal, expectedVotersWeight) } - if eval.block.CompactCertLastRound != 0 { - return fmt.Errorf("CompactCertLastRound wrong: %v != %v", eval.block.CompactCertLastRound, 0) + if eval.block.CompactCertLastRound != eval.state.compactCertLast() { + return fmt.Errorf("CompactCertLastRound wrong: %v != %v", eval.block.CompactCertLastRound, eval.state.compactCertLast()) } } diff --git a/protocol/hash.go b/protocol/hash.go index 4ab2786e69..c87ab0d420 100644 --- a/protocol/hash.go +++ b/protocol/hash.go @@ -47,6 +47,7 @@ const ( ProgramData HashID = "ProgData" ProposerSeed HashID = "PS" Seed HashID = "SD" + SpecialAddr HashID = "SpecialAddr" TestHashable HashID = "TE" TxGroup HashID = "TG" Transaction HashID = "TX" diff --git a/protocol/txntype.go b/protocol/txntype.go index d8e50ecc00..dbd60603cd 100644 --- a/protocol/txntype.go +++ b/protocol/txntype.go @@ -41,6 +41,9 @@ const ( // ApplicationCallTx allows creating, deleting, and interacting with an application ApplicationCallTx TxType = "appl" + // CompactCertTx records a compact certificate + CompactCertTx TxType = "cert" + // UnknownTx signals an error UnknownTx TxType = "unknown" ) From ecd0e2b2c3d3170dfe3454c09ea8ac9ba4b8b0a5 Mon Sep 17 00:00:00 2001 From: nicholasguoalgorand <67928479+nicholasguoalgorand@users.noreply.github.com> Date: Mon, 21 Sep 2020 19:05:46 -0700 Subject: [PATCH 057/136] Filter Timeout Change Refactor (#1520) Add AgreementFilterTimeout and make it use nanosecond resolution --- agreement/types.go | 8 +++----- config/consensus.go | 15 +++++++++------ node/node_test.go | 2 +- .../features/transactions/accountv2_test.go | 3 +++ test/e2e-go/perf/basic_test.go | 2 +- test/e2e-go/upgrades/send_receive_upgrade_test.go | 4 ++++ .../consensus/catchpointtestingprotocol.json | 4 ++-- 7 files changed, 23 insertions(+), 15 deletions(-) diff --git a/agreement/types.go b/agreement/types.go index 6cc35211f6..97e77388d0 100644 --- a/agreement/types.go +++ b/agreement/types.go @@ -31,13 +31,11 @@ var recoveryExtraTimeout = config.Protocol.SmallLambda // FilterTimeout is the duration of the first agreement step. func FilterTimeout(p period, v protocol.ConsensusVersion) time.Duration { - var smallLambdas uint64 if p == 0 { - smallLambdas = config.Consensus[v].FilterTimeoutPeriod0SmallLambdas - } else { - smallLambdas = config.Consensus[v].FilterTimeoutSmallLambdas + return config.Consensus[v].AgreementFilterTimeoutPeriod0 } - return config.Protocol.SmallLambda * time.Duration(smallLambdas) + // timeout is expected to be 2 * SmallLambda, value moved to consensusParams + return config.Consensus[v].AgreementFilterTimeout } // DeadlineTimeout is the duration of the second agreement step. diff --git a/config/consensus.go b/config/consensus.go index 4c4fdda5ba..9d5b58aaba 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -137,8 +137,11 @@ type ConsensusParams struct { DownCommitteeSize uint64 DownCommitteeThreshold uint64 - FilterTimeoutSmallLambdas uint64 - FilterTimeoutPeriod0SmallLambdas uint64 + // time for nodes to wait for block proposal headers for period > 0, value should be set to 2 * SmallLambda + AgreementFilterTimeout time.Duration + // time for nodes to wait for block proposal headers for period = 0, value should be configured to suit best case + // critical path + AgreementFilterTimeoutPeriod0 time.Duration FastRecoveryLambda time.Duration // time between fast recovery attempts FastPartitionRecovery bool // set when fast partition recovery is enabled @@ -539,8 +542,8 @@ func initConsensusProtocols() { DownCommitteeSize: 10000, DownCommitteeThreshold: 7750, - FilterTimeoutSmallLambdas: 2, - FilterTimeoutPeriod0SmallLambdas: 2, + AgreementFilterTimeout: 4 * time.Second, + AgreementFilterTimeoutPeriod0: 4 * time.Second, FastRecoveryLambda: 5 * time.Minute, @@ -806,8 +809,8 @@ func initConsensusProtocols() { vFuture := v24 vFuture.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{} - // FilterTimeout 2s instead of 4s for period 0 - vFuture.FilterTimeoutPeriod0SmallLambdas = 1 + // FilterTimeout for period 0 should take a new optimized, configured value, need to revisit this later + vFuture.AgreementFilterTimeoutPeriod0 = 4 * time.Second Consensus[protocol.ConsensusFuture] = vFuture } diff --git a/node/node_test.go b/node/node_test.go index e3aa4f0afa..56f2564cb4 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -42,7 +42,7 @@ import ( "github.com/algorand/go-algorand/util/execpool" ) -var expectedAgreementTime = 2*config.Protocol.BigLambda + 3*config.Protocol.SmallLambda + 2*time.Second +var expectedAgreementTime = 2*config.Protocol.BigLambda + config.Protocol.SmallLambda + config.Consensus[protocol.ConsensusCurrentVersion].AgreementFilterTimeout + 2*time.Second var sinkAddr = basics.Address{0x7, 0xda, 0xcb, 0x4b, 0x6d, 0x9e, 0xd1, 0x41, 0xb1, 0x75, 0x76, 0xbd, 0x45, 0x9a, 0xe6, 0x42, 0x1d, 0x48, 0x6d, 0xa3, 0xd4, 0xef, 0x22, 0x47, 0xc4, 0x9, 0xa3, 0x96, 0xb8, 0x2e, 0xa2, 0x21} var poolAddr = basics.Address{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} diff --git a/test/e2e-go/features/transactions/accountv2_test.go b/test/e2e-go/features/transactions/accountv2_test.go index 8792ccd077..2f37850c18 100644 --- a/test/e2e-go/features/transactions/accountv2_test.go +++ b/test/e2e-go/features/transactions/accountv2_test.go @@ -20,6 +20,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/require" @@ -39,6 +40,8 @@ func TestAccountInformationV2(t *testing.T) { proto, ok := config.Consensus[protocol.ConsensusFuture] a.True(ok) os.Setenv("ALGOSMALLLAMBDAMSEC", "200") + proto.AgreementFilterTimeoutPeriod0 = 400 * time.Millisecond + proto.AgreementFilterTimeout = 400 * time.Millisecond fixture.SetConsensus(config.ConsensusProtocols{protocol.ConsensusFuture: proto}) defer func() { os.Unsetenv("ALGOSMALLLAMBDAMSEC") diff --git a/test/e2e-go/perf/basic_test.go b/test/e2e-go/perf/basic_test.go index e275740118..78370c8614 100644 --- a/test/e2e-go/perf/basic_test.go +++ b/test/e2e-go/perf/basic_test.go @@ -51,7 +51,7 @@ func queuePayments(b *testing.B, wg *sync.WaitGroup, c libgoal.Client, q <-chan } fmt.Printf("Error broadcasting transaction: %v\n", err) - time.Sleep(2 * config.Protocol.SmallLambda) + time.Sleep(config.Consensus[protocol.ConsensusCurrentVersion].AgreementFilterTimeout) } } diff --git a/test/e2e-go/upgrades/send_receive_upgrade_test.go b/test/e2e-go/upgrades/send_receive_upgrade_test.go index 817324ac46..278d4d1224 100644 --- a/test/e2e-go/upgrades/send_receive_upgrade_test.go +++ b/test/e2e-go/upgrades/send_receive_upgrade_test.go @@ -117,6 +117,10 @@ func generateFastUpgradeConsensus() (fastUpgradeProtocols config.ConsensusProtoc } fastUpgradeProtocols[consensusTestFastUpgrade(proto)] = fastParams + + // support the ALGOSMALLLAMBDAMSEC = 500 env variable + fastParams.AgreementFilterTimeout = time.Second + fastParams.AgreementFilterTimeoutPeriod0 = time.Second } return } diff --git a/test/testdata/consensus/catchpointtestingprotocol.json b/test/testdata/consensus/catchpointtestingprotocol.json index ba6e81373e..05c0f2dd9d 100644 --- a/test/testdata/consensus/catchpointtestingprotocol.json +++ b/test/testdata/consensus/catchpointtestingprotocol.json @@ -33,8 +33,8 @@ "RedoCommitteeThreshold": 1768, "DownCommitteeSize": 6000, "DownCommitteeThreshold": 4560, - "FilterTimeoutSmallLambdas": 2, - "FilterTimeoutPeriod0SmallLambdas": 2, + "AgreementFilterTimeout": 1000000000, + "AgreementFilterTimeoutPeriod0": 1000000000, "FastRecoveryLambda": 300000000000, "FastPartitionRecovery": true, "PaysetCommitFlat": true, From c815ba421686eb21519a023de7c1a1b6a53ba82c Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Tue, 22 Sep 2020 12:18:08 -0400 Subject: [PATCH 058/136] Cache the most recent Seed/Circulation calls to reduce pressure on ledger (#1518) This PR adds a cache for the data.Ledger for the Circulation and Seed calls. --- data/ledger.go | 74 +++++++++++ data/ledger_test.go | 314 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 388 insertions(+) create mode 100644 data/ledger_test.go diff --git a/data/ledger.go b/data/ledger.go index 9e0adf86c0..3d7ec8903a 100644 --- a/data/ledger.go +++ b/data/ledger.go @@ -18,6 +18,7 @@ package data import ( "fmt" + "sync/atomic" "time" "github.com/algorand/go-algorand/agreement" @@ -43,6 +44,35 @@ type Ledger struct { *ledger.Ledger log logging.Logger + + // a two-item moving window cache for the total number of online circulating coins + lastRoundCirculation atomic.Value + // a two-item moving window cache for the round seed + lastRoundSeed atomic.Value +} + +// roundCirculationPair used to hold a pair of matching round number and the amount of online money +type roundCirculationPair struct { + round basics.Round + onlineMoney basics.MicroAlgos +} + +// roundCirculation is the cache for the circulating coins +type roundCirculation struct { + // elements holds several round-onlineMoney pairs + elements [2]roundCirculationPair +} + +// roundSeedPair is the cache for a single seed at a given round +type roundSeedPair struct { + round basics.Round + seed committee.Seed +} + +// roundSeed is the cache for the seed +type roundSeed struct { + // elements holds several round-seed pairs + elements [2]roundSeedPair } func makeGenesisBlock(proto protocol.ConsensusVersion, genesisBal GenesisBalances, genesisID string, genesisHash crypto.Digest) (bookkeeping.Block, error) { @@ -188,11 +218,32 @@ func (l *Ledger) NextRound() basics.Round { // Circulation implements agreement.Ledger.Circulation. func (l *Ledger) Circulation(r basics.Round) (basics.MicroAlgos, error) { + circulation, cached := l.lastRoundCirculation.Load().(roundCirculation) + if cached && r != basics.Round(0) { + for _, element := range circulation.elements { + if element.round == r { + return element.onlineMoney, nil + } + } + } + totals, err := l.Totals(r) if err != nil { return basics.MicroAlgos{}, err } + if !cached || r > circulation.elements[1].round { + l.lastRoundCirculation.Store( + roundCirculation{ + elements: [2]roundCirculationPair{ + circulation.elements[1], + roundCirculationPair{ + round: r, + onlineMoney: totals.Online.Money}, + }, + }) + } + return totals.Online.Money, nil } @@ -201,10 +252,33 @@ func (l *Ledger) Circulation(r basics.Round) (basics.MicroAlgos, error) { // I/O error. // Implements agreement.Ledger.Seed func (l *Ledger) Seed(r basics.Round) (committee.Seed, error) { + seed, cached := l.lastRoundSeed.Load().(roundSeed) + if cached && r != basics.Round(0) { + for _, roundSeed := range seed.elements { + if roundSeed.round == r { + return roundSeed.seed, nil + } + } + } + blockhdr, err := l.BlockHdr(r) if err != nil { return committee.Seed{}, err } + + if !cached || r > seed.elements[1].round { + l.lastRoundSeed.Store( + roundSeed{ + elements: [2]roundSeedPair{ + seed.elements[1], + roundSeedPair{ + round: r, + seed: blockhdr.Seed, + }, + }, + }) + } + return blockhdr.Seed, nil } diff --git a/data/ledger_test.go b/data/ledger_test.go new file mode 100644 index 0000000000..472c12d8e9 --- /dev/null +++ b/data/ledger_test.go @@ -0,0 +1,314 @@ +// Copyright (C) 2019-2020 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package data + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" +) + +var testPoolAddr = basics.Address{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} +var testSinkAddr = basics.Address{0x2c, 0x2a, 0x6c, 0xe9, 0xa9, 0xa7, 0xc2, 0x8c, 0x22, 0x95, 0xfd, 0x32, 0x4f, 0x77, 0xa5, 0x4, 0x8b, 0x42, 0xc2, 0xb7, 0xa8, 0x54, 0x84, 0xb6, 0x80, 0xb1, 0xe1, 0x3d, 0x59, 0x9b, 0xeb, 0x36} + +func testGenerateInitState(tb testing.TB, proto protocol.ConsensusVersion) (genesisInitState ledger.InitState, initKeys map[basics.Address]*crypto.SignatureSecrets) { + + var poolSecret, sinkSecret *crypto.SignatureSecrets + var seed crypto.Seed + + incentivePoolName := []byte("incentive pool") + copy(seed[:], incentivePoolName) + poolSecret = crypto.GenerateSignatureSecrets(seed) + + feeSinkName := []byte("fee sink") + copy(seed[:], feeSinkName) + sinkSecret = crypto.GenerateSignatureSecrets(seed) + + params := config.Consensus[proto] + poolAddr := testPoolAddr + sinkAddr := testSinkAddr + + var zeroSeed crypto.Seed + var genaddrs [10]basics.Address + var gensecrets [10]*crypto.SignatureSecrets + for i := range genaddrs { + seed := zeroSeed + seed[0] = byte(i) + x := crypto.GenerateSignatureSecrets(seed) + genaddrs[i] = basics.Address(x.SignatureVerifier) + gensecrets[i] = x + } + + initKeys = make(map[basics.Address]*crypto.SignatureSecrets) + initAccounts := make(map[basics.Address]basics.AccountData) + for i := range genaddrs { + initKeys[genaddrs[i]] = gensecrets[i] + // Give each account quite a bit more balance than MinFee or MinBalance + accountStatus := basics.Online + if i%2 == 0 { + accountStatus = basics.NotParticipating + } + initAccounts[genaddrs[i]] = basics.MakeAccountData(accountStatus, basics.MicroAlgos{Raw: uint64((i + 100) * 100000)}) + } + initKeys[poolAddr] = poolSecret + initAccounts[poolAddr] = basics.MakeAccountData(basics.NotParticipating, basics.MicroAlgos{Raw: 1234567}) + initKeys[sinkAddr] = sinkSecret + initAccounts[sinkAddr] = basics.MakeAccountData(basics.NotParticipating, basics.MicroAlgos{Raw: 7654321}) + + incentivePoolBalanceAtGenesis := initAccounts[poolAddr].MicroAlgos + initialRewardsPerRound := incentivePoolBalanceAtGenesis.Raw / uint64(params.RewardsRateRefreshInterval) + var emptyPayset transactions.Payset + + initBlock := bookkeeping.Block{ + BlockHeader: bookkeeping.BlockHeader{ + GenesisID: tb.Name(), + Round: 0, + RewardsState: bookkeeping.RewardsState{ + RewardsRate: initialRewardsPerRound, + RewardsPool: poolAddr, + FeeSink: sinkAddr, + }, + UpgradeState: bookkeeping.UpgradeState{ + CurrentProtocol: proto, + }, + TxnRoot: emptyPayset.Commit(params.PaysetCommitFlat), + }, + } + if params.SupportGenesisHash { + initBlock.BlockHeader.GenesisHash = crypto.Hash([]byte(tb.Name())) + } + + genesisInitState.Block = initBlock + genesisInitState.Accounts = initAccounts + genesisInitState.GenesisHash = crypto.Hash([]byte(tb.Name())) + + return +} + +func TestLedgerCirculation(t *testing.T) { + genesisInitState, keys := testGenerateInitState(t, protocol.ConsensusCurrentVersion) + + const inMem = true + cfg := config.GetDefaultLocal() + cfg.Archival = true + log := logging.TestingLog(t) + log.SetLevel(logging.Warn) + realLedger, err := ledger.OpenLedger(log, t.Name(), inMem, genesisInitState, cfg) + require.NoError(t, err, "could not open ledger") + defer realLedger.Close() + + l := Ledger{Ledger: realLedger} + require.NotNil(t, &l) + + var sourceAccount basics.Address + var destAccount basics.Address + for addr, acctData := range genesisInitState.Accounts { + if addr == testPoolAddr || addr == testSinkAddr { + continue + } + if acctData.Status == basics.Online { + sourceAccount = addr + break + } + } + for addr, acctData := range genesisInitState.Accounts { + if addr == testPoolAddr || addr == testSinkAddr { + continue + } + if acctData.Status == basics.NotParticipating { + destAccount = addr + break + } + } + require.False(t, sourceAccount.IsZero()) + require.False(t, destAccount.IsZero()) + + data, err := realLedger.Lookup(basics.Round(0), destAccount) + require.NoError(t, err) + baseDestValue := data.MicroAlgos.Raw + + blk := genesisInitState.Block + totals, _ := realLedger.Totals(basics.Round(0)) + baseCirculation := totals.Online.Money.Raw + + srcAccountKey := keys[sourceAccount] + require.NotNil(t, srcAccountKey) + + for rnd := basics.Round(1); rnd < basics.Round(600); rnd++ { + blk.BlockHeader.Round++ + blk.BlockHeader.TimeStamp += int64(crypto.RandUint64() % 100 * 1000) + var tx transactions.Transaction + tx.Sender = sourceAccount + tx.Fee = basics.MicroAlgos{Raw: 10000} + tx.FirstValid = rnd - 1 + tx.LastValid = tx.FirstValid + 999 + tx.Receiver = destAccount + tx.Amount = basics.MicroAlgos{Raw: 1} + tx.Type = protocol.PaymentTx + signedTx := tx.Sign(srcAccountKey) + blk.Payset = transactions.Payset{transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: signedTx, + }, + }} + require.NoError(t, l.AddBlock(blk, agreement.Certificate{})) + l.WaitForCommit(rnd) + + // test most recent round + if rnd < basics.Round(500) { + data, err = realLedger.Lookup(rnd, destAccount) + require.NoError(t, err) + require.Equal(t, baseDestValue+uint64(rnd), data.MicroAlgos.Raw) + data, err = l.Lookup(rnd, destAccount) + require.NoError(t, err) + require.Equal(t, baseDestValue+uint64(rnd), data.MicroAlgos.Raw) + + totals, err = realLedger.Totals(rnd) + require.NoError(t, err) + roundCirculation := totals.Online.Money.Raw + require.Equal(t, baseCirculation-uint64(rnd)*(10001), roundCirculation) + + totals, err = l.Totals(rnd) + require.NoError(t, err) + roundCirculation = totals.Online.Money.Raw + require.Equal(t, baseCirculation-uint64(rnd)*(10001), roundCirculation) + } else if rnd < basics.Round(510) { + // test one round ago + data, err = realLedger.Lookup(rnd-1, destAccount) + require.NoError(t, err) + require.Equal(t, baseDestValue+uint64(rnd)-1, data.MicroAlgos.Raw) + data, err = l.Lookup(rnd-1, destAccount) + require.NoError(t, err) + require.Equal(t, baseDestValue+uint64(rnd)-1, data.MicroAlgos.Raw) + + totals, err = realLedger.Totals(rnd - 1) + require.NoError(t, err) + roundCirculation := totals.Online.Money.Raw + require.Equal(t, baseCirculation-uint64(rnd-1)*(10001), roundCirculation) + + totals, err = l.Totals(rnd - 1) + require.NoError(t, err) + roundCirculation = totals.Online.Money.Raw + require.Equal(t, baseCirculation-uint64(rnd-1)*(10001), roundCirculation) + } else if rnd < basics.Round(520) { + // test one round in the future ( expected error ) + data, err = realLedger.Lookup(rnd+1, destAccount) + require.Error(t, err) + require.Equal(t, uint64(0), data.MicroAlgos.Raw) + data, err = l.Lookup(rnd+1, destAccount) + require.Error(t, err) + require.Equal(t, uint64(0), data.MicroAlgos.Raw) + + _, err = realLedger.Totals(rnd + 1) + require.Error(t, err) + + _, err = l.Totals(rnd + 1) + require.Error(t, err) + } else if rnd < basics.Round(520) { + // test expired round ( expected error ) + _, err = realLedger.Totals(rnd - 500) + require.Error(t, err) + + _, err = l.Totals(rnd - 500) + require.Error(t, err) + } + } + return +} + +func TestLedgerSeed(t *testing.T) { + genesisInitState, _ := testGenerateInitState(t, protocol.ConsensusCurrentVersion) + + const inMem = true + cfg := config.GetDefaultLocal() + cfg.Archival = true + log := logging.TestingLog(t) + log.SetLevel(logging.Warn) + realLedger, err := ledger.OpenLedger(log, t.Name(), inMem, genesisInitState, cfg) + require.NoError(t, err, "could not open ledger") + defer realLedger.Close() + + l := Ledger{Ledger: realLedger} + require.NotNil(t, &l) + + blk := genesisInitState.Block + for rnd := basics.Round(1); rnd < basics.Round(32); rnd++ { + blk.BlockHeader.Round++ + blk.BlockHeader.Seed[0] = byte(uint64(rnd)) + blk.BlockHeader.Seed[1] = byte(uint64(rnd) / 256) + blk.BlockHeader.Seed[2] = byte(uint64(rnd) / 65536) + blk.BlockHeader.TimeStamp += int64(crypto.RandUint64() % 100 * 1000) + require.NoError(t, l.AddBlock(blk, agreement.Certificate{})) + l.WaitForCommit(rnd) + if rnd < basics.Round(16) { + // test the current round + expectedHdr, err := realLedger.BlockHdr(rnd) + require.NoError(t, err) + + // ensure the item is not in the cache + seed, cached := l.lastRoundSeed.Load().(roundSeed) + if cached { + require.NotEqual(t, seed.elements[1].seed, expectedHdr.Seed) + } + + actualSeed, err := l.Seed(rnd) + require.NoError(t, err) + + require.Equal(t, expectedHdr.Seed, actualSeed) + + seed, cached = l.lastRoundSeed.Load().(roundSeed) + require.True(t, cached) + require.Equal(t, seed.elements[1].seed, expectedHdr.Seed) + } else if rnd < basics.Round(32) { + // test against the previous round + expectedHdr, err := realLedger.BlockHdr(rnd - 1) + require.NoError(t, err) + + // ensure the cache is aligned with the previous round + seed, cached := l.lastRoundSeed.Load().(roundSeed) + require.True(t, cached) + require.Equal(t, seed.elements[1].round, rnd-1) + require.Equal(t, seed.elements[1].seed, expectedHdr.Seed) + + actualSeed, err := l.Seed(rnd) + require.NoError(t, err) + + expectedHdr, err = realLedger.BlockHdr(rnd) + require.NoError(t, err) + + require.Equal(t, expectedHdr.Seed, actualSeed) + + // ensure the cache is aligned with the updated round + seed, cached = l.lastRoundSeed.Load().(roundSeed) + require.True(t, cached) + require.Equal(t, seed.elements[1].round, rnd) + require.Equal(t, seed.elements[1].seed, expectedHdr.Seed) + } + } + return +} From 98d62387a7f1659ba9c942603327a3d287a3f679 Mon Sep 17 00:00:00 2001 From: nicholasguoalgorand <67928479+nicholasguoalgorand@users.noreply.github.com> Date: Tue, 22 Sep 2020 11:57:44 -0700 Subject: [PATCH 059/136] Deprecate algolambdamsec environment variable (#1536) Deprecate algolambdamsec environment variable in favor of using the new AgreementFilterTimeout --- config/consensus.go | 10 ------ .../cli/goal/expect/catchpointCatchupTest.exp | 2 -- .../catchup/catchpointCatchup_test.go | 12 +++---- .../features/transactions/accountv2_test.go | 5 --- .../upgrades/application_support_test.go | 32 +++---------------- test/e2e-go/upgrades/rekey_support_test.go | 17 ++-------- .../upgrades/send_receive_upgrade_test.go | 4 +-- 7 files changed, 13 insertions(+), 69 deletions(-) diff --git a/config/consensus.go b/config/consensus.go index 9d5b58aaba..0450aadd62 100644 --- a/config/consensus.go +++ b/config/consensus.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "os" "path/filepath" - "strconv" "time" "github.com/algorand/go-algorand/protocol" @@ -833,15 +832,6 @@ func init() { initConsensusProtocols() - // Allow tuning SmallLambda for faster consensus in single-machine e2e - // tests. Useful for development. This might make sense to fold into - // a protocol-version-specific setting, once we move SmallLambda into - // ConsensusParams. - algoSmallLambda, err := strconv.ParseInt(os.Getenv("ALGOSMALLLAMBDAMSEC"), 10, 64) - if err == nil { - Protocol.SmallLambda = time.Duration(algoSmallLambda) * time.Millisecond - } - // Set allocation limits for _, p := range Consensus { checkSetAllocBounds(p) diff --git a/test/e2e-go/cli/goal/expect/catchpointCatchupTest.exp b/test/e2e-go/cli/goal/expect/catchpointCatchupTest.exp index f09ec8d0a9..476546da90 100755 --- a/test/e2e-go/cli/goal/expect/catchpointCatchupTest.exp +++ b/test/e2e-go/cli/goal/expect/catchpointCatchupTest.exp @@ -71,8 +71,6 @@ if { [catch { set ::GLOBAL_TEST_ROOT_DIR $TEST_ROOT_DIR set ::GLOBAL_NETWORK_NAME $NETWORK_NAME - set ::env(ALGOSMALLLAMBDAMSEC) 500 - # Start the Primary Node ::AlgorandGoal::StartNode $TEST_ROOT_DIR/Primary diff --git a/test/e2e-go/features/catchup/catchpointCatchup_test.go b/test/e2e-go/features/catchup/catchpointCatchup_test.go index 3ade3d372e..16de802892 100644 --- a/test/e2e-go/features/catchup/catchpointCatchup_test.go +++ b/test/e2e-go/features/catchup/catchpointCatchup_test.go @@ -19,7 +19,6 @@ package catchup import ( "fmt" "net/http" - "os" "os/exec" "path/filepath" "runtime" @@ -86,11 +85,6 @@ func TestBasicCatchpointCatchup(t *testing.T) { a := require.New(t) log := logging.TestingLog(t) - if runtime.GOARCH == "amd64" { - // amd64 platforms are generally quite capable, so exceletate the round times to make the test run faster. - os.Setenv("ALGOSMALLLAMBDAMSEC", "500") - } - // Overview of this test: // Start a two-node network (primary has 100%, secondary has 0%) // Nodes are having a consensus allowing balances history of 32 rounds and transaction history of 33 rounds. @@ -110,6 +104,12 @@ func TestBasicCatchpointCatchup(t *testing.T) { catchpointCatchupProtocol.MaxBalLookback = 2 * catchpointCatchupProtocol.SeedLookback * catchpointCatchupProtocol.SeedRefreshInterval // 32 catchpointCatchupProtocol.MaxTxnLife = 33 + if runtime.GOARCH == "amd64" { + // amd64 platforms are generally quite capable, so accelerate the round times to make the test run faster. + catchpointCatchupProtocol.AgreementFilterTimeoutPeriod0 = 1 * time.Second + catchpointCatchupProtocol.AgreementFilterTimeout = 1 * time.Second + } + consensus[consensusCatchpointCatchupTestProtocol] = catchpointCatchupProtocol var fixture fixtures.RestClientFixture diff --git a/test/e2e-go/features/transactions/accountv2_test.go b/test/e2e-go/features/transactions/accountv2_test.go index 2f37850c18..56fdd44244 100644 --- a/test/e2e-go/features/transactions/accountv2_test.go +++ b/test/e2e-go/features/transactions/accountv2_test.go @@ -17,7 +17,6 @@ package transactions import ( - "os" "path/filepath" "testing" "time" @@ -39,13 +38,9 @@ func TestAccountInformationV2(t *testing.T) { var fixture fixtures.RestClientFixture proto, ok := config.Consensus[protocol.ConsensusFuture] a.True(ok) - os.Setenv("ALGOSMALLLAMBDAMSEC", "200") proto.AgreementFilterTimeoutPeriod0 = 400 * time.Millisecond proto.AgreementFilterTimeout = 400 * time.Millisecond fixture.SetConsensus(config.ConsensusProtocols{protocol.ConsensusFuture: proto}) - defer func() { - os.Unsetenv("ALGOSMALLLAMBDAMSEC") - }() fixture.Setup(t, filepath.Join("nettemplates", "TwoNodes50EachFuture.json")) defer fixture.Shutdown() diff --git a/test/e2e-go/upgrades/application_support_test.go b/test/e2e-go/upgrades/application_support_test.go index fa1ca6943d..2717fae961 100644 --- a/test/e2e-go/upgrades/application_support_test.go +++ b/test/e2e-go/upgrades/application_support_test.go @@ -17,8 +17,6 @@ package upgrades import ( - "fmt" - "os" "path/filepath" "testing" "time" @@ -67,18 +65,7 @@ func makeApplicationUpgradeConsensus(t *testing.T) (appConsensus config.Consensu // to a version that supports applications. It verify that prior to supporting applications, the node would not accept // any application transaction and after the upgrade is complete, it would support that. func TestApplicationsUpgradeOverREST(t *testing.T) { - // set the small lambda to 500 for the duration of this test. - roundTimeMs := 500 - lambda := os.Getenv("ALGOSMALLLAMBDAMSEC") - os.Setenv("ALGOSMALLLAMBDAMSEC", fmt.Sprintf("%d", roundTimeMs)) - defer func() { - if lambda == "" { - os.Unsetenv("ALGOSMALLLAMBDAMSEC") - } else { - os.Setenv("ALGOSMALLLAMBDAMSEC", lambda) - } - }() - + smallLambdaMs := 500 consensus := makeApplicationUpgradeConsensus(t) var fixture fixtures.RestClientFixture @@ -177,7 +164,7 @@ int 1 require.NoError(t, err) require.Less(t, int64(time.Now().Sub(startLoopTime)), int64(3*time.Minute)) - time.Sleep(time.Duration(roundTimeMs) * time.Millisecond) + time.Sleep(time.Duration(smallLambdaMs) * time.Millisecond) round = curStatus.LastRound } @@ -291,18 +278,7 @@ int 1 // to a version that supports applications. It verify that prior to supporting applications, the node would not accept // any application transaction and after the upgrade is complete, it would support that. func TestApplicationsUpgradeOverGossip(t *testing.T) { - // set the small lambda to 500 for the duration of this test. - roundTimeMs := 500 - lambda := os.Getenv("ALGOSMALLLAMBDAMSEC") - os.Setenv("ALGOSMALLLAMBDAMSEC", fmt.Sprintf("%d", roundTimeMs)) - defer func() { - if lambda == "" { - os.Unsetenv("ALGOSMALLLAMBDAMSEC") - } else { - os.Setenv("ALGOSMALLLAMBDAMSEC", lambda) - } - }() - + smallLambdaMs := 500 consensus := makeApplicationUpgradeConsensus(t) var fixture fixtures.RestClientFixture @@ -436,7 +412,7 @@ int 1 require.NoError(t, err) require.Less(t, int64(time.Now().Sub(startLoopTime)), int64(3*time.Minute)) - time.Sleep(time.Duration(roundTimeMs) * time.Millisecond) + time.Sleep(time.Duration(smallLambdaMs) * time.Millisecond) round = curStatus.LastRound } diff --git a/test/e2e-go/upgrades/rekey_support_test.go b/test/e2e-go/upgrades/rekey_support_test.go index 88d6d7c0b7..d3263e0717 100644 --- a/test/e2e-go/upgrades/rekey_support_test.go +++ b/test/e2e-go/upgrades/rekey_support_test.go @@ -17,8 +17,6 @@ package upgrades import ( - "fmt" - "os" "path/filepath" "testing" "time" @@ -33,18 +31,7 @@ import ( func TestRekeyUpgrade(t *testing.T) { a := require.New(t) - // set the small lambda to 500 for the duration of this test. - roundTimeMs := 500 - lambda := os.Getenv("ALGOSMALLLAMBDAMSEC") - os.Setenv("ALGOSMALLLAMBDAMSEC", fmt.Sprintf("%d", roundTimeMs)) - defer func() { - if lambda == "" { - os.Unsetenv("ALGOSMALLLAMBDAMSEC") - } else { - os.Setenv("ALGOSMALLLAMBDAMSEC", lambda) - } - }() - + smallLambdaMs := 500 consensus := makeApplicationUpgradeConsensus(t) var fixture fixtures.RestClientFixture @@ -108,7 +95,7 @@ func TestRekeyUpgrade(t *testing.T) { require.NoError(t, err) require.Less(t, int64(time.Now().Sub(startLoopTime)), int64(3*time.Minute)) - time.Sleep(time.Duration(roundTimeMs) * time.Millisecond) + time.Sleep(time.Duration(smallLambdaMs) * time.Millisecond) round = curStatus.LastRound } diff --git a/test/e2e-go/upgrades/send_receive_upgrade_test.go b/test/e2e-go/upgrades/send_receive_upgrade_test.go index 278d4d1224..6cd8f115ba 100644 --- a/test/e2e-go/upgrades/send_receive_upgrade_test.go +++ b/test/e2e-go/upgrades/send_receive_upgrade_test.go @@ -18,7 +18,6 @@ package upgrades import ( "math/rand" - "os" "path/filepath" "testing" "time" @@ -118,7 +117,7 @@ func generateFastUpgradeConsensus() (fastUpgradeProtocols config.ConsensusProtoc fastUpgradeProtocols[consensusTestFastUpgrade(proto)] = fastParams - // support the ALGOSMALLLAMBDAMSEC = 500 env variable + // set the small lambda to 500 for the duration of dependent tests. fastParams.AgreementFilterTimeout = time.Second fastParams.AgreementFilterTimeoutPeriod0 = time.Second } @@ -128,7 +127,6 @@ func generateFastUpgradeConsensus() (fastUpgradeProtocols config.ConsensusProtoc func testAccountsCanSendMoneyAcrossUpgrade(t *testing.T, templatePath string) { t.Parallel() a := require.New(t) - os.Setenv("ALGOSMALLLAMBDAMSEC", "500") consensus := generateFastUpgradeConsensus() From e14f3d5ea1c0d2f6921c1fef659778b93cf0b60e Mon Sep 17 00:00:00 2001 From: Tsachi Herman Date: Tue, 22 Sep 2020 16:12:31 -0400 Subject: [PATCH 060/136] Fix heuristicFormatStr logic (#1528) Fix heuristicFormatStr logic --- cmd/goal/application.go | 126 +++++++++++++++++++++++++++++++++++++++- cmd/goal/interact.go | 2 +- data/basics/address.go | 2 + 3 files changed, 127 insertions(+), 3 deletions(-) diff --git a/cmd/goal/application.go b/cmd/goal/application.go index 1a4be147c7..4121564adc 100644 --- a/cmd/goal/application.go +++ b/cmd/goal/application.go @@ -25,6 +25,7 @@ import ( "strconv" "strings" "unicode" + "unicode/utf8" "github.com/spf13/cobra" @@ -919,7 +920,7 @@ var deleteAppCmd = &cobra.Command{ }, } -func printable(str string) bool { +func unicodePrintable(str string) bool { for _, r := range str { if !unicode.IsPrint(r) { return false @@ -928,17 +929,138 @@ func printable(str string) bool { return true } +func jsonPrintable(str string) bool { + // htmlSafeSet holds the value true if the ASCII character with the given + // array position can be safely represented inside a JSON string, embedded + // inside of HTML