From d1e589f11fd854a7ec8ac7fee05e91c88c199d1d Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 8 Jun 2019 19:00:50 +0100 Subject: [PATCH 001/405] Remove eth2 genesis in favour of genesis trigger --- .../contracts/validator_registration.json | 2 +- .../contracts/validator_registration.v.py | 22 -------- deposit_contract/tests/contracts/conftest.py | 26 ++-------- .../tests/contracts/test_deposit.py | 50 ------------------ specs/core/0_beacon-chain.md | 51 ++++++++++++++----- specs/core/0_deposit-contract.md | 20 -------- 6 files changed, 42 insertions(+), 129 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index 08d57f80a5..a885ef0a17 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "merkle_tree_index", "indexed": false}], "anonymous": false, "type": "event"}, {"name": "Eth2Genesis", "inputs": [{"type": "bytes32", "name": "deposit_root", "indexed": false}, {"type": "bytes", "name": "deposit_count", "indexed": false}, {"type": "bytes", "name": "time", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 7077}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 11026}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 445994}, {"name": "chainStarted", "outputs": [{"type": "bool", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 603}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600060c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600060c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600060c052602060c02001555b81516001018083528114156100aa575b505061140756600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561026b57602060046101403734156100b457600080fd5b67ffffffffffffffff6101405111156100cc57600080fd5b60006101605261014051610180526101a060006008818352015b6101605160086000811215610103578060000360020a820461010a565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561013557600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8600081121561017e578060000360020a8204610185565b8060020a82025b90509050610180525b81516001018083528114156100e6575b505060186008602082066101e001602082840111156101bc57600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101f957600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b8261030051111561022b57610247565b6000610300516102c001535b815160010180835281141561021b575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b63c5f2892f60005114156103c357341561028457600080fd5b6000610140526002546101605261018060006020818352015b600160016101605116141561031e57600061018051602081106102bf57600080fd5b600160c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af161031057600080fd5b60c05190506101405261038c565b6000610140516020826101a0010152602081019050610180516020811061034457600080fd5b600060c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161038257600080fd5b60c0519050610140525b610160600261039a57600080fd5b60028151048152505b815160010180835281141561029d575b50506101405160005260206000f3005b63621fd13060005114156104995734156103dc57600080fd5b60606101c060246380673289610140526002546101605261015c6000305af161040457600080fd5b6101e0805160200180610260828460006004600a8704601201f161042757600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c051111561045957610475565b60006102c05161028001535b8151600101808352811415610449575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d600051141561125657606060046101403760506004356004016101a03760306004356004013511156104cf57600080fd5b60406024356004016102203760206024356004013511156104ef57600080fd5b608060443560040161028037606060443560040135111561050f57600080fd5b63ffffffff6002541061052157600080fd5b60306101a0511461053157600080fd5b6020610220511461054157600080fd5b6060610280511461055157600080fd5b633b9aca00610340526103405161056757600080fd5b61034051340461032052633b9aca0061032051101561058557600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af16105ae57600080fd5b610460805160200180610360828460006004600a8704601201f16105d157600080fd5b50506002546104a05260006104c0526104a05160016104a0510110156105f657600080fd5b60016104a051016104e05261050060006020818352015b600160016104e05116141561062157610674565b6104c060605160018251018060405190131561063c57600080fd5b809190121561064a57600080fd5b8152506104e0600261065b57600080fd5b60028151048152505b815160010180835281141561060d575b505060006101a06030806020846105e001018260208501600060046016f1505080518201915050600060106020820661056001602082840111156106b757600080fd5b60208061058082610520600060046015f15050818152809050905090506010806020846105e001018260208501600060046013f1505080518201915050806105e0526105e09050602060c0825160208401600060025af161071757600080fd5b60c051905061054052600060006040602082066106800161028051828401111561074057600080fd5b6060806106a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af161078057600080fd5b60c051905060208261088001015260208101905060006040602060208206610740016102805182840111156107b457600080fd5b606080610760826020602088068803016102800160006004601bf150508181528090509050905060208060208461080001018260208501600060046015f15050805182019150506105205160208261080001015260208101905080610800526108009050602060c0825160208401600060025af161083157600080fd5b60c051905060208261088001015260208101905080610880526108809050602060c0825160208401600060025af161086857600080fd5b60c051905061066052600060006105405160208261092001015260208101905061022060208060208461092001018260208501600060046015f150508051820191505080610920526109209050602060c0825160208401600060025af16108ce57600080fd5b60c0519050602082610aa00101526020810190506000610360600880602084610a2001018260208501600060046012f150508051820191505060006018602082066109a0016020828401111561092357600080fd5b6020806109c082610520600060046015f1505081815280905090509050601880602084610a2001018260208501600060046014f150508051820191505061066051602082610a2001015260208101905080610a2052610a209050602060c0825160208401600060025af161099657600080fd5b60c0519050602082610aa001015260208101905080610aa052610aa09050602060c0825160208401600060025af16109cd57600080fd5b60c051905061090052610b2060006020818352015b6104c051610b20511215610a62576000610b205160208110610a0357600080fd5b600160c052602060c0200154602082610b4001015260208101905061090051602082610b4001015260208101905080610b4052610b409050602060c0825160208401600060025af1610a5457600080fd5b60c051905061090052610a67565b610a78565b5b81516001018083528114156109e2575b5050610900516104c05160208110610a8f57600080fd5b600160c052602060c02001556002805460018254011015610aaf57600080fd5b60018154018155506060610c4060246380673289610bc0526104a051610be052610bdc6000305af1610ae057600080fd5b610c60805160200180610ca0828460006004600a8704601201f1610b0357600080fd5b505060a0610d2052610d2051610d60526101a0805160200180610d2051610d6001828460006004600a8704601201f1610b3b57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516040818352015b83610d0051101515610b7957610b96565b6000610d00516020850101535b8151600101808352811415610b68575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610d8052610220805160200180610d2051610d6001828460006004600a8704601201f1610bed57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610c2b57610c48565b6000610d00516020850101535b8151600101808352811415610c1a575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610da052610360805160200180610d2051610d6001828460006004600a8704601201f1610c9f57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610cdd57610cfa565b6000610d00516020850101535b8151600101808352811415610ccc575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610dc052610280805160200180610d2051610d6001828460006004600a8704601201f1610d5157600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516060818352015b83610d0051101515610d8f57610dac565b6000610d00516020850101535b8151600101808352811415610d7e575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610de052610ca0805160200180610d2051610d6001828460006004600a8704601201f1610e0357600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610e4157610e5e565b6000610d00516020850101535b8151600101808352811415610e30575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d20527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce610d2051610d60a164077359400061032051101515611254576003805460018254011015610ed357600080fd5b60018154018155506201000060035414156112535742610e205242610e405262015180610eff57600080fd5b62015180610e405106610e20511015610f1757600080fd5b42610e405262015180610f2957600080fd5b62015180610e405106610e2051036202a30042610e205242610e405262015180610f5257600080fd5b62015180610e405106610e20511015610f6a57600080fd5b42610e405262015180610f7c57600080fd5b62015180610e405106610e205103011015610f9657600080fd5b6202a30042610e205242610e405262015180610fb157600080fd5b62015180610e405106610e20511015610fc957600080fd5b42610e405262015180610fdb57600080fd5b62015180610e405106610e20510301610e00526020610ee0600463c5f2892f610e8052610e9c6000305af161100f57600080fd5b610ee051610e60526060610f8060246380673289610f0052600254610f2052610f1c6000305af161103f57600080fd5b610fa0805160200180610fe0828460006004600a8704601201f161106257600080fd5b505060606110c06024638067328961104052610e00516110605261105c6000305af161108d57600080fd5b6110e0805160200180611120828460006004600a8704601201f16110b057600080fd5b5050610e60516111e05260606111a0526111a05161120052610fe08051602001806111a0516111e001828460006004600a8704601201f16110f057600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b836111805110151561112e5761114b565b6000611180516020850101535b815160010180835281141561111d575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0526111a051611220526111208051602001806111a0516111e001828460006004600a8704601201f16111a257600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b83611180511015156111e0576111fd565b6000611180516020850101535b81516001018083528114156111cf575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0527f08b71ef3f1b58f7a23ffb82e27f12f0888c8403f1ceb0ea7ea26b274e2189d4c6111a0516111e0a160016004555b5b005b63845980e8600051141561127c57341561126f57600080fd5b60045460005260206000f3005b60006000fd5b61018561140703610185600039610185611407036000f3"} \ No newline at end of file +{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "merkle_tree_index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 7077}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 11026}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 255244}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600060c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600060c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600060c052602060c02001555b81516001018083528114156100aa575b505061103b56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561026b57602060046101403734156100b457600080fd5b67ffffffffffffffff6101405111156100cc57600080fd5b60006101605261014051610180526101a060006008818352015b6101605160086000811215610103578060000360020a820461010a565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561013557600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8600081121561017e578060000360020a8204610185565b8060020a82025b90509050610180525b81516001018083528114156100e6575b505060186008602082066101e001602082840111156101bc57600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101f957600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b8261030051111561022b57610247565b6000610300516102c001535b815160010180835281141561021b575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b63c5f2892f60005114156103c357341561028457600080fd5b6000610140526002546101605261018060006020818352015b600160016101605116141561031e57600061018051602081106102bf57600080fd5b600160c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af161031057600080fd5b60c05190506101405261038c565b6000610140516020826101a0010152602081019050610180516020811061034457600080fd5b600060c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161038257600080fd5b60c0519050610140525b610160600261039a57600080fd5b60028151048152505b815160010180835281141561029d575b50506101405160005260206000f3005b63621fd13060005114156104995734156103dc57600080fd5b60606101c060246380673289610140526002546101605261015c6000305af161040457600080fd5b6101e0805160200180610260828460006004600a8704601201f161042757600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c051111561045957610475565b60006102c05161028001535b8151600101808352811415610449575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d6000511415610eb057606060046101403760506004356004016101a03760306004356004013511156104cf57600080fd5b60406024356004016102203760206024356004013511156104ef57600080fd5b608060443560040161028037606060443560040135111561050f57600080fd5b63ffffffff6002541061052157600080fd5b60306101a0511461053157600080fd5b6020610220511461054157600080fd5b6060610280511461055157600080fd5b633b9aca00610340526103405161056757600080fd5b61034051340461032052633b9aca0061032051101561058557600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af16105ae57600080fd5b610460805160200180610360828460006004600a8704601201f16105d157600080fd5b50506002546104a05260006104c0526104a05160016104a0510110156105f657600080fd5b60016104a051016104e05261050060006020818352015b600160016104e05116141561062157610674565b6104c060605160018251018060405190131561063c57600080fd5b809190121561064a57600080fd5b8152506104e0600261065b57600080fd5b60028151048152505b815160010180835281141561060d575b505060006101a06030806020846105e001018260208501600060046016f1505080518201915050600060106020820661056001602082840111156106b757600080fd5b60208061058082610520600060046015f15050818152809050905090506010806020846105e001018260208501600060046013f1505080518201915050806105e0526105e09050602060c0825160208401600060025af161071757600080fd5b60c051905061054052600060006040602082066106800161028051828401111561074057600080fd5b6060806106a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af161078057600080fd5b60c051905060208261088001015260208101905060006040602060208206610740016102805182840111156107b457600080fd5b606080610760826020602088068803016102800160006004601bf150508181528090509050905060208060208461080001018260208501600060046015f15050805182019150506105205160208261080001015260208101905080610800526108009050602060c0825160208401600060025af161083157600080fd5b60c051905060208261088001015260208101905080610880526108809050602060c0825160208401600060025af161086857600080fd5b60c051905061066052600060006105405160208261092001015260208101905061022060208060208461092001018260208501600060046015f150508051820191505080610920526109209050602060c0825160208401600060025af16108ce57600080fd5b60c0519050602082610aa00101526020810190506000610360600880602084610a2001018260208501600060046012f150508051820191505060006018602082066109a0016020828401111561092357600080fd5b6020806109c082610520600060046015f1505081815280905090509050601880602084610a2001018260208501600060046014f150508051820191505061066051602082610a2001015260208101905080610a2052610a209050602060c0825160208401600060025af161099657600080fd5b60c0519050602082610aa001015260208101905080610aa052610aa09050602060c0825160208401600060025af16109cd57600080fd5b60c051905061090052610b2060006020818352015b6104c051610b20511215610a62576000610b205160208110610a0357600080fd5b600160c052602060c0200154602082610b4001015260208101905061090051602082610b4001015260208101905080610b4052610b409050602060c0825160208401600060025af1610a5457600080fd5b60c051905061090052610a67565b610a78565b5b81516001018083528114156109e2575b5050610900516104c05160208110610a8f57600080fd5b600160c052602060c02001556002805460018254011015610aaf57600080fd5b60018154018155506060610c4060246380673289610bc0526104a051610be052610bdc6000305af1610ae057600080fd5b610c60805160200180610ca0828460006004600a8704601201f1610b0357600080fd5b505060a0610d2052610d2051610d60526101a0805160200180610d2051610d6001828460006004600a8704601201f1610b3b57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516040818352015b83610d0051101515610b7957610b96565b6000610d00516020850101535b8151600101808352811415610b68575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610d8052610220805160200180610d2051610d6001828460006004600a8704601201f1610bed57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610c2b57610c48565b6000610d00516020850101535b8151600101808352811415610c1a575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610da052610360805160200180610d2051610d6001828460006004600a8704601201f1610c9f57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610cdd57610cfa565b6000610d00516020850101535b8151600101808352811415610ccc575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610dc052610280805160200180610d2051610d6001828460006004600a8704601201f1610d5157600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516060818352015b83610d0051101515610d8f57610dac565b6000610d00516020850101535b8151600101808352811415610d7e575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610de052610ca0805160200180610d2051610d6001828460006004600a8704601201f1610e0357600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610e4157610e5e565b6000610d00516020850101535b8151600101808352811415610e30575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d20527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce610d2051610d60a1005b60006000fd5b61018561103b0361018560003961018561103b036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index 1d475311ae..c67ed28410 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -1,8 +1,5 @@ MIN_DEPOSIT_AMOUNT: constant(uint256) = 1000000000 # Gwei -FULL_DEPOSIT_AMOUNT: constant(uint256) = 32000000000 # Gwei -CHAIN_START_FULL_DEPOSIT_THRESHOLD: constant(uint256) = 65536 # 2**16 DEPOSIT_CONTRACT_TREE_DEPTH: constant(uint256) = 32 -SECONDS_PER_DAY: constant(uint256) = 86400 MAX_64_BIT_VALUE: constant(uint256) = 18446744073709551615 # 2**64 - 1 PUBKEY_LENGTH: constant(uint256) = 48 # bytes WITHDRAWAL_CREDENTIALS_LENGTH: constant(uint256) = 32 # bytes @@ -16,13 +13,10 @@ signature: bytes[96], merkle_tree_index: bytes[8], }) -Eth2Genesis: event({deposit_root: bytes32, deposit_count: bytes[8], time: bytes[8]}) zerohashes: bytes32[DEPOSIT_CONTRACT_TREE_DEPTH] branch: bytes32[DEPOSIT_CONTRACT_TREE_DEPTH] deposit_count: uint256 -full_deposit_count: uint256 -chainStarted: public(bool) @public @@ -30,7 +24,6 @@ def __init__(): for i in range(DEPOSIT_CONTRACT_TREE_DEPTH - 1): self.zerohashes[i+1] = sha256(concat(self.zerohashes[i], self.zerohashes[i])) - @public @constant def to_little_endian_64(value: uint256) -> bytes[8]: @@ -47,7 +40,6 @@ def to_little_endian_64(value: uint256) -> bytes[8]: return slice(convert(y, bytes32), start=24, len=8) - @public @constant def get_deposit_root() -> bytes32: @@ -124,17 +116,3 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], signature, self.to_little_endian_64(index), ) - - if deposit_amount >= FULL_DEPOSIT_AMOUNT: - self.full_deposit_count += 1 - if self.full_deposit_count == CHAIN_START_FULL_DEPOSIT_THRESHOLD: - timestamp_day_boundary: uint256 = ( - as_unitless_number(block.timestamp) - - as_unitless_number(block.timestamp) % SECONDS_PER_DAY + - 2 * SECONDS_PER_DAY - ) - new_deposit_root: bytes32 = self.get_deposit_root() - log.Eth2Genesis(new_deposit_root, - self.to_little_endian_64(self.deposit_count), - self.to_little_endian_64(timestamp_day_boundary)) - self.chainStarted = True diff --git a/deposit_contract/tests/contracts/conftest.py b/deposit_contract/tests/contracts/conftest.py index 69ece247d6..b8b488abe3 100644 --- a/deposit_contract/tests/contracts/conftest.py +++ b/deposit_contract/tests/contracts/conftest.py @@ -26,7 +26,6 @@ # Constants MIN_DEPOSIT_AMOUNT = 1000000000 # Gwei FULL_DEPOSIT_AMOUNT = 32000000000 # Gwei -CHAIN_START_FULL_DEPOSIT_THRESHOLD = 65536 # 2**16 DEPOSIT_CONTRACT_TREE_DEPTH = 32 TWO_TO_POWER_OF_TREE_DEPTH = 2**DEPOSIT_CONTRACT_TREE_DEPTH @@ -63,28 +62,14 @@ def registration_contract(w3, tester): return registration_deployed -@pytest.fixture(scope="session") -def chain_start_full_deposit_thresholds(): - return [randint(1, 5), randint(6, 10), randint(11, 15)] - - @pytest.fixture(params=[0, 1, 2]) def modified_registration_contract( request, w3, - tester, - chain_start_full_deposit_thresholds): - # Set CHAIN_START_FULL_DEPOSIT_THRESHOLD to different threshold t + tester): registration_code = get_deposit_contract_code() - t = str(chain_start_full_deposit_thresholds[request.param]) - modified_registration_code = re.sub( - r'CHAIN_START_FULL_DEPOSIT_THRESHOLD: constant\(uint256\) = [0-9]+', - 'CHAIN_START_FULL_DEPOSIT_THRESHOLD: constant(uint256) = ' + t, - registration_code, - ) - assert modified_registration_code != registration_code - contract_bytecode = compiler.compile_code(modified_registration_code)['bytecode'] - contract_abi = compiler.mk_full_signature(modified_registration_code) + contract_bytecode = compiler.compile_code(registration_code)['bytecode'] + contract_abi = compiler.mk_full_signature(registration_code) registration = w3.eth.contract( abi=contract_abi, bytecode=contract_bytecode) @@ -94,11 +79,6 @@ def modified_registration_contract( address=tx_receipt.contractAddress, abi=contract_abi ) - setattr( - registration_deployed, - 'chain_start_full_deposit_threshold', - chain_start_full_deposit_thresholds[request.param] - ) return registration_deployed diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 8492d63478..25c1a73bbd 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -184,53 +184,3 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input leaf_nodes.append(hash_tree_root_result) root = compute_merkle_root(leaf_nodes) assert root == registration_contract.functions.get_deposit_root().call() - - -def test_chain_start(modified_registration_contract, w3, assert_tx_failed, deposit_input): - t = getattr(modified_registration_contract, 'chain_start_full_deposit_threshold') - # CHAIN_START_FULL_DEPOSIT_THRESHOLD is set to t - min_deposit_amount = MIN_DEPOSIT_AMOUNT * eth_utils.denoms.gwei # in wei - full_deposit_amount = FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei - log_filter = modified_registration_contract.events.Eth2Genesis.createFilter( - fromBlock='latest', - ) - - index_not_full_deposit = randint(0, t - 1) - for i in range(t): - if i == index_not_full_deposit: - # Deposit with value below FULL_DEPOSIT_AMOUNT - modified_registration_contract.functions.deposit( - *deposit_input, - ).transact({"value": min_deposit_amount}) - logs = log_filter.get_new_entries() - # Eth2Genesis event should not be triggered - assert len(logs) == 0 - else: - # Deposit with value FULL_DEPOSIT_AMOUNT - modified_registration_contract.functions.deposit( - *deposit_input, - ).transact({"value": full_deposit_amount}) - logs = log_filter.get_new_entries() - # Eth2Genesis event should not be triggered - assert len(logs) == 0 - - # Make 1 more deposit with value FULL_DEPOSIT_AMOUNT to trigger Eth2Genesis event - modified_registration_contract.functions.deposit( - *deposit_input, - ).transact({"value": full_deposit_amount}) - logs = log_filter.get_new_entries() - assert len(logs) == 1 - timestamp = int(w3.eth.getBlock(w3.eth.blockNumber)['timestamp']) - timestamp_day_boundary = timestamp + (86400 - timestamp % 86400) + 86400 - log = logs[0]['args'] - assert log['deposit_root'] == modified_registration_contract.functions.get_deposit_root().call() - assert int.from_bytes(log['time'], byteorder='little') == timestamp_day_boundary - assert modified_registration_contract.functions.chainStarted().call() is True - - # Make 1 deposit with value FULL_DEPOSIT_AMOUNT and - # check that Eth2Genesis event is not triggered - modified_registration_contract.functions.deposit( - *deposit_input, - ).transact({"value": full_deposit_amount}) - logs = log_filter.get_new_entries() - assert len(logs) == 0 diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a6d9d23c58..70e9e2b27d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -95,7 +95,7 @@ - [`initiate_validator_exit`](#initiate_validator_exit) - [`slash_validator`](#slash_validator) - [Genesis](#genesis) - - [`Eth2Genesis`](#eth2genesis) + - [Genesis trigger](#genesis-trigger) - [Genesis state](#genesis-state) - [Genesis block](#genesis-block) - [Beacon chain state transition function](#beacon-chain-state-transition-function) @@ -370,12 +370,12 @@ class PendingAttestation(Container): ```python class Eth1Data(Container): + # Block hash + block_hash: Bytes32 # Root of the deposit tree deposit_root: Bytes32 # Total number of deposits deposit_count: uint64 - # Block hash - block_hash: Bytes32 ``` #### `HistoricalBatch` @@ -1156,20 +1156,45 @@ def slash_validator(state: BeaconState, ## Genesis -### `Eth2Genesis` +### Genesis trigger + +Whenever the deposit contract emits a `Deposit` log call the function `is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool` where: + +* `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log +* `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log + +When `is_genesis_trigger(deposits, timestamp) == True` for the first time let: + +* `genesis_deposits = deposits` +* `genesis_time = timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` +* `genesis_eth1_data` be the object of type `Eth1Data` where: + * `genesis_eth1_data.block_hash` is the block hash corresponding to `deposits` + * `genesis_eth1_data.deposit_root` is the deposit root corresponding to `deposits` + * `genesis_eth1_data.deposit_count = len(genesis_deposits)` -When enough deposits of size `MAX_EFFECTIVE_BALANCE` have been made to the deposit contract an `Eth2Genesis` log is emitted triggering the genesis of the beacon chain. Let: +*Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: -* `eth2genesis` be the object corresponding to `Eth2Genesis` -* `genesis_eth1_data` be object of type `Eth1Data` where - * `genesis_eth1_data.deposit_root = eth2genesis.deposit_root` - * `genesis_eth1_data.deposit_count = eth2genesis.deposit_count` - * `genesis_eth1_data.block_hash` is the hash of the Ethereum 1.0 block that emitted the `Eth2Genesis` log -* `genesis_deposits` be the object of type `List[Deposit]` with deposits ordered chronologically up to and including the deposit that triggered the `Eth2Genesis` log +```python +def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: + # Process deposits + state = BeaconState() + for deposit in deposits: + process_deposit(state, deposit) + + # Count active validators at genesis + active_validator_count = 0 + for validator in state.validator_registry: + if validator.effective_balance >= MAX_EFFECTIVE_BALANCE: + active_validator_count += 1 + + # Check effective balance to trigger genesis + GENESIS_EFFECTIVE_BALANCE = 2**21 + return active_validator_count * MAX_EFFECTIVE_BALANCE == GENESIS_EFFECTIVE_BALANCE +``` ### Genesis state -Let `genesis_state = get_genesis_beacon_state(genesis_deposits, eth2genesis.genesis_time, genesis_eth1_data)`. +Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_data)`. ```python def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis_eth1_data: Eth1Data) -> BeaconState: @@ -1246,7 +1271,7 @@ def process_slot(state: BeaconState) -> None: ### Epoch processing -Note: the `# @LabelHere` lines below are placeholders to show that code will be inserted here in a future phase. +*Note*: the `# @LabelHere` lines below are placeholders to show that code will be inserted here in a future phase. ```python def process_epoch(state: BeaconState) -> None: diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index 5d93985894..047e992c08 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -9,7 +9,6 @@ - [Table of contents](#table-of-contents) - [Introduction](#introduction) - [Constants](#constants) - - [Gwei values](#gwei-values) - [Contract](#contract) - [Ethereum 1.0 deposit contract](#ethereum-10-deposit-contract) - [Arguments](#arguments) @@ -17,7 +16,6 @@ - [Amount](#amount) - [Event logs](#event-logs) - [`Deposit` logs](#deposit-logs) - - [`Eth2Genesis` log](#eth2genesis-log) - [Vyper code](#vyper-code) @@ -28,19 +26,12 @@ This document represents the specification for the beacon chain deposit contract ## Constants -### Gwei values - -| Name | Value | Unit | -| - | - | - | -| `FULL_DEPOSIT_AMOUNT` | `32 * 10**9` | Gwei | - ### Contract | Name | Value | | - | - | | `DEPOSIT_CONTRACT_ADDRESS` | **TBD** | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | -| `CHAIN_START_FULL_DEPOSIT_THRESHOLD` | `2**16` (= 65,536) | ## Ethereum 1.0 deposit contract @@ -62,7 +53,6 @@ The private key corresponding to `withdrawal_pubkey` will be required to initiat #### Amount * A valid deposit amount should be at least `MIN_DEPOSIT_AMOUNT` in Gwei. -* A deposit with an amount greater than or equal to `FULL_DEPOSIT_AMOUNT` in Gwei is considered as a full deposit. ## Event logs @@ -70,16 +60,6 @@ The private key corresponding to `withdrawal_pubkey` will be required to initiat Every Ethereum 1.0 deposit, of size at least `MIN_DEPOSIT_AMOUNT`, emits a `Deposit` log for consumption by the beacon chain. The deposit contract does little validation, pushing most of the validator onboarding logic to the beacon chain. In particular, the proof of possession (a BLS12-381 signature) is not verified by the deposit contract. -### `Eth2Genesis` log - -When `CHAIN_START_FULL_DEPOSIT_THRESHOLD` of full deposits have been made, the deposit contract emits the `Eth2Genesis` log. The beacon chain state may then be initialized by calling the `get_genesis_beacon_state` function (defined [here](./0_beacon-chain.md#genesis-state)) where: - -* `genesis_time` equals `time` in the `Eth2Genesis` log -* `latest_eth1_data.deposit_root` equals `deposit_root` in the `Eth2Genesis` log -* `latest_eth1_data.deposit_count` equals `deposit_count` in the `Eth2Genesis` log -* `latest_eth1_data.block_hash` equals the hash of the block that included the log -* `genesis_validator_deposits` is a list of `Deposit` objects built according to the `Deposit` logs up to the deposit that triggered the `Eth2Genesis` log, processed in the order in which they were emitted (oldest to newest) - ## Vyper code The source for the Vyper contract lives in a [separate repository](https://github.com/ethereum/deposit_contract) at [https://github.com/ethereum/deposit_contract/blob/master/deposit_contract/contracts/validator_registration.v.py](https://github.com/ethereum/deposit_contract/blob/master/deposit_contract/contracts/validator_registration.v.py). From 4ee00c9cbddd0872b2878897a1b56b43c0e56094 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 9 Jun 2019 11:03:38 +0100 Subject: [PATCH 002/405] Address HW's comments --- .../contracts/validator_registration.v.py | 4 ++ specs/core/0_beacon-chain.md | 12 +++--- specs/core/0_deposit-contract.md | 37 +++++++------------ 3 files changed, 24 insertions(+), 29 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index c67ed28410..0411bdb78a 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -24,6 +24,7 @@ def __init__(): for i in range(DEPOSIT_CONTRACT_TREE_DEPTH - 1): self.zerohashes[i+1] = sha256(concat(self.zerohashes[i], self.zerohashes[i])) + @public @constant def to_little_endian_64(value: uint256) -> bytes[8]: @@ -40,6 +41,7 @@ def to_little_endian_64(value: uint256) -> bytes[8]: return slice(convert(y, bytes32), start=24, len=8) + @public @constant def get_deposit_root() -> bytes32: @@ -53,11 +55,13 @@ def get_deposit_root() -> bytes32: size /= 2 return root + @public @constant def get_deposit_count() -> bytes[8]: return self.to_little_endian_64(self.deposit_count) + @payable @public def deposit(pubkey: bytes[PUBKEY_LENGTH], diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 70e9e2b27d..8cacde7465 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1163,13 +1163,13 @@ Whenever the deposit contract emits a `Deposit` log call the function `is_genesi * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log -When `is_genesis_trigger(deposits, timestamp) == True` for the first time let: +When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: * `genesis_deposits = deposits` * `genesis_time = timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` * `genesis_eth1_data` be the object of type `Eth1Data` where: - * `genesis_eth1_data.block_hash` is the block hash corresponding to `deposits` - * `genesis_eth1_data.deposit_root` is the deposit root corresponding to `deposits` + * `genesis_eth1_data.block_hash` is the block hash for the last deposit in `deposits` + * `genesis_eth1_data.deposit_root` is the deposit root for the last deposit in `deposits` * `genesis_eth1_data.deposit_count = len(genesis_deposits)` *Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: @@ -1184,12 +1184,12 @@ def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: # Count active validators at genesis active_validator_count = 0 for validator in state.validator_registry: - if validator.effective_balance >= MAX_EFFECTIVE_BALANCE: + if validator.effective_balance == MAX_EFFECTIVE_BALANCE: active_validator_count += 1 # Check effective balance to trigger genesis - GENESIS_EFFECTIVE_BALANCE = 2**21 - return active_validator_count * MAX_EFFECTIVE_BALANCE == GENESIS_EFFECTIVE_BALANCE + GENESIS_ACTIVE_VALIDATOR_COUNT = 2**16 + return active_validator_count == GENESIS_ACTIVE_VALIDATOR_COUNT ``` ### Genesis state diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index 047e992c08..50f52fd37d 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -11,11 +11,10 @@ - [Constants](#constants) - [Contract](#contract) - [Ethereum 1.0 deposit contract](#ethereum-10-deposit-contract) - - [Arguments](#arguments) + - [`deposit` function](#deposit-function) + - [Deposit amount](#deposit-amount) - [Withdrawal credentials](#withdrawal-credentials) - - [Amount](#amount) - - [Event logs](#event-logs) - - [`Deposit` logs](#deposit-logs) + - [`Deposit` log](#deposit-log) - [Vyper code](#vyper-code) @@ -37,37 +36,29 @@ This document represents the specification for the beacon chain deposit contract The initial deployment phases of Ethereum 2.0 are implemented without consensus changes to Ethereum 1.0. A deposit contract at address `DEPOSIT_CONTRACT_ADDRESS` is added to Ethereum 1.0 for deposits of ETH to the beacon chain. Validator balances will be withdrawable to the shards in Phase 2 (i.e. when the EVM 2.0 is deployed and the shards have state). -### Arguments +### `deposit` function -The deposit contract has a `deposit` function which takes the amount in Ethereum 1.0 transaction, and arguments `pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96]` corresponding to `DepositData`. +The deposit contract has a public `deposit` function to make deposits. It takes as arguments `pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96]` corresponding to a `DepositData` object. + +#### Deposit amount + +The ETH sent to the deposit contract, i.e. the deposit amount, is burnt on Ethereum 1.0. The minimum deposit amount is `MIN_DEPOSIT_AMOUNT` Gwei. #### Withdrawal credentials -One of the `DepositData` fields is `withdrawal_credentials`. It is a commitment to credentials for withdrawals to shards. The first byte of `withdrawal_credentials` is a version number. As of now, the only expected format is as follows: +One of the `DepositData` fields is `withdrawal_credentials`. It is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). The first byte of `withdrawal_credentials` is a version number. As of now, the only expected format is as follows: * `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX_BYTE` * `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]` where `withdrawal_pubkey` is a BLS pubkey The private key corresponding to `withdrawal_pubkey` will be required to initiate a withdrawal. It can be stored separately until a withdrawal is required, e.g. in cold storage. -#### Amount - -* A valid deposit amount should be at least `MIN_DEPOSIT_AMOUNT` in Gwei. - -## Event logs +#### `Deposit` log -### `Deposit` logs - -Every Ethereum 1.0 deposit, of size at least `MIN_DEPOSIT_AMOUNT`, emits a `Deposit` log for consumption by the beacon chain. The deposit contract does little validation, pushing most of the validator onboarding logic to the beacon chain. In particular, the proof of possession (a BLS12-381 signature) is not verified by the deposit contract. +Every Ethereum 1.0 deposit emits a `Deposit` log for consumption by the beacon chain. The deposit contract does little validation, pushing most of the validator onboarding logic to the beacon chain. In particular, the proof of possession (a BLS12-381 signature) is not verified by the deposit contract. ## Vyper code -The source for the Vyper contract lives in a [separate repository](https://github.com/ethereum/deposit_contract) at [https://github.com/ethereum/deposit_contract/blob/master/deposit_contract/contracts/validator_registration.v.py](https://github.com/ethereum/deposit_contract/blob/master/deposit_contract/contracts/validator_registration.v.py). - -*Note*: To save ~10x on gas, this contract uses a somewhat unintuitive progressive Merkle root calculation algo that requires only O(log(n)) storage. See https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py for an implementation of the same algo in Python tested for correctness. - -For convenience, we provide the interface to the contract here: +The deposit contract source code, written in Vyper, is available [here](https://github.com/ethereum/eth2.0-specs/blob/dev/deposit_contract/contracts/validator_registration.v.py). -* `__init__()`: initializes the contract -* `get_deposit_root() -> bytes32`: returns the current root of the deposit tree -* `deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])`: adds a deposit instance to the deposit tree, incorporating the input arguments and the value transferred in the given call. *Note*: The amount of value transferred *must* be at least `MIN_DEPOSIT_AMOUNT`. Each of these constants are specified in units of Gwei. +*Note*: To save on gas the deposit contract uses a progressive Merkle root calculation algorithm that requires only O(log(n)) storage. See [here](https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py) for a Python implementation, and [here](https://github.com/runtimeverification/verified-smart-contracts/blob/master/deposit/formal-incremental-merkle-tree-algorithm.pdf) for a formal correctness proof. From c293b9dcef4d25a3a0ae4ebfe2a0405811b0d86d Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 9 Jun 2019 11:29:22 +0100 Subject: [PATCH 003/405] Cleanups --- .../contracts/validator_registration.json | 2 +- .../contracts/validator_registration.v.py | 91 +++++++------------ .../tests/contracts/test_deposit.py | 5 +- 3 files changed, 36 insertions(+), 62 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index a885ef0a17..9da4a015c4 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "merkle_tree_index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 7077}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 11026}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 255244}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600060c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600060c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600060c052602060c02001555b81516001018083528114156100aa575b505061103b56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561026b57602060046101403734156100b457600080fd5b67ffffffffffffffff6101405111156100cc57600080fd5b60006101605261014051610180526101a060006008818352015b6101605160086000811215610103578060000360020a820461010a565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561013557600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8600081121561017e578060000360020a8204610185565b8060020a82025b90509050610180525b81516001018083528114156100e6575b505060186008602082066101e001602082840111156101bc57600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101f957600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b8261030051111561022b57610247565b6000610300516102c001535b815160010180835281141561021b575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b63c5f2892f60005114156103c357341561028457600080fd5b6000610140526002546101605261018060006020818352015b600160016101605116141561031e57600061018051602081106102bf57600080fd5b600160c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af161031057600080fd5b60c05190506101405261038c565b6000610140516020826101a0010152602081019050610180516020811061034457600080fd5b600060c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161038257600080fd5b60c0519050610140525b610160600261039a57600080fd5b60028151048152505b815160010180835281141561029d575b50506101405160005260206000f3005b63621fd13060005114156104995734156103dc57600080fd5b60606101c060246380673289610140526002546101605261015c6000305af161040457600080fd5b6101e0805160200180610260828460006004600a8704601201f161042757600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c051111561045957610475565b60006102c05161028001535b8151600101808352811415610449575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d6000511415610eb057606060046101403760506004356004016101a03760306004356004013511156104cf57600080fd5b60406024356004016102203760206024356004013511156104ef57600080fd5b608060443560040161028037606060443560040135111561050f57600080fd5b63ffffffff6002541061052157600080fd5b60306101a0511461053157600080fd5b6020610220511461054157600080fd5b6060610280511461055157600080fd5b633b9aca00610340526103405161056757600080fd5b61034051340461032052633b9aca0061032051101561058557600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af16105ae57600080fd5b610460805160200180610360828460006004600a8704601201f16105d157600080fd5b50506002546104a05260006104c0526104a05160016104a0510110156105f657600080fd5b60016104a051016104e05261050060006020818352015b600160016104e05116141561062157610674565b6104c060605160018251018060405190131561063c57600080fd5b809190121561064a57600080fd5b8152506104e0600261065b57600080fd5b60028151048152505b815160010180835281141561060d575b505060006101a06030806020846105e001018260208501600060046016f1505080518201915050600060106020820661056001602082840111156106b757600080fd5b60208061058082610520600060046015f15050818152809050905090506010806020846105e001018260208501600060046013f1505080518201915050806105e0526105e09050602060c0825160208401600060025af161071757600080fd5b60c051905061054052600060006040602082066106800161028051828401111561074057600080fd5b6060806106a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af161078057600080fd5b60c051905060208261088001015260208101905060006040602060208206610740016102805182840111156107b457600080fd5b606080610760826020602088068803016102800160006004601bf150508181528090509050905060208060208461080001018260208501600060046015f15050805182019150506105205160208261080001015260208101905080610800526108009050602060c0825160208401600060025af161083157600080fd5b60c051905060208261088001015260208101905080610880526108809050602060c0825160208401600060025af161086857600080fd5b60c051905061066052600060006105405160208261092001015260208101905061022060208060208461092001018260208501600060046015f150508051820191505080610920526109209050602060c0825160208401600060025af16108ce57600080fd5b60c0519050602082610aa00101526020810190506000610360600880602084610a2001018260208501600060046012f150508051820191505060006018602082066109a0016020828401111561092357600080fd5b6020806109c082610520600060046015f1505081815280905090509050601880602084610a2001018260208501600060046014f150508051820191505061066051602082610a2001015260208101905080610a2052610a209050602060c0825160208401600060025af161099657600080fd5b60c0519050602082610aa001015260208101905080610aa052610aa09050602060c0825160208401600060025af16109cd57600080fd5b60c051905061090052610b2060006020818352015b6104c051610b20511215610a62576000610b205160208110610a0357600080fd5b600160c052602060c0200154602082610b4001015260208101905061090051602082610b4001015260208101905080610b4052610b409050602060c0825160208401600060025af1610a5457600080fd5b60c051905061090052610a67565b610a78565b5b81516001018083528114156109e2575b5050610900516104c05160208110610a8f57600080fd5b600160c052602060c02001556002805460018254011015610aaf57600080fd5b60018154018155506060610c4060246380673289610bc0526104a051610be052610bdc6000305af1610ae057600080fd5b610c60805160200180610ca0828460006004600a8704601201f1610b0357600080fd5b505060a0610d2052610d2051610d60526101a0805160200180610d2051610d6001828460006004600a8704601201f1610b3b57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516040818352015b83610d0051101515610b7957610b96565b6000610d00516020850101535b8151600101808352811415610b68575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610d8052610220805160200180610d2051610d6001828460006004600a8704601201f1610bed57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610c2b57610c48565b6000610d00516020850101535b8151600101808352811415610c1a575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610da052610360805160200180610d2051610d6001828460006004600a8704601201f1610c9f57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610cdd57610cfa565b6000610d00516020850101535b8151600101808352811415610ccc575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610dc052610280805160200180610d2051610d6001828460006004600a8704601201f1610d5157600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516060818352015b83610d0051101515610d8f57610dac565b6000610d00516020850101535b8151600101808352811415610d7e575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610de052610ca0805160200180610d2051610d6001828460006004600a8704601201f1610e0357600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610e4157610e5e565b6000610d00516020850101535b8151600101808352811415610e30575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d20527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce610d2051610d60a1005b60006000fd5b61018561103b0361018560003961018561103b036000f3"} \ No newline at end of file +{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 6973}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10922}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1318334}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b5050610e9956600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561025357602060046101403734156100b457600080fd5b60006101605261014051610180526101a060006008818352015b61016051600860008112156100eb578060000360020a82046100f2565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561011d57600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610166578060000360020a820461016d565b8060020a82025b90509050610180525b81516001018083528114156100ce575b505060186008602082066101e001602082840111156101a457600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101e157600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b826103005111156102135761022f565b6000610300516102c001535b8151600101808352811415610203575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b63c5f2892f60005114156103ab57341561026c57600080fd5b6000610140526001546101605261018060006020818352015b600160016101605116141561030657600061018051602081106102a757600080fd5b600060c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af16102f857600080fd5b60c051905061014052610374565b6000610140516020826101a0010152602081019050610180516020811061032c57600080fd5b600260c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161036a57600080fd5b60c0519050610140525b610160600261038257600080fd5b60028151048152505b8151600101808352811415610285575b50506101405160005260206000f3005b63621fd13060005114156104815734156103c457600080fd5b60606101c060246380673289610140526001546101605261015c6000305af16103ec57600080fd5b6101e0805160200180610260828460006004600a8704601201f161040f57600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c05111156104415761045d565b60006102c05161028001535b8151600101808352811415610431575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d6000511415610d0e57606060046101403760506004356004016101a03760306004356004013511156104b757600080fd5b60406024356004016102203760206024356004013511156104d757600080fd5b60806044356004016102803760606044356004013511156104f757600080fd5b63ffffffff6001541061050957600080fd5b633b9aca00610340526103405161051f57600080fd5b61034051340461032052633b9aca0061032051101561053d57600080fd5b60306101a0511461054d57600080fd5b6020610220511461055d57600080fd5b6060610280511461056d57600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af161059657600080fd5b610460805160200180610360828460006004600a8704601201f16105b957600080fd5b505060806104c0526104c051610500526101a08051602001806104c05161050001828460006004600a8704601201f16105f157600080fd5b50506104c051610500015160206001820306601f82010390506104c051610500016104a081516040818352015b836104a05110151561062f5761064c565b60006104a0516020850101535b815160010180835281141561061e575b5050505060206104c051610500015160206001820306601f82010390506104c05101016104c0526104c051610520526102208051602001806104c05161050001828460006004600a8704601201f16106a357600080fd5b50506104c051610500015160206001820306601f82010390506104c051610500016104a081516020818352015b836104a0511015156106e1576106fe565b60006104a0516020850101535b81516001018083528114156106d0575b5050505060206104c051610500015160206001820306601f82010390506104c05101016104c0526104c051610540526103608051602001806104c05161050001828460006004600a8704601201f161075557600080fd5b50506104c051610500015160206001820306601f82010390506104c051610500016104a081516020818352015b836104a051101515610793576107b0565b60006104a0516020850101535b8151600101808352811415610782575b5050505060206104c051610500015160206001820306601f82010390506104c05101016104c0526104c051610560526102808051602001806104c05161050001828460006004600a8704601201f161080757600080fd5b50506104c051610500015160206001820306601f82010390506104c051610500016104a081516060818352015b836104a05110151561084557610862565b60006104a0516020850101535b8151600101808352811415610834575b5050505060206104c051610500015160206001820306601f82010390506104c05101016104c0527f9d0a206c338cfcaf9198c04fe61b39a988e26b623ef97cb2f72bafcfbf8bb93e6104c051610500a160006101a060308060208461064001018260208501600060046016f150508051820191505060006010602082066105c001602082840111156108f357600080fd5b6020806105e082610580600060046015f150508181528090509050905060108060208461064001018260208501600060046013f150508051820191505080610640526106409050602060c0825160208401600060025af161095357600080fd5b60c05190506105a052600060006040602082066106e00161028051828401111561097c57600080fd5b606080610700826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af16109bc57600080fd5b60c05190506020826108e0010152602081019050600060406020602082066107a0016102805182840111156109f057600080fd5b6060806107c0826020602088068803016102800160006004601bf150508181528090509050905060208060208461086001018260208501600060046015f15050805182019150506105805160208261086001015260208101905080610860526108609050602060c0825160208401600060025af1610a6d57600080fd5b60c05190506020826108e0010152602081019050806108e0526108e09050602060c0825160208401600060025af1610aa457600080fd5b60c05190506106c052600060006105a05160208261098001015260208101905061022060208060208461098001018260208501600060046015f150508051820191505080610980526109809050602060c0825160208401600060025af1610b0a57600080fd5b60c0519050602082610b000101526020810190506000610360600880602084610a8001018260208501600060046012f15050805182019150506000601860208206610a000160208284011115610b5f57600080fd5b602080610a2082610580600060046015f1505081815280905090509050601880602084610a8001018260208501600060046014f15050805182019150506106c051602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610bd257600080fd5b60c0519050602082610b0001015260208101905080610b0052610b009050602060c0825160208401600060025af1610c0957600080fd5b60c0519050610960526001805460018254011015610c2657600080fd5b6001815401815550600154610b8052610ba060006020818352015b60016001610b8051161415610c765761096051610ba05160208110610c6557600080fd5b600060c052602060c0200155610d0a565b6000610ba05160208110610c8957600080fd5b600060c052602060c0200154602082610bc001015260208101905061096051602082610bc001015260208101905080610bc052610bc09050602060c0825160208401600060025af1610cda57600080fd5b60c051905061096052610b806002610cf157600080fd5b60028151048152505b8151600101808352811415610c41575b5050005b60006000fd5b610185610e9903610185600039610185610e99036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index 0411bdb78a..6c4bd7dc7a 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -1,59 +1,54 @@ MIN_DEPOSIT_AMOUNT: constant(uint256) = 1000000000 # Gwei DEPOSIT_CONTRACT_TREE_DEPTH: constant(uint256) = 32 -MAX_64_BIT_VALUE: constant(uint256) = 18446744073709551615 # 2**64 - 1 PUBKEY_LENGTH: constant(uint256) = 48 # bytes WITHDRAWAL_CREDENTIALS_LENGTH: constant(uint256) = 32 # bytes +AMOUNT_LENGTH: constant(uint256) = 8 # bytes SIGNATURE_LENGTH: constant(uint256) = 96 # bytes -MAX_DEPOSIT_COUNT: constant(uint256) = 4294967295 # 2**DEPOSIT_CONTRACT_TREE_DEPTH - 1 Deposit: event({ pubkey: bytes[48], withdrawal_credentials: bytes[32], amount: bytes[8], signature: bytes[96], - merkle_tree_index: bytes[8], }) -zerohashes: bytes32[DEPOSIT_CONTRACT_TREE_DEPTH] branch: bytes32[DEPOSIT_CONTRACT_TREE_DEPTH] deposit_count: uint256 - +# Compute hashes in empty sparse Merkle tree +zero_hashes: bytes32[DEPOSIT_CONTRACT_TREE_DEPTH] @public def __init__(): for i in range(DEPOSIT_CONTRACT_TREE_DEPTH - 1): - self.zerohashes[i+1] = sha256(concat(self.zerohashes[i], self.zerohashes[i])) + self.zero_hashes[i + 1] = sha256(concat(self.zero_hashes[i], self.zero_hashes[i])) @public @constant def to_little_endian_64(value: uint256) -> bytes[8]: - assert value <= MAX_64_BIT_VALUE - - # array access for bytes[] not currently supported in vyper so - # reversing bytes using bitwise uint256 manipulations + # Reversing bytes using bitwise uint256 manipulations + # (array accesses of bytes[] are not currently supported in Vyper) y: uint256 = 0 x: uint256 = value - for i in range(8): + for _ in range(8): y = shift(y, 8) y = y + bitwise_and(x, 255) x = shift(x, -8) - return slice(convert(y, bytes32), start=24, len=8) @public @constant def get_deposit_root() -> bytes32: - root: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000 + node: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000 size: uint256 = self.deposit_count - for h in range(DEPOSIT_CONTRACT_TREE_DEPTH): - if bitwise_and(size, 1) == 1: - root = sha256(concat(self.branch[h], root)) + for height in range(DEPOSIT_CONTRACT_TREE_DEPTH): + if bitwise_and(size, 1) == 1: # More gas efficient than `size % 2 == 1` + node = sha256(concat(self.branch[height], node)) else: - root = sha256(concat(root, self.zerohashes[h])) + node = sha256(concat(node, self.zero_hashes[height])) size /= 2 - return root + return node @public @@ -67,56 +62,38 @@ def get_deposit_count() -> bytes[8]: def deposit(pubkey: bytes[PUBKEY_LENGTH], withdrawal_credentials: bytes[WITHDRAWAL_CREDENTIALS_LENGTH], signature: bytes[SIGNATURE_LENGTH]): - # Prevent edge case in computing `self.branch` when `self.deposit_count == MAX_DEPOSIT_COUNT` - # NOTE: reaching this point with the constants as currently defined is impossible due to the - # uni-directional nature of transfers from eth1 to eth2 and the total ether supply (< 130M). - assert self.deposit_count < MAX_DEPOSIT_COUNT + # Avoid overflowing the Merkle tree + assert self.deposit_count < 2**DEPOSIT_CONTRACT_TREE_DEPTH - 1 + # Validate deposit data + deposit_amount: uint256 = msg.value / as_wei_value(1, "gwei") + assert deposit_amount >= MIN_DEPOSIT_AMOUNT assert len(pubkey) == PUBKEY_LENGTH assert len(withdrawal_credentials) == WITHDRAWAL_CREDENTIALS_LENGTH assert len(signature) == SIGNATURE_LENGTH - deposit_amount: uint256 = msg.value / as_wei_value(1, "gwei") - assert deposit_amount >= MIN_DEPOSIT_AMOUNT + # Emit `Deposit` log amount: bytes[8] = self.to_little_endian_64(deposit_amount) + log.Deposit(pubkey, withdrawal_credentials, amount, signature) - index: uint256 = self.deposit_count - - # add deposit to merkle tree - i: int128 = 0 - size: uint256 = index + 1 - for _ in range(DEPOSIT_CONTRACT_TREE_DEPTH): - if bitwise_and(size, 1) == 1: - break - i += 1 - size /= 2 - - zero_bytes_32: bytes32 - pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes_32, start=0, len=16))) + # Compute `DepositData` root + zero_bytes32: bytes32 + pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes32, start=0, len=64 - PUBKEY_LENGTH))) signature_root: bytes32 = sha256(concat( sha256(slice(signature, start=0, len=64)), - sha256(concat(slice(signature, start=64, len=32), zero_bytes_32)) + sha256(concat(slice(signature, start=64, len=SIGNATURE_LENGTH - 64), zero_bytes32)), )) - value: bytes32 = sha256(concat( + node: bytes32 = sha256(concat( sha256(concat(pubkey_root, withdrawal_credentials)), - sha256(concat( - amount, - slice(zero_bytes_32, start=0, len=24), - signature_root, - )) + sha256(concat(amount, slice(zero_bytes32, start=0, len=32 - AMOUNT_LENGTH), signature_root)), )) - for j in range(DEPOSIT_CONTRACT_TREE_DEPTH): - if j < i: - value = sha256(concat(self.branch[j], value)) - else: - break - self.branch[i] = value + # Add `DepositData` root to Merkle tree (update a single `branch` node) self.deposit_count += 1 - log.Deposit( - pubkey, - withdrawal_credentials, - amount, - signature, - self.to_little_endian_64(index), - ) + size: uint256 = self.deposit_count + for height in range(DEPOSIT_CONTRACT_TREE_DEPTH): + if bitwise_and(size, 1) == 1: # More gas efficient than `size % 2 == 1` + self.branch[height] = node + break + node = sha256(concat(self.branch[height], node)) + size /= 2 diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 25c1a73bbd..c1c07aac97 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -56,7 +56,7 @@ def deposit_input(): (10, True), (55555, True), (2**64 - 1, True), - (2**64, False), + (2**64, True), # Note that all calls to `to_little_endian_64` have an input less than 2**64 ] ) def test_to_little_endian_64(registration_contract, value, success, assert_tx_failed): @@ -151,7 +151,6 @@ def test_deposit_log(registration_contract, a0, w3, deposit_input): assert log['withdrawal_credentials'] == deposit_input[1] assert log['amount'] == deposit_amount_list[i].to_bytes(8, 'little') assert log['signature'] == deposit_input[2] - assert log['merkle_tree_index'] == i.to_bytes(8, 'little') def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input): @@ -172,8 +171,6 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input assert len(logs) == 1 log = logs[0]['args'] - assert log["merkle_tree_index"] == i.to_bytes(8, 'little') - deposit_data = DepositData( pubkey=deposit_input[0], withdrawal_credentials=deposit_input[1], From 29129d06cf762889312ed6eeeec783751d76cda7 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 9 Jun 2019 17:03:35 +0100 Subject: [PATCH 004/405] Fix tests --- deposit_contract/contracts/validator_registration.json | 2 +- deposit_contract/contracts/validator_registration.v.py | 8 ++++---- deposit_contract/tests/contracts/test_deposit.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index 9da4a015c4..676bf470ab 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 6973}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10922}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1318334}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b5050610e9956600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561025357602060046101403734156100b457600080fd5b60006101605261014051610180526101a060006008818352015b61016051600860008112156100eb578060000360020a82046100f2565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561011d57600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610166578060000360020a820461016d565b8060020a82025b90509050610180525b81516001018083528114156100ce575b505060186008602082066101e001602082840111156101a457600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101e157600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b826103005111156102135761022f565b6000610300516102c001535b8151600101808352811415610203575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b63c5f2892f60005114156103ab57341561026c57600080fd5b6000610140526001546101605261018060006020818352015b600160016101605116141561030657600061018051602081106102a757600080fd5b600060c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af16102f857600080fd5b60c051905061014052610374565b6000610140516020826101a0010152602081019050610180516020811061032c57600080fd5b600260c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161036a57600080fd5b60c0519050610140525b610160600261038257600080fd5b60028151048152505b8151600101808352811415610285575b50506101405160005260206000f3005b63621fd13060005114156104815734156103c457600080fd5b60606101c060246380673289610140526001546101605261015c6000305af16103ec57600080fd5b6101e0805160200180610260828460006004600a8704601201f161040f57600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c05111156104415761045d565b60006102c05161028001535b8151600101808352811415610431575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d6000511415610d0e57606060046101403760506004356004016101a03760306004356004013511156104b757600080fd5b60406024356004016102203760206024356004013511156104d757600080fd5b60806044356004016102803760606044356004013511156104f757600080fd5b63ffffffff6001541061050957600080fd5b633b9aca00610340526103405161051f57600080fd5b61034051340461032052633b9aca0061032051101561053d57600080fd5b60306101a0511461054d57600080fd5b6020610220511461055d57600080fd5b6060610280511461056d57600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af161059657600080fd5b610460805160200180610360828460006004600a8704601201f16105b957600080fd5b505060806104c0526104c051610500526101a08051602001806104c05161050001828460006004600a8704601201f16105f157600080fd5b50506104c051610500015160206001820306601f82010390506104c051610500016104a081516040818352015b836104a05110151561062f5761064c565b60006104a0516020850101535b815160010180835281141561061e575b5050505060206104c051610500015160206001820306601f82010390506104c05101016104c0526104c051610520526102208051602001806104c05161050001828460006004600a8704601201f16106a357600080fd5b50506104c051610500015160206001820306601f82010390506104c051610500016104a081516020818352015b836104a0511015156106e1576106fe565b60006104a0516020850101535b81516001018083528114156106d0575b5050505060206104c051610500015160206001820306601f82010390506104c05101016104c0526104c051610540526103608051602001806104c05161050001828460006004600a8704601201f161075557600080fd5b50506104c051610500015160206001820306601f82010390506104c051610500016104a081516020818352015b836104a051101515610793576107b0565b60006104a0516020850101535b8151600101808352811415610782575b5050505060206104c051610500015160206001820306601f82010390506104c05101016104c0526104c051610560526102808051602001806104c05161050001828460006004600a8704601201f161080757600080fd5b50506104c051610500015160206001820306601f82010390506104c051610500016104a081516060818352015b836104a05110151561084557610862565b60006104a0516020850101535b8151600101808352811415610834575b5050505060206104c051610500015160206001820306601f82010390506104c05101016104c0527f9d0a206c338cfcaf9198c04fe61b39a988e26b623ef97cb2f72bafcfbf8bb93e6104c051610500a160006101a060308060208461064001018260208501600060046016f150508051820191505060006010602082066105c001602082840111156108f357600080fd5b6020806105e082610580600060046015f150508181528090509050905060108060208461064001018260208501600060046013f150508051820191505080610640526106409050602060c0825160208401600060025af161095357600080fd5b60c05190506105a052600060006040602082066106e00161028051828401111561097c57600080fd5b606080610700826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af16109bc57600080fd5b60c05190506020826108e0010152602081019050600060406020602082066107a0016102805182840111156109f057600080fd5b6060806107c0826020602088068803016102800160006004601bf150508181528090509050905060208060208461086001018260208501600060046015f15050805182019150506105805160208261086001015260208101905080610860526108609050602060c0825160208401600060025af1610a6d57600080fd5b60c05190506020826108e0010152602081019050806108e0526108e09050602060c0825160208401600060025af1610aa457600080fd5b60c05190506106c052600060006105a05160208261098001015260208101905061022060208060208461098001018260208501600060046015f150508051820191505080610980526109809050602060c0825160208401600060025af1610b0a57600080fd5b60c0519050602082610b000101526020810190506000610360600880602084610a8001018260208501600060046012f15050805182019150506000601860208206610a000160208284011115610b5f57600080fd5b602080610a2082610580600060046015f1505081815280905090509050601880602084610a8001018260208501600060046014f15050805182019150506106c051602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610bd257600080fd5b60c0519050602082610b0001015260208101905080610b0052610b009050602060c0825160208401600060025af1610c0957600080fd5b60c0519050610960526001805460018254011015610c2657600080fd5b6001815401815550600154610b8052610ba060006020818352015b60016001610b8051161415610c765761096051610ba05160208110610c6557600080fd5b600060c052602060c0200155610d0a565b6000610ba05160208110610c8957600080fd5b600060c052602060c0200154602082610bc001015260208101905061096051602082610bc001015260208101905080610bc052610bc09050602060c0825160208401600060025af1610cda57600080fd5b60c051905061096052610b806002610cf157600080fd5b60028151048152505b8151600101808352811415610c41575b5050005b60006000fd5b610185610e9903610185600039610185610e99036000f3"} \ No newline at end of file +{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 6973}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10922}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1318334}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b5050610e9956600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561025357602060046101403734156100b457600080fd5b60006101605261014051610180526101a060006008818352015b61016051600860008112156100eb578060000360020a82046100f2565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561011d57600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610166578060000360020a820461016d565b8060020a82025b90509050610180525b81516001018083528114156100ce575b505060186008602082066101e001602082840111156101a457600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101e157600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b826103005111156102135761022f565b6000610300516102c001535b8151600101808352811415610203575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b63c5f2892f60005114156103ab57341561026c57600080fd5b6000610140526001546101605261018060006020818352015b600160016101605116141561030657600061018051602081106102a757600080fd5b600060c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af16102f857600080fd5b60c051905061014052610374565b6000610140516020826101a0010152602081019050610180516020811061032c57600080fd5b600260c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161036a57600080fd5b60c0519050610140525b610160600261038257600080fd5b60028151048152505b8151600101808352811415610285575b50506101405160005260206000f3005b63621fd13060005114156104815734156103c457600080fd5b60606101c060246380673289610140526001546101605261015c6000305af16103ec57600080fd5b6101e0805160200180610260828460006004600a8704601201f161040f57600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c05111156104415761045d565b60006102c05161028001535b8151600101808352811415610431575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d6000511415610d0e57606060046101403760506004356004016101a03760306004356004013511156104b757600080fd5b60406024356004016102203760206024356004013511156104d757600080fd5b60806044356004016102803760606044356004013511156104f757600080fd5b63ffffffff6001541061050957600080fd5b633b9aca00610340526103405161051f57600080fd5b61034051340461032052633b9aca0061032051101561053d57600080fd5b60306101a0511461054d57600080fd5b6020610220511461055d57600080fd5b6060610280511461056d57600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af161059657600080fd5b610460805160200180610360828460006004600a8704601201f16105b957600080fd5b505060006101a060308060208461056001018260208501600060046016f150508051820191505060006010602082066104e001602082840111156105fc57600080fd5b602080610500826104a0600060046015f150508181528090509050905060108060208461056001018260208501600060046013f150508051820191505080610560526105609050602060c0825160208401600060025af161065c57600080fd5b60c05190506104c052600060006040602082066106000161028051828401111561068557600080fd5b606080610620826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af16106c557600080fd5b60c0519050602082610800010152602081019050600060406020602082066106c0016102805182840111156106f957600080fd5b6060806106e0826020602088068803016102800160006004601bf150508181528090509050905060208060208461078001018260208501600060046015f15050805182019150506104a05160208261078001015260208101905080610780526107809050602060c0825160208401600060025af161077657600080fd5b60c051905060208261080001015260208101905080610800526108009050602060c0825160208401600060025af16107ad57600080fd5b60c05190506105e052600060006104c0516020826108a00101526020810190506102206020806020846108a001018260208501600060046015f1505080518201915050806108a0526108a09050602060c0825160208401600060025af161081357600080fd5b60c0519050602082610a2001015260208101905060006103606008806020846109a001018260208501600060046012f15050805182019150506000601860208206610920016020828401111561086857600080fd5b602080610940826104a0600060046015f15050818152809050905090506018806020846109a001018260208501600060046014f15050805182019150506105e0516020826109a0010152602081019050806109a0526109a09050602060c0825160208401600060025af16108db57600080fd5b60c0519050602082610a2001015260208101905080610a2052610a209050602060c0825160208401600060025af161091257600080fd5b60c051905061088052600180546001825401101561092f57600080fd5b6001815401815550600154610aa052610ac060006020818352015b60016001610aa05116141561097f5761088051610ac0516020811061096e57600080fd5b600060c052602060c0200155610a13565b6000610ac0516020811061099257600080fd5b600060c052602060c0200154602082610ae001015260208101905061088051602082610ae001015260208101905080610ae052610ae09050602060c0825160208401600060025af16109e357600080fd5b60c051905061088052610aa060026109fa57600080fd5b60028151048152505b815160010180835281141561094a575b50506080610b8052610b8051610bc0526101a0805160200180610b8051610bc001828460006004600a8704601201f1610a4b57600080fd5b5050610b8051610bc0015160206001820306601f8201039050610b8051610bc001610b6081516040818352015b83610b6051101515610a8957610aa6565b6000610b60516020850101535b8151600101808352811415610a78575b505050506020610b8051610bc0015160206001820306601f8201039050610b80510101610b8052610b8051610be052610220805160200180610b8051610bc001828460006004600a8704601201f1610afd57600080fd5b5050610b8051610bc0015160206001820306601f8201039050610b8051610bc001610b6081516020818352015b83610b6051101515610b3b57610b58565b6000610b60516020850101535b8151600101808352811415610b2a575b505050506020610b8051610bc0015160206001820306601f8201039050610b80510101610b8052610b8051610c0052610360805160200180610b8051610bc001828460006004600a8704601201f1610baf57600080fd5b5050610b8051610bc0015160206001820306601f8201039050610b8051610bc001610b6081516020818352015b83610b6051101515610bed57610c0a565b6000610b60516020850101535b8151600101808352811415610bdc575b505050506020610b8051610bc0015160206001820306601f8201039050610b80510101610b8052610b8051610c2052610280805160200180610b8051610bc001828460006004600a8704601201f1610c6157600080fd5b5050610b8051610bc0015160206001820306601f8201039050610b8051610bc001610b6081516060818352015b83610b6051101515610c9f57610cbc565b6000610b60516020850101535b8151600101808352811415610c8e575b505050506020610b8051610bc0015160206001820306601f8201039050610b80510101610b80527f9d0a206c338cfcaf9198c04fe61b39a988e26b623ef97cb2f72bafcfbf8bb93e610b8051610bc0a1005b60006000fd5b610185610e9903610185600039610185610e99036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index 6c4bd7dc7a..b99a307840 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -72,11 +72,8 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], assert len(withdrawal_credentials) == WITHDRAWAL_CREDENTIALS_LENGTH assert len(signature) == SIGNATURE_LENGTH - # Emit `Deposit` log - amount: bytes[8] = self.to_little_endian_64(deposit_amount) - log.Deposit(pubkey, withdrawal_credentials, amount, signature) - # Compute `DepositData` root + amount: bytes[8] = self.to_little_endian_64(deposit_amount) zero_bytes32: bytes32 pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes32, start=0, len=64 - PUBKEY_LENGTH))) signature_root: bytes32 = sha256(concat( @@ -97,3 +94,6 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], break node = sha256(concat(self.branch[height], node)) size /= 2 + + # Emit `Deposit` log + log.Deposit(pubkey, withdrawal_credentials, amount, signature) diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index c1c07aac97..12f3e7f8bb 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -56,7 +56,7 @@ def deposit_input(): (10, True), (55555, True), (2**64 - 1, True), - (2**64, True), # Note that all calls to `to_little_endian_64` have an input less than 2**64 + # (2**64, True), # Note that all calls to `to_little_endian_64` have an input less than 2**64 ] ) def test_to_little_endian_64(registration_contract, value, success, assert_tx_failed): From 565f61dfaa520708cc83a0a593e8eb2df9e5d516 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 9 Jun 2019 20:41:21 +0100 Subject: [PATCH 005/405] Cleanup containers --- configs/constant_presets/mainnet.yaml | 6 +- configs/constant_presets/minimal.yaml | 6 +- scripts/build_spec.py | 11 +- specs/core/0_beacon-chain.md | 458 ++++++++---------- specs/core/0_deposit-contract.md | 6 +- specs/core/0_fork-choice.md | 12 +- specs/core/1_custody-game.md | 20 +- specs/core/1_shard-data-chains.md | 12 +- specs/light_client/sync_protocol.md | 8 +- specs/networking/rpc-interface.md | 8 +- specs/validator/0_beacon-chain-validator.md | 16 +- .../pyspec/eth2spec/test/helpers/block.py | 4 +- .../pyspec/eth2spec/test/helpers/deposits.py | 6 +- .../pyspec/eth2spec/test/helpers/genesis.py | 12 +- .../test/helpers/proposer_slashings.py | 2 +- .../pyspec/eth2spec/test/helpers/state.py | 2 +- .../pyspec/eth2spec/test/helpers/transfers.py | 2 +- .../test_process_attestation.py | 8 +- .../test_process_attester_slashing.py | 4 +- .../test_process_block_header.py | 2 +- .../block_processing/test_process_deposit.py | 28 +- .../test_process_proposer_slashing.py | 10 +- .../block_processing/test_process_transfer.py | 20 +- .../test_process_voluntary_exit.py | 34 +- .../test_process_crosslinks.py | 8 +- .../test_process_registry_updates.py | 26 +- ...est_process_early_derived_secret_reveal.py | 4 +- .../eth2spec/test/sanity/test_blocks.py | 46 +- 28 files changed, 366 insertions(+), 415 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 6ac3f422f5..d72b276c12 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -80,11 +80,11 @@ MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 # State list lengths # --------------------------------------------------------------- # 2**13 (= 8,192) epochs ~36 days -LATEST_RANDAO_MIXES_LENGTH: 8192 +RANDAO_MIXES_LENGTH: 8192 # 2**13 (= 8,192) epochs ~36 days -LATEST_ACTIVE_INDEX_ROOTS_LENGTH: 8192 +ACTIVE_INDEX_ROOTS_LENGTH: 8192 # 2**13 (= 8,192) epochs ~36 days -LATEST_SLASHED_EXIT_LENGTH: 8192 +SLASHED_EXIT_LENGTH: 8192 # Reward and penalty quotients diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 73448c3c60..25f1e84192 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -81,11 +81,11 @@ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 # State list lengths # --------------------------------------------------------------- # [customized] smaller state -LATEST_RANDAO_MIXES_LENGTH: 64 +RANDAO_MIXES_LENGTH: 64 # [customized] smaller state -LATEST_ACTIVE_INDEX_ROOTS_LENGTH: 64 +ACTIVE_INDEX_ROOTS_LENGTH: 64 # [customized] smaller state -LATEST_SLASHED_EXIT_LENGTH: 64 +SLASHED_EXIT_LENGTH: 64 # Reward and penalty quotients diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 7a51970e38..e25147a7f0 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -62,6 +62,7 @@ from eth2spec.utils.hash_function import hash ''' +BYTE_TYPES = [4, 32, 48, 96] NEW_TYPES = { 'Slot': 'int', 'Epoch': 'int', @@ -69,7 +70,6 @@ 'ValidatorIndex': 'int', 'Gwei': 'int', } -BYTE_TYPES = [4, 32, 48, 96] SUNDRY_FUNCTIONS = ''' def get_ssz_type_by_name(name: str) -> Container: return globals()[name] @@ -130,9 +130,10 @@ def objects_to_spec(functions: Dict[str, str], """ Given all the objects that constitute a spec, combine them into a single pyfile. """ - new_type_definitions = \ - '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in new_types.items()]) - new_type_definitions += '\n' + '\n'.join(['Bytes%s = BytesN[%s]' % (n, n) for n in byte_types]) + new_type_definitions = '\n'.join(['Bytes%s = BytesN[%s]' % (n, n) for n in byte_types]) + new_type_definitions += '\n' + '\n'.join(['Hash = Bytes32', 'BLSPubkey = Bytes48', 'BLSSignature = Bytes96']) + new_type_definitions += \ + '\n' + '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in new_types.items()]) functions_spec = '\n\n'.join(functions.values()) constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants)) ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) @@ -177,7 +178,7 @@ def dependency_order_ssz_objects(objects: Dict[str, str]) -> None: items = list(objects.items()) for key, value in items: dependencies = re.findall(r'(: [A-Z][\w[]*)', value) - dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|uint\d+|Bytes\d+|bytes', '', x), dependencies) + dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|Bytes\d+|bytes', '', x), dependencies) for dep in dependencies: if dep in NEW_TYPES or len(dep) == 0: continue diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a6d9d23c58..9fd38d7dbb 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -20,7 +20,8 @@ - [Rewards and penalties](#rewards-and-penalties) - [Max operations per block](#max-operations-per-block) - [Signature domains](#signature-domains) - - [Data structures](#data-structures) + - [Custom types](#custom-types) + - [Containers](#containers) - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) - [`Validator`](#validator) @@ -45,7 +46,6 @@ - [`BeaconBlock`](#beaconblock) - [Beacon state](#beacon-state) - [`BeaconState`](#beaconstate) - - [Custom types](#custom-types) - [Helper functions](#helper-functions) - [`xor`](#xor) - [`hash`](#hash) @@ -216,9 +216,9 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `LATEST_RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `LATEST_ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `LATEST_SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | ### Rewards and penalties @@ -255,11 +255,25 @@ These configurations are updated for releases, but may be out of sync during `de | `DOMAIN_VOLUNTARY_EXIT` | `4` | | `DOMAIN_TRANSFER` | `5` | -## Data structures +## Custom types -The following data structures are defined as [SimpleSerialize (SSZ)](../simple-serialize.md) objects. +We define the following Python custom types for type hinting and readability: -The types are defined topologically to aid in facilitating an executable version of the spec. +| Name | SSZ equivalent | Description | +| - | - | - | +| `Slot` | `uint64` | a slot number | +| `Epoch` | `uint64` | an epoch number | +| `Shard` | `uint64` | a shard number | +| `ValidatorIndex` | `uint64` | a validator registry index | +| `Gwei` | `uint64` | an amount in Gwei | +| `BLSPubkey` | `Bytes48` | a BLS12-381 public key | +| `BLSSignature` | `Bytes96` | a BLS12-381 signature | + +## Containers + +The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containers. + +*Note*: The definitions are ordered topologically to facilitate execution of the spec. ### Misc dependencies @@ -267,49 +281,36 @@ The types are defined topologically to aid in facilitating an executable version ```python class Fork(Container): - # Previous fork version previous_version: Bytes4 - # Current fork version current_version: Bytes4 - # Fork epoch number - epoch: uint64 + epoch: Epoch ``` #### `Validator` ```python class Validator(Container): - # BLS public key - pubkey: Bytes48 - # Withdrawal credentials - withdrawal_credentials: Bytes32 - # Epoch when became eligible for activation - activation_eligibility_epoch: uint64 - # Epoch when validator activated - activation_epoch: uint64 - # Epoch when validator exited - exit_epoch: uint64 - # Epoch when validator is eligible to withdraw - withdrawable_epoch: uint64 - # Was the validator slashed + pubkey: BLSPubkey + withdrawal_credentials: Hash + effective_balance: Gwei slashed: bool - # Effective balance - effective_balance: uint64 + # Status epochs + activation_eligibility_epoch: Epoch + activation_epoch: Epoch + exit_epoch: Epoch + withdrawable_epoch: Epoch ``` #### `Crosslink` ```python class Crosslink(Container): - # Shard number - shard: uint64 - # Crosslinking data from epochs [start....end-1] - start_epoch: uint64 - end_epoch: uint64 - # Root of the previous crosslink - parent_root: Bytes32 - # Root of the crosslinked shard data since the previous crosslink - data_root: Bytes32 + shard: Shard + parent_root: Hash + # Crosslinking data + start_epoch: Epoch + end_epoch: Epoch + data_root: Hash ``` #### `AttestationData` @@ -317,14 +318,12 @@ class Crosslink(Container): ```python class AttestationData(Container): # LMD GHOST vote - beacon_block_root: Bytes32 - + beacon_block_root: Hash # FFG vote - source_epoch: uint64 - source_root: Bytes32 - target_epoch: uint64 - target_root: Bytes32 - + source_epoch: Epoch + source_root: Hash + target_epoch: Epoch + target_root: Hash # Crosslink vote crosslink: Crosslink ``` @@ -333,9 +332,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): - # Attestation data data: AttestationData - # Custody bit custody_bit: bool ``` @@ -343,74 +340,58 @@ class AttestationDataAndCustodyBit(Container): ```python class IndexedAttestation(Container): - # Validator indices - custody_bit_0_indices: List[uint64] - custody_bit_1_indices: List[uint64] - # Attestation data + custody_bit_0_indices: List[ValidatorIndex] # Indices with custody bit equal to 0 + custody_bit_1_indices: List[ValidatorIndex] # Indices with custody bit equal to 1 data: AttestationData - # Aggregate signature - signature: Bytes96 + signature: BLSSignature ``` #### `PendingAttestation` ```python class PendingAttestation(Container): - # Attester aggregation bitfield aggregation_bitfield: bytes - # Attestation data data: AttestationData - # Inclusion delay - inclusion_delay: uint64 - # Proposer index - proposer_index: uint64 + inclusion_delay: Slot + proposer_index: ValidatorIndex ``` #### `Eth1Data` ```python class Eth1Data(Container): - # Root of the deposit tree - deposit_root: Bytes32 - # Total number of deposits + deposit_root: Hash deposit_count: uint64 - # Block hash - block_hash: Bytes32 + block_hash: Hash ``` #### `HistoricalBatch` ```python class HistoricalBatch(Container): - # Block roots - block_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] - # State roots - state_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] + block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] ``` #### `DepositData` ```python class DepositData(Container): - # BLS pubkey - pubkey: Bytes48 - # Withdrawal credentials - withdrawal_credentials: Bytes32 - # Amount in Gwei - amount: uint64 - # Container self-signature - signature: Bytes96 + pubkey: BLSPubkey + withdrawal_credentials: Hash + amount: Gwei + signature: BLSSignature ``` #### `BeaconBlockHeader` ```python class BeaconBlockHeader(Container): - slot: uint64 - parent_root: Bytes32 - state_root: Bytes32 - body_root: Bytes32 - signature: Bytes96 + slot: Slot + parent_root: Hash + state_root: Hash + body_root: Hash + signature: BLSSignature ``` ### Beacon operations @@ -419,11 +400,8 @@ class BeaconBlockHeader(Container): ```python class ProposerSlashing(Container): - # Proposer index - proposer_index: uint64 - # First block header + proposer_index: ValidatorIndex header_1: BeaconBlockHeader - # Second block header header_2: BeaconBlockHeader ``` @@ -431,9 +409,7 @@ class ProposerSlashing(Container): ```python class AttesterSlashing(Container): - # First attestation attestation_1: IndexedAttestation - # Second attestation attestation_2: IndexedAttestation ``` @@ -441,23 +417,17 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - # Attester aggregation bitfield aggregation_bitfield: bytes - # Attestation data data: AttestationData - # Custody bitfield custody_bitfield: bytes - # BLS aggregate signature - signature: Bytes96 + signature: BLSSignature ``` #### `Deposit` ```python class Deposit(Container): - # Branch in the deposit tree - proof: Vector[Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] - # Data + proof: Vector[Hash, DEPOSIT_CONTRACT_TREE_DEPTH] data: DepositData ``` @@ -465,32 +435,22 @@ class Deposit(Container): ```python class VoluntaryExit(Container): - # Minimum epoch for processing exit - epoch: uint64 - # Index of the exiting validator - validator_index: uint64 - # Validator signature - signature: Bytes96 + epoch: Epoch + validator_index: ValidatorIndex + signature: BLSSignature ``` #### `Transfer` ```python class Transfer(Container): - # Sender index - sender: uint64 - # Recipient index - recipient: uint64 - # Amount in Gwei - amount: uint64 - # Fee in Gwei for block proposer - fee: uint64 - # Inclusion slot - slot: uint64 - # Sender withdrawal pubkey - pubkey: Bytes48 - # Sender signature - signature: Bytes96 + sender: ValidatorIndex + recipient: ValidatorIndex + amount: Gwei + fee: Gwei + slot: Slot + pubkey: BLSPubkey + signature: BLSSignature ``` ### Beacon blocks @@ -499,9 +459,10 @@ class Transfer(Container): ```python class BeaconBlockBody(Container): - randao_reveal: Bytes96 + randao_reveal: BLSSignature eth1_data: Eth1Data graffiti: Bytes32 + # Operations proposer_slashings: List[ProposerSlashing] attester_slashings: List[AttesterSlashing] attestations: List[Attestation] @@ -514,12 +475,11 @@ class BeaconBlockBody(Container): ```python class BeaconBlock(Container): - # Header - slot: uint64 - parent_root: Bytes32 - state_root: Bytes32 + slot: Slot + parent_root: Hash + state_root: Hash body: BeaconBlockBody - signature: Bytes96 + signature: BLSSignature ``` ### Beacon state @@ -528,55 +488,45 @@ class BeaconBlock(Container): ```python class BeaconState(Container): - # Misc - slot: uint64 + # Versioning genesis_time: uint64 - fork: Fork # For versioning hard forks - # Validator registry - validator_registry: List[Validator] - balances: List[uint64] - # Randomness and committees - latest_randao_mixes: Vector[Bytes32, LATEST_RANDAO_MIXES_LENGTH] - latest_start_shard: uint64 - # Finality - previous_epoch_attestations: List[PendingAttestation] - current_epoch_attestations: List[PendingAttestation] - previous_justified_epoch: uint64 - current_justified_epoch: uint64 - previous_justified_root: Bytes32 - current_justified_root: Bytes32 - justification_bitfield: uint64 - finalized_epoch: uint64 - finalized_root: Bytes32 - # Recent state - current_crosslinks: Vector[Crosslink, SHARD_COUNT] - previous_crosslinks: Vector[Crosslink, SHARD_COUNT] - latest_block_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_state_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_active_index_roots: Vector[Bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] - latest_slashed_balances: Vector[uint64, LATEST_SLASHED_EXIT_LENGTH] - latest_block_header: BeaconBlockHeader - historical_roots: List[Bytes32] - # Ethereum 1.0 chain data - latest_eth1_data: Eth1Data + slot: Slot + fork: Fork + # History + block_header: BeaconBlockHeader + block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] + historical_roots: List[Hash] + # Eth1 + eth1_data: Eth1Data eth1_data_votes: List[Eth1Data] - deposit_index: uint64 + eth1_deposit_index: uint64 + # Registry + validators: List[Validator] + balances: List[Gwei] + # Shuffling + start_shard: Shard + randao_mixes: Vector[Hash, RANDAO_MIXES_LENGTH] + active_index_roots: Vector[Hash, ACTIVE_INDEX_ROOTS_LENGTH] + # Slashings + slashed_balances: Vector[Gwei, SLASHED_EXIT_LENGTH] + # Attestations + previous_attestations: List[PendingAttestation] + current_attestations: List[PendingAttestation] + # Crosslinks + previous_crosslinks: Vector[Crosslink, SHARD_COUNT] + current_crosslinks: Vector[Crosslink, SHARD_COUNT] + # Justification + previous_justified_epoch: Epoch + previous_justified_root: Hash + current_justified_epoch: Epoch + current_justified_root: Hash + justification_bitfield: uint64 + # Finality + finalized_epoch: Epoch + finalized_root: Hash ``` -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `Slot` | `uint64` | a slot number | -| `Epoch` | `uint64` | an epoch number | -| `Shard` | `uint64` | a shard number | -| `ValidatorIndex` | `uint64` | a validator registry index | -| `Gwei` | `uint64` | an amount in Gwei | -| `BLSPubkey` | `Bytes48` | a BLS12-381 public key | -| `BLSSignature` | `Bytes96` | a BLS12-381 signature | - ## Helper functions *Note*: The definitions below are for specification purposes and are not necessarily optimal implementations. @@ -596,11 +546,11 @@ The `hash` function is SHA256. ### `hash_tree_root` -`def hash_tree_root(object: SSZSerializable) -> Bytes32` is a function for hashing objects into a single root utilizing a hash tree structure. `hash_tree_root` is defined in the [SimpleSerialize spec](../simple-serialize.md#merkleization). +`def hash_tree_root(object: SSZSerializable) -> Hash` is a function for hashing objects into a single root utilizing a hash tree structure. `hash_tree_root` is defined in the [SimpleSerialize spec](../simple-serialize.md#merkleization). ### `signing_root` -`def signing_root(object: Container) -> Bytes32` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. +`def signing_root(object: Container) -> Hash` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. ### `bls_domain` @@ -681,7 +631,7 @@ def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[Valid """ Get active validator indices at ``epoch``. """ - return [i for i, v in enumerate(state.validator_registry) if is_active_validator(v, epoch)] + return [i for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] ``` ### `increase_balance` @@ -726,7 +676,7 @@ def get_epoch_committee_count(state: BeaconState, epoch: Epoch) -> int: ```python def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: """ - Return the number of shards to increment ``state.latest_start_shard`` during ``epoch``. + Return the number of shards to increment ``state.start_shard`` during ``epoch``. """ return min(get_epoch_committee_count(state, epoch), SHARD_COUNT - SHARD_COUNT // SLOTS_PER_EPOCH) ``` @@ -737,7 +687,7 @@ def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: assert epoch <= get_current_epoch(state) + 1 check_epoch = get_current_epoch(state) + 1 - shard = (state.latest_start_shard + get_shard_delta(state, get_current_epoch(state))) % SHARD_COUNT + shard = (state.start_shard + get_shard_delta(state, get_current_epoch(state))) % SHARD_COUNT while check_epoch > epoch: check_epoch -= 1 shard = (shard + SHARD_COUNT - get_shard_delta(state, check_epoch)) % SHARD_COUNT @@ -757,19 +707,19 @@ def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot ```python def get_block_root_at_slot(state: BeaconState, - slot: Slot) -> Bytes32: + slot: Slot) -> Hash: """ Return the block root at a recent ``slot``. """ assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT - return state.latest_block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] + return state.block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] ``` ### `get_block_root` ```python def get_block_root(state: BeaconState, - epoch: Epoch) -> Bytes32: + epoch: Epoch) -> Hash: """ Return the block root at a recent ``epoch``. """ @@ -780,37 +730,37 @@ def get_block_root(state: BeaconState, ```python def get_randao_mix(state: BeaconState, - epoch: Epoch) -> Bytes32: + epoch: Epoch) -> Hash: """ Return the randao mix at a recent ``epoch``. - ``epoch`` expected to be between (current_epoch - LATEST_RANDAO_MIXES_LENGTH, current_epoch]. + ``epoch`` expected to be between (current_epoch - RANDAO_MIXES_LENGTH, current_epoch]. """ - return state.latest_randao_mixes[epoch % LATEST_RANDAO_MIXES_LENGTH] + return state.randao_mixes[epoch % RANDAO_MIXES_LENGTH] ``` ### `get_active_index_root` ```python def get_active_index_root(state: BeaconState, - epoch: Epoch) -> Bytes32: + epoch: Epoch) -> Hash: """ Return the index root at a recent ``epoch``. ``epoch`` expected to be between - (current_epoch - LATEST_ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY, current_epoch + ACTIVATION_EXIT_DELAY]. + (current_epoch - ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY, current_epoch + ACTIVATION_EXIT_DELAY]. """ - return state.latest_active_index_roots[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH] + return state.active_index_roots[epoch % ACTIVE_INDEX_ROOTS_LENGTH] ``` ### `generate_seed` ```python def generate_seed(state: BeaconState, - epoch: Epoch) -> Bytes32: + epoch: Epoch) -> Hash: """ Generate a seed for the given ``epoch``. """ return hash( - get_randao_mix(state, epoch + LATEST_RANDAO_MIXES_LENGTH - MIN_SEED_LOOKAHEAD) + + get_randao_mix(state, epoch + RANDAO_MIXES_LENGTH - MIN_SEED_LOOKAHEAD) + get_active_index_root(state, epoch) + int_to_bytes(epoch, length=32) ) @@ -834,7 +784,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: while True: candidate_index = first_committee[(epoch + i) % len(first_committee)] random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] - effective_balance = state.validator_registry[candidate_index].effective_balance + effective_balance = state.validators[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: return candidate_index i += 1 @@ -843,7 +793,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: ### `verify_merkle_branch` ```python -def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: int, root: Bytes32) -> bool: +def verify_merkle_branch(leaf: Hash, proof: List[Hash], depth: int, index: int, root: Hash) -> bool: """ Verify that the given ``leaf`` is on the merkle branch ``proof`` starting with the given ``root``. @@ -860,7 +810,7 @@ def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: ### `get_shuffled_index` ```python -def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Bytes32) -> ValidatorIndex: +def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Hash) -> ValidatorIndex: """ Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). """ @@ -884,7 +834,7 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Bytes32) - ### `compute_committee` ```python -def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]: +def compute_committee(indices: List[ValidatorIndex], seed: Hash, index: int, count: int) -> List[ValidatorIndex]: start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count return [indices[get_shuffled_index(i, len(indices), seed)] for i in range(start, end)] @@ -937,7 +887,7 @@ def get_total_balance(state: BeaconState, indices: List[ValidatorIndex]) -> Gwei """ Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ - return max(sum([state.validator_registry[index].effective_balance for index in indices]), 1) + return max(sum([state.validators[index].effective_balance for index in indices]), 1) ``` ### `get_domain` @@ -1022,8 +972,8 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify aggregate signature assert bls_verify_multiple( pubkeys=[ - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in bit_0_indices]), - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in bit_1_indices]), + bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_0_indices]), + bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_1_indices]), ], message_hashes=[ hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), @@ -1112,14 +1062,14 @@ def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: Initiate the exit of the validator of the given ``index``. """ # Return if validator already initiated exit - validator = state.validator_registry[index] + validator = state.validators[index] if validator.exit_epoch != FAR_FUTURE_EPOCH: return # Compute exit queue epoch - exit_epochs = [v.exit_epoch for v in state.validator_registry if v.exit_epoch != FAR_FUTURE_EPOCH] + exit_epochs = [v.exit_epoch for v in state.validators if v.exit_epoch != FAR_FUTURE_EPOCH] exit_queue_epoch = max(exit_epochs + [get_delayed_activation_exit_epoch(get_current_epoch(state))]) - exit_queue_churn = len([v for v in state.validator_registry if v.exit_epoch == exit_queue_epoch]) + exit_queue_churn = len([v for v in state.validators if v.exit_epoch == exit_queue_epoch]) if exit_queue_churn >= get_churn_limit(state): exit_queue_epoch += 1 @@ -1139,10 +1089,10 @@ def slash_validator(state: BeaconState, """ current_epoch = get_current_epoch(state) initiate_validator_exit(state, slashed_index) - state.validator_registry[slashed_index].slashed = True - state.validator_registry[slashed_index].withdrawable_epoch = current_epoch + LATEST_SLASHED_EXIT_LENGTH - slashed_balance = state.validator_registry[slashed_index].effective_balance - state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] += slashed_balance + state.validators[slashed_index].slashed = True + state.validators[slashed_index].withdrawable_epoch = current_epoch + SLASHED_EXIT_LENGTH + slashed_balance = state.validators[slashed_index].effective_balance + state.slashed_balances[current_epoch % SLASHED_EXIT_LENGTH] += slashed_balance proposer_index = get_beacon_proposer_index(state) if whistleblower_index is None: @@ -1175,8 +1125,8 @@ Let `genesis_state = get_genesis_beacon_state(genesis_deposits, eth2genesis.gene def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis_eth1_data: Eth1Data) -> BeaconState: state = BeaconState( genesis_time=genesis_time, - latest_eth1_data=genesis_eth1_data, - latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + eth1_data=genesis_eth1_data, + block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) # Process genesis deposits @@ -1184,15 +1134,15 @@ def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis process_deposit(state, deposit) # Process genesis activations - for validator in state.validator_registry: + for validator in state.validators: if validator.effective_balance >= MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate latest_active_index_roots + # Populate active_index_roots genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, GENESIS_EPOCH)) - for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): - state.latest_active_index_roots[index] = genesis_active_index_root + for index in range(ACTIVE_INDEX_ROOTS_LENGTH): + state.active_index_roots[index] = genesis_active_index_root return state ``` @@ -1233,15 +1183,15 @@ def process_slots(state: BeaconState, slot: Slot) -> None: def process_slot(state: BeaconState) -> None: # Cache state root previous_state_root = hash_tree_root(state) - state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root + state.state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root # Cache latest block header state root - if state.latest_block_header.state_root == ZERO_HASH: - state.latest_block_header.state_root = previous_state_root + if state.block_header.state_root == ZERO_HASH: + state.block_header.state_root = previous_state_root # Cache block root - previous_block_root = signing_root(state.latest_block_header) - state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root + previous_block_root = signing_root(state.block_header) + state.block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root ``` ### Epoch processing @@ -1271,7 +1221,7 @@ def get_total_active_balance(state: BeaconState) -> Gwei: ```python def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: assert epoch in (get_current_epoch(state), get_previous_epoch(state)) - return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations + return state.current_attestations if epoch == get_current_epoch(state) else state.previous_attestations ``` ```python @@ -1296,7 +1246,7 @@ def get_unslashed_attesting_indices(state: BeaconState, output = set() for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) - return sorted(filter(lambda index: not state.validator_registry[index].slashed, list(output))) + return sorted(filter(lambda index: not state.validators[index].slashed, list(output))) ``` ```python @@ -1391,7 +1341,7 @@ def process_crosslinks(state: BeaconState) -> None: ```python def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: total_balance = get_total_active_balance(state) - effective_balance = state.validator_registry[index].effective_balance + effective_balance = state.validators[index].effective_balance return effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH ``` @@ -1399,10 +1349,10 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - rewards = [0 for _ in range(len(state.validator_registry))] - penalties = [0 for _ in range(len(state.validator_registry))] + rewards = [0 for _ in range(len(state.validators))] + penalties = [0 for _ in range(len(state.validators))] eligible_validator_indices = [ - index for index, v in enumerate(state.validator_registry) + index for index, v in enumerate(state.validators) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) ] @@ -1436,7 +1386,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: penalties[index] += BASE_REWARDS_PER_EPOCH * get_base_reward(state, index) if index not in matching_target_attesting_indices: penalties[index] += ( - state.validator_registry[index].effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT + state.validators[index].effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT ) return rewards, penalties @@ -1444,8 +1394,8 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: ```python def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - rewards = [0 for index in range(len(state.validator_registry))] - penalties = [0 for index in range(len(state.validator_registry))] + rewards = [0 for index in range(len(state.validators))] + penalties = [0 for index in range(len(state.validators))] epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = (get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT @@ -1469,7 +1419,7 @@ def process_rewards_and_penalties(state: BeaconState) -> None: rewards1, penalties1 = get_attestation_deltas(state) rewards2, penalties2 = get_crosslink_deltas(state) - for i in range(len(state.validator_registry)): + for i in range(len(state.validators)): increase_balance(state, i, rewards1[i] + rewards2[i]) decrease_balance(state, i, penalties1[i] + penalties2[i]) ``` @@ -1479,7 +1429,7 @@ def process_rewards_and_penalties(state: BeaconState) -> None: ```python def process_registry_updates(state: BeaconState) -> None: # Process activation eligibility and ejections - for index, validator in enumerate(state.validator_registry): + for index, validator in enumerate(state.validators): if ( validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and validator.effective_balance >= MAX_EFFECTIVE_BALANCE @@ -1491,13 +1441,13 @@ def process_registry_updates(state: BeaconState) -> None: # Queue validators eligible for activation and not dequeued for activation prior to finalized epoch activation_queue = sorted([ - index for index, validator in enumerate(state.validator_registry) if + index for index, validator in enumerate(state.validators) if validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH and validator.activation_epoch >= get_delayed_activation_exit_epoch(state.finalized_epoch) - ], key=lambda index: state.validator_registry[index].activation_eligibility_epoch) + ], key=lambda index: state.validators[index].activation_eligibility_epoch) # Dequeued validators for activation up to churn limit (without resetting activation epoch) for index in activation_queue[:get_churn_limit(state)]: - validator = state.validator_registry[index] + validator = state.validators[index] if validator.activation_epoch == FAR_FUTURE_EPOCH: validator.activation_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) ``` @@ -1510,12 +1460,12 @@ def process_slashings(state: BeaconState) -> None: total_balance = get_total_active_balance(state) # Compute slashed balances in the current epoch - total_at_start = state.latest_slashed_balances[(current_epoch + 1) % LATEST_SLASHED_EXIT_LENGTH] - total_at_end = state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] + total_at_start = state.slashed_balances[(current_epoch + 1) % SLASHED_EXIT_LENGTH] + total_at_end = state.slashed_balances[current_epoch % SLASHED_EXIT_LENGTH] total_penalties = total_at_end - total_at_start - for index, validator in enumerate(state.validator_registry): - if validator.slashed and current_epoch == validator.withdrawable_epoch - LATEST_SLASHED_EXIT_LENGTH // 2: + for index, validator in enumerate(state.validators): + if validator.slashed and current_epoch == validator.withdrawable_epoch - SLASHED_EXIT_LENGTH // 2: penalty = max( validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT @@ -1533,34 +1483,34 @@ def process_final_updates(state: BeaconState) -> None: if (state.slot + 1) % SLOTS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] # Update effective balances with hysteresis - for index, validator in enumerate(state.validator_registry): + for index, validator in enumerate(state.validators): balance = state.balances[index] HALF_INCREMENT = EFFECTIVE_BALANCE_INCREMENT // 2 if balance < validator.effective_balance or validator.effective_balance + 3 * HALF_INCREMENT < balance: validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) # Update start shard - state.latest_start_shard = (state.latest_start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT + state.start_shard = (state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT # Set active index root - index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % LATEST_ACTIVE_INDEX_ROOTS_LENGTH - state.latest_active_index_roots[index_root_position] = hash_tree_root( + index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % ACTIVE_INDEX_ROOTS_LENGTH + state.active_index_roots[index_root_position] = hash_tree_root( get_active_validator_indices(state, next_epoch + ACTIVATION_EXIT_DELAY) ) # Set total slashed balances - state.latest_slashed_balances[next_epoch % LATEST_SLASHED_EXIT_LENGTH] = ( - state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] + state.slashed_balances[next_epoch % SLASHED_EXIT_LENGTH] = ( + state.slashed_balances[current_epoch % SLASHED_EXIT_LENGTH] ) # Set randao mix - state.latest_randao_mixes[next_epoch % LATEST_RANDAO_MIXES_LENGTH] = get_randao_mix(state, current_epoch) + state.randao_mixes[next_epoch % RANDAO_MIXES_LENGTH] = get_randao_mix(state, current_epoch) # Set historical root accumulator if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: historical_batch = HistoricalBatch( - block_roots=state.latest_block_roots, - state_roots=state.latest_state_roots, + block_roots=state.block_roots, + state_roots=state.state_roots, ) state.historical_roots.append(hash_tree_root(historical_batch)) # Rotate current/previous epoch attestations - state.previous_epoch_attestations = state.current_epoch_attestations - state.current_epoch_attestations = [] + state.previous_attestations = state.current_attestations + state.current_attestations = [] ``` ### Block processing @@ -1580,15 +1530,15 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: # Verify that the slots match assert block.slot == state.slot # Verify that the parent matches - assert block.parent_root == signing_root(state.latest_block_header) + assert block.parent_root == signing_root(state.block_header) # Save current block as the new latest block - state.latest_block_header = BeaconBlockHeader( + state.block_header = BeaconBlockHeader( slot=block.slot, parent_root=block.parent_root, body_root=hash_tree_root(block.body), ) # Verify proposer is not slashed - proposer = state.validator_registry[get_beacon_proposer_index(state)] + proposer = state.validators[get_beacon_proposer_index(state)] assert not proposer.slashed # Verify proposer signature assert bls_verify(proposer.pubkey, signing_root(block), block.signature, get_domain(state, DOMAIN_BEACON_PROPOSER)) @@ -1598,7 +1548,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: ```python def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: - proposer = state.validator_registry[get_beacon_proposer_index(state)] + proposer = state.validators[get_beacon_proposer_index(state)] # Verify that the provided randao value is valid assert bls_verify( proposer.pubkey, @@ -1607,7 +1557,7 @@ def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: get_domain(state, DOMAIN_RANDAO), ) # Mix it in - state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( + state.randao_mixes[get_current_epoch(state) % RANDAO_MIXES_LENGTH] = ( xor(get_randao_mix(state, get_current_epoch(state)), hash(body.randao_reveal)) ) @@ -1619,7 +1569,7 @@ def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: def process_eth1_data(state: BeaconState, body: BeaconBlockBody) -> None: state.eth1_data_votes.append(body.eth1_data) if state.eth1_data_votes.count(body.eth1_data) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD: - state.latest_eth1_data = body.eth1_data + state.eth1_data = body.eth1_data ``` #### Operations @@ -1627,7 +1577,7 @@ def process_eth1_data(state: BeaconState, body: BeaconBlockBody) -> None: ```python def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that outstanding deposits are processed up to the maximum number of deposits - assert len(body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index) + assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) @@ -1651,7 +1601,7 @@ def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSla """ Process ``ProposerSlashing`` operation. """ - proposer = state.validator_registry[proposer_slashing.proposer_index] + proposer = state.validators[proposer_slashing.proposer_index] # Verify that the epoch is the same assert slot_to_epoch(proposer_slashing.header_1.slot) == slot_to_epoch(proposer_slashing.header_2.slot) # But the headers are different @@ -1683,7 +1633,7 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla attesting_indices_1 = attestation_1.custody_bit_0_indices + attestation_1.custody_bit_1_indices attesting_indices_2 = attestation_2.custody_bit_0_indices + attestation_2.custody_bit_1_indices for index in sorted(set(attesting_indices_1).intersection(attesting_indices_2)): - if is_slashable_validator(state.validator_registry[index], get_current_epoch(state)): + if is_slashable_validator(state.validators[index], get_current_epoch(state)): slash_validator(state, index) slashed_any = True assert slashed_any @@ -1711,11 +1661,11 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: if data.target_epoch == get_current_epoch(state): ffg_data = (state.current_justified_epoch, state.current_justified_root, get_current_epoch(state)) parent_crosslink = state.current_crosslinks[data.crosslink.shard] - state.current_epoch_attestations.append(pending_attestation) + state.current_attestations.append(pending_attestation) else: ffg_data = (state.previous_justified_epoch, state.previous_justified_root, get_previous_epoch(state)) parent_crosslink = state.previous_crosslinks[data.crosslink.shard] - state.previous_epoch_attestations.append(pending_attestation) + state.previous_attestations.append(pending_attestation) # Check FFG data, crosslink data, and signature assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) @@ -1738,16 +1688,16 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: leaf=hash_tree_root(deposit.data), proof=deposit.proof, depth=DEPOSIT_CONTRACT_TREE_DEPTH, - index=state.deposit_index, - root=state.latest_eth1_data.deposit_root, + index=state.eth1_deposit_index, + root=state.eth1_data.deposit_root, ) # Deposits must be processed in order - state.deposit_index += 1 + state.eth1_deposit_index += 1 pubkey = deposit.data.pubkey amount = deposit.data.amount - validator_pubkeys = [v.pubkey for v in state.validator_registry] + validator_pubkeys = [v.pubkey for v in state.validators] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession). # Invalid signatures are allowed by the deposit contract, @@ -1759,7 +1709,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: return # Add validator and balance entries - state.validator_registry.append(Validator( + state.validators.append(Validator( pubkey=pubkey, withdrawal_credentials=deposit.data.withdrawal_credentials, activation_eligibility_epoch=FAR_FUTURE_EPOCH, @@ -1782,7 +1732,7 @@ def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: """ Process ``VoluntaryExit`` operation. """ - validator = state.validator_registry[exit.validator_index] + validator = state.validators[exit.validator_index] # Verify the validator is active assert is_active_validator(validator, get_current_epoch(state)) # Verify the validator has not yet exited @@ -1811,13 +1761,13 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: assert state.slot == transfer.slot # Sender must be not yet eligible for activation, withdrawn, or transfer balance over MAX_EFFECTIVE_BALANCE assert ( - state.validator_registry[transfer.sender].activation_eligibility_epoch == FAR_FUTURE_EPOCH or - get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or + state.validators[transfer.sender].activation_eligibility_epoch == FAR_FUTURE_EPOCH or + get_current_epoch(state) >= state.validators[transfer.sender].withdrawable_epoch or transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE <= state.balances[transfer.sender] ) # Verify that the pubkey is valid assert ( - state.validator_registry[transfer.sender].withdrawal_credentials == + state.validators[transfer.sender].withdrawal_credentials == int_to_bytes(BLS_WITHDRAWAL_PREFIX, length=1) + hash(transfer.pubkey)[1:] ) # Verify that the signature is valid diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index e80dad1c50..d40553f432 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -75,9 +75,9 @@ Every Ethereum 1.0 deposit, of size at least `MIN_DEPOSIT_AMOUNT`, emits a `Depo When `CHAIN_START_FULL_DEPOSIT_THRESHOLD` of full deposits have been made, the deposit contract emits the `Eth2Genesis` log. The beacon chain state may then be initialized by calling the `get_genesis_beacon_state` function (defined [here](./0_beacon-chain.md#genesis-state)) where: * `genesis_time` equals `time` in the `Eth2Genesis` log -* `latest_eth1_data.deposit_root` equals `deposit_root` in the `Eth2Genesis` log -* `latest_eth1_data.deposit_count` equals `deposit_count` in the `Eth2Genesis` log -* `latest_eth1_data.block_hash` equals the hash of the block that included the log +* `eth1_data.deposit_root` equals `deposit_root` in the `Eth2Genesis` log +* `eth1_data.deposit_count` equals `deposit_count` in the `Eth2Genesis` log +* `eth1_data.block_hash` equals the hash of the block that included the log * `genesis_validator_deposits` is a list of `Deposit` objects built according to the `Deposit` logs up to the deposit that triggered the `Eth2Genesis` log, processed in the order in which they were emitted (oldest to newest) ## Vyper code diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 91c3e27ee4..739c06d596 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -39,7 +39,7 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph Processing the beacon chain is similar to processing the Ethereum 1.0 chain. Clients download and process blocks and maintain a view of what is the current "canonical chain", terminating at the current "head". For a beacon block, `block`, to be processed by a node, the following conditions must be met: * The parent block with root `block.parent_root` has been processed and accepted. -* An Ethereum 1.0 block pointed to by the `state.latest_eth1_data.block_hash` has been processed and accepted. +* An Ethereum 1.0 block pointed to by the `state.eth1_data.block_hash` has been processed and accepted. * The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`. *Note*: Leap seconds mean that slots will occasionally last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds, possibly several times a year. @@ -68,8 +68,8 @@ def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: return get_ancestor(store, store.get_parent(block), slot) ``` -* Let `get_latest_attestation(store: Store, index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `index`. If several such attestations exist, use the one the validator `v` observed first. -* Let `get_latest_attestation_target(store: Store, index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_latest_attestation(store, index)`. +* Let `get_attestation(store: Store, index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `index`. If several such attestations exist, use the one the validator `v` observed first. +* Let `get_attestation_target(store: Store, index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_attestation(store, index)`. * Let `get_children(store: Store, block: BeaconBlock) -> List[BeaconBlock]` return the child blocks of the given `block`. * Let `justified_head_state` be the resulting `BeaconState` object from processing the chain up to the `justified_head`. * The `head` is `lmd_ghost(store, justified_head_state, justified_head)` where the function `lmd_ghost` is defined below. Note that the implementation below is suboptimal; there are implementations that compute the head in time logarithmic in slot count. @@ -79,16 +79,16 @@ def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) """ Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``. """ - validators = start_state.validator_registry + validators = start_state.validators active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot)) - attestation_targets = [(i, get_latest_attestation_target(store, i)) for i in active_validator_indices] + attestation_targets = [(i, get_attestation_target(store, i)) for i in active_validator_indices] # Use the rounded-balance-with-hysteresis supplied by the protocol for fork # choice voting. This reduces the number of recomputations that need to be # made for optimized implementations that precompute and save data def get_vote_count(block: BeaconBlock) -> int: return sum( - start_state.validator_registry[validator_index].effective_balance + start_state.validators[validator_index].effective_balance for validator_index, target in attestation_targets if get_ancestor(store, target, block.slot) == block ) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 6c89ef8531..75ab366097 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -339,7 +339,7 @@ def process_custody_key_reveal(state: BeaconState, Note that this function mutates ``state``. """ - revealer = state.validator_registry[reveal.revealer_index] + revealer = state.validators[reveal.revealer_index] epoch_to_sign = get_randao_epoch_for_custody_period(revealer.next_custody_reveal_period, reveal.revealed_index) assert revealer.next_custody_reveal_period < get_validators_custody_reveal_period(state, reveal.revealed_index) @@ -389,8 +389,8 @@ def process_early_derived_secret_reveal(state: BeaconState, Note that this function mutates ``state``. """ - revealed_validator = state.validator_registry[reveal.revealed_index] - masker = state.validator_registry[reveal.masker_index] + revealed_validator = state.validators[reveal.revealed_index] + masker = state.validators[reveal.masker_index] derived_secret_location = reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS assert reveal.epoch >= get_current_epoch(state) + RANDAO_PENALTY_EPOCHS @@ -399,7 +399,7 @@ def process_early_derived_secret_reveal(state: BeaconState, assert reveal.revealed_index not in state.exposed_derived_secrets[derived_secret_location] # Verify signature correctness - masker = state.validator_registry[reveal.masker_index] + masker = state.validators[reveal.masker_index] pubkeys = [revealed_validator.pubkey, masker.pubkey] message_hashes = [ hash_tree_root(reveal.epoch), @@ -465,7 +465,7 @@ def process_chunk_challenge(state: BeaconState, validate_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) # Verify it is not too late to challenge assert slot_to_epoch(challenge.attestation.data.slot) >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY - responder = state.validator_registry[challenge.responder_index] + responder = state.validators[challenge.responder_index] assert responder.exit_epoch >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY # Verify the responder participated in the attestation attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bitfield) @@ -507,7 +507,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> None: # Verify challenge signature - challenger = state.validator_registry[challenge.challenger_index] + challenger = state.validators[challenge.challenger_index] assert bls_verify( pubkey=challenger.pubkey, message_hash=signing_root(challenge), @@ -520,7 +520,7 @@ def process_bit_challenge(state: BeaconState, attestation = challenge.attestation validate_indexed_attestation(state, convert_to_indexed(state, attestation)) # Verify the attestation is eligible for challenging - responder = state.validator_registry[challenge.responder_index] + responder = state.validators[challenge.responder_index] assert (slot_to_epoch(attestation.data.slot) + responder.max_reveal_lateness <= get_validators_custody_reveal_period(state, challenge.responder_index)) @@ -630,7 +630,7 @@ def process_bit_challenge_response(state: BeaconState, # Verify chunk index assert response.chunk_index < challenge.chunk_count # Verify responder has not been slashed - responder = state.validator_registry[challenge.responder_index] + responder = state.validators[challenge.responder_index] assert not responder.slashed # Verify the chunk matches the crosslink data root assert verify_merkle_branch( @@ -669,7 +669,7 @@ Run `process_reveal_deadlines(state)` immediately after `process_registry_update process_reveal_deadlines(state) # end insert @process_reveal_deadlines def process_reveal_deadlines(state: BeaconState) -> None: - for index, validator in enumerate(state.validator_registry): + for index, validator in enumerate(state.validators): deadline = validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) if get_validators_custody_reveal_period(state, index) > deadline: slash_validator(state, index) @@ -710,7 +710,7 @@ def after_process_final_updates(state: BeaconState) -> None: validator_indices_in_records = set( [record.challenger_index for record in records] + [record.responder_index for record in records] ) - for index, validator in enumerate(state.validator_registry): + for index, validator in enumerate(state.validators): if index not in validator_indices_in_records: if validator.exit_epoch != FAR_FUTURE_EPOCH and validator.withdrawable_epoch == FAR_FUTURE_EPOCH: validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 21e08e7c96..a895b3c389 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -158,9 +158,9 @@ def get_persistent_committee(state: BeaconState, later_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD committee_count = max( - len(get_active_validator_indices(state.validator_registry, earlier_start_epoch)) // + len(get_active_validator_indices(state.validators, earlier_start_epoch)) // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), - len(get_active_validator_indices(state.validator_registry, later_start_epoch)) // + len(get_active_validator_indices(state.validators, later_start_epoch)) // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), ) + 1 @@ -190,7 +190,7 @@ def get_shard_proposer_index(state: BeaconState, # Search for an active proposer for index in persistent_committee: - if is_active_validator(state.validator_registry[index], get_current_epoch(state)): + if is_active_validator(state.validators[index], get_current_epoch(state)): return index # No block can be proposed if no validator is active @@ -224,7 +224,7 @@ def verify_shard_attestation_signature(state: BeaconState, pubkeys = [] for i, index in enumerate(persistent_committee): if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b1: - validator = state.validator_registry[index] + validator = state.validators[index] assert is_active_validator(validator, get_current_epoch(state)) pubkeys.append(validator.pubkey) assert bls_verify( @@ -325,7 +325,7 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], proposer_index = get_shard_proposer_index(beacon_state, candidate.shard, candidate.slot) assert proposer_index is not None assert bls_verify( - pubkey=beacon_state.validator_registry[proposer_index].pubkey, + pubkey=beacon_state.validators[proposer_index].pubkey, message_hash=signing_root(block), signature=candidate.signature, domain=get_domain(beacon_state, slot_to_epoch(candidate.slot), DOMAIN_SHARD_PROPOSER), @@ -395,7 +395,7 @@ def is_valid_beacon_attestation(shard: Shard, assert candidate.data.previous_attestation.epoch < slot_to_epoch(candidate.data.slot) # Check crosslink data root - start_epoch = beacon_state.latest_crosslinks[shard].epoch + start_epoch = beacon_state.crosslinks[shard].epoch end_epoch = min(slot_to_epoch(candidate.data.slot) - CROSSLINK_LOOKBACK, start_epoch + MAX_EPOCHS_PER_CROSSLINK) blocks = [] for slot in range(start_epoch * SLOTS_PER_EPOCH, end_epoch * SLOTS_PER_EPOCH): diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 8501c58690..eb4bf09eb2 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -31,7 +31,7 @@ We define an "expansion" of an object as an object where a field in an object th We define two expansions: -* `ExtendedBeaconState`, which is identical to a `BeaconState` except `latest_active_index_roots: List[Bytes32]` is replaced by `latest_active_indices: List[List[ValidatorIndex]]`, where `BeaconState.latest_active_index_roots[i] = hash_tree_root(ExtendedBeaconState.latest_active_indices[i])`. +* `ExtendedBeaconState`, which is identical to a `BeaconState` except `active_index_roots: List[Bytes32]` is replaced by `active_indices: List[List[ValidatorIndex]]`, where `BeaconState.active_index_roots[i] = hash_tree_root(ExtendedBeaconState.active_indices[i])`. * `ExtendedBeaconBlock`, which is identical to a `BeaconBlock` except `state_root` is replaced with the corresponding `state: ExtendedBeaconState`. ### `get_active_validator_indices` @@ -40,10 +40,10 @@ Note that there is now a new way to compute `get_active_validator_indices`: ```python def get_active_validator_indices(state: ExtendedBeaconState, epoch: Epoch) -> List[ValidatorIndex]: - return state.latest_active_indices[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH] + return state.active_indices[epoch % ACTIVE_INDEX_ROOTS_LENGTH] ``` -Note that it takes `state` instead of `state.validator_registry` as an argument. This does not affect its use in `get_shuffled_committee`, because `get_shuffled_committee` has access to the full `state` as one of its arguments. +Note that it takes `state` instead of `state.validators` as an argument. This does not affect its use in `get_shuffled_committee`, because `get_shuffled_committee` has access to the full `state` as one of its arguments. ### `MerklePartial` @@ -85,7 +85,7 @@ def get_period_data(block: ExtendedBeaconBlock, shard_id: Shard, later: bool) -> return PeriodData( validator_count, generate_seed(block.state, period_start), - [block.state.validator_registry[i] for i in indices], + [block.state.validators[i] for i in indices], ) ``` diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md index b81f78408f..be154075c1 100644 --- a/specs/networking/rpc-interface.md +++ b/specs/networking/rpc-interface.md @@ -95,8 +95,8 @@ Since some clients are waiting for `libp2p` implementations in their respective ( network_id: uint8 chain_id: uint64 - latest_finalized_root: bytes32 - latest_finalized_epoch: uint64 + finalized_root: bytes32 + finalized_epoch: uint64 best_root: bytes32 best_slot: uint64 ) @@ -107,7 +107,7 @@ Clients exchange `hello` messages upon connection, forming a two-phase handshake Clients SHOULD immediately disconnect from one another following the handshake above under the following conditions: 1. If `network_id` belongs to a different chain, since the client definitionally cannot sync with this client. -2. If the `latest_finalized_root` shared by the peer is not in the client's chain at the expected epoch. For example, if Peer 1 in the diagram below has `(root, epoch)` of `(A, 5)` and Peer 2 has `(B, 3)`, Peer 1 would disconnect because it knows that `B` is not the root in their chain at epoch 3: +2. If the `finalized_root` shared by the peer is not in the client's chain at the expected epoch. For example, if Peer 1 in the diagram below has `(root, epoch)` of `(A, 5)` and Peer 2 has `(B, 3)`, Peer 1 would disconnect because it knows that `B` is not the root in their chain at epoch 3: ``` Root A @@ -136,7 +136,7 @@ Root B ^ +---+ ``` -Once the handshake completes, the client with the higher `latest_finalized_epoch` or `best_slot` (if the clients have equal `latest_finalized_epoch`s) SHOULD request beacon block roots from its counterparty via `beacon_block_roots` (i.e. RPC method `10`). +Once the handshake completes, the client with the higher `finalized_epoch` or `best_slot` (if the clients have equal `finalized_epoch`s) SHOULD request beacon block roots from its counterparty via `beacon_block_roots` (i.e. RPC method `10`). ### Goodbye diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 2f5aa4264b..d062e1d3b2 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -101,15 +101,15 @@ To submit a deposit: * Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=bls_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `bls_domain` will default to zeroes there). * Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. -*Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. +*Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validators` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. ### Process deposit -Deposits cannot be processed into the beacon chain until the Eth 1.0 block in which they were deposited or any of its descendants is added to the beacon chain `state.eth1_data`. This takes _a minimum_ of `ETH1_FOLLOW_DISTANCE` Eth 1.0 blocks (~4 hours) plus `ETH1_DATA_VOTING_PERIOD` epochs (~1.7 hours). Once the requisite Eth 1.0 data is added, the deposit will normally be added to a beacon chain block and processed into the `state.validator_registry` within an epoch or two. The validator is then in a queue to be activated. +Deposits cannot be processed into the beacon chain until the Eth 1.0 block in which they were deposited or any of its descendants is added to the beacon chain `state.eth1_data`. This takes _a minimum_ of `ETH1_FOLLOW_DISTANCE` Eth 1.0 blocks (~4 hours) plus `ETH1_DATA_VOTING_PERIOD` epochs (~1.7 hours). Once the requisite Eth 1.0 data is added, the deposit will normally be added to a beacon chain block and processed into the `state.validators` within an epoch or two. The validator is then in a queue to be activated. ### Validator index -Once a validator has been processed and added to the beacon state's `validator_registry`, the validator's `validator_index` is defined by the index into the registry at which the [`ValidatorRecord`](../core/0_beacon-chain.md#validator) contains the `pubkey` specified in the validator's deposit. A validator's `validator_index` is guaranteed to not change from the time of initial deposit until the validator exits and fully withdraws. This `validator_index` is used throughout the specification to dictate validator roles and responsibilities at any point and should be stored locally. +Once a validator has been processed and added to the beacon state's `validators`, the validator's `validator_index` is defined by the index into the registry at which the [`ValidatorRecord`](../core/0_beacon-chain.md#validator) contains the `pubkey` specified in the validator's deposit. A validator's `validator_index` is guaranteed to not change from the time of initial deposit until the validator exits and fully withdraws. This `validator_index` is used throughout the specification to dictate validator roles and responsibilities at any point and should be stored locally. ### Activation @@ -118,7 +118,7 @@ In normal operation, the validator is quickly activated at which point the valid The function [`is_active_validator`](../core/0_beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows: ```python -validator = state.validator_registry[validator_index] +validator = state.validators[validator_index] is_active = is_active_validator(validator, get_current_epoch(state)) ``` @@ -221,10 +221,10 @@ epoch_signature = bls_sign( ##### Eth1 Data -`block.eth1_data` is a mechanism used by block proposers vote on a recent Ethereum 1.0 block hash and an associated deposit root found in the Ethereum 1.0 deposit contract. When consensus is formed, `state.latest_eth1_data` is updated, and validator deposits up to this root can be processed. The deposit root can be calculated by calling the `get_deposit_root()` function of the deposit contract using the post-state of the block hash. +`block.eth1_data` is a mechanism used by block proposers vote on a recent Ethereum 1.0 block hash and an associated deposit root found in the Ethereum 1.0 deposit contract. When consensus is formed, `state.eth1_data` is updated, and validator deposits up to this root can be processed. The deposit root can be calculated by calling the `get_deposit_root()` function of the deposit contract using the post-state of the block hash. * Let `D` be the list of `Eth1DataVote` objects `vote` in `state.eth1_data_votes` where: - * `vote.eth1_data.block_hash` is the hash of an Eth 1.0 block that is (i) part of the canonical chain, (ii) >= `ETH1_FOLLOW_DISTANCE` blocks behind the head, and (iii) newer than `state.latest_eth1_data.block_hash`. + * `vote.eth1_data.block_hash` is the hash of an Eth 1.0 block that is (i) part of the canonical chain, (ii) >= `ETH1_FOLLOW_DISTANCE` blocks behind the head, and (iii) newer than `state.eth1_data.block_hash`. * `vote.eth1_data.deposit_count` is the deposit count of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. * `vote.eth1_data.deposit_root` is the deposit root of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. * If `D` is empty: @@ -267,9 +267,9 @@ Up to `MAX_ATTESTATIONS` aggregate attestations can be included in the `block`. ##### Deposits -If there are any unprocessed deposits for the existing `state.latest_eth1_data` (i.e. `state.latest_eth1_data.deposit_count > state.deposit_index`), then pending deposits _must_ be added to the block. The expected number of deposits is exactly `min(MAX_DEPOSITS, latest_eth1_data.deposit_count - state.deposit_index)`. These [`deposits`](../core/0_beacon-chain.md#deposit) are constructed from the `Deposit` logs from the [Eth 1.0 deposit contract](../core/0_deposit-contract) and must be processed in sequential order. The deposits included in the `block` must satisfy the verification conditions found in [deposits processing](../core/0_beacon-chain.md#deposits). +If there are any unprocessed deposits for the existing `state.eth1_data` (i.e. `state.eth1_data.deposit_count > state.eth1_deposit_index`), then pending deposits _must_ be added to the block. The expected number of deposits is exactly `min(MAX_DEPOSITS, eth1_data.deposit_count - state.eth1_deposit_index)`. These [`deposits`](../core/0_beacon-chain.md#deposit) are constructed from the `Deposit` logs from the [Eth 1.0 deposit contract](../core/0_deposit-contract) and must be processed in sequential order. The deposits included in the `block` must satisfy the verification conditions found in [deposits processing](../core/0_beacon-chain.md#deposits). -The `proof` for each deposit must be constructed against the deposit root contained in `state.latest_eth1_data` rather than the deposit root at the time the deposit was initially logged from the 1.0 chain. This entails storing a full deposit merkle tree locally and computing updated proofs against the `latest_eth1_data.deposit_root` as needed. See [`minimal_merkle.py`](https://github.com/ethereum/research/blob/master/spec_pythonizer/utils/merkle_minimal.py) for a sample implementation. +The `proof` for each deposit must be constructed against the deposit root contained in `state.eth1_data` rather than the deposit root at the time the deposit was initially logged from the 1.0 chain. This entails storing a full deposit merkle tree locally and computing updated proofs against the `eth1_data.deposit_root` as needed. See [`minimal_merkle.py`](https://github.com/ethereum/research/blob/master/spec_pythonizer/utils/merkle_minimal.py) for a sample implementation. ##### Voluntary exits diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 5c7cb02a0f..ebd20f1b5b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -57,8 +57,8 @@ def build_empty_block(spec, state, slot=None, signed=False): slot = state.slot empty_block = spec.BeaconBlock() empty_block.slot = slot - empty_block.body.eth1_data.deposit_count = state.deposit_index - previous_block_header = deepcopy(state.latest_block_header) + empty_block.body.eth1_data.deposit_count = state.eth1_deposit_index + previous_block_header = deepcopy(state.block_header) if previous_block_header.state_root == spec.ZERO_HASH: previous_block_header.state_root = state.hash_tree_root() empty_block.parent_root = signing_root(previous_block_header) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index c85d265eb6..e3bd839aa3 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -58,7 +58,7 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c """ Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. """ - pre_validator_count = len(state.validator_registry) + pre_validator_count = len(state.validators) # fill previous deposits with zero-hash deposit_data_leaves = [spec.ZERO_HASH] * pre_validator_count @@ -80,6 +80,6 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c signed ) - state.latest_eth1_data.deposit_root = root - state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + state.eth1_data.deposit_root = root + state.eth1_data.deposit_count = len(deposit_data_leaves) return deposit diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 83af566211..241706957a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -22,8 +22,8 @@ def create_genesis_state(spec, num_validators): state = spec.BeaconState( genesis_time=0, - deposit_index=num_validators, - latest_eth1_data=spec.Eth1Data( + eth1_deposit_index=num_validators, + eth1_data=spec.Eth1Data( deposit_root=deposit_root, deposit_count=num_validators, block_hash=spec.ZERO_HASH, @@ -32,16 +32,16 @@ def create_genesis_state(spec, num_validators): # We "hack" in the initial validators, # as it is much faster than creating and processing genesis deposits for every single test case. state.balances = [spec.MAX_EFFECTIVE_BALANCE] * num_validators - state.validator_registry = [build_mock_validator(spec, i, state.balances[i]) for i in range(num_validators)] + state.validators = [build_mock_validator(spec, i, state.balances[i]) for i in range(num_validators)] # Process genesis activations - for validator in state.validator_registry: + for validator in state.validators: if validator.effective_balance >= spec.MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH genesis_active_index_root = hash_tree_root(spec.get_active_validator_indices(state, spec.GENESIS_EPOCH)) - for index in range(spec.LATEST_ACTIVE_INDEX_ROOTS_LENGTH): - state.latest_active_index_roots[index] = genesis_active_index_root + for index in range(spec.ACTIVE_INDEX_ROOTS_LENGTH): + state.active_index_roots[index] = genesis_active_index_root return state diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index 86c6acf47c..d5b7f7b7fd 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -7,7 +7,7 @@ def get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=False): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] slot = state.slot header_1 = spec.BeaconBlockHeader( diff --git a/test_libs/pyspec/eth2spec/test/helpers/state.py b/test_libs/pyspec/eth2spec/test/helpers/state.py index 63aa27d70a..19a04b04ff 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/state.py +++ b/test_libs/pyspec/eth2spec/test/helpers/state.py @@ -22,4 +22,4 @@ def get_state_root(spec, state, slot) -> bytes: Return the state root at a recent ``slot``. """ assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT - return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT] + return state.state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT] diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index e619c5569d..f7397ca386 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -31,7 +31,7 @@ def get_valid_transfer(spec, state, slot=None, sender_index=None, amount=None, f sign_transfer(spec, state, transfer, transfer_privkey) # ensure withdrawal_credentials reproducible - state.validator_registry[transfer.sender].withdrawal_credentials = ( + state.validators[transfer.sender].withdrawal_credentials = ( spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] ) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 2b34ab405e..6cd2d11946 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -31,17 +31,17 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', None return - current_epoch_count = len(state.current_epoch_attestations) - previous_epoch_count = len(state.previous_epoch_attestations) + current_epoch_count = len(state.current_attestations) + previous_epoch_count = len(state.previous_attestations) # process attestation spec.process_attestation(state, attestation) # Make sure the attestation has been processed if attestation.data.target_epoch == spec.get_current_epoch(state): - assert len(state.current_epoch_attestations) == current_epoch_count + 1 + assert len(state.current_attestations) == current_epoch_count + 1 else: - assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 + assert len(state.previous_attestations) == previous_epoch_count + 1 # yield post-state yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 6c7637d599..c51f5a8a90 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -34,7 +34,7 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) # Process slashing spec.process_attester_slashing(state, attester_slashing) - slashed_validator = state.validator_registry[slashed_index] + slashed_validator = state.validators[slashed_index] # Check slashing assert slashed_validator.slashed @@ -135,7 +135,7 @@ def test_participants_already_slashed(spec, state): attestation_1 = attester_slashing.attestation_1 validator_indices = attestation_1.custody_bit_0_indices + attestation_1.custody_bit_1_indices for index in validator_indices: - state.validator_registry[index].slashed = True + state.validators[index].slashed = True yield from run_attester_slashing_processing(spec, state, attester_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py index f3c017982d..a2306ef4d9 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py @@ -78,7 +78,7 @@ def test_proposer_slashed(spec, state): proposer_index = spec.get_beacon_proposer_index(stub_state) # set proposer to slashed - state.validator_registry[proposer_index].slashed = True + state.validators[proposer_index].slashed = True block = build_empty_block_for_next_slot(spec, state, signed=True) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index c50b11f2ef..7efb578718 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -16,7 +16,7 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ - pre_validator_count = len(state.validator_registry) + pre_validator_count = len(state.validators) pre_balance = 0 if validator_index < pre_validator_count: pre_balance = get_balance(state, validator_index) @@ -34,29 +34,29 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef yield 'post', state if not effective: - assert len(state.validator_registry) == pre_validator_count + assert len(state.validators) == pre_validator_count assert len(state.balances) == pre_validator_count if validator_index < pre_validator_count: assert get_balance(state, validator_index) == pre_balance else: if validator_index < pre_validator_count: # top-up - assert len(state.validator_registry) == pre_validator_count + assert len(state.validators) == pre_validator_count assert len(state.balances) == pre_validator_count else: # new validator - assert len(state.validator_registry) == pre_validator_count + 1 + assert len(state.validators) == pre_validator_count + 1 assert len(state.balances) == pre_validator_count + 1 assert get_balance(state, validator_index) == pre_balance + deposit.data.amount - assert state.deposit_index == state.latest_eth1_data.deposit_count + assert state.eth1_deposit_index == state.eth1_data.deposit_count @with_all_phases @spec_state_test def test_new_deposit(spec, state): # fresh deposit = next validator index = validator appended to registry - validator_index = len(state.validator_registry) + validator_index = len(state.validators) amount = spec.MAX_EFFECTIVE_BALANCE deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) @@ -68,7 +68,7 @@ def test_new_deposit(spec, state): @spec_state_test def test_invalid_sig_new_deposit(spec, state): # fresh deposit = next validator index = validator appended to registry - validator_index = len(state.validator_registry) + validator_index = len(state.validators) amount = spec.MAX_EFFECTIVE_BALANCE deposit = prepare_state_and_deposit(spec, state, validator_index, amount) yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=False) @@ -117,12 +117,12 @@ def test_invalid_withdrawal_credentials_top_up(spec, state): @with_all_phases @spec_state_test def test_wrong_index(spec, state): - validator_index = len(state.validator_registry) + validator_index = len(state.validators) amount = spec.MAX_EFFECTIVE_BALANCE deposit = prepare_state_and_deposit(spec, state, validator_index, amount) - # mess up deposit_index - deposit.index = state.deposit_index + 1 + # mess up eth1_deposit_index + deposit.index = state.eth1_deposit_index + 1 sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) @@ -132,7 +132,7 @@ def test_wrong_index(spec, state): @with_all_phases @spec_state_test def test_wrong_deposit_for_deposit_count(spec, state): - deposit_data_leaves = [spec.ZERO_HASH] * len(state.validator_registry) + deposit_data_leaves = [spec.ZERO_HASH] * len(state.validators) # build root for deposit_1 index_1 = len(deposit_data_leaves) @@ -166,8 +166,8 @@ def test_wrong_deposit_for_deposit_count(spec, state): ) # state has root for deposit_2 but is at deposit_count for deposit_1 - state.latest_eth1_data.deposit_root = root_2 - state.latest_eth1_data.deposit_count = deposit_count_1 + state.eth1_data.deposit_root = root_2 + state.eth1_data.deposit_count = deposit_count_1 yield from run_deposit_processing(spec, state, deposit_2, index_2, valid=False) @@ -178,7 +178,7 @@ def test_wrong_deposit_for_deposit_count(spec, state): @with_all_phases @spec_state_test def test_bad_merkle_proof(spec, state): - validator_index = len(state.validator_registry) + validator_index = len(state.validators) amount = spec.MAX_EFFECTIVE_BALANCE deposit = prepare_state_and_deposit(spec, state, validator_index, amount) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py index b35241859c..af34ea7099 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py @@ -28,7 +28,7 @@ def run_proposer_slashing_processing(spec, state, proposer_slashing, valid=True) yield 'post', state # check if slashed - slashed_validator = state.validator_registry[proposer_slashing.proposer_index] + slashed_validator = state.validators[proposer_slashing.proposer_index] assert slashed_validator.slashed assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH @@ -77,7 +77,7 @@ def test_invalid_sig_1_and_2(spec, state): def test_invalid_proposer_index(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) # Index just too high (by 1) - proposer_slashing.proposer_index = len(state.validator_registry) + proposer_slashing.proposer_index = len(state.validators) yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) @@ -111,7 +111,7 @@ def test_proposer_is_not_activated(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) # set proposer to be not active yet - state.validator_registry[proposer_slashing.proposer_index].activation_epoch = spec.get_current_epoch(state) + 1 + state.validators[proposer_slashing.proposer_index].activation_epoch = spec.get_current_epoch(state) + 1 yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) @@ -122,7 +122,7 @@ def test_proposer_is_slashed(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) # set proposer to slashed - state.validator_registry[proposer_slashing.proposer_index].slashed = True + state.validators[proposer_slashing.proposer_index].slashed = True yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) @@ -137,6 +137,6 @@ def test_proposer_is_withdrawn(spec, state): # set proposer withdrawable_epoch in past current_epoch = spec.get_current_epoch(state) proposer_index = proposer_slashing.proposer_index - state.validator_registry[proposer_index].withdrawable_epoch = current_epoch - 1 + state.validators[proposer_index].withdrawable_epoch = current_epoch - 1 yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 1294ca84a2..e9d282b3a6 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -41,7 +41,7 @@ def run_transfer_processing(spec, state, transfer, valid=True): def test_success_non_activated(spec, state): transfer = get_valid_transfer(spec, state, signed=True) # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer) @@ -55,7 +55,7 @@ def test_success_withdrawable(spec, state): transfer = get_valid_transfer(spec, state, signed=True) # withdrawable_epoch in past so can transfer - state.validator_registry[transfer.sender].withdrawable_epoch = spec.get_current_epoch(state) - 1 + state.validators[transfer.sender].withdrawable_epoch = spec.get_current_epoch(state) - 1 yield from run_transfer_processing(spec, state, transfer) @@ -86,7 +86,7 @@ def test_success_active_above_max_effective_fee(spec, state): def test_invalid_signature(spec, state): transfer = get_valid_transfer(spec, state) # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -107,7 +107,7 @@ def test_active_but_transfer_past_effective_balance(spec, state): def test_incorrect_slot(spec, state): transfer = get_valid_transfer(spec, state, slot=state.slot + 1, signed=True) # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -120,7 +120,7 @@ def test_insufficient_balance_for_fee(spec, state): transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -133,7 +133,7 @@ def test_insufficient_balance(spec, state): transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -153,7 +153,7 @@ def test_no_dust_sender(spec, state): ) # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -167,7 +167,7 @@ def test_no_dust_recipient(spec, state): state.balances[transfer.recipient] = 0 # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -176,9 +176,9 @@ def test_no_dust_recipient(spec, state): @spec_state_test def test_invalid_pubkey(spec, state): transfer = get_valid_transfer(spec, state, signed=True) - state.validator_registry[transfer.sender].withdrawal_credentials = spec.ZERO_HASH + state.validators[transfer.sender].withdrawal_credentials = spec.ZERO_HASH # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index 3359c5e789..33cacc4e20 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -21,14 +21,14 @@ def run_voluntary_exit_processing(spec, state, voluntary_exit, valid=True): yield 'post', None return - pre_exit_epoch = state.validator_registry[validator_index].exit_epoch + pre_exit_epoch = state.validators[validator_index].exit_epoch spec.process_voluntary_exit(state, voluntary_exit) yield 'post', state assert pre_exit_epoch == spec.FAR_FUTURE_EPOCH - assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @with_all_phases @@ -39,7 +39,7 @@ def test_success(spec, state): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit(spec, state, current_epoch, validator_index, privkey, signed=True) @@ -55,7 +55,7 @@ def test_invalid_signature(spec, state): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit(spec, state, current_epoch, validator_index, privkey) @@ -76,7 +76,7 @@ def test_success_exit_queue(spec, state): # Prepare a bunch of exits, based on the current state exit_queue = [] for index in initial_indices: - privkey = pubkey_to_privkey[state.validator_registry[index].pubkey] + privkey = pubkey_to_privkey[state.validators[index].pubkey] exit_queue.append(build_voluntary_exit( spec, state, @@ -94,7 +94,7 @@ def test_success_exit_queue(spec, state): # exit an additional validator validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit( spec, state, @@ -109,8 +109,8 @@ def test_success_exit_queue(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit) assert ( - state.validator_registry[validator_index].exit_epoch == - state.validator_registry[initial_indices[0]].exit_epoch + 1 + state.validators[validator_index].exit_epoch == + state.validators[initial_indices[0]].exit_epoch + 1 ) @@ -122,7 +122,7 @@ def test_validator_exit_in_future(spec, state): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit( spec, @@ -146,7 +146,7 @@ def test_validator_invalid_validator_index(spec, state): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit( spec, @@ -156,7 +156,7 @@ def test_validator_invalid_validator_index(spec, state): privkey, signed=False, ) - voluntary_exit.validator_index = len(state.validator_registry) + voluntary_exit.validator_index = len(state.validators) sign_voluntary_exit(spec, state, voluntary_exit, privkey) yield from run_voluntary_exit_processing(spec, state, voluntary_exit, False) @@ -167,9 +167,9 @@ def test_validator_invalid_validator_index(spec, state): def test_validator_not_active(spec, state): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] - state.validator_registry[validator_index].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[validator_index].activation_epoch = spec.FAR_FUTURE_EPOCH # build and test voluntary exit voluntary_exit = build_voluntary_exit( @@ -192,10 +192,10 @@ def test_validator_already_exited(spec, state): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] # but validator already has exited - state.validator_registry[validator_index].exit_epoch = current_epoch + 2 + state.validators[validator_index].exit_epoch = current_epoch + 2 voluntary_exit = build_voluntary_exit( spec, @@ -214,7 +214,7 @@ def test_validator_already_exited(spec, state): def test_validator_not_active_long_enough(spec, state): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit( spec, @@ -226,7 +226,7 @@ def test_validator_not_active_long_enough(spec, state): ) assert ( - current_epoch - state.validator_registry[validator_index].activation_epoch < + current_epoch - state.validators[validator_index].activation_epoch < spec.PERSISTENT_COMMITTEE_PERIOD ) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index 65d9586788..cd45324576 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -56,7 +56,7 @@ def test_single_crosslink_update_from_current_epoch(spec, state): fill_aggregate_attestation(spec, state, attestation) add_attestation_to_state(spec, state, attestation, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) - assert len(state.current_epoch_attestations) == 1 + assert len(state.current_attestations) == 1 shard = attestation.data.crosslink.shard pre_crosslink = deepcopy(state.current_crosslinks[shard]) @@ -77,7 +77,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): fill_aggregate_attestation(spec, state, attestation) add_attestation_to_state(spec, state, attestation, state.slot + spec.SLOTS_PER_EPOCH) - assert len(state.previous_epoch_attestations) == 1 + assert len(state.previous_attestations) == 1 shard = attestation.data.crosslink.shard pre_crosslink = deepcopy(state.current_crosslinks[shard]) @@ -130,8 +130,8 @@ def test_double_late_crosslink(spec, state): next_epoch(spec, state) add_attestation_to_state(spec, state, attestation_2, state.slot + 1) - assert len(state.previous_epoch_attestations) == 1 - assert len(state.current_epoch_attestations) == 0 + assert len(state.previous_attestations) == 1 + assert len(state.current_attestations) == 0 crosslink_deltas = spec.get_crosslink_deltas(state) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index e6679f8448..6863af3b1e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -35,23 +35,23 @@ def run_process_registry_updates(spec, state, valid=True): @spec_state_test def test_activation(spec, state): index = 0 - assert spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) + assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) # Mock a new deposit - state.validator_registry[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - state.validator_registry[index].activation_epoch = spec.FAR_FUTURE_EPOCH - state.validator_registry[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE - assert not spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) + state.validators[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + state.validators[index].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE + assert not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): next_epoch(spec, state) yield from run_process_registry_updates(spec, state) - assert state.validator_registry[index].activation_eligibility_epoch != spec.FAR_FUTURE_EPOCH - assert state.validator_registry[index].activation_epoch != spec.FAR_FUTURE_EPOCH + assert state.validators[index].activation_eligibility_epoch != spec.FAR_FUTURE_EPOCH + assert state.validators[index].activation_epoch != spec.FAR_FUTURE_EPOCH assert spec.is_active_validator( - state.validator_registry[index], + state.validators[index], spec.get_current_epoch(state), ) @@ -60,19 +60,19 @@ def test_activation(spec, state): @spec_state_test def test_ejection(spec, state): index = 0 - assert spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) - assert state.validator_registry[index].exit_epoch == spec.FAR_FUTURE_EPOCH + assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) + assert state.validators[index].exit_epoch == spec.FAR_FUTURE_EPOCH # Mock an ejection - state.validator_registry[index].effective_balance = spec.EJECTION_BALANCE + state.validators[index].effective_balance = spec.EJECTION_BALANCE for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): next_epoch(spec, state) yield from run_process_registry_updates(spec, state) - assert state.validator_registry[index].exit_epoch != spec.FAR_FUTURE_EPOCH + assert state.validators[index].exit_epoch != spec.FAR_FUTURE_EPOCH assert not spec.is_active_validator( - state.validator_registry[index], + state.validators[index], spec.get_current_epoch(state), ) diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py index 110231d77e..87297d443c 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py @@ -24,7 +24,7 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v spec.process_early_derived_secret_reveal(state, randao_key_reveal) - slashed_validator = state.validator_registry[randao_key_reveal.revealed_index] + slashed_validator = state.validators[randao_key_reveal.revealed_index] if randao_key_reveal.epoch >= spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING: assert slashed_validator.slashed @@ -111,7 +111,7 @@ def test_double_reveal(spec, state): @spec_state_test def test_revealer_is_slashed(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) - state.validator_registry[randao_key_reveal.revealed_index].slashed = True + state.validators[randao_key_reveal.revealed_index].slashed = True yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 587c377429..a5079b901f 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -91,7 +91,7 @@ def test_empty_epoch_transition(spec, state): # assert state.slot == block.slot # assert state.finalized_epoch < spec.get_current_epoch(state) - 4 -# for index in range(len(state.validator_registry)): +# for index in range(len(state.validators)): # assert get_balance(state, index) < get_balance(pre_state, index) @@ -103,7 +103,7 @@ def test_proposer_slashing(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) validator_index = proposer_slashing.proposer_index - assert not state.validator_registry[validator_index].slashed + assert not state.validators[validator_index].slashed yield 'pre', state @@ -119,7 +119,7 @@ def test_proposer_slashing(spec, state): yield 'post', state # check if slashed - slashed_validator = state.validator_registry[validator_index] + slashed_validator = state.validators[validator_index] assert slashed_validator.slashed assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH @@ -137,7 +137,7 @@ def test_attester_slashing(spec, state): validator_index = (attester_slashing.attestation_1.custody_bit_0_indices + attester_slashing.attestation_1.custody_bit_1_indices)[0] - assert not state.validator_registry[validator_index].slashed + assert not state.validators[validator_index].slashed yield 'pre', state @@ -152,7 +152,7 @@ def test_attester_slashing(spec, state): spec.state_transition(state, block) yield 'post', state - slashed_validator = state.validator_registry[validator_index] + slashed_validator = state.validators[validator_index] assert slashed_validator.slashed assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH @@ -172,10 +172,10 @@ def test_attester_slashing(spec, state): @with_all_phases @spec_state_test def test_deposit_in_block(spec, state): - initial_registry_len = len(state.validator_registry) + initial_registry_len = len(state.validators) initial_balances_len = len(state.balances) - validator_index = len(state.validator_registry) + validator_index = len(state.validators) amount = spec.MAX_EFFECTIVE_BALANCE deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) @@ -190,10 +190,10 @@ def test_deposit_in_block(spec, state): spec.state_transition(state, block) yield 'post', state - assert len(state.validator_registry) == initial_registry_len + 1 + assert len(state.validators) == initial_registry_len + 1 assert len(state.balances) == initial_balances_len + 1 assert get_balance(state, validator_index) == spec.MAX_EFFECTIVE_BALANCE - assert state.validator_registry[validator_index].pubkey == pubkeys[validator_index] + assert state.validators[validator_index].pubkey == pubkeys[validator_index] @with_all_phases @@ -203,7 +203,7 @@ def test_deposit_top_up(spec, state): amount = spec.MAX_EFFECTIVE_BALANCE // 4 deposit = prepare_state_and_deposit(spec, state, validator_index, amount) - initial_registry_len = len(state.validator_registry) + initial_registry_len = len(state.validators) initial_balances_len = len(state.balances) validator_pre_balance = get_balance(state, validator_index) @@ -218,7 +218,7 @@ def test_deposit_top_up(spec, state): spec.state_transition(state, block) yield 'post', state - assert len(state.validator_registry) == initial_registry_len + assert len(state.validators) == initial_registry_len assert len(state.balances) == initial_balances_len assert get_balance(state, validator_index) == validator_pre_balance + amount @@ -233,17 +233,17 @@ def test_attestation(spec, state): attestation = get_valid_attestation(spec, state, signed=True) # Add to state via block transition - pre_current_attestations_len = len(state.current_epoch_attestations) + pre_current_attestations_len = len(state.current_attestations) attestation_block = build_empty_block_for_next_slot(spec, state) attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation_block.body.attestations.append(attestation) sign_block(spec, state, attestation_block) spec.state_transition(state, attestation_block) - assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 + assert len(state.current_attestations) == pre_current_attestations_len + 1 - # Epoch transition should move to previous_epoch_attestations - pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) + # Epoch transition should move to previous_attestations + pre_current_attestations_root = spec.hash_tree_root(state.current_attestations) epoch_block = build_empty_block_for_next_slot(spec, state) epoch_block.slot += spec.SLOTS_PER_EPOCH @@ -253,8 +253,8 @@ def test_attestation(spec, state): yield 'blocks', [attestation_block, epoch_block], List[spec.BeaconBlock] yield 'post', state - assert len(state.current_epoch_attestations) == 0 - assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root + assert len(state.current_attestations) == 0 + assert spec.hash_tree_root(state.previous_attestations) == pre_current_attestations_root @with_all_phases @@ -289,7 +289,7 @@ def test_voluntary_exit(spec, state): sign_block(spec, state, initiate_exit_block) spec.state_transition(state, initiate_exit_block) - assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH # Process within epoch transition exit_block = build_empty_block_for_next_slot(spec, state) @@ -300,7 +300,7 @@ def test_voluntary_exit(spec, state): yield 'blocks', [initiate_exit_block, exit_block], List[spec.BeaconBlock] yield 'post', state - assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @with_all_phases @@ -317,7 +317,7 @@ def test_transfer(spec, state): pre_transfer_recipient_balance = get_balance(state, recipient_index) # un-activate so validator can transfer - state.validator_registry[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + state.validators[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield 'pre', state @@ -343,10 +343,10 @@ def test_balance_driven_status_transitions(spec, state): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] - assert state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH + assert state.validators[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH # set validator balance to below ejection threshold - state.validator_registry[validator_index].effective_balance = spec.EJECTION_BALANCE + state.validators[validator_index].effective_balance = spec.EJECTION_BALANCE yield 'pre', state @@ -359,7 +359,7 @@ def test_balance_driven_status_transitions(spec, state): yield 'blocks', [block], List[spec.BeaconBlock] yield 'post', state - assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @with_all_phases From 36a6c1bf1f56266c763ab5a20922c46cfebd5013 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 9 Jun 2019 21:30:42 +0100 Subject: [PATCH 006/405] Set MIN_ATTESTATION_INCLUSION_DELAY to 1 See item 7 of #1054. We should consider increasing the slot duration as well. --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a6d9d23c58..6f273de4bf 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -199,7 +199,7 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `MIN_ATTESTATION_INCLUSION_DELAY` | `2**2` (= 4) | slots | 24 seconds | +| `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 6 seconds | | `SLOTS_PER_EPOCH` | `2**6` (= 64) | slots | 6.4 minutes | | `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | | `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | epochs | 25.6 minutes | From 9bb0f25f18198a5e02b3290f5d5eee025d8c68a1 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 10 Jun 2019 13:41:28 +0100 Subject: [PATCH 007/405] Update specs/core/0_beacon-chain.md Co-Authored-By: NIC Lin --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8cacde7465..543f20d592 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1166,7 +1166,7 @@ Whenever the deposit contract emits a `Deposit` log call the function `is_genesi When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: * `genesis_deposits = deposits` -* `genesis_time = timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` +* `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` * `genesis_eth1_data` be the object of type `Eth1Data` where: * `genesis_eth1_data.block_hash` is the block hash for the last deposit in `deposits` * `genesis_eth1_data.deposit_root` is the deposit root for the last deposit in `deposits` From 4d6a25f16101782ff41ac4dbf62c8c321e5d8dbe Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 10 Jun 2019 13:43:00 +0100 Subject: [PATCH 008/405] Update conftest.py --- deposit_contract/tests/contracts/conftest.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/deposit_contract/tests/contracts/conftest.py b/deposit_contract/tests/contracts/conftest.py index b8b488abe3..d4c7da9aac 100644 --- a/deposit_contract/tests/contracts/conftest.py +++ b/deposit_contract/tests/contracts/conftest.py @@ -62,26 +62,6 @@ def registration_contract(w3, tester): return registration_deployed -@pytest.fixture(params=[0, 1, 2]) -def modified_registration_contract( - request, - w3, - tester): - registration_code = get_deposit_contract_code() - contract_bytecode = compiler.compile_code(registration_code)['bytecode'] - contract_abi = compiler.mk_full_signature(registration_code) - registration = w3.eth.contract( - abi=contract_abi, - bytecode=contract_bytecode) - tx_hash = registration.constructor().transact() - tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash) - registration_deployed = w3.eth.contract( - address=tx_receipt.contractAddress, - abi=contract_abi - ) - return registration_deployed - - @pytest.fixture def assert_tx_failed(tester): def assert_tx_failed(function_to_test, exception=eth_tester.exceptions.TransactionFailed): From 05a35c7228f0ae12c44e46cb1d202c6989af434e Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 10 Jun 2019 15:14:32 +0100 Subject: [PATCH 009/405] Tweak inclusion delay rewards and set BASE_REWARD_FACTOR Substantive changes: 1) Split the inclusion delay reward between attester and proposer to add up to at most one base reward. This is analogous to the reward logic in `slash_validator`, and makes the `BASE_REWARDS_PER_EPOCH` constant include proposer rewards. 2) Double `BASE_REWARD_FACTOR` to 2^6 (addressing item 4 in #1054). When the total effective balance is 2^17 ETH then maximum annual issuance is a bit below 2^21 ETH. Maximum annual issuance happens when a) all validators make perfect attestations (matching source, target, head, as well as consistent crosslink data), b) all attestations are included as fast as possible (in particular, no skip blocks), and c) there are no slashings. ```python BASE_REWARD_FACTOR = 2**6 SLOTS_PER_EPOCH = 2**6 SECONDS_PER_SLOT = 6 BASE_REWARDS_PER_EPOCH = 5 GWEI_PER_ETH = 10**9 MAX_TOTAL_EFFECTIVE_BALANCE = 2**27 * GWEI_PER_ETH TARGET_MAX_ISSUANCE = 2**21 * GWEI_PER_ETH def integer_squareroot(n: int) -> int: """ The largest integer ``x`` such that ``x**2`` is less than or equal to ``n``. """ assert n >= 0 x = n y = (x + 1) // 2 while y < x: x = y y = (x + n // x) // 2 return x MAX_REWARDS_PER_EPOCH = MAX_TOTAL_EFFECTIVE_BALANCE * BASE_REWARD_FACTOR // integer_squareroot(MAX_TOTAL_EFFECTIVE_BALANCE) // BASE_REWARDS_PER_EPOCH EPOCHS_PER_YEAR = 365.25*24*60*60 / (SECONDS_PER_SLOT * SLOTS_PER_EPOCH) MAX_REWARDS_PER_YEAR = EPOCHS_PER_YEAR * MAX_REWARDS_PER_EPOCH * BASE_REWARDS_PER_EPOCH print(MAX_REWARDS_PER_YEAR / TARGET_MAX_ISSUANCE) ``` --- specs/core/0_beacon-chain.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a6d9d23c58..8a60b6f34b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -224,14 +224,13 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | | - | - | -| `BASE_REWARD_FACTOR` | `2**5` (= 32) | +| `BASE_REWARD_FACTOR` | `2**6` (= 64) | | `WHISTLEBLOWING_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -* **The `BASE_REWARD_FACTOR` is NOT final. Once all other protocol details are finalized, it will be adjusted to target a theoretical maximum total issuance of `2**21` ETH per year if `2**27` ETH is validating (and therefore `2**20` per year if `2**25` ETH is validating, etc.)** -* The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (~18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)` so after `INVERSE_SQRT_E_DROP_TIME` epochs it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. +* The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)` so after `INVERSE_SQRT_E_DROP_TIME` epochs it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block @@ -1425,8 +1424,10 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: a for a in matching_source_attestations if index in get_attesting_indices(state, a.data, a.aggregation_bitfield) ], key=lambda a: a.inclusion_delay) - rewards[attestation.proposer_index] += get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT - rewards[index] += get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // attestation.inclusion_delay + proposer_reward = get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT + rewards[attestation.proposer_index] += proposer_reward + max_attester_reward = get_base_reward(state, index) - proposer_reward + rewards[index] += max_attester_reward * MIN_ATTESTATION_INCLUSION_DELAY // attestation.inclusion_delay # Inactivity penalty finality_delay = previous_epoch - state.finalized_epoch From ef91ee5698ce5a82fa47c62efa29cb8890df0216 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 10 Jun 2019 15:55:08 +0100 Subject: [PATCH 010/405] Address Danny's comments --- deposit_contract/contracts/validator_registration.v.py | 8 +++++--- specs/core/0_beacon-chain.md | 4 ++-- specs/core/0_deposit-contract.md | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index b99a307840..8510380ee2 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -1,5 +1,6 @@ MIN_DEPOSIT_AMOUNT: constant(uint256) = 1000000000 # Gwei DEPOSIT_CONTRACT_TREE_DEPTH: constant(uint256) = 32 +MAX_DEPOSIT_COUNT: constant(uint256) = 4294967295 # 2**DEPOSIT_CONTRACT_TREE_DEPTH - 1 PUBKEY_LENGTH: constant(uint256) = 48 # bytes WITHDRAWAL_CREDENTIALS_LENGTH: constant(uint256) = 32 # bytes AMOUNT_LENGTH: constant(uint256) = 8 # bytes @@ -27,7 +28,8 @@ def __init__(): @constant def to_little_endian_64(value: uint256) -> bytes[8]: # Reversing bytes using bitwise uint256 manipulations - # (array accesses of bytes[] are not currently supported in Vyper) + # Note: array accesses of bytes[] are not currently supported in Vyper + # Note: this function is only called when `value < 2**64` y: uint256 = 0 x: uint256 = value for _ in range(8): @@ -62,8 +64,8 @@ def get_deposit_count() -> bytes[8]: def deposit(pubkey: bytes[PUBKEY_LENGTH], withdrawal_credentials: bytes[WITHDRAWAL_CREDENTIALS_LENGTH], signature: bytes[SIGNATURE_LENGTH]): - # Avoid overflowing the Merkle tree - assert self.deposit_count < 2**DEPOSIT_CONTRACT_TREE_DEPTH - 1 + # Avoid overflowing the Merkle tree (and prevent edge case in computing `self.branch`) + assert self.deposit_count < MAX_DEPOSIT_COUNT # Validate deposit data deposit_amount: uint256 = msg.value / as_wei_value(1, "gwei") diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8cacde7465..fa15bc5dc0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1158,7 +1158,7 @@ def slash_validator(state: BeaconState, ### Genesis trigger -Whenever the deposit contract emits a `Deposit` log call the function `is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool` where: +Before genesis has been triggered and whenever the deposit contract emits a `Deposit`, log call the function `is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool` where: * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log @@ -1168,7 +1168,7 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: * `genesis_deposits = deposits` * `genesis_time = timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` * `genesis_eth1_data` be the object of type `Eth1Data` where: - * `genesis_eth1_data.block_hash` is the block hash for the last deposit in `deposits` + * `genesis_eth1_data.block_hash` is the Ethereum 1.0 block hash that emitted to log for the last deposit in `deposits` * `genesis_eth1_data.deposit_root` is the deposit root for the last deposit in `deposits` * `genesis_eth1_data.deposit_count = len(genesis_deposits)` diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index 50f52fd37d..338ece4875 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -42,7 +42,7 @@ The deposit contract has a public `deposit` function to make deposits. It takes #### Deposit amount -The ETH sent to the deposit contract, i.e. the deposit amount, is burnt on Ethereum 1.0. The minimum deposit amount is `MIN_DEPOSIT_AMOUNT` Gwei. +The amount of ETH (rounded down to the closest Gwei) sent to the deposit contract is the deposit amount, which must be of size at least `MIN_DEPOSIT_AMOUNT` Gwei. Note that ETH consumed by the deposit contract is no longer usable on Ethereum 1.0. #### Withdrawal credentials From 9fc197af672fd2a237d80efa987f5e840341091f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 10 Jun 2019 11:10:13 -0400 Subject: [PATCH 011/405] class Bytes32 --- scripts/build_spec.py | 7 ++++--- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 7a51970e38..087a88f983 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,8 @@ ) from eth2spec.utils.ssz.ssz_typing import ( # unused: uint8, uint16, uint32, uint128, uint256, - uint64, Container, Vector, BytesN + uint64, Container, Vector, + Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -52,7 +53,8 @@ ) from eth2spec.utils.ssz.ssz_typing import ( # unused: uint8, uint16, uint32, uint128, uint256, - uint64, Container, Vector, BytesN + uint64, Container, Vector, + Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -132,7 +134,6 @@ def objects_to_spec(functions: Dict[str, str], """ new_type_definitions = \ '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in new_types.items()]) - new_type_definitions += '\n' + '\n'.join(['Bytes%s = BytesN[%s]' % (n, n) for n in byte_types]) functions_spec = '\n\n'.join(functions.values()) constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants)) ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 368041f90a..e40c904cad 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -390,6 +390,22 @@ def hash_tree_root(self): return hash_tree_root(self, self.__class__) +class Bytes4(BytesN): + length = 4 + + +class Bytes32(BytesN): + length = 32 + + +class Bytes48(BytesN): + length = 48 + + +class Bytes96(BytesN): + length = 96 + + # SSZ Defaults # ----------------------------- def get_zero_value(typ): From dc56d87eef3a367783e57723cf8be060f3de7926 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 10 Jun 2019 21:16:51 +0100 Subject: [PATCH 012/405] Revert a couple of renamings --- specs/core/0_beacon-chain.md | 28 +++++++++---------- specs/test_formats/operations/README.md | 6 ++-- test_generators/operations/main.py | 4 +-- .../pyspec/eth2spec/test/helpers/block.py | 2 +- .../test/helpers/proposer_slashings.py | 2 +- .../test_process_attestation.py | 8 +++--- .../test_process_proposer_slashing.py | 2 +- .../test_process_crosslinks.py | 8 +++--- .../eth2spec/test/sanity/test_blocks.py | 12 ++++---- 9 files changed, 36 insertions(+), 36 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9fd38d7dbb..013d002c9d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -493,7 +493,7 @@ class BeaconState(Container): slot: Slot fork: Fork # History - block_header: BeaconBlockHeader + parent_block_header: BeaconBlockHeader block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] historical_roots: List[Hash] @@ -511,8 +511,8 @@ class BeaconState(Container): # Slashings slashed_balances: Vector[Gwei, SLASHED_EXIT_LENGTH] # Attestations - previous_attestations: List[PendingAttestation] - current_attestations: List[PendingAttestation] + previous_epoch_attestations: List[PendingAttestation] + current_epoch_attestations: List[PendingAttestation] # Crosslinks previous_crosslinks: Vector[Crosslink, SHARD_COUNT] current_crosslinks: Vector[Crosslink, SHARD_COUNT] @@ -1126,7 +1126,7 @@ def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis state = BeaconState( genesis_time=genesis_time, eth1_data=genesis_eth1_data, - block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + parent_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) # Process genesis deposits @@ -1186,11 +1186,11 @@ def process_slot(state: BeaconState) -> None: state.state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root # Cache latest block header state root - if state.block_header.state_root == ZERO_HASH: - state.block_header.state_root = previous_state_root + if state.parent_block_header.state_root == ZERO_HASH: + state.parent_block_header.state_root = previous_state_root # Cache block root - previous_block_root = signing_root(state.block_header) + previous_block_root = signing_root(state.parent_block_header) state.block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root ``` @@ -1221,7 +1221,7 @@ def get_total_active_balance(state: BeaconState) -> Gwei: ```python def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: assert epoch in (get_current_epoch(state), get_previous_epoch(state)) - return state.current_attestations if epoch == get_current_epoch(state) else state.previous_attestations + return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` ```python @@ -1509,8 +1509,8 @@ def process_final_updates(state: BeaconState) -> None: ) state.historical_roots.append(hash_tree_root(historical_batch)) # Rotate current/previous epoch attestations - state.previous_attestations = state.current_attestations - state.current_attestations = [] + state.previous_epoch_attestations = state.current_epoch_attestations + state.current_epoch_attestations = [] ``` ### Block processing @@ -1530,9 +1530,9 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: # Verify that the slots match assert block.slot == state.slot # Verify that the parent matches - assert block.parent_root == signing_root(state.block_header) + assert block.parent_root == signing_root(state.parent_block_header) # Save current block as the new latest block - state.block_header = BeaconBlockHeader( + state.parent_block_header = BeaconBlockHeader( slot=block.slot, parent_root=block.parent_root, body_root=hash_tree_root(block.body), @@ -1661,11 +1661,11 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: if data.target_epoch == get_current_epoch(state): ffg_data = (state.current_justified_epoch, state.current_justified_root, get_current_epoch(state)) parent_crosslink = state.current_crosslinks[data.crosslink.shard] - state.current_attestations.append(pending_attestation) + state.current_epoch_attestations.append(pending_attestation) else: ffg_data = (state.previous_justified_epoch, state.previous_justified_root, get_previous_epoch(state)) parent_crosslink = state.previous_crosslinks[data.crosslink.shard] - state.previous_attestations.append(pending_attestation) + state.previous_epoch_attestations.append(pending_attestation) # Check FFG data, crosslink data, and signature assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md index 32cf880b36..dead3722b6 100644 --- a/specs/test_formats/operations/README.md +++ b/specs/test_formats/operations/README.md @@ -14,7 +14,7 @@ post: BeaconState -- state after applying the operation. No ## Condition -A handler of the `operations` test-runner should process these cases, +A handler of the `operations` test-runner should process these cases, calling the corresponding processing implementation. Operations: @@ -23,13 +23,13 @@ Operations: |-------------------------|----------------------|----------------------|--------------------------------------------------------| | `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` | | `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | -| `block_header` | `Block` | `block` | `process_block_header(state, block)` | +| `parent_block_header` | `Block` | `block` | `process_block_header(state, block)` | | `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | | `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | | `transfer` | `Transfer` | `transfer` | `process_transfer(state, transfer)` | | `voluntary_exit` | `VoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | -Note that `block_header` is not strictly an operation (and is a full `Block`), but processed in the same manner, and hence included here. +Note that `parent_block_header` is not strictly an operation (and is a full `Block`), but processed in the same manner, and hence included here. The resulting state should match the expected `post` state, or if the `post` state is left blank, the handler should reject the input operation as invalid. diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index 82e05b3075..70089ebe8e 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -42,8 +42,8 @@ def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: create_suite('attestation', 'mainnet', lambda: generate_from_tests(test_process_attestation)), create_suite('attester_slashing', 'minimal', lambda: generate_from_tests(test_process_attester_slashing)), create_suite('attester_slashing', 'mainnet', lambda: generate_from_tests(test_process_attester_slashing)), - create_suite('block_header', 'minimal', lambda: generate_from_tests(test_process_block_header)), - create_suite('block_header', 'mainnet', lambda: generate_from_tests(test_process_block_header)), + create_suite('parent_block_header', 'minimal', lambda: generate_from_tests(test_process_block_header)), + create_suite('parent_block_header', 'mainnet', lambda: generate_from_tests(test_process_block_header)), create_suite('deposit', 'minimal', lambda: generate_from_tests(test_process_deposit)), create_suite('deposit', 'mainnet', lambda: generate_from_tests(test_process_deposit)), create_suite('proposer_slashing', 'minimal', lambda: generate_from_tests(test_process_proposer_slashing)), diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index ebd20f1b5b..03f8d5b7e9 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -58,7 +58,7 @@ def build_empty_block(spec, state, slot=None, signed=False): empty_block = spec.BeaconBlock() empty_block.slot = slot empty_block.body.eth1_data.deposit_count = state.eth1_deposit_index - previous_block_header = deepcopy(state.block_header) + previous_block_header = deepcopy(state.parent_block_header) if previous_block_header.state_root == spec.ZERO_HASH: previous_block_header.state_root = state.hash_tree_root() empty_block.parent_root = signing_root(previous_block_header) diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index d5b7f7b7fd..d725f3316f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -1,6 +1,6 @@ from copy import deepcopy -from eth2spec.test.helpers.block_header import sign_block_header +from eth2spec.test.helpers.parent_block_header import sign_block_header from eth2spec.test.helpers.keys import pubkey_to_privkey diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 6cd2d11946..2b34ab405e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -31,17 +31,17 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', None return - current_epoch_count = len(state.current_attestations) - previous_epoch_count = len(state.previous_attestations) + current_epoch_count = len(state.current_epoch_attestations) + previous_epoch_count = len(state.previous_epoch_attestations) # process attestation spec.process_attestation(state, attestation) # Make sure the attestation has been processed if attestation.data.target_epoch == spec.get_current_epoch(state): - assert len(state.current_attestations) == current_epoch_count + 1 + assert len(state.current_epoch_attestations) == current_epoch_count + 1 else: - assert len(state.previous_attestations) == previous_epoch_count + 1 + assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 # yield post-state yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py index af34ea7099..41039ab06d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py @@ -1,5 +1,5 @@ from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases -from eth2spec.test.helpers.block_header import sign_block_header +from eth2spec.test.helpers.parent_block_header import sign_block_header from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing from eth2spec.test.helpers.state import get_balance diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index cd45324576..65d9586788 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -56,7 +56,7 @@ def test_single_crosslink_update_from_current_epoch(spec, state): fill_aggregate_attestation(spec, state, attestation) add_attestation_to_state(spec, state, attestation, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) - assert len(state.current_attestations) == 1 + assert len(state.current_epoch_attestations) == 1 shard = attestation.data.crosslink.shard pre_crosslink = deepcopy(state.current_crosslinks[shard]) @@ -77,7 +77,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): fill_aggregate_attestation(spec, state, attestation) add_attestation_to_state(spec, state, attestation, state.slot + spec.SLOTS_PER_EPOCH) - assert len(state.previous_attestations) == 1 + assert len(state.previous_epoch_attestations) == 1 shard = attestation.data.crosslink.shard pre_crosslink = deepcopy(state.current_crosslinks[shard]) @@ -130,8 +130,8 @@ def test_double_late_crosslink(spec, state): next_epoch(spec, state) add_attestation_to_state(spec, state, attestation_2, state.slot + 1) - assert len(state.previous_attestations) == 1 - assert len(state.current_attestations) == 0 + assert len(state.previous_epoch_attestations) == 1 + assert len(state.current_epoch_attestations) == 0 crosslink_deltas = spec.get_crosslink_deltas(state) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index a5079b901f..967846cb7f 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -233,17 +233,17 @@ def test_attestation(spec, state): attestation = get_valid_attestation(spec, state, signed=True) # Add to state via block transition - pre_current_attestations_len = len(state.current_attestations) + pre_current_attestations_len = len(state.current_epoch_attestations) attestation_block = build_empty_block_for_next_slot(spec, state) attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation_block.body.attestations.append(attestation) sign_block(spec, state, attestation_block) spec.state_transition(state, attestation_block) - assert len(state.current_attestations) == pre_current_attestations_len + 1 + assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 - # Epoch transition should move to previous_attestations - pre_current_attestations_root = spec.hash_tree_root(state.current_attestations) + # Epoch transition should move to previous_epoch_attestations + pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) epoch_block = build_empty_block_for_next_slot(spec, state) epoch_block.slot += spec.SLOTS_PER_EPOCH @@ -253,8 +253,8 @@ def test_attestation(spec, state): yield 'blocks', [attestation_block, epoch_block], List[spec.BeaconBlock] yield 'post', state - assert len(state.current_attestations) == 0 - assert spec.hash_tree_root(state.previous_attestations) == pre_current_attestations_root + assert len(state.current_epoch_attestations) == 0 + assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root @with_all_phases From 05f1a44a3846b207a089f6ce454060325d4ea77d Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 10 Jun 2019 21:20:45 +0100 Subject: [PATCH 013/405] Fix tests --- specs/test_formats/operations/README.md | 4 ++-- test_generators/operations/main.py | 4 ++-- test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py | 2 +- .../block_processing/test_process_proposer_slashing.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md index dead3722b6..cc7e43f3f2 100644 --- a/specs/test_formats/operations/README.md +++ b/specs/test_formats/operations/README.md @@ -23,13 +23,13 @@ Operations: |-------------------------|----------------------|----------------------|--------------------------------------------------------| | `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` | | `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | -| `parent_block_header` | `Block` | `block` | `process_block_header(state, block)` | +| `block_header` | `Block` | `block` | `process_block_header(state, block)` | | `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | | `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | | `transfer` | `Transfer` | `transfer` | `process_transfer(state, transfer)` | | `voluntary_exit` | `VoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | -Note that `parent_block_header` is not strictly an operation (and is a full `Block`), but processed in the same manner, and hence included here. +Note that `block_header` is not strictly an operation (and is a full `Block`), but processed in the same manner, and hence included here. The resulting state should match the expected `post` state, or if the `post` state is left blank, the handler should reject the input operation as invalid. diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index 70089ebe8e..82e05b3075 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -42,8 +42,8 @@ def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: create_suite('attestation', 'mainnet', lambda: generate_from_tests(test_process_attestation)), create_suite('attester_slashing', 'minimal', lambda: generate_from_tests(test_process_attester_slashing)), create_suite('attester_slashing', 'mainnet', lambda: generate_from_tests(test_process_attester_slashing)), - create_suite('parent_block_header', 'minimal', lambda: generate_from_tests(test_process_block_header)), - create_suite('parent_block_header', 'mainnet', lambda: generate_from_tests(test_process_block_header)), + create_suite('block_header', 'minimal', lambda: generate_from_tests(test_process_block_header)), + create_suite('block_header', 'mainnet', lambda: generate_from_tests(test_process_block_header)), create_suite('deposit', 'minimal', lambda: generate_from_tests(test_process_deposit)), create_suite('deposit', 'mainnet', lambda: generate_from_tests(test_process_deposit)), create_suite('proposer_slashing', 'minimal', lambda: generate_from_tests(test_process_proposer_slashing)), diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index d725f3316f..d5b7f7b7fd 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -1,6 +1,6 @@ from copy import deepcopy -from eth2spec.test.helpers.parent_block_header import sign_block_header +from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import pubkey_to_privkey diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py index 41039ab06d..af34ea7099 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py @@ -1,5 +1,5 @@ from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases -from eth2spec.test.helpers.parent_block_header import sign_block_header +from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing from eth2spec.test.helpers.state import get_balance From 8b64f37d225bf0d5971735bd8088418dd68b096c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 10 Jun 2019 23:16:59 -0400 Subject: [PATCH 014/405] Make uint64 be `class` for type hinting --- scripts/build_spec.py | 36 +++++++++++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 21 ++++++----- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 087a88f983..0ede5f1d98 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -15,7 +15,6 @@ Any, Dict, List, - NewType, Tuple, ) @@ -41,7 +40,6 @@ Any, Dict, List, - NewType, Tuple, ) @@ -65,11 +63,11 @@ from eth2spec.utils.hash_function import hash ''' NEW_TYPES = { - 'Slot': 'int', - 'Epoch': 'int', - 'Shard': 'int', - 'ValidatorIndex': 'int', - 'Gwei': 'int', + 'Slot': 'uint64', + 'Epoch': 'uint64', + 'Shard': 'uint64', + 'ValidatorIndex': 'uint64', + 'Gwei': 'uint64', } BYTE_TYPES = [4, 32, 48, 96] SUNDRY_FUNCTIONS = ''' @@ -79,7 +77,7 @@ def get_ssz_type_by_name(name: str) -> Container: # Monkey patch validator compute committee code _compute_committee = compute_committee -committee_cache = {} +committee_cache = {} # type: Dict[Tuple[Bytes32, Bytes32, ValidatorIndex, int], List[ValidatorIndex]] def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]: @@ -95,10 +93,10 @@ def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, # Monkey patch hash cache _hash = hash -hash_cache = {} +hash_cache: Dict[bytes, Bytes32] = {} -def hash(x): +def hash(x: bytes) -> Bytes32: if x in hash_cache: return hash_cache[x] else: @@ -108,7 +106,7 @@ def hash(x): # Access to overwrite spec constants based on configuration -def apply_constants_preset(preset: Dict[str, Any]): +def apply_constants_preset(preset: Dict[str, Any]) -> None: global_vars = globals() for k, v in preset.items(): global_vars[k] = v @@ -132,20 +130,28 @@ def objects_to_spec(functions: Dict[str, str], """ Given all the objects that constitute a spec, combine them into a single pyfile. """ - new_type_definitions = \ - '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in new_types.items()]) + new_type_definitions = ( + '\n\n'.join( + [ + f"class {key}({value}):\n" + f" def __init__(self, _x: uint64) -> None:\n" + f" ...\n" + for key, value in new_types.items() + ] + ) + ) functions_spec = '\n\n'.join(functions.values()) constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants)) ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) ssz_objects_reinitialization_spec = ( - 'def init_SSZ_types():\n global_vars = globals()\n\n ' + 'def init_SSZ_types() -> None:\n global_vars = globals()\n\n ' + '\n\n '.join([re.sub(r'(?!\n\n)\n', r'\n ', value[:-1]) for value in ssz_objects.values()]) + '\n\n' + '\n'.join(map(lambda x: ' global_vars[\'%s\'] = %s' % (x, x), ssz_objects.keys())) ) spec = ( imports - + '\n' + new_type_definitions + + '\n\n' + new_type_definitions + '\n\n' + constants_spec + '\n\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index e40c904cad..cde1586c14 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -46,8 +46,13 @@ def __new__(cls, value, *args, **kwargs): return super().__new__(cls, value) -# We simply default to uint64. But do give it a name, for readability -uint64 = NewType('uint64', int) +class uint64(uint): + byte_len = 8 + + def __new__(cls, value, *args, **kwargs): + if value.bit_length() > 64: + raise ValueError("value out of bounds for uint128") + return super().__new__(cls, value) class uint128(uint): @@ -409,12 +414,12 @@ class Bytes96(BytesN): # SSZ Defaults # ----------------------------- def get_zero_value(typ): - if is_uint_type(typ): - return 0 - elif is_list_type(typ): + if is_list_type(typ): return [] elif is_bool_type(typ): return False + elif is_uint_type(typ): + return uint64(0) elif is_vector_type(typ): return typ() elif is_bytesn_type(typ): @@ -432,12 +437,12 @@ def get_zero_value(typ): def infer_type(obj): - if is_uint_type(obj.__class__): - return obj.__class__ - elif isinstance(obj, int): + if isinstance(obj, int): return uint64 elif isinstance(obj, list): return List[infer_type(obj[0])] + elif is_uint_type(obj.__class__): + return obj.__class__ elif isinstance(obj, (Vector, Container, bool, BytesN, bytes)): return obj.__class__ else: From 9f454185f8448772a9cea4bc19a71f7b3808c7f2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 11 Jun 2019 00:15:52 -0400 Subject: [PATCH 015/405] WIP! 1. Use custom types in SSZ declaration 2. Casting --- scripts/build_spec.py | 9 +- specs/core/0_beacon-chain.md | 172 +++++++++--------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 3 files changed, 95 insertions(+), 88 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0ede5f1d98..f396d3b9c5 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -15,6 +15,7 @@ Any, Dict, List, + Set, Tuple, ) @@ -40,6 +41,7 @@ Any, Dict, List, + Set, Tuple, ) @@ -77,10 +79,13 @@ def get_ssz_type_by_name(name: str) -> Container: # Monkey patch validator compute committee code _compute_committee = compute_committee -committee_cache = {} # type: Dict[Tuple[Bytes32, Bytes32, ValidatorIndex, int], List[ValidatorIndex]] +committee_cache: Dict[Tuple[Bytes32, Bytes32, int, int], List[ValidatorIndex]] = {} -def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]: +def compute_committee(indices: List[ValidatorIndex], # type: ignore + seed: Bytes32, + index: int, + count: int) -> List[ValidatorIndex]: param_hash = (hash_tree_root(indices), seed, index, count) if param_hash in committee_cache: diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a6d9d23c58..a6bfd5e1ac 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -180,18 +180,18 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | Unit | | - | - | :-: | -| `MIN_DEPOSIT_AMOUNT` | `2**0 * 10**9` (= 1,000,000,000) | Gwei | -| `MAX_EFFECTIVE_BALANCE` | `2**5 * 10**9` (= 32,000,000,000) | Gwei | -| `EJECTION_BALANCE` | `2**4 * 10**9` (= 16,000,000,000) | Gwei | -| `EFFECTIVE_BALANCE_INCREMENT` | `2**0 * 10**9` (= 1,000,000,000) | Gwei | +| `MIN_DEPOSIT_AMOUNT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | Gwei | +| `MAX_EFFECTIVE_BALANCE` | `Gwei(2**5 * 10**9)` (= 32,000,000,000) | Gwei | +| `EJECTION_BALANCE` | `Gwei(2**4 * 10**9)` (= 16,000,000,000) | Gwei | +| `EFFECTIVE_BALANCE_INCREMENT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | Gwei | ### Initial values | Name | Value | | - | - | -| `GENESIS_SLOT` | `0` | -| `GENESIS_EPOCH` | `0` | -| `FAR_FUTURE_EPOCH` | `2**64 - 1` | +| `GENESIS_SLOT` | `Slot(0)` | +| `GENESIS_EPOCH` | `Epoch(0)` | +| `FAR_FUTURE_EPOCH` | `Epoch(2**64 - 1)` | | `ZERO_HASH` | `b'\x00' * 32` | | `BLS_WITHDRAWAL_PREFIX` | `0` | @@ -272,7 +272,7 @@ class Fork(Container): # Current fork version current_version: Bytes4 # Fork epoch number - epoch: uint64 + epoch: Epoch ``` #### `Validator` @@ -284,17 +284,17 @@ class Validator(Container): # Withdrawal credentials withdrawal_credentials: Bytes32 # Epoch when became eligible for activation - activation_eligibility_epoch: uint64 + activation_eligibility_epoch: Epoch # Epoch when validator activated - activation_epoch: uint64 + activation_epoch: Epoch # Epoch when validator exited - exit_epoch: uint64 + exit_epoch: Epoch # Epoch when validator is eligible to withdraw - withdrawable_epoch: uint64 + withdrawable_epoch: Epoch # Was the validator slashed slashed: bool # Effective balance - effective_balance: uint64 + effective_balance: Gwei ``` #### `Crosslink` @@ -302,10 +302,10 @@ class Validator(Container): ```python class Crosslink(Container): # Shard number - shard: uint64 + shard: Shard # Crosslinking data from epochs [start....end-1] - start_epoch: uint64 - end_epoch: uint64 + start_epoch: Epoch + end_epoch: Epoch # Root of the previous crosslink parent_root: Bytes32 # Root of the crosslinked shard data since the previous crosslink @@ -320,9 +320,9 @@ class AttestationData(Container): beacon_block_root: Bytes32 # FFG vote - source_epoch: uint64 + source_epoch: Epoch source_root: Bytes32 - target_epoch: uint64 + target_epoch: Epoch target_root: Bytes32 # Crosslink vote @@ -344,8 +344,8 @@ class AttestationDataAndCustodyBit(Container): ```python class IndexedAttestation(Container): # Validator indices - custody_bit_0_indices: List[uint64] - custody_bit_1_indices: List[uint64] + custody_bit_0_indices: List[ValidatorIndex] + custody_bit_1_indices: List[ValidatorIndex] # Attestation data data: AttestationData # Aggregate signature @@ -363,7 +363,7 @@ class PendingAttestation(Container): # Inclusion delay inclusion_delay: uint64 # Proposer index - proposer_index: uint64 + proposer_index: ValidatorIndex ``` #### `Eth1Data` @@ -397,7 +397,7 @@ class DepositData(Container): # Withdrawal credentials withdrawal_credentials: Bytes32 # Amount in Gwei - amount: uint64 + amount: Gwei # Container self-signature signature: Bytes96 ``` @@ -406,7 +406,7 @@ class DepositData(Container): ```python class BeaconBlockHeader(Container): - slot: uint64 + slot: Slot parent_root: Bytes32 state_root: Bytes32 body_root: Bytes32 @@ -420,7 +420,7 @@ class BeaconBlockHeader(Container): ```python class ProposerSlashing(Container): # Proposer index - proposer_index: uint64 + proposer_index: ValidatorIndex # First block header header_1: BeaconBlockHeader # Second block header @@ -466,9 +466,9 @@ class Deposit(Container): ```python class VoluntaryExit(Container): # Minimum epoch for processing exit - epoch: uint64 + epoch: Epoch # Index of the exiting validator - validator_index: uint64 + validator_index: ValidatorIndex # Validator signature signature: Bytes96 ``` @@ -478,15 +478,15 @@ class VoluntaryExit(Container): ```python class Transfer(Container): # Sender index - sender: uint64 + sender: ValidatorIndex # Recipient index - recipient: uint64 + recipient: ValidatorIndex # Amount in Gwei - amount: uint64 + amount: Gwei # Fee in Gwei for block proposer - fee: uint64 + fee: Gwei # Inclusion slot - slot: uint64 + slot: Slot # Sender withdrawal pubkey pubkey: Bytes48 # Sender signature @@ -515,7 +515,7 @@ class BeaconBlockBody(Container): ```python class BeaconBlock(Container): # Header - slot: uint64 + slot: Slot parent_root: Bytes32 state_root: Bytes32 body: BeaconBlockBody @@ -529,24 +529,24 @@ class BeaconBlock(Container): ```python class BeaconState(Container): # Misc - slot: uint64 + slot: Slot genesis_time: uint64 fork: Fork # For versioning hard forks # Validator registry validator_registry: List[Validator] - balances: List[uint64] + balances: List[Gwei] # Randomness and committees latest_randao_mixes: Vector[Bytes32, LATEST_RANDAO_MIXES_LENGTH] - latest_start_shard: uint64 + latest_start_shard: Shard # Finality previous_epoch_attestations: List[PendingAttestation] current_epoch_attestations: List[PendingAttestation] - previous_justified_epoch: uint64 - current_justified_epoch: uint64 + previous_justified_epoch: Epoch + current_justified_epoch: Epoch previous_justified_root: Bytes32 current_justified_root: Bytes32 justification_bitfield: uint64 - finalized_epoch: uint64 + finalized_epoch: Epoch finalized_root: Bytes32 # Recent state current_crosslinks: Vector[Crosslink, SHARD_COUNT] @@ -554,7 +554,7 @@ class BeaconState(Container): latest_block_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] latest_state_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] latest_active_index_roots: Vector[Bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] - latest_slashed_balances: Vector[uint64, LATEST_SLASHED_EXIT_LENGTH] + latest_slashed_balances: Vector[Gwei, LATEST_SLASHED_EXIT_LENGTH] latest_block_header: BeaconBlockHeader historical_roots: List[Bytes32] # Ethereum 1.0 chain data @@ -619,7 +619,7 @@ def slot_to_epoch(slot: Slot) -> Epoch: """ Return the epoch number of the given ``slot``. """ - return slot // SLOTS_PER_EPOCH + return Epoch(slot // SLOTS_PER_EPOCH) ``` ### `get_previous_epoch` @@ -631,7 +631,7 @@ def get_previous_epoch(state: BeaconState) -> Epoch: Return the current epoch if it's genesis epoch. """ current_epoch = get_current_epoch(state) - return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else current_epoch - 1 + return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else Epoch(current_epoch - 1) ``` ### `get_current_epoch` @@ -651,7 +651,7 @@ def get_epoch_start_slot(epoch: Epoch) -> Slot: """ Return the starting slot of the given ``epoch``. """ - return epoch * SLOTS_PER_EPOCH + return Slot(epoch * SLOTS_PER_EPOCH) ``` ### `is_active_validator` @@ -681,7 +681,7 @@ def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[Valid """ Get active validator indices at ``epoch``. """ - return [i for i, v in enumerate(state.validator_registry) if is_active_validator(v, epoch)] + return [ValidatorIndex(i) for i, v in enumerate(state.validator_registry) if is_active_validator(v, epoch)] ``` ### `increase_balance` @@ -737,10 +737,10 @@ def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: assert epoch <= get_current_epoch(state) + 1 check_epoch = get_current_epoch(state) + 1 - shard = (state.latest_start_shard + get_shard_delta(state, get_current_epoch(state))) % SHARD_COUNT + shard = Shard((state.latest_start_shard + get_shard_delta(state, get_current_epoch(state))) % SHARD_COUNT) while check_epoch > epoch: check_epoch -= 1 - shard = (shard + SHARD_COUNT - get_shard_delta(state, check_epoch)) % SHARD_COUNT + shard = Shard((shard + SHARD_COUNT - get_shard_delta(state, Epoch(check_epoch))) % SHARD_COUNT) return shard ``` @@ -750,7 +750,7 @@ def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot: committee_count = get_epoch_committee_count(state, data.target_epoch) offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, data.target_epoch)) % SHARD_COUNT - return get_epoch_start_slot(data.target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH) + return Slot(get_epoch_start_slot(data.target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` ### `get_block_root_at_slot` @@ -810,7 +810,7 @@ def generate_seed(state: BeaconState, Generate a seed for the given ``epoch``. """ return hash( - get_randao_mix(state, epoch + LATEST_RANDAO_MIXES_LENGTH - MIN_SEED_LOOKAHEAD) + + get_randao_mix(state, Epoch(epoch + LATEST_RANDAO_MIXES_LENGTH - MIN_SEED_LOOKAHEAD)) + get_active_index_root(state, epoch) + int_to_bytes(epoch, length=32) ) @@ -826,7 +826,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: epoch = get_current_epoch(state) committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH offset = committees_per_slot * (state.slot % SLOTS_PER_EPOCH) - shard = (get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT + shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) first_committee = get_crosslink_committee(state, epoch, shard) MAX_RANDOM_BYTE = 2**8 - 1 seed = generate_seed(state, epoch) @@ -836,7 +836,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] effective_balance = state.validator_registry[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - return candidate_index + return ValidatorIndex(candidate_index) i += 1 ``` @@ -860,7 +860,7 @@ def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: ### `get_shuffled_index` ```python -def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Bytes32) -> ValidatorIndex: +def get_shuffled_index(index: int, index_count: int, seed: Bytes32) -> ValidatorIndex: """ Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). """ @@ -869,16 +869,16 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Bytes32) - # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) # See the 'generalized domain' algorithm on page 3 - for round in range(SHUFFLE_ROUND_COUNT): - pivot = bytes_to_int(hash(seed + int_to_bytes(round, length=1))[0:8]) % index_count + for current_round in range(SHUFFLE_ROUND_COUNT): + pivot = bytes_to_int(hash(seed + int_to_bytes(current_round, length=1))[0:8]) % index_count flip = (pivot + index_count - index) % index_count position = max(index, flip) - source = hash(seed + int_to_bytes(round, length=1) + int_to_bytes(position // 256, length=4)) + source = hash(seed + int_to_bytes(current_round, length=1) + int_to_bytes(position // 256, length=4)) byte = source[(position % 256) // 8] bit = (byte >> (position % 8)) % 2 index = flip if bit else index - return index + return ValidatorIndex(index) ``` ### `compute_committee` @@ -887,7 +887,7 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Bytes32) - def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]: start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return [indices[get_shuffled_index(i, len(indices), seed)] for i in range(start, end)] + return [indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] ``` ### `get_crosslink_committee` @@ -937,7 +937,7 @@ def get_total_balance(state: BeaconState, indices: List[ValidatorIndex]) -> Gwei """ Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ - return max(sum([state.validator_registry[index].effective_balance for index in indices]), 1) + return Gwei(max(sum([state.validator_registry[index].effective_balance for index in indices]), 1)) ``` ### `get_domain` @@ -945,7 +945,7 @@ def get_total_balance(state: BeaconState, indices: List[ValidatorIndex]) -> Gwei ```python def get_domain(state: BeaconState, domain_type: int, - message_epoch: int=None) -> int: + message_epoch: Epoch=None) -> int: """ Return the signature domain (fork version concatenated with domain type) of a message. """ @@ -1072,7 +1072,7 @@ def get_delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: """ Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. """ - return epoch + 1 + ACTIVATION_EXIT_DELAY + return Epoch(epoch + 1 + ACTIVATION_EXIT_DELAY) ``` ### `get_churn_limit` @@ -1121,11 +1121,11 @@ def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: exit_queue_epoch = max(exit_epochs + [get_delayed_activation_exit_epoch(get_current_epoch(state))]) exit_queue_churn = len([v for v in state.validator_registry if v.exit_epoch == exit_queue_epoch]) if exit_queue_churn >= get_churn_limit(state): - exit_queue_epoch += 1 + exit_queue_epoch += Epoch(1) # Set validator exit epoch and withdrawable epoch validator.exit_epoch = exit_queue_epoch - validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY + validator.withdrawable_epoch = Epoch(validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY) ``` #### `slash_validator` @@ -1140,15 +1140,15 @@ def slash_validator(state: BeaconState, current_epoch = get_current_epoch(state) initiate_validator_exit(state, slashed_index) state.validator_registry[slashed_index].slashed = True - state.validator_registry[slashed_index].withdrawable_epoch = current_epoch + LATEST_SLASHED_EXIT_LENGTH + state.validator_registry[slashed_index].withdrawable_epoch = Epoch(current_epoch + LATEST_SLASHED_EXIT_LENGTH) slashed_balance = state.validator_registry[slashed_index].effective_balance state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] += slashed_balance proposer_index = get_beacon_proposer_index(state) if whistleblower_index is None: whistleblower_index = proposer_index - whistleblowing_reward = slashed_balance // WHISTLEBLOWING_REWARD_QUOTIENT - proposer_reward = whistleblowing_reward // PROPOSER_REWARD_QUOTIENT + whistleblowing_reward = Gwei(slashed_balance // WHISTLEBLOWING_REWARD_QUOTIENT) + proposer_reward = Gwei(whistleblowing_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) increase_balance(state, whistleblower_index, whistleblowing_reward - proposer_reward) decrease_balance(state, slashed_index, whistleblowing_reward) @@ -1226,7 +1226,7 @@ def process_slots(state: BeaconState, slot: Slot) -> None: # Process epoch on the first slot of the next epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) - state.slot += 1 + state.slot += Slot(1) ``` ```python @@ -1293,7 +1293,7 @@ def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> List[Pen ```python def get_unslashed_attesting_indices(state: BeaconState, attestations: List[PendingAttestation]) -> List[ValidatorIndex]: - output = set() + output = set() # type: Set[ValidatorIndex] for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) return sorted(filter(lambda index: not state.validator_registry[index].slashed, list(output))) @@ -1379,7 +1379,7 @@ def process_crosslinks(state: BeaconState) -> None: state.previous_crosslinks = [c for c in state.current_crosslinks] for epoch in (get_previous_epoch(state), get_current_epoch(state)): for offset in range(get_epoch_committee_count(state, epoch)): - shard = (get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT + shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) crosslink_committee = get_crosslink_committee(state, epoch, shard) winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) if 3 * get_total_balance(state, attesting_indices) >= 2 * get_total_balance(state, crosslink_committee): @@ -1392,17 +1392,17 @@ def process_crosslinks(state: BeaconState) -> None: def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: total_balance = get_total_active_balance(state) effective_balance = state.validator_registry[index].effective_balance - return effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH + return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH) ``` ```python def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - rewards = [0 for _ in range(len(state.validator_registry))] - penalties = [0 for _ in range(len(state.validator_registry))] + rewards = [Gwei(0) for index in range(len(state.validator_registry))] + penalties = [Gwei(0) for index in range(len(state.validator_registry))] eligible_validator_indices = [ - index for index, v in enumerate(state.validator_registry) + ValidatorIndex(index) for index, v in enumerate(state.validator_registry) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) ] @@ -1421,21 +1421,23 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: # Proposer and inclusion delay micro-rewards for index in get_unslashed_attesting_indices(state, matching_source_attestations): + index = ValidatorIndex(index) attestation = min([ a for a in matching_source_attestations if index in get_attesting_indices(state, a.data, a.aggregation_bitfield) ], key=lambda a: a.inclusion_delay) - rewards[attestation.proposer_index] += get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT - rewards[index] += get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // attestation.inclusion_delay + rewards[attestation.proposer_index] += Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) + rewards[index] += Gwei(get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // attestation.inclusion_delay) # Inactivity penalty finality_delay = previous_epoch - state.finalized_epoch if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in eligible_validator_indices: - penalties[index] += BASE_REWARDS_PER_EPOCH * get_base_reward(state, index) + index = ValidatorIndex(index) + penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * get_base_reward(state, index)) if index not in matching_target_attesting_indices: - penalties[index] += ( + penalties[index] += Gwei( state.validator_registry[index].effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT ) @@ -1444,11 +1446,11 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: ```python def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - rewards = [0 for index in range(len(state.validator_registry))] - penalties = [0 for index in range(len(state.validator_registry))] + rewards = [Gwei(0) for index in range(len(state.validator_registry))] + penalties = [Gwei(0) for index in range(len(state.validator_registry))] epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): - shard = (get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT + shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) crosslink_committee = get_crosslink_committee(state, epoch, shard) winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) attesting_balance = get_total_balance(state, attesting_indices) @@ -1469,9 +1471,9 @@ def process_rewards_and_penalties(state: BeaconState) -> None: rewards1, penalties1 = get_attestation_deltas(state) rewards2, penalties2 = get_crosslink_deltas(state) - for i in range(len(state.validator_registry)): - increase_balance(state, i, rewards1[i] + rewards2[i]) - decrease_balance(state, i, penalties1[i] + penalties2[i]) + for index in range(len(state.validator_registry)): + increase_balance(state, ValidatorIndex(index), rewards1[index] + rewards2[index]) + decrease_balance(state, ValidatorIndex(index), penalties1[index] + penalties2[index]) ``` #### Registry updates @@ -1487,7 +1489,7 @@ def process_registry_updates(state: BeaconState) -> None: validator.activation_eligibility_epoch = get_current_epoch(state) if is_active_validator(validator, get_current_epoch(state)) and validator.effective_balance <= EJECTION_BALANCE: - initiate_validator_exit(state, index) + initiate_validator_exit(state, ValidatorIndex(index)) # Queue validators eligible for activation and not dequeued for activation prior to finalized epoch activation_queue = sorted([ @@ -1520,7 +1522,7 @@ def process_slashings(state: BeaconState) -> None: validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT ) - decrease_balance(state, index, penalty) + decrease_balance(state, ValidatorIndex(index), penalty) ``` #### Final updates @@ -1539,11 +1541,11 @@ def process_final_updates(state: BeaconState) -> None: if balance < validator.effective_balance or validator.effective_balance + 3 * HALF_INCREMENT < balance: validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) # Update start shard - state.latest_start_shard = (state.latest_start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT + state.latest_start_shard = Shard((state.latest_start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT) # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % LATEST_ACTIVE_INDEX_ROOTS_LENGTH state.latest_active_index_roots[index_root_position] = hash_tree_root( - get_active_validator_indices(state, next_epoch + ACTIVATION_EXIT_DELAY) + get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) ) # Set total slashed balances state.latest_slashed_balances[next_epoch % LATEST_SLASHED_EXIT_LENGTH] = ( @@ -1771,7 +1773,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: state.balances.append(amount) else: # Increase balance by deposit amount - index = validator_pubkeys.index(pubkey) + index = ValidatorIndex(validator_pubkeys.index(pubkey)) increase_balance(state, index, amount) ``` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index cde1586c14..25851f331b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -254,7 +254,7 @@ def __init__(self, *args: Iterable): # cannot check non-type objects, or parametrized types if isinstance(cls.elem_type, type) and not hasattr(cls.elem_type, '__args__'): for i, item in enumerate(self.items): - if not issubclass(type(item), cls.elem_type): + if not issubclass(cls.elem_type, type(item)): raise TypeError("Typed vector cannot hold differently typed value" " at index %d. Got type: %s, expected type: %s" % (i, type(item), cls.elem_type)) From 6f526add79d904f6230400eff7dbcb6dbab0c8b3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 11 Jun 2019 00:40:07 -0400 Subject: [PATCH 016/405] flake8 length --- specs/core/0_beacon-chain.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a6bfd5e1ac..57bc2673f1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1427,7 +1427,9 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: if index in get_attesting_indices(state, a.data, a.aggregation_bitfield) ], key=lambda a: a.inclusion_delay) rewards[attestation.proposer_index] += Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) - rewards[index] += Gwei(get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // attestation.inclusion_delay) + rewards[index] += Gwei( + get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // attestation.inclusion_delay + ) # Inactivity penalty finality_delay = previous_epoch - state.finalized_epoch From f2c33529df6c7d724277fb4329c3cf623444ca4a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 11 Jun 2019 00:40:25 -0400 Subject: [PATCH 017/405] Add mypy check in CI --- Makefile | 4 +++- test_libs/pyspec/requirements-testing.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f79b89dada..87439559fd 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,9 @@ citest: $(PY_SPEC_ALL_TARGETS) lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ - flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec; + flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec; \ + cd ./eth2spec; \ + mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs --disallow-any-generics -p phase0 install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS) cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt diff --git a/test_libs/pyspec/requirements-testing.txt b/test_libs/pyspec/requirements-testing.txt index 331d0fa284..1ae21a2f24 100644 --- a/test_libs/pyspec/requirements-testing.txt +++ b/test_libs/pyspec/requirements-testing.txt @@ -2,3 +2,4 @@ pytest>=3.6,<3.7 ../config_helpers flake8==3.7.7 +mypy==0.701 From abe48cc98857c3350d45656d3d4a6f2e6af2784d Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Tue, 11 Jun 2019 15:25:25 +0100 Subject: [PATCH 018/405] Address final comments by HW and Danny --- .../contracts/validator_registration.json | 2 +- .../contracts/validator_registration.v.py | 8 +++--- .../tests/contracts/test_deposit.py | 26 +++---------------- 3 files changed, 9 insertions(+), 27 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index 676bf470ab..1779423d2a 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 6973}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10922}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1318334}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b5050610e9956600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561025357602060046101403734156100b457600080fd5b60006101605261014051610180526101a060006008818352015b61016051600860008112156100eb578060000360020a82046100f2565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561011d57600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610166578060000360020a820461016d565b8060020a82025b90509050610180525b81516001018083528114156100ce575b505060186008602082066101e001602082840111156101a457600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101e157600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b826103005111156102135761022f565b6000610300516102c001535b8151600101808352811415610203575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b63c5f2892f60005114156103ab57341561026c57600080fd5b6000610140526001546101605261018060006020818352015b600160016101605116141561030657600061018051602081106102a757600080fd5b600060c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af16102f857600080fd5b60c051905061014052610374565b6000610140516020826101a0010152602081019050610180516020811061032c57600080fd5b600260c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161036a57600080fd5b60c0519050610140525b610160600261038257600080fd5b60028151048152505b8151600101808352811415610285575b50506101405160005260206000f3005b63621fd13060005114156104815734156103c457600080fd5b60606101c060246380673289610140526001546101605261015c6000305af16103ec57600080fd5b6101e0805160200180610260828460006004600a8704601201f161040f57600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c05111156104415761045d565b60006102c05161028001535b8151600101808352811415610431575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d6000511415610d0e57606060046101403760506004356004016101a03760306004356004013511156104b757600080fd5b60406024356004016102203760206024356004013511156104d757600080fd5b60806044356004016102803760606044356004013511156104f757600080fd5b63ffffffff6001541061050957600080fd5b633b9aca00610340526103405161051f57600080fd5b61034051340461032052633b9aca0061032051101561053d57600080fd5b60306101a0511461054d57600080fd5b6020610220511461055d57600080fd5b6060610280511461056d57600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af161059657600080fd5b610460805160200180610360828460006004600a8704601201f16105b957600080fd5b505060006101a060308060208461056001018260208501600060046016f150508051820191505060006010602082066104e001602082840111156105fc57600080fd5b602080610500826104a0600060046015f150508181528090509050905060108060208461056001018260208501600060046013f150508051820191505080610560526105609050602060c0825160208401600060025af161065c57600080fd5b60c05190506104c052600060006040602082066106000161028051828401111561068557600080fd5b606080610620826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af16106c557600080fd5b60c0519050602082610800010152602081019050600060406020602082066106c0016102805182840111156106f957600080fd5b6060806106e0826020602088068803016102800160006004601bf150508181528090509050905060208060208461078001018260208501600060046015f15050805182019150506104a05160208261078001015260208101905080610780526107809050602060c0825160208401600060025af161077657600080fd5b60c051905060208261080001015260208101905080610800526108009050602060c0825160208401600060025af16107ad57600080fd5b60c05190506105e052600060006104c0516020826108a00101526020810190506102206020806020846108a001018260208501600060046015f1505080518201915050806108a0526108a09050602060c0825160208401600060025af161081357600080fd5b60c0519050602082610a2001015260208101905060006103606008806020846109a001018260208501600060046012f15050805182019150506000601860208206610920016020828401111561086857600080fd5b602080610940826104a0600060046015f15050818152809050905090506018806020846109a001018260208501600060046014f15050805182019150506105e0516020826109a0010152602081019050806109a0526109a09050602060c0825160208401600060025af16108db57600080fd5b60c0519050602082610a2001015260208101905080610a2052610a209050602060c0825160208401600060025af161091257600080fd5b60c051905061088052600180546001825401101561092f57600080fd5b6001815401815550600154610aa052610ac060006020818352015b60016001610aa05116141561097f5761088051610ac0516020811061096e57600080fd5b600060c052602060c0200155610a13565b6000610ac0516020811061099257600080fd5b600060c052602060c0200154602082610ae001015260208101905061088051602082610ae001015260208101905080610ae052610ae09050602060c0825160208401600060025af16109e357600080fd5b60c051905061088052610aa060026109fa57600080fd5b60028151048152505b815160010180835281141561094a575b50506080610b8052610b8051610bc0526101a0805160200180610b8051610bc001828460006004600a8704601201f1610a4b57600080fd5b5050610b8051610bc0015160206001820306601f8201039050610b8051610bc001610b6081516040818352015b83610b6051101515610a8957610aa6565b6000610b60516020850101535b8151600101808352811415610a78575b505050506020610b8051610bc0015160206001820306601f8201039050610b80510101610b8052610b8051610be052610220805160200180610b8051610bc001828460006004600a8704601201f1610afd57600080fd5b5050610b8051610bc0015160206001820306601f8201039050610b8051610bc001610b6081516020818352015b83610b6051101515610b3b57610b58565b6000610b60516020850101535b8151600101808352811415610b2a575b505050506020610b8051610bc0015160206001820306601f8201039050610b80510101610b8052610b8051610c0052610360805160200180610b8051610bc001828460006004600a8704601201f1610baf57600080fd5b5050610b8051610bc0015160206001820306601f8201039050610b8051610bc001610b6081516020818352015b83610b6051101515610bed57610c0a565b6000610b60516020850101535b8151600101808352811415610bdc575b505050506020610b8051610bc0015160206001820306601f8201039050610b80510101610b8052610b8051610c2052610280805160200180610b8051610bc001828460006004600a8704601201f1610c6157600080fd5b5050610b8051610bc0015160206001820306601f8201039050610b8051610bc001610b6081516060818352015b83610b6051101515610c9f57610cbc565b6000610b60516020850101535b8151600101808352811415610c8e575b505050506020610b8051610bc0015160206001820306601f8201039050610b80510101610b80527f9d0a206c338cfcaf9198c04fe61b39a988e26b623ef97cb2f72bafcfbf8bb93e610b8051610bc0a1005b60006000fd5b610185610e9903610185600039610185610e99036000f3"} \ No newline at end of file +{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334984}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b505061134056600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63c5f2892f60005114156103cf57341561029057600080fd5b6000610140526001546101605261018060006020818352015b600160016101605116141561032a57600061018051602081106102cb57600080fd5b600060c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af161031c57600080fd5b60c051905061014052610398565b6000610140516020826101a0010152602081019050610180516020811061035057600080fd5b600260c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161038e57600080fd5b60c0519050610140525b61016060026103a657600080fd5b60028151048152505b81516001018083528114156102a9575b50506101405160005260206000f3005b63621fd13060005114156104e15734156103e857600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f8201039050610220511015156104335761044c565b610220516101e001526102205160200161022052610411565b6101c0805160200180610280828460006004600a8704601201f161046f57600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156104a1576104bd565b60006102e0516102a001535b8151600101808352811415610491575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d60005114156111b557606060046101403760506004356004016101a037603060043560040135111561051757600080fd5b604060243560040161022037602060243560040135111561053757600080fd5b608060443560040161028037606060443560040135111561055757600080fd5b63ffffffff6001541061056957600080fd5b633b9aca00610340526103405161057f57600080fd5b61034051340461032052633b9aca0061032051101561059d57600080fd5b60306101a051146105ad57600080fd5b602061022051146105bd57600080fd5b606061028051146105cd57600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a05110151561065d57610676565b6104a05161046001526104a0516020016104a05261063b565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f16106dd57600080fd5b505060006101a060308060208461058001018260208501600060046016f15050805182019150506000601060208206610500016020828401111561072057600080fd5b602080610520826104c0600060046015f150508181528090509050905060108060208461058001018260208501600060046013f150508051820191505080610580526105809050602060c0825160208401600060025af161078057600080fd5b60c05190506104e05260006000604060208206610620016102805182840111156107a957600080fd5b606080610640826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af16107e957600080fd5b60c0519050602082610820010152602081019050600060406020602082066106e00161028051828401111561081d57600080fd5b606080610700826020602088068803016102800160006004601bf15050818152809050905090506020806020846107a001018260208501600060046015f15050805182019150506104c0516020826107a0010152602081019050806107a0526107a09050602060c0825160208401600060025af161089a57600080fd5b60c051905060208261082001015260208101905080610820526108209050602060c0825160208401600060025af16108d157600080fd5b60c051905061060052600060006104e0516020826108c00101526020810190506102206020806020846108c001018260208501600060046015f1505080518201915050806108c0526108c09050602060c0825160208401600060025af161093757600080fd5b60c0519050602082610a4001015260208101905060006103606008806020846109c001018260208501600060046012f15050805182019150506000601860208206610940016020828401111561098c57600080fd5b602080610960826104c0600060046015f15050818152809050905090506018806020846109c001018260208501600060046014f1505080518201915050610600516020826109c0010152602081019050806109c0526109c09050602060c0825160208401600060025af16109ff57600080fd5b60c0519050602082610a4001015260208101905080610a4052610a409050602060c0825160208401600060025af1610a3657600080fd5b60c05190506108a0526101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a0516104c0516104e05161050051610520516105405161056051610580516105a0516105c0516105e05161060051610620516106405161066051610680516106a0516106c0516106e05161070051610720516107405161076051610780516107a0516107c0516107e05161080051610820516108405161086051610880516108a0516108c0516108e05161090051610920516109405161096051610980516109a0516109c0516109e051610a0051610a2051610a4051610a6051610a8051610aa0516380673289610ac052600154610ae052610ae0516006580161009b565b50610b40526000610ba0525b610b405160206001820306601f8201039050610ba051101515610bae57610bc7565b610ba051610b600152610ba051602001610ba052610b8c565b610aa052610a8052610a6052610a4052610a2052610a00526109e0526109c0526109a05261098052610960526109405261092052610900526108e0526108c0526108a05261088052610860526108405261082052610800526107e0526107c0526107a05261078052610760526107405261072052610700526106e0526106c0526106a05261068052610660526106405261062052610600526105e0526105c0526105a05261058052610560526105405261052052610500526104e0526104c0526104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610b40805160200180610bc0828460006004600a8704601201f1610d0e57600080fd5b505060a0610c4052610c4051610c80526101a0805160200180610c4051610c8001828460006004600a8704601201f1610d4657600080fd5b5050610c4051610c80015160206001820306601f8201039050610c4051610c8001610c2081516040818352015b83610c2051101515610d8457610da1565b6000610c20516020850101535b8151600101808352811415610d73575b505050506020610c4051610c80015160206001820306601f8201039050610c40510101610c4052610c4051610ca052610220805160200180610c4051610c8001828460006004600a8704601201f1610df857600080fd5b5050610c4051610c80015160206001820306601f8201039050610c4051610c8001610c2081516020818352015b83610c2051101515610e3657610e53565b6000610c20516020850101535b8151600101808352811415610e25575b505050506020610c4051610c80015160206001820306601f8201039050610c40510101610c4052610c4051610cc052610360805160200180610c4051610c8001828460006004600a8704601201f1610eaa57600080fd5b5050610c4051610c80015160206001820306601f8201039050610c4051610c8001610c2081516020818352015b83610c2051101515610ee857610f05565b6000610c20516020850101535b8151600101808352811415610ed7575b505050506020610c4051610c80015160206001820306601f8201039050610c40510101610c4052610c4051610ce052610280805160200180610c4051610c8001828460006004600a8704601201f1610f5c57600080fd5b5050610c4051610c80015160206001820306601f8201039050610c4051610c8001610c2081516060818352015b83610c2051101515610f9a57610fb7565b6000610c20516020850101535b8151600101808352811415610f89575b505050506020610c4051610c80015160206001820306601f8201039050610c40510101610c4052610c4051610d0052610bc0805160200180610c4051610c8001828460006004600a8704601201f161100e57600080fd5b5050610c4051610c80015160206001820306601f8201039050610c4051610c8001610c2081516020818352015b83610c205110151561104c57611069565b6000610c20516020850101535b815160010180835281141561103b575b505050506020610c4051610c80015160206001820306601f8201039050610c40510101610c40527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce610c4051610c80a160018054600182540110156110cd57600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d205116141561111d576108a051610d40516020811061110c57600080fd5b600060c052602060c02001556111b1565b6000610d40516020811061113057600080fd5b600060c052602060c0200154602082610d600101526020810190506108a051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af161118157600080fd5b60c05190506108a052610d20600261119857600080fd5b60028151048152505b81516001018083528114156110e8575b5050005b60006000fd5b61018561134003610185600039610185611340036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index 8510380ee2..10daf42b93 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -11,6 +11,7 @@ withdrawal_credentials: bytes[32], amount: bytes[8], signature: bytes[96], + index: bytes[8], }) branch: bytes32[DEPOSIT_CONTRACT_TREE_DEPTH] @@ -24,7 +25,7 @@ def __init__(): self.zero_hashes[i + 1] = sha256(concat(self.zero_hashes[i], self.zero_hashes[i])) -@public +@private @constant def to_little_endian_64(value: uint256) -> bytes[8]: # Reversing bytes using bitwise uint256 manipulations @@ -87,6 +88,9 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], sha256(concat(amount, slice(zero_bytes32, start=0, len=32 - AMOUNT_LENGTH), signature_root)), )) + # Emit `Deposit` log + log.Deposit(pubkey, withdrawal_credentials, amount, signature, self.to_little_endian_64(self.deposit_count)) + # Add `DepositData` root to Merkle tree (update a single `branch` node) self.deposit_count += 1 size: uint256 = self.deposit_count @@ -97,5 +101,3 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], node = sha256(concat(self.branch[height], node)) size /= 2 - # Emit `Deposit` log - log.Deposit(pubkey, withdrawal_credentials, amount, signature) diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 12f3e7f8bb..783af33567 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -49,28 +49,6 @@ def deposit_input(): ) -@pytest.mark.parametrize( - 'value,success', - [ - (0, True), - (10, True), - (55555, True), - (2**64 - 1, True), - # (2**64, True), # Note that all calls to `to_little_endian_64` have an input less than 2**64 - ] -) -def test_to_little_endian_64(registration_contract, value, success, assert_tx_failed): - call = registration_contract.functions.to_little_endian_64(value) - - if success: - little_endian_64 = call.call() - assert little_endian_64 == (value).to_bytes(8, 'little') - else: - assert_tx_failed( - lambda: call.call() - ) - - @pytest.mark.parametrize( 'success,deposit_amount', [ @@ -151,7 +129,7 @@ def test_deposit_log(registration_contract, a0, w3, deposit_input): assert log['withdrawal_credentials'] == deposit_input[1] assert log['amount'] == deposit_amount_list[i].to_bytes(8, 'little') assert log['signature'] == deposit_input[2] - + assert log['index'] == i.to_bytes(8, 'little') def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input): log_filter = registration_contract.events.Deposit.createFilter( @@ -171,6 +149,8 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input assert len(logs) == 1 log = logs[0]['args'] + assert log["index"] == i.to_bytes(8, 'little') + deposit_data = DepositData( pubkey=deposit_input[0], withdrawal_credentials=deposit_input[1], From 6a83205420cde658f9ac1a0f9e324313d3dacb2f Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Tue, 11 Jun 2019 15:29:34 +0100 Subject: [PATCH 019/405] Minor copy edit --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 61729e8d98..c25d43d96a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1168,7 +1168,7 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: * `genesis_deposits = deposits` * `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` * `genesis_eth1_data` be the object of type `Eth1Data` where: - * `genesis_eth1_data.block_hash` is the Ethereum 1.0 block hash that emitted to log for the last deposit in `deposits` + * `genesis_eth1_data.block_hash` is the Ethereum 1.0 block hash that emitted the log for the last deposit in `deposits` * `genesis_eth1_data.deposit_root` is the deposit root for the last deposit in `deposits` * `genesis_eth1_data.deposit_count = len(genesis_deposits)` From 193b628fb4fb7c3ef08cf160a9bbafade61a839e Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Tue, 11 Jun 2019 19:03:26 +0100 Subject: [PATCH 020/405] Bypass Vyper compiler bug --- deposit_contract/contracts/validator_registration.json | 2 +- deposit_contract/contracts/validator_registration.v.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index 1779423d2a..1988b28c09 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334984}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b505061134056600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63c5f2892f60005114156103cf57341561029057600080fd5b6000610140526001546101605261018060006020818352015b600160016101605116141561032a57600061018051602081106102cb57600080fd5b600060c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af161031c57600080fd5b60c051905061014052610398565b6000610140516020826101a0010152602081019050610180516020811061035057600080fd5b600260c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161038e57600080fd5b60c0519050610140525b61016060026103a657600080fd5b60028151048152505b81516001018083528114156102a9575b50506101405160005260206000f3005b63621fd13060005114156104e15734156103e857600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f8201039050610220511015156104335761044c565b610220516101e001526102205160200161022052610411565b6101c0805160200180610280828460006004600a8704601201f161046f57600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156104a1576104bd565b60006102e0516102a001535b8151600101808352811415610491575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d60005114156111b557606060046101403760506004356004016101a037603060043560040135111561051757600080fd5b604060243560040161022037602060243560040135111561053757600080fd5b608060443560040161028037606060443560040135111561055757600080fd5b63ffffffff6001541061056957600080fd5b633b9aca00610340526103405161057f57600080fd5b61034051340461032052633b9aca0061032051101561059d57600080fd5b60306101a051146105ad57600080fd5b602061022051146105bd57600080fd5b606061028051146105cd57600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a05110151561065d57610676565b6104a05161046001526104a0516020016104a05261063b565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f16106dd57600080fd5b505060006101a060308060208461058001018260208501600060046016f15050805182019150506000601060208206610500016020828401111561072057600080fd5b602080610520826104c0600060046015f150508181528090509050905060108060208461058001018260208501600060046013f150508051820191505080610580526105809050602060c0825160208401600060025af161078057600080fd5b60c05190506104e05260006000604060208206610620016102805182840111156107a957600080fd5b606080610640826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af16107e957600080fd5b60c0519050602082610820010152602081019050600060406020602082066106e00161028051828401111561081d57600080fd5b606080610700826020602088068803016102800160006004601bf15050818152809050905090506020806020846107a001018260208501600060046015f15050805182019150506104c0516020826107a0010152602081019050806107a0526107a09050602060c0825160208401600060025af161089a57600080fd5b60c051905060208261082001015260208101905080610820526108209050602060c0825160208401600060025af16108d157600080fd5b60c051905061060052600060006104e0516020826108c00101526020810190506102206020806020846108c001018260208501600060046015f1505080518201915050806108c0526108c09050602060c0825160208401600060025af161093757600080fd5b60c0519050602082610a4001015260208101905060006103606008806020846109c001018260208501600060046012f15050805182019150506000601860208206610940016020828401111561098c57600080fd5b602080610960826104c0600060046015f15050818152809050905090506018806020846109c001018260208501600060046014f1505080518201915050610600516020826109c0010152602081019050806109c0526109c09050602060c0825160208401600060025af16109ff57600080fd5b60c0519050602082610a4001015260208101905080610a4052610a409050602060c0825160208401600060025af1610a3657600080fd5b60c05190506108a0526101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a0516104c0516104e05161050051610520516105405161056051610580516105a0516105c0516105e05161060051610620516106405161066051610680516106a0516106c0516106e05161070051610720516107405161076051610780516107a0516107c0516107e05161080051610820516108405161086051610880516108a0516108c0516108e05161090051610920516109405161096051610980516109a0516109c0516109e051610a0051610a2051610a4051610a6051610a8051610aa0516380673289610ac052600154610ae052610ae0516006580161009b565b50610b40526000610ba0525b610b405160206001820306601f8201039050610ba051101515610bae57610bc7565b610ba051610b600152610ba051602001610ba052610b8c565b610aa052610a8052610a6052610a4052610a2052610a00526109e0526109c0526109a05261098052610960526109405261092052610900526108e0526108c0526108a05261088052610860526108405261082052610800526107e0526107c0526107a05261078052610760526107405261072052610700526106e0526106c0526106a05261068052610660526106405261062052610600526105e0526105c0526105a05261058052610560526105405261052052610500526104e0526104c0526104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610b40805160200180610bc0828460006004600a8704601201f1610d0e57600080fd5b505060a0610c4052610c4051610c80526101a0805160200180610c4051610c8001828460006004600a8704601201f1610d4657600080fd5b5050610c4051610c80015160206001820306601f8201039050610c4051610c8001610c2081516040818352015b83610c2051101515610d8457610da1565b6000610c20516020850101535b8151600101808352811415610d73575b505050506020610c4051610c80015160206001820306601f8201039050610c40510101610c4052610c4051610ca052610220805160200180610c4051610c8001828460006004600a8704601201f1610df857600080fd5b5050610c4051610c80015160206001820306601f8201039050610c4051610c8001610c2081516020818352015b83610c2051101515610e3657610e53565b6000610c20516020850101535b8151600101808352811415610e25575b505050506020610c4051610c80015160206001820306601f8201039050610c40510101610c4052610c4051610cc052610360805160200180610c4051610c8001828460006004600a8704601201f1610eaa57600080fd5b5050610c4051610c80015160206001820306601f8201039050610c4051610c8001610c2081516020818352015b83610c2051101515610ee857610f05565b6000610c20516020850101535b8151600101808352811415610ed7575b505050506020610c4051610c80015160206001820306601f8201039050610c40510101610c4052610c4051610ce052610280805160200180610c4051610c8001828460006004600a8704601201f1610f5c57600080fd5b5050610c4051610c80015160206001820306601f8201039050610c4051610c8001610c2081516060818352015b83610c2051101515610f9a57610fb7565b6000610c20516020850101535b8151600101808352811415610f89575b505050506020610c4051610c80015160206001820306601f8201039050610c40510101610c4052610c4051610d0052610bc0805160200180610c4051610c8001828460006004600a8704601201f161100e57600080fd5b5050610c4051610c80015160206001820306601f8201039050610c4051610c8001610c2081516020818352015b83610c205110151561104c57611069565b6000610c20516020850101535b815160010180835281141561103b575b505050506020610c4051610c80015160206001820306601f8201039050610c40510101610c40527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce610c4051610c80a160018054600182540110156110cd57600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d205116141561111d576108a051610d40516020811061110c57600080fd5b600060c052602060c02001556111b1565b6000610d40516020811061113057600080fd5b600060c052602060c0200154602082610d600101526020810190506108a051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af161118157600080fd5b60c05190506108a052610d20600261119857600080fd5b60028151048152505b81516001018083528114156110e8575b5050005b60006000fd5b61018561134003610185600039610185611340036000f3"} \ No newline at end of file +{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334417}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b50506111c656600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63c5f2892f60005114156103cf57341561029057600080fd5b6000610140526001546101605261018060006020818352015b600160016101605116141561032a57600061018051602081106102cb57600080fd5b600060c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af161031c57600080fd5b60c051905061014052610398565b6000610140516020826101a0010152602081019050610180516020811061035057600080fd5b600260c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161038e57600080fd5b60c0519050610140525b61016060026103a657600080fd5b60028151048152505b81516001018083528114156102a9575b50506101405160005260206000f3005b63621fd13060005114156104e15734156103e857600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f8201039050610220511015156104335761044c565b610220516101e001526102205160200161022052610411565b6101c0805160200180610280828460006004600a8704601201f161046f57600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156104a1576104bd565b60006102e0516102a001535b8151600101808352811415610491575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d600051141561103b57606060046101403760506004356004016101a037603060043560040135111561051757600080fd5b604060243560040161022037602060243560040135111561053757600080fd5b608060443560040161028037606060443560040135111561055757600080fd5b63ffffffff6001541061056957600080fd5b633b9aca00610340526103405161057f57600080fd5b61034051340461032052633b9aca0061032051101561059d57600080fd5b60306101a051146105ad57600080fd5b602061022051146105bd57600080fd5b606061028051146105cd57600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a05110151561065d57610676565b6104a05161046001526104a0516020016104a05261063b565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f16106dd57600080fd5b50506101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a05163806732896104c0526001546104e0526104e0516006580161009b565b506105405260006105a0525b6105405160206001820306601f82010390506105a05110151561078e576107a7565b6105a05161056001526105a0516020016105a05261076c565b6104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526105408051602001806105c0828460006004600a8704601201f161082e57600080fd5b505060a06106405261064051610680526101a08051602001806106405161068001828460006004600a8704601201f161086657600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516040818352015b83610620511015156108a4576108c1565b6000610620516020850101535b8151600101808352811415610893575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106a0526102208051602001806106405161068001828460006004600a8704601201f161091857600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b836106205110151561095657610973565b6000610620516020850101535b8151600101808352811415610945575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106c0526103608051602001806106405161068001828460006004600a8704601201f16109ca57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610a0857610a25565b6000610620516020850101535b81516001018083528114156109f7575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106e0526102808051602001806106405161068001828460006004600a8704601201f1610a7c57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516060818352015b8361062051101515610aba57610ad7565b6000610620516020850101535b8151600101808352811415610aa9575b50505050602061064051610680015160206001820306601f82010390506106405101016106405261064051610700526105c08051602001806106405161068001828460006004600a8704601201f1610b2e57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610b6c57610b89565b6000610620516020850101535b8151600101808352811415610b5b575b50505050602061064051610680015160206001820306601f8201039050610640510101610640527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce61064051610680a160006107205260006101a06030806020846107e001018260208501600060046016f150508051820191505060006010602082066107600160208284011115610c2057600080fd5b60208061078082610720600060046015f15050818152809050905090506010806020846107e001018260208501600060046013f1505080518201915050806107e0526107e09050602060c0825160208401600060025af1610c8057600080fd5b60c0519050610740526000600060406020820661088001610280518284011115610ca957600080fd5b6060806108a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af1610ce957600080fd5b60c0519050602082610a800101526020810190506000604060206020820661094001610280518284011115610d1d57600080fd5b606080610960826020602088068803016102800160006004601bf1505081815280905090509050602080602084610a0001018260208501600060046015f150508051820191505061072051602082610a0001015260208101905080610a0052610a009050602060c0825160208401600060025af1610d9a57600080fd5b60c0519050602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610dd157600080fd5b60c0519050610860526000600061074051602082610b20010152602081019050610220602080602084610b2001018260208501600060046015f150508051820191505080610b2052610b209050602060c0825160208401600060025af1610e3757600080fd5b60c0519050602082610ca00101526020810190506000610360600880602084610c2001018260208501600060046012f15050805182019150506000601860208206610ba00160208284011115610e8c57600080fd5b602080610bc082610720600060046015f1505081815280905090509050601880602084610c2001018260208501600060046014f150508051820191505061086051602082610c2001015260208101905080610c2052610c209050602060c0825160208401600060025af1610eff57600080fd5b60c0519050602082610ca001015260208101905080610ca052610ca09050602060c0825160208401600060025af1610f3657600080fd5b60c0519050610b00526001805460018254011015610f5357600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d2051161415610fa357610b0051610d405160208110610f9257600080fd5b600060c052602060c0200155611037565b6000610d405160208110610fb657600080fd5b600060c052602060c0200154602082610d60010152602081019050610b0051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af161100757600080fd5b60c0519050610b0052610d20600261101e57600080fd5b60028151048152505b8151600101808352811415610f6e575b5050005b60006000fd5b6101856111c6036101856000396101856111c6036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index 10daf42b93..14f1005209 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -75,9 +75,12 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], assert len(withdrawal_credentials) == WITHDRAWAL_CREDENTIALS_LENGTH assert len(signature) == SIGNATURE_LENGTH - # Compute `DepositData` root + # Emit `Deposit` log amount: bytes[8] = self.to_little_endian_64(deposit_amount) - zero_bytes32: bytes32 + log.Deposit(pubkey, withdrawal_credentials, amount, signature, self.to_little_endian_64(self.deposit_count)) + + # Compute `DepositData` root + zero_bytes32: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000 pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes32, start=0, len=64 - PUBKEY_LENGTH))) signature_root: bytes32 = sha256(concat( sha256(slice(signature, start=0, len=64)), @@ -88,9 +91,6 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], sha256(concat(amount, slice(zero_bytes32, start=0, len=32 - AMOUNT_LENGTH), signature_root)), )) - # Emit `Deposit` log - log.Deposit(pubkey, withdrawal_credentials, amount, signature, self.to_little_endian_64(self.deposit_count)) - # Add `DepositData` root to Merkle tree (update a single `branch` node) self.deposit_count += 1 size: uint256 = self.deposit_count From 7a366828baf1a3a1ba34074b45becc01cbedd2e8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 12 Jun 2019 14:54:00 -0400 Subject: [PATCH 021/405] Make phase0 pass --- scripts/build_spec.py | 2 ++ specs/core/0_beacon-chain.md | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 12 ++++++------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index f396d3b9c5..a4a97ab575 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -13,6 +13,7 @@ PHASE0_IMPORTS = '''from typing import ( Any, + Callable, Dict, List, Set, @@ -39,6 +40,7 @@ ''' PHASE1_IMPORTS = '''from typing import ( Any, + Callable, Dict, List, Set, diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 57bc2673f1..86c22c9764 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1634,15 +1634,15 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: assert len(body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index) # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) - - for operations, max_operations, function in ( + all_operations = [ (body.proposer_slashings, MAX_PROPOSER_SLASHINGS, process_proposer_slashing), (body.attester_slashings, MAX_ATTESTER_SLASHINGS, process_attester_slashing), (body.attestations, MAX_ATTESTATIONS, process_attestation), (body.deposits, MAX_DEPOSITS, process_deposit), (body.voluntary_exits, MAX_VOLUNTARY_EXITS, process_voluntary_exit), (body.transfers, MAX_TRANSFERS, process_transfer), - ): + ] # type: List[Tuple[List[Container], int, Callable]] + for operations, max_operations, function in all_operations: assert len(operations) <= max_operations for operation in operations: function(state, operation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 25851f331b..b75a40403a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -414,12 +414,12 @@ class Bytes96(BytesN): # SSZ Defaults # ----------------------------- def get_zero_value(typ): - if is_list_type(typ): + if is_uint_type(typ): + return uint64(0) + elif is_list_type(typ): return [] elif is_bool_type(typ): return False - elif is_uint_type(typ): - return uint64(0) elif is_vector_type(typ): return typ() elif is_bytesn_type(typ): @@ -437,12 +437,12 @@ def get_zero_value(typ): def infer_type(obj): - if isinstance(obj, int): + if is_uint_type(obj.__class__): + return obj.__class__ + elif isinstance(obj, int): return uint64 elif isinstance(obj, list): return List[infer_type(obj[0])] - elif is_uint_type(obj.__class__): - return obj.__class__ elif isinstance(obj, (Vector, Container, bool, BytesN, bytes)): return obj.__class__ else: From 8a54203796503a873780377e180c8c19d143ec6a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 12 Jun 2019 16:50:49 -0400 Subject: [PATCH 022/405] Modify the mypy config --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 87439559fd..26fe3a3876 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec; \ cd ./eth2spec; \ - mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs --disallow-any-generics -p phase0 + mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS) cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt From 48e8164e28aeb73d7bc9108a89c40bff60ef2e42 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 12 Jun 2019 20:08:19 -0400 Subject: [PATCH 023/405] Add phase1 type hinting checks and fix many bugs --- scripts/build_spec.py | 1 + specs/core/1_custody-game.md | 62 ++++++++++++++++--------------- specs/core/1_shard-data-chains.md | 34 ++++++++--------- 3 files changed, 51 insertions(+), 46 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index a4a97ab575..3fb1dda7ea 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -43,6 +43,7 @@ Callable, Dict, List, + Optional, Set, Tuple, ) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 6c89ef8531..9e8414a427 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -184,7 +184,7 @@ class CustodyResponse(Container): ```python class CustodyKeyReveal(Container): # Index of the validator whose key is being revealed - revealer_index: uint64 + revealer_index: ValidatorIndex # Reveal (masked signature) reveal: Bytes96 ``` @@ -198,7 +198,7 @@ class EarlyDerivedSecretReveal(Container): # Index of the validator whose key is being revealed revealed_index: uint64 # RANDAO epoch of the key that is being revealed - epoch: uint64 + epoch: Epoch # Reveal (masked signature) reveal: Bytes96 # Index of the validator who revealed (whistleblower) @@ -251,7 +251,7 @@ class BeaconBlockBody(Container): ### `ceillog2` ```python -def ceillog2(x): +def ceillog2(x: int) -> int: return x.bit_length() ``` @@ -269,7 +269,7 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int: ```python def get_custody_chunk_bit(key: Bytes96, chunk: bytes) -> bool: # TODO: Replace with something MPC-friendly, e.g. the Legendre symbol - return get_bitfield_bit(hash(key + chunk), 0) + return bool(get_bitfield_bit(hash(key + chunk), 0)) ``` ### `get_chunk_bits_root` @@ -288,7 +288,7 @@ def get_chunk_bits_root(chunk_bitfield: bytes) -> Bytes32: ```python def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorIndex) -> Epoch: next_period_start = (period + 1) * EPOCHS_PER_CUSTODY_PERIOD - validator_index % EPOCHS_PER_CUSTODY_PERIOD - return next_period_start + CUSTODY_PERIOD_TO_RANDAO_PADDING + return Epoch(next_period_start + CUSTODY_PERIOD_TO_RANDAO_PADDING) ``` ### `get_validators_custody_reveal_period` @@ -372,7 +372,11 @@ def process_custody_key_reveal(state: BeaconState, # Reward Block Preposer proposer_index = get_beacon_proposer_index(state) - increase_balance(state, proposer_index, get_base_reward(state, reveal.revealer_index) // MINOR_REWARD_QUOTIENT) + increase_balance( + state, + proposer_index, + Gwei(get_base_reward(state, reveal.revealer_index) // MINOR_REWARD_QUOTIENT) + ) ``` #### Early derived secret reveals @@ -433,7 +437,7 @@ def process_early_derived_secret_reveal(state: BeaconState, // len(get_active_validator_indices(state, get_current_epoch(state))) // PROPOSER_REWARD_QUOTIENT ) - penalty = ( + penalty = Gwei( max_proposer_slot_reward * EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE * (len(state.exposed_derived_secrets[derived_secret_location]) + 1) @@ -442,8 +446,8 @@ def process_early_derived_secret_reveal(state: BeaconState, # Apply penalty proposer_index = get_beacon_proposer_index(state) whistleblower_index = reveal.masker_index - whistleblowing_reward = penalty // WHISTLEBLOWING_REWARD_QUOTIENT - proposer_reward = whistleblowing_reward // PROPOSER_REWARD_QUOTIENT + whistleblowing_reward = Gwei(penalty // WHISTLEBLOWING_REWARD_QUOTIENT) + proposer_reward = Gwei(whistleblowing_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) increase_balance(state, whistleblower_index, whistleblowing_reward - proposer_reward) decrease_balance(state, reveal.revealed_index, penalty) @@ -512,7 +516,7 @@ def process_bit_challenge(state: BeaconState, pubkey=challenger.pubkey, message_hash=signing_root(challenge), signature=challenge.signature, - domain=get_domain(state, get_current_epoch(state), DOMAIN_CUSTODY_BIT_CHALLENGE), + domain=get_domain(state, DOMAIN_CUSTODY_BIT_CHALLENGE, get_current_epoch(state)), ) assert is_slashable_validator(challenger, get_current_epoch(state)) @@ -535,8 +539,8 @@ def process_bit_challenge(state: BeaconState, # Verify the responder is a valid custody key epoch_to_sign = get_randao_epoch_for_custody_period( get_validators_custody_reveal_period( - state=state, - index=challenge.responder_index, + state, + challenge.responder_index, epoch=slot_to_epoch(attestation.data.slot)), challenge.responder_index ) @@ -610,7 +614,7 @@ def process_chunk_challenge_response(state: BeaconState, # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), - branch=response.data_branch, + proof=response.data_branch, depth=challenge.depth, index=response.chunk_index, root=challenge.data_root, @@ -620,7 +624,7 @@ def process_chunk_challenge_response(state: BeaconState, records[records.index(challenge)] = CustodyChunkChallengeRecord() # Reward the proposer proposer_index = get_beacon_proposer_index(state) - increase_balance(state, proposer_index, get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT) + increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) ``` ```python @@ -635,7 +639,7 @@ def process_bit_challenge_response(state: BeaconState, # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), - branch=response.data_branch, + proof=response.data_branch, depth=ceillog2(challenge.chunk_count), index=response.chunk_index, root=challenge.data_root, @@ -643,7 +647,7 @@ def process_bit_challenge_response(state: BeaconState, # Verify the chunk bit leaf matches the challenge data assert verify_merkle_branch( leaf=response.chunk_bits_leaf, - branch=response.chunk_bits_branch, + proof=response.chunk_bits_branch, depth=ceillog2(challenge.chunk_count) >> 8, index=response.chunk_index // 256, root=challenge.chunk_bits_merkle_root @@ -671,8 +675,8 @@ Run `process_reveal_deadlines(state)` immediately after `process_registry_update def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): deadline = validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) - if get_validators_custody_reveal_period(state, index) > deadline: - slash_validator(state, index) + if get_validators_custody_reveal_period(state, ValidatorIndex(index)) > deadline: + slash_validator(state, ValidatorIndex(index)) ``` Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadlines(state)`: @@ -682,17 +686,17 @@ Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadl process_challenge_deadlines(state) # end insert @process_challenge_deadlines def process_challenge_deadlines(state: BeaconState) -> None: - for challenge in state.custody_chunk_challenge_records: - if get_current_epoch(state) > challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: - slash_validator(state, challenge.responder_index, challenge.challenger_index) - records = state.custody_chunk_challenge_records - records[records.index(challenge)] = CustodyChunkChallengeRecord() - - for challenge in state.custody_bit_challenge_records: - if get_current_epoch(state) > challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: - slash_validator(state, challenge.responder_index, challenge.challenger_index) + for custody_chunk_challenge in state.custody_chunk_challenge_records: + if get_current_epoch(state) > custody_chunk_challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: + slash_validator(state, custody_chunk_challenge.responder_index, custody_chunk_challenge.challenger_index) + records = state.custody_chunk_challenge + records[records.index(custody_chunk_challenge)] = CustodyChunkChallengeRecord() + + for custody_bit_challenge in state.custody_bit_challenge_records: + if get_current_epoch(state) > custody_bit_challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: + slash_validator(state, custody_bit_challenge.responder_index, custody_bit_challenge.challenger_index) records = state.custody_bit_challenge_records - records[records.index(challenge)] = CustodyBitChallengeRecord() + records[records.index(custody_bit_challenge)] = CustodyBitChallengeRecord() ``` Append this to `process_final_updates(state)`: @@ -713,5 +717,5 @@ def after_process_final_updates(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): if index not in validator_indices_in_records: if validator.exit_epoch != FAR_FUTURE_EPOCH and validator.withdrawable_epoch == FAR_FUTURE_EPOCH: - validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY + validator.withdrawable_epoch = Epoch(validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY) ``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 21e08e7c96..ce6f0a6c28 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -79,8 +79,8 @@ class ShardBlockBody(Container): ```python class ShardAttestation(Container): class data(Container): - slot: uint64 - shard: uint64 + slot: Slot + shard: Shard shard_block_root: Bytes32 aggregation_bitfield: bytes aggregate_signature: Bytes96 @@ -90,8 +90,8 @@ class ShardAttestation(Container): ```python class ShardBlock(Container): - slot: uint64 - shard: uint64 + slot: Slot + shard: Shard beacon_chain_root: Bytes32 parent_root: Bytes32 data: ShardBlockBody @@ -104,8 +104,8 @@ class ShardBlock(Container): ```python class ShardBlockHeader(Container): - slot: uint64 - shard: uint64 + slot: Slot + shard: Shard beacon_chain_root: Bytes32 parent_root: Bytes32 body_root: Bytes32 @@ -138,8 +138,8 @@ def get_period_committee(state: BeaconState, ### `get_switchover_epoch` ```python -def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex): - earlier_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2 +def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex) -> int: + earlier_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2) return (bytes_to_int(hash(generate_seed(state, earlier_start_epoch) + int_to_bytes(index, length=3)[0:8])) % PERSISTENT_COMMITTEE_PERIOD) ``` @@ -154,19 +154,19 @@ def get_persistent_committee(state: BeaconState, Return the persistent committee for the given ``shard`` at the given ``slot``. """ epoch = slot_to_epoch(slot) - earlier_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2 - later_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD + earlier_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2) + later_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD) committee_count = max( - len(get_active_validator_indices(state.validator_registry, earlier_start_epoch)) // + len(get_active_validator_indices(state, earlier_start_epoch)) // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), - len(get_active_validator_indices(state.validator_registry, later_start_epoch)) // + len(get_active_validator_indices(state, later_start_epoch)) // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), ) + 1 index = slot % committee_count - earlier_committee = get_period_committee(state, shard, earlier_start_epoch, index, committee_count) - later_committee = get_period_committee(state, shard, later_start_epoch, index, committee_count) + earlier_committee = get_period_committee(state, earlier_start_epoch, shard, index, committee_count) + later_committee = get_period_committee(state, later_start_epoch, shard, index, committee_count) # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated @@ -181,7 +181,7 @@ def get_persistent_committee(state: BeaconState, ```python def get_shard_proposer_index(state: BeaconState, shard: Shard, - slot: Slot) -> ValidatorIndex: + slot: Slot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee persistent_committee = get_persistent_committee(state, shard, slot) seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) @@ -231,7 +231,7 @@ def verify_shard_attestation_signature(state: BeaconState, pubkey=bls_aggregate_pubkeys(pubkeys), message_hash=data.shard_block_root, signature=attestation.aggregate_signature, - domain=get_domain(state, slot_to_epoch(data.slot), DOMAIN_SHARD_ATTESTER) + domain=get_domain(state, DOMAIN_SHARD_ATTESTER, slot_to_epoch(data.slot)) ) ``` @@ -328,7 +328,7 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], pubkey=beacon_state.validator_registry[proposer_index].pubkey, message_hash=signing_root(block), signature=candidate.signature, - domain=get_domain(beacon_state, slot_to_epoch(candidate.slot), DOMAIN_SHARD_PROPOSER), + domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, slot_to_epoch(candidate.slot)), ) return True From 8577cff72eedc2d99eec7e2cc288715834de8a36 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 12 Jun 2019 20:10:24 -0400 Subject: [PATCH 024/405] enable mypy check in CI --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 26fe3a3876..ca59d6c8a9 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,8 @@ lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec; \ cd ./eth2spec; \ - mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 + mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0; \ + mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1 install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS) cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt From f095ab43bb2df7934ef03fcaca3be1b87ea058e7 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 13 Jun 2019 10:40:02 +0100 Subject: [PATCH 025/405] Update 0_beacon-chain.md (#1170) minor typo fix --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c25d43d96a..e594ab898c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1158,7 +1158,7 @@ def slash_validator(state: BeaconState, ### Genesis trigger -Before genesis has been triggered and whenever the deposit contract emits a `Deposit`, log call the function `is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool` where: +Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool` where: * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log From aabd2b08adcc2771f741a50f71cb7120645487f0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 13 Jun 2019 15:46:22 +0200 Subject: [PATCH 026/405] attestations: check shard, and check epoch earlier --- specs/core/0_beacon-chain.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e594ab898c..86df188981 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1722,6 +1722,10 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: Process ``Attestation`` operation. """ data = attestation.data + + assert data.crosslink.shard < SHARD_COUNT + assert data.target_epoch in (get_previous_epoch(state), get_current_epoch(state)) + attestation_slot = get_attestation_data_slot(state, data) assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH @@ -1732,7 +1736,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_index=get_beacon_proposer_index(state), ) - assert data.target_epoch in (get_previous_epoch(state), get_current_epoch(state)) if data.target_epoch == get_current_epoch(state): ffg_data = (state.current_justified_epoch, state.current_justified_root, get_current_epoch(state)) parent_crosslink = state.current_crosslinks[data.crosslink.shard] From 67d2585ec0e24a0b59d1d5db0804faf04ac3f128 Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 13 Jun 2019 21:01:10 +0100 Subject: [PATCH 027/405] Fix #1173 The bug is that it's possible to include a participating validator which has custody bit one *without* specifying that validator in `attestation.aggregation_bitfield`. In other words, we want to check that every bit in `custody_bitfield` is zero whenever the corresponding bit in `aggregation_bitfield` is zero. Well spotted @protolambda --- specs/core/0_beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 86df188981..0725f5211b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -991,6 +991,7 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedA """ attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bitfield) + assert set(custody_bit_1_indices).issubset(attesting_indices) custody_bit_0_indices = [index for index in attesting_indices if index not in custody_bit_1_indices] return IndexedAttestation( From ed748a7d769e3d81697a2df92f1638cccf9c5057 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 15 Jun 2019 15:09:50 +0100 Subject: [PATCH 028/405] Address Danny's comments --- scripts/build_spec.py | 2 +- specs/core/0_beacon-chain.md | 14 +++++++------- specs/core/0_fork-choice.md | 4 ++-- test_libs/pyspec/eth2spec/test/helpers/block.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index e25147a7f0..eae9b60d57 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -62,7 +62,6 @@ from eth2spec.utils.hash_function import hash ''' -BYTE_TYPES = [4, 32, 48, 96] NEW_TYPES = { 'Slot': 'int', 'Epoch': 'int', @@ -70,6 +69,7 @@ 'ValidatorIndex': 'int', 'Gwei': 'int', } +BYTE_TYPES = [4, 32, 48, 96] SUNDRY_FUNCTIONS = ''' def get_ssz_type_by_name(name: str) -> Container: return globals()[name] diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b4049bb309..367a1ccebc 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -493,7 +493,7 @@ class BeaconState(Container): slot: Slot fork: Fork # History - parent_block_header: BeaconBlockHeader + latest_block_header: BeaconBlockHeader block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] historical_roots: List[Hash] @@ -1152,7 +1152,7 @@ def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis state = BeaconState( genesis_time=genesis_time, eth1_data=genesis_eth1_data, - parent_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) # Process genesis deposits @@ -1212,11 +1212,11 @@ def process_slot(state: BeaconState) -> None: state.state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root # Cache latest block header state root - if state.parent_block_header.state_root == ZERO_HASH: - state.parent_block_header.state_root = previous_state_root + if state.latest_block_header.state_root == ZERO_HASH: + state.latest_block_header.state_root = previous_state_root # Cache block root - previous_block_root = signing_root(state.parent_block_header) + previous_block_root = signing_root(state.latest_block_header) state.block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root ``` @@ -1556,9 +1556,9 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: # Verify that the slots match assert block.slot == state.slot # Verify that the parent matches - assert block.parent_root == signing_root(state.parent_block_header) + assert block.parent_root == signing_root(state.latest_block_header) # Save current block as the new latest block - state.parent_block_header = BeaconBlockHeader( + state.latest_block_header = BeaconBlockHeader( slot=block.slot, parent_root=block.parent_root, body_root=hash_tree_root(block.body), diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 739c06d596..8103bbeb2f 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -68,8 +68,8 @@ def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: return get_ancestor(store, store.get_parent(block), slot) ``` -* Let `get_attestation(store: Store, index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `index`. If several such attestations exist, use the one the validator `v` observed first. -* Let `get_attestation_target(store: Store, index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_attestation(store, index)`. +* Let `get_latest_attestation(store: Store, index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `index`. If several such attestations exist, use the one the validator `v` observed first. +* Let `get_attestation_target(store: Store, index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_latest_attestation(store, index)`. * Let `get_children(store: Store, block: BeaconBlock) -> List[BeaconBlock]` return the child blocks of the given `block`. * Let `justified_head_state` be the resulting `BeaconState` object from processing the chain up to the `justified_head`. * The `head` is `lmd_ghost(store, justified_head_state, justified_head)` where the function `lmd_ghost` is defined below. Note that the implementation below is suboptimal; there are implementations that compute the head in time logarithmic in slot count. diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 03f8d5b7e9..16249fe93d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -58,7 +58,7 @@ def build_empty_block(spec, state, slot=None, signed=False): empty_block = spec.BeaconBlock() empty_block.slot = slot empty_block.body.eth1_data.deposit_count = state.eth1_deposit_index - previous_block_header = deepcopy(state.parent_block_header) + previous_block_header = deepcopy(state.latest_block_header) if previous_block_header.state_root == spec.ZERO_HASH: previous_block_header.state_root = state.hash_tree_root() empty_block.parent_root = signing_root(previous_block_header) From 7cd7659a4b68ad515231ba635a800ae69e5f6798 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 15 Jun 2019 15:51:17 +0100 Subject: [PATCH 029/405] Add comments to non-obvious container fields --- specs/core/0_beacon-chain.md | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 367a1ccebc..cc4500d3fc 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -283,7 +283,7 @@ The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containe class Fork(Container): previous_version: Bytes4 current_version: Bytes4 - epoch: Epoch + epoch: Epoch # Epoch of latest fork ``` #### `Validator` @@ -291,14 +291,14 @@ class Fork(Container): ```python class Validator(Container): pubkey: BLSPubkey - withdrawal_credentials: Hash - effective_balance: Gwei + withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers + effective_balance: Gwei # Balance at stake slashed: bool # Status epochs - activation_eligibility_epoch: Epoch + activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch exit_epoch: Epoch - withdrawable_epoch: Epoch + withdrawable_epoch: Epoch # When validator can withdraw or transfer funds ``` #### `Crosslink` @@ -333,7 +333,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): data: AttestationData - custody_bit: bool + custody_bit: bool # Challengeable bit for the custody of crosslink data ``` #### `IndexedAttestation` @@ -350,7 +350,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: bytes + aggregation_bitfield: bytes # Bit set for every attesting participant within a committee data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -427,7 +427,7 @@ class Attestation(Container): ```python class Deposit(Container): - proof: Vector[Hash, DEPOSIT_CONTRACT_TREE_DEPTH] + proof: Vector[Hash, DEPOSIT_CONTRACT_TREE_DEPTH] # Merkle path to deposit root data: DepositData ``` @@ -435,7 +435,7 @@ class Deposit(Container): ```python class VoluntaryExit(Container): - epoch: Epoch + epoch: Epoch # Earliest epoch when voluntary exit can be processed validator_index: ValidatorIndex signature: BLSSignature ``` @@ -448,9 +448,9 @@ class Transfer(Container): recipient: ValidatorIndex amount: Gwei fee: Gwei - slot: Slot - pubkey: BLSPubkey - signature: BLSSignature + slot: Slot # Slot at which transfer must be processed + pubkey: BLSPubkey # Withdrawal pubkey + signature: BLSSignature # Signature checked against withdrawal pubkey ``` ### Beacon blocks @@ -460,8 +460,8 @@ class Transfer(Container): ```python class BeaconBlockBody(Container): randao_reveal: BLSSignature - eth1_data: Eth1Data - graffiti: Bytes32 + eth1_data: Eth1Data # Eth1 data vote + graffiti: Bytes32 # Arbitrary data # Operations proposer_slashings: List[ProposerSlashing] attester_slashings: List[AttesterSlashing] @@ -507,21 +507,21 @@ class BeaconState(Container): # Shuffling start_shard: Shard randao_mixes: Vector[Hash, RANDAO_MIXES_LENGTH] - active_index_roots: Vector[Hash, ACTIVE_INDEX_ROOTS_LENGTH] + active_index_roots: Vector[Hash, ACTIVE_INDEX_ROOTS_LENGTH] # Digests of the active registry, for light clients # Slashings - slashed_balances: Vector[Gwei, SLASHED_EXIT_LENGTH] + slashed_balances: Vector[Gwei, SLASHED_EXIT_LENGTH] # Sums of the effective balances of slashed validators # Attestations previous_epoch_attestations: List[PendingAttestation] current_epoch_attestations: List[PendingAttestation] # Crosslinks - previous_crosslinks: Vector[Crosslink, SHARD_COUNT] + previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] # Justification - previous_justified_epoch: Epoch - previous_justified_root: Hash + previous_justified_epoch: Epoch # Previous epoch snapshot + previous_justified_root: Hash # Previous epoch snapshot current_justified_epoch: Epoch current_justified_root: Hash - justification_bitfield: uint64 + justification_bitfield: uint64 # Bit set for every recent justified epoch # Finality finalized_epoch: Epoch finalized_root: Hash From 75b469281e870523a4145e8c7bdcf2da9a166aac Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 15 Jun 2019 18:05:01 +0200 Subject: [PATCH 030/405] fix linting issue --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cc4500d3fc..9680988950 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -298,7 +298,7 @@ class Validator(Container): activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch exit_epoch: Epoch - withdrawable_epoch: Epoch # When validator can withdraw or transfer funds + withdrawable_epoch: Epoch # When validator can withdraw or transfer funds ``` #### `Crosslink` From 00a68e28b50cb0c1067319d31c75d368b352b822 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 15 Jun 2019 16:57:50 -0400 Subject: [PATCH 031/405] Define Custom Types via function_puller --- scripts/build_spec.py | 17 ++++---- scripts/function_puller.py | 22 +++++----- specs/core/0_beacon-chain.md | 40 +++++++++---------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 4 files changed, 43 insertions(+), 38 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 3fb1dda7ea..6bcb04656d 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -128,11 +128,11 @@ def apply_constants_preset(preset: Dict[str, Any]) -> None: def objects_to_spec(functions: Dict[str, str], + custom_types: Dict[str, str], constants: Dict[str, str], ssz_objects: Dict[str, str], inserts: Dict[str, str], imports: Dict[str, str], - new_types: Dict[str, str], byte_types: List[int], ) -> str: """ @@ -144,7 +144,7 @@ def objects_to_spec(functions: Dict[str, str], f"class {key}({value}):\n" f" def __init__(self, _x: uint64) -> None:\n" f" ...\n" - for key, value in new_types.items() + for key, value in custom_types.items() ] ) ) @@ -225,18 +225,19 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: """ Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function. """ - functions0, constants0, ssz_objects0, inserts0 = spec0 - functions1, constants1, ssz_objects1, inserts1 = spec1 + functions0, custom_types0, constants0, ssz_objects0, inserts0 = spec0 + functions1, custom_types1, constants1, ssz_objects1, inserts1 = spec1 functions = combine_functions(functions0, functions1) + custom_types = combine_constants(custom_types0, custom_types1) constants = combine_constants(constants0, constants1) ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1) inserts = combine_inserts(inserts0, inserts1) - return functions, constants, ssz_objects, inserts + return functions, custom_types, constants, ssz_objects, inserts def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]: - functions, constants, ssz_objects, inserts = get_spec(sourcefile) - spec = objects_to_spec(functions, constants, ssz_objects, inserts, PHASE0_IMPORTS, NEW_TYPES, BYTE_TYPES) + functions, custom_types, constants, ssz_objects, inserts = get_spec(sourcefile) + spec = objects_to_spec(functions, custom_types, constants, ssz_objects, inserts, PHASE0_IMPORTS, BYTE_TYPES) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) @@ -253,7 +254,7 @@ def build_phase1_spec(phase0_sourcefile: str, spec_objects = phase0_spec for value in [phase1_custody, phase1_shard_data]: spec_objects = combine_spec_objects(spec_objects, value) - spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS, NEW_TYPES, BYTE_TYPES) + spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS, BYTE_TYPES) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 303d4ec2f5..4ad9eef57c 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -29,6 +29,7 @@ def get_spec(file_name: str) -> SpecObject: inserts = {} function_matcher = re.compile(FUNCTION_REGEX) inserts_matcher = re.compile(BEGIN_INSERT_REGEX) + custom_types = {} for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': @@ -64,7 +65,7 @@ def get_spec(file_name: str) -> SpecObject: ssz_objects[current_name] = ssz_objects.get(current_name, '') + line + '\n' else: functions[current_name] = functions.get(current_name, '') + line + '\n' - # Handle constant table entries + # Handle constant and custom types table entries elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') if len(row) >= 2: @@ -72,12 +73,15 @@ def get_spec(file_name: str) -> SpecObject: row[i] = row[i].strip().strip('`') if '`' in row[i]: row[i] = row[i][:row[i].find('`')] - eligible = True - if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': - eligible = False - for c in row[0]: - if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': + if row[1].startswith('uint') or row[1].startswith('bytes'): + custom_types[row[0]] = row[1] + else: + eligible = True + if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': eligible = False - if eligible: - constants[row[0]] = row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890') - return functions, constants, ssz_objects, inserts + for c in row[0]: + if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': + eligible = False + if eligible: + constants[row[0]] = row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890') + return functions, custom_types, constants, ssz_objects, inserts diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 86c22c9764..8baba6928f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -10,6 +10,7 @@ - [Introduction](#introduction) - [Notation](#notation) - [Terminology](#terminology) + - [Custom types](#custom-types) - [Constants](#constants) - [Misc](#misc) - [Deposit contract](#deposit-contract) @@ -45,7 +46,6 @@ - [`BeaconBlock`](#beaconblock) - [Beacon state](#beacon-state) - [`BeaconState`](#beaconstate) - - [Custom types](#custom-types) - [Helper functions](#helper-functions) - [`xor`](#xor) - [`hash`](#hash) @@ -150,6 +150,20 @@ Code snippets appearing in `this style` are to be interpreted as Python code. * **Withdrawal period**—the number of slots between a [validator](#dfn-validator) exit and the [validator](#dfn-validator) balance being withdrawable. * **Genesis time**—the Unix time of the genesis beacon chain block at slot 0. +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `Slot` | `uint64` | a slot number | +| `Epoch` | `uint64` | an epoch number | +| `Shard` | `uint64` | a shard number | +| `ValidatorIndex` | `uint64` | a validator registry index | +| `Gwei` | `uint64` | an amount in Gwei | +| `BLSPubkey` | `Bytes48` | a BLS12-381 public key | +| `BLSSignature` | `Bytes96` | a BLS12-381 signature | + ## Constants *Note*: The default mainnet values for the constants are included here for spec-design purposes. @@ -178,12 +192,12 @@ These configurations are updated for releases, but may be out of sync during `de ### Gwei values -| Name | Value | Unit | +| Name | Value | | - | - | :-: | -| `MIN_DEPOSIT_AMOUNT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | Gwei | -| `MAX_EFFECTIVE_BALANCE` | `Gwei(2**5 * 10**9)` (= 32,000,000,000) | Gwei | -| `EJECTION_BALANCE` | `Gwei(2**4 * 10**9)` (= 16,000,000,000) | Gwei | -| `EFFECTIVE_BALANCE_INCREMENT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | Gwei | +| `MIN_DEPOSIT_AMOUNT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | +| `MAX_EFFECTIVE_BALANCE` | `Gwei(2**5 * 10**9)` (= 32,000,000,000) | +| `EJECTION_BALANCE` | `Gwei(2**4 * 10**9)` (= 16,000,000,000) | +| `EFFECTIVE_BALANCE_INCREMENT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | ### Initial values @@ -563,20 +577,6 @@ class BeaconState(Container): deposit_index: uint64 ``` -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `Slot` | `uint64` | a slot number | -| `Epoch` | `uint64` | an epoch number | -| `Shard` | `uint64` | a shard number | -| `ValidatorIndex` | `uint64` | a validator registry index | -| `Gwei` | `uint64` | an amount in Gwei | -| `BLSPubkey` | `Bytes48` | a BLS12-381 public key | -| `BLSSignature` | `Bytes96` | a BLS12-381 signature | - ## Helper functions *Note*: The definitions below are for specification purposes and are not necessarily optimal implementations. diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index b75a40403a..cbd313d85f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -51,7 +51,7 @@ class uint64(uint): def __new__(cls, value, *args, **kwargs): if value.bit_length() > 64: - raise ValueError("value out of bounds for uint128") + raise ValueError("value out of bounds for uint64") return super().__new__(cls, value) From b772b03847d90e52b5c0f79df893f416b655fb6f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 15 Jun 2019 17:23:44 -0400 Subject: [PATCH 032/405] Handle `BLSPubkey` and `BLSSignature` --- scripts/build_spec.py | 14 ++++++++------ scripts/function_puller.py | 2 +- specs/core/0_beacon-chain.md | 22 +++++++++++----------- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 6bcb04656d..8f0048b8f7 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -142,8 +142,10 @@ def objects_to_spec(functions: Dict[str, str], '\n\n'.join( [ f"class {key}({value}):\n" - f" def __init__(self, _x: uint64) -> None:\n" + f" def __init__(self, _x: {value}) -> None:\n" f" ...\n" + if value.startswith("uint") + else f"class {key}({value}):\n pass\n" for key, value in custom_types.items() ] ) @@ -185,7 +187,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st return old_constants -def dependency_order_ssz_objects(objects: Dict[str, str]) -> None: +def dependency_order_ssz_objects(objects: Dict[str, str], custom_types: Dict[str, str]) -> None: """ Determines which SSZ Object is depenedent on which other and orders them appropriately """ @@ -194,14 +196,14 @@ def dependency_order_ssz_objects(objects: Dict[str, str]) -> None: dependencies = re.findall(r'(: [A-Z][\w[]*)', value) dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|uint\d+|Bytes\d+|bytes', '', x), dependencies) for dep in dependencies: - if dep in NEW_TYPES or len(dep) == 0: + if dep in custom_types or len(dep) == 0: continue key_list = list(objects.keys()) for item in [dep, key] + key_list[key_list.index(dep)+1:]: objects[item] = objects.pop(item) -def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str]) -> Dict[str, str]: +def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str], custom_types) -> Dict[str, str]: """ Takes in old spec and new spec ssz objects, combines them, and returns the newer versions of the objects in dependency order. @@ -213,7 +215,7 @@ def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str] # remove leading variable name value = re.sub(r'^class [\w]*\(Container\):\n', '', value) old_objects[key] = old_objects.get(key, '') + value - dependency_order_ssz_objects(old_objects) + dependency_order_ssz_objects(old_objects, custom_types) return old_objects @@ -230,7 +232,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: functions = combine_functions(functions0, functions1) custom_types = combine_constants(custom_types0, custom_types1) constants = combine_constants(constants0, constants1) - ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1) + ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1, custom_types) inserts = combine_inserts(inserts0, inserts1) return functions, custom_types, constants, ssz_objects, inserts diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 4ad9eef57c..0fd2fa4763 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -73,7 +73,7 @@ def get_spec(file_name: str) -> SpecObject: row[i] = row[i].strip().strip('`') if '`' in row[i]: row[i] = row[i][:row[i].find('`')] - if row[1].startswith('uint') or row[1].startswith('bytes'): + if row[1].startswith('uint') or row[1].startswith('Bytes'): custom_types[row[0]] = row[1] else: eligible = True diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8baba6928f..a45da28725 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -294,7 +294,7 @@ class Fork(Container): ```python class Validator(Container): # BLS public key - pubkey: Bytes48 + pubkey: BLSPubkey # Withdrawal credentials withdrawal_credentials: Bytes32 # Epoch when became eligible for activation @@ -363,7 +363,7 @@ class IndexedAttestation(Container): # Attestation data data: AttestationData # Aggregate signature - signature: Bytes96 + signature: BLSSignature ``` #### `PendingAttestation` @@ -407,13 +407,13 @@ class HistoricalBatch(Container): ```python class DepositData(Container): # BLS pubkey - pubkey: Bytes48 + pubkey: BLSPubkey # Withdrawal credentials withdrawal_credentials: Bytes32 # Amount in Gwei amount: Gwei # Container self-signature - signature: Bytes96 + signature: BLSSignature ``` #### `BeaconBlockHeader` @@ -424,7 +424,7 @@ class BeaconBlockHeader(Container): parent_root: Bytes32 state_root: Bytes32 body_root: Bytes32 - signature: Bytes96 + signature: BLSSignature ``` ### Beacon operations @@ -462,7 +462,7 @@ class Attestation(Container): # Custody bitfield custody_bitfield: bytes # BLS aggregate signature - signature: Bytes96 + signature: BLSSignature ``` #### `Deposit` @@ -484,7 +484,7 @@ class VoluntaryExit(Container): # Index of the exiting validator validator_index: ValidatorIndex # Validator signature - signature: Bytes96 + signature: BLSSignature ``` #### `Transfer` @@ -502,9 +502,9 @@ class Transfer(Container): # Inclusion slot slot: Slot # Sender withdrawal pubkey - pubkey: Bytes48 + pubkey: BLSPubkey # Sender signature - signature: Bytes96 + signature: BLSSignature ``` ### Beacon blocks @@ -513,7 +513,7 @@ class Transfer(Container): ```python class BeaconBlockBody(Container): - randao_reveal: Bytes96 + randao_reveal: BLSSignature eth1_data: Eth1Data graffiti: Bytes32 proposer_slashings: List[ProposerSlashing] @@ -533,7 +533,7 @@ class BeaconBlock(Container): parent_root: Bytes32 state_root: Bytes32 body: BeaconBlockBody - signature: Bytes96 + signature: BLSSignature ``` ### Beacon state From 9b77ec11f8e091992b74258f39e5633663c5903e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 15 Jun 2019 17:32:52 -0400 Subject: [PATCH 033/405] Version: Bytes4 --- scripts/build_spec.py | 7 ------- specs/core/0_beacon-chain.md | 5 +++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 8f0048b8f7..9f5d07b294 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -67,13 +67,6 @@ from eth2spec.utils.hash_function import hash ''' -NEW_TYPES = { - 'Slot': 'uint64', - 'Epoch': 'uint64', - 'Shard': 'uint64', - 'ValidatorIndex': 'uint64', - 'Gwei': 'uint64', -} BYTE_TYPES = [4, 32, 48, 96] SUNDRY_FUNCTIONS = ''' def get_ssz_type_by_name(name: str) -> Container: diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a45da28725..8b0c51b044 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -161,6 +161,7 @@ We define the following Python custom types for type hinting and readability: | `Shard` | `uint64` | a shard number | | `ValidatorIndex` | `uint64` | a validator registry index | | `Gwei` | `uint64` | an amount in Gwei | +| `Version` | `Bytes4` | a fork version number | | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | @@ -282,9 +283,9 @@ The types are defined topologically to aid in facilitating an executable version ```python class Fork(Container): # Previous fork version - previous_version: Bytes4 + previous_version: Version # Current fork version - current_version: Bytes4 + current_version: Version # Fork epoch number epoch: Epoch ``` From e93ba51ef807e7ae87c5f793c385112da1b8c0ab Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 15 Jun 2019 18:25:37 -0400 Subject: [PATCH 034/405] More clean up --- specs/core/1_custody-game.md | 18 +++++++++--------- specs/core/1_shard-data-chains.md | 6 +++--- specs/light_client/sync_protocol.md | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 9e8414a427..315084c301 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -133,9 +133,9 @@ class CustodyBitChallenge(Container): responder_index: ValidatorIndex attestation: Attestation challenger_index: ValidatorIndex - responder_key: Bytes96 + responder_key: BLSSignature chunk_bits: bytes - signature: Bytes96 + signature: BLSSignature ``` #### `CustodyChunkChallengeRecord` @@ -162,7 +162,7 @@ class CustodyBitChallengeRecord(Container): data_root: Bytes32 chunk_count: uint64 chunk_bits_merkle_root: Bytes32 - responder_key: Bytes96 + responder_key: BLSSignature ``` #### `CustodyResponse` @@ -186,7 +186,7 @@ class CustodyKeyReveal(Container): # Index of the validator whose key is being revealed revealer_index: ValidatorIndex # Reveal (masked signature) - reveal: Bytes96 + reveal: BLSSignature ``` #### `EarlyDerivedSecretReveal` @@ -196,13 +196,13 @@ Represents an early (punishable) reveal of one of the derived secrets, where der ```python class EarlyDerivedSecretReveal(Container): # Index of the validator whose key is being revealed - revealed_index: uint64 + revealed_index: ValidatorIndex # RANDAO epoch of the key that is being revealed epoch: Epoch # Reveal (masked signature) - reveal: Bytes96 + reveal: BLSSignature # Index of the validator who revealed (whistleblower) - masker_index: uint64 + masker_index: ValidatorIndex # Mask used to hide the actual reveal signature (prevent reveal from being stolen) mask: Bytes32 ``` @@ -232,7 +232,7 @@ class BeaconState(Container): # Future derived secrets already exposed; contains the indices of the exposed validator # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS - exposed_derived_secrets: Vector[List[uint64], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] + exposed_derived_secrets: Vector[List[ValidatorIndex], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] ``` #### `BeaconBlockBody` @@ -267,7 +267,7 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int: ### `get_custody_chunk_bit` ```python -def get_custody_chunk_bit(key: Bytes96, chunk: bytes) -> bool: +def get_custody_chunk_bit(key: BLSSignature, chunk: bytes) -> bool: # TODO: Replace with something MPC-friendly, e.g. the Legendre symbol return bool(get_bitfield_bit(hash(key + chunk), 0)) ``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index ce6f0a6c28..6b18d13f64 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -83,7 +83,7 @@ class ShardAttestation(Container): shard: Shard shard_block_root: Bytes32 aggregation_bitfield: bytes - aggregate_signature: Bytes96 + aggregate_signature: BLSSignature ``` ### `ShardBlock` @@ -97,7 +97,7 @@ class ShardBlock(Container): data: ShardBlockBody state_root: Bytes32 attestations: List[ShardAttestation] - signature: Bytes96 + signature: BLSSignature ``` ### `ShardBlockHeader` @@ -111,7 +111,7 @@ class ShardBlockHeader(Container): body_root: Bytes32 state_root: Bytes32 attestations: List[ShardAttestation] - signature: Bytes96 + signature: BLSSignature ``` ## Helper functions diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 8501c58690..dc2ce397be 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -167,7 +167,7 @@ If a client wants to update its `finalized_header` it asks the network for a `Bl ```python { 'header': BeaconBlockHeader, - 'shard_aggregate_signature': 'bytes96', + 'shard_aggregate_signature': BLSSignature, 'shard_bitfield': 'bytes', 'shard_parent_block': ShardBlock, } From 11edda64ed91dafa801cb58f831e27b28a153a64 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 15 Jun 2019 18:42:03 -0400 Subject: [PATCH 035/405] Adds fork-choice to spec builder --- Makefile | 4 +- scripts/build_spec.py | 28 +++-- specs/core/0_fork-choice.md | 186 ++++++++++++++++++++---------- specs/core/1_shard-data-chains.md | 1 - 4 files changed, 141 insertions(+), 78 deletions(-) diff --git a/Makefile b/Makefile index f79b89dada..7e20254188 100644 --- a/Makefile +++ b/Makefile @@ -66,10 +66,10 @@ test_deposit_contract: pyspec: $(PY_SPEC_ALL_TARGETS) $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@ CURRENT_DIR = ${CURDIR} diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 7a51970e38..4213c38b68 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -177,7 +177,7 @@ def dependency_order_ssz_objects(objects: Dict[str, str]) -> None: items = list(objects.items()) for key, value in items: dependencies = re.findall(r'(: [A-Z][\w[]*)', value) - dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|uint\d+|Bytes\d+|bytes', '', x), dependencies) + dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Dict|uint\d+|Bytes\d+|bytes', '', x), dependencies) for dep in dependencies: if dep in NEW_TYPES or len(dep) == 0: continue @@ -219,9 +219,11 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: return functions, constants, ssz_objects, inserts -def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]: - functions, constants, ssz_objects, inserts = get_spec(sourcefile) - spec = objects_to_spec(functions, constants, ssz_objects, inserts, PHASE0_IMPORTS, NEW_TYPES, BYTE_TYPES) +def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]: + phase0_spec = get_spec(phase0_sourcefile) + fork_choice_spec = get_spec(fork_choice_sourcefile) + spec_objects = combine_spec_objects(phase0_spec, fork_choice_spec) + spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS, NEW_TYPES, BYTE_TYPES) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) @@ -231,12 +233,14 @@ def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]: def build_phase1_spec(phase0_sourcefile: str, phase1_custody_sourcefile: str, phase1_shard_sourcefile: str, + fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]: phase0_spec = get_spec(phase0_sourcefile) phase1_custody = get_spec(phase1_custody_sourcefile) phase1_shard_data = get_spec(phase1_shard_sourcefile) + fork_choice_spec = get_spec(fork_choice_sourcefile) spec_objects = phase0_spec - for value in [phase1_custody, phase1_shard_data]: + for value in [phase1_custody, phase1_shard_data, fork_choice_spec]: spec_objects = combine_spec_objects(spec_objects, value) spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS, NEW_TYPES, BYTE_TYPES) if outfile is not None: @@ -250,13 +254,15 @@ def build_phase1_spec(phase0_sourcefile: str, Build the specs from the md docs. If building phase 0: 1st argument is input spec.md - 2nd argument is output spec.py + 2nd argument is input fork_choice.md + 3rd argument is output spec.py If building phase 1: 1st argument is input spec_phase0.md 2nd argument is input spec_phase1_custody.md 3rd argument is input spec_phase1_shard_data.md - 4th argument is output spec.py + 4th argument is input fork_choice.md + 5th argument is output spec.py ''' parser = ArgumentParser(description=description) parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #") @@ -264,14 +270,14 @@ def build_phase1_spec(phase0_sourcefile: str, args = parser.parse_args() if args.phase == 0: - if len(args.files) == 2: + if len(args.files) == 3: build_phase0_spec(*args.files) else: - print(" Phase 0 requires an output as well as an input file.") + print(" Phase 0 requires an output as well as spec and forkchoice files.") elif args.phase == 1: - if len(args.files) == 4: + if len(args.files) == 5: build_phase1_spec(*args.files) else: - print(" Phase 1 requires an output as well as 3 input files (phase0.md and phase1.md, phase1.md)") + print(" Phase 1 requires an output as well as 4 input files (phase0.md and phase1.md, phase1.md, fork_choice.md)") else: print("Invalid phase: {0}".format(args.phase)) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 91c3e27ee4..c840179b95 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -8,23 +8,27 @@ - [Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice](#ethereum-20-phase-0----beacon-chain-fork-choice) - [Table of contents](#table-of-contents) - [Introduction](#introduction) - - [Prerequisites](#prerequisites) - [Constants](#constants) - [Time parameters](#time-parameters) - - [Beacon chain processing](#beacon-chain-processing) - - [Beacon chain fork choice rule](#beacon-chain-fork-choice-rule) - - [Implementation notes](#implementation-notes) - - [Justification and finality at genesis](#justification-and-finality-at-genesis) + - [Fork choice](#fork-choice) + - [Containers](#containers) + - [`Target`](#target) + - [`Store`](#store) + - [Helpers](#helpers) + - [`get_genesis_store`](#get_genesis_store) + - [`get_ancestor`](#get_ancestor) + - [`get_attesting_balance_from_store`](#get_attesting_balance_from_store) + - [`get_head`](#get_head) + - [Handlers](#handlers) + - [`on_tick`](#on_tick) + - [`on_block`](#on_block) + - [`on_attestation`](#on_attestation) ## Introduction -This document represents the specification for the beacon chain fork choice rule, part of Ethereum 2.0 Phase 0. - -## Prerequisites - -All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./0_beacon-chain.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. +This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./0_beacon-chain.md). ## Constants @@ -34,76 +38,130 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | - | - | :-: | :-: | | `SECONDS_PER_SLOT` | `6` | seconds | 6 seconds | -## Beacon chain processing +## Fork choice + +The head block root associated with a `store` is defined as `get_head(store)`. At genesis let `store = get_genesis_store(genesis_state)` and update `store` by running: + +* `on_tick(time)` whenever `time > store.time` where `time` is the current Unix time +* `on_block(block)` whenever a block `block` is received +* `on_attestation(attestation)` whenever an attestation `attestation` is received + +*Notes*: + +1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. +2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. +3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. +4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. + +### Containers -Processing the beacon chain is similar to processing the Ethereum 1.0 chain. Clients download and process blocks and maintain a view of what is the current "canonical chain", terminating at the current "head". For a beacon block, `block`, to be processed by a node, the following conditions must be met: +#### `Target` + +```python +class Target(Container): + epoch: Epoch + root: Bytes32 +``` + +#### `Store` + +```python +class Store(Container): + blocks: Dict[Bytes32, BeaconBlock] + states: Dict[Bytes32, BeaconState] + time: uint64 + latest_targets: Dict[ValidatorIndex, Target] + justified_root: Bytes32 + finalized_root: Bytes32 +``` -* The parent block with root `block.parent_root` has been processed and accepted. -* An Ethereum 1.0 block pointed to by the `state.latest_eth1_data.block_hash` has been processed and accepted. -* The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`. +### Helpers -*Note*: Leap seconds mean that slots will occasionally last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds, possibly several times a year. +#### `get_genesis_store` -*Note*: Nodes needs to have a clock that is roughly (i.e. within `SECONDS_PER_SLOT` seconds) synchronized with the other nodes. +```python +def get_genesis_store(genesis_state: BeaconState) -> Store: + genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) + root = signing_root(genesis_block) + return Store(blocks={root: genesis_block}, states={root: genesis_state}, finalized_root=root, justified_root=root) +``` -### Beacon chain fork choice rule +#### `get_ancestor` -The beacon chain fork choice rule is a hybrid that combines justification and finality with Latest Message Driven (LMD) Greediest Heaviest Observed SubTree (GHOST). At any point in time, a validator `v` subjectively calculates the beacon chain head as follows. +```python +def get_ancestor(store: Store, root: Bytes32, slot: Slot) -> Bytes32: + block = store.blocks[root] + assert block.slot >= slot + return root if block.slot == slot else get_ancestor(store, block.parent_root, slot) +``` -* Abstractly define `Store` as the type of storage object for the chain data, and let `store` be the set of attestations and blocks that the validator `v` has observed and verified (in particular, block ancestors must be recursively verified). Attestations not yet included in any chain are still included in `store`. -* Let `finalized_head` be the finalized block with the highest epoch. (A block `B` is finalized if there is a descendant of `B` in `store`, the processing of which sets `B` as finalized.) -* Let `justified_head` be the descendant of `finalized_head` with the highest epoch that has been justified for at least 1 epoch. (A block `B` is justified if there is a descendant of `B` in `store` the processing of which sets `B` as justified.) If no such descendant exists, set `justified_head` to `finalized_head`. -* Let `get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock` be the ancestor of `block` with slot number `slot`. The `get_ancestor` function can be defined recursively as: +#### `get_attesting_balance_from_store` ```python -def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: - """ - Get the ancestor of ``block`` with slot number ``slot``; return ``None`` if not found. - """ - if block.slot == slot: - return block - elif block.slot < slot: - return None - else: - return get_ancestor(store, store.get_parent(block), slot) +def get_attesting_balance_from_store(store: Store, root: Bytes32) -> Gwei: + state = store.states[store.justified_root] + active_indices = get_active_validator_indices(state.validator_registry, slot_to_epoch(state.slot)) + return sum( + state.validator_registry[i].effective_balance for i in active_indices + if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root + ) ``` -* Let `get_latest_attestation(store: Store, index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `index`. If several such attestations exist, use the one the validator `v` observed first. -* Let `get_latest_attestation_target(store: Store, index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_latest_attestation(store, index)`. -* Let `get_children(store: Store, block: BeaconBlock) -> List[BeaconBlock]` return the child blocks of the given `block`. -* Let `justified_head_state` be the resulting `BeaconState` object from processing the chain up to the `justified_head`. -* The `head` is `lmd_ghost(store, justified_head_state, justified_head)` where the function `lmd_ghost` is defined below. Note that the implementation below is suboptimal; there are implementations that compute the head in time logarithmic in slot count. +#### `get_head` ```python -def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) -> BeaconBlock: - """ - Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``. - """ - validators = start_state.validator_registry - active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot)) - attestation_targets = [(i, get_latest_attestation_target(store, i)) for i in active_validator_indices] - - # Use the rounded-balance-with-hysteresis supplied by the protocol for fork - # choice voting. This reduces the number of recomputations that need to be - # made for optimized implementations that precompute and save data - def get_vote_count(block: BeaconBlock) -> int: - return sum( - start_state.validator_registry[validator_index].effective_balance - for validator_index, target in attestation_targets - if get_ancestor(store, target, block.slot) == block - ) - - head = start_block - while 1: - children = get_children(store, head) +def get_head(store: Store) -> Bytes32: + # Execute the LMD-GHOST fork choice + head = store.justified_root + while True: + children = [root for root in store.blocks.keys() if store.blocks[root].parent_root == head] if len(children) == 0: return head - # Ties broken by favoring block with lexicographically higher root - head = max(children, key=lambda x: (get_vote_count(x), hash_tree_root(x))) + # Sort by attesting balance with ties broken lexicographically + head = max(children, key=lambda root: (get_attesting_balance_from_store(store, root), root)) +``` + +### Handlers + +#### `on_tick` + +```python +def on_tick(store: Store, time: int) -> None: + store.time = time ``` -## Implementation notes +#### `on_block` -### Justification and finality at genesis +```python +def on_block(store: Store, block: BeaconBlock) -> None: + # Add new block to the store + store.blocks[signing_root(block)] = block + # Check block is a descendant of the finalized block + assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root + # Check block slot against Unix time + pre_state = store.states[block.parent_root].copy() + assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT + # Check the block is valid and compute the post-state + state = state_transition(pre_state, block) + # Add new state to the store + store.states[signing_root(block)] = state + # Update justified and finalized blocks + if state.finalized_epoch > slot_to_epoch(store.blocks[store.finalized_root].slot): + store.finalized_root = state.finalized_root + if state.current_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + store.justified_root = state.current_justified_root + if state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + store.justified_root = state.previous_justified_root +``` -During genesis, justification and finality root fields within the `BeaconState` reference `ZERO_HASH` rather than a known block. `ZERO_HASH` in `previous_justified_root`, `current_justified_root`, and `finalized_root` should be considered as an alias to the root of the genesis block. +#### `on_attestation` + +```python +def on_attestation(store: Store, attestation: Attestation) -> None: + state = store.states[get_head(store)] + indexed_attestation = convert_to_indexed(state, attestation) + validate_indexed_attestation(state, indexed_attestation) + for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: + if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: + store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) +``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 21e08e7c96..84af66ffb1 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -56,7 +56,6 @@ This document describes the shard data layer and the shard fork choice rule in P | - | - | :-: | :-: | | `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.2 minutes | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | -| `SECONDS_PER_SLOT` | `2**1 * 3**1` (= 6) | 6 seconds | ### Signature domains From 061ecf7d0a870bf5336cffa1a3567054e5978c27 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 15 Jun 2019 19:13:56 -0400 Subject: [PATCH 036/405] Adds fork-choice tests --- .../pyspec/eth2spec/test/test_fork_choice.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 test_libs/pyspec/eth2spec/test/test_fork_choice.py diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py new file mode 100644 index 0000000000..46b60b93c5 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -0,0 +1,50 @@ +from eth2spec.utils.ssz.ssz_impl import signing_root + +from eth2spec.test.context import with_all_phases, spec_state_test + +from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.state import next_slot + + +@with_all_phases +@spec_state_test +def test_basic(spec, state): + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + assert store.time == time + + # On receiving a block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(state) + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + + # On receiving a block of next epoch + store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + block = build_empty_block_for_next_slot(state) + block.slot += spec.SLOTS_PER_EPOCH + + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + + # TODO: add tests for justified_root and finalized_root + + +@with_all_phases +@spec_state_test +def test_on_attestation(spec, state): + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + next_slot(state) + + attestation = get_valid_attestation(state, slot=1) + indexed_attestation = spec.convert_to_indexed(state, attestation) + spec.on_attestation(store, attestation) + assert ( + store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == + spec.Target(attestation.data.target_epoch, attestation.data.target_root) + ) \ No newline at end of file From 5d10cd63c7301e68ef13dbc008e7a34eccdc4763 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 16 Jun 2019 09:52:52 -0400 Subject: [PATCH 037/405] SSZ (partially) handles Dicts --- specs/core/0_fork-choice.md | 5 +- .../pyspec/eth2spec/test/test_fork_choice.py | 65 ++++++++++++------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 37 ++++++++++- 3 files changed, 80 insertions(+), 27 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index c840179b95..3ed794584a 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -163,5 +163,8 @@ def on_attestation(store: Store, attestation: Attestation) -> None: validate_indexed_attestation(state, indexed_attestation) for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) + store.latest_targets[i] = Target( + epoch = attestation.data.target_epoch, + root = attestation.data.target_root, + ) ``` diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index 46b60b93c5..3744660326 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -10,41 +10,56 @@ @with_all_phases @spec_state_test def test_basic(spec, state): + yield 'pre', state + # Initialization - store = spec.get_genesis_store(state) - time = 100 - spec.on_tick(store, time) - assert store.time == time + store = spec.get_genesis_store(state) + blocks = [] + time = 100 + spec.on_tick(store, time) + assert store.time == time # On receiving a block of `GENESIS_SLOT + 1` slot - block = build_empty_block_for_next_slot(state) - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block + block = build_empty_block_for_next_slot(spec, state) + blocks.append(block) + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block # On receiving a block of next epoch - store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - block = build_empty_block_for_next_slot(state) - block.slot += spec.SLOTS_PER_EPOCH + store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + block = build_empty_block_for_next_slot(spec, state) + block.slot += spec.SLOTS_PER_EPOCH + blocks.append(block) - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + yield 'blocks', blocks, List[spec.BeaconBlock] # TODO: add tests for justified_root and finalized_root + yield 'post', state @with_all_phases @spec_state_test def test_on_attestation(spec, state): - store = spec.get_genesis_store(state) - time = 100 - spec.on_tick(store, time) - - next_slot(state) - - attestation = get_valid_attestation(state, slot=1) - indexed_attestation = spec.convert_to_indexed(state, attestation) - spec.on_attestation(store, attestation) - assert ( - store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == - spec.Target(attestation.data.target_epoch, attestation.data.target_root) - ) \ No newline at end of file + yield 'pre', state + + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=1) + yield 'attestation', attestation + indexed_attestation = spec.convert_to_indexed(state, attestation) + spec.on_attestation(store, attestation) + assert ( + store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == + spec.Target( + epoch = attestation.data.target_epoch, + root = attestation.data.target_root + ) + ) + + yield 'post', state \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 55ced4ee2a..7119f2e5ba 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,4 @@ -from typing import List, Iterable, TypeVar, Type, NewType +from typing import List, Iterable, TypeVar, Type, NewType, Dict from typing import Union from typing_inspect import get_origin @@ -280,6 +280,32 @@ def __eq__(self, other): return self.hash_tree_root() == other.hash_tree_root() +# # Super Secret Un-documented SSZ Dict (for forkchoice) +# # ----------------------------- +class Dict(dict): + def __init__(self,*args,**kwargs) : dict.__init__(self,*args,**kwargs) + + def serialize(self): + raise NotImplementedError + + def hash_tree_root(self): + raise NotImplementedError + + def __getitem__(self, key): + return self.items[key] + + def __setitem__(self, key, value): + self.items[key] = value + + def __iter__(self): + return iter(self.items) + + def __len__(self): + return len(self.items) + + def __eq__(self, other): + raise NotImplementedError + # SSZ BytesN # ----------------------------- @@ -407,6 +433,8 @@ def get_zero_value(typ): return b'' elif is_container_type(typ): return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) + elif is_dict_type(typ): + return dict() else: raise Exception("Type not supported: {}".format(typ)) @@ -498,6 +526,13 @@ def is_container_type(typ): return isinstance(typ, type) and issubclass(typ, Container) +def is_dict_type(typ): + """ + Check of the given type is a Dict. (Which are a part of the super-secret undocumented SSZ spec) + """ + return get_origin(typ) is Dict or get_origin(typ) is dict + + T = TypeVar('T') L = TypeVar('L') From 4deb311b71ea6e1dabe956dc2de374d517d9dc35 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 16 Jun 2019 12:17:31 -0400 Subject: [PATCH 038/405] Debugging 1st test --- specs/core/0_fork-choice.md | 7 +- .../pyspec/eth2spec/test/test_finality.py | 406 +++++++++--------- .../pyspec/eth2spec/test/test_fork_choice.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 41 +- 4 files changed, 223 insertions(+), 233 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3ed794584a..3d1e70ca47 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -83,6 +83,7 @@ class Store(Container): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) + print('groot', root) return Store(blocks={root: genesis_block}, states={root: genesis_state}, finalized_root=root, justified_root=root) ``` @@ -90,6 +91,7 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: ```python def get_ancestor(store: Store, root: Bytes32, slot: Slot) -> Bytes32: + print('ruut', root) block = store.blocks[root] assert block.slot >= slot return root if block.slot == slot else get_ancestor(store, block.parent_root, slot) @@ -135,14 +137,17 @@ def on_tick(store: Store, time: int) -> None: ```python def on_block(store: Store, block: BeaconBlock) -> None: # Add new block to the store + print('setting', signing_root(block)) store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root # Check block slot against Unix time pre_state = store.states[block.parent_root].copy() + print('store.states[block.parent_root]', hash_tree_root(store.states[block.parent_root])) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) + print('store.states[block.parent_root]', hash_tree_root(store.states[block.parent_root])) # Add new state to the store store.states[signing_root(block)] = state # Update justified and finalized blocks @@ -150,7 +155,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.finalized_root = state.finalized_root if state.current_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): store.justified_root = state.current_justified_root - if state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + elif state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): store.justified_root = state.previous_justified_root ``` diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 801e8b4fd5..9556abbd13 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,203 +1,203 @@ -from copy import deepcopy -from typing import List - -from eth2spec.test.context import spec_state_test, never_bls, with_all_phases -from eth2spec.test.helpers.state import next_epoch -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block -from eth2spec.test.helpers.attestations import get_valid_attestation - - -def check_finality(spec, - state, - prev_state, - current_justified_changed, - previous_justified_changed, - finalized_changed): - if current_justified_changed: - assert state.current_justified_epoch > prev_state.current_justified_epoch - assert state.current_justified_root != prev_state.current_justified_root - else: - assert state.current_justified_epoch == prev_state.current_justified_epoch - assert state.current_justified_root == prev_state.current_justified_root - - if previous_justified_changed: - assert state.previous_justified_epoch > prev_state.previous_justified_epoch - assert state.previous_justified_root != prev_state.previous_justified_root - else: - assert state.previous_justified_epoch == prev_state.previous_justified_epoch - assert state.previous_justified_root == prev_state.previous_justified_root - - if finalized_changed: - assert state.finalized_epoch > prev_state.finalized_epoch - assert state.finalized_root != prev_state.finalized_root - else: - assert state.finalized_epoch == prev_state.finalized_epoch - assert state.finalized_root == prev_state.finalized_root - - -def next_epoch_with_attestations(spec, - state, - fill_cur_epoch, - fill_prev_epoch): - post_state = deepcopy(state) - blocks = [] - for _ in range(spec.SLOTS_PER_EPOCH): - block = build_empty_block_for_next_slot(spec, post_state) - if fill_cur_epoch: - slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): - cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) - block.body.attestations.append(cur_attestation) - - if fill_prev_epoch: - slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 - prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest) - block.body.attestations.append(prev_attestation) - - spec.state_transition(post_state, block) - blocks.append(block) - - return state, blocks, post_state - - -@with_all_phases -@never_bls -@spec_state_test -def test_finality_rule_4(spec, state): - yield 'pre', state - - blocks = [] - for epoch in range(4): - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) - blocks += new_blocks - - # justification/finalization skipped at GENESIS_EPOCH - if epoch == 0: - check_finality(spec, state, prev_state, False, False, False) - # justification/finalization skipped at GENESIS_EPOCH + 1 - elif epoch == 1: - check_finality(spec, state, prev_state, False, False, False) - elif epoch == 2: - check_finality(spec, state, prev_state, True, False, False) - elif epoch >= 3: - # rule 4 of finality - check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root - - yield 'blocks', blocks, List[spec.BeaconBlock] - yield 'post', state - - -@with_all_phases -@never_bls -@spec_state_test -def test_finality_rule_1(spec, state): - # get past first two epochs that finality does not run on - next_epoch(spec, state) - apply_empty_block(spec, state) - next_epoch(spec, state) - apply_empty_block(spec, state) - - yield 'pre', state - - blocks = [] - for epoch in range(3): - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) - blocks += new_blocks - - if epoch == 0: - check_finality(spec, state, prev_state, True, False, False) - elif epoch == 1: - check_finality(spec, state, prev_state, True, True, False) - elif epoch == 2: - # finalized by rule 1 - check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root - - yield 'blocks', blocks, List[spec.BeaconBlock] - yield 'post', state - - -@with_all_phases -@never_bls -@spec_state_test -def test_finality_rule_2(spec, state): - # get past first two epochs that finality does not run on - next_epoch(spec, state) - apply_empty_block(spec, state) - next_epoch(spec, state) - apply_empty_block(spec, state) - - yield 'pre', state - - blocks = [] - for epoch in range(3): - if epoch == 0: - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) - check_finality(spec, state, prev_state, True, False, False) - elif epoch == 1: - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) - check_finality(spec, state, prev_state, False, True, False) - elif epoch == 2: - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) - # finalized by rule 2 - check_finality(spec, state, prev_state, True, False, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root - - blocks += new_blocks - - yield 'blocks', blocks, List[spec.BeaconBlock] - yield 'post', state - - -@with_all_phases -@never_bls -@spec_state_test -def test_finality_rule_3(spec, state): - """ - Test scenario described here - https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 - """ - # get past first two epochs that finality does not run on - next_epoch(spec, state) - apply_empty_block(spec, state) - next_epoch(spec, state) - apply_empty_block(spec, state) - - yield 'pre', state - - blocks = [] - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) - blocks += new_blocks - check_finality(spec, state, prev_state, True, False, False) - - # In epoch N, JE is set to N, prev JE is set to N-1 - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) - blocks += new_blocks - check_finality(spec, state, prev_state, True, True, True) - - # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) - blocks += new_blocks - check_finality(spec, state, prev_state, False, True, False) - - # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. - # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) - blocks += new_blocks - # rule 2 - check_finality(spec, state, prev_state, True, False, True) - - # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, True) - blocks += new_blocks - # rule 3 - check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root - - yield 'blocks', blocks, List[spec.BeaconBlock] - yield 'post', state +# from copy import deepcopy +# from typing import List + +# from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +# from eth2spec.test.helpers.state import next_epoch +# from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block +# from eth2spec.test.helpers.attestations import get_valid_attestation + + +# def check_finality(spec, +# state, +# prev_state, +# current_justified_changed, +# previous_justified_changed, +# finalized_changed): +# if current_justified_changed: +# assert state.current_justified_epoch > prev_state.current_justified_epoch +# assert state.current_justified_root != prev_state.current_justified_root +# else: +# assert state.current_justified_epoch == prev_state.current_justified_epoch +# assert state.current_justified_root == prev_state.current_justified_root + +# if previous_justified_changed: +# assert state.previous_justified_epoch > prev_state.previous_justified_epoch +# assert state.previous_justified_root != prev_state.previous_justified_root +# else: +# assert state.previous_justified_epoch == prev_state.previous_justified_epoch +# assert state.previous_justified_root == prev_state.previous_justified_root + +# if finalized_changed: +# assert state.finalized_epoch > prev_state.finalized_epoch +# assert state.finalized_root != prev_state.finalized_root +# else: +# assert state.finalized_epoch == prev_state.finalized_epoch +# assert state.finalized_root == prev_state.finalized_root + + +# def next_epoch_with_attestations(spec, +# state, +# fill_cur_epoch, +# fill_prev_epoch): +# post_state = deepcopy(state) +# blocks = [] +# for _ in range(spec.SLOTS_PER_EPOCH): +# block = build_empty_block_for_next_slot(spec, post_state) +# if fill_cur_epoch: +# slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 +# if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): +# cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) +# block.body.attestations.append(cur_attestation) + +# if fill_prev_epoch: +# slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 +# prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest) +# block.body.attestations.append(prev_attestation) + +# spec.state_transition(post_state, block) +# blocks.append(block) + +# return state, blocks, post_state + + +# @with_all_phases +# @never_bls +# @spec_state_test +# def test_finality_rule_4(spec, state): +# yield 'pre', state + +# blocks = [] +# for epoch in range(4): +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) +# blocks += new_blocks + +# # justification/finalization skipped at GENESIS_EPOCH +# if epoch == 0: +# check_finality(spec, state, prev_state, False, False, False) +# # justification/finalization skipped at GENESIS_EPOCH + 1 +# elif epoch == 1: +# check_finality(spec, state, prev_state, False, False, False) +# elif epoch == 2: +# check_finality(spec, state, prev_state, True, False, False) +# elif epoch >= 3: +# # rule 4 of finality +# check_finality(spec, state, prev_state, True, True, True) +# assert state.finalized_epoch == prev_state.current_justified_epoch +# assert state.finalized_root == prev_state.current_justified_root + +# yield 'blocks', blocks, List[spec.BeaconBlock] +# yield 'post', state + + +# @with_all_phases +# @never_bls +# @spec_state_test +# def test_finality_rule_1(spec, state): +# # get past first two epochs that finality does not run on +# next_epoch(spec, state) +# apply_empty_block(spec, state) +# next_epoch(spec, state) +# apply_empty_block(spec, state) + +# yield 'pre', state + +# blocks = [] +# for epoch in range(3): +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) +# blocks += new_blocks + +# if epoch == 0: +# check_finality(spec, state, prev_state, True, False, False) +# elif epoch == 1: +# check_finality(spec, state, prev_state, True, True, False) +# elif epoch == 2: +# # finalized by rule 1 +# check_finality(spec, state, prev_state, True, True, True) +# assert state.finalized_epoch == prev_state.previous_justified_epoch +# assert state.finalized_root == prev_state.previous_justified_root + +# yield 'blocks', blocks, List[spec.BeaconBlock] +# yield 'post', state + + +# @with_all_phases +# @never_bls +# @spec_state_test +# def test_finality_rule_2(spec, state): +# # get past first two epochs that finality does not run on +# next_epoch(spec, state) +# apply_empty_block(spec, state) +# next_epoch(spec, state) +# apply_empty_block(spec, state) + +# yield 'pre', state + +# blocks = [] +# for epoch in range(3): +# if epoch == 0: +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) +# check_finality(spec, state, prev_state, True, False, False) +# elif epoch == 1: +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) +# check_finality(spec, state, prev_state, False, True, False) +# elif epoch == 2: +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) +# # finalized by rule 2 +# check_finality(spec, state, prev_state, True, False, True) +# assert state.finalized_epoch == prev_state.previous_justified_epoch +# assert state.finalized_root == prev_state.previous_justified_root + +# blocks += new_blocks + +# yield 'blocks', blocks, List[spec.BeaconBlock] +# yield 'post', state + + +# @with_all_phases +# @never_bls +# @spec_state_test +# def test_finality_rule_3(spec, state): +# """ +# Test scenario described here +# https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 +# """ +# # get past first two epochs that finality does not run on +# next_epoch(spec, state) +# apply_empty_block(spec, state) +# next_epoch(spec, state) +# apply_empty_block(spec, state) + +# yield 'pre', state + +# blocks = [] +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) +# blocks += new_blocks +# check_finality(spec, state, prev_state, True, False, False) + +# # In epoch N, JE is set to N, prev JE is set to N-1 +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) +# blocks += new_blocks +# check_finality(spec, state, prev_state, True, True, True) + +# # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) +# blocks += new_blocks +# check_finality(spec, state, prev_state, False, True, False) + +# # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. +# # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) +# blocks += new_blocks +# # rule 2 +# check_finality(spec, state, prev_state, True, False, True) + +# # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, True) +# blocks += new_blocks +# # rule 3 +# check_finality(spec, state, prev_state, True, True, True) +# assert state.finalized_epoch == prev_state.current_justified_epoch +# assert state.finalized_root == prev_state.current_justified_root + +# yield 'blocks', blocks, List[spec.BeaconBlock] +# yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index 3744660326..c1df605fe2 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -58,7 +58,7 @@ def test_on_attestation(spec, state): store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == spec.Target( epoch = attestation.data.target_epoch, - root = attestation.data.target_root + root = attestation.data.target_root, ) ) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7119f2e5ba..5b3500d3ef 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,5 +1,13 @@ -from typing import List, Iterable, TypeVar, Type, NewType, Dict -from typing import Union +import copy +from typing import ( + List, + Iterable, + TypeVar, + Type, + NewType, + Dict, + Union, +) from typing_inspect import get_origin # SSZ integers @@ -123,6 +131,9 @@ def signing_root(self): def get_field_values(self): cls = self.__class__ return [getattr(self, field) for field in cls.get_field_names()] + + def copy(self): + return self.deepcopy(self) def __repr__(self): return repr({field: getattr(self, field) for field in self.get_field_names()}) @@ -280,32 +291,6 @@ def __eq__(self, other): return self.hash_tree_root() == other.hash_tree_root() -# # Super Secret Un-documented SSZ Dict (for forkchoice) -# # ----------------------------- -class Dict(dict): - def __init__(self,*args,**kwargs) : dict.__init__(self,*args,**kwargs) - - def serialize(self): - raise NotImplementedError - - def hash_tree_root(self): - raise NotImplementedError - - def __getitem__(self, key): - return self.items[key] - - def __setitem__(self, key, value): - self.items[key] = value - - def __iter__(self): - return iter(self.items) - - def __len__(self): - return len(self.items) - - def __eq__(self, other): - raise NotImplementedError - # SSZ BytesN # ----------------------------- From f421850dc91db05144b5792912f1c4296ba25cce Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 16 Jun 2019 15:53:42 -0400 Subject: [PATCH 039/405] Fixes tests! Thanks @hwwhww! --- specs/core/0_fork-choice.md | 5 - .../pyspec/eth2spec/test/test_finality.py | 406 +++++++++--------- .../pyspec/eth2spec/test/test_fork_choice.py | 5 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 4 files changed, 208 insertions(+), 210 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3d1e70ca47..9a06570816 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -83,7 +83,6 @@ class Store(Container): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) - print('groot', root) return Store(blocks={root: genesis_block}, states={root: genesis_state}, finalized_root=root, justified_root=root) ``` @@ -91,7 +90,6 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: ```python def get_ancestor(store: Store, root: Bytes32, slot: Slot) -> Bytes32: - print('ruut', root) block = store.blocks[root] assert block.slot >= slot return root if block.slot == slot else get_ancestor(store, block.parent_root, slot) @@ -137,17 +135,14 @@ def on_tick(store: Store, time: int) -> None: ```python def on_block(store: Store, block: BeaconBlock) -> None: # Add new block to the store - print('setting', signing_root(block)) store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root # Check block slot against Unix time pre_state = store.states[block.parent_root].copy() - print('store.states[block.parent_root]', hash_tree_root(store.states[block.parent_root])) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) - print('store.states[block.parent_root]', hash_tree_root(store.states[block.parent_root])) # Add new state to the store store.states[signing_root(block)] = state # Update justified and finalized blocks diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 9556abbd13..801e8b4fd5 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,203 +1,203 @@ -# from copy import deepcopy -# from typing import List - -# from eth2spec.test.context import spec_state_test, never_bls, with_all_phases -# from eth2spec.test.helpers.state import next_epoch -# from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block -# from eth2spec.test.helpers.attestations import get_valid_attestation - - -# def check_finality(spec, -# state, -# prev_state, -# current_justified_changed, -# previous_justified_changed, -# finalized_changed): -# if current_justified_changed: -# assert state.current_justified_epoch > prev_state.current_justified_epoch -# assert state.current_justified_root != prev_state.current_justified_root -# else: -# assert state.current_justified_epoch == prev_state.current_justified_epoch -# assert state.current_justified_root == prev_state.current_justified_root - -# if previous_justified_changed: -# assert state.previous_justified_epoch > prev_state.previous_justified_epoch -# assert state.previous_justified_root != prev_state.previous_justified_root -# else: -# assert state.previous_justified_epoch == prev_state.previous_justified_epoch -# assert state.previous_justified_root == prev_state.previous_justified_root - -# if finalized_changed: -# assert state.finalized_epoch > prev_state.finalized_epoch -# assert state.finalized_root != prev_state.finalized_root -# else: -# assert state.finalized_epoch == prev_state.finalized_epoch -# assert state.finalized_root == prev_state.finalized_root - - -# def next_epoch_with_attestations(spec, -# state, -# fill_cur_epoch, -# fill_prev_epoch): -# post_state = deepcopy(state) -# blocks = [] -# for _ in range(spec.SLOTS_PER_EPOCH): -# block = build_empty_block_for_next_slot(spec, post_state) -# if fill_cur_epoch: -# slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 -# if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): -# cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) -# block.body.attestations.append(cur_attestation) - -# if fill_prev_epoch: -# slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 -# prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest) -# block.body.attestations.append(prev_attestation) - -# spec.state_transition(post_state, block) -# blocks.append(block) - -# return state, blocks, post_state - - -# @with_all_phases -# @never_bls -# @spec_state_test -# def test_finality_rule_4(spec, state): -# yield 'pre', state - -# blocks = [] -# for epoch in range(4): -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) -# blocks += new_blocks - -# # justification/finalization skipped at GENESIS_EPOCH -# if epoch == 0: -# check_finality(spec, state, prev_state, False, False, False) -# # justification/finalization skipped at GENESIS_EPOCH + 1 -# elif epoch == 1: -# check_finality(spec, state, prev_state, False, False, False) -# elif epoch == 2: -# check_finality(spec, state, prev_state, True, False, False) -# elif epoch >= 3: -# # rule 4 of finality -# check_finality(spec, state, prev_state, True, True, True) -# assert state.finalized_epoch == prev_state.current_justified_epoch -# assert state.finalized_root == prev_state.current_justified_root - -# yield 'blocks', blocks, List[spec.BeaconBlock] -# yield 'post', state - - -# @with_all_phases -# @never_bls -# @spec_state_test -# def test_finality_rule_1(spec, state): -# # get past first two epochs that finality does not run on -# next_epoch(spec, state) -# apply_empty_block(spec, state) -# next_epoch(spec, state) -# apply_empty_block(spec, state) - -# yield 'pre', state - -# blocks = [] -# for epoch in range(3): -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) -# blocks += new_blocks - -# if epoch == 0: -# check_finality(spec, state, prev_state, True, False, False) -# elif epoch == 1: -# check_finality(spec, state, prev_state, True, True, False) -# elif epoch == 2: -# # finalized by rule 1 -# check_finality(spec, state, prev_state, True, True, True) -# assert state.finalized_epoch == prev_state.previous_justified_epoch -# assert state.finalized_root == prev_state.previous_justified_root - -# yield 'blocks', blocks, List[spec.BeaconBlock] -# yield 'post', state - - -# @with_all_phases -# @never_bls -# @spec_state_test -# def test_finality_rule_2(spec, state): -# # get past first two epochs that finality does not run on -# next_epoch(spec, state) -# apply_empty_block(spec, state) -# next_epoch(spec, state) -# apply_empty_block(spec, state) - -# yield 'pre', state - -# blocks = [] -# for epoch in range(3): -# if epoch == 0: -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) -# check_finality(spec, state, prev_state, True, False, False) -# elif epoch == 1: -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) -# check_finality(spec, state, prev_state, False, True, False) -# elif epoch == 2: -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) -# # finalized by rule 2 -# check_finality(spec, state, prev_state, True, False, True) -# assert state.finalized_epoch == prev_state.previous_justified_epoch -# assert state.finalized_root == prev_state.previous_justified_root - -# blocks += new_blocks - -# yield 'blocks', blocks, List[spec.BeaconBlock] -# yield 'post', state - - -# @with_all_phases -# @never_bls -# @spec_state_test -# def test_finality_rule_3(spec, state): -# """ -# Test scenario described here -# https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 -# """ -# # get past first two epochs that finality does not run on -# next_epoch(spec, state) -# apply_empty_block(spec, state) -# next_epoch(spec, state) -# apply_empty_block(spec, state) - -# yield 'pre', state - -# blocks = [] -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) -# blocks += new_blocks -# check_finality(spec, state, prev_state, True, False, False) - -# # In epoch N, JE is set to N, prev JE is set to N-1 -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) -# blocks += new_blocks -# check_finality(spec, state, prev_state, True, True, True) - -# # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) -# blocks += new_blocks -# check_finality(spec, state, prev_state, False, True, False) - -# # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. -# # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) -# blocks += new_blocks -# # rule 2 -# check_finality(spec, state, prev_state, True, False, True) - -# # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, True) -# blocks += new_blocks -# # rule 3 -# check_finality(spec, state, prev_state, True, True, True) -# assert state.finalized_epoch == prev_state.current_justified_epoch -# assert state.finalized_root == prev_state.current_justified_root - -# yield 'blocks', blocks, List[spec.BeaconBlock] -# yield 'post', state +from copy import deepcopy +from typing import List + +from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block +from eth2spec.test.helpers.attestations import get_valid_attestation + + +def check_finality(spec, + state, + prev_state, + current_justified_changed, + previous_justified_changed, + finalized_changed): + if current_justified_changed: + assert state.current_justified_epoch > prev_state.current_justified_epoch + assert state.current_justified_root != prev_state.current_justified_root + else: + assert state.current_justified_epoch == prev_state.current_justified_epoch + assert state.current_justified_root == prev_state.current_justified_root + + if previous_justified_changed: + assert state.previous_justified_epoch > prev_state.previous_justified_epoch + assert state.previous_justified_root != prev_state.previous_justified_root + else: + assert state.previous_justified_epoch == prev_state.previous_justified_epoch + assert state.previous_justified_root == prev_state.previous_justified_root + + if finalized_changed: + assert state.finalized_epoch > prev_state.finalized_epoch + assert state.finalized_root != prev_state.finalized_root + else: + assert state.finalized_epoch == prev_state.finalized_epoch + assert state.finalized_root == prev_state.finalized_root + + +def next_epoch_with_attestations(spec, + state, + fill_cur_epoch, + fill_prev_epoch): + post_state = deepcopy(state) + blocks = [] + for _ in range(spec.SLOTS_PER_EPOCH): + block = build_empty_block_for_next_slot(spec, post_state) + if fill_cur_epoch: + slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 + if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): + cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) + block.body.attestations.append(cur_attestation) + + if fill_prev_epoch: + slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 + prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest) + block.body.attestations.append(prev_attestation) + + spec.state_transition(post_state, block) + blocks.append(block) + + return state, blocks, post_state + + +@with_all_phases +@never_bls +@spec_state_test +def test_finality_rule_4(spec, state): + yield 'pre', state + + blocks = [] + for epoch in range(4): + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + blocks += new_blocks + + # justification/finalization skipped at GENESIS_EPOCH + if epoch == 0: + check_finality(spec, state, prev_state, False, False, False) + # justification/finalization skipped at GENESIS_EPOCH + 1 + elif epoch == 1: + check_finality(spec, state, prev_state, False, False, False) + elif epoch == 2: + check_finality(spec, state, prev_state, True, False, False) + elif epoch >= 3: + # rule 4 of finality + check_finality(spec, state, prev_state, True, True, True) + assert state.finalized_epoch == prev_state.current_justified_epoch + assert state.finalized_root == prev_state.current_justified_root + + yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'post', state + + +@with_all_phases +@never_bls +@spec_state_test +def test_finality_rule_1(spec, state): + # get past first two epochs that finality does not run on + next_epoch(spec, state) + apply_empty_block(spec, state) + next_epoch(spec, state) + apply_empty_block(spec, state) + + yield 'pre', state + + blocks = [] + for epoch in range(3): + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) + blocks += new_blocks + + if epoch == 0: + check_finality(spec, state, prev_state, True, False, False) + elif epoch == 1: + check_finality(spec, state, prev_state, True, True, False) + elif epoch == 2: + # finalized by rule 1 + check_finality(spec, state, prev_state, True, True, True) + assert state.finalized_epoch == prev_state.previous_justified_epoch + assert state.finalized_root == prev_state.previous_justified_root + + yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'post', state + + +@with_all_phases +@never_bls +@spec_state_test +def test_finality_rule_2(spec, state): + # get past first two epochs that finality does not run on + next_epoch(spec, state) + apply_empty_block(spec, state) + next_epoch(spec, state) + apply_empty_block(spec, state) + + yield 'pre', state + + blocks = [] + for epoch in range(3): + if epoch == 0: + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + check_finality(spec, state, prev_state, True, False, False) + elif epoch == 1: + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) + check_finality(spec, state, prev_state, False, True, False) + elif epoch == 2: + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) + # finalized by rule 2 + check_finality(spec, state, prev_state, True, False, True) + assert state.finalized_epoch == prev_state.previous_justified_epoch + assert state.finalized_root == prev_state.previous_justified_root + + blocks += new_blocks + + yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'post', state + + +@with_all_phases +@never_bls +@spec_state_test +def test_finality_rule_3(spec, state): + """ + Test scenario described here + https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 + """ + # get past first two epochs that finality does not run on + next_epoch(spec, state) + apply_empty_block(spec, state) + next_epoch(spec, state) + apply_empty_block(spec, state) + + yield 'pre', state + + blocks = [] + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + blocks += new_blocks + check_finality(spec, state, prev_state, True, False, False) + + # In epoch N, JE is set to N, prev JE is set to N-1 + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + blocks += new_blocks + check_finality(spec, state, prev_state, True, True, True) + + # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) + blocks += new_blocks + check_finality(spec, state, prev_state, False, True, False) + + # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. + # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) + blocks += new_blocks + # rule 2 + check_finality(spec, state, prev_state, True, False, True) + + # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, True) + blocks += new_blocks + # rule 3 + check_finality(spec, state, prev_state, True, True, True) + assert state.finalized_epoch == prev_state.current_justified_epoch + assert state.finalized_root == prev_state.current_justified_root + + yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index c1df605fe2..6f054fc528 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -1,4 +1,6 @@ -from eth2spec.utils.ssz.ssz_impl import signing_root +from typing import List + +from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root from eth2spec.test.context import with_all_phases, spec_state_test @@ -10,6 +12,7 @@ @with_all_phases @spec_state_test def test_basic(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) yield 'pre', state # Initialization diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 5b3500d3ef..7df95d1192 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -133,7 +133,7 @@ def get_field_values(self): return [getattr(self, field) for field in cls.get_field_names()] def copy(self): - return self.deepcopy(self) + return copy.deepcopy(self) def __repr__(self): return repr({field: getattr(self, field) for field in self.get_field_names()}) From 9f2d06b2e7e74f977f20c35dbda7617d44511bda Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 16 Jun 2019 16:02:56 -0400 Subject: [PATCH 040/405] Somehow I had indented with 5 spaces everywhere. --- specs/core/0_fork-choice.md | 4 +- .../pyspec/eth2spec/test/test_fork_choice.py | 86 +++++++++---------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 9a06570816..f3d57feaa0 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -164,7 +164,7 @@ def on_attestation(store: Store, attestation: Attestation) -> None: for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: store.latest_targets[i] = Target( - epoch = attestation.data.target_epoch, - root = attestation.data.target_root, + epoch=attestation.data.target_epoch, + root=attestation.data.target_root, ) ``` diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index 6f054fc528..ab17282519 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -12,57 +12,57 @@ @with_all_phases @spec_state_test def test_basic(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - yield 'pre', state + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + yield 'pre', state # Initialization - store = spec.get_genesis_store(state) - blocks = [] - time = 100 - spec.on_tick(store, time) - assert store.time == time + store = spec.get_genesis_store(state) + blocks = [] + time = 100 + spec.on_tick(store, time) + assert store.time == time - # On receiving a block of `GENESIS_SLOT + 1` slot - block = build_empty_block_for_next_slot(spec, state) - blocks.append(block) - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block + # On receiving a block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + blocks.append(block) + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block - # On receiving a block of next epoch - store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - block = build_empty_block_for_next_slot(spec, state) - block.slot += spec.SLOTS_PER_EPOCH - blocks.append(block) + # On receiving a block of next epoch + store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + block = build_empty_block_for_next_slot(spec, state) + block.slot += spec.SLOTS_PER_EPOCH + blocks.append(block) - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block - yield 'blocks', blocks, List[spec.BeaconBlock] + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + yield 'blocks', blocks, List[spec.BeaconBlock] - # TODO: add tests for justified_root and finalized_root - yield 'post', state + # TODO: add tests for justified_root and finalized_root + yield 'post', state @with_all_phases @spec_state_test def test_on_attestation(spec, state): - yield 'pre', state - - store = spec.get_genesis_store(state) - time = 100 - spec.on_tick(store, time) - - next_slot(spec, state) - - attestation = get_valid_attestation(spec, state, slot=1) - yield 'attestation', attestation - indexed_attestation = spec.convert_to_indexed(state, attestation) - spec.on_attestation(store, attestation) - assert ( - store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == - spec.Target( - epoch = attestation.data.target_epoch, - root = attestation.data.target_root, - ) - ) - - yield 'post', state \ No newline at end of file + yield 'pre', state + + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=1) + yield 'attestation', attestation + indexed_attestation = spec.convert_to_indexed(state, attestation) + spec.on_attestation(store, attestation) + assert ( + store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == + spec.Target( + epoch=attestation.data.target_epoch, + root=attestation.data.target_root, + ) + ) + + yield 'post', state diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7df95d1192..3bd7dd0624 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -131,7 +131,7 @@ def signing_root(self): def get_field_values(self): cls = self.__class__ return [getattr(self, field) for field in cls.get_field_names()] - + def copy(self): return copy.deepcopy(self) From 2ea6cede3edad9df50088378fd82f2037558f513 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 17 Jun 2019 10:48:33 -0400 Subject: [PATCH 041/405] Moves fork-choice objects away from SSZ --- scripts/build_spec.py | 14 +++++++++++++ scripts/function_puller.py | 1 + specs/core/0_fork-choice.md | 20 ++++++++++--------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 14 ------------- test_libs/pyspec/requirements.txt | 1 + 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 4213c38b68..1091e7bed1 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -19,6 +19,13 @@ Tuple, ) +from dataclasses import ( + dataclass, + field, +) + +from copy import deepcopy + from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, @@ -44,6 +51,13 @@ Tuple, ) +from dataclasses import ( + dataclass, + field, +) + +from copy import deepcopy + from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 303d4ec2f5..776fcfa82e 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -29,6 +29,7 @@ def get_spec(file_name: str) -> SpecObject: inserts = {} function_matcher = re.compile(FUNCTION_REGEX) inserts_matcher = re.compile(BEGIN_INSERT_REGEX) + is_ssz = False for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index f3d57feaa0..1a1465d257 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -58,7 +58,8 @@ The head block root associated with a `store` is defined as `get_head(store)`. A #### `Target` ```python -class Target(Container): +@dataclass +class Target(object): epoch: Epoch root: Bytes32 ``` @@ -66,13 +67,14 @@ class Target(Container): #### `Store` ```python -class Store(Container): - blocks: Dict[Bytes32, BeaconBlock] - states: Dict[Bytes32, BeaconState] - time: uint64 - latest_targets: Dict[ValidatorIndex, Target] - justified_root: Bytes32 - finalized_root: Bytes32 +@dataclass +class Store(object): + blocks: Dict[Bytes32, BeaconBlock] = field(default_factory=dict) + states: Dict[Bytes32, BeaconState] = field(default_factory=dict) + time: int = 0 + latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) + justified_root: Bytes32 = ZERO_HASH + finalized_root: Bytes32 = ZERO_HASH ``` ### Helpers @@ -139,7 +141,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root # Check block slot against Unix time - pre_state = store.states[block.parent_root].copy() + pre_state = deepcopy(store.states[block.parent_root]) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 3bd7dd0624..d4b481ef2d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,11 +1,9 @@ -import copy from typing import ( List, Iterable, TypeVar, Type, NewType, - Dict, Union, ) from typing_inspect import get_origin @@ -132,9 +130,6 @@ def get_field_values(self): cls = self.__class__ return [getattr(self, field) for field in cls.get_field_names()] - def copy(self): - return copy.deepcopy(self) - def __repr__(self): return repr({field: getattr(self, field) for field in self.get_field_names()}) @@ -418,8 +413,6 @@ def get_zero_value(typ): return b'' elif is_container_type(typ): return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) - elif is_dict_type(typ): - return dict() else: raise Exception("Type not supported: {}".format(typ)) @@ -511,13 +504,6 @@ def is_container_type(typ): return isinstance(typ, type) and issubclass(typ, Container) -def is_dict_type(typ): - """ - Check of the given type is a Dict. (Which are a part of the super-secret undocumented SSZ spec) - """ - return get_origin(typ) is Dict or get_origin(typ) is dict - - T = TypeVar('T') L = TypeVar('L') diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 3b38930bda..18e5d2a4cf 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -3,3 +3,4 @@ eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 typing_inspect==0.4.0 +dataclasses==0.6 From bb36660d569d98ef295fcc148ab9ef10f35c4403 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 17 Jun 2019 11:08:22 -0400 Subject: [PATCH 042/405] Kick the CI cache --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e9a77c49b..21118e88d3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,13 +35,13 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v2-pyspec + venv_name: v3-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v2-pyspec + venv_name: v3-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv restore_deposit_contract_cached_venv: From c31a42632fdb9d41096f2cc6d12496525114c3ce Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 17 Jun 2019 11:11:38 -0400 Subject: [PATCH 043/405] Kick the other CI cache --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21118e88d3..dbecd13b31 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,13 +48,13 @@ commands: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v4-deposit-contract + venv_name: v5-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v4-deposit-contract + venv_name: v5-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: From d911236fbb3fffca524318d8016161688b0d3f1e Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 17 Jun 2019 14:19:17 -0700 Subject: [PATCH 044/405] Update slots.md --- specs/test_formats/sanity/slots.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/test_formats/sanity/slots.md b/specs/test_formats/sanity/slots.md index 81866d47b9..eb0f336b4e 100644 --- a/specs/test_formats/sanity/slots.md +++ b/specs/test_formats/sanity/slots.md @@ -8,7 +8,7 @@ Sanity tests to cover a series of one or more empty-slot transitions being proce description: string -- description of test case, purely for debugging purposes bls_setting: int -- see general test-format spec. pre: BeaconState -- state before running through the transitions. -slots: N -- amount of slots to process, N being a positive numer. +slots: N -- amount of slots to process, N being a positive number. post: BeaconState -- state after applying all the transitions. ``` From 207eb808a0cad10d0ec20b5fd0ae23004fd59705 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 17 Jun 2019 15:19:44 -0600 Subject: [PATCH 045/405] split constants vs configuration --- configs/constant_presets/mainnet.yaml | 4 ---- configs/constant_presets/minimal.yaml | 4 ---- specs/core/0_beacon-chain.md | 21 ++++++++++++--------- specs/core/0_fork-choice.md | 4 ++-- specs/core/1_shard-data-chains.md | 9 +++++++-- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index c0b7c6e9ca..892111ce0f 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -25,8 +25,6 @@ SHUFFLE_ROUND_COUNT: 90 # --------------------------------------------------------------- # **TBD** DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 -# 2**5 (= 32) -DEPOSIT_CONTRACT_TREE_DEPTH: 32 # Gwei values @@ -46,8 +44,6 @@ EFFECTIVE_BALANCE_INCREMENT: 1000000000 GENESIS_FORK_VERSION: 0x00000000 # 0, GENESIS_EPOCH is derived from this constant GENESIS_SLOT: 0 -# 2**64 - 1 -FAR_FUTURE_EPOCH: 18446744073709551615 BLS_WITHDRAWAL_PREFIX: 0 diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index cdd919927b..5f4087d898 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -24,8 +24,6 @@ SHUFFLE_ROUND_COUNT: 10 # --------------------------------------------------------------- # **TBD** DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 -# 2**5 (= 32) -DEPOSIT_CONTRACT_TREE_DEPTH: 32 # Gwei values @@ -45,8 +43,6 @@ EFFECTIVE_BALANCE_INCREMENT: 1000000000 GENESIS_FORK_VERSION: 0x00000000 # 0, GENESIS_EPOCH is derived from this constant GENESIS_SLOT: 0 -# 2**64 - 1 -FAR_FUTURE_EPOCH: 18446744073709551615 BLS_WITHDRAWAL_PREFIX: 0 diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9680988950..6f6b6e74e4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -11,6 +11,7 @@ - [Notation](#notation) - [Terminology](#terminology) - [Constants](#constants) + - [Configuration](#configuration) - [Misc](#misc) - [Deposit contract](#deposit-contract) - [Gwei values](#gwei-values) @@ -152,7 +153,17 @@ Code snippets appearing in `this style` are to be interpreted as Python code. ## Constants -*Note*: The default mainnet values for the constants are included here for spec-design purposes. +The following values are (non-configurable) constants used throughout the specification. + +| Name | Value | +| - | - | +| `FAR_FUTURE_EPOCH` | `2**64 - 1` | +| `ZERO_HASH` | `b'\x00' * 32` | +| `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | + +## Configuration + +*Note*: The default mainnet configuration values are included here for spec-design purposes. The different configurations for mainnet, testnets, and YAML-based testing can be found in the `configs/constant_presets/` directory. These configurations are updated for releases, but may be out of sync during `dev` changes. @@ -170,12 +181,6 @@ These configurations are updated for releases, but may be out of sync during `de * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) -### Deposit contract - -| Name | Value | -| - | - | -| `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | - ### Gwei values | Name | Value | Unit | @@ -191,8 +196,6 @@ These configurations are updated for releases, but may be out of sync during `de | - | - | | `GENESIS_SLOT` | `0` | | `GENESIS_EPOCH` | `0` | -| `FAR_FUTURE_EPOCH` | `2**64 - 1` | -| `ZERO_HASH` | `b'\x00' * 32` | | `BLS_WITHDRAWAL_PREFIX` | `0` | ### Time parameters diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 8103bbeb2f..1910b6d04d 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -9,7 +9,7 @@ - [Table of contents](#table-of-contents) - [Introduction](#introduction) - [Prerequisites](#prerequisites) - - [Constants](#constants) + - [Configuration](#configuration) - [Time parameters](#time-parameters) - [Beacon chain processing](#beacon-chain-processing) - [Beacon chain fork choice rule](#beacon-chain-fork-choice-rule) @@ -26,7 +26,7 @@ This document represents the specification for the beacon chain fork choice rule All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./0_beacon-chain.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. -## Constants +## Configuration ### Time parameters diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index a895b3c389..1b2b2b2da1 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -9,8 +9,9 @@ - [Ethereum 2.0 Phase 1 -- Shard Data Chains](#ethereum-20-phase-1----shard-data-chains) - [Table of contents](#table-of-contents) - [Introduction](#introduction) - - [Constants](#constants) + - [Configuration](#configuration) - [Misc](#misc) + - [Initial values](#initial-values) - [Time parameters](#time-parameters) - [Signature domains](#signature-domains) - [Data structures](#data-structures) @@ -38,7 +39,7 @@ This document describes the shard data layer and the shard fork choice rule in Phase 1 of Ethereum 2.0. -## Constants +## Configuration ### Misc @@ -46,6 +47,10 @@ This document describes the shard data layer and the shard fork choice rule in P | - | - | | `BYTES_PER_SHARD_BLOCK_BODY` | `2**14` (= 16,384) | | `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) | + +### Initial values + +| Name | Value | | `PHASE_1_FORK_EPOCH` | **TBD** | | `PHASE_1_FORK_SLOT` | **TBD** | | `GENESIS_SHARD_SLOT` | 0 | From 18ebd2aa9093b16b708be7261f510f5d96e2d3bd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 17 Jun 2019 17:21:45 -0400 Subject: [PATCH 046/405] Bytes32 -> Hash --- scripts/build_spec.py | 20 ++++------ specs/core/0_beacon-chain.md | 71 ++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 47 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 9f5d07b294..01bcac76e3 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -75,21 +75,19 @@ def get_ssz_type_by_name(name: str) -> Container: # Monkey patch validator compute committee code _compute_committee = compute_committee -committee_cache: Dict[Tuple[Bytes32, Bytes32, int, int], List[ValidatorIndex]] = {} +committee_cache: Dict[Tuple[Hash, Hash, int, int], List[ValidatorIndex]] = {} def compute_committee(indices: List[ValidatorIndex], # type: ignore - seed: Bytes32, + seed: Hash, index: int, count: int) -> List[ValidatorIndex]: param_hash = (hash_tree_root(indices), seed, index, count) - if param_hash in committee_cache: - return committee_cache[param_hash] - else: + if param_hash not in committee_cache: ret = _compute_committee(indices, seed, index, count) committee_cache[param_hash] = ret - return ret + return committee_cache[param_hash] # Monkey patch hash cache @@ -97,13 +95,11 @@ def compute_committee(indices: List[ValidatorIndex], # type: ignore hash_cache: Dict[bytes, Bytes32] = {} -def hash(x: bytes) -> Bytes32: - if x in hash_cache: - return hash_cache[x] - else: +def hash(x: bytes) -> Hash: + if x not in hash_cache: ret = _hash(x) - hash_cache[x] = ret - return ret + hash_cache[x] = Hash(ret) + return hash_cache[x] # Access to overwrite spec constants based on configuration diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8b0c51b044..76dc6eb740 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -162,6 +162,7 @@ We define the following Python custom types for type hinting and readability: | `ValidatorIndex` | `uint64` | a validator registry index | | `Gwei` | `uint64` | an amount in Gwei | | `Version` | `Bytes4` | a fork version number | +| `Hash` | `Bytes32` | a hashed result | | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | @@ -297,7 +298,7 @@ class Validator(Container): # BLS public key pubkey: BLSPubkey # Withdrawal credentials - withdrawal_credentials: Bytes32 + withdrawal_credentials: Hash # Epoch when became eligible for activation activation_eligibility_epoch: Epoch # Epoch when validator activated @@ -322,9 +323,9 @@ class Crosslink(Container): start_epoch: Epoch end_epoch: Epoch # Root of the previous crosslink - parent_root: Bytes32 + parent_root: Hash # Root of the crosslinked shard data since the previous crosslink - data_root: Bytes32 + data_root: Hash ``` #### `AttestationData` @@ -332,13 +333,13 @@ class Crosslink(Container): ```python class AttestationData(Container): # LMD GHOST vote - beacon_block_root: Bytes32 + beacon_block_root: Hash # FFG vote source_epoch: Epoch - source_root: Bytes32 + source_root: Hash target_epoch: Epoch - target_root: Bytes32 + target_root: Hash # Crosslink vote crosslink: Crosslink @@ -386,11 +387,11 @@ class PendingAttestation(Container): ```python class Eth1Data(Container): # Root of the deposit tree - deposit_root: Bytes32 + deposit_root: Hash # Total number of deposits deposit_count: uint64 # Block hash - block_hash: Bytes32 + block_hash: Hash ``` #### `HistoricalBatch` @@ -398,9 +399,9 @@ class Eth1Data(Container): ```python class HistoricalBatch(Container): # Block roots - block_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] + block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] # State roots - state_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] ``` #### `DepositData` @@ -410,7 +411,7 @@ class DepositData(Container): # BLS pubkey pubkey: BLSPubkey # Withdrawal credentials - withdrawal_credentials: Bytes32 + withdrawal_credentials: Hash # Amount in Gwei amount: Gwei # Container self-signature @@ -422,9 +423,9 @@ class DepositData(Container): ```python class BeaconBlockHeader(Container): slot: Slot - parent_root: Bytes32 - state_root: Bytes32 - body_root: Bytes32 + parent_root: Hash + state_root: Hash + body_root: Hash signature: BLSSignature ``` @@ -471,7 +472,7 @@ class Attestation(Container): ```python class Deposit(Container): # Branch in the deposit tree - proof: Vector[Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] + proof: Vector[Hash, DEPOSIT_CONTRACT_TREE_DEPTH] # Data data: DepositData ``` @@ -531,8 +532,8 @@ class BeaconBlockBody(Container): class BeaconBlock(Container): # Header slot: Slot - parent_root: Bytes32 - state_root: Bytes32 + parent_root: Hash + state_root: Hash body: BeaconBlockBody signature: BLSSignature ``` @@ -551,27 +552,27 @@ class BeaconState(Container): validator_registry: List[Validator] balances: List[Gwei] # Randomness and committees - latest_randao_mixes: Vector[Bytes32, LATEST_RANDAO_MIXES_LENGTH] + latest_randao_mixes: Vector[Hash, LATEST_RANDAO_MIXES_LENGTH] latest_start_shard: Shard # Finality previous_epoch_attestations: List[PendingAttestation] current_epoch_attestations: List[PendingAttestation] previous_justified_epoch: Epoch current_justified_epoch: Epoch - previous_justified_root: Bytes32 - current_justified_root: Bytes32 + previous_justified_root: Hash + current_justified_root: Hash justification_bitfield: uint64 finalized_epoch: Epoch - finalized_root: Bytes32 + finalized_root: Hash # Recent state current_crosslinks: Vector[Crosslink, SHARD_COUNT] previous_crosslinks: Vector[Crosslink, SHARD_COUNT] - latest_block_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_state_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_active_index_roots: Vector[Bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] + latest_block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] + latest_state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] + latest_active_index_roots: Vector[Hash, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] latest_slashed_balances: Vector[Gwei, LATEST_SLASHED_EXIT_LENGTH] latest_block_header: BeaconBlockHeader - historical_roots: List[Bytes32] + historical_roots: List[Hash] # Ethereum 1.0 chain data latest_eth1_data: Eth1Data eth1_data_votes: List[Eth1Data] @@ -597,11 +598,11 @@ The `hash` function is SHA256. ### `hash_tree_root` -`def hash_tree_root(object: SSZSerializable) -> Bytes32` is a function for hashing objects into a single root utilizing a hash tree structure. `hash_tree_root` is defined in the [SimpleSerialize spec](../simple-serialize.md#merkleization). +`def hash_tree_root(object: SSZSerializable) -> Hash` is a function for hashing objects into a single root utilizing a hash tree structure. `hash_tree_root` is defined in the [SimpleSerialize spec](../simple-serialize.md#merkleization). ### `signing_root` -`def signing_root(object: Container) -> Bytes32` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. +`def signing_root(object: Container) -> Hash` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. ### `bls_domain` @@ -758,7 +759,7 @@ def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot ```python def get_block_root_at_slot(state: BeaconState, - slot: Slot) -> Bytes32: + slot: Slot) -> Hash: """ Return the block root at a recent ``slot``. """ @@ -770,7 +771,7 @@ def get_block_root_at_slot(state: BeaconState, ```python def get_block_root(state: BeaconState, - epoch: Epoch) -> Bytes32: + epoch: Epoch) -> Hash: """ Return the block root at a recent ``epoch``. """ @@ -781,7 +782,7 @@ def get_block_root(state: BeaconState, ```python def get_randao_mix(state: BeaconState, - epoch: Epoch) -> Bytes32: + epoch: Epoch) -> Hash: """ Return the randao mix at a recent ``epoch``. ``epoch`` expected to be between (current_epoch - LATEST_RANDAO_MIXES_LENGTH, current_epoch]. @@ -793,7 +794,7 @@ def get_randao_mix(state: BeaconState, ```python def get_active_index_root(state: BeaconState, - epoch: Epoch) -> Bytes32: + epoch: Epoch) -> Hash: """ Return the index root at a recent ``epoch``. ``epoch`` expected to be between @@ -806,7 +807,7 @@ def get_active_index_root(state: BeaconState, ```python def generate_seed(state: BeaconState, - epoch: Epoch) -> Bytes32: + epoch: Epoch) -> Hash: """ Generate a seed for the given ``epoch``. """ @@ -844,7 +845,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: ### `verify_merkle_branch` ```python -def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: int, root: Bytes32) -> bool: +def verify_merkle_branch(leaf: Hash, proof: List[Hash], depth: int, index: int, root: Hash) -> bool: """ Verify that the given ``leaf`` is on the merkle branch ``proof`` starting with the given ``root``. @@ -861,7 +862,7 @@ def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: ### `get_shuffled_index` ```python -def get_shuffled_index(index: int, index_count: int, seed: Bytes32) -> ValidatorIndex: +def get_shuffled_index(index: int, index_count: int, seed: Hash) -> ValidatorIndex: """ Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). """ @@ -885,7 +886,7 @@ def get_shuffled_index(index: int, index_count: int, seed: Bytes32) -> Validator ### `compute_committee` ```python -def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]: +def compute_committee(indices: List[ValidatorIndex], seed: Hash, index: int, count: int) -> List[ValidatorIndex]: start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count return [indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] From 89d9d80b1c54fef13bf3135aa7fbcb534ce5e700 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 17 Jun 2019 16:12:47 -0600 Subject: [PATCH 047/405] move BASE_REWARDS_PER_EPOCH to constants --- configs/constant_presets/mainnet.yaml | 2 -- configs/constant_presets/minimal.yaml | 2 -- specs/core/0_beacon-chain.md | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 892111ce0f..ed839fc7d6 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -15,8 +15,6 @@ MAX_INDICES_PER_ATTESTATION: 4096 MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**16 (= 65,536) CHURN_LIMIT_QUOTIENT: 65536 -# Normalizes base rewards -BASE_REWARDS_PER_EPOCH: 5 # See issue 563 SHUFFLE_ROUND_COUNT: 90 diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 5f4087d898..d631300e27 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -14,8 +14,6 @@ MAX_INDICES_PER_ATTESTATION: 4096 MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**16 (= 65,536) CHURN_LIMIT_QUOTIENT: 65536 -# Normalizes base rewards -BASE_REWARDS_PER_EPOCH: 5 # [customized] Faster, but unsecure. SHUFFLE_ROUND_COUNT: 10 diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6f6b6e74e4..4a8e316eae 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -159,6 +159,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `FAR_FUTURE_EPOCH` | `2**64 - 1` | | `ZERO_HASH` | `b'\x00' * 32` | +| `BASE_REWARDS_PER_EPOCH` | `5` | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | ## Configuration @@ -176,7 +177,6 @@ These configurations are updated for releases, but may be out of sync during `de | `MAX_INDICES_PER_ATTESTATION` | `2**12` (= 4,096) | | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | -| `BASE_REWARDS_PER_EPOCH` | `5` | | `SHUFFLE_ROUND_COUNT` | `90` | * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) From d5d35b1a3ef521d14124c7f2e924fd932a887c16 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 15:09:55 +0200 Subject: [PATCH 048/405] Trail upgrade CI to python 3.7 --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dbecd13b31..70c51b9781 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,7 +60,7 @@ commands: jobs: checkout_specs: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: # Restore git repo at point close to target branch/revision, to speed up checkout @@ -80,7 +80,7 @@ jobs: - ~/specs-repo install_pyspec_test: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: - restore_cache: @@ -92,7 +92,7 @@ jobs: - save_pyspec_cached_venv test: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: - restore_cache: @@ -105,7 +105,7 @@ jobs: path: test_libs/pyspec/test-reports lint: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: - restore_cache: @@ -116,7 +116,7 @@ jobs: command: make lint install_deposit_contract_test: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: - restore_cache: @@ -128,7 +128,7 @@ jobs: - save_deposit_contract_cached_venv deposit_contract: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: - restore_cache: From 28a3e54db094942ce4b37f78239df7455f3147af Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 15:13:04 +0200 Subject: [PATCH 049/405] Actually resolve some merge conflicts --- specs/core/0_fork-choice.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 05735e0814..e4ea773c84 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -8,12 +8,7 @@ - [Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice](#ethereum-20-phase-0----beacon-chain-fork-choice) - [Table of contents](#table-of-contents) - [Introduction](#introduction) -<<<<<<< HEAD - [Constants](#constants) -======= - - [Prerequisites](#prerequisites) - - [Configuration](#configuration) ->>>>>>> dev - [Time parameters](#time-parameters) - [Fork choice](#fork-choice) - [Containers](#containers) @@ -82,13 +77,7 @@ class Store(object): finalized_root: Bytes32 = ZERO_HASH ``` -<<<<<<< HEAD ### Helpers -======= -* The parent block with root `block.parent_root` has been processed and accepted. -* An Ethereum 1.0 block pointed to by the `state.eth1_data.block_hash` has been processed and accepted. -* The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`. ->>>>>>> dev #### `get_genesis_store` From eb9fc7b518944db129ac673180cad039206983f4 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 15:26:12 +0200 Subject: [PATCH 050/405] Remove circle ci repo caching --- .circleci/config.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 70c51b9781..2b5e7a2925 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,15 +60,9 @@ commands: jobs: checkout_specs: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - # Restore git repo at point close to target branch/revision, to speed up checkout - - restore_cache: - keys: - - v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - v1-specs-repo-{{ .Branch }}- - - v1-specs-repo- - checkout - run: name: Clean up git repo to reduce cache size @@ -80,7 +74,7 @@ jobs: - ~/specs-repo install_pyspec_test: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: @@ -92,7 +86,7 @@ jobs: - save_pyspec_cached_venv test: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: @@ -105,7 +99,7 @@ jobs: path: test_libs/pyspec/test-reports lint: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: @@ -116,7 +110,7 @@ jobs: command: make lint install_deposit_contract_test: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: @@ -128,7 +122,7 @@ jobs: - save_deposit_contract_cached_venv deposit_contract: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: From dfdd283908b2c74280f0ef3ef04c959fa00d8cc6 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 15:30:01 +0200 Subject: [PATCH 051/405] Python 3.7 -> 3.6 again --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b5e7a2925..dbecd13b31 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,6 +63,12 @@ jobs: - image: circleci/python:3.6 working_directory: ~/specs-repo steps: + # Restore git repo at point close to target branch/revision, to speed up checkout + - restore_cache: + keys: + - v1-specs-repo-{{ .Branch }}-{{ .Revision }} + - v1-specs-repo-{{ .Branch }}- + - v1-specs-repo- - checkout - run: name: Clean up git repo to reduce cache size From 686273e1d6a37f434dc77259d119a2142d8eaf53 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 15:34:07 +0200 Subject: [PATCH 052/405] Maybe venv cache name collision --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dbecd13b31..9ba9934a6d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,26 +35,26 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v3-pyspec + venv_name: v3-pyspec-bump2 reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v3-pyspec + venv_name: v3-pyspec-bump2 reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv restore_deposit_contract_cached_venv: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v5-deposit-contract + venv_name: v5-deposit-contract-bump2 reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v5-deposit-contract + venv_name: v5-deposit-contract-bump2 reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: From 3bcddf5e49521192d415e89727fe38dea4a975ac Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 20:59:17 +0200 Subject: [PATCH 053/405] Removes defaults from Store to try get CI to pass --- specs/core/0_fork-choice.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index e4ea773c84..f57fd84b8e 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -73,8 +73,8 @@ class Store(object): states: Dict[Bytes32, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) - justified_root: Bytes32 = ZERO_HASH - finalized_root: Bytes32 = ZERO_HASH + justified_root: Bytes32 + finalized_root: Bytes32 ``` ### Helpers From d804cb307bc4a684abe875bb1d0a402769191f3d Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 21:50:40 +0200 Subject: [PATCH 054/405] Removes defaults from Store to try get CI to pass --- specs/core/0_fork-choice.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index f57fd84b8e..11d6453e97 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -73,8 +73,8 @@ class Store(object): states: Dict[Bytes32, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) - justified_root: Bytes32 - finalized_root: Bytes32 + justified_root: Bytes32 = 0x0 + finalized_root: Bytes32 = 0x0 ``` ### Helpers From f4de5e3c72b1a178f55379a8fb6468d5f1a9eb52 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 21:50:53 +0200 Subject: [PATCH 055/405] fix review comment: one line cache set --- scripts/build_spec.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index d672db419f..4890beaac9 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -85,8 +85,7 @@ def compute_committee(indices: List[ValidatorIndex], # type: ignore param_hash = (hash_tree_root(indices), seed, index, count) if param_hash not in committee_cache: - ret = _compute_committee(indices, seed, index, count) - committee_cache[param_hash] = ret + committee_cache[param_hash] = _compute_committee(indices, seed, index, count) return committee_cache[param_hash] From 207f632e1a4c609a3edd2724fc98f2fa341e94a6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 21:53:39 +0200 Subject: [PATCH 056/405] resolve other ret comment --- scripts/build_spec.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 4890beaac9..b7605df60b 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -91,13 +91,12 @@ def compute_committee(indices: List[ValidatorIndex], # type: ignore # Monkey patch hash cache _hash = hash -hash_cache: Dict[bytes, Bytes32] = {} +hash_cache: Dict[bytes, Hash] = {} def hash(x: bytes) -> Hash: if x not in hash_cache: - ret = _hash(x) - hash_cache[x] = Hash(ret) + hash_cache[x] = Hash(_hash(x)) return hash_cache[x] From f0e909c8d853b3db38dbe8bdef25c1f80a9e5b4c Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 21:54:00 +0200 Subject: [PATCH 057/405] add mypy cache to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 16d39a4342..4dff5fbcb3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,9 @@ build/ output/ eth2.0-spec-tests/ + .pytest_cache +.mypy_cache # Dynamically built from Markdown spec test_libs/pyspec/eth2spec/phase0/spec.py From 7a71919c534fa291209bd4e476b69970c0a3f27e Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 21:55:41 +0200 Subject: [PATCH 058/405] Test removing all the Bytes objects --- specs/core/0_fork-choice.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 11d6453e97..a5d202ffac 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -61,7 +61,6 @@ The head block root associated with a `store` is defined as `get_head(store)`. A @dataclass class Target(object): epoch: Epoch - root: Bytes32 ``` #### `Store` @@ -69,12 +68,8 @@ class Target(object): ```python @dataclass class Store(object): - blocks: Dict[Bytes32, BeaconBlock] = field(default_factory=dict) - states: Dict[Bytes32, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) - justified_root: Bytes32 = 0x0 - finalized_root: Bytes32 = 0x0 ``` ### Helpers From 346e61dfeb1d4924ec0921a91e99dc4450764041 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 21:57:37 +0200 Subject: [PATCH 059/405] make epoch pattern similar to exit-epoch loop --- specs/core/0_beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6534ce4f2d..8e81566340 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -704,11 +704,11 @@ def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: ```python def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: assert epoch <= get_current_epoch(state) + 1 - check_epoch = get_current_epoch(state) + 1 + check_epoch = Epoch(get_current_epoch(state) + 1) shard = Shard((state.start_shard + get_shard_delta(state, get_current_epoch(state))) % SHARD_COUNT) while check_epoch > epoch: - check_epoch -= 1 - shard = Shard((shard + SHARD_COUNT - get_shard_delta(state, Epoch(check_epoch))) % SHARD_COUNT) + check_epoch -= Epoch(1) + shard = Shard((shard + SHARD_COUNT - get_shard_delta(state, check_epoch)) % SHARD_COUNT) return shard ``` From 060041945cc25c95a8ec477800abc11dda21af11 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 22:00:22 +0200 Subject: [PATCH 060/405] remove unnecessary cast --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8e81566340..3016b52d88 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -843,7 +843,7 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Hash) -> V position = max(index, flip) source = hash( seed + int_to_bytes(current_round, length=1) + - int_to_bytes(ValidatorIndex(position // 256), length=4) + int_to_bytes(position // 256, length=4) ) byte = source[(position % 256) // 8] bit = (byte >> (position % 8)) % 2 From 7d2f0a9dc0df1c69fc8178160325043b06cbb92c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 18 Jun 2019 14:07:05 -0600 Subject: [PATCH 061/405] clean up --- scripts/build_spec.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index b7605df60b..e209e2c56a 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -132,17 +132,11 @@ def objects_to_spec(functions: Dict[str, str], f" def __init__(self, _x: {value}) -> None:\n" f" ...\n" if value.startswith("uint") - # else "" - # else f"{key} = {value}\n" else f"class {key}({value}):\n pass\n" for key, value in custom_types.items() ] ) ) - # new_type_definitions += '\n'.join(['Bytes%s = BytesN[%s]' % (n, n) for n in byte_types]) - # new_type_definitions += '\n' + '\n'.join(['Hash = Bytes32', 'BLSPubkey = Bytes48', 'BLSSignature = Bytes96']) - # new_type_definitions += \ - # '\n' + '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in new_types.items()]) functions_spec = '\n\n'.join(functions.values()) constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants)) ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) From 6df75ec3d479faf0eba764033e2d66822dba89f8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 22:16:21 +0200 Subject: [PATCH 062/405] cleanup unused byte part of builder --- scripts/build_spec.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index e209e2c56a..3a1b22efd3 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -120,7 +120,6 @@ def objects_to_spec(functions: Dict[str, str], ssz_objects: Dict[str, str], inserts: Dict[str, str], imports: Dict[str, str], - byte_types: List[int], ) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -226,7 +225,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]: functions, custom_types, constants, ssz_objects, inserts = get_spec(sourcefile) - spec = objects_to_spec(functions, custom_types, constants, ssz_objects, inserts, PHASE0_IMPORTS, BYTE_TYPES) + spec = objects_to_spec(functions, custom_types, constants, ssz_objects, inserts, PHASE0_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) @@ -243,7 +242,7 @@ def build_phase1_spec(phase0_sourcefile: str, spec_objects = phase0_spec for value in [phase1_custody, phase1_shard_data]: spec_objects = combine_spec_objects(spec_objects, value) - spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS, BYTE_TYPES) + spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) From dc01218182e91c471bf011391b6ca3df77ef1835 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 18 Jun 2019 14:23:31 -0600 Subject: [PATCH 063/405] Add `dataclasses==0.6` to setup.py --- test_libs/pyspec/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 3856640abf..fe14c498cb 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -10,6 +10,7 @@ "pycryptodome==3.7.3", "py_ecc>=1.6.0", "typing_inspect==0.4.0", - "ssz==0.1.0a10" + "ssz==0.1.0a10", + "dataclasses==0.6", ] ) From 04c7c438745580660d538ab68747deb7955821b7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 18 Jun 2019 14:36:49 -0600 Subject: [PATCH 064/405] kick deposit contract cache --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ba9934a6d..a43430f772 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,13 +48,13 @@ commands: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v5-deposit-contract-bump2 + venv_name: v6-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v5-deposit-contract-bump2 + venv_name: v6-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: From 61a51417ed4f58091e0555c82ff18d098df2dd93 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 23:00:30 +0200 Subject: [PATCH 065/405] Unbreak things again --- specs/core/0_fork-choice.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index a5d202ffac..e4ea773c84 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -61,6 +61,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A @dataclass class Target(object): epoch: Epoch + root: Bytes32 ``` #### `Store` @@ -68,8 +69,12 @@ class Target(object): ```python @dataclass class Store(object): + blocks: Dict[Bytes32, BeaconBlock] = field(default_factory=dict) + states: Dict[Bytes32, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) + justified_root: Bytes32 = ZERO_HASH + finalized_root: Bytes32 = ZERO_HASH ``` ### Helpers From 29dbe1b880b904a4dfb42bc0433608606560e69d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 19 Jun 2019 13:59:44 -0600 Subject: [PATCH 066/405] Increase historical length and cleanups (#1196) * increase historical length and a few cleanups --- configs/constant_presets/mainnet.yaml | 10 ++-- configs/constant_presets/minimal.yaml | 8 +-- specs/core/0_beacon-chain.md | 59 ++++++++----------- .../pyspec/eth2spec/test/helpers/genesis.py | 2 +- 4 files changed, 34 insertions(+), 45 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index ed839fc7d6..09ec7bdb79 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -71,14 +71,12 @@ MAX_EPOCHS_PER_CROSSLINK: 64 MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 -# State list lengths +# State vector lengths # --------------------------------------------------------------- +# 2**16 (= 65,536) epochs ~0.8 years +EPOCHS_PER_HISTORICAL_VECTOR: 65536 # 2**13 (= 8,192) epochs ~36 days -RANDAO_MIXES_LENGTH: 8192 -# 2**13 (= 8,192) epochs ~36 days -ACTIVE_INDEX_ROOTS_LENGTH: 8192 -# 2**13 (= 8,192) epochs ~36 days -SLASHED_EXIT_LENGTH: 8192 +EPOCHS_PER_SLASHED_BALANCES_VECTOR: 8192 # Reward and penalty quotients diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index d631300e27..c8c2853c70 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -72,14 +72,12 @@ MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 -# State list lengths +# State vector lengths # --------------------------------------------------------------- # [customized] smaller state -RANDAO_MIXES_LENGTH: 64 +EPOCHS_PER_HISTORICAL_VECTOR: 64 # [customized] smaller state -ACTIVE_INDEX_ROOTS_LENGTH: 64 -# [customized] smaller state -SLASHED_EXIT_LENGTH: 64 +EPOCHS_PER_SLASHED_BALANCES_VECTOR: 64 # Reward and penalty quotients diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3016b52d88..1dcdbdef94 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -235,9 +235,8 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | +| `EPOCHS_PER_SLASHED_BALANCES_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | ### Rewards and penalties @@ -524,10 +523,10 @@ class BeaconState(Container): balances: List[Gwei] # Shuffling start_shard: Shard - randao_mixes: Vector[Hash, RANDAO_MIXES_LENGTH] - active_index_roots: Vector[Hash, ACTIVE_INDEX_ROOTS_LENGTH] # Digests of the active registry, for light clients + randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] + active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Digests of the active registry, for light clients # Slashings - slashed_balances: Vector[Gwei, SLASHED_EXIT_LENGTH] # Sums of the effective balances of slashed validators + slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators # Attestations previous_epoch_attestations: List[PendingAttestation] current_epoch_attestations: List[PendingAttestation] @@ -751,9 +750,9 @@ def get_randao_mix(state: BeaconState, epoch: Epoch) -> Hash: """ Return the randao mix at a recent ``epoch``. - ``epoch`` expected to be between (current_epoch - RANDAO_MIXES_LENGTH, current_epoch]. + ``epoch`` expected to be between (current_epoch - EPOCHS_PER_HISTORICAL_VECTOR, current_epoch]. """ - return state.randao_mixes[epoch % RANDAO_MIXES_LENGTH] + return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` ### `get_active_index_root` @@ -764,9 +763,9 @@ def get_active_index_root(state: BeaconState, """ Return the index root at a recent ``epoch``. ``epoch`` expected to be between - (current_epoch - ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY, current_epoch + ACTIVATION_EXIT_DELAY]. + (current_epoch - EPOCHS_PER_HISTORICAL_VECTOR + ACTIVATION_EXIT_DELAY, current_epoch + ACTIVATION_EXIT_DELAY]. """ - return state.active_index_roots[epoch % ACTIVE_INDEX_ROOTS_LENGTH] + return state.active_index_roots[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` ### `generate_seed` @@ -778,7 +777,7 @@ def generate_seed(state: BeaconState, Generate a seed for the given ``epoch``. """ return hash( - get_randao_mix(state, Epoch(epoch + RANDAO_MIXES_LENGTH - MIN_SEED_LOOKAHEAD)) + + get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + get_active_index_root(state, epoch) + int_to_bytes(epoch, length=32) ) @@ -1112,9 +1111,9 @@ def slash_validator(state: BeaconState, current_epoch = get_current_epoch(state) initiate_validator_exit(state, slashed_index) state.validators[slashed_index].slashed = True - state.validators[slashed_index].withdrawable_epoch = Epoch(current_epoch + SLASHED_EXIT_LENGTH) + state.validators[slashed_index].withdrawable_epoch = Epoch(current_epoch + EPOCHS_PER_SLASHED_BALANCES_VECTOR) slashed_balance = state.validators[slashed_index].effective_balance - state.slashed_balances[current_epoch % SLASHED_EXIT_LENGTH] += slashed_balance + state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] += slashed_balance proposer_index = get_beacon_proposer_index(state) if whistleblower_index is None: @@ -1188,7 +1187,7 @@ def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis # Populate active_index_roots genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, GENESIS_EPOCH)) - for index in range(ACTIVE_INDEX_ROOTS_LENGTH): + for index in range(EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root return state @@ -1511,12 +1510,12 @@ def process_slashings(state: BeaconState) -> None: total_balance = get_total_active_balance(state) # Compute slashed balances in the current epoch - total_at_start = state.slashed_balances[(current_epoch + 1) % SLASHED_EXIT_LENGTH] - total_at_end = state.slashed_balances[current_epoch % SLASHED_EXIT_LENGTH] + total_at_start = state.slashed_balances[(current_epoch + 1) % EPOCHS_PER_SLASHED_BALANCES_VECTOR] + total_at_end = state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] total_penalties = total_at_end - total_at_start for index, validator in enumerate(state.validators): - if validator.slashed and current_epoch == validator.withdrawable_epoch - SLASHED_EXIT_LENGTH // 2: + if validator.slashed and current_epoch == validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2: penalty = max( validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT @@ -1542,16 +1541,16 @@ def process_final_updates(state: BeaconState) -> None: # Update start shard state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT) # Set active index root - index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % ACTIVE_INDEX_ROOTS_LENGTH + index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR state.active_index_roots[index_root_position] = hash_tree_root( get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) ) # Set total slashed balances - state.slashed_balances[next_epoch % SLASHED_EXIT_LENGTH] = ( - state.slashed_balances[current_epoch % SLASHED_EXIT_LENGTH] + state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( + state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] ) # Set randao mix - state.randao_mixes[next_epoch % RANDAO_MIXES_LENGTH] = get_randao_mix(state, current_epoch) + state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) # Set historical root accumulator if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: historical_batch = HistoricalBatch( @@ -1599,19 +1598,13 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: ```python def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: + epoch = get_current_epoch(state) + # Verify RANDAO reveal proposer = state.validators[get_beacon_proposer_index(state)] - # Verify that the provided randao value is valid - assert bls_verify( - proposer.pubkey, - hash_tree_root(get_current_epoch(state)), - body.randao_reveal, - get_domain(state, DOMAIN_RANDAO), - ) - # Mix it in - state.randao_mixes[get_current_epoch(state) % RANDAO_MIXES_LENGTH] = ( - xor(get_randao_mix(state, get_current_epoch(state)), - hash(body.randao_reveal)) - ) + assert bls_verify(proposer.pubkey, hash_tree_root(epoch), body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) + # Mix in RANDAO reveal + mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) + state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = mix ``` #### Eth1 data diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 87b0446afd..d1d8189080 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -41,7 +41,7 @@ def create_genesis_state(spec, num_validators): validator.activation_epoch = spec.GENESIS_EPOCH genesis_active_index_root = hash_tree_root(spec.get_active_validator_indices(state, spec.GENESIS_EPOCH)) - for index in range(spec.ACTIVE_INDEX_ROOTS_LENGTH): + for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root return state From 241fe34da77d35ee48f9848ce41c684e51055206 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 19 Jun 2019 22:37:51 +0200 Subject: [PATCH 067/405] Workaround for python3.6 --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ed303833f1..64d50b5794 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -349,6 +349,8 @@ def __instancecheck__(self, other): return _is_bytes_n_instance_of(self, other.__class__) def __eq__(self, other): + if other == (): + return False return _is_equal_bytes_n_type(self, other) def __ne__(self, other): From d73aa31ef6d893598b942e6c67373da278bafc5c Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 20 Jun 2019 11:58:05 +0100 Subject: [PATCH 068/405] Cleanups --- specs/core/0_fork-choice.md | 47 +++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index fead7d55b7..6b7e4bdbea 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -11,13 +11,12 @@ - [Constants](#constants) - [Time parameters](#time-parameters) - [Fork choice](#fork-choice) - - [Containers](#containers) + - [Helpers](#helpers) - [`Target`](#target) - [`Store`](#store) - - [Helpers](#helpers) - [`get_genesis_store`](#get_genesis_store) - [`get_ancestor`](#get_ancestor) - - [`get_attesting_balance_from_store`](#get_attesting_balance_from_store) + - [`get_latest_attesting_balance`](#get_latest_attesting_balance) - [`get_head`](#get_head) - [Handlers](#handlers) - [`on_tick`](#on_tick) @@ -51,9 +50,9 @@ The head block root associated with a `store` is defined as `get_head(store)`. A 1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. 2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. 3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. -4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. +4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. -### Containers +### Helpers #### `Target` @@ -61,7 +60,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A @dataclass class Target(object): epoch: Epoch - root: Bytes32 + root: Hash ``` #### `Store` @@ -69,38 +68,36 @@ class Target(object): ```python @dataclass class Store(object): - blocks: Dict[Bytes32, BeaconBlock] = field(default_factory=dict) - states: Dict[Bytes32, BeaconState] = field(default_factory=dict) + blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) + states: Dict[Hash, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) - justified_root: Bytes32 = ZERO_HASH - finalized_root: Bytes32 = ZERO_HASH + justified_root: Hash = ZERO_HASH + finalized_root: Hash = ZERO_HASH ``` -### Helpers - #### `get_genesis_store` ```python def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) - return Store(blocks={root: genesis_block}, states={root: genesis_state}, finalized_root=root, justified_root=root) + return Store(blocks={root: genesis_block}, states={root: genesis_state}, justified_root=root, finalized_root=root) ``` #### `get_ancestor` ```python -def get_ancestor(store: Store, root: Bytes32, slot: Slot) -> Bytes32: +def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: block = store.blocks[root] assert block.slot >= slot return root if block.slot == slot else get_ancestor(store, block.parent_root, slot) ``` -#### `get_attesting_balance_from_store` +#### `get_latest_attesting_balance` ```python -def get_attesting_balance_from_store(store: Store, root: Bytes32) -> Gwei: +def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: state = store.states[store.justified_root] active_indices = get_active_validator_indices(state.validator_registry, slot_to_epoch(state.slot)) return Gwei(sum( @@ -112,15 +109,15 @@ def get_attesting_balance_from_store(store: Store, root: Bytes32) -> Gwei: #### `get_head` ```python -def get_head(store: Store) -> Bytes32: +def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice head = store.justified_root while True: children = [root for root in store.blocks.keys() if store.blocks[root].parent_root == head] if len(children) == 0: return head - # Sort by attesting balance with ties broken lexicographically - head = max(children, key=lambda root: (get_attesting_balance_from_store(store, root), root)) + # Sort by latest attesting balance with ties broken lexicographically + head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) ``` ### Handlers @@ -147,13 +144,14 @@ def on_block(store: Store, block: BeaconBlock) -> None: state = state_transition(pre_state, block) # Add new state to the store store.states[signing_root(block)] = state - # Update justified and finalized blocks - if state.finalized_epoch > slot_to_epoch(store.blocks[store.finalized_root].slot): - store.finalized_root = state.finalized_root + # Update justified block root if state.current_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): store.justified_root = state.current_justified_root elif state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): store.justified_root = state.previous_justified_root + # Update finalized block root + if state.finalized_epoch > slot_to_epoch(store.blocks[store.finalized_root].slot): + store.finalized_root = state.finalized_root ``` #### `on_attestation` @@ -165,8 +163,5 @@ def on_attestation(store: Store, attestation: Attestation) -> None: validate_indexed_attestation(state, indexed_attestation) for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = Target( - epoch=attestation.data.target_epoch, - root=attestation.data.target_root, - ) + store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) ``` From 5ddfe34f0c3363589b402e88abde2c0aa1b05225 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:51:38 +0200 Subject: [PATCH 069/405] Simplified SSZ impl --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 13 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 393 ++++-------------- 2 files changed, 95 insertions(+), 311 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index b3c877d484..c88cfed1f3 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,9 +1,8 @@ -from ..merkle_minimal import merkleize_chunks, hash -from eth2spec.utils.ssz.ssz_typing import ( +from ..merkle_minimal import merkleize_chunks, hash, ZERO_BYTES32 +from .ssz_typing import ( is_uint_type, is_bool_type, is_container_type, is_list_kind, is_vector_kind, - read_vector_elem_type, read_elem_type, - uint_byte_size, + read_elem_type, infer_input_type, get_zero_value, ) @@ -20,7 +19,7 @@ def is_basic_type(typ): def serialize_basic(value, typ): if is_uint_type(typ): - return value.to_bytes(uint_byte_size(typ), 'little') + return value.to_bytes(typ.byte_len, 'little') elif is_bool_type(typ): if value: return b'\x01' @@ -140,6 +139,8 @@ def get_typed_values(obj, typ=None): else: raise Exception("Invalid type") +def item_length(typ): + return 1 if typ == bool else typ.byte_len if is_uint_type(typ) else 32 @infer_input_type def hash_tree_root(obj, typ=None): @@ -150,6 +151,8 @@ def hash_tree_root(obj, typ=None): fields = get_typed_values(obj, typ=typ) leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] if is_list_kind(typ): + full_chunk_length = (item_length(read_elem_type(typ)) * typ.length + 31) // 32 + leaves += [ZERO_BYTES32] * (full_chunk_length - len(obj)) return mix_in_length(merkleize_chunks(leaves), len(obj)) else: return merkleize_chunks(leaves) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index f870336e84..dbc3f95233 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -6,74 +6,38 @@ # SSZ integers # ----------------------------- - class uint(int): byte_len = 0 def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") + if value.byte_len and value.bit_length() > value.byte_len: + raise ValueError("value out of bounds for uint{}".format(value.byte_len)) return super().__new__(cls, value) class uint8(uint): byte_len = 1 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 8: - raise ValueError("value out of bounds for uint8") - return super().__new__(cls, value) - - # Alias for uint8 byte = NewType('byte', uint8) - class uint16(uint): byte_len = 2 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 16: - raise ValueError("value out of bounds for uint16") - return super().__new__(cls, value) - - class uint32(uint): byte_len = 4 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 32: - raise ValueError("value out of bounds for uint16") - return super().__new__(cls, value) - - class uint64(uint): byte_len = 8 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 64: - raise ValueError("value out of bounds for uint64") - return super().__new__(cls, value) - - class uint128(uint): byte_len = 16 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 128: - raise ValueError("value out of bounds for uint128") - return super().__new__(cls, value) - - class uint256(uint): byte_len = 32 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 256: - raise ValueError("value out of bounds for uint256") - return super().__new__(cls, value) - - def is_uint_type(typ): # All integers are uint in the scope of the spec here. # Since we default to uint64. Bounds can be checked elsewhere. @@ -84,21 +48,6 @@ def is_uint_type(typ): return isinstance(typ, type) and issubclass(typ, int) and not issubclass(typ, bool) - -def uint_byte_size(typ): - if hasattr(typ, '__supertype__'): - typ = typ.__supertype__ - - if isinstance(typ, type): - if issubclass(typ, uint): - return typ.byte_len - elif issubclass(typ, int): - # Default to uint64 - return 8 - else: - raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) - - # SSZ Container base class # ----------------------------- @@ -166,45 +115,19 @@ def get_field_types(cls): return list(cls.__annotations__.values()) -# SSZ vector -# ----------------------------- - - -def _is_vector_instance_of(a, b): - # Other must not be a BytesN - if issubclass(b, bytes): - return False - elif not hasattr(b, 'elem_type') or not hasattr(b, 'length'): - # Vector (b) is not an instance of Vector[X, Y] (a) - return False - elif not hasattr(a, 'elem_type') or not hasattr(a, 'length'): - # Vector[X, Y] (b) is an instance of Vector (a) - return True - else: - # Vector[X, Y] (a) is an instance of Vector[X, Y] (b) - return a.elem_type == b.elem_type and a.length == b.length - - -def _is_equal_vector_type(a, b): - # Other must not be a BytesN - if issubclass(b, bytes): - return False - elif not hasattr(a, 'elem_type') or not hasattr(a, 'length'): - if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): - # Vector == Vector - return True - else: - # Vector != Vector[X, Y] - return False - elif not hasattr(b, 'elem_type') or not hasattr(b, 'length'): - # Vector[X, Y] != Vector - return False +def get_zero_value(typ): + if typ == int: + return 0 else: - # Vector[X, Y] == Vector[X, Y] - return a.elem_type == b.elem_type and a.length == b.length + return typ.default() +def type_check(typ, value): + if typ == int or typ == uint64: + return isinstance(value, int) + else: + return typ.value_check(value) -class VectorMeta(type): +class AbstractListMeta(type): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) if 'elem_type' in attrs and 'length' in attrs: @@ -214,239 +137,115 @@ def __new__(cls, class_name, parents, attrs): def __getitem__(self, params): if not isinstance(params, tuple) or len(params) != 2: - raise Exception("Vector must be instantiated with two args: elem type and length") - o = self.__class__(self.__name__, (Vector,), {'elem_type': params[0], 'length': params[1]}) - o._name = 'Vector' + raise Exception("List must be instantiated with two args: elem type and length") + o = self.__class__(self.__name__, (self,), {'elem_type': params[0], 'length': params[1]}) + o._name = 'AbstractList' return o - def __subclasscheck__(self, sub): - return _is_vector_instance_of(self, sub) - - def __instancecheck__(self, other): - return _is_vector_instance_of(self, other.__class__) - - def __eq__(self, other): - return _is_equal_vector_type(self, other) - - def __ne__(self, other): - return not _is_equal_vector_type(self, other) - - def __hash__(self): - return hash(self.__class__) - + def __instancecheck__(self, obj): + if obj.__class__.__name__ != self.__name__: + return False + if hasattr(self, 'elem_type') and obj.__class__.elem_type != self.elem_type: + return False + if hasattr(self, 'length') and obj.__class__.length != self.length: + return False + return True -class Vector(metaclass=VectorMeta): +class ValueCheckError(Exception): + pass + +class AbstractList(metaclass=AbstractListMeta): + def __init__(self, *args): + items = self.extract_args(args) + + if not self.value_check(items): + raise ValueCheckError("Bad input for class {}: {}".format(self.__class__, items)) + self.items = items + + def value_check(self, value): + for v in value: + if not type_check(self.__class__.elem_type, v): + return False + return True - def __init__(self, *args: Iterable): - cls = self.__class__ - if not hasattr(cls, 'elem_type'): - raise TypeError("Type Vector without elem_type data cannot be instantiated") - elif not hasattr(cls, 'length'): - raise TypeError("Type Vector without length data cannot be instantiated") - - if len(args) != cls.length: - if len(args) == 0: - args = [get_zero_value(cls.elem_type) for _ in range(cls.length)] - else: - raise TypeError("Typed vector with length %d cannot hold %d items" % (cls.length, len(args))) + def extract_args(self, args): + return list(args) if len(args) > 0 else self.default() - self.items = list(args) + def default(self): + raise Exception("Not implemented") - # cannot check non-type objects, or parametrized types - if isinstance(cls.elem_type, type) and not hasattr(cls.elem_type, '__args__'): - for i, item in enumerate(self.items): - if not issubclass(cls.elem_type, type(item)): - raise TypeError("Typed vector cannot hold differently typed value" - " at index %d. Got type: %s, expected type: %s" % (i, type(item), cls.elem_type)) + def __getitem__(self, i): + return self.items[i] - def serialize(self): - from .ssz_impl import serialize - return serialize(self, self.__class__) + def __setitem__(self, k, v): + self.items[k] = v - def hash_tree_root(self): - from .ssz_impl import hash_tree_root - return hash_tree_root(self, self.__class__) + def __len__(self): + return len(self.items) def __repr__(self): - return repr({'length': self.__class__.length, 'items': self.items}) - - def __getitem__(self, key): - return self.items[key] - - def __setitem__(self, key, value): - self.items[key] = value + return repr(self.items) def __iter__(self): return iter(self.items) - def __len__(self): - return len(self.items) - def __eq__(self, other): - return self.hash_tree_root() == other.hash_tree_root() + return self.items == other.items +class List(AbstractList, metaclass=AbstractListMeta): + def value_check(self, value): + return len(value) <= self.__class__.length and super().value_check(value) -# SSZ BytesN -# ----------------------------- - - -def _is_bytes_n_instance_of(a, b): - # Other has to be a Bytes derivative class to be a BytesN - if not issubclass(b, bytes): - return False - elif not hasattr(b, 'length'): - # BytesN (b) is not an instance of BytesN[X] (a) - return False - elif not hasattr(a, 'length'): - # BytesN[X] (b) is an instance of BytesN (a) - return True - else: - # BytesN[X] (a) is an instance of BytesN[X] (b) - return a.length == b.length - - -def _is_equal_bytes_n_type(a, b): - # Other has to be a Bytes derivative class to be a BytesN - if not issubclass(b, bytes): - return False - elif not hasattr(a, 'length'): - if not hasattr(b, 'length'): - # BytesN == BytesN - return True - else: - # BytesN != BytesN[X] - return False - elif not hasattr(b, 'length'): - # BytesN[X] != BytesN - return False - else: - # BytesN[X] == BytesN[X] - return a.length == b.length - - -class BytesNMeta(type): - def __new__(cls, class_name, parents, attrs): - out = type.__new__(cls, class_name, parents, attrs) - if 'length' in attrs: - setattr(out, 'length', attrs['length']) - out._name = 'BytesN' - out.elem_type = byte - return out - - def __getitem__(self, n): - return self.__class__(self.__name__, (BytesN,), {'length': n}) - - def __subclasscheck__(self, sub): - return _is_bytes_n_instance_of(self, sub) - - def __instancecheck__(self, other): - return _is_bytes_n_instance_of(self, other.__class__) - - def __eq__(self, other): - return _is_equal_bytes_n_type(self, other) - - def __ne__(self, other): - return not _is_equal_bytes_n_type(self, other) - - def __hash__(self): - return hash(self.__class__) - - -def parse_bytes(val): - if val is None: - return None - elif isinstance(val, str): - # TODO: import from eth-utils instead, and do: hexstr_if_str(to_bytes, val) - return None - elif isinstance(val, bytes): - return val - elif isinstance(val, int): - return bytes([val]) - elif isinstance(val, (list, GeneratorType)): - return bytes(val) - else: - return None - - -class BytesN(bytes, metaclass=BytesNMeta): - def __new__(cls, *args): - if not hasattr(cls, 'length'): - return - bytesval = None - if len(args) == 1: - val: Union[bytes, int, str] = args[0] - bytesval = parse_bytes(val) - elif len(args) > 1: - # TODO: each int is 1 byte, check size, create bytesval - bytesval = bytes(args) - - if bytesval is None: - if cls.length == 0: - bytesval = b'' - else: - bytesval = b'\x00' * cls.length - if len(bytesval) != cls.length: - raise TypeError("BytesN[%d] cannot be initialized with value of %d bytes" % (cls.length, len(bytesval))) - return super().__new__(cls, bytesval) - - def serialize(self): - from .ssz_impl import serialize - return serialize(self, self.__class__) - - def hash_tree_root(self): - from .ssz_impl import hash_tree_root - return hash_tree_root(self, self.__class__) + def default(self): + return [] +class Vector(AbstractList, metaclass=AbstractListMeta): + def value_check(self, value): + return len(value) == self.__class__.length and super().value_check(value) -class Bytes4(BytesN): - length = 4 + def default(self): + return [get_zero_value(self.__class__.elem_type) for _ in range(self.__class__.length)] +class BytesMeta(AbstractListMeta): + def __getitem__(self, params): + if not isinstance(params, int): + raise Exception("Bytes must be instantiated with one arg: length") + o = self.__class__(self.__name__, (self,), {'length': params}) + o._name = 'Bytes' + return o -class Bytes32(BytesN): - length = 32 +def single_item_extractor(cls, args): + assert len(args) < 2 + return args[0] if len(args) > 0 else cls.default() +class Bytes(AbstractList, metaclass=BytesMeta): + def value_check(self, value): + return len(value) <= self.__class__.length and isinstance(value, bytes) -class Bytes48(BytesN): - length = 48 + extract_args = single_item_extractor + def default(self): + return b'' -class Bytes96(BytesN): - length = 96 +class BytesN(AbstractList, metaclass=BytesMeta): + def value_check(self, value): + return len(value) == self.__class__.length and isinstance(value, bytes) + extract_args = single_item_extractor -# SSZ Defaults -# ----------------------------- -def get_zero_value(typ): - if is_uint_type(typ): - return uint64(0) - elif is_list_type(typ): - return [] - elif is_bool_type(typ): - return False - elif is_vector_type(typ): - return typ() - elif is_bytesn_type(typ): - return typ() - elif is_bytes_type(typ): - return b'' - elif is_container_type(typ): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) - else: - raise Exception("Type not supported: {}".format(typ)) + def default(self): + return b'\x00' * self.__class__.length # Type helpers # ----------------------------- - def infer_type(obj): if is_uint_type(obj.__class__): return obj.__class__ elif isinstance(obj, int): return uint64 - elif isinstance(obj, list): - return List[infer_type(obj[0])] - elif isinstance(obj, (Vector, Container, bool, BytesN, bytes)): + elif isinstance(obj, (List, Vector, Container, bool, BytesN, Bytes)): return obj.__class__ else: raise Exception("Unknown type for {}".format(obj)) @@ -476,15 +275,14 @@ def is_list_type(typ): """ Check if the given type is a list. """ - return get_origin(typ) is List or get_origin(typ) is list + return isinstance(typ, type) and issubclass(typ, List) def is_bytes_type(typ): """ Check if the given type is a ``bytes``. """ - # Do not accept subclasses of bytes here, to avoid confusion with BytesN - return typ == bytes + return isinstance(typ, type) and issubclass(typ, Bytes) def is_bytesn_type(typ): @@ -526,22 +324,5 @@ def is_container_type(typ): L = TypeVar('L') -def read_list_elem_type(list_typ: Type[List[T]]) -> T: - if list_typ.__args__ is None or len(list_typ.__args__) != 1: - raise TypeError("Supplied list-type is invalid, no element type found.") - return list_typ.__args__[0] - - -def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: - return vector_typ.elem_type - - def read_elem_type(typ): - if typ == bytes or (isinstance(typ, type) and issubclass(typ, bytes)): # bytes or bytesN - return byte - elif is_list_type(typ): - return read_list_elem_type(typ) - elif is_vector_type(typ): - return read_vector_elem_type(typ) - else: - raise TypeError("Unexpected type: {}".format(typ)) + return typ.elem_type From 7c4232455c305b58143d9f62dfd29ebbaa2eb5e2 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Fri, 14 Jun 2019 14:07:25 -0400 Subject: [PATCH 070/405] Added get_container_type to get_zero_value --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index dbc3f95233..de54bbf059 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -118,6 +118,8 @@ def get_field_types(cls): def get_zero_value(typ): if typ == int: return 0 + elif is_container_type(typ): + return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) else: return typ.default() From 8919f628cbb9f17472716cb8cd850b963a7f37c7 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 14 Jun 2019 18:32:30 -0400 Subject: [PATCH 071/405] Update test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py Co-Authored-By: Diederik Loerakker --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index c88cfed1f3..b08a3d4e21 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,4 +1,5 @@ -from ..merkle_minimal import merkleize_chunks, hash, ZERO_BYTES32 +from ..merkle_minimal import merkleize_chunks, ZERO_BYTES32 +from .hash_function import hash from .ssz_typing import ( is_uint_type, is_bool_type, is_container_type, is_list_kind, is_vector_kind, From d1ecfd510ec2738dd267994a1e2b305176bb4eda Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:53:32 +0200 Subject: [PATCH 072/405] typing improvements --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 291 ++++++++---------- 1 file changed, 125 insertions(+), 166 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index de54bbf059..9aafb5294c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,52 +1,56 @@ from types import GeneratorType -from typing import List, Iterable, TypeVar, Type, NewType -from typing import Union -from typing_inspect import get_origin + + +class DefaultingTypeMeta(type): + def default(cls): + raise Exception("Not implemented") # SSZ integers # ----------------------------- -class uint(int): + +class uint(int, metaclass=DefaultingTypeMeta): byte_len = 0 def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") - if value.byte_len and value.bit_length() > value.byte_len: - raise ValueError("value out of bounds for uint{}".format(value.byte_len)) + if cls.byte_len and (value.bit_length() >> 3) > cls.byte_len: + raise ValueError("value out of bounds for uint{}".format(cls.byte_len)) return super().__new__(cls, value) + @classmethod + def default(cls): + return cls(0) + class uint8(uint): byte_len = 1 + # Alias for uint8 byte = NewType('byte', uint8) + class uint16(uint): byte_len = 2 + class uint32(uint): byte_len = 4 + class uint64(uint): byte_len = 8 + class uint128(uint): byte_len = 16 + class uint256(uint): byte_len = 32 -def is_uint_type(typ): - # All integers are uint in the scope of the spec here. - # Since we default to uint64. Bounds can be checked elsewhere. - # However, some are wrapped in a NewType - if hasattr(typ, '__supertype__'): - # get the type that the NewType is wrapping - typ = typ.__supertype__ - - return isinstance(typ, type) and issubclass(typ, int) and not issubclass(typ, bool) # SSZ Container base class # ----------------------------- @@ -59,7 +63,7 @@ def __init__(self, **kwargs): cls = self.__class__ for f, t in cls.get_fields(): if f not in kwargs: - setattr(self, f, get_zero_value(t)) + setattr(self, f, t.default()) else: setattr(self, f, kwargs[f]) @@ -83,9 +87,9 @@ def __repr__(self): return repr({field: getattr(self, field) for field in self.get_field_names()}) def __str__(self): - output = [] + output = [f'{self.__class__.__name__}'] for field in self.get_field_names(): - output.append(f'{field}: {getattr(self, field)}') + output.append(f' {field}: {getattr(self, field)}') return "\n".join(output) def __eq__(self, other): @@ -114,67 +118,94 @@ def get_field_types(cls): # values of annotations are the types corresponding to the fields, not instance values. return list(cls.__annotations__.values()) + @classmethod + def default(cls): + return cls(**{f: t.default() for f, t in cls.get_fields()}) + -def get_zero_value(typ): - if typ == int: - return 0 - elif is_container_type(typ): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) - else: - return typ.default() +class ParamsBase: + _bare = True -def type_check(typ, value): - if typ == int or typ == uint64: - return isinstance(value, int) - else: - return typ.value_check(value) + def __new__(cls, *args, **kwargs): + if cls._bare: + raise Exception("cannot init bare type without params") + return super().__new__(cls, **kwargs) + + +class ParamsMeta(DefaultingTypeMeta): -class AbstractListMeta(type): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) - if 'elem_type' in attrs and 'length' in attrs: - setattr(out, 'elem_type', attrs['elem_type']) - setattr(out, 'length', attrs['length']) + for k, v in attrs.items(): + setattr(out, k, v) return out def __getitem__(self, params): - if not isinstance(params, tuple) or len(params) != 2: - raise Exception("List must be instantiated with two args: elem type and length") - o = self.__class__(self.__name__, (self,), {'elem_type': params[0], 'length': params[1]}) - o._name = 'AbstractList' + o = self.__class__(self.__name__, (self,), self.attr_from_params(params)) + o._bare = False return o + def attr_from_params(self, p): + # single key params are valid too. Wrap them in a tuple. + params = p if isinstance(p, tuple) else (p,) + res = {} + i = 0 + for (name, typ) in self.__annotations__.items(): + param = params[i] + if hasattr(self.__class__, name): + res[name] = getattr(self.__class__, name) + else: + if not isinstance(param, typ): + raise TypeError( + "cannot create parametrized class with param {} as {} of type {}".format(param, name, typ)) + res[name] = param + i += 1 + if len(params) != i: + raise TypeError("provided parameters {} mismatch required parameter count {}".format(params, i)) + return res + def __instancecheck__(self, obj): if obj.__class__.__name__ != self.__name__: return False - if hasattr(self, 'elem_type') and obj.__class__.elem_type != self.elem_type: - return False - if hasattr(self, 'length') and obj.__class__.length != self.length: - return False + for name, typ in self.__annotations__: + if hasattr(self, name) and hasattr(obj.__class__, name) \ + and getattr(obj.__class__, name) != getattr(self, name): + return False return True + class ValueCheckError(Exception): pass -class AbstractList(metaclass=AbstractListMeta): + +class AbstractListMeta(ParamsMeta): + elem_type: DefaultingTypeMeta + length: int + + +class AbstractList(ParamsBase, metaclass=AbstractListMeta): + def __init__(self, *args): - items = self.extract_args(args) - + items = self.extract_args(*args) + if not self.value_check(items): raise ValueCheckError("Bad input for class {}: {}".format(self.__class__, items)) self.items = items - - def value_check(self, value): - for v in value: - if not type_check(self.__class__.elem_type, v): - return False - return True - def extract_args(self, args): - return list(args) if len(args) > 0 else self.default() + @classmethod + def value_check(cls, value): + return all(isinstance(v, cls.elem_type) for v in value) - def default(self): - raise Exception("Not implemented") + @classmethod + def extract_args(cls, *args): + x = list(args) + if len(x) == 1 and isinstance(x[0], GeneratorType): + x = list(x[0]) + return x if len(x) > 0 else cls.default() + + def __str__(self): + cls = self.__class__ + return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self.items)})" def __getitem__(self, i): return self.items[i] @@ -194,137 +225,65 @@ def __iter__(self): def __eq__(self, other): return self.items == other.items -class List(AbstractList, metaclass=AbstractListMeta): + +class List(AbstractList): def value_check(self, value): return len(value) <= self.__class__.length and super().value_check(value) - def default(self): - return [] + @classmethod + def default(cls): + return cls() + class Vector(AbstractList, metaclass=AbstractListMeta): def value_check(self, value): return len(value) == self.__class__.length and super().value_check(value) - def default(self): - return [get_zero_value(self.__class__.elem_type) for _ in range(self.__class__.length)] - -class BytesMeta(AbstractListMeta): - def __getitem__(self, params): - if not isinstance(params, int): - raise Exception("Bytes must be instantiated with one arg: length") - o = self.__class__(self.__name__, (self,), {'length': params}) - o._name = 'Bytes' - return o - -def single_item_extractor(cls, args): - assert len(args) < 2 - return args[0] if len(args) > 0 else cls.default() - -class Bytes(AbstractList, metaclass=BytesMeta): - def value_check(self, value): - return len(value) <= self.__class__.length and isinstance(value, bytes) - - extract_args = single_item_extractor - - def default(self): - return b'' - -class BytesN(AbstractList, metaclass=BytesMeta): - def value_check(self, value): - return len(value) == self.__class__.length and isinstance(value, bytes) - - extract_args = single_item_extractor - - def default(self): - return b'\x00' * self.__class__.length - - -# Type helpers -# ----------------------------- - -def infer_type(obj): - if is_uint_type(obj.__class__): - return obj.__class__ - elif isinstance(obj, int): - return uint64 - elif isinstance(obj, (List, Vector, Container, bool, BytesN, Bytes)): - return obj.__class__ - else: - raise Exception("Unknown type for {}".format(obj)) - - -def infer_input_type(fn): - """ - Decorator to run infer_type on the obj if typ argument is None - """ - def infer_helper(obj, typ=None, **kwargs): - if typ is None: - typ = infer_type(obj) - return fn(obj, typ=typ, **kwargs) - return infer_helper - - -def is_bool_type(typ): - """ - Check if the given type is a bool. - """ - if hasattr(typ, '__supertype__'): - typ = typ.__supertype__ - return isinstance(typ, type) and issubclass(typ, bool) - - -def is_list_type(typ): - """ - Check if the given type is a list. - """ - return isinstance(typ, type) and issubclass(typ, List) - + @classmethod + def default(cls): + return [cls.elem_type.default() for _ in range(cls.length)] -def is_bytes_type(typ): - """ - Check if the given type is a ``bytes``. - """ - return isinstance(typ, type) and issubclass(typ, Bytes) +class BytesMeta(AbstractListMeta): + elem_type: DefaultingTypeMeta = byte + length: int -def is_bytesn_type(typ): - """ - Check if the given type is a BytesN. - """ - return isinstance(typ, type) and issubclass(typ, BytesN) +class BytesLike(AbstractList, metaclass=BytesMeta): -def is_list_kind(typ): - """ - Check if the given type is a kind of list. Can be bytes. - """ - return is_list_type(typ) or is_bytes_type(typ) + @classmethod + def extract_args(cls, args): + if isinstance(args, bytes): + return args + elif isinstance(args, BytesLike): + return args.items + elif isinstance(args, GeneratorType): + return bytes(args) + else: + return bytes(args) + @classmethod + def value_check(cls, value): + return len(value) == cls.length and isinstance(value, bytes) -def is_vector_type(typ): - """ - Check if the given type is a vector. - """ - return isinstance(typ, type) and issubclass(typ, Vector) + def __str__(self): + cls = self.__class__ + return f"{cls.__name__}[{cls.length}]: {self.items.hex()}" -def is_vector_kind(typ): - """ - Check if the given type is a kind of vector. Can be BytesN. - """ - return is_vector_type(typ) or is_bytesn_type(typ) +class Bytes(BytesLike): + def value_check(self, value): + return len(value) <= self.__class__.length and isinstance(value, bytes) -def is_container_type(typ): - """ - Check if the given type is a container. - """ - return isinstance(typ, type) and issubclass(typ, Container) + @classmethod + def default(cls): + return b'' -T = TypeVar('T') -L = TypeVar('L') +class BytesN(BytesLike): + @classmethod + def default(cls): + return b'\x00' * cls.length -def read_elem_type(typ): - return typ.elem_type From b6cf809d9b686f090dc4e19a68fbf311cd247831 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:54:59 +0200 Subject: [PATCH 073/405] more improvements, and implement new space-efficient merkleization with padding support --- .../pyspec/eth2spec/utils/merkle_minimal.py | 36 ++++++++--- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 64 +++++++++++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 32 ++++++---- 3 files changed, 85 insertions(+), 47 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index c508f0df29..ebfb4faf6d 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -44,11 +44,31 @@ def next_power_of_two(v: int) -> int: return 1 << (v - 1).bit_length() -def merkleize_chunks(chunks): - tree = chunks[::] - margin = next_power_of_two(len(chunks)) - len(chunks) - tree.extend([ZERO_BYTES32] * margin) - tree = [ZERO_BYTES32] * len(tree) + tree - for i in range(len(tree) // 2 - 1, 0, -1): - tree[i] = hash(tree[i * 2] + tree[i * 2 + 1]) - return tree[1] +def merkleize_chunks(chunks, pad_to: int = None): + count = len(chunks) + depth = max(count - 1, 0).bit_length() + max_depth = max(depth, (pad_to - 1).bit_length()) + tmp = [None for _ in range(max_depth + 1)] + + def merge(h, i): + j = 0 + while True: + if i & (1 << j) == 0: + if i == count and j < depth: + h = hash(h + zerohashes[j]) + else: + break + else: + h = hash(tmp[j] + h) + j += 1 + tmp[j] = h + + for i in range(count): + merge(chunks[i], i) + + merge(zerohashes[0], count) + + for j in range(depth, max_depth): + tmp[j + 1] = hash(tmp[j] + zerohashes[j]) + + return tmp[max_depth] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index b08a3d4e21..1a556bc7db 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,11 +1,7 @@ from ..merkle_minimal import merkleize_chunks, ZERO_BYTES32 -from .hash_function import hash +from ..hash_function import hash from .ssz_typing import ( - is_uint_type, is_bool_type, is_container_type, - is_list_kind, is_vector_kind, - read_elem_type, - infer_input_type, - get_zero_value, + get_zero_value, Container, List, Vector, Bytes, BytesN, uint ) # SSZ Serialization @@ -15,13 +11,13 @@ def is_basic_type(typ): - return is_uint_type(typ) or is_bool_type(typ) + return issubclass(typ, (bool, uint)) def serialize_basic(value, typ): - if is_uint_type(typ): + if issubclass(typ, uint): return value.to_bytes(typ.byte_len, 'little') - elif is_bool_type(typ): + elif issubclass(typ, bool): if value: return b'\x01' else: @@ -31,22 +27,34 @@ def serialize_basic(value, typ): def deserialize_basic(value, typ): - if is_uint_type(typ): + if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif is_bool_type(typ): + elif issubclass(typ, bool): assert value in (b'\x00', b'\x01') return True if value == b'\x01' else False else: raise Exception("Type not supported: {}".format(typ)) +def is_list_kind(typ): + return issubclass(typ, (List, Bytes)) + + +def is_vector_kind(typ): + return issubclass(typ, (Vector, BytesN)) + + +def is_container_type(typ): + return issubclass(typ, Container) + + def is_fixed_size(typ): if is_basic_type(typ): return True elif is_list_kind(typ): return False elif is_vector_kind(typ): - return is_fixed_size(read_vector_elem_type(typ)) + return is_fixed_size(typ.elem_type) elif is_container_type(typ): return all(is_fixed_size(t) for t in typ.get_field_types()) else: @@ -57,12 +65,11 @@ def is_empty(obj): return get_zero_value(type(obj)) == obj -@infer_input_type -def serialize(obj, typ=None): +def serialize(obj, typ): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_kind(typ) or is_vector_kind(typ): - return encode_series(obj, [read_elem_type(typ)] * len(obj)) + return encode_series(obj, [typ.elem_type] * len(obj)) elif is_container_type(typ): return encode_series(obj.get_field_values(), typ.get_field_types()) else: @@ -126,40 +133,41 @@ def mix_in_length(root, length): def is_bottom_layer_kind(typ): return ( is_basic_type(typ) or - (is_list_kind(typ) or is_vector_kind(typ)) and is_basic_type(read_elem_type(typ)) + (is_list_kind(typ) or is_vector_kind(typ)) and is_basic_type(typ.elem_type) ) -@infer_input_type -def get_typed_values(obj, typ=None): +def get_typed_values(obj, typ): if is_container_type(typ): return obj.get_typed_values() elif is_list_kind(typ) or is_vector_kind(typ): - elem_type = read_elem_type(typ) - return list(zip(obj, [elem_type] * len(obj))) + return list(zip(obj, [typ.elem_type] * len(obj))) else: raise Exception("Invalid type") + def item_length(typ): - return 1 if typ == bool else typ.byte_len if is_uint_type(typ) else 32 + if typ == bool: + return 1 + elif issubclass(typ, uint): + return typ.byte_len + else: + return 32 + -@infer_input_type -def hash_tree_root(obj, typ=None): +def hash_tree_root(obj, typ): if is_bottom_layer_kind(typ): - data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_type(typ)) + data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, typ.elem_type) leaves = chunkify(data) else: fields = get_typed_values(obj, typ=typ) leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] if is_list_kind(typ): - full_chunk_length = (item_length(read_elem_type(typ)) * typ.length + 31) // 32 - leaves += [ZERO_BYTES32] * (full_chunk_length - len(obj)) - return mix_in_length(merkleize_chunks(leaves), len(obj)) + return mix_in_length(merkleize_chunks(leaves, pad_to=typ.length), len(obj)) else: return merkleize_chunks(leaves) -@infer_input_type def signing_root(obj, typ): assert is_container_type(typ) # ignore last field diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 9aafb5294c..30f71f87dd 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,10 +1,26 @@ +from typing import NewType, Union from types import GeneratorType +class ValueCheckError(Exception): + pass + + class DefaultingTypeMeta(type): def default(cls): raise Exception("Not implemented") +# Every type is subclassed and has a default() method, except bool. +TypeWithDefault = Union[DefaultingTypeMeta, bool] + + +def get_zero_value(typ: TypeWithDefault): + if issubclass(typ, bool): + return False + else: + return typ.default() + + # SSZ integers # ----------------------------- @@ -63,7 +79,7 @@ def __init__(self, **kwargs): cls = self.__class__ for f, t in cls.get_fields(): if f not in kwargs: - setattr(self, f, t.default()) + setattr(self, f, get_zero_value(t)) else: setattr(self, f, kwargs[f]) @@ -120,7 +136,7 @@ def get_field_types(cls): @classmethod def default(cls): - return cls(**{f: t.default() for f, t in cls.get_fields()}) + return cls(**{f: get_zero_value(t) for f, t in cls.get_fields()}) class ParamsBase: @@ -174,12 +190,8 @@ def __instancecheck__(self, obj): return True -class ValueCheckError(Exception): - pass - - class AbstractListMeta(ParamsMeta): - elem_type: DefaultingTypeMeta + elem_type: TypeWithDefault length: int @@ -227,8 +239,6 @@ def __eq__(self, other): class List(AbstractList): - def value_check(self, value): - return len(value) <= self.__class__.length and super().value_check(value) @classmethod def default(cls): @@ -241,11 +251,11 @@ def value_check(self, value): @classmethod def default(cls): - return [cls.elem_type.default() for _ in range(cls.length)] + return [get_zero_value(cls.elem_type) for _ in range(cls.length)] class BytesMeta(AbstractListMeta): - elem_type: DefaultingTypeMeta = byte + elem_type: TypeWithDefault = byte length: int From cd5f59eb74cf8c398033ccd7e862d89c4d6ceffa Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:55:53 +0200 Subject: [PATCH 074/405] fix bytes value check, fix default-type checking --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 30f71f87dd..6f578796d0 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -171,7 +171,10 @@ def attr_from_params(self, p): if hasattr(self.__class__, name): res[name] = getattr(self.__class__, name) else: - if not isinstance(param, typ): + if typ == TypeWithDefault: + if not (isinstance(param, bool) or isinstance(param, DefaultingTypeMeta)): + raise TypeError("expected param {} as {} to have a type default".format(param, name, typ)) + elif not isinstance(param, typ): raise TypeError( "cannot create parametrized class with param {} as {} of type {}".format(param, name, typ)) res[name] = param @@ -246,8 +249,10 @@ def default(cls): class Vector(AbstractList, metaclass=AbstractListMeta): - def value_check(self, value): - return len(value) == self.__class__.length and super().value_check(value) + + @classmethod + def value_check(cls, value): + return len(value) == cls.length and super().value_check(value) @classmethod def default(cls): @@ -274,7 +279,7 @@ def extract_args(cls, args): @classmethod def value_check(cls, value): - return len(value) == cls.length and isinstance(value, bytes) + return isinstance(value, bytes) def __str__(self): cls = self.__class__ @@ -283,9 +288,6 @@ def __str__(self): class Bytes(BytesLike): - def value_check(self, value): - return len(value) <= self.__class__.length and isinstance(value, bytes) - @classmethod def default(cls): return b'' @@ -297,3 +299,7 @@ class BytesN(BytesLike): def default(cls): return b'\x00' * cls.length + @classmethod + def value_check(cls, value): + return len(value) == cls.length and super().value_check(value) + From 54a1fa9abec52a747ea47bf73c679a7b2f2bef11 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Sat, 15 Jun 2019 20:57:22 +0200 Subject: [PATCH 075/405] Update test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py Co-Authored-By: vbuterin --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 1a556bc7db..ed8dbf1e07 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -153,7 +153,13 @@ def item_length(typ): return typ.byte_len else: return 32 - +def chunk_count(typ): + if is_basic_type(typ): + return 1 + elif is_list_kind(typ) or is_vector_kind(typ): + return (typ.length * item_length(typ.elem_type) + 31) // 32 + else: + return len(typ.get_fields()) def hash_tree_root(obj, typ): if is_bottom_layer_kind(typ): From 3a9b1fb72c97e4512a935fb3c99c084ac411cae1 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Sat, 15 Jun 2019 20:57:30 +0200 Subject: [PATCH 076/405] Update test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py Co-Authored-By: vbuterin --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index ed8dbf1e07..bcdef3988b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -169,7 +169,7 @@ def hash_tree_root(obj, typ): fields = get_typed_values(obj, typ=typ) leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] if is_list_kind(typ): - return mix_in_length(merkleize_chunks(leaves, pad_to=typ.length), len(obj)) + return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(typ)), len(obj)) else: return merkleize_chunks(leaves) From 82e7392b17f7d63f58a2745d00555bb8e479cf18 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 15 Jun 2019 21:13:01 +0200 Subject: [PATCH 077/405] default method for container is recognized now --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6f578796d0..7cf44f6b34 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -73,7 +73,7 @@ class uint256(uint): # Note: importing ssz functionality locally, to avoid import loop -class Container(object): +class Container(object, metaclass=DefaultingTypeMeta): def __init__(self, **kwargs): cls = self.__class__ From 108410d8626a2db20bb7e44422180f98d41162c6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 15 Jun 2019 22:12:59 +0200 Subject: [PATCH 078/405] Change byte to explict class instead of newtype --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7cf44f6b34..40901ad970 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -45,7 +45,8 @@ class uint8(uint): # Alias for uint8 -byte = NewType('byte', uint8) +class byte(uint8): + pass class uint16(uint): From 4ebdceaf129e40cd27c2b95ffbd36af0d72e9db6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:57:50 +0200 Subject: [PATCH 079/405] highly experimental typing --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 140 +++++++----------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 137 +++++++++++------ 2 files changed, 143 insertions(+), 134 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index bcdef3988b..6795748917 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ -from ..merkle_minimal import merkleize_chunks, ZERO_BYTES32 +from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - get_zero_value, Container, List, Vector, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint ) # SSZ Serialization @@ -10,79 +10,48 @@ BYTES_PER_LENGTH_OFFSET = 4 -def is_basic_type(typ): - return issubclass(typ, (bool, uint)) - - -def serialize_basic(value, typ): - if issubclass(typ, uint): - return value.to_bytes(typ.byte_len, 'little') - elif issubclass(typ, bool): +def serialize_basic(value: SSZValue): + if isinstance(value, uint): + return value.to_bytes(value.type().byte_len, 'little') + elif isinstance(value, Bit): if value: return b'\x01' else: return b'\x00' else: - raise Exception("Type not supported: {}".format(typ)) + raise Exception(f"Type not supported: {type(value)}") -def deserialize_basic(value, typ): +def deserialize_basic(value, typ: BasicType): if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif issubclass(typ, bool): + elif issubclass(typ, Bit): assert value in (b'\x00', b'\x01') - return True if value == b'\x01' else False - else: - raise Exception("Type not supported: {}".format(typ)) - - -def is_list_kind(typ): - return issubclass(typ, (List, Bytes)) - - -def is_vector_kind(typ): - return issubclass(typ, (Vector, BytesN)) - - -def is_container_type(typ): - return issubclass(typ, Container) - - -def is_fixed_size(typ): - if is_basic_type(typ): - return True - elif is_list_kind(typ): - return False - elif is_vector_kind(typ): - return is_fixed_size(typ.elem_type) - elif is_container_type(typ): - return all(is_fixed_size(t) for t in typ.get_field_types()) + return Bit(value == b'\x01') else: - raise Exception("Type not supported: {}".format(typ)) + raise Exception(f"Type not supported: {typ}") -def is_empty(obj): - return get_zero_value(type(obj)) == obj +def is_empty(obj: SSZValue): + return type(obj).default() == obj -def serialize(obj, typ): - if is_basic_type(typ): - return serialize_basic(obj, typ) - elif is_list_kind(typ) or is_vector_kind(typ): - return encode_series(obj, [typ.elem_type] * len(obj)) - elif is_container_type(typ): - return encode_series(obj.get_field_values(), typ.get_field_types()) +def serialize(obj: SSZValue): + if isinstance(obj, BasicValue): + return serialize_basic(obj) + elif isinstance(obj, Series): + return encode_series(obj) else: - raise Exception("Type not supported: {}".format(typ)) + raise Exception(f"Type not supported: {type(obj)}") -def encode_series(values, types): +def encode_series(values: Series): # bytes and bytesN are already in the right format. if isinstance(values, bytes): return values # Recursively serialize - parts = [(is_fixed_size(types[i]), serialize(values[i], typ=types[i])) for i in range(len(values))] + parts = [(v.type().is_fixed_size(), serialize(v)) for v in values] # Compute and check lengths fixed_lengths = [len(serialized) if constant_size else BYTES_PER_LENGTH_OFFSET @@ -114,10 +83,10 @@ def encode_series(values, types): # ----------------------------- -def pack(values, subtype): +def pack(values: Series): if isinstance(values, bytes): return values - return b''.join([serialize_basic(value, subtype) for value in values]) + return b''.join([serialize_basic(value) for value in values]) def chunkify(bytez): @@ -130,52 +99,49 @@ def mix_in_length(root, length): return hash(root + length.to_bytes(32, 'little')) -def is_bottom_layer_kind(typ): +def is_bottom_layer_kind(typ: SSZType): return ( - is_basic_type(typ) or - (is_list_kind(typ) or is_vector_kind(typ)) and is_basic_type(typ.elem_type) + issubclass(typ, BasicType) or + (issubclass(typ, Elements) and issubclass(typ.elem_type, BasicType)) ) -def get_typed_values(obj, typ): - if is_container_type(typ): - return obj.get_typed_values() - elif is_list_kind(typ) or is_vector_kind(typ): - return list(zip(obj, [typ.elem_type] * len(obj))) +def item_length(typ: SSZType) -> int: + if issubclass(typ, BasicType): + return typ.byte_len else: - raise Exception("Invalid type") + return 32 -def item_length(typ): - if typ == bool: +def chunk_count(typ: SSZType) -> int: + if issubclass(typ, BasicType): return 1 - elif issubclass(typ, uint): - return typ.byte_len + elif issubclass(typ, Elements): + return (typ.length * item_length(typ.elem_type) + 31) // 32 + elif issubclass(typ, Container): + return len(typ.get_fields()) else: - return 32 -def chunk_count(typ): - if is_basic_type(typ): - return 1 - elif is_list_kind(typ) or is_vector_kind(typ): - return (typ.length * item_length(typ.elem_type) + 31) // 32 - else: - return len(typ.get_fields()) - -def hash_tree_root(obj, typ): - if is_bottom_layer_kind(typ): - data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, typ.elem_type) - leaves = chunkify(data) + raise Exception(f"Type not supported: {typ}") + + +def hash_tree_root(obj: SSZValue): + if isinstance(obj, Series): + if is_bottom_layer_kind(obj.type()): + leaves = chunkify(pack(obj)) + else: + leaves = [hash_tree_root(value) for value in obj] + elif isinstance(obj, BasicValue): + leaves = chunkify(serialize_basic(obj)) else: - fields = get_typed_values(obj, typ=typ) - leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] - if is_list_kind(typ): - return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(typ)), len(obj)) + raise Exception(f"Type not supported: {obj.type()}") + + if isinstance(obj, (List, Bytes)): + return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(obj.type())), len(obj)) else: return merkleize_chunks(leaves) -def signing_root(obj, typ): - assert is_container_type(typ) +def signing_root(obj: Container): # ignore last field - leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] + leaves = [hash_tree_root(field) for field in obj[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 40901ad970..b79789f27f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,4 @@ -from typing import NewType, Union +from typing import Tuple, Dict, Iterator from types import GeneratorType @@ -10,24 +10,42 @@ class DefaultingTypeMeta(type): def default(cls): raise Exception("Not implemented") -# Every type is subclassed and has a default() method, except bool. -TypeWithDefault = Union[DefaultingTypeMeta, bool] +class SSZType(DefaultingTypeMeta): -def get_zero_value(typ: TypeWithDefault): - if issubclass(typ, bool): - return False - else: - return typ.default() + def is_fixed_size(cls): + raise Exception("Not implemented") -# SSZ integers -# ----------------------------- +class SSZValue(object, metaclass=SSZType): + def type(self): + return self.__class__ -class uint(int, metaclass=DefaultingTypeMeta): + +class BasicType(SSZType): byte_len = 0 + def is_fixed_size(cls): + return True + + +class BasicValue(int, SSZValue, metaclass=BasicType): + pass + + +class Bit(BasicValue): # can't subclass bool. + + @classmethod + def default(cls): + return cls(False) + + def __bool__(self): + return self > 0 + + +class uint(BasicValue, metaclass=BasicType): + def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") @@ -69,36 +87,39 @@ class uint256(uint): byte_len = 32 -# SSZ Container base class -# ----------------------------- +class Series(SSZValue): + + def __iter__(self) -> Iterator[SSZValue]: + raise Exception("Not implemented") + # Note: importing ssz functionality locally, to avoid import loop -class Container(object, metaclass=DefaultingTypeMeta): +class Container(Series, metaclass=SSZType): def __init__(self, **kwargs): cls = self.__class__ for f, t in cls.get_fields(): if f not in kwargs: - setattr(self, f, get_zero_value(t)) + setattr(self, f, t.default()) else: setattr(self, f, kwargs[f]) def serialize(self): from .ssz_impl import serialize - return serialize(self, self.__class__) + return serialize(self) def hash_tree_root(self): from .ssz_impl import hash_tree_root - return hash_tree_root(self, self.__class__) + return hash_tree_root(self) def signing_root(self): from .ssz_impl import signing_root - return signing_root(self, self.__class__) + return signing_root(self) - def get_field_values(self): + def get_field_values(self) -> Tuple[SSZValue, ...]: cls = self.__class__ - return [getattr(self, field) for field in cls.get_field_names()] + return tuple(getattr(self, field) for field in cls.get_field_names()) def __repr__(self): return repr({field: getattr(self, field) for field in self.get_field_names()}) @@ -116,31 +137,38 @@ def __hash__(self): return hash(self.hash_tree_root()) @classmethod - def get_fields_dict(cls): + def get_fields_dict(cls) -> Dict[str, SSZType]: return dict(cls.__annotations__) @classmethod - def get_fields(cls): - return list(dict(cls.__annotations__).items()) + def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: + return tuple((f, SSZType(t)) for f, t in dict(cls.__annotations__).items()) def get_typed_values(self): - return list(zip(self.get_field_values(), self.get_field_types())) + return tuple(zip(self.get_field_values(), self.get_field_types())) @classmethod - def get_field_names(cls): - return list(cls.__annotations__.keys()) + def get_field_names(cls) -> Tuple[str]: + return tuple(cls.__annotations__.keys()) @classmethod - def get_field_types(cls): + def get_field_types(cls) -> Tuple[SSZType, ...]: # values of annotations are the types corresponding to the fields, not instance values. - return list(cls.__annotations__.values()) + return tuple(cls.__annotations__.values()) @classmethod def default(cls): - return cls(**{f: get_zero_value(t) for f, t in cls.get_fields()}) + return cls(**{f: t.default() for f, t in cls.get_fields()}) + + @classmethod + def is_fixed_size(cls): + return all(t.is_fixed_size() for t in cls.get_field_types()) + + def __iter__(self) -> Iterator[SSZValue]: + return iter(self.get_field_values()) -class ParamsBase: +class ParamsBase(Series): _bare = True def __new__(cls, *args, **kwargs): @@ -149,7 +177,7 @@ def __new__(cls, *args, **kwargs): return super().__new__(cls, **kwargs) -class ParamsMeta(DefaultingTypeMeta): +class ParamsMeta(SSZType): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) @@ -168,14 +196,14 @@ def attr_from_params(self, p): res = {} i = 0 for (name, typ) in self.__annotations__.items(): - param = params[i] if hasattr(self.__class__, name): res[name] = getattr(self.__class__, name) else: - if typ == TypeWithDefault: - if not (isinstance(param, bool) or isinstance(param, DefaultingTypeMeta)): - raise TypeError("expected param {} as {} to have a type default".format(param, name, typ)) - elif not isinstance(param, typ): + if i >= len(params): + i += 1 + continue + param = params[i] + if not isinstance(param, typ): raise TypeError( "cannot create parametrized class with param {} as {} of type {}".format(param, name, typ)) res[name] = param @@ -194,12 +222,12 @@ def __instancecheck__(self, obj): return True -class AbstractListMeta(ParamsMeta): - elem_type: TypeWithDefault +class Elements(ParamsMeta): + elem_type: SSZType length: int -class AbstractList(ParamsBase, metaclass=AbstractListMeta): +class ElementsBase(ParamsBase, metaclass=Elements): def __init__(self, *args): items = self.extract_args(*args) @@ -223,7 +251,7 @@ def __str__(self): cls = self.__class__ return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self.items)})" - def __getitem__(self, i): + def __getitem__(self, i) -> SSZValue: return self.items[i] def __setitem__(self, k, v): @@ -235,21 +263,25 @@ def __len__(self): def __repr__(self): return repr(self.items) - def __iter__(self): + def __iter__(self) -> Iterator[SSZValue]: return iter(self.items) def __eq__(self, other): return self.items == other.items -class List(AbstractList): +class List(ElementsBase): @classmethod def default(cls): return cls() + @classmethod + def is_fixed_size(cls): + return False + -class Vector(AbstractList, metaclass=AbstractListMeta): +class Vector(ElementsBase): @classmethod def value_check(cls, value): @@ -257,15 +289,19 @@ def value_check(cls, value): @classmethod def default(cls): - return [get_zero_value(cls.elem_type) for _ in range(cls.length)] + return [cls.elem_type.default() for _ in range(cls.length)] + + @classmethod + def is_fixed_size(cls): + return cls.elem_type.is_fixed_size() -class BytesMeta(AbstractListMeta): - elem_type: TypeWithDefault = byte +class BytesMeta(Elements): + elem_type: SSZType = byte length: int -class BytesLike(AbstractList, metaclass=BytesMeta): +class BytesLike(ElementsBase, metaclass=BytesMeta): @classmethod def extract_args(cls, args): @@ -293,6 +329,10 @@ class Bytes(BytesLike): def default(cls): return b'' + @classmethod + def is_fixed_size(cls): + return False + class BytesN(BytesLike): @@ -304,3 +344,6 @@ def default(cls): def value_check(cls, value): return len(value) == cls.length and super().value_check(value) + @classmethod + def is_fixed_size(cls): + return True From 97025c51ac399140445ee103f20ab488c2f3ca2f Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:03:11 +0200 Subject: [PATCH 080/405] start updating virtual sizes of lists --- specs/core/0_beacon-chain.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1dcdbdef94..ddc4c8f611 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -367,7 +367,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: bytes # Bit set for every attesting participant within a committee + aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] # Bit set for every attesting participant within a committee data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -434,9 +434,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: bytes + aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] data: AttestationData - custody_bitfield: bytes + custody_bitfield: Bytes[MAX_COMMITTEE_SIZE] signature: BLSSignature ``` @@ -480,12 +480,12 @@ class BeaconBlockBody(Container): eth1_data: Eth1Data # Eth1 data vote graffiti: Bytes32 # Arbitrary data # Operations - proposer_slashings: List[ProposerSlashing] - attester_slashings: List[AttesterSlashing] - attestations: List[Attestation] - deposits: List[Deposit] - voluntary_exits: List[VoluntaryExit] - transfers: List[Transfer] + proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] + attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] + attestations: List[Attestation, MAX_ATTESTATIONS] + deposits: List[Deposit, MAX_DEPOSITS] + voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] + transfers: List[Transfer, MAX_TRANSFERS] ``` #### `BeaconBlock` From 08e6f32f3897889a62dcdf0887e4ba42403aa3a0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 16 Jun 2019 03:21:58 +0200 Subject: [PATCH 081/405] typing improvements, type testing --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 21 ++- .../eth2spec/utils/ssz/test_ssz_typing.py | 131 ++++++++++++++++++ 2 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index b79789f27f..5cab68aa88 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -36,6 +36,11 @@ class BasicValue(int, SSZValue, metaclass=BasicType): class Bit(BasicValue): # can't subclass bool. + def __new__(cls, value, *args, **kwargs): + if value < 0 or value > 1: + raise ValueError(f"value {value} out of bounds for bit") + return super().__new__(cls, value) + @classmethod def default(cls): return cls(False) @@ -49,7 +54,7 @@ class uint(BasicValue, metaclass=BasicType): def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") - if cls.byte_len and (value.bit_length() >> 3) > cls.byte_len: + if cls.byte_len and value.bit_length() > (cls.byte_len << 3): raise ValueError("value out of bounds for uint{}".format(cls.byte_len)) return super().__new__(cls, value) @@ -142,7 +147,7 @@ def get_fields_dict(cls) -> Dict[str, SSZType]: @classmethod def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: - return tuple((f, SSZType(t)) for f, t in dict(cls.__annotations__).items()) + return tuple((f, t) for f, t in cls.__annotations__.items()) def get_typed_values(self): return tuple(zip(self.get_field_values(), self.get_field_types())) @@ -190,6 +195,12 @@ def __getitem__(self, params): o._bare = False return o + def __str__(self): + return f"{self.__name__}~{self.__class__.__name__}" + + def __repr__(self): + return self, self.__class__ + def attr_from_params(self, p): # single key params are valid too. Wrap them in a tuple. params = p if isinstance(p, tuple) else (p,) @@ -215,7 +226,7 @@ def attr_from_params(self, p): def __instancecheck__(self, obj): if obj.__class__.__name__ != self.__name__: return False - for name, typ in self.__annotations__: + for name, typ in self.__annotations__.items(): if hasattr(self, name) and hasattr(obj.__class__, name) \ and getattr(obj.__class__, name) != getattr(self, name): return False @@ -233,7 +244,7 @@ def __init__(self, *args): items = self.extract_args(*args) if not self.value_check(items): - raise ValueCheckError("Bad input for class {}: {}".format(self.__class__, items)) + raise ValueCheckError(f"Bad input for class {self.__class__}: {items}") self.items = items @classmethod @@ -245,7 +256,7 @@ def extract_args(cls, *args): x = list(args) if len(x) == 1 and isinstance(x[0], GeneratorType): x = list(x[0]) - return x if len(x) > 0 else cls.default() + return x def __str__(self): cls = self.__class__ diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py new file mode 100644 index 0000000000..a0705f8d36 --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -0,0 +1,131 @@ +from .ssz_typing import ( + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, + uint, uint8, uint16, uint32, uint64, uint128, uint256 +) + + +def test_subclasses(): + for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: + assert issubclass(u, uint) + assert issubclass(u, int) + assert issubclass(u, BasicValue) + assert issubclass(u, SSZValue) + assert isinstance(u, SSZType) + assert isinstance(u, BasicType) + assert issubclass(Bit, BasicValue) + assert isinstance(Bit, BasicType) + + for c in [Container, List, Vector, Bytes, BytesN]: + assert issubclass(c, Series) + assert issubclass(c, SSZValue) + assert isinstance(c, SSZType) + assert not issubclass(c, BasicValue) + assert not isinstance(c, BasicType) + + for c in [List, Vector, Bytes, BytesN]: + assert isinstance(c, Elements) + + +def test_basic_instances(): + for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: + v = u(123) + assert isinstance(v, uint) + assert isinstance(v, int) + assert isinstance(v, BasicValue) + assert isinstance(v, SSZValue) + + assert isinstance(Bit(True), BasicValue) + assert isinstance(Bit(False), BasicValue) + + +def test_basic_value_bounds(): + max = { + Bit: 2**1, + uint8: 2**(8 * 1), + uint16: 2**(8 * 2), + uint32: 2**(8 * 4), + uint64: 2**(8 * 8), + uint128: 2**(8 * 16), + uint256: 2**(8 * 32), + } + for k, v in max.items(): + # this should work + assert k(v - 1) == v - 1 + # but we do not allow overflows + try: + k(v) + assert False + except ValueError: + pass + + for k, _ in max.items(): + # this should work + assert k(0) == 0 + # but we do not allow underflows + try: + k(-1) + assert False + except ValueError: + pass + + +def test_container(): + class Foo(Container): + a: uint8 + b: uint32 + + assert issubclass(Foo, Container) + assert issubclass(Foo, SSZValue) + assert issubclass(Foo, Series) + + assert Foo.is_fixed_size() + x = Foo(a=uint8(123), b=uint32(45)) + assert x.a == 123 + assert x.b == 45 + assert isinstance(x.a, uint8) + assert isinstance(x.b, uint32) + assert x.type().is_fixed_size() + + class Bar(Container): + a: uint8 + b: List[uint8, 1024] + + assert not Bar.is_fixed_size() + + y = Bar(a=uint8(123), b=List[uint8, 1024](uint8(1), uint8(2))) + assert y.a == 123 + assert len(y.b) == 2 + assert isinstance(y.a, uint8) + assert isinstance(y.b, List[uint8, 1024]) + assert not y.type().is_fixed_size() + assert y.b[0] == 1 + v: List = y.b + assert v.type().elem_type == uint8 + assert v.type().length == 1024 + + +def test_list(): + typ = List[uint64, 128] + assert issubclass(typ, List) + assert issubclass(typ, SSZValue) + assert issubclass(typ, Series) + assert isinstance(typ, Elements) + + assert not typ.is_fixed_size() + + assert len(typ()) == 0 # empty + assert len(typ(uint64(0))) == 1 # single arg + assert len(typ(uint64(i) for i in range(10))) == 10 # generator + assert len(typ(uint64(0), uint64(1), uint64(2))) == 3 # args + + v = typ(uint64(0)) + v[0] = uint64(123) + assert v[0] == 123 + assert isinstance(v[0], uint64) + + assert isinstance(v, List) + assert isinstance(v, List[uint64, 128]) + assert isinstance(v, typ) + assert isinstance(v, SSZValue) + assert isinstance(v, Series) + assert isinstance(v.type(), Elements) From 8bd2e878ef308c58da54f35bd8dd67c13756048c Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 17 Jun 2019 01:39:39 +0200 Subject: [PATCH 082/405] bugfixes and typing improvements --- .../pyspec/eth2spec/utils/merkle_minimal.py | 10 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 18 +-- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 69 ++++++---- .../eth2spec/utils/ssz/test_ssz_impl.py | 118 ++++++++++++++++++ .../eth2spec/utils/ssz/test_ssz_typing.py | 11 +- 5 files changed, 187 insertions(+), 39 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index ebfb4faf6d..21583ee921 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -44,7 +44,7 @@ def next_power_of_two(v: int) -> int: return 1 << (v - 1).bit_length() -def merkleize_chunks(chunks, pad_to: int = None): +def merkleize_chunks(chunks, pad_to: int = 1): count = len(chunks) depth = max(count - 1, 0).bit_length() max_depth = max(depth, (pad_to - 1).bit_length()) @@ -55,7 +55,7 @@ def merge(h, i): while True: if i & (1 << j) == 0: if i == count and j < depth: - h = hash(h + zerohashes[j]) + h = hash(h + zerohashes[j]) # keep going if we are complementing the void to the next power of 2 else: break else: @@ -63,11 +63,15 @@ def merge(h, i): j += 1 tmp[j] = h + # merge in leaf by leaf. for i in range(count): merge(chunks[i], i) - merge(zerohashes[0], count) + # complement with 0 if empty, or if not the right power of 2 + if 1 << depth != count: + merge(zerohashes[0], count) + # the next power of two may be smaller than the ultimate virtual size, complement with zero-hashes at each depth. for j in range(depth, max_depth): tmp[j + 1] = hash(tmp[j] + zerohashes[j]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 6795748917..1b59b276bb 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint ) # SSZ Serialization @@ -47,8 +47,8 @@ def serialize(obj: SSZValue): def encode_series(values: Series): # bytes and bytesN are already in the right format. - if isinstance(values, bytes): - return values + if isinstance(values, (Bytes, BytesN)): + return values.items # Recursively serialize parts = [(v.type().is_fixed_size(), serialize(v)) for v in values] @@ -84,8 +84,8 @@ def encode_series(values: Series): def pack(values: Series): - if isinstance(values, bytes): - return values + if isinstance(values, (Bytes, BytesN)): + return values.items return b''.join([serialize_basic(value) for value in values]) @@ -101,8 +101,8 @@ def mix_in_length(root, length): def is_bottom_layer_kind(typ: SSZType): return ( - issubclass(typ, BasicType) or - (issubclass(typ, Elements) and issubclass(typ.elem_type, BasicType)) + isinstance(typ, BasicType) or + (issubclass(typ, Elements) and isinstance(typ.elem_type, BasicType)) ) @@ -114,7 +114,7 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: - if issubclass(typ, BasicType): + if isinstance(typ, BasicType): return 1 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 @@ -133,7 +133,7 @@ def hash_tree_root(obj: SSZValue): elif isinstance(obj, BasicValue): leaves = chunkify(serialize_basic(obj)) else: - raise Exception(f"Type not supported: {obj.type()}") + raise Exception(f"Type not supported: {type(obj)}") if isinstance(obj, (List, Bytes)): return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(obj.type())), len(obj)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 5cab68aa88..7662971c48 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -92,6 +92,22 @@ class uint256(uint): byte_len = 32 +def coerce_type_maybe(v, typ: SSZType): + v_typ = type(v) + # shortcut if it's already the type we are looking for + if v_typ == typ: + return v + elif isinstance(v, int): + return typ(v) + elif isinstance(v, (list, tuple)): + return typ(*v) + elif isinstance(v, GeneratorType): + return typ(v) + else: + # just return as-is, Value-checkers will take care of it not being coerced. + return v + + class Series(SSZValue): def __iter__(self) -> Iterator[SSZValue]: @@ -108,7 +124,11 @@ def __init__(self, **kwargs): if f not in kwargs: setattr(self, f, t.default()) else: - setattr(self, f, kwargs[f]) + value = coerce_type_maybe(kwargs[f], t) + if not isinstance(value, t): + raise ValueCheckError(f"Bad input for class {self.__class__}:" + f" field: {f} type: {t} value: {value} value type: {type(value)}") + setattr(self, f, value) def serialize(self): from .ssz_impl import serialize @@ -141,23 +161,22 @@ def __eq__(self, other): def __hash__(self): return hash(self.hash_tree_root()) - @classmethod - def get_fields_dict(cls) -> Dict[str, SSZType]: - return dict(cls.__annotations__) - @classmethod def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: + if not hasattr(cls, '__annotations__'): # no container fields + return () return tuple((f, t) for f, t in cls.__annotations__.items()) - def get_typed_values(self): - return tuple(zip(self.get_field_values(), self.get_field_types())) - @classmethod - def get_field_names(cls) -> Tuple[str]: + def get_field_names(cls) -> Tuple[str, ...]: + if not hasattr(cls, '__annotations__'): # no container fields + return () return tuple(cls.__annotations__.keys()) @classmethod def get_field_types(cls) -> Tuple[SSZType, ...]: + if not hasattr(cls, '__annotations__'): # no container fields + return () # values of annotations are the types corresponding to the fields, not instance values. return tuple(cls.__annotations__.values()) @@ -233,12 +252,12 @@ def __instancecheck__(self, obj): return True -class Elements(ParamsMeta): +class ElementsType(ParamsMeta): elem_type: SSZType length: int -class ElementsBase(ParamsBase, metaclass=Elements): +class Elements(ParamsBase, metaclass=ElementsType): def __init__(self, *args): items = self.extract_args(*args) @@ -256,6 +275,7 @@ def extract_args(cls, *args): x = list(args) if len(x) == 1 and isinstance(x[0], GeneratorType): x = list(x[0]) + x = [coerce_type_maybe(v, cls.elem_type) for v in x] return x def __str__(self): @@ -281,7 +301,7 @@ def __eq__(self, other): return self.items == other.items -class List(ElementsBase): +class List(Elements): @classmethod def default(cls): @@ -292,7 +312,7 @@ def is_fixed_size(cls): return False -class Vector(ElementsBase): +class Vector(Elements): @classmethod def value_check(cls, value): @@ -307,23 +327,26 @@ def is_fixed_size(cls): return cls.elem_type.is_fixed_size() -class BytesMeta(Elements): +class BytesType(ElementsType): elem_type: SSZType = byte length: int -class BytesLike(ElementsBase, metaclass=BytesMeta): +class BytesLike(Elements, metaclass=BytesType): @classmethod - def extract_args(cls, args): - if isinstance(args, bytes): - return args - elif isinstance(args, BytesLike): - return args.items - elif isinstance(args, GeneratorType): - return bytes(args) + def extract_args(cls, *args): + x = list(args) + if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes, BytesLike)): + x = x[0] + if isinstance(x, bytes): + return x + elif isinstance(x, BytesLike): + return x.items + elif isinstance(x, GeneratorType): + return bytes(x) else: - return bytes(args) + return bytes(x) @classmethod def value_check(cls, value): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py new file mode 100644 index 0000000000..8dd04a86d1 --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -0,0 +1,118 @@ +from .ssz_impl import serialize, serialize_basic, encode_series, signing_root, hash_tree_root +from .ssz_typing import ( + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Bit, Container, List, Vector, Bytes, BytesN, + uint, uint8, uint16, uint32, uint64, uint128, uint256, byte +) + +import pytest + + +class EmptyTestStruct(Container): + pass + + +class SingleFieldTestStruct(Container): + A: byte + + +class SmallTestStruct(Container): + A: uint16 + B: uint16 + + +class FixedTestStruct(Container): + A: uint8 + B: uint64 + C: uint32 + + +class VarTestStruct(Container): + A: uint16 + B: List[uint16, 1024] + C: uint8 + + +class ComplexTestStruct(Container): + A: uint16 + B: List[uint16, 128] + C: uint8 + D: Bytes[256] + E: VarTestStruct + F: Vector[FixedTestStruct, 4] + G: Vector[VarTestStruct, 2] + + +sig_test_data = [0 for i in range(96)] +for k, v in {0: 1, 32: 2, 64: 3, 95: 0xff}.items(): + sig_test_data[k] = v + +test_data = [ + ("bool F", Bit(False), "00"), + ("bool T", Bit(True), "01"), + ("uint8 00", uint8(0x00), "00"), + ("uint8 01", uint8(0x01), "01"), + ("uint8 ab", uint8(0xab), "ab"), + ("uint16 0000", uint16(0x0000), "0000"), + ("uint16 abcd", uint16(0xabcd), "cdab"), + ("uint32 00000000", uint32(0x00000000), "00000000"), + ("uint32 01234567", uint32(0x01234567), "67452301"), + ("small (4567, 0123)", SmallTestStruct(A=0x4567, B=0x0123), "67452301"), + ("small [4567, 0123]::2", Vector[uint16, 2](uint16(0x4567), uint16(0x0123)), "67452301"), + ("uint32 01234567", uint32(0x01234567), "67452301"), + ("uint64 0000000000000000", uint64(0x00000000), "0000000000000000"), + ("uint64 0123456789abcdef", uint64(0x0123456789abcdef), "efcdab8967452301"), + ("sig", BytesN[96](*sig_test_data), + "0100000000000000000000000000000000000000000000000000000000000000" + "0200000000000000000000000000000000000000000000000000000000000000" + "03000000000000000000000000000000000000000000000000000000000000ff"), + ("emptyTestStruct", EmptyTestStruct(), ""), + ("singleFieldTestStruct", SingleFieldTestStruct(A=0xab), "ab"), + ("fixedTestStruct", FixedTestStruct(A=0xab, B=0xaabbccdd00112233, C=0x12345678), "ab33221100ddccbbaa78563412"), + ("varTestStruct nil", VarTestStruct(A=0xabcd, C=0xff), "cdab07000000ff"), + ("varTestStruct empty", VarTestStruct(A=0xabcd, B=List[uint16, 1024](), C=0xff), "cdab07000000ff"), + ("varTestStruct some", VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), + "cdab07000000ff010002000300"), + ("complexTestStruct", + ComplexTestStruct( + A=0xaabb, + B=List[uint16, 128](0x1122, 0x3344), + C=0xff, + D=Bytes[256](b"foobar"), + E=VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), + F=Vector[FixedTestStruct, 4]( + FixedTestStruct(A=0xcc, B=0x4242424242424242, C=0x13371337), + FixedTestStruct(A=0xdd, B=0x3333333333333333, C=0xabcdabcd), + FixedTestStruct(A=0xee, B=0x4444444444444444, C=0x00112233), + FixedTestStruct(A=0xff, B=0x5555555555555555, C=0x44556677)), + G=Vector[VarTestStruct, 2]( + VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), + VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff)), + ), + "bbaa" + "47000000" # offset of B, []uint16 + "ff" + "4b000000" # offset of foobar + "51000000" # offset of E + "cc424242424242424237133713" + "dd3333333333333333cdabcdab" + "ee444444444444444433221100" + "ff555555555555555577665544" + "5e000000" # pointer to G + "22114433" # contents of B + "666f6f626172" # foobar + "cdab07000000ff010002000300" # contents of E + "08000000" "15000000" # [start G]: local offsets of [2]varTestStruct + "cdab07000000ff010002000300" + "cdab07000000ff010002000300", + ) +] + + +@pytest.mark.parametrize("name, value, serialized", test_data) +def test_serialize(name, value, serialized): + assert serialize(value) == bytes.fromhex(serialized) + + +@pytest.mark.parametrize("name, value, _", test_data) +def test_hash_tree_root(name, value, _): + hash_tree_root(value) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index a0705f8d36..e59d29b91b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,5 +1,5 @@ from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint, uint8, uint16, uint32, uint64, uint128, uint256 ) @@ -23,7 +23,8 @@ def test_subclasses(): assert not isinstance(c, BasicType) for c in [List, Vector, Bytes, BytesN]: - assert isinstance(c, Elements) + assert issubclass(c, Elements) + assert isinstance(c, ElementsType) def test_basic_instances(): @@ -109,7 +110,8 @@ def test_list(): assert issubclass(typ, List) assert issubclass(typ, SSZValue) assert issubclass(typ, Series) - assert isinstance(typ, Elements) + assert issubclass(typ, Elements) + assert isinstance(typ, ElementsType) assert not typ.is_fixed_size() @@ -128,4 +130,5 @@ def test_list(): assert isinstance(v, typ) assert isinstance(v, SSZValue) assert isinstance(v, Series) - assert isinstance(v.type(), Elements) + assert issubclass(v.type(), Elements) + assert isinstance(v.type(), ElementsType) From 0a43003b426b581de664ff27bc0e5e766a960b29 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 17 Jun 2019 02:11:50 +0200 Subject: [PATCH 083/405] minor test improvements --- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index e59d29b91b..99416e333c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -93,8 +93,9 @@ class Bar(Container): assert not Bar.is_fixed_size() - y = Bar(a=uint8(123), b=List[uint8, 1024](uint8(1), uint8(2))) + y = Bar(a=123, b=List[uint8, 1024](uint8(1), uint8(2))) assert y.a == 123 + assert isinstance(y.a, uint8) assert len(y.b) == 2 assert isinstance(y.a, uint8) assert isinstance(y.b, List[uint8, 1024]) @@ -119,6 +120,8 @@ def test_list(): assert len(typ(uint64(0))) == 1 # single arg assert len(typ(uint64(i) for i in range(10))) == 10 # generator assert len(typ(uint64(0), uint64(1), uint64(2))) == 3 # args + assert isinstance(typ(1, 2, 3, 4, 5)[4], uint64) # coercion + assert isinstance(typ(i for i in range(10))[9], uint64) # coercion in generator v = typ(uint64(0)) v[0] = uint64(123) From b89183ae69faf82a1eec798928da7c358d759621 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:05:34 +0200 Subject: [PATCH 084/405] Update spec for new SSZ with list max length --- specs/core/0_beacon-chain.md | 37 ++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ddc4c8f611..44c20ec848 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -224,6 +224,7 @@ These configurations are updated for releases, but may be out of sync during `de | `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | epochs | 25.6 minutes | | `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~1.7 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | +| `HISTORICAL_ROOTS_LENGTH` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | | `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | @@ -237,6 +238,12 @@ These configurations are updated for releases, but may be out of sync during `de | - | - | :-: | :-: | | `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | | `EPOCHS_PER_SLASHED_BALANCES_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | +| `RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `VALIDATOR_REGISTRY_SIZE` | `2**40 (= 1,099,511,627,776)` | | | + +* Assuming the maximum 16 deposits per slot, the validator registry will last for at least 13,065 years (but likely much much longer) ### Rewards and penalties @@ -357,8 +364,8 @@ class AttestationDataAndCustodyBit(Container): ```python class IndexedAttestation(Container): - custody_bit_0_indices: List[ValidatorIndex] # Indices with custody bit equal to 0 - custody_bit_1_indices: List[ValidatorIndex] # Indices with custody bit equal to 1 + custody_bit_0_indices: List[ValidatorIndex, MAX_INDICES_PER_ATTESTATION] # Indices with custody bit equal to 0 + custody_bit_1_indices: List[ValidatorIndex, MAX_INDICES_PER_ATTESTATION] # Indices with custody bit equal to 1 data: AttestationData signature: BLSSignature ``` @@ -513,14 +520,14 @@ class BeaconState(Container): latest_block_header: BeaconBlockHeader block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] - historical_roots: List[Hash] + historical_roots: List[Hash, HISTORICAL_ROOTS_LENGTH] # Eth1 eth1_data: Eth1Data - eth1_data_votes: List[Eth1Data] + eth1_data_votes: List[Eth1Data, SLOTS_PER_ETH1_VOTING_PERIOD] eth1_deposit_index: uint64 # Registry - validators: List[Validator] - balances: List[Gwei] + validators: List[Validator, VALIDATOR_REGISTRY_SIZE] + balances: List[Gwei, VALIDATOR_REGISTRY_SIZE] # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] @@ -528,8 +535,8 @@ class BeaconState(Container): # Slashings slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators # Attestations - previous_epoch_attestations: List[PendingAttestation] - current_epoch_attestations: List[PendingAttestation] + previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] + current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] # Crosslinks previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] @@ -984,8 +991,6 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify no index has custody bit equal to 1 [to be removed in phase 1] assert len(bit_1_indices) == 0 - # Verify max number of indices - assert len(bit_0_indices) + len(bit_1_indices) <= MAX_INDICES_PER_ATTESTATION # Verify index sets are disjoint assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 # Verify indices are sorted @@ -1624,6 +1629,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) +<<<<<<< HEAD all_operations = [ (body.proposer_slashings, MAX_PROPOSER_SLASHINGS, process_proposer_slashing), (body.attester_slashings, MAX_ATTESTER_SLASHINGS, process_attester_slashing), @@ -1634,6 +1640,17 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: ] # type: List[Tuple[List[Container], int, Callable]] for operations, max_operations, function in all_operations: assert len(operations) <= max_operations +======= + + for operations, max_operations, function in ( + (body.proposer_slashings, process_proposer_slashing), + (body.attester_slashings, process_attester_slashing), + (body.attestations, process_attestation), + (body.deposits, process_deposit), + (body.voluntary_exits, process_voluntary_exit), + (body.transfers, process_transfer), + ): +>>>>>>> f6a2345f... Update spec for new SSZ with list max length for operation in operations: function(state, operation) ``` From 4c2adcc5e643279c462ba3d25ce474ff4bb741f2 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 17 Jun 2019 19:07:20 -0400 Subject: [PATCH 085/405] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 44c20ec848..8f2078956b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -243,8 +243,6 @@ These configurations are updated for releases, but may be out of sync during `de | `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | | `VALIDATOR_REGISTRY_SIZE` | `2**40 (= 1,099,511,627,776)` | | | -* Assuming the maximum 16 deposits per slot, the validator registry will last for at least 13,065 years (but likely much much longer) - ### Rewards and penalties | Name | Value | @@ -991,6 +989,8 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify no index has custody bit equal to 1 [to be removed in phase 1] assert len(bit_1_indices) == 0 + # Verify max number of indices + assert len(bit_0_indices) + len(bit_1_indices) <= MAX_INDICES_PER_ATTESTATION # Verify index sets are disjoint assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 # Verify indices are sorted From 73ba419d640920a1427c357e487625ba70ae87b4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 02:04:10 +0200 Subject: [PATCH 086/405] check virtual lengths, fix imports --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7662971c48..efcb8f2078 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -268,7 +268,7 @@ def __init__(self, *args): @classmethod def value_check(cls, value): - return all(isinstance(v, cls.elem_type) for v in value) + return all(isinstance(v, cls.elem_type) for v in value) and len(value) <= cls.length @classmethod def extract_args(cls, *args): @@ -316,6 +316,7 @@ class Vector(Elements): @classmethod def value_check(cls, value): + # check length limit strictly return len(value) == cls.length and super().value_check(value) @classmethod @@ -350,7 +351,8 @@ def extract_args(cls, *args): @classmethod def value_check(cls, value): - return isinstance(value, bytes) + # check type and virtual length limit + return isinstance(value, bytes) and len(value) <= cls.length def __str__(self): cls = self.__class__ @@ -376,6 +378,7 @@ def default(cls): @classmethod def value_check(cls, value): + # check length limit strictly return len(value) == cls.length and super().value_check(value) @classmethod From 8c6ddd5233d51bc8937dc6a7e84d8b95f381fc80 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 02:18:00 +0200 Subject: [PATCH 087/405] container field coercion --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 12 +++++++++++- .../pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index efcb8f2078..942095622d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,4 @@ -from typing import Tuple, Dict, Iterator +from typing import Tuple, Iterator from types import GeneratorType @@ -142,6 +142,16 @@ def signing_root(self): from .ssz_impl import signing_root return signing_root(self) + def __setattr__(self, name, value): + if name not in self.__class__.__annotations__: + raise AttributeError("Cannot change non-existing SSZ-container attribute") + field_typ = self.__class__.__annotations__[name] + value = coerce_type_maybe(value, field_typ) + if not isinstance(value, field_typ): + raise ValueCheckError(f"Cannot set field of {self.__class__}:" + f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") + super().__setattr__(name, value) + def get_field_values(self) -> Tuple[SSZValue, ...]: cls = self.__class__ return tuple(getattr(self, field) for field in cls.get_field_names()) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 99416e333c..291c4b955b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -105,6 +105,20 @@ class Bar(Container): assert v.type().elem_type == uint8 assert v.type().length == 1024 + y.a = 42 + y.a = uint16(255) + try: + y.a = uint16(256) + assert False + except ValueError: + pass + + try: + y.not_here = 5 + assert False + except AttributeError: + pass + def test_list(): typ = List[uint64, 128] From 4aefc078e9e6cd9a11757681c4cb55abc2b908dc Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 02:54:08 +0200 Subject: [PATCH 088/405] list-rework type fixes --- specs/core/0_beacon-chain.md | 25 +++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 22 ++++++---- .../eth2spec/utils/ssz/test_ssz_typing.py | 43 ++++++++++++++++++- 3 files changed, 61 insertions(+), 29 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8f2078956b..a505fb4511 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -241,7 +241,7 @@ These configurations are updated for releases, but may be out of sync during `de | `RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | | `ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | | `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `VALIDATOR_REGISTRY_SIZE` | `2**40 (= 1,099,511,627,776)` | | | +| `VALIDATOR_REGISTRY_SIZE` | `2**40` (= 1,099,511,627,776) | | | ### Rewards and penalties @@ -372,7 +372,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] # Bit set for every attesting participant within a committee + aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -439,9 +439,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] + aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] data: AttestationData - custody_bitfield: Bytes[MAX_COMMITTEE_SIZE] + custody_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] signature: BLSSignature ``` @@ -1629,20 +1629,8 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) -<<<<<<< HEAD - all_operations = [ - (body.proposer_slashings, MAX_PROPOSER_SLASHINGS, process_proposer_slashing), - (body.attester_slashings, MAX_ATTESTER_SLASHINGS, process_attester_slashing), - (body.attestations, MAX_ATTESTATIONS, process_attestation), - (body.deposits, MAX_DEPOSITS, process_deposit), - (body.voluntary_exits, MAX_VOLUNTARY_EXITS, process_voluntary_exit), - (body.transfers, MAX_TRANSFERS, process_transfer), - ] # type: List[Tuple[List[Container], int, Callable]] - for operations, max_operations, function in all_operations: - assert len(operations) <= max_operations -======= - - for operations, max_operations, function in ( + + for operations, function in ( (body.proposer_slashings, process_proposer_slashing), (body.attester_slashings, process_attester_slashing), (body.attestations, process_attestation), @@ -1650,7 +1638,6 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), ): ->>>>>>> f6a2345f... Update spec for new SSZ with list max length for operation in operations: function(state, operation) ``` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 942095622d..5aadfae329 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -2,10 +2,6 @@ from types import GeneratorType -class ValueCheckError(Exception): - pass - - class DefaultingTypeMeta(type): def default(cls): raise Exception("Not implemented") @@ -97,7 +93,7 @@ def coerce_type_maybe(v, typ: SSZType): # shortcut if it's already the type we are looking for if v_typ == typ: return v - elif isinstance(v, int): + elif isinstance(v, int) and not isinstance(v, uint): # do not coerce from one uintX to another uintY return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) @@ -126,7 +122,7 @@ def __init__(self, **kwargs): else: value = coerce_type_maybe(kwargs[f], t) if not isinstance(value, t): - raise ValueCheckError(f"Bad input for class {self.__class__}:" + raise ValueError(f"Bad input for class {self.__class__}:" f" field: {f} type: {t} value: {value} value type: {type(value)}") setattr(self, f, value) @@ -148,7 +144,7 @@ def __setattr__(self, name, value): field_typ = self.__class__.__annotations__[name] value = coerce_type_maybe(value, field_typ) if not isinstance(value, field_typ): - raise ValueCheckError(f"Cannot set field of {self.__class__}:" + raise ValueError(f"Cannot set field of {self.__class__}:" f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") super().__setattr__(name, value) @@ -273,7 +269,7 @@ def __init__(self, *args): items = self.extract_args(*args) if not self.value_check(items): - raise ValueCheckError(f"Bad input for class {self.__class__}: {items}") + raise ValueError(f"Bad input for class {self.__class__}: {items}") self.items = items @classmethod @@ -296,6 +292,16 @@ def __getitem__(self, i) -> SSZValue: return self.items[i] def __setitem__(self, k, v): + if k < 0: + raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") + if k > len(self.items): + raise IndexError(f"cannot set item in type {self.__class__}" + f" at out of bounds index {k} (to {v}, bound: {len(self.items)})") + typ = self.__class__.elem_type + v = coerce_type_maybe(v, typ) + if not isinstance(v, typ): + raise ValueError(f"Cannot set item in type {self.__class__}," + f" mismatched element type: {v} of {type(v)}, expected {typ}") self.items[k] = v def __len__(self): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 291c4b955b..0f4a06c5fb 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -106,9 +106,14 @@ class Bar(Container): assert v.type().length == 1024 y.a = 42 - y.a = uint16(255) try: - y.a = uint16(256) + y.a = 256 # out of bounds + assert False + except ValueError: + pass + + try: + y.a = uint16(255) # within bounds, wrong type assert False except ValueError: pass @@ -149,3 +154,37 @@ def test_list(): assert isinstance(v, Series) assert issubclass(v.type(), Elements) assert isinstance(v.type(), ElementsType) + + foo = List[uint32, 128](0 for i in range(128)) + foo[0] = 123 + foo[1] = 654 + foo[127] = 222 + assert sum(foo) == 999 + try: + foo[3] = 2**32 # out of bounds + except ValueError: + pass + + try: + foo[3] = uint64(2**32 - 1) # within bounds, wrong type + assert False + except ValueError: + pass + + try: + foo[128] = 100 + assert False + except IndexError: + pass + + try: + foo[-1] = 100 # valid in normal python lists + assert False + except IndexError: + pass + + try: + foo[128] = 100 # out of bounds + assert False + except IndexError: + pass From 439e4d485938b07a9cc52b5e7745d052245569b4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:12:17 +0200 Subject: [PATCH 089/405] Build spec --- scripts/build_spec.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 3a1b22efd3..61613376fc 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -26,8 +26,7 @@ ) from eth2spec.utils.ssz.ssz_typing import ( # unused: uint8, uint16, uint32, uint128, uint256, - uint64, Container, Vector, - Bytes4, Bytes32, Bytes48, Bytes96, + Bit, Container, List, Vector, Bytes, BytesN, uint64 ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -179,8 +178,9 @@ def dependency_order_ssz_objects(objects: Dict[str, str], custom_types: Dict[str """ items = list(objects.items()) for key, value in items: - dependencies = re.findall(r'(: [A-Z][\w[]*)', value) - dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|Bytes\d+|bytes', '', x), dependencies) + dependencies = re.findall(r'(: [A-Z][\w\[]*)', value) + dependencies = filter(lambda x: '_' not in x and x.upper() != x, dependencies) # filter out constants + dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|BytesN\[.+\\]|Bytes\[.+\\]|Bytes\d*|bytes\d*', '', x), dependencies) for dep in dependencies: if dep in custom_types or len(dep) == 0: continue From c9747b634f1cca8994af1b97207769b222b4165e Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:13:55 +0200 Subject: [PATCH 090/405] improve build spec, get clean dependencies list --- scripts/build_spec.py | 48 ++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 61613376fc..ca648a754c 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,12 +12,7 @@ PHASE0_IMPORTS = '''from typing import ( - Any, - Callable, - Dict, - List, - Set, - Tuple, + Any, Callable, Iterable, Dict, Set, Tuple ) from eth2spec.utils.ssz.ssz_impl import ( @@ -33,18 +28,14 @@ bls_verify, bls_verify_multiple, ) -# Note: 'int' type defaults to being interpreted as a uint64 by SSZ implementation. from eth2spec.utils.hash_function import hash + + +Deltas = list ''' PHASE1_IMPORTS = '''from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Set, - Tuple, + Any, Callable, Dict, Optional, Set, Tuple, Iterable ) from eth2spec.utils.ssz.ssz_impl import ( @@ -54,8 +45,7 @@ is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - # unused: uint8, uint16, uint32, uint128, uint256, - uint64, Container, Vector, + Bit, Container, List, Vector, Bytes, BytesN, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -65,8 +55,10 @@ ) from eth2spec.utils.hash_function import hash + + +Deltas = list ''' -BYTE_TYPES = [4, 32, 48, 96] SUNDRY_FUNCTIONS = ''' def get_ssz_type_by_name(name: str) -> Container: return globals()[name] @@ -172,18 +164,32 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st return old_constants +ignored_dependencies = [ + 'Bit', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' + 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', + 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', + 'bytes' # to be removed after updating spec doc +] + + def dependency_order_ssz_objects(objects: Dict[str, str], custom_types: Dict[str, str]) -> None: """ Determines which SSZ Object is depenedent on which other and orders them appropriately """ items = list(objects.items()) for key, value in items: - dependencies = re.findall(r'(: [A-Z][\w\[]*)', value) + dependencies = [] + for line in value.split('\n'): + if not re.match(r'\s+\w+: .+', line): + continue # skip whitespace etc. + line = line[line.index(':') + 1:] # strip of field name + if '#' in line: + line = line[:line.index('#')] # strip of comment + dependencies.extend(re.findall(r'(\w+)', line)) # catch all legible words, potential dependencies dependencies = filter(lambda x: '_' not in x and x.upper() != x, dependencies) # filter out constants - dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|BytesN\[.+\\]|Bytes\[.+\\]|Bytes\d*|bytes\d*', '', x), dependencies) + dependencies = filter(lambda x: x not in ignored_dependencies, dependencies) + dependencies = filter(lambda x: x not in custom_types, dependencies) for dep in dependencies: - if dep in custom_types or len(dep) == 0: - continue key_list = list(objects.keys()) for item in [dep, key] + key_list[key_list.index(dep)+1:]: objects[item] = objects.pop(item) From 8344d50ae5ceaa20998eda8f8f063dc1f53e7f03 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:15:48 +0200 Subject: [PATCH 091/405] update beacon chain doc, use new types, avoid List --- specs/core/0_beacon-chain.md | 64 +++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a505fb4511..7cbb9b67bd 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -291,6 +291,8 @@ We define the following Python custom types for type hinting and readability: | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | +`Deltas` is a non-SSZ type, a series of changes applied to balances, optimized by clients. + ## Containers The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containers. @@ -315,7 +317,7 @@ class Validator(Container): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers effective_balance: Gwei # Balance at stake - slashed: bool + slashed: Bit # Status epochs activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch @@ -355,7 +357,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): data: AttestationData - custody_bit: bool # Challengeable bit for the custody of crosslink data + custody_bit: Bit # Challengeable bit for the custody of crosslink data ``` #### `IndexedAttestation` @@ -649,11 +651,11 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: ### `get_active_validator_indices` ```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex]: +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Tuple[ValidatorIndex, ...]: """ Get active validator indices at ``epoch``. """ - return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] + return tuple(ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)) ``` ### `increase_balance` @@ -815,7 +817,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: ### `verify_merkle_branch` ```python -def verify_merkle_branch(leaf: Hash, proof: List[Hash], depth: int, index: int, root: Hash) -> bool: +def verify_merkle_branch(leaf: Hash, proof: Tuple[Hash, ...], depth: int, index: int, root: Hash) -> bool: """ Verify that the given ``leaf`` is on the merkle branch ``proof`` starting with the given ``root``. @@ -859,16 +861,17 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Hash) -> V ### `compute_committee` ```python -def compute_committee(indices: List[ValidatorIndex], seed: Hash, index: int, count: int) -> List[ValidatorIndex]: +def compute_committee(indices: Tuple[ValidatorIndex, ...], + seed: Hash, index: int, count: int) -> Tuple[ValidatorIndex, ...]: start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return [indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] + return tuple(indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)) ``` ### `get_crosslink_committee` ```python -def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> List[ValidatorIndex]: +def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[ValidatorIndex, ...]: return compute_committee( indices=get_active_validator_indices(state, epoch), seed=generate_seed(state, epoch), @@ -882,13 +885,13 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> L ```python def get_attesting_indices(state: BeaconState, attestation_data: AttestationData, - bitfield: bytes) -> List[ValidatorIndex]: + bitfield: bytes) -> Tuple[ValidatorIndex, ...]: """ Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. """ committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) - return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) + return tuple(sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1])) ``` ### `int_to_bytes` @@ -908,7 +911,7 @@ def bytes_to_int(data: bytes) -> int: ### `get_total_balance` ```python -def get_total_balance(state: BeaconState, indices: List[ValidatorIndex]) -> Gwei: +def get_total_balance(state: BeaconState, indices: Iterable[ValidatorIndex]) -> Gwei: """ Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ @@ -1134,7 +1137,7 @@ def slash_validator(state: BeaconState, ### Genesis trigger -Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool` where: +Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool` where: * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log @@ -1151,7 +1154,7 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: *Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: ```python -def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: +def is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool: # Process deposits state = BeaconState() for deposit in deposits: @@ -1173,10 +1176,10 @@ def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_data)`. ```python -def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis_eth1_data: Eth1Data) -> BeaconState: +def get_genesis_beacon_state(deposits: Iterable[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: state = BeaconState( genesis_time=genesis_time, - eth1_data=genesis_eth1_data, + eth1_data=eth1_data, latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) @@ -1270,21 +1273,21 @@ def get_total_active_balance(state: BeaconState) -> Gwei: ``` ```python -def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: +def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: assert epoch in (get_current_epoch(state), get_previous_epoch(state)) return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` ```python -def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: - return [ +def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: + return [ a for a in get_matching_source_attestations(state, epoch) if a.data.target_root == get_block_root(state, epoch) ] ``` ```python -def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: +def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) if a.data.beacon_block_root == get_block_root_at_slot(state, get_attestation_data_slot(state, a.data)) @@ -1293,7 +1296,7 @@ def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> List[Pen ```python def get_unslashed_attesting_indices(state: BeaconState, - attestations: List[PendingAttestation]) -> List[ValidatorIndex]: + attestations: Iterable[PendingAttestation]) -> Iterable[ValidatorIndex]: output = set() # type: Set[ValidatorIndex] for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) @@ -1301,14 +1304,14 @@ def get_unslashed_attesting_indices(state: BeaconState, ``` ```python -def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: +def get_attesting_balance(state: BeaconState, attestations: Iterable[PendingAttestation]) -> Gwei: return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) ``` ```python def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, - shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]: + shard: Shard) -> Tuple[Crosslink, Iterable[ValidatorIndex]]: attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] crosslinks = list(filter( lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_root, hash_tree_root(c)), @@ -1397,11 +1400,11 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ``` ```python -def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: +def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - rewards = [Gwei(0) for _ in range(len(state.validators))] - penalties = [Gwei(0) for _ in range(len(state.validators))] + rewards = Deltas(0 for _ in range(len(state.validators))) + penalties = Deltas(0 for _ in range(len(state.validators))) eligible_validator_indices = [ ValidatorIndex(index) for index, v in enumerate(state.validators) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) @@ -1448,9 +1451,9 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: ``` ```python -def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - rewards = [Gwei(0) for index in range(len(state.validators))] - penalties = [Gwei(0) for index in range(len(state.validators))] +def get_crosslink_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: + rewards = Deltas(0 for _ in range(len(state.validators))) + penalties = Deltas(0 for _ in range(len(state.validators))) epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) @@ -1630,14 +1633,15 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) - for operations, function in ( + all_operations = ( (body.proposer_slashings, process_proposer_slashing), (body.attester_slashings, process_attester_slashing), (body.attestations, process_attestation), (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), - ): + ) # type: Tuple[Tuple[List, Callable], ...] + for operations, function in all_operations: for operation in operations: function(state, operation) ``` From 4b4bf87e476769d56227bdd3dc2486a6886947fd Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 00:55:01 +0200 Subject: [PATCH 092/405] update shard doc, use new types, avoid List --- specs/core/1_shard-data-chains.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index b83cd54f7b..79e2478fef 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -154,7 +154,7 @@ def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex ```python def get_persistent_committee(state: BeaconState, shard: Shard, - slot: Slot) -> List[ValidatorIndex]: + slot: Slot) -> Tuple[ValidatorIndex, ...]: """ Return the persistent committee for the given ``shard`` at the given ``slot``. """ @@ -175,10 +175,10 @@ def get_persistent_committee(state: BeaconState, # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated - return sorted(list(set( + return tuple(sorted(list(set( [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(state, epoch, i)] + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(state, epoch, i)] - ))) + )))) ``` ### `get_shard_proposer_index` @@ -284,7 +284,7 @@ Let: ```python def is_valid_shard_block(beacon_blocks: List[BeaconBlock], beacon_state: BeaconState, - valid_shard_blocks: List[ShardBlock], + valid_shard_blocks: Iterable[ShardBlock], candidate: ShardBlock) -> bool: # Check if block is already determined valid for _, block in enumerate(valid_shard_blocks): @@ -348,7 +348,7 @@ Let: * `candidate` be a candidate `ShardAttestation` for which validity is to be determined by running `is_valid_shard_attestation` ```python -def is_valid_shard_attestation(valid_shard_blocks: List[ShardBlock], +def is_valid_shard_attestation(valid_shard_blocks: Iterable[ShardBlock], beacon_state: BeaconState, candidate: ShardAttestation) -> bool: # Check shard block @@ -380,7 +380,7 @@ Let: def is_valid_beacon_attestation(shard: Shard, shard_blocks: List[ShardBlock], beacon_state: BeaconState, - valid_attestations: List[Attestation], + valid_attestations: Set[Attestation], candidate: Attestation) -> bool: # Check if attestation is already determined valid for _, attestation in enumerate(valid_attestations): From 5be0c57aade56b95c0d8b8b2d54d437cdc14ddb4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 01:03:04 +0200 Subject: [PATCH 093/405] fix linting + mypy --- scripts/build_spec.py | 6 +++--- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 13 +++++++++--- .../eth2spec/utils/ssz/test_ssz_impl.py | 6 +++--- .../eth2spec/utils/ssz/test_ssz_typing.py | 21 ++++++++++--------- 5 files changed, 28 insertions(+), 20 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index ca648a754c..20e90ee339 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -20,8 +20,8 @@ signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - # unused: uint8, uint16, uint32, uint128, uint256, - Bit, Container, List, Vector, Bytes, BytesN, uint64 + Bit, Container, List, Vector, Bytes, uint64, + Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -45,7 +45,7 @@ is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Container, List, Vector, Bytes, BytesN, uint64, + Bit, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 1b59b276bb..fd17e29f94 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint ) # SSZ Serialization diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 5aadfae329..082e3ed30f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -123,7 +123,7 @@ def __init__(self, **kwargs): value = coerce_type_maybe(kwargs[f], t) if not isinstance(value, t): raise ValueError(f"Bad input for class {self.__class__}:" - f" field: {f} type: {t} value: {value} value type: {type(value)}") + f" field: {f} type: {t} value: {value} value type: {type(value)}") setattr(self, f, value) def serialize(self): @@ -145,7 +145,7 @@ def __setattr__(self, name, value): value = coerce_type_maybe(value, field_typ) if not isinstance(value, field_typ): raise ValueError(f"Cannot set field of {self.__class__}:" - f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") + f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") super().__setattr__(name, value) def get_field_values(self) -> Tuple[SSZValue, ...]: @@ -301,7 +301,7 @@ def __setitem__(self, k, v): v = coerce_type_maybe(v, typ) if not isinstance(v, typ): raise ValueError(f"Cannot set item in type {self.__class__}," - f" mismatched element type: {v} of {type(v)}, expected {typ}") + f" mismatched element type: {v} of {type(v)}, expected {typ}") self.items[k] = v def __len__(self): @@ -400,3 +400,10 @@ def value_check(cls, value): @classmethod def is_fixed_size(cls): return True + + +# Helpers for common BytesN types. +Bytes4 = BytesN[4] +Bytes32 = BytesN[32] +Bytes48 = BytesN[48] +Bytes96 = BytesN[96] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 8dd04a86d1..ae0849098d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,7 +1,7 @@ -from .ssz_impl import serialize, serialize_basic, encode_series, signing_root, hash_tree_root +from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Bit, Container, List, Vector, Bytes, BytesN, - uint, uint8, uint16, uint32, uint64, uint128, uint256, byte + Bit, Container, List, Vector, Bytes, BytesN, + uint8, uint16, uint32, uint64, byte ) import pytest diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 0f4a06c5fb..f604b6468e 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,5 +1,6 @@ from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, + Elements, Bit, Container, List, Vector, Bytes, BytesN, uint, uint8, uint16, uint32, uint64, uint128, uint256 ) @@ -41,13 +42,13 @@ def test_basic_instances(): def test_basic_value_bounds(): max = { - Bit: 2**1, - uint8: 2**(8 * 1), - uint16: 2**(8 * 2), - uint32: 2**(8 * 4), - uint64: 2**(8 * 8), - uint128: 2**(8 * 16), - uint256: 2**(8 * 32), + Bit: 2 ** 1, + uint8: 2 ** (8 * 1), + uint16: 2 ** (8 * 2), + uint32: 2 ** (8 * 4), + uint64: 2 ** (8 * 8), + uint128: 2 ** (8 * 16), + uint256: 2 ** (8 * 32), } for k, v in max.items(): # this should work @@ -161,12 +162,12 @@ def test_list(): foo[127] = 222 assert sum(foo) == 999 try: - foo[3] = 2**32 # out of bounds + foo[3] = 2 ** 32 # out of bounds except ValueError: pass try: - foo[3] = uint64(2**32 - 1) # within bounds, wrong type + foo[3] = uint64(2 ** 32 - 1) # within bounds, wrong type assert False except ValueError: pass From 6f46c1d837646bd6906845253c13f2e6ee1d2d72 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 01:12:51 +0200 Subject: [PATCH 094/405] fix typing in spec builder monkey patch --- scripts/build_spec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 20e90ee339..7d42c218e1 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -66,13 +66,13 @@ def get_ssz_type_by_name(name: str) -> Container: # Monkey patch validator compute committee code _compute_committee = compute_committee -committee_cache: Dict[Tuple[Hash, Hash, int, int], List[ValidatorIndex]] = {} +committee_cache: Dict[Tuple[Hash, Hash, int, int], Tuple[ValidatorIndex, ...]] = {} -def compute_committee(indices: List[ValidatorIndex], # type: ignore +def compute_committee(indices: Tuple[ValidatorIndex, ...], # type: ignore seed: Hash, index: int, - count: int) -> List[ValidatorIndex]: + count: int) -> Tuple[ValidatorIndex, ...]: param_hash = (hash_tree_root(indices), seed, index, count) if param_hash not in committee_cache: From 6b82e3faa56707b0c1fd11cd82e2f8eb36037370 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:20:07 +0200 Subject: [PATCH 095/405] Modifications from Vitalik, to enable SSZ Partials --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 54 ++++++++----------- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index fd17e29f94..a9c36649b5 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -107,7 +107,7 @@ def is_bottom_layer_kind(typ: SSZType): def item_length(typ: SSZType) -> int: - if issubclass(typ, BasicType): + if issubclass(typ, BasicValue): return typ.byte_len else: return 32 diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 082e3ed30f..981f30d9bc 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -31,6 +31,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): class Bit(BasicValue): # can't subclass bool. + byte_len = 1 def __new__(cls, value, *args, **kwargs): if value < 0 or value > 1: @@ -88,7 +89,7 @@ class uint256(uint): byte_len = 32 -def coerce_type_maybe(v, typ: SSZType): +def coerce_type_maybe(v, typ: SSZType, strict: bool = False): v_typ = type(v) # shortcut if it's already the type we are looking for if v_typ == typ: @@ -97,10 +98,14 @@ def coerce_type_maybe(v, typ: SSZType): return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) + elif isinstance(v, bytes): + return typ(v) elif isinstance(v, GeneratorType): return typ(v) else: # just return as-is, Value-checkers will take care of it not being coerced. + if strict and not isinstance(v, typ): + raise ValueError("Type coercion of {} to {} failed".format(v, typ)) return v @@ -116,7 +121,7 @@ class Container(Series, metaclass=SSZType): def __init__(self, **kwargs): cls = self.__class__ - for f, t in cls.get_fields(): + for f, t in cls.get_fields().items(): if f not in kwargs: setattr(self, f, t.default()) else: @@ -148,16 +153,12 @@ def __setattr__(self, name, value): f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") super().__setattr__(name, value) - def get_field_values(self) -> Tuple[SSZValue, ...]: - cls = self.__class__ - return tuple(getattr(self, field) for field in cls.get_field_names()) - def __repr__(self): - return repr({field: getattr(self, field) for field in self.get_field_names()}) + return repr({field: getattr(self, field) for field in self.get_fields()}) def __str__(self): output = [f'{self.__class__.__name__}'] - for field in self.get_field_names(): + for field in self.get_fields(): output.append(f' {field}: {getattr(self, field)}') return "\n".join(output) @@ -168,23 +169,10 @@ def __hash__(self): return hash(self.hash_tree_root()) @classmethod - def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: - if not hasattr(cls, '__annotations__'): # no container fields - return () - return tuple((f, t) for f, t in cls.__annotations__.items()) - - @classmethod - def get_field_names(cls) -> Tuple[str, ...]: + def get_fields(cls) -> Dict[str, SSZType]: if not hasattr(cls, '__annotations__'): # no container fields - return () - return tuple(cls.__annotations__.keys()) - - @classmethod - def get_field_types(cls) -> Tuple[SSZType, ...]: - if not hasattr(cls, '__annotations__'): # no container fields - return () - # values of annotations are the types corresponding to the fields, not instance values. - return tuple(cls.__annotations__.values()) + return {} + return dict(cls.__annotations__) @classmethod def default(cls): @@ -195,7 +183,7 @@ def is_fixed_size(cls): return all(t.is_fixed_size() for t in cls.get_field_types()) def __iter__(self) -> Iterator[SSZValue]: - return iter(self.get_field_values()) + return iter([getattr(self, field) for field in self.get_fields()]) class ParamsBase(Series): @@ -297,12 +285,16 @@ def __setitem__(self, k, v): if k > len(self.items): raise IndexError(f"cannot set item in type {self.__class__}" f" at out of bounds index {k} (to {v}, bound: {len(self.items)})") - typ = self.__class__.elem_type - v = coerce_type_maybe(v, typ) - if not isinstance(v, typ): - raise ValueError(f"Cannot set item in type {self.__class__}," - f" mismatched element type: {v} of {type(v)}, expected {typ}") - self.items[k] = v + self.items[k] = coerce_type_maybe(v, self.__class__.elem_type, strict=True) + + def append(self, v): + self.items.append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) + + def pop(self): + if len(self.items) == 0: + raise IndexError("Pop from empty list") + else: + return self.items.pop() def __len__(self): return len(self.items) From 5048b9e87a1f260c4c307100e489407adcade2ac Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 02:14:13 +0200 Subject: [PATCH 096/405] temporary fix for phase-1 spec typing --- scripts/build_spec.py | 3 ++- specs/core/1_custody-game.md | 34 +++++++++++++++++++------------ specs/core/1_shard-data-chains.md | 24 ++++++++++++++-------- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 7d42c218e1..612edbd00a 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -35,7 +35,8 @@ Deltas = list ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Tuple, Iterable + Any, Callable, Dict, Optional, Set, Tuple, Iterable, + List as TypingList ) from eth2spec.utils.ssz.ssz_impl import ( diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 3fe132c07a..99c85a0342 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -113,6 +113,13 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | - | - | | `DOMAIN_CUSTODY_BIT_CHALLENGE` | `6` | + +### TODO PLACEHOLDER + +| Name | Value | +| - | - | +| `PLACEHOLDER` | `2**32` | + ## Data structures ### Custody objects @@ -134,7 +141,7 @@ class CustodyBitChallenge(Container): attestation: Attestation challenger_index: ValidatorIndex responder_key: BLSSignature - chunk_bits: bytes + chunk_bits: Bytes[PLACEHOLDER] signature: BLSSignature ``` @@ -171,9 +178,9 @@ class CustodyBitChallengeRecord(Container): class CustodyResponse(Container): challenge_index: uint64 chunk_index: uint64 - chunk: Vector[bytes, BYTES_PER_CUSTODY_CHUNK] - data_branch: List[Bytes32] - chunk_bits_branch: List[Bytes32] + chunk: Vector[Bytes[PLACEHOLDER], BYTES_PER_CUSTODY_CHUNK] + data_branch: List[Bytes32, PLACEHOLDER] + chunk_bits_branch: List[Bytes32, PLACEHOLDER] chunk_bits_leaf: Bytes32 ``` @@ -226,24 +233,25 @@ class Validator(Container): ```python class BeaconState(Container): - custody_chunk_challenge_records: List[CustodyChunkChallengeRecord] - custody_bit_challenge_records: List[CustodyBitChallengeRecord] + custody_chunk_challenge_records: List[CustodyChunkChallengeRecord, PLACEHOLDER] + custody_bit_challenge_records: List[CustodyBitChallengeRecord, PLACEHOLDER] custody_challenge_index: uint64 # Future derived secrets already exposed; contains the indices of the exposed validator # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS - exposed_derived_secrets: Vector[List[ValidatorIndex], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] + exposed_derived_secrets: Vector[List[ValidatorIndex, PLACEHOLDER], + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] ``` #### `BeaconBlockBody` ```python class BeaconBlockBody(Container): - custody_chunk_challenges: List[CustodyChunkChallenge] - custody_bit_challenges: List[CustodyBitChallenge] - custody_responses: List[CustodyResponse] - custody_key_reveals: List[CustodyKeyReveal] - early_derived_secret_reveals: List[EarlyDerivedSecretReveal] + custody_chunk_challenges: List[CustodyChunkChallenge, PLACEHOLDER] + custody_bit_challenges: List[CustodyBitChallenge, PLACEHOLDER] + custody_responses: List[CustodyResponse, PLACEHOLDER] + custody_key_reveals: List[CustodyKeyReveal, PLACEHOLDER] + early_derived_secret_reveals: List[EarlyDerivedSecretReveal, PLACEHOLDER] ``` ## Helpers @@ -310,7 +318,7 @@ def get_validators_custody_reveal_period(state: BeaconState, ### `replace_empty_or_append` ```python -def replace_empty_or_append(list: List[Any], new_element: Any) -> int: +def replace_empty_or_append(list: TypingList[Any], new_element: Any) -> int: for i in range(len(list)): if is_empty(list[i]): list[i] = new_element diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 79e2478fef..bdf6550ff6 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -70,13 +70,19 @@ This document describes the shard data layer and the shard fork choice rule in P | `DOMAIN_SHARD_PROPOSER` | `128` | | `DOMAIN_SHARD_ATTESTER` | `129` | +### TODO PLACEHOLDER + +| Name | Value | +| - | - | +| `PLACEHOLDER` | `2**32` | + ## Data structures ### `ShardBlockBody` ```python class ShardBlockBody(Container): - data: Vector[bytes, BYTES_PER_SHARD_BLOCK_BODY] + data: Vector[Bytes[PLACEHOLDER], BYTES_PER_SHARD_BLOCK_BODY] ``` ### `ShardAttestation` @@ -87,7 +93,7 @@ class ShardAttestation(Container): slot: Slot shard: Shard shard_block_root: Bytes32 - aggregation_bitfield: bytes + aggregation_bitfield: Bytes[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -101,7 +107,7 @@ class ShardBlock(Container): parent_root: Bytes32 data: ShardBlockBody state_root: Bytes32 - attestations: List[ShardAttestation] + attestations: List[ShardAttestation, PLACEHOLDER] signature: BLSSignature ``` @@ -115,7 +121,7 @@ class ShardBlockHeader(Container): parent_root: Bytes32 body_root: Bytes32 state_root: Bytes32 - attestations: List[ShardAttestation] + attestations: List[ShardAttestation, PLACEHOLDER] signature: BLSSignature ``` @@ -128,7 +134,7 @@ def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard, index: int, - count: int) -> List[ValidatorIndex]: + count: int) -> Tuple[ValidatorIndex, ...]: """ Return committee for a period. Used to construct persistent committees. """ @@ -243,11 +249,11 @@ def verify_shard_attestation_signature(state: BeaconState, ### `compute_crosslink_data_root` ```python -def compute_crosslink_data_root(blocks: List[ShardBlock]) -> Bytes32: +def compute_crosslink_data_root(blocks: Iterable[ShardBlock]) -> Bytes32: def is_power_of_two(value: int) -> bool: return (value > 0) and (value & (value - 1) == 0) - def pad_to_power_of_2(values: List[bytes]) -> List[bytes]: + def pad_to_power_of_2(values: TypingList[bytes]) -> TypingList[bytes]: while not is_power_of_two(len(values)): values += [b'\x00' * BYTES_PER_SHARD_BLOCK_BODY] return values @@ -282,7 +288,7 @@ Let: * `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block` ```python -def is_valid_shard_block(beacon_blocks: List[BeaconBlock], +def is_valid_shard_block(beacon_blocks: TypingList[BeaconBlock], beacon_state: BeaconState, valid_shard_blocks: Iterable[ShardBlock], candidate: ShardBlock) -> bool: @@ -378,7 +384,7 @@ Let: ```python def is_valid_beacon_attestation(shard: Shard, - shard_blocks: List[ShardBlock], + shard_blocks: TypingList[ShardBlock], beacon_state: BeaconState, valid_attestations: Set[Attestation], candidate: Attestation) -> bool: From a33c67894ebd9593fea94d38fd6914a3725ffe04 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 02:37:22 +0200 Subject: [PATCH 097/405] update ssz testing/debug utils --- test_libs/pyspec/eth2spec/debug/decode.py | 44 +++---- test_libs/pyspec/eth2spec/debug/encode.py | 34 +++--- .../pyspec/eth2spec/debug/random_value.py | 112 ++++++++---------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 5 +- 4 files changed, 84 insertions(+), 111 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index 5ce1160258..743479371d 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,39 +1,29 @@ +from typing import Any from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - is_uint_type, is_bool_type, is_list_type, - is_vector_type, is_bytes_type, is_bytesn_type, is_container_type, - read_vector_elem_type, read_list_elem_type, + SSZType, SSZValue, uint, Container, Bytes, List, Bit, Vector, BytesN ) -def decode(data, typ): - if is_uint_type(typ): - return data - elif is_bool_type(typ): - assert data in (True, False) - return data - elif is_list_type(typ): - elem_typ = read_list_elem_type(typ) - return [decode(element, elem_typ) for element in data] - elif is_vector_type(typ): - elem_typ = read_vector_elem_type(typ) - return Vector(decode(element, elem_typ) for element in data) - elif is_bytes_type(typ): - return bytes.fromhex(data[2:]) - elif is_bytesn_type(typ): - return BytesN(bytes.fromhex(data[2:])) - elif is_container_type(typ): +def decode(data: Any, typ: SSZType) -> SSZValue: + if issubclass(typ, (uint, Bit)): + return typ(data) + elif issubclass(typ, (List, Vector)): + return typ(decode(element, typ.elem_type) for element in data) + elif issubclass(typ, (Bytes, BytesN)): + return typ(bytes.fromhex(data[2:])) + elif issubclass(typ, Container): temp = {} - for field, subtype in typ.get_fields(): - temp[field] = decode(data[field], subtype) - if field + "_hash_tree_root" in data: - assert(data[field + "_hash_tree_root"][2:] == - hash_tree_root(temp[field], subtype).hex()) + for field_name, field_type in typ.get_fields().items(): + temp[field_name] = decode(data[field_name], field_type) + if field_name + "_hash_tree_root" in data: + assert (data[field_name + "_hash_tree_root"][2:] == + hash_tree_root(temp[field_name]).hex()) ret = typ(**temp) if "hash_tree_root" in data: - assert(data["hash_tree_root"][2:] == - hash_tree_root(ret, typ).hex()) + assert (data["hash_tree_root"][2:] == + hash_tree_root(ret).hex()) return ret else: raise Exception(f"Type not recognized: data={data}, typ={typ}") diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 61dd87928b..d264bd7ff1 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,36 +1,30 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - is_uint_type, is_bool_type, is_list_type, is_vector_type, is_container_type, - read_elem_type, - uint + SSZValue, uint, Container, Bytes, BytesN, List, Vector, Bit ) -def encode(value, typ, include_hash_tree_roots=False): - if is_uint_type(typ): - if hasattr(typ, '__supertype__'): - typ = typ.__supertype__ +def encode(value: SSZValue, include_hash_tree_roots=False): + if isinstance(value, uint): # Larger uints are boxed and the class declares their byte length - if issubclass(typ, uint) and typ.byte_len > 8: + if value.type().byte_len > 8: return str(value) return value - elif is_bool_type(typ): + elif isinstance(value, Bit): assert value in (True, False) return value - elif is_list_type(typ) or is_vector_type(typ): - elem_typ = read_elem_type(typ) - return [encode(element, elem_typ, include_hash_tree_roots) for element in value] - elif isinstance(typ, type) and issubclass(typ, bytes): # both bytes and BytesN + elif isinstance(value, (List, Vector)): + return [encode(element, include_hash_tree_roots) for element in value] + elif isinstance(value, (Bytes, BytesN)): # both bytes and BytesN return '0x' + value.hex() - elif is_container_type(typ): + elif isinstance(value, Container): ret = {} - for field, subtype in typ.get_fields(): - field_value = getattr(value, field) - ret[field] = encode(field_value, subtype, include_hash_tree_roots) + for field_value, field_name in zip(value, value.get_fields().keys()): + ret[field_name] = encode(field_value, include_hash_tree_roots) if include_hash_tree_roots: - ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(field_value, subtype).hex() + ret[field_name + "_hash_tree_root"] = '0x' + hash_tree_root(field_value).hex() if include_hash_tree_roots: - ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() + ret["hash_tree_root"] = '0x' + hash_tree_root(value).hex() return ret else: - raise Exception(f"Type not recognized: value={value}, typ={typ}") + raise Exception(f"Type not recognized: value={value}, typ={value.type()}") diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 3edcc88084..bd40cb8320 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -1,18 +1,13 @@ from random import Random -from typing import Any from enum import Enum -from eth2spec.utils.ssz.ssz_impl import is_basic_type - from eth2spec.utils.ssz.ssz_typing import ( - is_uint_type, is_bool_type, is_list_type, - is_vector_type, is_bytes_type, is_bytesn_type, is_container_type, - read_vector_elem_type, read_list_elem_type, - uint_byte_size + SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bit, + Vector, BytesN ) # in bytes -UINT_SIZES = (1, 2, 4, 8, 16, 32) +UINT_BYTE_SIZES = (1, 2, 4, 8, 16, 32) random_mode_names = ("random", "zero", "max", "nil", "one", "lengthy") @@ -39,11 +34,11 @@ def is_changing(self): def get_random_ssz_object(rng: Random, - typ: Any, + typ: SSZType, max_bytes_length: int, max_list_length: int, mode: RandomizationMode, - chaos: bool) -> Any: + chaos: bool) -> SSZValue: """ Create an object for a given type, filled with random data. :param rng: The random number generator to use. @@ -56,33 +51,31 @@ def get_random_ssz_object(rng: Random, """ if chaos: mode = rng.choice(list(RandomizationMode)) - if is_bytes_type(typ): + if issubclass(typ, Bytes): # Bytes array if mode == RandomizationMode.mode_nil_count: - return b'' + return typ(b'') elif mode == RandomizationMode.mode_max_count: - return get_random_bytes_list(rng, max_bytes_length) + return typ(get_random_bytes_list(rng, max_bytes_length)) elif mode == RandomizationMode.mode_one_count: - return get_random_bytes_list(rng, 1) + return typ(get_random_bytes_list(rng, 1)) elif mode == RandomizationMode.mode_zero: - return b'\x00' + return typ(b'\x00') elif mode == RandomizationMode.mode_max: - return b'\xff' + return typ(b'\xff') else: - return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) - elif is_bytesn_type(typ): - # BytesN - length = typ.length + return typ(get_random_bytes_list(rng, rng.randint(0, max_bytes_length))) + elif issubclass(typ, BytesN): # Sanity, don't generate absurdly big random values # If a client is aiming to performance-test, they should create a benchmark suite. - assert length <= max_bytes_length + assert typ.length <= max_bytes_length if mode == RandomizationMode.mode_zero: - return b'\x00' * length + return typ(b'\x00' * typ.length) elif mode == RandomizationMode.mode_max: - return b'\xff' * length + return typ(b'\xff' * typ.length) else: - return get_random_bytes_list(rng, length) - elif is_basic_type(typ): + return typ(get_random_bytes_list(rng, typ.length)) + elif issubclass(typ, BasicValue): # Basic types if mode == RandomizationMode.mode_zero: return get_min_basic_value(typ) @@ -90,32 +83,28 @@ def get_random_ssz_object(rng: Random, return get_max_basic_value(typ) else: return get_random_basic_value(rng, typ) - elif is_vector_type(typ): - # Vector - elem_typ = read_vector_elem_type(typ) - return [ - get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) + elif issubclass(typ, Vector): + return typ( + get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(typ.length) - ] - elif is_list_type(typ): - # List - elem_typ = read_list_elem_type(typ) - length = rng.randint(0, max_list_length) + ) + elif issubclass(typ, List): + length = rng.randint(0, min(typ.length, max_list_length)) if mode == RandomizationMode.mode_one_count: length = 1 elif mode == RandomizationMode.mode_max_count: length = max_list_length - return [ - get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) + return typ( + get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(length) - ] - elif is_container_type(typ): + ) + elif issubclass(typ, Container): # Container return typ(**{ - field: - get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) - for field, subtype in typ.get_fields() + field_name: + get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) + for field_name, field_type in typ.get_fields().items() }) else: raise Exception(f"Type not recognized: typ={typ}") @@ -125,34 +114,31 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: return bytes(rng.getrandbits(8) for _ in range(length)) -def get_random_basic_value(rng: Random, typ) -> Any: - if is_bool_type(typ): - return rng.choice((True, False)) - elif is_uint_type(typ): - size = uint_byte_size(typ) - assert size in UINT_SIZES - return rng.randint(0, 256**size - 1) +def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: + if issubclass(typ, Bit): + return typ(rng.choice((True, False))) + elif issubclass(typ, uint): + assert typ.byte_len in UINT_BYTE_SIZES + return typ(rng.randint(0, 256 ** typ.byte_len - 1)) else: raise ValueError(f"Not a basic type: typ={typ}") -def get_min_basic_value(typ) -> Any: - if is_bool_type(typ): - return False - elif is_uint_type(typ): - size = uint_byte_size(typ) - assert size in UINT_SIZES - return 0 +def get_min_basic_value(typ: BasicType) -> BasicValue: + if issubclass(typ, Bit): + return typ(False) + elif issubclass(typ, uint): + assert typ.byte_len in UINT_BYTE_SIZES + return typ(0) else: raise ValueError(f"Not a basic type: typ={typ}") -def get_max_basic_value(typ) -> Any: - if is_bool_type(typ): - return True - elif is_uint_type(typ): - size = uint_byte_size(typ) - assert size in UINT_SIZES - return 256**size - 1 +def get_max_basic_value(typ: BasicType) -> BasicValue: + if issubclass(typ, Bit): + return typ(True) + elif issubclass(typ, uint): + assert typ.byte_len in UINT_BYTE_SIZES + return typ(256 ** typ.byte_len - 1) else: raise ValueError(f"Not a basic type: typ={typ}") diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 981f30d9bc..ea4d85e829 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -364,7 +364,10 @@ def value_check(cls, value): def __str__(self): cls = self.__class__ - return f"{cls.__name__}[{cls.length}]: {self.items.hex()}" + return f"{cls.__name__}[{cls.length}]: {self.hex()}" + + def hex(self) -> str: + return self.items.hex() class Bytes(BytesLike): From 7cdec746b47bacff0e2d4352cb4838dd6d28182b Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 02:41:43 +0200 Subject: [PATCH 098/405] fix field iteration crash in ssz typing --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ea4d85e829..51a7908536 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -154,11 +154,11 @@ def __setattr__(self, name, value): super().__setattr__(name, value) def __repr__(self): - return repr({field: getattr(self, field) for field in self.get_fields()}) + return repr({field: getattr(self, field) for field in self.get_fields().keys()}) def __str__(self): output = [f'{self.__class__.__name__}'] - for field in self.get_fields(): + for field in self.get_fields().keys(): output.append(f' {field}: {getattr(self, field)}') return "\n".join(output) @@ -176,7 +176,7 @@ def get_fields(cls) -> Dict[str, SSZType]: @classmethod def default(cls): - return cls(**{f: t.default() for f, t in cls.get_fields()}) + return cls(**{f: t.default() for f, t in cls.get_fields().items()}) @classmethod def is_fixed_size(cls): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index f604b6468e..6bb56f4e5d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -76,6 +76,10 @@ class Foo(Container): a: uint8 b: uint32 + empty = Foo() + assert empty.a == uint8(0) + assert empty.b == uint32(0) + assert issubclass(Foo, Container) assert issubclass(Foo, SSZValue) assert issubclass(Foo, Series) From 4e747fb8879540012655e534e9d8fd214e47be87 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:25:22 +0200 Subject: [PATCH 099/405] fixes for class based ssz typing --- scripts/build_spec.py | 24 ++++++------- specs/core/0_beacon-chain.md | 8 +++-- .../pyspec/eth2spec/test/helpers/custody.py | 4 +-- .../pyspec/eth2spec/utils/merkle_minimal.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 5 +-- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 36 ++++++++++--------- .../eth2spec/utils/ssz/test_ssz_typing.py | 20 ++++++++++- 7 files changed, 62 insertions(+), 37 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 612edbd00a..d33ba66424 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -65,6 +65,17 @@ def get_ssz_type_by_name(name: str) -> Container: return globals()[name] +# Monkey patch hash cache +_hash = hash +hash_cache: Dict[bytes, Hash] = {} + + +def hash(x: bytes) -> Hash: + if x not in hash_cache: + hash_cache[x] = Hash(_hash(x)) + return hash_cache[x] + + # Monkey patch validator compute committee code _compute_committee = compute_committee committee_cache: Dict[Tuple[Hash, Hash, int, int], Tuple[ValidatorIndex, ...]] = {} @@ -74,24 +85,13 @@ def compute_committee(indices: Tuple[ValidatorIndex, ...], # type: ignore seed: Hash, index: int, count: int) -> Tuple[ValidatorIndex, ...]: - param_hash = (hash_tree_root(indices), seed, index, count) + param_hash = (hash(b''.join(index.to_bytes(length=4, byteorder='little') for index in indices)), seed, index, count) if param_hash not in committee_cache: committee_cache[param_hash] = _compute_committee(indices, seed, index, count) return committee_cache[param_hash] -# Monkey patch hash cache -_hash = hash -hash_cache: Dict[bytes, Hash] = {} - - -def hash(x: bytes) -> Hash: - if x not in hash_cache: - hash_cache[x] = Hash(_hash(x)) - return hash_cache[x] - - # Access to overwrite spec constants based on configuration def apply_constants_preset(preset: Dict[str, Any]) -> None: global_vars = globals() diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7cbb9b67bd..9a02c16e47 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -651,11 +651,13 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: ### `get_active_validator_indices` ```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Tuple[ValidatorIndex, ...]: +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]: """ Get active validator indices at ``epoch``. """ - return tuple(ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)) + return List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + i for i, v in enumerate(state.validators) if is_active_validator(v, epoch) + ) ``` ### `increase_balance` @@ -873,7 +875,7 @@ def compute_committee(indices: Tuple[ValidatorIndex, ...], ```python def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[ValidatorIndex, ...]: return compute_committee( - indices=get_active_validator_indices(state, epoch), + indices=tuple(get_active_validator_indices(state, epoch)), seed=generate_seed(state, epoch), index=(shard + SHARD_COUNT - get_epoch_start_shard(state, epoch)) % SHARD_COUNT, count=get_epoch_committee_count(state, epoch), diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 67df12fcdb..b49a6be1f2 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -11,7 +11,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): epoch = current_epoch + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING reveal = bls_sign( - message_hash=spec.hash_tree_root(epoch), + message_hash=spec.hash_tree_root(spec.Epoch(epoch)), privkey=privkeys[revealed_index], domain=spec.get_domain( state=state, @@ -20,7 +20,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): ), ) mask = bls_sign( - message_hash=spec.hash_tree_root(epoch), + message_hash=spec.hash_tree_root(spec.Epoch(epoch)), privkey=privkeys[masker_index], domain=spec.get_domain( state=state, diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 21583ee921..038b555cf8 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -4,7 +4,7 @@ ZERO_BYTES32 = b'\x00' * 32 zerohashes = [ZERO_BYTES32] -for layer in range(1, 32): +for layer in range(1, 100): zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1])) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index a9c36649b5..4b64c91624 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint, ) # SSZ Serialization @@ -143,5 +143,6 @@ def hash_tree_root(obj: SSZValue): def signing_root(obj: Container): # ignore last field - leaves = [hash_tree_root(field) for field in obj[:-1]] + fields = [field for field in obj][:-1] + leaves = [hash_tree_root(f) for f in fields] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 51a7908536..381dadf9e1 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -98,7 +98,7 @@ def coerce_type_maybe(v, typ: SSZType, strict: bool = False): return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) - elif isinstance(v, bytes): + elif isinstance(v, (bytes, BytesN, Bytes)): return typ(v) elif isinstance(v, GeneratorType): return typ(v) @@ -154,7 +154,8 @@ def __setattr__(self, name, value): super().__setattr__(name, value) def __repr__(self): - return repr({field: getattr(self, field) for field in self.get_fields().keys()}) + return repr({field: (getattr(self, field) if hasattr(self, field) else 'unset') + for field in self.get_fields().keys()}) def __str__(self): output = [f'{self.__class__.__name__}'] @@ -236,15 +237,24 @@ def attr_from_params(self, p): raise TypeError("provided parameters {} mismatch required parameter count {}".format(params, i)) return res - def __instancecheck__(self, obj): - if obj.__class__.__name__ != self.__name__: + def __subclasscheck__(self, subclass): + # check regular class system if we can, solves a lot of the normal cases. + if super().__subclasscheck__(subclass): + return True + # if they are not normal subclasses, they are of the same class. + # then they should have the same name + if subclass.__name__ != self.__name__: return False + # If they do have the same name, they should also have the same params. for name, typ in self.__annotations__.items(): - if hasattr(self, name) and hasattr(obj.__class__, name) \ - and getattr(obj.__class__, name) != getattr(self, name): + if hasattr(self, name) and hasattr(subclass, name) \ + and getattr(subclass, name) != getattr(self, name): return False return True + def __instancecheck__(self, obj): + return self.__subclasscheck__(obj.__class__) + class ElementsType(ParamsMeta): elem_type: SSZType @@ -305,9 +315,6 @@ def __repr__(self): def __iter__(self) -> Iterator[SSZValue]: return iter(self.items) - def __eq__(self, other): - return self.items == other.items - class List(Elements): @@ -366,9 +373,6 @@ def __str__(self): cls = self.__class__ return f"{cls.__name__}[{cls.length}]: {self.hex()}" - def hex(self) -> str: - return self.items.hex() - class Bytes(BytesLike): @@ -398,7 +402,7 @@ def is_fixed_size(cls): # Helpers for common BytesN types. -Bytes4 = BytesN[4] -Bytes32 = BytesN[32] -Bytes48 = BytesN[48] -Bytes96 = BytesN[96] +Bytes4: BytesType = BytesN[4] +Bytes32: BytesType = BytesN[32] +Bytes48: BytesType = BytesN[48] +Bytes96: BytesType = BytesN[96] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 6bb56f4e5d..daa923aa73 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,7 +1,8 @@ from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, - uint, uint8, uint16, uint32, uint64, uint128, uint256 + uint, uint8, uint16, uint32, uint64, uint128, uint256, + Bytes32, Bytes48 ) @@ -193,3 +194,20 @@ def test_list(): assert False except IndexError: pass + + +def test_bytesn_subclass(): + assert isinstance(BytesN[32](b'\xab' * 32), Bytes32) + assert not isinstance(BytesN[32](b'\xab' * 32), Bytes48) + assert issubclass(BytesN[32](b'\xab' * 32).type(), Bytes32) + assert issubclass(BytesN[32], Bytes32) + + class Hash(Bytes32): + pass + + assert isinstance(Hash(b'\xab' * 32), Bytes32) + assert not isinstance(Hash(b'\xab' * 32), Bytes48) + assert issubclass(Hash(b'\xab' * 32).type(), Bytes32) + assert issubclass(Hash, Bytes32) + + assert not issubclass(Bytes48, Bytes32) From 977856b06fa324440f87cf912b6f250ce60a3982 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:30:42 +0200 Subject: [PATCH 100/405] ssz typing now subclasses list/bytes, much easier to work with than wrapped list/bytes functionality --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 11 ++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 90 +++++++++++-------- .../eth2spec/utils/ssz/test_ssz_typing.py | 2 + 3 files changed, 58 insertions(+), 45 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 4b64c91624..144201d837 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, uint, ) # SSZ Serialization @@ -46,9 +46,8 @@ def serialize(obj: SSZValue): def encode_series(values: Series): - # bytes and bytesN are already in the right format. - if isinstance(values, (Bytes, BytesN)): - return values.items + if isinstance(values, bytes): # Bytes and BytesN are already like serialized output + return values # Recursively serialize parts = [(v.type().is_fixed_size(), serialize(v)) for v in values] @@ -84,8 +83,8 @@ def encode_series(values: Series): def pack(values: Series): - if isinstance(values, (Bytes, BytesN)): - return values.items + if isinstance(values, bytes): # Bytes and BytesN are already packed + return values return b''.join([serialize_basic(value) for value in values]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 381dadf9e1..341df880ae 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -188,10 +188,10 @@ def __iter__(self) -> Iterator[SSZValue]: class ParamsBase(Series): - _bare = True + _has_params = False def __new__(cls, *args, **kwargs): - if cls._bare: + if not cls._has_params: raise Exception("cannot init bare type without params") return super().__new__(cls, **kwargs) @@ -200,13 +200,13 @@ class ParamsMeta(SSZType): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) - for k, v in attrs.items(): - setattr(out, k, v) + if hasattr(out, "_has_params") and getattr(out, "_has_params"): + for k, v in attrs.items(): + setattr(out, k, v) return out def __getitem__(self, params): o = self.__class__(self.__name__, (self,), self.attr_from_params(params)) - o._bare = False return o def __str__(self): @@ -218,7 +218,7 @@ def __repr__(self): def attr_from_params(self, p): # single key params are valid too. Wrap them in a tuple. params = p if isinstance(p, tuple) else (p,) - res = {} + res = {'_has_params': True} i = 0 for (name, typ) in self.__annotations__.items(): if hasattr(self.__class__, name): @@ -262,13 +262,17 @@ class ElementsType(ParamsMeta): class Elements(ParamsBase, metaclass=ElementsType): + pass + + +class BaseList(list, Elements): def __init__(self, *args): items = self.extract_args(*args) if not self.value_check(items): raise ValueError(f"Bad input for class {self.__class__}: {items}") - self.items = items + super().__init__(items) @classmethod def value_check(cls, value): @@ -284,39 +288,32 @@ def extract_args(cls, *args): def __str__(self): cls = self.__class__ - return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self.items)})" + return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" def __getitem__(self, i) -> SSZValue: - return self.items[i] + if i < 0: + raise IndexError(f"cannot get item in type {self.__class__} at negative index {i}") + if i > len(self): + raise IndexError(f"cannot get item in type {self.__class__}" + f" at out of bounds index {i}") + return super().__getitem__(i) def __setitem__(self, k, v): if k < 0: raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") - if k > len(self.items): + if k > len(self): raise IndexError(f"cannot set item in type {self.__class__}" - f" at out of bounds index {k} (to {v}, bound: {len(self.items)})") - self.items[k] = coerce_type_maybe(v, self.__class__.elem_type, strict=True) + f" at out of bounds index {k} (to {v}, bound: {len(self)})") + super().__setitem__(k, coerce_type_maybe(v, self.__class__.elem_type, strict=True)) def append(self, v): - self.items.append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) - - def pop(self): - if len(self.items) == 0: - raise IndexError("Pop from empty list") - else: - return self.items.pop() - - def __len__(self): - return len(self.items) - - def __repr__(self): - return repr(self.items) + super().append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) def __iter__(self) -> Iterator[SSZValue]: - return iter(self.items) + return super().__iter__() -class List(Elements): +class List(BaseList): @classmethod def default(cls): @@ -327,7 +324,7 @@ def is_fixed_size(cls): return False -class Vector(Elements): +class Vector(BaseList): @classmethod def value_check(cls, value): @@ -342,27 +339,35 @@ def default(cls): def is_fixed_size(cls): return cls.elem_type.is_fixed_size() + def append(self, v): + raise Exception("cannot modify vector length") + + def pop(self, *args): + raise Exception("cannot modify vector length") + class BytesType(ElementsType): elem_type: SSZType = byte length: int -class BytesLike(Elements, metaclass=BytesType): +class BaseBytes(bytes, Elements, metaclass=BytesType): + + def __new__(cls, *args) -> "BaseBytes": + extracted_val = cls.extract_args(*args) + if not cls.value_check(extracted_val): + raise ValueError(f"Bad input for class {cls}: {extracted_val}") + return super().__new__(cls, extracted_val) @classmethod def extract_args(cls, *args): - x = list(args) - if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes, BytesLike)): + x = args + if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes)): x = x[0] - if isinstance(x, bytes): + if isinstance(x, bytes): # Includes BytesLike return x - elif isinstance(x, BytesLike): - return x.items - elif isinstance(x, GeneratorType): - return bytes(x) else: - return bytes(x) + return bytes(x) # E.g. GeneratorType put into bytes. @classmethod def value_check(cls, value): @@ -374,7 +379,7 @@ def __str__(self): return f"{cls.__name__}[{cls.length}]: {self.hex()}" -class Bytes(BytesLike): +class Bytes(BaseBytes): @classmethod def default(cls): @@ -385,7 +390,14 @@ def is_fixed_size(cls): return False -class BytesN(BytesLike): +class BytesN(BaseBytes): + + @classmethod + def extract_args(cls, *args): + if len(args) == 0: + return cls.default() + else: + return super().extract_args(*args) @classmethod def default(cls): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index daa923aa73..895a074a91 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -211,3 +211,5 @@ class Hash(Bytes32): assert issubclass(Hash, Bytes32) assert not issubclass(Bytes48, Bytes32) + + assert len(Bytes32() + Bytes48()) == 80 From 82240d8dbd0f083efb66c8c5b2388eb153d69977 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 18:01:57 +0200 Subject: [PATCH 101/405] fix vector default type --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 341df880ae..6595c966f9 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -333,7 +333,7 @@ def value_check(cls, value): @classmethod def default(cls): - return [cls.elem_type.default() for _ in range(cls.length)] + return cls(cls.elem_type.default() for _ in range(cls.length)) @classmethod def is_fixed_size(cls): From f157745248492114da19b14c45e8831493368853 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:38:42 +0200 Subject: [PATCH 102/405] resolve some remaining list-rework rebase details --- specs/core/0_beacon-chain.md | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9a02c16e47..9c871f5f2a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -535,8 +535,8 @@ class BeaconState(Container): # Slashings slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators # Attestations - previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] - current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] + previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] + current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] # Crosslinks previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] @@ -1282,7 +1282,7 @@ def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Iterab ```python def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: - return [ + return [ a for a in get_matching_source_attestations(state, epoch) if a.data.target_root == get_block_root(state, epoch) ] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6595c966f9..bd7fe09506 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,4 @@ -from typing import Tuple, Iterator +from typing import Dict, Iterator from types import GeneratorType @@ -40,7 +40,7 @@ def __new__(cls, value, *args, **kwargs): @classmethod def default(cls): - return cls(False) + return cls(0) def __bool__(self): return self > 0 @@ -103,7 +103,7 @@ def coerce_type_maybe(v, typ: SSZType, strict: bool = False): elif isinstance(v, GeneratorType): return typ(v) else: - # just return as-is, Value-checkers will take care of it not being coerced. + # just return as-is, Value-checkers will take care of it not being coerced, if we are not strict. if strict and not isinstance(v, typ): raise ValueError("Type coercion of {} to {} failed".format(v, typ)) return v @@ -181,10 +181,10 @@ def default(cls): @classmethod def is_fixed_size(cls): - return all(t.is_fixed_size() for t in cls.get_field_types()) + return all(t.is_fixed_size() for t in cls.get_fields().values()) def __iter__(self) -> Iterator[SSZValue]: - return iter([getattr(self, field) for field in self.get_fields()]) + return iter([getattr(self, field) for field in self.get_fields().keys()]) class ParamsBase(Series): From 5f8edd6b55d13a7eb91a8ea7950896bab7a165fa Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 20 Jun 2019 20:50:17 +0200 Subject: [PATCH 103/405] Genesis block store uses genesis time Co-Authored-By: Hsiao-Wei Wang --- specs/core/0_fork-choice.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 6b7e4bdbea..5a4a52db4c 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -82,7 +82,12 @@ class Store(object): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) - return Store(blocks={root: genesis_block}, states={root: genesis_state}, justified_root=root, finalized_root=root) + return Store( + blocks={root: genesis_block}, + states={root: genesis_state}, + time=genesis_state.genesis_time, + justified_root=root, finalized_root=root, + ) ``` #### `get_ancestor` From 224c98a094a5d537fe05377955d8f30868cb6ba7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:55:17 +0200 Subject: [PATCH 104/405] last() method, no negative index lookups --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- .../eth2spec/test/helpers/proposer_slashings.py | 2 +- .../pyspec/eth2spec/test/helpers/transfers.py | 2 +- .../block_processing/test_process_transfer.py | 14 +++++++------- .../test_process_voluntary_exit.py | 2 +- .../pyspec/eth2spec/test/sanity/test_blocks.py | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 ++++ 7 files changed, 18 insertions(+), 14 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index b49a6be1f2..681add457d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -4,7 +4,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): current_epoch = spec.get_current_epoch(state) - revealed_index = spec.get_active_validator_indices(state, current_epoch)[-1] + revealed_index = spec.get_active_validator_indices(state, current_epoch).last() masker_index = spec.get_active_validator_indices(state, current_epoch)[0] if epoch is None: diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index d5b7f7b7fd..7b6f713740 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -6,7 +6,7 @@ def get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=False): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] + validator_index = spec.get_active_validator_indices(state, current_epoch).last() privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] slot = state.slot diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index acc6a35c53..215cd6fcb5 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -9,7 +9,7 @@ def get_valid_transfer(spec, state, slot=None, sender_index=None, amount=None, f slot = state.slot current_epoch = spec.get_current_epoch(state) if sender_index is None: - sender_index = spec.get_active_validator_indices(state, current_epoch)[-1] + sender_index = spec.get_active_validator_indices(state, current_epoch).last() recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] transfer_pubkey = pubkeys[-1] transfer_privkey = privkeys[-1] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index e9d282b3a6..2bed94dc81 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -63,7 +63,7 @@ def test_success_withdrawable(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -73,7 +73,7 @@ def test_success_active_above_max_effective(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -94,7 +94,7 @@ def test_invalid_signature(spec, state): @with_all_phases @spec_state_test def test_active_but_transfer_past_effective_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=amount, fee=0, signed=True) @@ -115,7 +115,7 @@ def test_incorrect_slot(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance_for_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -128,7 +128,7 @@ def test_insufficient_balance_for_fee(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -141,7 +141,7 @@ def test_insufficient_balance(spec, state): @with_all_phases @spec_state_test def test_no_dust_sender(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() balance = state.balances[sender_index] transfer = get_valid_transfer( spec, @@ -161,7 +161,7 @@ def test_no_dust_sender(spec, state): @with_all_phases @spec_state_test def test_no_dust_recipient(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) state.balances[transfer.recipient] = 0 diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index 33cacc4e20..b9f7d60981 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -93,7 +93,7 @@ def test_success_exit_queue(spec, state): continue # exit an additional validator - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] + validator_index = spec.get_active_validator_indices(state, current_epoch).last() privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit( spec, diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index f8590005e1..c86394a0b1 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -269,7 +269,7 @@ def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices( state, spec.get_current_epoch(state) - )[-1] + ).last() # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -315,7 +315,7 @@ def test_voluntary_exit(spec, state): # overwrite default 0 to test # spec.MAX_TRANSFERS = 1 - # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() # amount = get_balance(state, sender_index) # transfer = get_valid_transfer(spec, state, state.slot + 1, sender_index, amount, signed=True) @@ -347,7 +347,7 @@ def test_voluntary_exit(spec, state): @spec_state_test def test_balance_driven_status_transitions(spec, state): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] + validator_index = spec.get_active_validator_indices(state, current_epoch).last() assert state.validators[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index bd7fe09506..e5f9f66a8c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -312,6 +312,10 @@ def append(self, v): def __iter__(self) -> Iterator[SSZValue]: return super().__iter__() + def last(self): + # be explict about getting the last item, for the non-python readers, and negative-index safety + return self[len(self)-1] + class List(BaseList): From c26fffc15481ca282c75307a122f2012f4090cd1 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 20 Jun 2019 20:55:28 +0200 Subject: [PATCH 105/405] Moves copy into SSZ container --- scripts/build_spec.py | 4 ---- specs/core/0_fork-choice.md | 5 +++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 ++++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 9232cf00b8..a16fa79ac3 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,8 +25,6 @@ field, ) -from copy import deepcopy - from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, @@ -60,8 +58,6 @@ field, ) -from copy import deepcopy - from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 5a4a52db4c..ba97398a22 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -86,7 +86,8 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: blocks={root: genesis_block}, states={root: genesis_state}, time=genesis_state.genesis_time, - justified_root=root, finalized_root=root, + justified_root=root, + finalized_root=root, ) ``` @@ -143,7 +144,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root # Check block slot against Unix time - pre_state = deepcopy(store.states[block.parent_root]) + pre_state = store.states[block.parent_root].copy() assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 64d50b5794..077f94b494 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,3 +1,4 @@ +import copy from types import GeneratorType from typing import ( List, @@ -151,6 +152,9 @@ def __eq__(self, other): def __hash__(self): return hash(self.hash_tree_root()) + def copy(self): + return copy.deepcopy(self) + @classmethod def get_fields_dict(cls): return dict(cls.__annotations__) From 8c6d2b42d885f14758f57958f922b66128b2cf8f Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:07:23 +0200 Subject: [PATCH 106/405] update ssz-pyssz decoder for fuzzing --- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 73 +++++++++----------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index a5d3dfd97a..a4ebb7e708 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -8,32 +8,31 @@ def translate_typ(typ) -> ssz.BaseSedes: :param typ: The spec type, a class. :return: The Py-SSZ equivalent. """ - if spec_ssz.is_container_type(typ): + if issubclass(typ, spec_ssz.Container): return ssz.Container( - [translate_typ(field_typ) for (field_name, field_typ) in typ.get_fields()]) - elif spec_ssz.is_bytesn_type(typ): + [translate_typ(field_typ) for field_name, field_typ in typ.get_fields().items()]) + elif issubclass(typ, spec_ssz.BytesN): return ssz.ByteVector(typ.length) - elif spec_ssz.is_bytes_type(typ): + elif issubclass(typ, spec_ssz.Bytes): return ssz.ByteList() - elif spec_ssz.is_vector_type(typ): - return ssz.Vector(translate_typ(spec_ssz.read_vector_elem_type(typ)), typ.length) - elif spec_ssz.is_list_type(typ): - return ssz.List(translate_typ(spec_ssz.read_list_elem_type(typ))) - elif spec_ssz.is_bool_type(typ): + elif issubclass(typ, spec_ssz.Vector): + return ssz.Vector(translate_typ(typ.elem_type), typ.length) + elif issubclass(typ, spec_ssz.List): + return ssz.List(translate_typ(typ.elem_type)) + elif issubclass(typ, spec_ssz.Bit): return ssz.boolean - elif spec_ssz.is_uint_type(typ): - size = spec_ssz.uint_byte_size(typ) - if size == 1: + elif issubclass(typ, spec_ssz.uint): + if typ.byte_len == 1: return ssz.uint8 - elif size == 2: + elif typ.byte_len == 2: return ssz.uint16 - elif size == 4: + elif typ.byte_len == 4: return ssz.uint32 - elif size == 8: + elif typ.byte_len == 8: return ssz.uint64 - elif size == 16: + elif typ.byte_len == 16: return ssz.uint128 - elif size == 32: + elif typ.byte_len == 32: return ssz.uint256 else: raise TypeError("invalid uint size") @@ -48,37 +47,33 @@ def translate_value(value, typ): :param typ: The type from the spec to translate into :return: the translated value """ - if spec_ssz.is_uint_type(typ): - size = spec_ssz.uint_byte_size(typ) - if size == 1: + if issubclass(typ, spec_ssz.uint): + if typ.byte_len == 1: return spec_ssz.uint8(value) - elif size == 2: + elif typ.byte_len == 2: return spec_ssz.uint16(value) - elif size == 4: + elif typ.byte_len == 4: return spec_ssz.uint32(value) - elif size == 8: - # uint64 is default (TODO this is changing soon) - return value - elif size == 16: + elif typ.byte_len == 8: + return spec_ssz.uint64(value) + elif typ.byte_len == 16: return spec_ssz.uint128(value) - elif size == 32: + elif typ.byte_len == 32: return spec_ssz.uint256(value) else: raise TypeError("invalid uint size") - elif spec_ssz.is_list_type(typ): - elem_typ = spec_ssz.read_elem_type(typ) - return [translate_value(elem, elem_typ) for elem in value] - elif spec_ssz.is_bool_type(typ): + elif issubclass(typ, spec_ssz.List): + return [translate_value(elem, typ.elem_type) for elem in value] + elif issubclass(typ, spec_ssz.Bit): return value - elif spec_ssz.is_vector_type(typ): - elem_typ = spec_ssz.read_elem_type(typ) - return typ(*(translate_value(elem, elem_typ) for elem in value)) - elif spec_ssz.is_bytesn_type(typ): + elif issubclass(typ, spec_ssz.Vector): + return typ(*(translate_value(elem, typ.elem_type) for elem in value)) + elif issubclass(typ, spec_ssz.BytesN): return typ(value) - elif spec_ssz.is_bytes_type(typ): + elif issubclass(typ, spec_ssz.Bytes): return value - elif spec_ssz.is_container_type(typ): - return typ(**{f_name: translate_value(f_val, f_typ) for (f_name, f_val, f_typ) - in zip(typ.get_field_names(), value, typ.get_field_types())}) + if issubclass(typ, spec_ssz.Container): + return typ(**{f_name: translate_value(f_val, f_typ) for (f_val, (f_name, f_typ)) + in zip(value, typ.get_fields().items())}) else: raise TypeError("Type not supported: {}".format(typ)) From 8bd204827bd5e2d94091ede1e9ff973b011a7678 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:08:34 +0200 Subject: [PATCH 107/405] improve type coercion; coerce between equal-length uint subclasses --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index e5f9f66a8c..55acde44bc 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -94,19 +94,24 @@ def coerce_type_maybe(v, typ: SSZType, strict: bool = False): # shortcut if it's already the type we are looking for if v_typ == typ: return v - elif isinstance(v, int) and not isinstance(v, uint): # do not coerce from one uintX to another uintY - return typ(v) + elif isinstance(v, int): + if isinstance(v, uint): # do not coerce from one uintX to another uintY + if issubclass(typ, uint) and v.type().byte_len == typ.byte_len: + return typ(v) + # revert to default behavior below if-else. (ValueError/bare) + else: + return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) elif isinstance(v, (bytes, BytesN, Bytes)): return typ(v) elif isinstance(v, GeneratorType): return typ(v) - else: - # just return as-is, Value-checkers will take care of it not being coerced, if we are not strict. - if strict and not isinstance(v, typ): - raise ValueError("Type coercion of {} to {} failed".format(v, typ)) - return v + + # just return as-is, Value-checkers will take care of it not being coerced, if we are not strict. + if strict and not isinstance(v, typ): + raise ValueError("Type coercion of {} to {} failed".format(v, typ)) + return v class Series(SSZValue): From b4ef672f87cb3d3afbd0a7244feabbb3b7158258 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:12:46 +0200 Subject: [PATCH 108/405] deal with deepcopy modifying vector length from 0 to full length during copy --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 55acde44bc..39fed9ac87 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -349,7 +349,12 @@ def is_fixed_size(cls): return cls.elem_type.is_fixed_size() def append(self, v): - raise Exception("cannot modify vector length") + # Deep-copy and other utils like to change the internals during work. + # Only complain if we had the right size. + if len(self) == self.__class__.length: + raise Exception("cannot modify vector length") + else: + super().append(v) def pop(self, *args): raise Exception("cannot modify vector length") From 2d6771707925558b54fc5fe6bbc8154198dca2fd Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:42:55 +0200 Subject: [PATCH 109/405] fix linting issues + make spec builder remove comments in container re-initialization part --- scripts/build_spec.py | 15 ++++++++++++++- specs/core/0_beacon-chain.md | 5 +++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index d33ba66424..338093e94c 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -106,6 +106,18 @@ def apply_constants_preset(preset: Dict[str, Any]) -> None: ''' +def strip_comments(raw: str) -> str: + comment_line_regex = re.compile('^\s+# ') + lines = raw.split('\n') + out = [] + for line in lines: + if not comment_line_regex.match(line): + if ' #' in line: + line = line[:line.index(' #')] + out.append(line) + return '\n'.join(out) + + def objects_to_spec(functions: Dict[str, str], custom_types: Dict[str, str], constants: Dict[str, str], @@ -133,7 +145,8 @@ def objects_to_spec(functions: Dict[str, str], ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) ssz_objects_reinitialization_spec = ( 'def init_SSZ_types() -> None:\n global_vars = globals()\n\n ' - + '\n\n '.join([re.sub(r'(?!\n\n)\n', r'\n ', value[:-1]) for value in ssz_objects.values()]) + + '\n\n '.join([strip_comments(re.sub(r'(?!\n\n)\n', r'\n ', value[:-1])) + for value in ssz_objects.values()]) + '\n\n' + '\n'.join(map(lambda x: ' global_vars[\'%s\'] = %s' % (x, x), ssz_objects.keys())) ) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9c871f5f2a..52d0872814 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -533,7 +533,7 @@ class BeaconState(Container): randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Digests of the active registry, for light clients # Slashings - slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators + slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the slashed effective balances # Attestations previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] @@ -1525,7 +1525,8 @@ def process_slashings(state: BeaconState) -> None: total_penalties = total_at_end - total_at_start for index, validator in enumerate(state.validators): - if validator.slashed and current_epoch == validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2: + if validator.slashed and current_epoch == ( + validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2): penalty = max( validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 39fed9ac87..c9516bebf2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -319,7 +319,7 @@ def __iter__(self) -> Iterator[SSZValue]: def last(self): # be explict about getting the last item, for the non-python readers, and negative-index safety - return self[len(self)-1] + return self[len(self) - 1] class List(BaseList): From 4dcfee2d2cd8ba978787eda8bf336f2756dc67a0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:45:17 +0200 Subject: [PATCH 110/405] remove unused spec-helper from spec builder --- scripts/build_spec.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 338093e94c..45cb0368f9 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -61,10 +61,6 @@ Deltas = list ''' SUNDRY_FUNCTIONS = ''' -def get_ssz_type_by_name(name: str) -> Container: - return globals()[name] - - # Monkey patch hash cache _hash = hash hash_cache: Dict[bytes, Hash] = {} From d8f470bb4a81ae5662f28d83daecc27769972ef3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:34:19 +0200 Subject: [PATCH 111/405] enable slicing of SSZ lists/vectors --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index c9516bebf2..22f76badaa 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -295,13 +295,14 @@ def __str__(self): cls = self.__class__ return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" - def __getitem__(self, i) -> SSZValue: - if i < 0: - raise IndexError(f"cannot get item in type {self.__class__} at negative index {i}") - if i > len(self): - raise IndexError(f"cannot get item in type {self.__class__}" - f" at out of bounds index {i}") - return super().__getitem__(i) + def __getitem__(self, k) -> SSZValue: + if isinstance(k, int): # check if we are just doing a lookup, and not slicing + if k < 0: + raise IndexError(f"cannot get item in type {self.__class__} at negative index {k}") + if k > len(self): + raise IndexError(f"cannot get item in type {self.__class__}" + f" at out of bounds index {k}") + return super().__getitem__(k) def __setitem__(self, k, v): if k < 0: From c5143ccefaddd9929bfb68401c6c00e1afa75e6d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 20 Jun 2019 14:48:10 -0600 Subject: [PATCH 112/405] modify fork choice to utilize epochs as first class citizens --- specs/core/0_fork-choice.md | 69 ++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 6b7e4bdbea..3bcb6b8894 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -63,17 +63,30 @@ class Target(object): root: Hash ``` +#### `RootSlot` + +```python +@dataclass +class RootSlot(object): + slot: Slot + root: Hash +``` + + + #### `Store` ```python @dataclass class Store(object): blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) - states: Dict[Hash, BeaconState] = field(default_factory=dict) + states: Dict[RootSlot, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) justified_root: Hash = ZERO_HASH + justified_epoch: Epoch = 0 finalized_root: Hash = ZERO_HASH + finalized_epoch: Epoch = 0 ``` #### `get_genesis_store` @@ -98,8 +111,8 @@ def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: ```python def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: - state = store.states[store.justified_root] - active_indices = get_active_validator_indices(state.validator_registry, slot_to_epoch(state.slot)) + state = store.states[RootSlot(store.justified_root, get_epoch_start_slot(store.justified_epoch)] + active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) return Gwei(sum( state.validator_registry[i].effective_balance for i in active_indices if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root @@ -112,8 +125,12 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice head = store.justified_root + justified_slot = get_epoch_start_slot(store.justified_epoch) while True: - children = [root for root in store.blocks.keys() if store.blocks[root].parent_root == head] + children = [ + root for root in store.blocks.keys() + if store.blocks[root].parent_root == head and store.blocks[root].slot > justified_slot + ] if len(children) == 0: return head # Sort by latest attesting balance with ties broken lexicographically @@ -137,30 +154,58 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root + # Check that block is later than the finalized epoch slot + assert blocks.slot > get_epoch_start_slot(store.finalized_epoch) # Check block slot against Unix time pre_state = deepcopy(store.states[block.parent_root]) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) - # Add new state to the store - store.states[signing_root(block)] = state - # Update justified block root - if state.current_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + # Add new state for this block to the store + store.states[RootSlot(signing_root(block), block.slot)] = state + # Update justified block root and epoch + previous_justified_epoch = store.justified_epoch + justified_root_slot = None + if state.current_justified_epoch > store.justified_epoch: store.justified_root = state.current_justified_root - elif state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + state.justified_epoch = state.current_justified_epoch + elif state.previous_justified_epoch > store.justified_epoch: store.justified_root = state.previous_justified_root - # Update finalized block root - if state.finalized_epoch > slot_to_epoch(store.blocks[store.finalized_root].slot): + state.justified_epoch = state.previous_justified_epoch + + # Store justified state + if previous_justified_epoch != store.justified_epoch: + justified_slot = get_epoch_start_slot(store.justified_epoch) + justified_root_slot = RootSlot(store.justified_root, justified_slot) + if justified_root_slot not in store.states: + base_state = store.states[RootSlot(store.justified_root, store.blocks[store.justified_root].slot)] + store.states[justified_root_slot] = process_slots(deepcopy(base_state), justified_slot) + + # Update finalized block root and epoch, and store finalized state + if state.finalized_epoch > state.finalized_epoch: store.finalized_root = state.finalized_root + store.finalized_epoch = state.finalized_epoch + finalized_slot = get_epoch_start_slot(store.finalized_epoch) + finalized_root_slot = RootSlot(store.finalized_root, finalized_slot) + if finalized_root_slot not in store.states: + base_state = store.states[RootSlot(store.finalized_root, store.blocks[store.finalized_root].slot)] + store.states[finalized_root_slot] = process_slots(deepcopy(base_state), finalized_slot) ``` #### `on_attestation` ```python def on_attestation(store: Store, attestation: Attestation) -> None: - state = store.states[get_head(store)] + # cannot calculate the current shuffling if have not seen the target + assert attestation.data.target_root in store.blocks + + # get state at the `target_root`/`target_epoch` to validate attestation and calculate the committees + state = store.states[(attestation.data.target_root, get_epoch_start_slot(attestation.data.target_epoch))] + indexed_attestation = convert_to_indexed(state, attestation) validate_indexed_attestation(state, indexed_attestation) + + # update latest targets for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) From 6338c5b88051dcefe515e424196bca708b44dd06 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:49:03 +0200 Subject: [PATCH 113/405] fix custody bug, needs review from Carl --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 681add457d..bc70c9fa85 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -27,7 +27,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): domain_type=spec.DOMAIN_RANDAO, message_epoch=epoch, ), - ) + )[:32] # TODO(Carl): mask is 32 bytes, and signature is 96? Correct to slice the first 32 out? return spec.EarlyDerivedSecretReveal( revealed_index=revealed_index, From f27c44b953789b49e533fe36f8be314073c9f844 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:49:34 +0200 Subject: [PATCH 114/405] fix deposit negative index fail --- .../test/phase_0/block_processing/test_process_deposit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 63b94c6385..a40c120fb9 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -183,7 +183,7 @@ def test_bad_merkle_proof(spec, state): deposit = prepare_state_and_deposit(spec, state, validator_index, amount) # mess up merkle branch - deposit.proof[-1] = spec.ZERO_HASH + deposit.proof[5] = spec.ZERO_HASH sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) From c20372409c543b52bf15e71bc1df02b7ccb37434 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:52:16 +0200 Subject: [PATCH 115/405] comment out old deposit test, re-introduced soon maybe, cc Justin --- .../block_processing/test_process_deposit.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index a40c120fb9..2f60c6be24 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -113,20 +113,21 @@ def test_invalid_withdrawal_credentials_top_up(spec, state): # inconsistent withdrawal credentials, in top-ups, are allowed! yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True) - -@with_all_phases -@spec_state_test -def test_wrong_index(spec, state): - validator_index = len(state.validators) - amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(spec, state, validator_index, amount) - - # mess up eth1_deposit_index - deposit.index = state.eth1_deposit_index + 1 - - sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) - - yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) +# Deposit data temporarily has no index field. +# SSZ typing catches that, and raises an exception. Test may be introduced back later +# @with_all_phases +# @spec_state_test +# def test_wrong_index(spec, state): +# validator_index = len(state.validators) +# amount = spec.MAX_EFFECTIVE_BALANCE +# deposit = prepare_state_and_deposit(spec, state, validator_index, amount) +# +# # mess up eth1_deposit_index +# deposit.index = state.eth1_deposit_index + 1 +# +# sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) +# +# yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) @with_all_phases From fb9a5f0bc5e2358fbb2a57f3324b83ca2f82ed07 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 20 Jun 2019 14:57:53 -0600 Subject: [PATCH 116/405] one more rootslot fix --- specs/core/0_fork-choice.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3bcb6b8894..668b207143 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -157,7 +157,8 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Check that block is later than the finalized epoch slot assert blocks.slot > get_epoch_start_slot(store.finalized_epoch) # Check block slot against Unix time - pre_state = deepcopy(store.states[block.parent_root]) + parent_block = store.blocks[block.parent_root] + pre_state = deepcopy(store.states[RootSlot(signing_root(parent_block), parent_block.slot)]) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) From 3d8466fd6e07dea808fe13b2719917867ffa28b5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 23:02:22 +0200 Subject: [PATCH 117/405] make Bit check not use "is", and remove duplicate line --- specs/core/0_beacon-chain.md | 2 +- specs/core/1_custody-game.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 52d0872814..f171bd4df4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -645,7 +645,7 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: """ Check if ``validator`` is slashable. """ - return validator.slashed is False and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) + return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) ``` ### `get_active_validator_indices` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 99c85a0342..24e9a19e2f 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -402,12 +402,11 @@ def process_early_derived_secret_reveal(state: BeaconState, """ revealed_validator = state.validators[reveal.revealed_index] - masker = state.validators[reveal.masker_index] derived_secret_location = reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS assert reveal.epoch >= get_current_epoch(state) + RANDAO_PENALTY_EPOCHS assert reveal.epoch < get_current_epoch(state) + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS - assert revealed_validator.slashed is False + assert not revealed_validator.slashed assert reveal.revealed_index not in state.exposed_derived_secrets[derived_secret_location] # Verify signature correctness From 6648b3c49e0cd5535c5a34519d33798cc461712d Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 21 Jun 2019 00:23:28 +0200 Subject: [PATCH 118/405] remove old deposits test, there is no deposit index in deposit data anymore --- .../block_processing/test_process_deposit.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 2f60c6be24..8b3d7b413e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -113,22 +113,6 @@ def test_invalid_withdrawal_credentials_top_up(spec, state): # inconsistent withdrawal credentials, in top-ups, are allowed! yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True) -# Deposit data temporarily has no index field. -# SSZ typing catches that, and raises an exception. Test may be introduced back later -# @with_all_phases -# @spec_state_test -# def test_wrong_index(spec, state): -# validator_index = len(state.validators) -# amount = spec.MAX_EFFECTIVE_BALANCE -# deposit = prepare_state_and_deposit(spec, state, validator_index, amount) -# -# # mess up eth1_deposit_index -# deposit.index = state.eth1_deposit_index + 1 -# -# sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) -# -# yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) - @with_all_phases @spec_state_test @@ -173,9 +157,6 @@ def test_wrong_deposit_for_deposit_count(spec, state): yield from run_deposit_processing(spec, state, deposit_2, index_2, valid=False) -# TODO: test invalid signature - - @with_all_phases @spec_state_test def test_bad_merkle_proof(spec, state): From e99c864ed18ef172adc5795f1777035ef517fc82 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 20 Jun 2019 17:17:12 -0600 Subject: [PATCH 119/405] Deltas = NewType('Deltas', TypingList[Gwei]) --- scripts/build_spec.py | 12 ++++-------- specs/core/0_beacon-chain.md | 8 ++++---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 45cb0368f9..bd7686ff5e 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,7 +12,8 @@ PHASE0_IMPORTS = '''from typing import ( - Any, Callable, Iterable, Dict, Set, Tuple + Any, Callable, Iterable, Dict, Set, Tuple, NewType, + List as TypingList, ) from eth2spec.utils.ssz.ssz_impl import ( @@ -30,12 +31,9 @@ ) from eth2spec.utils.hash_function import hash - - -Deltas = list ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Tuple, Iterable, + Any, Callable, Dict, Optional, Set, Tuple, Iterable, NewType, List as TypingList ) @@ -56,9 +54,6 @@ ) from eth2spec.utils.hash_function import hash - - -Deltas = list ''' SUNDRY_FUNCTIONS = ''' # Monkey patch hash cache @@ -149,6 +144,7 @@ def objects_to_spec(functions: Dict[str, str], spec = ( imports + '\n\n' + new_type_definitions + + '\n\n' + "Deltas = NewType('Deltas', TypingList[Gwei])" + '\n\n' + constants_spec + '\n\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f171bd4df4..e89f0baede 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1405,8 +1405,8 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - rewards = Deltas(0 for _ in range(len(state.validators))) - penalties = Deltas(0 for _ in range(len(state.validators))) + rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) + penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) eligible_validator_indices = [ ValidatorIndex(index) for index, v in enumerate(state.validators) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) @@ -1454,8 +1454,8 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: ```python def get_crosslink_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: - rewards = Deltas(0 for _ in range(len(state.validators))) - penalties = Deltas(0 for _ in range(len(state.validators))) + rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) + penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) From e88a96c45e2cd724f7a3949e78d4c4c5707d1d40 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 21 Jun 2019 12:13:22 +0200 Subject: [PATCH 120/405] Apply suggestions from @drjtwo's code review Co-Authored-By: Danny Ryan --- specs/core/0_fork-choice.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index ba97398a22..6ae7e11fab 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -47,10 +47,11 @@ The head block root associated with a `store` is defined as `get_head(store)`. A *Notes*: -1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. +1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. This is automatically handled by [UNIX time](https://en.wikipedia.org/wiki/Unix_time). 2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. 3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. 4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. +5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). ### Helpers From f90469ea25c3b28ec20881d1a4e17033a70a129a Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 21 Jun 2019 12:19:08 +0200 Subject: [PATCH 121/405] Move block timing assertion 1st --- specs/core/0_fork-choice.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 6ae7e11fab..eb7c1a579b 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -140,13 +140,14 @@ def on_tick(store: Store, time: int) -> None: ```python def on_block(store: Store, block: BeaconBlock) -> None: + # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. + assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root # Check block slot against Unix time pre_state = store.states[block.parent_root].copy() - assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state to the store From 0e59c6676a66bf1f0b40bf0eb76c3040da01656f Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 21 Jun 2019 13:00:42 +0200 Subject: [PATCH 122/405] Stop yielding from fork-choie tests --- specs/core/0_fork-choice.md | 4 ++-- .../pyspec/eth2spec/test/test_fork_choice.py | 16 +++++----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index eb7c1a579b..3d85cf5713 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -140,14 +140,14 @@ def on_tick(store: Store, time: int) -> None: ```python def on_block(store: Store, block: BeaconBlock) -> None: + # Make a copy of the state to avoid mutability issues + pre_state = store.states[block.parent_root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root - # Check block slot against Unix time - pre_state = store.states[block.parent_root].copy() # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state to the store diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index ab17282519..4bc7e8b0a0 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -2,7 +2,7 @@ from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root -from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.context import with_all_phases, with_state, bls_switch from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation @@ -10,10 +10,10 @@ @with_all_phases -@spec_state_test +@with_state +@bls_switch def test_basic(spec, state): state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - yield 'pre', state # Initialization store = spec.get_genesis_store(state) @@ -36,17 +36,14 @@ def test_basic(spec, state): spec.on_block(store, block) assert store.blocks[signing_root(block)] == block - yield 'blocks', blocks, List[spec.BeaconBlock] # TODO: add tests for justified_root and finalized_root - yield 'post', state @with_all_phases -@spec_state_test +@with_state +@bls_switch def test_on_attestation(spec, state): - yield 'pre', state - store = spec.get_genesis_store(state) time = 100 spec.on_tick(store, time) @@ -54,7 +51,6 @@ def test_on_attestation(spec, state): next_slot(spec, state) attestation = get_valid_attestation(spec, state, slot=1) - yield 'attestation', attestation indexed_attestation = spec.convert_to_indexed(state, attestation) spec.on_attestation(store, attestation) assert ( @@ -64,5 +60,3 @@ def test_on_attestation(spec, state): root=attestation.data.target_root, ) ) - - yield 'post', state From b46e047baa4fc06a0980904c25ef2137f51a06a2 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 21 Jun 2019 12:57:30 +0100 Subject: [PATCH 123/405] Minor simplification from #1198 --- specs/core/0_fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3d85cf5713..892c592e9f 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -106,7 +106,7 @@ def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: ```python def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: state = store.states[store.justified_root] - active_indices = get_active_validator_indices(state.validator_registry, slot_to_epoch(state.slot)) + active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) return Gwei(sum( state.validator_registry[i].effective_balance for i in active_indices if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root From acbccbc2a8eeed367167e22732c91987cdd06637 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 11:18:24 -0600 Subject: [PATCH 124/405] minor typo --- specs/core/0_fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 892c592e9f..25b33ab304 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -51,7 +51,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A 2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. 3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. 4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. -5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). +5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number of optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). ### Helpers From 1b66a1a2bd297d33f666deede5b403388197073a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 12:55:55 -0600 Subject: [PATCH 125/405] rework forkchoice to use Checkpoints --- specs/core/0_fork-choice.md | 106 +++++++++++++++--------------------- 1 file changed, 43 insertions(+), 63 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index ee318412df..1755c6df35 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -12,7 +12,7 @@ - [Time parameters](#time-parameters) - [Fork choice](#fork-choice) - [Helpers](#helpers) - - [`Target`](#target) + - [`Checkpoint`](#checkpoint) - [`Store`](#store) - [`get_genesis_store`](#get_genesis_store) - [`get_ancestor`](#get_ancestor) @@ -55,39 +55,27 @@ The head block root associated with a `store` is defined as `get_head(store)`. A ### Helpers -#### `Target` +#### `Checkpoint` ```python @dataclass -class Target(object): +class Checkpoint(object): epoch: Epoch root: Hash ``` -#### `RootSlot` - -```python -@dataclass -class RootSlot(object): - slot: Slot - root: Hash -``` - - - #### `Store` ```python @dataclass class Store(object): blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) - states: Dict[RootSlot, BeaconState] = field(default_factory=dict) - time: int = 0 - latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) - justified_root: Hash = ZERO_HASH - justified_epoch: Epoch = 0 - finalized_root: Hash = ZERO_HASH - finalized_epoch: Epoch = 0 + block_states: Dict[Hash, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + time: int + latest_targets: Dict[ValidatorIndex, Checkpoint] = field(default_factory=dict) + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint ``` #### `get_genesis_store` @@ -96,12 +84,15 @@ class Store(object): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) + justified_checkpoint = Checkpoint(GENESIS_EPOCH, root) + finalized_checkpoint = Checkpoint(GENESIS_EPOCH, root) return Store( blocks={root: genesis_block}, - states={root: genesis_state}, + block_states={root: genesis_state}, + checkpoint_states={justified_checkpoint: genesis_state.copy()}, time=genesis_state.genesis_time, - justified_root=root, - finalized_root=root, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, ) ``` @@ -118,7 +109,7 @@ def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: ```python def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: - state = store.states[RootSlot(store.justified_root, get_epoch_start_slot(store.justified_epoch)] + state = store.checkpoint_states[store.justified_checkpoint] active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) return Gwei(sum( state.validator_registry[i].effective_balance for i in active_indices @@ -131,8 +122,8 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: ```python def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice - head = store.justified_root - justified_slot = get_epoch_start_slot(store.justified_epoch) + head = store.justified_checkpoint.root + justified_slot = get_epoch_start_slot(store.justified_checkpoint.epoch) while True: children = [ root for root in store.blocks.keys() @@ -159,63 +150,52 @@ def on_tick(store: Store, time: int) -> None: def on_block(store: Store, block: BeaconBlock) -> None: # Make a copy of the state to avoid mutability issues parent_block = store.blocks[block.parent_root] - pre_state = store.states[RootSlot(signing_root(parent_block), parent_block.slot)].copy() + pre_state = store.block_states[parent_block.root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block - assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root + assert get_ancestor(store, signing_root(block), store.blocks[get_epoch_start_slot(store.finalized_checkpoint)].slot) == store.finalized_checkpoint.root # Check that block is later than the finalized epoch slot - assert blocks.slot > get_epoch_start_slot(store.finalized_epoch) + assert block.slot > get_epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state for this block to the store - store.states[RootSlot(signing_root(block), block.slot)] = state - # Update justified block root and epoch - previous_justified_epoch = store.justified_epoch - justified_root_slot = None - if state.current_justified_epoch > store.justified_epoch: - store.justified_root = state.current_justified_root - state.justified_epoch = state.current_justified_epoch + store.block_states[signing_root(block)] = state + + # Update justified checkpoint + if state.current_justified_epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = Checkpoint(state.current_justified_epoch, state.current_justified_root) elif state.previous_justified_epoch > store.justified_epoch: - store.justified_root = state.previous_justified_root - state.justified_epoch = state.previous_justified_epoch - - # Store justified state - if previous_justified_epoch != store.justified_epoch: - justified_slot = get_epoch_start_slot(store.justified_epoch) - justified_root_slot = RootSlot(store.justified_root, justified_slot) - if justified_root_slot not in store.states: - base_state = store.states[RootSlot(store.justified_root, store.blocks[store.justified_root].slot)] - store.states[justified_root_slot] = process_slots(deepcopy(base_state), justified_slot) - - # Update finalized block root and epoch, and store finalized state + store.justified_checkpoint = Checkpoint(state.previous_justified_epoch, state.previous_justified_root) + + # Update finalized checkpoint if state.finalized_epoch > state.finalized_epoch: - store.finalized_root = state.finalized_root - store.finalized_epoch = state.finalized_epoch - finalized_slot = get_epoch_start_slot(store.finalized_epoch) - finalized_root_slot = RootSlot(store.finalized_root, finalized_slot) - if finalized_root_slot not in store.states: - base_state = store.states[RootSlot(store.finalized_root, store.blocks[store.finalized_root].slot)] - store.states[finalized_root_slot] = process_slots(deepcopy(base_state), finalized_slot) + store.finalized_checkpoint = Checkpoint(state.finalized_epoch, state.finalized_root) ``` #### `on_attestation` ```python def on_attestation(store: Store, attestation: Attestation) -> None: - # cannot calculate the current shuffling if have not seen the target - assert attestation.data.target_root in store.blocks + target_checkpoint = Checkpoint(attestation.data.target_epoch, attestation.data, target_root) + + # Cannot calculate the current shuffling if have not seen the target + assert target_checkpoint.root in store.blocks - # get state at the `target_root`/`target_epoch` to validate attestation and calculate the committees - state = store.states[(attestation.data.target_root, get_epoch_start_slot(attestation.data.target_epoch))] + # Store target checkpoint state if not yet seen + if target_checkpoint not in store.checkpoint_states: + base_state = store.block_states[target_checkpoint.root].copy() + store.checkpoint_states[target_checkpoint] = process_slots(base_state, get_epoch_start_slot(target_checkpoint.epoch)) + # Get state at the `target_checkpoint` to validate attestation and calculate the committees + state = store.checkpoint_states[target_checkpoint] indexed_attestation = convert_to_indexed(state, attestation) validate_indexed_attestation(state, indexed_attestation) - # update latest targets + # Update latest targets for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: - if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) + if i not in store.latest_targets or target_checkpoint.epoch > store.latest_targets[i].epoch: + store.latest_targets[i] = target_checkpoint ``` From b7b2fee6350489ec4cd1c2e118b8bfba21db8055 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 21 Jun 2019 21:12:27 +0200 Subject: [PATCH 126/405] uint add/sub type checking, fixes #1029 --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 8 ++++- .../eth2spec/utils/ssz/test_ssz_typing.py | 32 +++++++++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 22f76badaa..971b2106b9 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -52,9 +52,15 @@ def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") if cls.byte_len and value.bit_length() > (cls.byte_len << 3): - raise ValueError("value out of bounds for uint{}".format(cls.byte_len)) + raise ValueError("value out of bounds for uint{}".format(cls.byte_len * 8)) return super().__new__(cls, value) + def __add__(self, other): + return self.__class__(super().__add__(coerce_type_maybe(other, self.__class__, strict=True))) + + def __sub__(self, other): + return self.__class__(super().__sub__(coerce_type_maybe(other, self.__class__, strict=True))) + @classmethod def default(cls): return cls(0) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 895a074a91..4325501aac 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -6,6 +6,14 @@ ) +def expect_value_error(fn, msg): + try: + fn() + raise AssertionError(msg) + except ValueError: + pass + + def test_subclasses(): for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: assert issubclass(u, uint) @@ -55,21 +63,13 @@ def test_basic_value_bounds(): # this should work assert k(v - 1) == v - 1 # but we do not allow overflows - try: - k(v) - assert False - except ValueError: - pass + expect_value_error(lambda: k(v), "no overflows allowed") for k, _ in max.items(): # this should work assert k(0) == 0 # but we do not allow underflows - try: - k(-1) - assert False - except ValueError: - pass + expect_value_error(lambda: k(-1), "no underflows allowed") def test_container(): @@ -213,3 +213,15 @@ class Hash(Bytes32): assert not issubclass(Bytes48, Bytes32) assert len(Bytes32() + Bytes48()) == 80 + + +def test_uint_math(): + assert uint8(0) + uint8(uint32(16)) == uint8(16) # allow explict casting to make invalid addition valid + + expect_value_error(lambda: uint8(0) - uint8(1), "no underflows allowed") + expect_value_error(lambda: uint8(1) + uint8(255), "no overflows allowed") + expect_value_error(lambda: uint8(0) + 256, "no overflows allowed") + expect_value_error(lambda: uint8(42) + uint32(123), "no mixed types") + expect_value_error(lambda: uint32(42) + uint8(123), "no mixed types") + + assert type(uint32(1234) + 56) == uint32 From d1fa3acb27802b56627ee71e0e3d292553838146 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 21 Jun 2019 21:27:26 +0200 Subject: [PATCH 127/405] remove unused dependency --- test_libs/pyspec/requirements.txt | 1 - test_libs/pyspec/setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index b96b0a80fc..83197af9c8 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -2,6 +2,5 @@ eth-utils>=1.3.0,<2 eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 -typing_inspect==0.4.0 dataclasses==0.6 ssz==0.1.0a10 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index fe14c498cb..d8d54eab74 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -9,7 +9,6 @@ "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.7.3", "py_ecc>=1.6.0", - "typing_inspect==0.4.0", "ssz==0.1.0a10", "dataclasses==0.6", ] From f12c32a6904417a4b8ed4b8bbbae2f6193c0f835 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 14:18:28 -0600 Subject: [PATCH 128/405] Xdist tests for parallelism (#1201) * add xdist parallelization * reduce circleci cpus to 8 * reduce cpus to 4 * reduce circleci cpus to 2 * circleci back to 4 cpus --- Makefile | 4 ++-- test_libs/pyspec/requirements-testing.txt | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9764b35286..9090c3ef0e 100644 --- a/Makefile +++ b/Makefile @@ -48,11 +48,11 @@ install_test: test: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; export PYTHONPATH="./"; \ - python -m pytest --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python -m pytest -n 4 --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \ - python -m pytest --junitxml=test-reports/eth2spec/test_results.xml eth2spec + python -m pytest -n 4 --junitxml=test-reports/eth2spec/test_results.xml eth2spec open_cov: ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & diff --git a/test_libs/pyspec/requirements-testing.txt b/test_libs/pyspec/requirements-testing.txt index ad9c4de3bc..b5229ae20f 100644 --- a/test_libs/pyspec/requirements-testing.txt +++ b/test_libs/pyspec/requirements-testing.txt @@ -1,6 +1,7 @@ -r requirements.txt -pytest>=3.6,<3.7 +pytest>=4.4 ../config_helpers flake8==3.7.7 mypy==0.701 pytest-cov +pytest-xdist From f55499c0baad5d22a6f7749a9153374c7edf9267 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 14:30:22 -0600 Subject: [PATCH 129/405] Fix Gwei value table (#1203) --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1dcdbdef94..5081dbf740 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -200,7 +200,7 @@ These configurations are updated for releases, but may be out of sync during `de ### Gwei values | Name | Value | -| - | - | :-: | +| - | - | | `MIN_DEPOSIT_AMOUNT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | | `MAX_EFFECTIVE_BALANCE` | `Gwei(2**5 * 10**9)` (= 32,000,000,000) | | `EJECTION_BALANCE` | `Gwei(2**4 * 10**9)` (= 16,000,000,000) | From c09e45c5fc49e81c5be5c6576642949b18ec0d73 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 14:43:55 -0600 Subject: [PATCH 130/405] fix rule_4 underflow and split out genesis finality test --- .../pyspec/eth2spec/test/test_finality.py | 42 ++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 5e81f52c88..160a32a4b9 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -39,11 +39,13 @@ def next_epoch_with_attestations(spec, state, fill_cur_epoch, fill_prev_epoch): + assert state.slot % spec.SLOTS_PER_EPOCH == 0 + post_state = deepcopy(state) blocks = [] for _ in range(spec.SLOTS_PER_EPOCH): block = build_empty_block_for_next_slot(spec, post_state) - if fill_cur_epoch: + if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) @@ -63,11 +65,13 @@ def next_epoch_with_attestations(spec, @with_all_phases @never_bls @spec_state_test -def test_finality_rule_4(spec, state): +def test_finality_no_updates_at_genesis(spec, state): + assert spec.get_current_epoch(state) == spec.GENESIS_EPOCH + yield 'pre', state blocks = [] - for epoch in range(4): + for epoch in range(2): prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) blocks += new_blocks @@ -77,9 +81,37 @@ def test_finality_rule_4(spec, state): # justification/finalization skipped at GENESIS_EPOCH + 1 elif epoch == 1: check_finality(spec, state, prev_state, False, False, False) - elif epoch == 2: + + yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'post', state + + +@with_all_phases +@never_bls +@spec_state_test +def test_finality_rule_4(spec, state): + # get past first two epochs that finality does not run on + next_epoch(spec, state) + apply_empty_block(spec, state) + next_epoch(spec, state) + apply_empty_block(spec, state) + + yield 'pre', state + + blocks = [] + for epoch in range(2): + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + blocks += new_blocks + + # justification/finalization skipped at GENESIS_EPOCH + # if epoch == 0: + # check_finality(spec, state, prev_state, False, False, False) + # justification/finalization skipped at GENESIS_EPOCH + 1 + # elif epoch == 1: + # check_finality(spec, state, prev_state, False, False, False) + if epoch == 0: check_finality(spec, state, prev_state, True, False, False) - elif epoch >= 3: + elif epoch == 1: # rule 4 of finality check_finality(spec, state, prev_state, True, True, True) assert state.finalized_epoch == prev_state.current_justified_epoch From 8f997413445d55a11a00c44ffe4f81a92ca0fe1e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 14:47:18 -0600 Subject: [PATCH 131/405] remove commented old code --- test_libs/pyspec/eth2spec/test/test_finality.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 160a32a4b9..7304562862 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -103,12 +103,6 @@ def test_finality_rule_4(spec, state): prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) blocks += new_blocks - # justification/finalization skipped at GENESIS_EPOCH - # if epoch == 0: - # check_finality(spec, state, prev_state, False, False, False) - # justification/finalization skipped at GENESIS_EPOCH + 1 - # elif epoch == 1: - # check_finality(spec, state, prev_state, False, False, False) if epoch == 0: check_finality(spec, state, prev_state, True, False, False) elif epoch == 1: From e15a649f37680ab086801f02b408e1b562381d6f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 15:34:36 -0600 Subject: [PATCH 132/405] reduce MAX_EPOCHS_PER_CROSSLINK in minimal config for testing purposes --- configs/constant_presets/minimal.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index c8c2853c70..201ac475ff 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -64,8 +64,8 @@ SLOTS_PER_HISTORICAL_ROOT: 64 MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 # 2**11 (= 2,048) epochs PERSISTENT_COMMITTEE_PERIOD: 2048 -# 2**6 (= 64) epochs -MAX_EPOCHS_PER_CROSSLINK: 64 +# [customized] fast catchup crosslinks +MAX_EPOCHS_PER_CROSSLINK: 4 # 2**2 (= 4) epochs MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 # [customized] 2**12 (= 4,096) epochs From 7a16db144c9658c6dad6da9d29bd849141a7a94a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 15:59:18 -0600 Subject: [PATCH 133/405] Add test_genesis --- scripts/build_spec.py | 1 + specs/core/0_beacon-chain.md | 14 +++-- .../pyspec/eth2spec/test/helpers/deposits.py | 52 ++++++++++++++++--- .../eth2spec/test/sanity/test_genesis.py | 29 +++++++++++ 4 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/sanity/test_genesis.py diff --git a/scripts/build_spec.py b/scripts/build_spec.py index a16fa79ac3..d7b6d0f594 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -16,6 +16,7 @@ Callable, Dict, List, + Optional, Set, Tuple, ) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1dcdbdef94..28d0d72cdd 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1176,8 +1176,12 @@ def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis ) # Process genesis deposits - for deposit in deposits: - process_deposit(state, deposit) + for deposit_index, deposit in enumerate(deposits): + process_deposit( + state, + deposit, + deposit_index=deposit_index, + ) # Process genesis activations for validator in state.validators: @@ -1726,16 +1730,18 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ##### Deposits ```python -def process_deposit(state: BeaconState, deposit: Deposit) -> None: +def process_deposit(state: BeaconState, deposit: Deposit, deposit_index: Optional[uint64]) -> None: """ Process an Eth1 deposit, registering a validator or increasing its balance. """ + if deposit_index is None: + deposit_index = state.eth1_deposit_index # Verify the Merkle branch assert verify_merkle_branch( leaf=hash_tree_root(deposit.data), proof=deposit.proof, depth=DEPOSIT_CONTRACT_TREE_DEPTH, - index=state.eth1_deposit_index, + index=deposit_index, root=state.eth1_data.deposit_root, ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 20ff7440f7..ca8b23361c 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -4,25 +4,31 @@ from eth2spec.utils.ssz.ssz_impl import signing_root -def build_deposit_data(spec, state, pubkey, privkey, amount, withdrawal_credentials, signed=False): +def build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=None, signed=False): deposit_data = spec.DepositData( pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, amount=amount, ) if signed: - sign_deposit_data(spec, state, deposit_data, privkey) + sign_deposit_data(spec, deposit_data, privkey, state) return deposit_data -def sign_deposit_data(spec, state, deposit_data, privkey): - signature = bls_sign( - message_hash=signing_root(deposit_data), - privkey=privkey, - domain=spec.get_domain( +def sign_deposit_data(spec, deposit_data, privkey, state=None): + if state is None: + # Genesis + domain = spec.bls_domain(spec.DOMAIN_DEPOSIT) + else: + domain = spec.get_domain( state, spec.DOMAIN_DEPOSIT, ) + + signature = bls_sign( + message_hash=signing_root(deposit_data), + privkey=privkey, + domain=domain, ) deposit_data.signature = signature @@ -35,7 +41,7 @@ def build_deposit(spec, amount, withdrawal_credentials, signed): - deposit_data = build_deposit_data(spec, state, pubkey, privkey, amount, withdrawal_credentials, signed) + deposit_data = build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=state, signed=signed) item = deposit_data.hash_tree_root() index = len(deposit_data_leaves) @@ -54,6 +60,36 @@ def build_deposit(spec, return deposit, root, deposit_data_leaves +def prepare_genesis_deposits(spec, genesis_validator_count, signed=False): + genesis_deposit_data_list = [] + deposit_data_leaves = [] + for validator_index in range(genesis_validator_count): + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + # insecurely use pubkey as withdrawal key if no credentials provided + withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(pubkey)[1:] + deposit_data = spec.DepositData( + pubkey=pubkey, + withdrawal_credentials=withdrawal_credentials, + amount=spec.MAX_EFFECTIVE_BALANCE, + ) + if signed: + sign_deposit_data(spec, deposit_data, privkey) # state=None + item = deposit_data.hash_tree_root() + deposit_data_leaves.append(item) + genesis_deposit_data_list.append(deposit_data) + + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) + root = get_merkle_root((tuple(deposit_data_leaves))) + + genesis_deposits = ( + spec.Deposit(proof=list(get_merkle_proof(tree, item_index=index)), data=deposit_data) + for index, deposit_data in enumerate(genesis_deposit_data_list) + ) + + return genesis_deposits, root + + def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_credentials=None, signed=False): """ Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py new file mode 100644 index 0000000000..088df84e84 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py @@ -0,0 +1,29 @@ +from eth2spec.test.context import spec_state_test, with_phases +from eth2spec.test.helpers.deposits import ( + prepare_genesis_deposits, +) + + +@with_phases(['phase0']) +@spec_state_test +def test_genesis(spec, state): + deposit_count = 2 + genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count) + genesis_time = 1234 + + yield genesis_deposits + yield genesis_time + + genesis_eth1_data = spec.Eth1Data( + deposit_root=deposit_root, + deposit_count=deposit_count, + block_hash=b'\x12' * 32, + ) + + yield genesis_eth1_data + genesis_state = spec.get_genesis_beacon_state( + genesis_deposits, + genesis_time, + genesis_eth1_data, + ) + yield genesis_state From b36ffd5c412c65327ba10f35f1f406a006bf1d20 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 16:10:19 -0600 Subject: [PATCH 134/405] default value of `deposit_index` --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 28d0d72cdd..4d35c65165 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1730,7 +1730,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ##### Deposits ```python -def process_deposit(state: BeaconState, deposit: Deposit, deposit_index: Optional[uint64]) -> None: +def process_deposit(state: BeaconState, deposit: Deposit, deposit_index: Optional[uint64]=None) -> None: """ Process an Eth1 deposit, registering a validator or increasing its balance. """ From ac34221f55aabc75934c4e4fa6704e29599a680c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 16:30:31 -0600 Subject: [PATCH 135/405] Fix `is_genesis_trigger` and add test case --- configs/constant_presets/mainnet.yaml | 2 ++ configs/constant_presets/minimal.yaml | 2 ++ specs/core/0_beacon-chain.md | 19 ++++++++------ .../eth2spec/test/sanity/test_genesis.py | 26 +++++++++++++++++++ 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 09ec7bdb79..572cfb7089 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -17,6 +17,8 @@ MIN_PER_EPOCH_CHURN_LIMIT: 4 CHURN_LIMIT_QUOTIENT: 65536 # See issue 563 SHUFFLE_ROUND_COUNT: 90 +# `2**16` (= 65,536) +GENESIS_ACTIVE_VALIDATOR_COUNT: 65536 # Deposit contract diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index c8c2853c70..edc73ab8b5 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -16,6 +16,8 @@ MIN_PER_EPOCH_CHURN_LIMIT: 4 CHURN_LIMIT_QUOTIENT: 65536 # [customized] Faster, but unsecure. SHUFFLE_ROUND_COUNT: 10 +# [customized] +GENESIS_ACTIVE_VALIDATOR_COUNT: 100 # Deposit contract diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4d35c65165..0b856636bb 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -194,6 +194,7 @@ These configurations are updated for releases, but may be out of sync during `de | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | +| `GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**16` (= 65,536) | * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -1134,10 +1135,11 @@ Before genesis has been triggered and whenever the deposit contract emits a `Dep * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log -When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: +When `is_genesis_trigger(deposits, timestamp, deposit_root) is True` for the first time let: * `genesis_deposits = deposits` * `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` +* `deposit_root` is the tree root of the given `deposits` * `genesis_eth1_data` be the object of type `Eth1Data` where: * `genesis_eth1_data.block_hash` is the Ethereum 1.0 block hash that emitted the log for the last deposit in `deposits` * `genesis_eth1_data.deposit_root` is the deposit root for the last deposit in `deposits` @@ -1146,20 +1148,19 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: *Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: ```python -def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: +def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64, deposit_root: Hash) -> bool: # Process deposits state = BeaconState() - for deposit in deposits: - process_deposit(state, deposit) + for index, deposit in enumerate(deposits): + process_deposit(state, deposit, deposit_index=index, deposit_root=deposit_root) # Count active validators at genesis active_validator_count = 0 - for validator in state.validator_registry: + for validator in state.validators: if validator.effective_balance == MAX_EFFECTIVE_BALANCE: active_validator_count += 1 # Check effective balance to trigger genesis - GENESIS_ACTIVE_VALIDATOR_COUNT = 2**16 return active_validator_count == GENESIS_ACTIVE_VALIDATOR_COUNT ``` @@ -1730,19 +1731,21 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ##### Deposits ```python -def process_deposit(state: BeaconState, deposit: Deposit, deposit_index: Optional[uint64]=None) -> None: +def process_deposit(state: BeaconState, deposit: Deposit, deposit_index: Optional[uint64]=None, deposit_root: Optional[Hash]=None) -> None: """ Process an Eth1 deposit, registering a validator or increasing its balance. """ if deposit_index is None: deposit_index = state.eth1_deposit_index + if deposit_root is None: + deposit_root = state.eth1_data.deposit_root # Verify the Merkle branch assert verify_merkle_branch( leaf=hash_tree_root(deposit.data), proof=deposit.proof, depth=DEPOSIT_CONTRACT_TREE_DEPTH, index=deposit_index, - root=state.eth1_data.deposit_root, + root=deposit_root, ) # Deposits must be processed in order diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py index 088df84e84..0a9db7bc09 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py @@ -4,6 +4,32 @@ ) +@with_phases(['phase0']) +@spec_state_test +def test_is_genesis_trigger_false(spec, state): + deposit_count = 2 + genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count) + genesis_time = 1234 + + is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time, deposit_root) + assert is_triggered is False + + yield is_triggered + + +@with_phases(['phase0']) +@spec_state_test +def test_is_genesis_trigger_true(spec, state): + deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT + genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count) + genesis_time = 1234 + + is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time, deposit_root) + assert is_triggered is True + + yield is_triggered + + @with_phases(['phase0']) @spec_state_test def test_genesis(spec, state): From e7c595d1d6200299a6d33a7a4ef0d06801297b53 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 16:33:24 -0600 Subject: [PATCH 136/405] Fix test_process_deposit.py --- .../test/phase_0/block_processing/test_process_deposit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 63b94c6385..1b8c446516 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -124,7 +124,7 @@ def test_wrong_index(spec, state): # mess up eth1_deposit_index deposit.index = state.eth1_deposit_index + 1 - sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) + sign_deposit_data(spec, deposit.data, privkeys[validator_index], state=state) yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) @@ -185,6 +185,6 @@ def test_bad_merkle_proof(spec, state): # mess up merkle branch deposit.proof[-1] = spec.ZERO_HASH - sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) + sign_deposit_data(spec, deposit.data, privkeys[validator_index], state) yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) From de5b9cc823dedd9673f3599bf8e209a363cf0dd7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 16:33:43 -0600 Subject: [PATCH 137/405] convert _root/_epoch tuples to Checkpoint obj in beacon spec --- specs/core/0_beacon-chain.md | 92 ++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 5081dbf740..cf1d368620 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -26,6 +26,7 @@ - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) - [`Validator`](#validator) + - [`Checkpoint`](#checkpoint) - [`Crosslink`](#crosslink) - [`AttestationData`](#attestationdata) - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) @@ -303,6 +304,14 @@ class Fork(Container): epoch: Epoch # Epoch of latest fork ``` +#### `Checkpoint` + +```python +class Checkpoint(Container): + epoch: Epoch + root: Hash +``` + #### `Validator` ```python @@ -337,10 +346,8 @@ class AttestationData(Container): # LMD GHOST vote beacon_block_root: Hash # FFG vote - source_epoch: Epoch - source_root: Hash - target_epoch: Epoch - target_root: Hash + source_checkpoint: Checkpoint + target_checkpoint: Checkpoint # Crosslink vote crosslink: Crosslink ``` @@ -534,14 +541,11 @@ class BeaconState(Container): previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] # Justification - previous_justified_epoch: Epoch # Previous epoch snapshot - previous_justified_root: Hash # Previous epoch snapshot - current_justified_epoch: Epoch - current_justified_root: Hash + previous_justified_checkpoint: Checkpoint # Previous epoch snapshot + current_justified_checkpoint: Checkpoint justification_bitfield: uint64 # Bit set for every recent justified epoch # Finality - finalized_epoch: Epoch - finalized_root: Hash + finalized_checkpoint: Checkpoint ``` ## Helper functions @@ -715,9 +719,10 @@ def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: ```python def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot: - committee_count = get_epoch_committee_count(state, data.target_epoch) - offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, data.target_epoch)) % SHARD_COUNT - return Slot(get_epoch_start_slot(data.target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) + target_epoch = data.target_checkpoint.epoch + committee_count = get_epoch_committee_count(state, target_epoch) + offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, target_epoch)) % SHARD_COUNT + return Slot(get_epoch_start_slot(target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` ### `get_block_root_at_slot` @@ -876,12 +881,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> L ```python def get_attesting_indices(state: BeaconState, - attestation_data: AttestationData, + data: AttestationData, bitfield: bytes) -> List[ValidatorIndex]: """ - Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. + Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. """ - committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) + committee = get_crosslink_committee(state, data.target_checkpoint.epoch, data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) ``` @@ -1001,7 +1006,7 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), ], signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_epoch), + domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_checkpoint.epoch), ) ``` @@ -1014,9 +1019,12 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa """ return ( # Double vote - (data_1 != data_2 and data_1.target_epoch == data_2.target_epoch) or + (data_1 != data_2 and data_1.target_checkpoint.epoch == data_2.target_checkpoint.epoch) or # Surround vote - (data_1.source_epoch < data_2.source_epoch and data_2.target_epoch < data_1.target_epoch) + ( + data_1.source_checkpoint.epoch < data_2.source_checkpoint.epoch and + data_2.target_checkpoint.epoch < data_1.target_checkpoint.epoch + ) ) ``` @@ -1274,7 +1282,7 @@ def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[P def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) - if a.data.target_root == get_block_root(state, epoch) + if a.data.target_checkpoint.root == get_block_root(state, epoch) ] ``` @@ -1326,46 +1334,39 @@ def process_justification_and_finalization(state: BeaconState) -> None: previous_epoch = get_previous_epoch(state) current_epoch = get_current_epoch(state) - old_previous_justified_epoch = state.previous_justified_epoch - old_current_justified_epoch = state.current_justified_epoch + old_previous_justified_checkpoint = state.previous_justified_checkpoint + old_current_justified_checkpoint = state.current_justified_checkpoint # Process justifications - state.previous_justified_epoch = state.current_justified_epoch - state.previous_justified_root = state.current_justified_root + state.previous_justified_checkpoint = state.current_justified_checkpoint state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_epoch = previous_epoch - state.current_justified_root = get_block_root(state, state.current_justified_epoch) + state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_epoch = current_epoch - state.current_justified_root = get_block_root(state, state.current_justified_epoch) + state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: - state.finalized_epoch = old_previous_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: - state.finalized_epoch = old_previous_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: - state.finalized_epoch = old_current_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_current_justified_checkpoint # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: - state.finalized_epoch = old_current_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_current_justified_checkpoint ``` #### Crosslinks @@ -1428,7 +1429,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: rewards[index] += Gwei(max_attester_reward * MIN_ATTESTATION_INCLUSION_DELAY // attestation.inclusion_delay) # Inactivity penalty - finality_delay = previous_epoch - state.finalized_epoch + finality_delay = previous_epoch - state.finalized_checkpoint.epoch if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in eligible_validator_indices: @@ -1493,7 +1494,7 @@ def process_registry_updates(state: BeaconState) -> None: activation_queue = sorted([ index for index, validator in enumerate(state.validators) if validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH and - validator.activation_epoch >= get_delayed_activation_exit_epoch(state.finalized_epoch) + validator.activation_epoch >= get_delayed_activation_exit_epoch(state.finalized_checkpoint.epoch) ], key=lambda index: state.validators[index].activation_eligibility_epoch) # Dequeued validators for activation up to churn limit (without resetting activation epoch) for index in activation_queue[:get_churn_limit(state)]: @@ -1691,9 +1692,10 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: Process ``Attestation`` operation. """ data = attestation.data + target_epoch = data.target_checkpoint.epoch assert data.crosslink.shard < SHARD_COUNT - assert data.target_epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert target_epoch in (get_previous_epoch(state), get_current_epoch(state)) attestation_slot = get_attestation_data_slot(state, data) assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH @@ -1705,19 +1707,19 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_index=get_beacon_proposer_index(state), ) - if data.target_epoch == get_current_epoch(state): - ffg_data = (state.current_justified_epoch, state.current_justified_root, get_current_epoch(state)) + if target_epoch == get_current_epoch(state): + ffg_source = state.current_justified_checkpoint parent_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) else: - ffg_data = (state.previous_justified_epoch, state.previous_justified_root, get_previous_epoch(state)) + ffg_source = state.previous_justified_checkpoint parent_crosslink = state.previous_crosslinks[data.crosslink.shard] state.previous_epoch_attestations.append(pending_attestation) - # Check FFG data, crosslink data, and signature - assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) + # Check FFG source, crosslink data, and signature + assert ffg_source == data.source_checkpoint assert data.crosslink.start_epoch == parent_crosslink.end_epoch - assert data.crosslink.end_epoch == min(data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) + assert data.crosslink.end_epoch == min(target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] validate_indexed_attestation(state, convert_to_indexed(state, attestation)) From 8828dad786271fa2b5a6c95f5549d7bf5927c68b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 17:07:44 -0600 Subject: [PATCH 138/405] minor updates --- specs/core/0_beacon-chain.md | 5 ++++- .../test/phase_0/block_processing/test_process_deposit.py | 2 +- test_libs/pyspec/eth2spec/test/sanity/test_genesis.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 0b856636bb..c82f1e34f0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1731,7 +1731,10 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ##### Deposits ```python -def process_deposit(state: BeaconState, deposit: Deposit, deposit_index: Optional[uint64]=None, deposit_root: Optional[Hash]=None) -> None: +def process_deposit(state: BeaconState, + deposit: Deposit, + deposit_index: Optional[uint64]=None, + deposit_root: Optional[Hash]=None) -> None: """ Process an Eth1 deposit, registering a validator or increasing its balance. """ diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 1b8c446516..6acf86cc79 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -185,6 +185,6 @@ def test_bad_merkle_proof(spec, state): # mess up merkle branch deposit.proof[-1] = spec.ZERO_HASH - sign_deposit_data(spec, deposit.data, privkeys[validator_index], state) + sign_deposit_data(spec, deposit.data, privkeys[validator_index], state=state) yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py index 0a9db7bc09..d6b3bf9403 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py @@ -33,7 +33,7 @@ def test_is_genesis_trigger_true(spec, state): @with_phases(['phase0']) @spec_state_test def test_genesis(spec, state): - deposit_count = 2 + deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count) genesis_time = 1234 From 47b29c84564e442cf9394028080d39c256bf9914 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 21:49:13 -0600 Subject: [PATCH 139/405] Loose condition of effective genesis deposit --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c82f1e34f0..1d2dead41c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1157,7 +1157,7 @@ def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64, deposit_root: # Count active validators at genesis active_validator_count = 0 for validator in state.validators: - if validator.effective_balance == MAX_EFFECTIVE_BALANCE: + if validator.effective_balance >= MAX_EFFECTIVE_BALANCE: active_validator_count += 1 # Check effective balance to trigger genesis From 7468fd034f5cb904513d6b07fc482f5b5478ad1e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 21:50:36 -0600 Subject: [PATCH 140/405] Add more test --- .../pyspec/eth2spec/test/helpers/deposits.py | 6 +++--- .../eth2spec/test/sanity/test_genesis.py | 20 ++++++++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index ca8b23361c..eea019e8b9 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -60,7 +60,7 @@ def build_deposit(spec, return deposit, root, deposit_data_leaves -def prepare_genesis_deposits(spec, genesis_validator_count, signed=False): +def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False): genesis_deposit_data_list = [] deposit_data_leaves = [] for validator_index in range(genesis_validator_count): @@ -71,7 +71,7 @@ def prepare_genesis_deposits(spec, genesis_validator_count, signed=False): deposit_data = spec.DepositData( pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, - amount=spec.MAX_EFFECTIVE_BALANCE, + amount=amount, ) if signed: sign_deposit_data(spec, deposit_data, privkey) # state=None @@ -84,7 +84,7 @@ def prepare_genesis_deposits(spec, genesis_validator_count, signed=False): genesis_deposits = ( spec.Deposit(proof=list(get_merkle_proof(tree, item_index=index)), data=deposit_data) - for index, deposit_data in enumerate(genesis_deposit_data_list) + for index, deposit_data in enumerate(genesis_deposit_data_list) ) return genesis_deposits, root diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py index d6b3bf9403..7e5fd5db46 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py @@ -8,7 +8,7 @@ @spec_state_test def test_is_genesis_trigger_false(spec, state): deposit_count = 2 - genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count) + genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) genesis_time = 1234 is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time, deposit_root) @@ -21,7 +21,7 @@ def test_is_genesis_trigger_false(spec, state): @spec_state_test def test_is_genesis_trigger_true(spec, state): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT - genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count) + genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) genesis_time = 1234 is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time, deposit_root) @@ -29,12 +29,26 @@ def test_is_genesis_trigger_true(spec, state): yield is_triggered +@with_phases(['phase0']) +@spec_state_test +def test_is_genesis_trigger_not_enough_balance(spec, state): + deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT + genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE - 1) + genesis_time = 1234 + yield genesis_deposits + yield genesis_time + + is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time, deposit_root) + assert is_triggered is False + + yield is_triggered + @with_phases(['phase0']) @spec_state_test def test_genesis(spec, state): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT - genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count) + genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) genesis_time = 1234 yield genesis_deposits From 24ad42663fd05fd9a09667ae74395feafae8dc37 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 22:02:03 -0600 Subject: [PATCH 141/405] `spectest_with_bls_switch` decorator --- test_libs/pyspec/eth2spec/test/context.py | 6 +++++- .../eth2spec/test/sanity/test_genesis.py | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 97266acf2c..e7560afc6d 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -27,9 +27,13 @@ def entry(*args, **kw): DEFAULT_BLS_ACTIVE = False +def spectest_with_bls_switch(fn): + return bls_switch(spectest()(fn)) + + # shorthand for decorating @with_state @spectest() def spec_state_test(fn): - return with_state(bls_switch(spectest()(fn))) + return with_state(spectest_with_bls_switch(fn)) def expect_assertion_error(fn): diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py index 7e5fd5db46..3c76654e6f 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py @@ -1,12 +1,12 @@ -from eth2spec.test.context import spec_state_test, with_phases +from eth2spec.test.context import with_phases, spectest_with_bls_switch from eth2spec.test.helpers.deposits import ( prepare_genesis_deposits, ) @with_phases(['phase0']) -@spec_state_test -def test_is_genesis_trigger_false(spec, state): +@spectest_with_bls_switch +def test_is_genesis_trigger_false(spec): deposit_count = 2 genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) genesis_time = 1234 @@ -18,8 +18,8 @@ def test_is_genesis_trigger_false(spec, state): @with_phases(['phase0']) -@spec_state_test -def test_is_genesis_trigger_true(spec, state): +@spectest_with_bls_switch +def test_is_genesis_trigger_true(spec): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) genesis_time = 1234 @@ -29,9 +29,10 @@ def test_is_genesis_trigger_true(spec, state): yield is_triggered + @with_phases(['phase0']) -@spec_state_test -def test_is_genesis_trigger_not_enough_balance(spec, state): +@spectest_with_bls_switch +def test_is_genesis_trigger_not_enough_balance(spec): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE - 1) genesis_time = 1234 @@ -45,8 +46,8 @@ def test_is_genesis_trigger_not_enough_balance(spec, state): @with_phases(['phase0']) -@spec_state_test -def test_genesis(spec, state): +@spectest_with_bls_switch +def test_genesis(spec): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) genesis_time = 1234 From 5f0921277739d5a137ba86cefa5bf91d0df852dd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 22:09:09 -0600 Subject: [PATCH 142/405] yield for `is_genesis_trigger` --- specs/core/0_beacon-chain.md | 6 +----- .../pyspec/eth2spec/test/sanity/test_genesis.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1d2dead41c..c94c4fcc90 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1178,11 +1178,7 @@ def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis # Process genesis deposits for deposit_index, deposit in enumerate(deposits): - process_deposit( - state, - deposit, - deposit_index=deposit_index, - ) + process_deposit(state, deposit, deposit_index=deposit_index) # Process genesis activations for validator in state.validators: diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py index 3c76654e6f..37d8a2b171 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py @@ -9,7 +9,11 @@ def test_is_genesis_trigger_false(spec): deposit_count = 2 genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - genesis_time = 1234 + genesis_time = 1546300800 + + yield genesis_deposits + yield genesis_time + yield deposit_root is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time, deposit_root) assert is_triggered is False @@ -22,7 +26,11 @@ def test_is_genesis_trigger_false(spec): def test_is_genesis_trigger_true(spec): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - genesis_time = 1234 + genesis_time = 1546300800 + + yield genesis_deposits + yield genesis_time + yield deposit_root is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time, deposit_root) assert is_triggered is True @@ -35,9 +43,10 @@ def test_is_genesis_trigger_true(spec): def test_is_genesis_trigger_not_enough_balance(spec): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE - 1) - genesis_time = 1234 + genesis_time = 1546300800 yield genesis_deposits yield genesis_time + yield deposit_root is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time, deposit_root) assert is_triggered is False @@ -50,7 +59,7 @@ def test_is_genesis_trigger_not_enough_balance(spec): def test_genesis(spec): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - genesis_time = 1234 + genesis_time = 1546300800 yield genesis_deposits yield genesis_time From 6aef6c5634acc0c378c1f03b2cefd32ba3501954 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 22:26:15 -0600 Subject: [PATCH 143/405] Clean up --- specs/core/0_beacon-chain.md | 4 ++-- test_libs/pyspec/eth2spec/test/sanity/test_genesis.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index bb1379b936..8f57967edd 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1139,11 +1139,11 @@ When `is_genesis_trigger(deposits, timestamp, deposit_root) is True` for the fir * `genesis_deposits = deposits` * `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` -* `deposit_root` is the tree root of the given `deposits` +* `deposit_root` is the Merkle tree root of the data of the given `deposits` * `genesis_eth1_data` be the object of type `Eth1Data` where: - * `genesis_eth1_data.block_hash` is the Ethereum 1.0 block hash that emitted the log for the last deposit in `deposits` * `genesis_eth1_data.deposit_root` is the deposit root for the last deposit in `deposits` * `genesis_eth1_data.deposit_count = len(genesis_deposits)` + * `genesis_eth1_data.block_hash` is the Ethereum 1.0 block hash that emitted the log for the last deposit in `deposits` *Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py index 37d8a2b171..780d039ebe 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py @@ -60,6 +60,7 @@ def test_genesis(spec): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) genesis_time = 1546300800 + block_hash = b'\x12' * 32 yield genesis_deposits yield genesis_time @@ -67,7 +68,7 @@ def test_genesis(spec): genesis_eth1_data = spec.Eth1Data( deposit_root=deposit_root, deposit_count=deposit_count, - block_hash=b'\x12' * 32, + block_hash=block_hash, ) yield genesis_eth1_data @@ -76,4 +77,11 @@ def test_genesis(spec): genesis_time, genesis_eth1_data, ) + + assert genesis_state.genesis_time == genesis_time + assert len(genesis_state.validators) == deposit_count + assert genesis_state.eth1_data.deposit_root == deposit_root + assert genesis_state.eth1_data.deposit_count == deposit_count + assert genesis_state.eth1_data.block_hash == block_hash + yield genesis_state From 99df7da94a026b1daf02cadb10cbb18ee091b71d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 22:11:58 -0600 Subject: [PATCH 144/405] Add `&&` between the commands --- Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 9764b35286..9c7bfcf592 100644 --- a/Makefile +++ b/Makefile @@ -59,10 +59,9 @@ open_cov: lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ - flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec; \ - cd ./eth2spec; \ - mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0; \ - mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1 + flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec \ + && cd ./eth2spec && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 \ + && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1; install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS) cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt From 183fa3c7762acfc2f68a295db58b1b9bb749581e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 23:00:30 -0600 Subject: [PATCH 145/405] Make linter happy --- specs/core/0_beacon-chain.md | 11 ++++++++--- test_libs/pyspec/eth2spec/test/test_fork_choice.py | 2 -- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1dcdbdef94..d478c2941f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -524,9 +524,11 @@ class BeaconState(Container): # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] - active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Digests of the active registry, for light clients + # Digests of the active registry, for light clients + active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Slashings - slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators + # Sums of the effective balances of slashed validators + slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Attestations previous_epoch_attestations: List[PendingAttestation] current_epoch_attestations: List[PendingAttestation] @@ -1515,7 +1517,10 @@ def process_slashings(state: BeaconState) -> None: total_penalties = total_at_end - total_at_start for index, validator in enumerate(state.validators): - if validator.slashed and current_epoch == validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2: + if ( + validator.slashed and + current_epoch == validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2 + ): penalty = max( validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index 4bc7e8b0a0..4706f0eaf5 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -1,5 +1,3 @@ -from typing import List - from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root from eth2spec.test.context import with_all_phases, with_state, bls_switch From 613380bff452f829de9342cb66abf01e945792cf Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 07:29:25 +0200 Subject: [PATCH 146/405] Shorter in-line comments --- specs/core/0_beacon-chain.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d478c2941f..7b5d2a42ae 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -524,11 +524,9 @@ class BeaconState(Container): # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] - # Digests of the active registry, for light clients - active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] + active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Active registry digests for light clients # Slashings - # Sums of the effective balances of slashed validators - slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] + slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of slashed effective balances # Attestations previous_epoch_attestations: List[PendingAttestation] current_epoch_attestations: List[PendingAttestation] From 1e1a3e5311fd54a15e1449340285624bc316a7e8 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 07:34:02 +0200 Subject: [PATCH 147/405] Avoid substraction in comparison --- specs/core/0_beacon-chain.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7b5d2a42ae..894b96a195 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1506,19 +1506,16 @@ def process_registry_updates(state: BeaconState) -> None: ```python def process_slashings(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) + epoch = get_current_epoch(state) total_balance = get_total_active_balance(state) # Compute slashed balances in the current epoch - total_at_start = state.slashed_balances[(current_epoch + 1) % EPOCHS_PER_SLASHED_BALANCES_VECTOR] - total_at_end = state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] + total_at_start = state.slashed_balances[(epoch + 1) % EPOCHS_PER_SLASHED_BALANCES_VECTOR] + total_at_end = state.slashed_balances[epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] total_penalties = total_at_end - total_at_start for index, validator in enumerate(state.validators): - if ( - validator.slashed and - current_epoch == validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2 - ): + if validator.slashed and epoch + EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2 == validator.withdrawable_epoch: penalty = max( validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT From a5b22e13b8b4bf8247b2e130b12917eee026e0f6 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 22 Jun 2019 16:56:16 +0200 Subject: [PATCH 148/405] Resolves make masker sign mask --- specs/core/1_custody-game.md | 2 +- .../pyspec/eth2spec/test/helpers/custody.py | 13 +++++++++---- .../test_process_early_derived_secret_reveal.py | 16 +++++++++++++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 3fe132c07a..b051d13c67 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -204,7 +204,7 @@ class EarlyDerivedSecretReveal(Container): # Index of the validator who revealed (whistleblower) masker_index: ValidatorIndex # Mask used to hide the actual reveal signature (prevent reveal from being stolen) - mask: Bytes32 + mask: Hash ``` ### Phase 0 container updates diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 67df12fcdb..0167edc6a6 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import bls_sign +from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures def get_valid_early_derived_secret_reveal(spec, state, epoch=None): @@ -10,6 +10,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): if epoch is None: epoch = current_epoch + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING + # Generate the secret that is being revealed reveal = bls_sign( message_hash=spec.hash_tree_root(epoch), privkey=privkeys[revealed_index], @@ -19,8 +20,11 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): message_epoch=epoch, ), ) - mask = bls_sign( - message_hash=spec.hash_tree_root(epoch), + # Generate the mask (any random 32 bytes will do) + mask = reveal[:32] + # Generate masker's signature on the mask + masker_signature = bls_sign( + message_hash=mask, privkey=privkeys[masker_index], domain=spec.get_domain( state=state, @@ -28,11 +32,12 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): message_epoch=epoch, ), ) + masked_reveal = bls_aggregate_signatures([reveal, masker_signature]) return spec.EarlyDerivedSecretReveal( revealed_index=revealed_index, epoch=epoch, - reveal=reveal, + reveal=masked_reveal, masker_index=masker_index, mask=mask, ) diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py index 87297d443c..831ad35a55 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py @@ -1,7 +1,13 @@ from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import next_epoch, get_balance -from eth2spec.test.context import with_all_phases_except, spec_state_test, expect_assertion_error +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, + expect_assertion_error, + always_bls, + never_bls, +) def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, valid=True): @@ -36,6 +42,7 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v @with_all_phases_except(['phase0']) +@always_bls @spec_state_test def test_success(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state) @@ -44,6 +51,7 @@ def test_success(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_reveal_from_current_epoch(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) @@ -52,6 +60,7 @@ def test_reveal_from_current_epoch(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_reveal_from_past_epoch(spec, state): next_epoch(spec, state) @@ -62,6 +71,7 @@ def test_reveal_from_past_epoch(spec, state): @with_all_phases_except(['phase0']) +@always_bls @spec_state_test def test_reveal_with_custody_padding(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( @@ -73,6 +83,7 @@ def test_reveal_with_custody_padding(spec, state): @with_all_phases_except(['phase0']) +@always_bls @spec_state_test def test_reveal_with_custody_padding_minus_one(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( @@ -84,6 +95,7 @@ def test_reveal_with_custody_padding_minus_one(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_double_reveal(spec, state): randao_key_reveal1 = get_valid_early_derived_secret_reveal( @@ -108,6 +120,7 @@ def test_double_reveal(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_revealer_is_slashed(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) @@ -117,6 +130,7 @@ def test_revealer_is_slashed(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_far_future_epoch(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( From b007d5aa92f183be7eb8001db09d205767b2838b Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Sat, 22 Jun 2019 17:38:30 +0200 Subject: [PATCH 149/405] Add note on default zero-values, and be explicit with state-root (#1208) * Add note on default zero-values, and be explicit with state-root --- specs/core/0_beacon-chain.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a955bcab85..094c04a115 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -180,9 +180,7 @@ The following values are (non-configurable) constants used throughout the specif ## Configuration -*Note*: The default mainnet configuration values are included here for spec-design purposes. -The different configurations for mainnet, testnets, and YAML-based testing can be found in the `configs/constant_presets/` directory. -These configurations are updated for releases, but may be out of sync during `dev` changes. +*Note*: The default mainnet configuration values are included here for spec-design purposes. The different configurations for mainnet, testnets, and YAML-based testing can be found in the `configs/constant_presets/` directory. These configurations are updated for releases and may be out of sync during `dev` changes. ### Misc @@ -292,6 +290,8 @@ The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containe *Note*: The definitions are ordered topologically to facilitate execution of the spec. +*Note*: Fields missing in container instantiations default to their zero value. + ### Misc dependencies #### `Fork` @@ -1585,6 +1585,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: state.latest_block_header = BeaconBlockHeader( slot=block.slot, parent_root=block.parent_root, + state_root=ZERO_HASH, # Overwritten in next `process_slot` call body_root=hash_tree_root(block.body), ) # Verify proposer is not slashed From 00aae07d4690417e70ee50b57a34b2382b6642b3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 18:12:42 +0200 Subject: [PATCH 150/405] type annotation clean up --- scripts/build_spec.py | 12 +++--- specs/core/0_beacon-chain.md | 64 +++++++++++++++---------------- specs/core/1_shard-data-chains.md | 18 ++++----- 3 files changed, 46 insertions(+), 48 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 943d378771..0b55dfa443 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,8 +12,7 @@ PHASE0_IMPORTS = '''from typing import ( - Any, Callable, Iterable, Dict, Set, Tuple, NewType, - List as TypingList, + Any, Callable, Iterable, Dict, Set, Sequence, Tuple, ) from dataclasses import ( @@ -38,7 +37,7 @@ from eth2spec.utils.hash_function import hash ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Tuple, Iterable, NewType, + Any, Callable, Dict, Iterable, Optional, Set, Sequence, Tuple, List as TypingList ) @@ -79,13 +78,13 @@ def hash(x: bytes) -> Hash: # Monkey patch validator compute committee code _compute_committee = compute_committee -committee_cache: Dict[Tuple[Hash, Hash, int, int], Tuple[ValidatorIndex, ...]] = {} +committee_cache: Dict[Tuple[Hash, Hash, int, int], Sequence[ValidatorIndex]] = {} -def compute_committee(indices: Tuple[ValidatorIndex, ...], # type: ignore +def compute_committee(indices: Sequence[ValidatorIndex], # type: ignore seed: Hash, index: int, - count: int) -> Tuple[ValidatorIndex, ...]: + count: int) -> Sequence[ValidatorIndex]: param_hash = (hash(b''.join(index.to_bytes(length=4, byteorder='little') for index in indices)), seed, index, count) if param_hash not in committee_cache: @@ -154,7 +153,6 @@ def objects_to_spec(functions: Dict[str, str], spec = ( imports + '\n\n' + new_type_definitions - + '\n\n' + "Deltas = NewType('Deltas', TypingList[Gwei])" + '\n\n' + constants_spec + '\n\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cf8e695358..f42b4cc133 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -651,13 +651,11 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: ### `get_active_validator_indices` ```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]: +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ Get active validator indices at ``epoch``. """ - return List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( - i for i, v in enumerate(state.validators) if is_active_validator(v, epoch) - ) + return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] ``` ### `increase_balance` @@ -819,7 +817,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: ### `verify_merkle_branch` ```python -def verify_merkle_branch(leaf: Hash, proof: Tuple[Hash, ...], depth: int, index: int, root: Hash) -> bool: +def verify_merkle_branch(leaf: Hash, proof: Sequence[Hash], depth: int, index: int, root: Hash) -> bool: """ Verify that the given ``leaf`` is on the merkle branch ``proof`` starting with the given ``root``. @@ -863,19 +861,19 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Hash) -> V ### `compute_committee` ```python -def compute_committee(indices: Tuple[ValidatorIndex, ...], - seed: Hash, index: int, count: int) -> Tuple[ValidatorIndex, ...]: +def compute_committee(indices: Sequence[ValidatorIndex], + seed: Hash, index: int, count: int) -> Sequence[ValidatorIndex]: start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return tuple(indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)) + return [indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] ``` ### `get_crosslink_committee` ```python -def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[ValidatorIndex, ...]: +def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: return compute_committee( - indices=tuple(get_active_validator_indices(state, epoch)), + indices=get_active_validator_indices(state, epoch), seed=generate_seed(state, epoch), index=(shard + SHARD_COUNT - get_epoch_start_shard(state, epoch)) % SHARD_COUNT, count=get_epoch_committee_count(state, epoch), @@ -887,13 +885,13 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> T ```python def get_attesting_indices(state: BeaconState, attestation_data: AttestationData, - bitfield: bytes) -> Tuple[ValidatorIndex, ...]: + bitfield: bytes) -> Sequence[ValidatorIndex]: """ Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. """ committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) - return tuple(sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1])) + return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) ``` ### `int_to_bytes` @@ -1139,7 +1137,7 @@ def slash_validator(state: BeaconState, ### Genesis trigger -Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool` where: +Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool` where: * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log @@ -1156,7 +1154,7 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: *Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: ```python -def is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool: +def is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool: # Process deposits state = BeaconState() for deposit in deposits: @@ -1178,7 +1176,7 @@ def is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool: Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_data)`. ```python -def get_genesis_beacon_state(deposits: Iterable[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: +def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: state = BeaconState( genesis_time=genesis_time, eth1_data=eth1_data, @@ -1195,8 +1193,10 @@ def get_genesis_beacon_state(deposits: Iterable[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots - genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, GENESIS_EPOCH)) + # Populate active_index_roots + genesis_active_index_root = hash_tree_root( + List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE](get_active_validator_indices(state, GENESIS_EPOCH)) + ) for index in range(EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root @@ -1271,17 +1271,17 @@ def process_epoch(state: BeaconState) -> None: ```python def get_total_active_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state, get_current_epoch(state))) + return get_total_balance(state, set(get_active_validator_indices(state, get_current_epoch(state)))) ``` ```python -def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: +def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: assert epoch in (get_current_epoch(state), get_previous_epoch(state)) return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` ```python -def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: +def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) if a.data.target_root == get_block_root(state, epoch) @@ -1289,7 +1289,7 @@ def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterab ``` ```python -def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: +def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) if a.data.beacon_block_root == get_block_root_at_slot(state, get_attestation_data_slot(state, a.data)) @@ -1298,22 +1298,22 @@ def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Iterable ```python def get_unslashed_attesting_indices(state: BeaconState, - attestations: Iterable[PendingAttestation]) -> Iterable[ValidatorIndex]: + attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: output = set() # type: Set[ValidatorIndex] for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) - return sorted(filter(lambda index: not state.validators[index].slashed, list(output))) + return set(filter(lambda index: not state.validators[index].slashed, list(output))) ``` ```python -def get_attesting_balance(state: BeaconState, attestations: Iterable[PendingAttestation]) -> Gwei: +def get_attesting_balance(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Gwei: return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) ``` ```python def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, - shard: Shard) -> Tuple[Crosslink, Iterable[ValidatorIndex]]: + shard: Shard) -> Tuple[Crosslink, Set[ValidatorIndex]]: attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] crosslinks = list(filter( lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_root, hash_tree_root(c)), @@ -1402,11 +1402,11 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ``` ```python -def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: +def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) - penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) + rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] eligible_validator_indices = [ ValidatorIndex(index) for index, v in enumerate(state.validators) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) @@ -1453,9 +1453,9 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: ``` ```python -def get_crosslink_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: - rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) - penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) +def get_crosslink_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) @@ -1642,7 +1642,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), - ) # type: Tuple[Tuple[List, Callable], ...] + ) # type: Sequence[Tuple[List, Callable]] for operations, function in all_operations: for operation in operations: function(state, operation) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 9c60062f48..9f0de2e6bc 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -133,7 +133,7 @@ def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard, index: int, - count: int) -> Tuple[ValidatorIndex, ...]: + count: int) -> Sequence[ValidatorIndex]: """ Return committee for a period. Used to construct persistent committees. """ @@ -159,7 +159,7 @@ def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex ```python def get_persistent_committee(state: BeaconState, shard: Shard, - slot: Slot) -> Tuple[ValidatorIndex, ...]: + slot: Slot) -> Sequence[ValidatorIndex]: """ Return the persistent committee for the given ``shard`` at the given ``slot``. """ @@ -193,7 +193,7 @@ def get_shard_proposer_index(state: BeaconState, shard: Shard, slot: Slot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee - persistent_committee = get_persistent_committee(state, shard, slot) + persistent_committee = list(get_persistent_committee(state, shard, slot)) seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) random_index = bytes_to_int(seed[0:8]) % len(persistent_committee) persistent_committee = persistent_committee[random_index:] + persistent_committee[:random_index] @@ -248,7 +248,7 @@ def verify_shard_attestation_signature(state: BeaconState, ### `compute_crosslink_data_root` ```python -def compute_crosslink_data_root(blocks: Iterable[ShardBlock]) -> Bytes32: +def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: def is_power_of_two(value: int) -> bool: return (value > 0) and (value & (value - 1) == 0) @@ -287,9 +287,9 @@ Let: * `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block` ```python -def is_valid_shard_block(beacon_blocks: TypingList[BeaconBlock], +def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], beacon_state: BeaconState, - valid_shard_blocks: Iterable[ShardBlock], + valid_shard_blocks: Sequence[ShardBlock], candidate: ShardBlock) -> bool: # Check if block is already determined valid for _, block in enumerate(valid_shard_blocks): @@ -336,7 +336,7 @@ def is_valid_shard_block(beacon_blocks: TypingList[BeaconBlock], assert proposer_index is not None assert bls_verify( pubkey=beacon_state.validators[proposer_index].pubkey, - message_hash=signing_root(block), + message_hash=signing_root(candidate), signature=candidate.signature, domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, slot_to_epoch(candidate.slot)), ) @@ -353,7 +353,7 @@ Let: * `candidate` be a candidate `ShardAttestation` for which validity is to be determined by running `is_valid_shard_attestation` ```python -def is_valid_shard_attestation(valid_shard_blocks: Iterable[ShardBlock], +def is_valid_shard_attestation(valid_shard_blocks: Sequence[ShardBlock], beacon_state: BeaconState, candidate: ShardAttestation) -> bool: # Check shard block @@ -383,7 +383,7 @@ Let: ```python def is_valid_beacon_attestation(shard: Shard, - shard_blocks: TypingList[ShardBlock], + shard_blocks: Sequence[ShardBlock], beacon_state: BeaconState, valid_attestations: Set[Attestation], candidate: Attestation) -> bool: From f95e7315b42afe8d879c270a6e354e92ad5840c5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 18:34:33 +0200 Subject: [PATCH 151/405] fix get_active_validator_indices typing usage --- specs/core/0_beacon-chain.md | 4 +++- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 3 ++- .../eth2spec/test/helpers/proposer_slashings.py | 2 +- .../pyspec/eth2spec/test/helpers/transfers.py | 2 +- .../block_processing/test_process_transfer.py | 14 +++++++------- .../test_process_voluntary_exit.py | 2 +- .../pyspec/eth2spec/test/sanity/test_blocks.py | 6 +++--- 8 files changed, 19 insertions(+), 16 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f42b4cc133..1f500bc05b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1553,7 +1553,9 @@ def process_final_updates(state: BeaconState) -> None: # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR state.active_index_roots[index_root_position] = hash_tree_root( - get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) + List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) + ) ) # Set total slashed balances state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index bc70c9fa85..cd34ee4c8d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -4,7 +4,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): current_epoch = spec.get_current_epoch(state) - revealed_index = spec.get_active_validator_indices(state, current_epoch).last() + revealed_index = spec.get_active_validator_indices(state, current_epoch)[-1] masker_index = spec.get_active_validator_indices(state, current_epoch)[0] if epoch is None: diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index d1d8189080..73738932a6 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -40,7 +40,8 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - genesis_active_index_root = hash_tree_root(spec.get_active_validator_indices(state, spec.GENESIS_EPOCH)) + genesis_active_index_root = hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index 7b6f713740..d5b7f7b7fd 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -6,7 +6,7 @@ def get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=False): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch).last() + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] slot = state.slot diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index 215cd6fcb5..acc6a35c53 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -9,7 +9,7 @@ def get_valid_transfer(spec, state, slot=None, sender_index=None, amount=None, f slot = state.slot current_epoch = spec.get_current_epoch(state) if sender_index is None: - sender_index = spec.get_active_validator_indices(state, current_epoch).last() + sender_index = spec.get_active_validator_indices(state, current_epoch)[-1] recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] transfer_pubkey = pubkeys[-1] transfer_privkey = privkeys[-1] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 2bed94dc81..e9d282b3a6 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -63,7 +63,7 @@ def test_success_withdrawable(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -73,7 +73,7 @@ def test_success_active_above_max_effective(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -94,7 +94,7 @@ def test_invalid_signature(spec, state): @with_all_phases @spec_state_test def test_active_but_transfer_past_effective_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=amount, fee=0, signed=True) @@ -115,7 +115,7 @@ def test_incorrect_slot(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance_for_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -128,7 +128,7 @@ def test_insufficient_balance_for_fee(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -141,7 +141,7 @@ def test_insufficient_balance(spec, state): @with_all_phases @spec_state_test def test_no_dust_sender(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] balance = state.balances[sender_index] transfer = get_valid_transfer( spec, @@ -161,7 +161,7 @@ def test_no_dust_sender(spec, state): @with_all_phases @spec_state_test def test_no_dust_recipient(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) state.balances[transfer.recipient] = 0 diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index b9f7d60981..33cacc4e20 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -93,7 +93,7 @@ def test_success_exit_queue(spec, state): continue # exit an additional validator - validator_index = spec.get_active_validator_indices(state, current_epoch).last() + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit( spec, diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index c86394a0b1..f8590005e1 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -269,7 +269,7 @@ def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices( state, spec.get_current_epoch(state) - ).last() + )[-1] # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -315,7 +315,7 @@ def test_voluntary_exit(spec, state): # overwrite default 0 to test # spec.MAX_TRANSFERS = 1 - # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] # amount = get_balance(state, sender_index) # transfer = get_valid_transfer(spec, state, state.slot + 1, sender_index, amount, signed=True) @@ -347,7 +347,7 @@ def test_voluntary_exit(spec, state): @spec_state_test def test_balance_driven_status_transitions(spec, state): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch).last() + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] assert state.validators[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH From dd5ad2e2c56b7bc47da3c47c8fa5ed87d1eaa650 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 19:48:06 +0200 Subject: [PATCH 152/405] remove unnecessary (and now outdated) type hints, update List encoding for generators --- test_libs/pyspec/eth2spec/debug/encode.py | 4 +-- .../pyspec/eth2spec/test/helpers/genesis.py | 1 + .../eth2spec/test/sanity/test_blocks.py | 28 +++++++++---------- .../pyspec/eth2spec/test/test_finality.py | 11 ++++---- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index d264bd7ff1..e45bd11174 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -13,9 +13,9 @@ def encode(value: SSZValue, include_hash_tree_roots=False): elif isinstance(value, Bit): assert value in (True, False) return value - elif isinstance(value, (List, Vector)): + elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] - elif isinstance(value, (Bytes, BytesN)): # both bytes and BytesN + elif isinstance(value, bytes): # both bytes and BytesN return '0x' + value.hex() elif isinstance(value, Container): ret = {} diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 73738932a6..680825165e 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -1,5 +1,6 @@ from eth2spec.test.helpers.keys import pubkeys from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import List def build_mock_validator(spec, i: int, balance: int): diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index f8590005e1..36ded7bc6c 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -28,7 +28,7 @@ def test_empty_block_transition(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert len(state.eth1_data_votes) == pre_eth1_votes + 1 @@ -48,7 +48,7 @@ def test_skipped_slots(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.slot == block.slot @@ -69,7 +69,7 @@ def test_empty_epoch_transition(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.slot == block.slot @@ -90,7 +90,7 @@ def test_empty_epoch_transition(spec, state): # state_transition_and_sign_block(spec, state, block) -# yield 'blocks', [block], List[spec.BeaconBlock] +# yield 'blocks', [block] # yield 'post', state # assert state.slot == block.slot @@ -120,7 +120,7 @@ def test_proposer_slashing(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state # check if slashed @@ -155,7 +155,7 @@ def test_attester_slashing(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state slashed_validator = state.validators[validator_index] @@ -193,7 +193,7 @@ def test_deposit_in_block(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert len(state.validators) == initial_registry_len + 1 @@ -221,7 +221,7 @@ def test_deposit_top_up(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert len(state.validators) == initial_registry_len @@ -256,7 +256,7 @@ def test_attestation(spec, state): sign_block(spec, state, epoch_block) state_transition_and_sign_block(spec, state, epoch_block) - yield 'blocks', [attestation_block, epoch_block], List[spec.BeaconBlock] + yield 'blocks', [attestation_block, epoch_block] yield 'post', state assert len(state.current_epoch_attestations) == 0 @@ -303,7 +303,7 @@ def test_voluntary_exit(spec, state): sign_block(spec, state, exit_block) state_transition_and_sign_block(spec, state, exit_block) - yield 'blocks', [initiate_exit_block, exit_block], List[spec.BeaconBlock] + yield 'blocks', [initiate_exit_block, exit_block] yield 'post', state assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @@ -334,7 +334,7 @@ def test_voluntary_exit(spec, state): # state_transition_and_sign_block(spec, state, block) - # yield 'blocks', [block], List[spec.BeaconBlock] + # yield 'blocks', [block] # yield 'post', state # sender_balance = get_balance(state, sender_index) @@ -362,7 +362,7 @@ def test_balance_driven_status_transitions(spec, state): sign_block(spec, state, block) state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @@ -379,7 +379,7 @@ def test_historical_batch(spec, state): block = build_empty_block_for_next_slot(spec, state, signed=True) state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.slot == block.slot @@ -408,7 +408,7 @@ def test_historical_batch(spec, state): # state_transition_and_sign_block(spec, state, block) -# yield 'blocks', [block], List[spec.BeaconBlock] +# yield 'blocks', [block] # yield 'post', state # assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 7304562862..cf9e94a715 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,5 +1,4 @@ from copy import deepcopy -from typing import List from eth2spec.test.context import spec_state_test, never_bls, with_all_phases from eth2spec.test.helpers.state import next_epoch, state_transition_and_sign_block @@ -82,7 +81,7 @@ def test_finality_no_updates_at_genesis(spec, state): elif epoch == 1: check_finality(spec, state, prev_state, False, False, False) - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -111,7 +110,7 @@ def test_finality_rule_4(spec, state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -142,7 +141,7 @@ def test_finality_rule_1(spec, state): assert state.finalized_epoch == prev_state.previous_justified_epoch assert state.finalized_root == prev_state.previous_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -175,7 +174,7 @@ def test_finality_rule_2(spec, state): blocks += new_blocks - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -225,5 +224,5 @@ def test_finality_rule_3(spec, state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state From e873bbd73bc3d05526b3c3f42ea3e1bd88049eb9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 19:59:15 +0200 Subject: [PATCH 153/405] support list casting --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 14415ac6b0..891480a5c2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -296,7 +296,7 @@ def value_check(cls, value): @classmethod def extract_args(cls, *args): x = list(args) - if len(x) == 1 and isinstance(x[0], GeneratorType): + if len(x) == 1 and isinstance(x[0], (GeneratorType, list, tuple)): x = list(x[0]) x = [coerce_type_maybe(v, cls.elem_type) for v in x] return x diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 4325501aac..bd86a58063 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -161,6 +161,8 @@ def test_list(): assert issubclass(v.type(), Elements) assert isinstance(v.type(), ElementsType) + assert len(typ([i for i in range(10)])) == 10 # cast py list to SSZ list + foo = List[uint32, 128](0 for i in range(128)) foo[0] = 123 foo[1] = 654 From 47034a6c8c40fbe5beb3aba36f264f92863682d2 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 19:59:44 +0200 Subject: [PATCH 154/405] fix imports in helper test file --- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 680825165e..d3d67e09e9 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -41,7 +41,7 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - genesis_active_index_root = hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_SIZE]( spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root From 8a732fbbcfbfae529cc34458681a965e135a36f8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 22 Jun 2019 12:00:26 -0600 Subject: [PATCH 155/405] pass on checkpoints working in testing --- specs/core/0_beacon-chain.md | 12 +++---- .../eth2spec/test/helpers/attestations.py | 22 ++++++------- .../test/helpers/attester_slashings.py | 2 +- .../test_process_attestation.py | 31 ++++++++--------- .../test_process_attester_slashing.py | 10 +++--- .../test_process_crosslinks.py | 4 +-- .../pyspec/eth2spec/test/test_finality.py | 33 ++++++++----------- 7 files changed, 52 insertions(+), 62 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cf1d368620..838178ea48 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1344,28 +1344,28 @@ def process_justification_and_finalization(state: BeaconState) -> None: state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: + if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_checkpoint.epoch + 3 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: + if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_checkpoint.epoch+ 2 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: + if (bitfield >> 0) % 8 == 0b111 and old_current_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: + if (bitfield >> 0) % 4 == 0b11 and old_current_justified_checkpoint.epoch + 1 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint ``` diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 4c8b5c7ebb..d2ce6e29b3 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -24,11 +24,11 @@ def build_attestation_data(spec, state, slot, shard): epoch_boundary_root = spec.get_block_root(state, spec.get_current_epoch(state)) if slot < current_epoch_start_slot: - justified_epoch = state.previous_justified_epoch - justified_block_root = state.previous_justified_root + source_epoch = state.previous_justified_checkpoint.epoch + source_root = state.previous_justified_checkpoint.root else: - justified_epoch = state.current_justified_epoch - justified_block_root = state.current_justified_root + source_epoch = state.current_justified_checkpoint.epoch + source_root = state.current_justified_checkpoint.root if spec.slot_to_epoch(slot) == spec.get_current_epoch(state): parent_crosslink = state.current_crosslinks[shard] @@ -37,10 +37,8 @@ def build_attestation_data(spec, state, slot, shard): return spec.AttestationData( beacon_block_root=block_root, - source_epoch=justified_epoch, - source_root=justified_block_root, - target_epoch=spec.slot_to_epoch(slot), - target_root=epoch_boundary_root, + source_checkpoint=spec.Checkpoint(epoch=source_epoch, root=source_root), + target_checkpoint=spec.Checkpoint(epoch=spec.slot_to_epoch(slot), root=epoch_boundary_root), crosslink=spec.Crosslink( shard=shard, start_epoch=parent_crosslink.end_epoch, @@ -64,8 +62,8 @@ def get_valid_attestation(spec, state, slot=None, signed=False): crosslink_committee = spec.get_crosslink_committee( state, - attestation_data.target_epoch, - attestation_data.crosslink.shard + attestation_data.target_checkpoint.epoch, + attestation_data.crosslink.shard, ) committee_size = len(crosslink_committee) @@ -126,7 +124,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi domain=spec.get_domain( state=state, domain_type=spec.DOMAIN_ATTESTATION, - message_epoch=attestation_data.target_epoch, + message_epoch=attestation_data.target_checkpoint.epoch, ) ) @@ -134,7 +132,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi def fill_aggregate_attestation(spec, state, attestation): crosslink_committee = spec.get_crosslink_committee( state, - attestation.data.target_epoch, + attestation.data.target_checkpoint.epoch, attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index 9fd34520c2..e339421e84 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -7,7 +7,7 @@ def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): attestation_1 = get_valid_attestation(spec, state, signed=signed_1) attestation_2 = deepcopy(attestation_1) - attestation_2.data.target_root = b'\x01' * 32 + attestation_2.data.target_checkpoint.root = b'\x01' * 32 if signed_2: sign_attestation(spec, state, attestation_2) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 2b34ab405e..9a718b0715 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -38,7 +38,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): spec.process_attestation(state, attestation) # Make sure the attestation has been processed - if attestation.data.target_epoch == spec.get_current_epoch(state): + if attestation.data.target_checkpoint.epoch == spec.get_current_epoch(state): assert len(state.current_epoch_attestations) == current_epoch_count + 1 else: assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 @@ -119,16 +119,16 @@ def test_after_epoch_slots(spec, state): @spec_state_test def test_old_source_epoch(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 - state.finalized_epoch = 2 - state.previous_justified_epoch = 3 - state.current_justified_epoch = 4 + state.finalized_checkpoint.epoch = 2 + state.previous_justified_checkpoint.epoch = 3 + state.current_justified_checkpoint.epoch = 4 attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) # test logic sanity check: make sure the attestation is pointing to oldest known source epoch - assert attestation.data.source_epoch == state.previous_justified_epoch + assert attestation.data.source_checkpoint.epoch == state.previous_justified_checkpoint.epoch # Now go beyond that, it will be invalid - attestation.data.source_epoch -= 1 + attestation.data.source_checkpoint.epoch -= 1 sign_attestation(spec, state, attestation) @@ -154,7 +154,7 @@ def test_new_source_epoch(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_epoch += 1 + attestation.data.source_checkpoint.epoch += 1 sign_attestation(spec, state, attestation) @@ -167,7 +167,7 @@ def test_source_root_is_target_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_root = attestation.data.target_root + attestation.data.source_checkpoint.root = attestation.data.target_checkpoint.root sign_attestation(spec, state, attestation) @@ -180,21 +180,18 @@ def test_invalid_current_source_root(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 state.finalized_epoch = 2 - state.previous_justified_epoch = 3 - state.previous_justified_root = b'\x01' * 32 - - state.current_justified_epoch = 4 - state.current_justified_root = b'\xff' * 32 + state.previous_justified_checkpoint = spec.Checkpoint(epoch=3, root=b'\x01'*32) + state.current_justified_checkpoint = spec.Checkpoint(epoch=4, root=b'\x32'*32) attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY # Test logic sanity checks: - assert state.current_justified_root != state.previous_justified_root - assert attestation.data.source_root == state.previous_justified_root + assert state.current_justified_checkpoint.root != state.previous_justified_checkpoint.root + assert attestation.data.source_checkpoint.root == state.previous_justified_checkpoint.root # Make attestation source root invalid: should be previous justified, not current one - attestation.data.source_root = state.current_justified_root + attestation.data.source_checkpoint.root = state.current_justified_checkpoint.root sign_attestation(spec, state, attestation) @@ -207,7 +204,7 @@ def test_bad_source_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_root = b'\x42' * 32 + attestation.data.source_checkpoint.root = b'\x42' * 32 sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index c51f5a8a90..71915f1d08 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -68,12 +68,14 @@ def test_success_surround(spec, state): next_epoch(spec, state) apply_empty_block(spec, state) - state.current_justified_epoch += 1 + state.current_justified_checkpoint.epoch += 1 attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) + attestation_1 = attester_slashing.attestation_1 + attestation_2 = attester_slashing.attestation_2 # set attestion1 to surround attestation 2 - attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1 - attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 + attestation_1.data.source_checkpoint.epoch = attestation_2.data.source_checkpoint.epoch - 1 + attestation_1.data.target_checkpoint.epoch = attestation_2.data.target_checkpoint.epoch + 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) @@ -120,7 +122,7 @@ def test_same_data(spec, state): def test_no_double_or_surround(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - attester_slashing.attestation_1.data.target_epoch += 1 + attester_slashing.attestation_1.data.target_checkpoint.epoch += 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index d51191efba..482085fd10 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -96,7 +96,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): # ensure rewarded for index in spec.get_crosslink_committee( state, - attestation.data.target_epoch, + attestation.data.target_checkpoint.epoch, attestation.data.crosslink.shard): assert crosslink_deltas[0][index] > 0 assert crosslink_deltas[1][index] == 0 @@ -148,7 +148,7 @@ def test_double_late_crosslink(spec, state): # ensure no reward, only penalties for the failed crosslink for index in spec.get_crosslink_committee( state, - attestation_2.data.target_epoch, + attestation_2.data.target_checkpoint.epoch, attestation_2.data.crosslink.shard): assert crosslink_deltas[0][index] == 0 assert crosslink_deltas[1][index] > 0 diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 5e81f52c88..e405312ebd 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -14,25 +14,22 @@ def check_finality(spec, previous_justified_changed, finalized_changed): if current_justified_changed: - assert state.current_justified_epoch > prev_state.current_justified_epoch - assert state.current_justified_root != prev_state.current_justified_root + assert state.current_justified_checkpoint.epoch > prev_state.current_justified_checkpoint.epoch + assert state.current_justified_checkpoint.root != prev_state.current_justified_checkpoint.root else: - assert state.current_justified_epoch == prev_state.current_justified_epoch - assert state.current_justified_root == prev_state.current_justified_root + assert state.current_justified_checkpoint == prev_state.current_justified_checkpoint if previous_justified_changed: - assert state.previous_justified_epoch > prev_state.previous_justified_epoch - assert state.previous_justified_root != prev_state.previous_justified_root + assert state.previous_justified_checkpoint.epoch > prev_state.previous_justified_checkpoint.epoch + assert state.previous_justified_checkpoint.root != prev_state.previous_justified_checkpoint.root else: - assert state.previous_justified_epoch == prev_state.previous_justified_epoch - assert state.previous_justified_root == prev_state.previous_justified_root + assert state.previous_justified_checkpoint == prev_state.previous_justified_checkpoint if finalized_changed: - assert state.finalized_epoch > prev_state.finalized_epoch - assert state.finalized_root != prev_state.finalized_root + assert state.finalized_checkpoint.epoch > prev_state.finalized_checkpoint.epoch + assert state.finalized_checkpoint.root != prev_state.finalized_checkpoint.root else: - assert state.finalized_epoch == prev_state.finalized_epoch - assert state.finalized_root == prev_state.finalized_root + assert state.finalized_checkpoint == prev_state.finalized_checkpoint def next_epoch_with_attestations(spec, @@ -82,8 +79,7 @@ def test_finality_rule_4(spec, state): elif epoch >= 3: # rule 4 of finality check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root + assert state.finalized_checkpoint == prev_state.current_justified_checkpoint yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -113,8 +109,7 @@ def test_finality_rule_1(spec, state): elif epoch == 2: # finalized by rule 1 check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root + assert state.finalized_checkpoint == prev_state.previous_justified_checkpoint yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -144,8 +139,7 @@ def test_finality_rule_2(spec, state): prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) # finalized by rule 2 check_finality(spec, state, prev_state, True, False, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root + assert state.finalized_checkpoint == prev_state.previous_justified_checkpoint blocks += new_blocks @@ -196,8 +190,7 @@ def test_finality_rule_3(spec, state): blocks += new_blocks # rule 3 check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root + assert state.finalized_checkpoint == prev_state.current_justified_checkpoint yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state From 0249cf651e2c30c6c343dd6c2b2440f17b59a2cd Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 20:04:17 +0200 Subject: [PATCH 156/405] fix lint, and update encoder to handle the few imported types well --- test_libs/pyspec/eth2spec/debug/encode.py | 5 ++--- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index e45bd11174..9a819a2563 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZValue, uint, Container, Bytes, BytesN, List, Vector, Bit + SSZValue, uint, Container, Bit ) @@ -11,8 +11,7 @@ def encode(value: SSZValue, include_hash_tree_roots=False): return str(value) return value elif isinstance(value, Bit): - assert value in (True, False) - return value + return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] elif isinstance(value, bytes): # both bytes and BytesN diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 36ded7bc6c..286c0150ce 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -1,5 +1,4 @@ from copy import deepcopy -from typing import List from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.utils.bls import bls_sign From 9befe09f82605c6cf103c9c4fbd120ad50977831 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 21:27:56 +0200 Subject: [PATCH 157/405] test merkleize chunks --- .../eth2spec/utils/test_merkle_minimal.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py diff --git a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py new file mode 100644 index 0000000000..1c72a649b6 --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py @@ -0,0 +1,58 @@ +import pytest +from .merkle_minimal import zerohashes, merkleize_chunks +from .hash_function import hash + + +def h(a: bytes, b: bytes) -> bytes: + return hash(a + b) + + +def e(v: int) -> bytes: + return v.to_bytes(length=32, byteorder='little') + + +def z(i: int) -> bytes: + return zerohashes[i] + + +cases = [ + (0, 0, 1, z(0)), + (0, 1, 1, e(0)), + (1, 0, 2, h(z(0), z(0))), + (1, 1, 2, h(e(0), z(0))), + (1, 2, 2, h(e(0), e(1))), + (2, 0, 4, h(h(z(0), z(0)), z(1))), + (2, 1, 4, h(h(e(0), z(0)), z(1))), + (2, 2, 4, h(h(e(0), e(1)), z(1))), + (2, 3, 4, h(h(e(0), e(1)), h(e(2), z(0)))), + (2, 4, 4, h(h(e(0), e(1)), h(e(2), e(3)))), + (3, 0, 8, h(h(h(z(0), z(0)), z(1)), z(2))), + (3, 1, 8, h(h(h(e(0), z(0)), z(1)), z(2))), + (3, 2, 8, h(h(h(e(0), e(1)), z(1)), z(2))), + (3, 3, 8, h(h(h(e(0), e(1)), h(e(2), z(0))), z(2))), + (3, 4, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), z(2))), + (3, 5, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), z(0)), z(1)))), + (3, 6, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(z(0), z(0))))), + (3, 7, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), z(0))))), + (3, 8, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7))))), + (4, 0, 16, h(h(h(h(z(0), z(0)), z(1)), z(2)), z(3))), + (4, 1, 16, h(h(h(h(e(0), z(0)), z(1)), z(2)), z(3))), + (4, 2, 16, h(h(h(h(e(0), e(1)), z(1)), z(2)), z(3))), + (4, 3, 16, h(h(h(h(e(0), e(1)), h(e(2), z(0))), z(2)), z(3))), + (4, 4, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), z(2)), z(3))), + (4, 5, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), z(0)), z(1))), z(3))), + (4, 6, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(z(0), z(0)))), z(3))), + (4, 7, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), z(0)))), z(3))), + (4, 8, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7)))), z(3))), + (4, 9, 16, + h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7)))), h(h(h(e(8), z(0)), z(1)), z(2)))), +] + + +@pytest.mark.parametrize( + 'depth,count,pow2,value', + cases, +) +def test_merkleize_chunks(depth, count, pow2, value): + chunks = [e(i) for i in range(count)] + assert merkleize_chunks(chunks, pad_to=pow2) == value From da858f1aae3ff5f906e041f025094d0a90ccc0c5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 21:49:42 +0200 Subject: [PATCH 158/405] fix int encoding, fix list randomization size limit. --- test_libs/pyspec/eth2spec/debug/encode.py | 4 ++-- test_libs/pyspec/eth2spec/debug/random_value.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 9a819a2563..0878a1f947 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -8,8 +8,8 @@ def encode(value: SSZValue, include_hash_tree_roots=False): if isinstance(value, uint): # Larger uints are boxed and the class declares their byte length if value.type().byte_len > 8: - return str(value) - return value + return str(int(value)) + return int(value) elif isinstance(value, Bit): return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index bd40cb8320..8e13cd5e1d 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -95,6 +95,9 @@ def get_random_ssz_object(rng: Random, elif mode == RandomizationMode.mode_max_count: length = max_list_length + if typ.length < length: # SSZ imposes a hard limit on lists, we can't put in more than that + length = typ.length + return typ( get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(length) From a5a2e29dfe8a424ea53cf0b9203b58463046a24b Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 22:15:42 +0200 Subject: [PATCH 159/405] remove unnecessary argument, typing is based on values fully now --- test_generators/ssz_static/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 9d9af8c5e5..0dfdebf5dc 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -21,8 +21,8 @@ @to_dict -def create_test_case_contents(value, typ): - yield "value", encode.encode(value, typ) +def create_test_case_contents(value): + yield "value", encode.encode(value) yield "serialized", '0x' + serialize(value).hex() yield "root", '0x' + hash_tree_root(value).hex() if hasattr(value, "signature"): @@ -32,7 +32,7 @@ def create_test_case_contents(value, typ): @to_dict def create_test_case(rng: Random, name: str, typ, mode: random_value.RandomizationMode, chaos: bool): value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) - yield name, create_test_case_contents(value, typ) + yield name, create_test_case_contents(value) def get_spec_ssz_types(): From 181a2a876a369051a8a6db3b0c972d13cdb81202 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:49:53 +0200 Subject: [PATCH 160/405] Cleanups --- specs/core/0_beacon-chain.md | 38 ++++++++----------- .../eth2spec/test/helpers/attestations.py | 10 ++--- .../test/helpers/attester_slashings.py | 2 +- .../test_process_attestation.py | 16 ++++---- .../test_process_attester_slashing.py | 6 +-- .../test_process_crosslinks.py | 4 +- 6 files changed, 35 insertions(+), 41 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 838178ea48..d8d3d6a76d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -346,8 +346,8 @@ class AttestationData(Container): # LMD GHOST vote beacon_block_root: Hash # FFG vote - source_checkpoint: Checkpoint - target_checkpoint: Checkpoint + source: Checkpoint + target: Checkpoint # Crosslink vote crosslink: Crosslink ``` @@ -719,10 +719,9 @@ def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: ```python def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot: - target_epoch = data.target_checkpoint.epoch - committee_count = get_epoch_committee_count(state, target_epoch) - offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, target_epoch)) % SHARD_COUNT - return Slot(get_epoch_start_slot(target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) + committee_count = get_epoch_committee_count(state, data.target.epoch) + offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, data.target.epoch)) % SHARD_COUNT + return Slot(get_epoch_start_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` ### `get_block_root_at_slot` @@ -886,7 +885,7 @@ def get_attesting_indices(state: BeaconState, """ Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. """ - committee = get_crosslink_committee(state, data.target_checkpoint.epoch, data.crosslink.shard) + committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) ``` @@ -1006,7 +1005,7 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), ], signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_checkpoint.epoch), + domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target.epoch), ) ``` @@ -1019,12 +1018,9 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa """ return ( # Double vote - (data_1 != data_2 and data_1.target_checkpoint.epoch == data_2.target_checkpoint.epoch) or + (data_1 != data_2 and data_1.target.epoch == data_2.target.epoch) or # Surround vote - ( - data_1.source_checkpoint.epoch < data_2.source_checkpoint.epoch and - data_2.target_checkpoint.epoch < data_1.target_checkpoint.epoch - ) + (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch) ) ``` @@ -1282,7 +1278,7 @@ def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[P def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) - if a.data.target_checkpoint.root == get_block_root(state, epoch) + if a.data.target.root == get_block_root(state, epoch) ] ``` @@ -1344,13 +1340,13 @@ def process_justification_and_finalization(state: BeaconState) -> None: state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations @@ -1692,10 +1688,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: Process ``Attestation`` operation. """ data = attestation.data - target_epoch = data.target_checkpoint.epoch - assert data.crosslink.shard < SHARD_COUNT - assert target_epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) attestation_slot = get_attestation_data_slot(state, data) assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH @@ -1707,7 +1701,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_index=get_beacon_proposer_index(state), ) - if target_epoch == get_current_epoch(state): + if data.target.epoch == get_current_epoch(state): ffg_source = state.current_justified_checkpoint parent_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) @@ -1717,9 +1711,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: state.previous_epoch_attestations.append(pending_attestation) # Check FFG source, crosslink data, and signature - assert ffg_source == data.source_checkpoint + assert ffg_source == data.source assert data.crosslink.start_epoch == parent_crosslink.end_epoch - assert data.crosslink.end_epoch == min(target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) + assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] validate_indexed_attestation(state, convert_to_indexed(state, attestation)) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index d2ce6e29b3..d717bdc0db 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -37,8 +37,8 @@ def build_attestation_data(spec, state, slot, shard): return spec.AttestationData( beacon_block_root=block_root, - source_checkpoint=spec.Checkpoint(epoch=source_epoch, root=source_root), - target_checkpoint=spec.Checkpoint(epoch=spec.slot_to_epoch(slot), root=epoch_boundary_root), + source=spec.Checkpoint(epoch=source_epoch, root=source_root), + target=spec.Checkpoint(epoch=spec.slot_to_epoch(slot), root=epoch_boundary_root), crosslink=spec.Crosslink( shard=shard, start_epoch=parent_crosslink.end_epoch, @@ -62,7 +62,7 @@ def get_valid_attestation(spec, state, slot=None, signed=False): crosslink_committee = spec.get_crosslink_committee( state, - attestation_data.target_checkpoint.epoch, + attestation_data.target.epoch, attestation_data.crosslink.shard, ) @@ -124,7 +124,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi domain=spec.get_domain( state=state, domain_type=spec.DOMAIN_ATTESTATION, - message_epoch=attestation_data.target_checkpoint.epoch, + message_epoch=attestation_data.target.epoch, ) ) @@ -132,7 +132,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi def fill_aggregate_attestation(spec, state, attestation): crosslink_committee = spec.get_crosslink_committee( state, - attestation.data.target_checkpoint.epoch, + attestation.data.target.epoch, attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index e339421e84..9c68b7bbe4 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -7,7 +7,7 @@ def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): attestation_1 = get_valid_attestation(spec, state, signed=signed_1) attestation_2 = deepcopy(attestation_1) - attestation_2.data.target_checkpoint.root = b'\x01' * 32 + attestation_2.data.target.root = b'\x01' * 32 if signed_2: sign_attestation(spec, state, attestation_2) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 9a718b0715..d6cb615f80 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -38,7 +38,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): spec.process_attestation(state, attestation) # Make sure the attestation has been processed - if attestation.data.target_checkpoint.epoch == spec.get_current_epoch(state): + if attestation.data.target.epoch == spec.get_current_epoch(state): assert len(state.current_epoch_attestations) == current_epoch_count + 1 else: assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 @@ -125,10 +125,10 @@ def test_old_source_epoch(spec, state): attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) # test logic sanity check: make sure the attestation is pointing to oldest known source epoch - assert attestation.data.source_checkpoint.epoch == state.previous_justified_checkpoint.epoch + assert attestation.data.source.epoch == state.previous_justified_checkpoint.epoch # Now go beyond that, it will be invalid - attestation.data.source_checkpoint.epoch -= 1 + attestation.data.source.epoch -= 1 sign_attestation(spec, state, attestation) @@ -154,7 +154,7 @@ def test_new_source_epoch(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_checkpoint.epoch += 1 + attestation.data.source.epoch += 1 sign_attestation(spec, state, attestation) @@ -167,7 +167,7 @@ def test_source_root_is_target_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_checkpoint.root = attestation.data.target_checkpoint.root + attestation.data.source.root = attestation.data.target.root sign_attestation(spec, state, attestation) @@ -188,10 +188,10 @@ def test_invalid_current_source_root(spec, state): # Test logic sanity checks: assert state.current_justified_checkpoint.root != state.previous_justified_checkpoint.root - assert attestation.data.source_checkpoint.root == state.previous_justified_checkpoint.root + assert attestation.data.source.root == state.previous_justified_checkpoint.root # Make attestation source root invalid: should be previous justified, not current one - attestation.data.source_checkpoint.root = state.current_justified_checkpoint.root + attestation.data.source.root = state.current_justified_checkpoint.root sign_attestation(spec, state, attestation) @@ -204,7 +204,7 @@ def test_bad_source_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_checkpoint.root = b'\x42' * 32 + attestation.data.source.root = b'\x42' * 32 sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 71915f1d08..e2b50ea0b2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -74,8 +74,8 @@ def test_success_surround(spec, state): attestation_2 = attester_slashing.attestation_2 # set attestion1 to surround attestation 2 - attestation_1.data.source_checkpoint.epoch = attestation_2.data.source_checkpoint.epoch - 1 - attestation_1.data.target_checkpoint.epoch = attestation_2.data.target_checkpoint.epoch + 1 + attestation_1.data.source.epoch = attestation_2.data.source.epoch - 1 + attestation_1.data.target.epoch = attestation_2.data.target.epoch + 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) @@ -122,7 +122,7 @@ def test_same_data(spec, state): def test_no_double_or_surround(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - attester_slashing.attestation_1.data.target_checkpoint.epoch += 1 + attester_slashing.attestation_1.data.target.epoch += 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index 482085fd10..090858a2eb 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -96,7 +96,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): # ensure rewarded for index in spec.get_crosslink_committee( state, - attestation.data.target_checkpoint.epoch, + attestation.data.target.epoch, attestation.data.crosslink.shard): assert crosslink_deltas[0][index] > 0 assert crosslink_deltas[1][index] == 0 @@ -148,7 +148,7 @@ def test_double_late_crosslink(spec, state): # ensure no reward, only penalties for the failed crosslink for index in spec.get_crosslink_committee( state, - attestation_2.data.target_checkpoint.epoch, + attestation_2.data.target.epoch, attestation_2.data.crosslink.shard): assert crosslink_deltas[0][index] == 0 assert crosslink_deltas[1][index] > 0 From e6e90c7736e69759c59dd6cf6fa2efc9c11e1d3c Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:51:04 +0200 Subject: [PATCH 161/405] Minor reorg --- specs/core/0_beacon-chain.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d8d3d6a76d..ae99a807ae 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -540,11 +540,10 @@ class BeaconState(Container): # Crosslinks previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] - # Justification + # Finality + justification_bitfield: uint64 # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint - justification_bitfield: uint64 # Bit set for every recent justified epoch - # Finality finalized_checkpoint: Checkpoint ``` From 98692bf9d69198c8079d4af9458e5b4e13243cb2 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:52:37 +0200 Subject: [PATCH 162/405] Fix ToC --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ae99a807ae..a40bef6d17 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -25,8 +25,8 @@ - [Containers](#containers) - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) - - [`Validator`](#validator) - [`Checkpoint`](#checkpoint) + - [`Validator`](#validator) - [`Crosslink`](#crosslink) - [`AttestationData`](#attestationdata) - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) From 2342c787c96b034e9c2db08445b15fdc3a7f8b50 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:54:07 +0200 Subject: [PATCH 163/405] Cleanup --- specs/core/0_beacon-chain.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a40bef6d17..30876ae297 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -878,9 +878,7 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> L ### `get_attesting_indices` ```python -def get_attesting_indices(state: BeaconState, - data: AttestationData, - bitfield: bytes) -> List[ValidatorIndex]: +def get_attesting_indices(state: BeaconState, data: AttestationData, bitfield: bytes) -> List[ValidatorIndex]: """ Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. """ From a3f8f50beed9f66801ce266b192b234c2dd11299 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 23:34:09 +0200 Subject: [PATCH 164/405] Initialise deposit root properly --- specs/core/0_beacon-chain.md | 38 ++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8f57967edd..8aa307d666 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1135,11 +1135,10 @@ Before genesis has been triggered and whenever the deposit contract emits a `Dep * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log -When `is_genesis_trigger(deposits, timestamp, deposit_root) is True` for the first time let: +When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: * `genesis_deposits = deposits` * `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` -* `deposit_root` is the Merkle tree root of the data of the given `deposits` * `genesis_eth1_data` be the object of type `Eth1Data` where: * `genesis_eth1_data.deposit_root` is the deposit root for the last deposit in `deposits` * `genesis_eth1_data.deposit_count = len(genesis_deposits)` @@ -1148,16 +1147,20 @@ When `is_genesis_trigger(deposits, timestamp, deposit_root) is True` for the fir *Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: ```python -def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64, deposit_root: Hash) -> bool: - # Process deposits +def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: + # Initialize deposit root state = BeaconState() - for index, deposit in enumerate(deposits): - process_deposit(state, deposit, deposit_index=index, deposit_root=deposit_root) + for i in range(DEPOSIT_CONTRACT_TREE_DEPTH - 1): + state.eth1_data.deposit_root = hash(state.eth1_data.deposit_root + state.eth1_data.deposit_root) + + # Process deposits + for deposit in deposits: + process_deposit(state, deposit) # Count active validators at genesis active_validator_count = 0 for validator in state.validators: - if validator.effective_balance >= MAX_EFFECTIVE_BALANCE: + if validator.effective_balance == MAX_EFFECTIVE_BALANCE: active_validator_count += 1 # Check effective balance to trigger genesis @@ -1177,12 +1180,12 @@ def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis ) # Process genesis deposits - for deposit_index, deposit in enumerate(deposits): - process_deposit(state, deposit, deposit_index=deposit_index) + for deposit in deposits: + process_deposit(state, deposit) # Process genesis activations for validator in state.validators: - if validator.effective_balance >= MAX_EFFECTIVE_BALANCE: + if validator.effective_balance == MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH @@ -1483,7 +1486,7 @@ def process_registry_updates(state: BeaconState) -> None: for index, validator in enumerate(state.validators): if ( validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and - validator.effective_balance >= MAX_EFFECTIVE_BALANCE + validator.effective_balance == MAX_EFFECTIVE_BALANCE ): validator.activation_eligibility_epoch = get_current_epoch(state) @@ -1727,24 +1730,17 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ##### Deposits ```python -def process_deposit(state: BeaconState, - deposit: Deposit, - deposit_index: Optional[uint64]=None, - deposit_root: Optional[Hash]=None) -> None: +def process_deposit(state: BeaconState, deposit: Deposit) -> None: """ Process an Eth1 deposit, registering a validator or increasing its balance. """ - if deposit_index is None: - deposit_index = state.eth1_deposit_index - if deposit_root is None: - deposit_root = state.eth1_data.deposit_root # Verify the Merkle branch assert verify_merkle_branch( leaf=hash_tree_root(deposit.data), proof=deposit.proof, depth=DEPOSIT_CONTRACT_TREE_DEPTH, - index=deposit_index, - root=deposit_root, + index=state.eth1_deposit_index, + root=state.eth1_data.deposit_root, ) # Deposits must be processed in order From 1408a1ee0daf918a4fdc72facad4e22bf56d9d79 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 23 Jun 2019 00:17:54 +0200 Subject: [PATCH 165/405] undo tuple wrapping --- specs/core/1_shard-data-chains.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 9f0de2e6bc..f84b0dd46e 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -180,10 +180,10 @@ def get_persistent_committee(state: BeaconState, # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated - return tuple(sorted(list(set( + return sorted(list(set( [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(state, epoch, i)] + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(state, epoch, i)] - )))) + ))) ``` ### `get_shard_proposer_index` From 1e2bb08a747abc3b9ef4c5434c074bc078360a53 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 23 Jun 2019 11:09:09 +0200 Subject: [PATCH 166/405] Cosmetic fix for consistency --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 30876ae297..c062bbf78d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1267,7 +1267,7 @@ def get_total_active_balance(state: BeaconState) -> Gwei: ```python def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: - assert epoch in (get_current_epoch(state), get_previous_epoch(state)) + assert epoch in (get_previous_epoch(state), get_current_epoch(state)) return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` From bb0a492fea2f4e2e3a036999325d6d8141b7b4b4 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 23 Jun 2019 11:18:24 +0200 Subject: [PATCH 167/405] Cleanups --- specs/core/0_beacon-chain.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c062bbf78d..68a121cd45 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1699,20 +1699,21 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ) if data.target.epoch == get_current_epoch(state): - ffg_source = state.current_justified_checkpoint + assert data.source == state.current_justified_checkpoint parent_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) else: - ffg_source = state.previous_justified_checkpoint + assert data.source == state.previous_justified_checkpoint parent_crosslink = state.previous_crosslinks[data.crosslink.shard] state.previous_epoch_attestations.append(pending_attestation) - # Check FFG source, crosslink data, and signature - assert ffg_source == data.source + # Check crosslink against expected parent crosslink + assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.start_epoch == parent_crosslink.end_epoch assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) - assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] + + # Check signature validate_indexed_attestation(state, convert_to_indexed(state, attestation)) ``` From d9644f518b6ce8a9269b08c18942d5ff1d6773f1 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 24 Jun 2019 16:08:13 +0200 Subject: [PATCH 168/405] mask is hash() in tests Co-Authored-By: dankrad --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 0167edc6a6..8037755bca 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -21,7 +21,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): ), ) # Generate the mask (any random 32 bytes will do) - mask = reveal[:32] + mask = hash(reveal) # Generate masker's signature on the mask masker_signature = bls_sign( message_hash=mask, From 139d0f56f10972d6677b70683702f819b5c2fe02 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 24 Jun 2019 16:26:21 +0200 Subject: [PATCH 169/405] Finishes moving mask to hash() --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 8037755bca..9ca770c11e 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -1,5 +1,6 @@ from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures +from eth2spec.utils.hash_function import hash def get_valid_early_derived_secret_reveal(spec, state, epoch=None): @@ -20,7 +21,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): message_epoch=epoch, ), ) - # Generate the mask (any random 32 bytes will do) + # Generate the mask (any random 32 bytes that don't reveal the masker's secret will do) mask = hash(reveal) # Generate masker's signature on the mask masker_signature = bls_sign( From 82ae1804900e06daf2446c8dfbb1b198d931a5ed Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 24 Jun 2019 23:38:36 +0200 Subject: [PATCH 170/405] clean up list limit constants --- configs/constant_presets/mainnet.yaml | 4 ++++ configs/constant_presets/minimal.yaml | 4 ++++ specs/core/0_beacon-chain.md | 17 +++++++---------- .../pyspec/eth2spec/test/helpers/genesis.py | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 09ec7bdb79..9f7ca950f1 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -77,6 +77,10 @@ MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 EPOCHS_PER_HISTORICAL_VECTOR: 65536 # 2**13 (= 8,192) epochs ~36 days EPOCHS_PER_SLASHED_BALANCES_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~26,131 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 # Reward and penalty quotients diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 201ac475ff..3e3f7ccb48 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -78,6 +78,10 @@ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 EPOCHS_PER_HISTORICAL_VECTOR: 64 # [customized] smaller state EPOCHS_PER_SLASHED_BALANCES_VECTOR: 64 +# 2**24 (= 16,777,216) historical roots +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 # Reward and penalty quotients diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1f500bc05b..7d2459eb35 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -224,7 +224,6 @@ These configurations are updated for releases, but may be out of sync during `de | `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | epochs | 25.6 minutes | | `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~1.7 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | -| `HISTORICAL_ROOTS_LENGTH` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | | `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | @@ -238,10 +237,8 @@ These configurations are updated for releases, but may be out of sync during `de | - | - | :-: | :-: | | `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | | `EPOCHS_PER_SLASHED_BALANCES_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | -| `RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `VALIDATOR_REGISTRY_SIZE` | `2**40` (= 1,099,511,627,776) | | | +| `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | +| `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validator spots | | ### Rewards and penalties @@ -520,14 +517,14 @@ class BeaconState(Container): latest_block_header: BeaconBlockHeader block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] - historical_roots: List[Hash, HISTORICAL_ROOTS_LENGTH] + historical_roots: List[Hash, HISTORICAL_ROOTS_LIMIT] # Eth1 eth1_data: Eth1Data eth1_data_votes: List[Eth1Data, SLOTS_PER_ETH1_VOTING_PERIOD] eth1_deposit_index: uint64 # Registry - validators: List[Validator, VALIDATOR_REGISTRY_SIZE] - balances: List[Gwei, VALIDATOR_REGISTRY_SIZE] + validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] + balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] @@ -1195,7 +1192,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth # Populate active_index_roots genesis_active_index_root = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE](get_active_validator_indices(state, GENESIS_EPOCH)) + List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) ) for index in range(EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root @@ -1553,7 +1550,7 @@ def process_final_updates(state: BeaconState) -> None: # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR state.active_index_roots[index_root_position] = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT]( get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) ) ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index d3d67e09e9..ce0be19bbd 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -41,7 +41,7 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_SIZE]( + genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_LIMIT]( spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root From c73417b4cae0335e348f1a4d06968506301aeb64 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 24 Jun 2019 23:40:47 +0200 Subject: [PATCH 171/405] deserialize-basic detail, make subclass --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 144201d837..e2a2649552 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -27,7 +27,7 @@ def deserialize_basic(value, typ: BasicType): return typ(int.from_bytes(value, 'little')) elif issubclass(typ, Bit): assert value in (b'\x00', b'\x01') - return Bit(value == b'\x01') + return typ(value == b'\x01') else: raise Exception(f"Type not supported: {typ}") From 5989e5cd2368a5e7b4913bcfa6ef5025019b0aa7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 24 Jun 2019 23:56:26 +0200 Subject: [PATCH 172/405] use Bool as base name, make Bit an alias --- scripts/build_spec.py | 6 +++--- specs/core/0_beacon-chain.md | 4 ++-- test_libs/pyspec/eth2spec/debug/decode.py | 4 ++-- test_libs/pyspec/eth2spec/debug/encode.py | 4 ++-- .../pyspec/eth2spec/debug/random_value.py | 8 ++++---- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 4 ++-- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 6 +++--- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 ++++++- .../pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 11 ++++++++--- .../eth2spec/utils/ssz/test_ssz_typing.py | 18 +++++++++++------- 10 files changed, 43 insertions(+), 29 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0b55dfa443..08793869f3 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Container, List, Vector, Bytes, uint64, + Bit, Bool, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -53,7 +53,7 @@ is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Container, List, Vector, Bytes, uint64, + Bit, Bool, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -179,7 +179,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ - 'Bit', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' + 'Bit', 'Bool', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7d2459eb35..b3e3ba0ea9 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -314,7 +314,7 @@ class Validator(Container): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers effective_balance: Gwei # Balance at stake - slashed: Bit + slashed: Bool # Status epochs activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch @@ -354,7 +354,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): data: AttestationData - custody_bit: Bit # Challengeable bit for the custody of crosslink data + custody_bit: Bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data ``` #### `IndexedAttestation` diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index 743479371d..c0b53b0efb 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,13 +1,13 @@ from typing import Any from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, uint, Container, Bytes, List, Bit, + SSZType, SSZValue, uint, Container, Bytes, List, Bool, Vector, BytesN ) def decode(data: Any, typ: SSZType) -> SSZValue: - if issubclass(typ, (uint, Bit)): + if issubclass(typ, (uint, Bool)): return typ(data) elif issubclass(typ, (List, Vector)): return typ(decode(element, typ.elem_type) for element in data) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 0878a1f947..02814e441a 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZValue, uint, Container, Bit + SSZValue, uint, Container, Bool ) @@ -10,7 +10,7 @@ def encode(value: SSZValue, include_hash_tree_roots=False): if value.type().byte_len > 8: return str(int(value)) return int(value) - elif isinstance(value, Bit): + elif isinstance(value, Bool): return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 8e13cd5e1d..c6efb722b9 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,7 +2,7 @@ from enum import Enum from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bit, + SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bool, Vector, BytesN ) @@ -118,7 +118,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: - if issubclass(typ, Bit): + if issubclass(typ, Bool): return typ(rng.choice((True, False))) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -128,7 +128,7 @@ def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: def get_min_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bit): + if issubclass(typ, Bool): return typ(False) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -138,7 +138,7 @@ def get_min_basic_value(typ: BasicType) -> BasicValue: def get_max_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bit): + if issubclass(typ, Bool): return typ(True) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index a4ebb7e708..e533ca5c21 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -19,7 +19,7 @@ def translate_typ(typ) -> ssz.BaseSedes: return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.List): return ssz.List(translate_typ(typ.elem_type)) - elif issubclass(typ, spec_ssz.Bit): + elif issubclass(typ, spec_ssz.Bool): return ssz.boolean elif issubclass(typ, spec_ssz.uint): if typ.byte_len == 1: @@ -64,7 +64,7 @@ def translate_value(value, typ): raise TypeError("invalid uint size") elif issubclass(typ, spec_ssz.List): return [translate_value(elem, typ.elem_type) for elem in value] - elif issubclass(typ, spec_ssz.Bit): + elif issubclass(typ, spec_ssz.Bool): return value elif issubclass(typ, spec_ssz.Vector): return typ(*(translate_value(elem, typ.elem_type) for elem in value)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index e2a2649552..b9c7b6d384 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, uint, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bool, Container, List, Bytes, uint, ) # SSZ Serialization @@ -13,7 +13,7 @@ def serialize_basic(value: SSZValue): if isinstance(value, uint): return value.to_bytes(value.type().byte_len, 'little') - elif isinstance(value, Bit): + elif isinstance(value, Bool): if value: return b'\x01' else: @@ -25,7 +25,7 @@ def serialize_basic(value: SSZValue): def deserialize_basic(value, typ: BasicType): if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif issubclass(typ, Bit): + elif issubclass(typ, Bool): assert value in (b'\x00', b'\x01') return typ(value == b'\x01') else: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 891480a5c2..6079f28662 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -31,7 +31,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): pass -class Bit(BasicValue): # can't subclass bool. +class Bool(BasicValue): # can't subclass bool. byte_len = 1 def __new__(cls, value, *args, **kwargs): @@ -47,6 +47,11 @@ def __bool__(self): return self > 0 +# Alias for Bool +class Bit(Bool): + pass + + class uint(BasicValue, metaclass=BasicType): def __new__(cls, value, *args, **kwargs): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index ae0849098d..aa7aee64a5 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,6 +1,6 @@ from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( - Bit, Container, List, Vector, Bytes, BytesN, + Bit, Bool, Container, List, Vector, Bytes, BytesN, uint8, uint16, uint32, uint64, byte ) @@ -47,11 +47,16 @@ class ComplexTestStruct(Container): sig_test_data[k] = v test_data = [ - ("bool F", Bit(False), "00"), - ("bool T", Bit(True), "01"), + ("bit F", Bit(False), "00"), + ("bit T", Bit(True), "01"), + ("bool F", Bool(False), "00"), + ("bool T", Bool(True), "01"), ("uint8 00", uint8(0x00), "00"), ("uint8 01", uint8(0x01), "01"), ("uint8 ab", uint8(0xab), "ab"), + ("byte 00", byte(0x00), "00"), + ("byte 01", byte(0x01), "01"), + ("byte ab", byte(0xab), "ab"), ("uint16 0000", uint16(0x0000), "0000"), ("uint16 abcd", uint16(0xabcd), "cdab"), ("uint32 00000000", uint32(0x00000000), "00000000"), diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index bd86a58063..2af7423604 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,7 +1,7 @@ from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, - Elements, Bit, Container, List, Vector, Bytes, BytesN, - uint, uint8, uint16, uint32, uint64, uint128, uint256, + Elements, Bit, Bool, Container, List, Vector, Bytes, BytesN, + byte, uint, uint8, uint16, uint32, uint64, uint128, uint256, Bytes32, Bytes48 ) @@ -22,8 +22,8 @@ def test_subclasses(): assert issubclass(u, SSZValue) assert isinstance(u, SSZType) assert isinstance(u, BasicType) - assert issubclass(Bit, BasicValue) - assert isinstance(Bit, BasicType) + assert issubclass(Bool, BasicValue) + assert isinstance(Bool, BasicType) for c in [Container, List, Vector, Bytes, BytesN]: assert issubclass(c, Series) @@ -38,21 +38,25 @@ def test_subclasses(): def test_basic_instances(): - for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: + for u in [uint, uint8, byte, uint16, uint32, uint64, uint128, uint256]: v = u(123) assert isinstance(v, uint) assert isinstance(v, int) assert isinstance(v, BasicValue) assert isinstance(v, SSZValue) - assert isinstance(Bit(True), BasicValue) - assert isinstance(Bit(False), BasicValue) + assert isinstance(Bool(True), BasicValue) + assert isinstance(Bool(False), BasicValue) + assert isinstance(Bit(True), Bool) + assert isinstance(Bit(False), Bool) def test_basic_value_bounds(): max = { + Bool: 2 ** 1, Bit: 2 ** 1, uint8: 2 ** (8 * 1), + byte: 2 ** (8 * 1), uint16: 2 ** (8 * 2), uint32: 2 ** (8 * 4), uint64: 2 ** (8 * 8), From 9fb58067645da1aa02dcfa995cb61fb86a2dff16 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 00:24:13 +0200 Subject: [PATCH 173/405] be explicit about input for balance sum --- scripts/build_spec.py | 4 ++-- specs/core/0_beacon-chain.md | 6 +++--- specs/core/1_shard-data-chains.md | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 08793869f3..237c243840 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,7 +12,7 @@ PHASE0_IMPORTS = '''from typing import ( - Any, Callable, Iterable, Dict, Set, Sequence, Tuple, + Any, Callable, Dict, Set, Sequence, Tuple, ) from dataclasses import ( @@ -37,7 +37,7 @@ from eth2spec.utils.hash_function import hash ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Iterable, Optional, Set, Sequence, Tuple, + Any, Callable, Dict, Optional, Set, Sequence, Tuple, List as TypingList ) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4d6be89f73..b8cab7eaf3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -908,7 +908,7 @@ def bytes_to_int(data: bytes) -> int: ### `get_total_balance` ```python -def get_total_balance(state: BeaconState, indices: Iterable[ValidatorIndex]) -> Gwei: +def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: """ Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ @@ -1383,7 +1383,7 @@ def process_crosslinks(state: BeaconState) -> None: for epoch in (get_previous_epoch(state), get_current_epoch(state)): for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) - crosslink_committee = get_crosslink_committee(state, epoch, shard) + crosslink_committee = set(get_crosslink_committee(state, epoch, shard)) winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) if 3 * get_total_balance(state, attesting_indices) >= 2 * get_total_balance(state, crosslink_committee): state.current_crosslinks[shard] = winning_crosslink @@ -1456,7 +1456,7 @@ def get_crosslink_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[G epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) - crosslink_committee = get_crosslink_committee(state, epoch, shard) + crosslink_committee = set(get_crosslink_committee(state, epoch, shard)) winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) attesting_balance = get_total_balance(state, attesting_indices) committee_balance = get_total_balance(state, crosslink_committee) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index f84b0dd46e..dd48a842ad 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -378,7 +378,7 @@ Let: * `shard` be a valid `Shard` * `shard_blocks` be the `ShardBlock` list such that `shard_blocks[slot]` is the canonical `ShardBlock` for shard `shard` at slot `slot` * `beacon_state` be the canonical `BeaconState` -* `valid_attestations` be the list of valid `Attestation`, recursively defined +* `valid_attestations` be the set of valid `Attestation` objects, recursively defined * `candidate` be a candidate `Attestation` which is valid under Phase 0 rules, and for which validity is to be determined under Phase 1 rules by running `is_valid_beacon_attestation` ```python @@ -388,7 +388,7 @@ def is_valid_beacon_attestation(shard: Shard, valid_attestations: Set[Attestation], candidate: Attestation) -> bool: # Check if attestation is already determined valid - for _, attestation in enumerate(valid_attestations): + for attestation in valid_attestations: if candidate == attestation: return True From 82167ff0a33a3476df5901bd6f0645cd1cdf852a Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 24 Jun 2019 15:40:37 -0700 Subject: [PATCH 174/405] Clarify comment on phase 0 transfers The comment seems to imply conjunction over each case, but the code implies disjunction; at the very least it is ambiguous. This PR makes the comment less ambiguous. --- specs/core/0_beacon-chain.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 094c04a115..103c436b5e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1807,10 +1807,13 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: assert state.balances[transfer.sender] >= max(transfer.amount, transfer.fee) # A transfer is valid in only one slot assert state.slot == transfer.slot - # Sender must be not yet eligible for activation, withdrawn, or transfer balance over MAX_EFFECTIVE_BALANCE + # Sender must satisfy at least one of the following conditions in the parenthesis: assert ( + # * Has not been activated state.validators[transfer.sender].activation_eligibility_epoch == FAR_FUTURE_EPOCH or + # * Is withdrawable get_current_epoch(state) >= state.validators[transfer.sender].withdrawable_epoch or + # * Balance after transfer is more than the effective balance threshold transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE <= state.balances[transfer.sender] ) # Verify that the pubkey is valid From 0e362d36b1eda660d013d32002acf7fce0c8ff91 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 17:18:22 -0600 Subject: [PATCH 175/405] pr feedback --- specs/core/0_fork-choice.md | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 1755c6df35..1818de6c4d 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -179,23 +179,31 @@ def on_block(store: Store, block: BeaconBlock) -> None: ```python def on_attestation(store: Store, attestation: Attestation) -> None: - target_checkpoint = Checkpoint(attestation.data.target_epoch, attestation.data, target_root) + target = Checkpoint(attestation.data.target_epoch, attestation.data.target_root) # Cannot calculate the current shuffling if have not seen the target - assert target_checkpoint.root in store.blocks + assert target.root in store.blocks + + # Attestations cannot be from future epochs. If they are, their consideration must be delayed until the are in the past. + assert store.time >= pre_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen - if target_checkpoint not in store.checkpoint_states: - base_state = store.block_states[target_checkpoint.root].copy() - store.checkpoint_states[target_checkpoint] = process_slots(base_state, get_epoch_start_slot(target_checkpoint.epoch)) + if target not in store.checkpoint_states: + base_state = store.block_states[target.root].copy() + store.checkpoint_states[target] = process_slots(base_state, get_epoch_start_slot(target.epoch)) + target_state = store.checkpoint_states[target] + + # Attestations can only affect the fork choice of subsequent slots. + # Delay consideration in the fork choice until their slot is in the past. + attestation_slot = get_attestation_data_slot(target_state, attestation.data) + assert store.time >= (attestation_slot + 1) * SECONDS_PER_SLOT - # Get state at the `target_checkpoint` to validate attestation and calculate the committees - state = store.checkpoint_states[target_checkpoint] - indexed_attestation = convert_to_indexed(state, attestation) - validate_indexed_attestation(state, indexed_attestation) + # Get state at the `target` to validate attestation and calculate the committees + indexed_attestation = convert_to_indexed(target_state, attestation) + validate_indexed_attestation(target_state, indexed_attestation) # Update latest targets for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: - if i not in store.latest_targets or target_checkpoint.epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = target_checkpoint + if i not in store.latest_targets or target.epoch > store.latest_targets[i].epoch: + store.latest_targets[i] = target ``` From a5b7564c5b6c337a9db8272beac7d9fe61e92f6d Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 02:35:55 +0200 Subject: [PATCH 176/405] hash-tree-root tests --- .../eth2spec/utils/ssz/test_ssz_impl.py | 192 ++++++++++++++---- 1 file changed, 155 insertions(+), 37 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index aa7aee64a5..82fb4ec684 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,8 +1,10 @@ +from typing import Iterable from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( Bit, Bool, Container, List, Vector, Bytes, BytesN, - uint8, uint16, uint32, uint64, byte + uint8, uint16, uint32, uint64, uint256, byte ) +from ..hash_function import hash as bytes_hash import pytest @@ -46,37 +48,117 @@ class ComplexTestStruct(Container): for k, v in {0: 1, 32: 2, 64: 3, 95: 0xff}.items(): sig_test_data[k] = v + +def chunk(hex: str) -> str: + return (hex + ("00" * 32))[:64] # just pad on the right, to 32 bytes (64 hex chars) + + +def h(a: str, b: str) -> str: + return bytes_hash(bytes.fromhex(a) + bytes.fromhex(b)).hex() + + +# zero hashes, as strings, for +zero_hashes = [chunk("")] +for layer in range(1, 32): + zero_hashes.append(h(zero_hashes[layer - 1], zero_hashes[layer - 1])) + + +def merge(a: str, branch: Iterable[str]) -> str: + """ + Merge (out on left, branch on right) leaf a with branch items, branch is from bottom to top. + """ + out = a + for b in branch: + out = h(out, b) + return out + + test_data = [ - ("bit F", Bit(False), "00"), - ("bit T", Bit(True), "01"), - ("bool F", Bool(False), "00"), - ("bool T", Bool(True), "01"), - ("uint8 00", uint8(0x00), "00"), - ("uint8 01", uint8(0x01), "01"), - ("uint8 ab", uint8(0xab), "ab"), - ("byte 00", byte(0x00), "00"), - ("byte 01", byte(0x01), "01"), - ("byte ab", byte(0xab), "ab"), - ("uint16 0000", uint16(0x0000), "0000"), - ("uint16 abcd", uint16(0xabcd), "cdab"), - ("uint32 00000000", uint32(0x00000000), "00000000"), - ("uint32 01234567", uint32(0x01234567), "67452301"), - ("small (4567, 0123)", SmallTestStruct(A=0x4567, B=0x0123), "67452301"), - ("small [4567, 0123]::2", Vector[uint16, 2](uint16(0x4567), uint16(0x0123)), "67452301"), - ("uint32 01234567", uint32(0x01234567), "67452301"), - ("uint64 0000000000000000", uint64(0x00000000), "0000000000000000"), - ("uint64 0123456789abcdef", uint64(0x0123456789abcdef), "efcdab8967452301"), + ("bit F", Bit(False), "00", chunk("00")), + ("bit T", Bit(True), "01", chunk("01")), + ("bool F", Bool(False), "00", chunk("00")), + ("bool T", Bool(True), "01", chunk("01")), + ("uint8 00", uint8(0x00), "00", chunk("00")), + ("uint8 01", uint8(0x01), "01", chunk("01")), + ("uint8 ab", uint8(0xab), "ab", chunk("ab")), + ("byte 00", byte(0x00), "00", chunk("00")), + ("byte 01", byte(0x01), "01", chunk("01")), + ("byte ab", byte(0xab), "ab", chunk("ab")), + ("uint16 0000", uint16(0x0000), "0000", chunk("0000")), + ("uint16 abcd", uint16(0xabcd), "cdab", chunk("cdab")), + ("uint32 00000000", uint32(0x00000000), "00000000", chunk("00000000")), + ("uint32 01234567", uint32(0x01234567), "67452301", chunk("67452301")), + ("small (4567, 0123)", SmallTestStruct(A=0x4567, B=0x0123), "67452301", h(chunk("6745"), chunk("2301"))), + ("small [4567, 0123]::2", Vector[uint16, 2](uint16(0x4567), uint16(0x0123)), "67452301", chunk("67452301")), + ("uint32 01234567", uint32(0x01234567), "67452301", chunk("67452301")), + ("uint64 0000000000000000", uint64(0x00000000), "0000000000000000", chunk("0000000000000000")), + ("uint64 0123456789abcdef", uint64(0x0123456789abcdef), "efcdab8967452301", chunk("efcdab8967452301")), ("sig", BytesN[96](*sig_test_data), "0100000000000000000000000000000000000000000000000000000000000000" "0200000000000000000000000000000000000000000000000000000000000000" - "03000000000000000000000000000000000000000000000000000000000000ff"), - ("emptyTestStruct", EmptyTestStruct(), ""), - ("singleFieldTestStruct", SingleFieldTestStruct(A=0xab), "ab"), - ("fixedTestStruct", FixedTestStruct(A=0xab, B=0xaabbccdd00112233, C=0x12345678), "ab33221100ddccbbaa78563412"), - ("varTestStruct nil", VarTestStruct(A=0xabcd, C=0xff), "cdab07000000ff"), - ("varTestStruct empty", VarTestStruct(A=0xabcd, B=List[uint16, 1024](), C=0xff), "cdab07000000ff"), + "03000000000000000000000000000000000000000000000000000000000000ff", + h(h(chunk("01"), chunk("02")), + h("03000000000000000000000000000000000000000000000000000000000000ff", chunk("")))), + ("emptyTestStruct", EmptyTestStruct(), "", chunk("")), + ("singleFieldTestStruct", SingleFieldTestStruct(A=0xab), "ab", chunk("ab")), + ("uint16 list", List[uint16, 32](uint16(0xaabb), uint16(0xc0ad), uint16(0xeeff)), "bbaaadc0ffee", + h(h(chunk("bbaaadc0ffee"), chunk("")), chunk("03000000")) # max length: 32 * 2 = 64 bytes = 2 chunks + ), + ("uint32 list", List[uint32, 128](uint32(0xaabb), uint32(0xc0ad), uint32(0xeeff)), "bbaa0000adc00000ffee0000", + # max length: 128 * 4 = 512 bytes = 16 chunks + h(merge(chunk("bbaa0000adc00000ffee0000"), zero_hashes[0:4]), chunk("03000000")) + ), + ("uint256 list", List[uint256, 32](uint256(0xaabb), uint256(0xc0ad), uint256(0xeeff)), + "bbaa000000000000000000000000000000000000000000000000000000000000" + "adc0000000000000000000000000000000000000000000000000000000000000" + "ffee000000000000000000000000000000000000000000000000000000000000", + h(merge(h(h(chunk("bbaa"), chunk("adc0")), h(chunk("ffee"), chunk(""))), zero_hashes[2:5]), chunk("03000000")) + ), + ("uint256 list long", List[uint256, 128](i for i in range(1, 20)), + "".join([i.to_bytes(length=32, byteorder='little').hex() for i in range(1, 20)]), + h(merge( + h( + h( + h( + h(h(chunk("01"), chunk("02")), h(chunk("03"), chunk("04"))), + h(h(chunk("05"), chunk("06")), h(chunk("07"), chunk("08"))), + ), + h( + h(h(chunk("09"), chunk("0a")), h(chunk("0b"), chunk("0c"))), + h(h(chunk("0d"), chunk("0e")), h(chunk("0f"), chunk("10"))), + ) + ), + h( + h( + h(h(chunk("11"), chunk("12")), h(chunk("13"), chunk(""))), + zero_hashes[2] + ), + zero_hashes[3] + ) + ), + zero_hashes[5:7]), chunk("13000000")) # 128 chunks = 7 deep + ), + ("fixedTestStruct", FixedTestStruct(A=0xab, B=0xaabbccdd00112233, C=0x12345678), "ab33221100ddccbbaa78563412", + h(h(chunk("ab"), chunk("33221100ddccbbaa")), h(chunk("78563412"), chunk("")))), + ("varTestStruct nil", VarTestStruct(A=0xabcd, C=0xff), "cdab07000000ff", + h(h(chunk("cdab"), h(zero_hashes[6], chunk("00000000"))), h(chunk("ff"), chunk("")))), + ("varTestStruct empty", VarTestStruct(A=0xabcd, B=List[uint16, 1024](), C=0xff), "cdab07000000ff", + h(h(chunk("cdab"), h(zero_hashes[6], chunk("00000000"))), h(chunk("ff"), chunk("")))), # log2(1024*2/32)= 6 deep ("varTestStruct some", VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), - "cdab07000000ff010002000300"), + "cdab07000000ff010002000300", + h( + h( + chunk("cdab"), + h( + merge( + chunk("010002000300"), + zero_hashes[0:6] + ), + chunk("03000000") # length mix in + ) + ), + h(chunk("ff"), chunk("")) + )), ("complexTestStruct", ComplexTestStruct( A=0xaabb, @@ -90,8 +172,8 @@ class ComplexTestStruct(Container): FixedTestStruct(A=0xee, B=0x4444444444444444, C=0x00112233), FixedTestStruct(A=0xff, B=0x5555555555555555, C=0x44556677)), G=Vector[VarTestStruct, 2]( - VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), - VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff)), + VarTestStruct(A=0xdead, B=List[uint16, 1024](1, 2, 3), C=0x11), + VarTestStruct(A=0xbeef, B=List[uint16, 1024](4, 5, 6), C=0x22)), ), "bbaa" "47000000" # offset of B, []uint16 @@ -107,17 +189,53 @@ class ComplexTestStruct(Container): "666f6f626172" # foobar "cdab07000000ff010002000300" # contents of E "08000000" "15000000" # [start G]: local offsets of [2]varTestStruct - "cdab07000000ff010002000300" - "cdab07000000ff010002000300", - ) + "adde0700000011010002000300" + "efbe0700000022040005000600", + h( + h( + h( # A and B + chunk("bbaa"), + h(merge(chunk("22114433"), zero_hashes[0:3]), chunk("02000000")) # 2*128/32 = 8 chunks + ), + h( # C and D + chunk("ff"), + h(merge(chunk("666f6f626172"), zero_hashes[0:3]), chunk("06000000")) # 256/32 = 8 chunks + ) + ), + h( + h( # E and F + h(h(chunk("cdab"), h(merge(chunk("010002000300"), zero_hashes[0:6]), chunk("03000000"))), + h(chunk("ff"), chunk(""))), + h( + h( + h(h(chunk("cc"), chunk("4242424242424242")), h(chunk("37133713"), chunk(""))), + h(h(chunk("dd"), chunk("3333333333333333")), h(chunk("cdabcdab"), chunk(""))), + ), + h( + h(h(chunk("ee"), chunk("4444444444444444")), h(chunk("33221100"), chunk(""))), + h(h(chunk("ff"), chunk("5555555555555555")), h(chunk("77665544"), chunk(""))), + ), + ) + ), + h( # G and padding + h( + h(h(chunk("adde"), h(merge(chunk("010002000300"), zero_hashes[0:6]), chunk("03000000"))), + h(chunk("11"), chunk(""))), + h(h(chunk("efbe"), h(merge(chunk("040005000600"), zero_hashes[0:6]), chunk("03000000"))), + h(chunk("22"), chunk(""))), + ), + chunk("") + ) + ) + )) ] -@pytest.mark.parametrize("name, value, serialized", test_data) -def test_serialize(name, value, serialized): +@pytest.mark.parametrize("name, value, serialized, _", test_data) +def test_serialize(name, value, serialized, _): assert serialize(value) == bytes.fromhex(serialized) -@pytest.mark.parametrize("name, value, _", test_data) -def test_hash_tree_root(name, value, _): - hash_tree_root(value) +@pytest.mark.parametrize("name, value, _, root", test_data) +def test_hash_tree_root(name, value, _, root): + assert hash_tree_root(value) == bytes.fromhex(root) From 45dbf5a107324e7f5b5c69e2d76950cca888bac3 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Tue, 25 Jun 2019 02:41:02 +0200 Subject: [PATCH 177/405] Remove old Deltas reference Co-Authored-By: Hsiao-Wei Wang --- specs/core/0_beacon-chain.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b8cab7eaf3..77fdb34e1e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -286,8 +286,6 @@ We define the following Python custom types for type hinting and readability: | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | -`Deltas` is a non-SSZ type, a series of changes applied to balances, optimized by clients. - ## Containers The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containers. From 054a1579536afdbdb71d6b6da0c778bcbae6a3b5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 02:56:49 +0200 Subject: [PATCH 178/405] get rid of TypingList, add MutableSequence --- scripts/build_spec.py | 3 +-- specs/core/1_custody-game.md | 2 +- specs/core/1_shard-data-chains.md | 6 ++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 237c243840..d67c0e5c68 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -37,8 +37,7 @@ from eth2spec.utils.hash_function import hash ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Sequence, Tuple, - List as TypingList + Any, Callable, Dict, Optional, Set, Sequence, MutableSequence, Tuple, ) from dataclasses import ( diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 24e9a19e2f..07f6ec6982 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -318,7 +318,7 @@ def get_validators_custody_reveal_period(state: BeaconState, ### `replace_empty_or_append` ```python -def replace_empty_or_append(list: TypingList[Any], new_element: Any) -> int: +def replace_empty_or_append(list: MutableSequence[Any], new_element: Any) -> int: for i in range(len(list)): if is_empty(list[i]): list[i] = new_element diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index dd48a842ad..d1c86eca26 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -252,9 +252,9 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: def is_power_of_two(value: int) -> bool: return (value > 0) and (value & (value - 1) == 0) - def pad_to_power_of_2(values: TypingList[bytes]) -> TypingList[bytes]: + def pad_to_power_of_2(values: MutableSequence[bytes]) -> Sequence[bytes]: while not is_power_of_two(len(values)): - values += [b'\x00' * BYTES_PER_SHARD_BLOCK_BODY] + values.append(b'\x00' * BYTES_PER_SHARD_BLOCK_BODY) return values def hash_tree_root_of_bytes(data: bytes) -> bytes: @@ -264,6 +264,8 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: return data + b'\x00' * (length - len(data)) return hash( + # TODO untested code. + # Need to either pass a typed list to hash-tree-root, or merkleize_chunks(values, pad_to=2**x) hash_tree_root(pad_to_power_of_2([ hash_tree_root_of_bytes( zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_BODY) From 751738f411842c3cf1ac7d3ac9ec828d754912b0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 21:01:15 -0600 Subject: [PATCH 179/405] enhance fork choice testing --- specs/core/0_fork-choice.md | 24 ++-- .../test/fork_choice/test_on_attestation.py | 134 ++++++++++++++++++ .../test/fork_choice/test_on_block.py | 95 +++++++++++++ .../pyspec/eth2spec/test/test_fork_choice.py | 59 -------- 4 files changed, 241 insertions(+), 71 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py create mode 100644 test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 1818de6c4d..3ab0418374 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -58,7 +58,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A #### `Checkpoint` ```python -@dataclass +@dataclass(eq=True, frozen=True) class Checkpoint(object): epoch: Epoch root: Hash @@ -69,13 +69,13 @@ class Checkpoint(object): ```python @dataclass class Store(object): + time: int + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) block_states: Dict[Hash, BeaconState] = field(default_factory=dict) checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - time: int latest_targets: Dict[ValidatorIndex, Checkpoint] = field(default_factory=dict) - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint ``` #### `get_genesis_store` @@ -87,12 +87,12 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: justified_checkpoint = Checkpoint(GENESIS_EPOCH, root) finalized_checkpoint = Checkpoint(GENESIS_EPOCH, root) return Store( - blocks={root: genesis_block}, - block_states={root: genesis_state}, - checkpoint_states={justified_checkpoint: genesis_state.copy()}, time=genesis_state.genesis_time, justified_checkpoint=justified_checkpoint, finalized_checkpoint=finalized_checkpoint, + blocks={root: genesis_block}, + block_states={root: genesis_state}, + checkpoint_states={justified_checkpoint: genesis_state.copy()}, ) ``` @@ -150,13 +150,13 @@ def on_tick(store: Store, time: int) -> None: def on_block(store: Store, block: BeaconBlock) -> None: # Make a copy of the state to avoid mutability issues parent_block = store.blocks[block.parent_root] - pre_state = store.block_states[parent_block.root].copy() + pre_state = store.block_states[block.parent_root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block - assert get_ancestor(store, signing_root(block), store.blocks[get_epoch_start_slot(store.finalized_checkpoint)].slot) == store.finalized_checkpoint.root + assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) == store.finalized_checkpoint.root # Check that block is later than the finalized epoch slot assert block.slot > get_epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state @@ -167,7 +167,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Update justified checkpoint if state.current_justified_epoch > store.justified_checkpoint.epoch: store.justified_checkpoint = Checkpoint(state.current_justified_epoch, state.current_justified_root) - elif state.previous_justified_epoch > store.justified_epoch: + elif state.previous_justified_epoch > store.justified_checkpoint.epoch: store.justified_checkpoint = Checkpoint(state.previous_justified_epoch, state.previous_justified_root) # Update finalized checkpoint @@ -185,11 +185,11 @@ def on_attestation(store: Store, attestation: Attestation) -> None: assert target.root in store.blocks # Attestations cannot be from future epochs. If they are, their consideration must be delayed until the are in the past. - assert store.time >= pre_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT + base_state = store.block_states[target.root].copy() + assert store.time >= base_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - base_state = store.block_states[target.root].copy() store.checkpoint_states[target] = process_slots(base_state, get_epoch_start_slot(target.epoch)) target_state = store.checkpoint_states[target] diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py new file mode 100644 index 0000000000..fd42a6b29b --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -0,0 +1,134 @@ +from eth2spec.utils.ssz.ssz_impl import hash_tree_root + +from eth2spec.test.context import with_all_phases, with_state, bls_switch + +from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.state import next_slot + + +def run_on_attestation(spec, state, store, attestation, valid=True): + if not valid: + try: + spec.on_attestation(store, attestation) + except: + return + else: + assert False + + indexed_attestation = spec.convert_to_indexed(state, attestation) + spec.on_attestation(store, attestation) + assert ( + store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == + spec.Checkpoint( + epoch=attestation.data.target_epoch, + root=attestation.data.target_root, + ) + ) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + # store block in store + spec.on_block(store, block) + + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=block.slot) + run_on_attestation(spec, state, store, attestation) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_target_not_in_store(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + # move to next epoch to make block new target + state.slot += spec.SLOTS_PER_EPOCH + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + # do not add block to store + + next_slot(spec, state) + attestation = get_valid_attestation(spec, state, slot=block.slot) + run_on_attestation(spec, state, store, attestation, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_future_epoch(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 3 * spec.SECONDS_PER_SLOT + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + # store block in store + spec.on_block(store, block) + next_slot(spec, state) + + # move state forward but not store + attestation_slot = block.slot + spec.SLOTS_PER_EPOCH + state.slot = attestation_slot + + attestation = get_valid_attestation(spec, state, slot=state.slot) + run_on_attestation(spec, state, store, attestation, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_same_slot(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 1 * spec.SECONDS_PER_SLOT + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + spec.on_block(store, block) + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=block.slot) + run_on_attestation(spec, state, store, attestation, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_invalid_attestation(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 3 * spec.SECONDS_PER_SLOT + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + spec.on_block(store, block) + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=block.slot) + # make attestation invalid + attestation.custody_bitfield = b'\xf0' + attestation.custody_bitfield[1:] + run_on_attestation(spec, state, store, attestation, False) \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py new file mode 100644 index 0000000000..ef25317b69 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -0,0 +1,95 @@ +from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root + +from eth2spec.test.context import with_all_phases, with_state, bls_switch + +from eth2spec.test.helpers.block import build_empty_block_for_next_slot + + +def run_on_block(spec, state, store, block, valid=True): + if not valid: + try: + spec.on_block(store, block) + except: + return + else: + assert False + + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + + +@with_all_phases +@with_state +@bls_switch +def test_basic(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + assert store.time == time + + # On receiving a block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + run_on_block(spec, state, store, block) + + # On receiving a block of next epoch + store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + block = build_empty_block_for_next_slot(spec, state) + block.slot += spec.SLOTS_PER_EPOCH + + run_on_block(spec, state, store, block) + + # TODO: add tests for justified_root and finalized_root + + +@with_all_phases +@with_state +@bls_switch +def test_on_block_future_block(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + + # do not tick time + + # Fail receiving block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + run_on_block(spec, state, store, block, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_block_bad_parent_root(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + # Fail receiving block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + block.parent_root = b'\x45' * 32 + run_on_block(spec, state, store, block, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_block_before_finalized(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + store.finalized_checkpoint = spec.Checkpoint(epoch=store.finalized_checkpoint.epoch + 2, root=store.finalized_checkpoint.root) + + # Fail receiving block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + run_on_block(spec, state, store, block, False) \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index 4706f0eaf5..8b13789179 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -1,60 +1 @@ -from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root -from eth2spec.test.context import with_all_phases, with_state, bls_switch - -from eth2spec.test.helpers.block import build_empty_block_for_next_slot -from eth2spec.test.helpers.attestations import get_valid_attestation -from eth2spec.test.helpers.state import next_slot - - -@with_all_phases -@with_state -@bls_switch -def test_basic(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - - # Initialization - store = spec.get_genesis_store(state) - blocks = [] - time = 100 - spec.on_tick(store, time) - assert store.time == time - - # On receiving a block of `GENESIS_SLOT + 1` slot - block = build_empty_block_for_next_slot(spec, state) - blocks.append(block) - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block - - # On receiving a block of next epoch - store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - block = build_empty_block_for_next_slot(spec, state) - block.slot += spec.SLOTS_PER_EPOCH - blocks.append(block) - - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block - - # TODO: add tests for justified_root and finalized_root - - -@with_all_phases -@with_state -@bls_switch -def test_on_attestation(spec, state): - store = spec.get_genesis_store(state) - time = 100 - spec.on_tick(store, time) - - next_slot(spec, state) - - attestation = get_valid_attestation(spec, state, slot=1) - indexed_attestation = spec.convert_to_indexed(state, attestation) - spec.on_attestation(store, attestation) - assert ( - store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == - spec.Target( - epoch=attestation.data.target_epoch, - root=attestation.data.target_root, - ) - ) From d9b97578c0f73fb0ccd03a5727596eaec4847654 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 21:09:57 -0600 Subject: [PATCH 180/405] lint --- specs/core/0_fork-choice.md | 12 ++++++++---- .../eth2spec/test/fork_choice/test_on_attestation.py | 4 ++-- .../eth2spec/test/fork_choice/test_on_block.py | 9 ++++++--- test_libs/pyspec/eth2spec/test/test_fork_choice.py | 1 - 4 files changed, 16 insertions(+), 10 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/test_fork_choice.py diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3ab0418374..71920d2bc8 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -149,14 +149,17 @@ def on_tick(store: Store, time: int) -> None: ```python def on_block(store: Store, block: BeaconBlock) -> None: # Make a copy of the state to avoid mutability issues - parent_block = store.blocks[block.parent_root] + assert block.parent_root in store.block_states pre_state = store.block_states[block.parent_root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block - assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) == store.finalized_checkpoint.root + assert ( + get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) == + store.finalized_checkpoint.root + ) # Check that block is later than the finalized epoch slot assert block.slot > get_epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state @@ -184,13 +187,14 @@ def on_attestation(store: Store, attestation: Attestation) -> None: # Cannot calculate the current shuffling if have not seen the target assert target.root in store.blocks - # Attestations cannot be from future epochs. If they are, their consideration must be delayed until the are in the past. + # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrivesr base_state = store.block_states[target.root].copy() assert store.time >= base_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - store.checkpoint_states[target] = process_slots(base_state, get_epoch_start_slot(target.epoch)) + process_slots(base_state, get_epoch_start_slot(target.epoch)) + store.checkpoint_states[target] = base_state target_state = store.checkpoint_states[target] # Attestations can only affect the fork choice of subsequent slots. diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index fd42a6b29b..cf0e7c9cb8 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -11,7 +11,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): if not valid: try: spec.on_attestation(store, attestation) - except: + except AssertionError: return else: assert False @@ -131,4 +131,4 @@ def test_on_attestation_invalid_attestation(spec, state): attestation = get_valid_attestation(spec, state, slot=block.slot) # make attestation invalid attestation.custody_bitfield = b'\xf0' + attestation.custody_bitfield[1:] - run_on_attestation(spec, state, store, attestation, False) \ No newline at end of file + run_on_attestation(spec, state, store, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py index ef25317b69..b18752f8c6 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -9,7 +9,7 @@ def run_on_block(spec, state, store, block, valid=True): if not valid: try: spec.on_block(store, block) - except: + except AssertionError: return else: assert False @@ -88,8 +88,11 @@ def test_on_block_before_finalized(spec, state): time = 100 spec.on_tick(store, time) - store.finalized_checkpoint = spec.Checkpoint(epoch=store.finalized_checkpoint.epoch + 2, root=store.finalized_checkpoint.root) + store.finalized_checkpoint = spec.Checkpoint( + epoch=store.finalized_checkpoint.epoch + 2, + root=store.finalized_checkpoint.root + ) # Fail receiving block of `GENESIS_SLOT + 1` slot block = build_empty_block_for_next_slot(spec, state) - run_on_block(spec, state, store, block, False) \ No newline at end of file + run_on_block(spec, state, store, block, False) diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py deleted file mode 100644 index 8b13789179..0000000000 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ /dev/null @@ -1 +0,0 @@ - From b8c0985e6063db80a99ec1b87127cf1106761034 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 21:43:05 -0600 Subject: [PATCH 181/405] merge in fork choice. tests pass --- specs/core/0_beacon-chain.md | 7 ++--- specs/core/0_fork-choice.md | 27 +++++++------------ .../test/fork_choice/test_on_attestation.py | 4 +-- .../test_process_attestation.py | 4 +-- 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 91e79e98f9..1c9d02c331 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1337,13 +1337,14 @@ def process_justification_and_finalization(state: BeaconState) -> None: state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations @@ -1352,7 +1353,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_checkpoint.epoch + 3 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_checkpoint.epoch+ 2 == current_epoch: + if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source if (bitfield >> 0) % 8 == 0b111 and old_current_justified_checkpoint.epoch + 2 == current_epoch: diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 71920d2bc8..77f8e407d7 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -55,15 +55,6 @@ The head block root associated with a `store` is defined as `get_head(store)`. A ### Helpers -#### `Checkpoint` - -```python -@dataclass(eq=True, frozen=True) -class Checkpoint(object): - epoch: Epoch - root: Hash -``` - #### `Store` ```python @@ -84,8 +75,8 @@ class Store(object): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) - justified_checkpoint = Checkpoint(GENESIS_EPOCH, root) - finalized_checkpoint = Checkpoint(GENESIS_EPOCH, root) + justified_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root) + finalized_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root) return Store( time=genesis_state.genesis_time, justified_checkpoint=justified_checkpoint, @@ -168,21 +159,21 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.block_states[signing_root(block)] = state # Update justified checkpoint - if state.current_justified_epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = Checkpoint(state.current_justified_epoch, state.current_justified_root) - elif state.previous_justified_epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = Checkpoint(state.previous_justified_epoch, state.previous_justified_root) + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + elif state.previous_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.previous_justified_checkpoint # Update finalized checkpoint - if state.finalized_epoch > state.finalized_epoch: - store.finalized_checkpoint = Checkpoint(state.finalized_epoch, state.finalized_root) + if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: + store.finalized_checkpoint = state.finalized_checkpoint ``` #### `on_attestation` ```python def on_attestation(store: Store, attestation: Attestation) -> None: - target = Checkpoint(attestation.data.target_epoch, attestation.data.target_root) + target = attestation.data.target # Cannot calculate the current shuffling if have not seen the target assert target.root in store.blocks diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index cf0e7c9cb8..20c482d4f5 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -21,8 +21,8 @@ def run_on_attestation(spec, state, store, attestation, valid=True): assert ( store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == spec.Checkpoint( - epoch=attestation.data.target_epoch, - root=attestation.data.target_root, + epoch=attestation.data.target.epoch, + root=attestation.data.target.root, ) ) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index d6cb615f80..5ceab40588 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -180,8 +180,8 @@ def test_invalid_current_source_root(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 state.finalized_epoch = 2 - state.previous_justified_checkpoint = spec.Checkpoint(epoch=3, root=b'\x01'*32) - state.current_justified_checkpoint = spec.Checkpoint(epoch=4, root=b'\x32'*32) + state.previous_justified_checkpoint = spec.Checkpoint(epoch=3, root=b'\x01' * 32) + state.current_justified_checkpoint = spec.Checkpoint(epoch=4, root=b'\x32' * 32) attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY From 37b1872634097f4d2b22242e0645f3175d3bcf2b Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 25 Jun 2019 08:30:59 -0500 Subject: [PATCH 182/405] Remove duplicate custom type definitions (#1214) Seems to be an accidental duplication of the type defs --- specs/core/0_beacon-chain.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 103c436b5e..6898bf4592 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -21,7 +21,6 @@ - [Rewards and penalties](#rewards-and-penalties) - [Max operations per block](#max-operations-per-block) - [Signature domains](#signature-domains) - - [Custom types](#custom-types-1) - [Containers](#containers) - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) @@ -270,20 +269,6 @@ The following values are (non-configurable) constants used throughout the specif | `DOMAIN_VOLUNTARY_EXIT` | `4` | | `DOMAIN_TRANSFER` | `5` | -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `Slot` | `uint64` | a slot number | -| `Epoch` | `uint64` | an epoch number | -| `Shard` | `uint64` | a shard number | -| `ValidatorIndex` | `uint64` | a validator registry index | -| `Gwei` | `uint64` | an amount in Gwei | -| `BLSPubkey` | `Bytes48` | a BLS12-381 public key | -| `BLSSignature` | `Bytes96` | a BLS12-381 signature | - ## Containers The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containers. From 20602bc92ba98a8b32a7b1ae81cfeb68018f55d5 Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 25 Jun 2019 08:32:56 -0500 Subject: [PATCH 183/405] phase 0 doc standardization b4 spec freeze (#1212) --- scripts/README.md | 12 +++---- specs/core/0_beacon-chain.md | 28 +++++++-------- specs/core/0_deposit-contract.md | 2 +- specs/core/0_fork-choice.md | 6 ++-- specs/simple-serialize.md | 8 ++--- specs/validator/0_beacon-chain-validator.md | 36 +++++++++---------- .../validator/0_beacon-node-validator-api.md | 19 +++++----- 7 files changed, 55 insertions(+), 56 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 25b46decfe..9d5849053f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,18 +1,18 @@ # Building pyspecs from specs.md -The benefit of the particular spec design is that the given markdown files can be converted to a `spec.py` file for the purposes of testing and linting. The result of this is that bugs are discovered and patched more quickly. +The benefit of the particular spec design is that the given Markdown files can be converted to a `spec.py` file for the purposes of testing and linting. As a result, bugs are discovered and patched more quickly. -Specs can be built from either a single markdown document or multiple files that must be combined in a given order. Given 2 spec objects, `build_spec.combine_spec_objects` will combine them into a single spec object which, subsequently, can be converted into a `specs.py`. +Specs can be built from either a single Markdown document or multiple files that must be combined in a given order. Given 2 spec objects, `build_spec.combine_spec_objects` will combine them into a single spec object which, subsequently, can be converted into a `specs.py`. ## Usage -For usage of the spec builder run `python3 -m build_spec --help`. +For usage of the spec builder, run `python3 -m build_spec --help`. ## `@Labels` and inserts The functioning of the spec combiner is largely automatic in that given `spec0.md` and `spec1.md`, SSZ Objects will be extended and old functions will be overwritten. Extra functionality is provided for more granular control over how files are combined. In the event that only a small portion of code is to be added to an existing function, insert functionality is provided. This saves having to completely redefine the old function from `spec0.md` in `spec1.md`. This is done by marking where the change is to occur in the old file and marking which code is to be inserted in the new file. This is done as follows: -* In the old file, a label is added as a python comment marking where the code is to be inserted. This would appear as follows in `spec0.md`: +* In the old file, a label is added as a Python comment marking where the code is to be inserted. This would appear as follows in `spec0.md`: ```python def foo(x): @@ -21,7 +21,7 @@ def foo(x): return x ``` -* In spec1, the new code could then be inserted by having a code-block that looked as follows: +* In spec1, the new code can then be inserted by having a code-block that looks as follows: ```python #begin insert @YourLabelHere @@ -29,4 +29,4 @@ def foo(x): #end insert @YourLabelHere ``` -**Note** that the code to be inserted has the **same level of indentation** as the surrounding code of its destination insert point. +*Note*: The code to be inserted has the **same level of indentation** as the surrounding code of its destination insert point. diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6898bf4592..b3b2b2166a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -140,7 +140,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code. * **Committee**—a (pseudo-) randomly sampled subset of [active validators](#dfn-active-validator). When a committee is referred to collectively, as in "this committee attests to X", this is assumed to mean "some subset of that committee that contains enough [validators](#dfn-validator) that the protocol recognizes it as representing the committee". * **Proposer**—the [validator](#dfn-validator) that creates a beacon chain block. * **Attester**—a [validator](#dfn-validator) that is part of a committee that needs to sign off on a beacon chain block while simultaneously creating a link (crosslink) to a recent shard block on a particular shard chain. -* **Beacon chain**—the central PoS chain that is the base of the sharding system. +* **Beacon chain**—the central proof-of-stake chain that is the base of the sharding system. * **Shard chain**—one of the chains on which user transactions take place and account data is stored. * **Block root**—a 32-byte Merkle root of a beacon chain block or shard chain block. Previously called "block hash". * **Crosslink**—a set of signatures from a committee attesting to a block in a shard chain that can be included into the beacon chain. Crosslinks are the main means by which the beacon chain "learns about" the updated state of shard chains. @@ -179,7 +179,7 @@ The following values are (non-configurable) constants used throughout the specif ## Configuration -*Note*: The default mainnet configuration values are included here for spec-design purposes. The different configurations for mainnet, testnets, and YAML-based testing can be found in the `configs/constant_presets/` directory. These configurations are updated for releases and may be out of sync during `dev` changes. +*Note*: The default mainnet configuration values are included here for spec-design purposes. The different configurations for mainnet, testnets, and YAML-based testing can be found in the [`configs/constant_presets`](../../configs/constant_presets) directory. These configurations are updated for releases and may be out of sync during `dev` changes. ### Misc @@ -192,7 +192,7 @@ The following values are (non-configurable) constants used throughout the specif | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | -* For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) +* For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) ### Gwei values @@ -226,7 +226,7 @@ The following values are (non-configurable) constants used throughout the specif | `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | | `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes | -* `MAX_EPOCHS_PER_CROSSLINK` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH` +* `MAX_EPOCHS_PER_CROSSLINK` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH`. ### State list lengths @@ -245,7 +245,7 @@ The following values are (non-configurable) constants used throughout the specif | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -* The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)` so after `INVERSE_SQRT_E_DROP_TIME` epochs it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. +* The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block @@ -548,11 +548,11 @@ The `hash` function is SHA256. ### `hash_tree_root` -`def hash_tree_root(object: SSZSerializable) -> Hash` is a function for hashing objects into a single root utilizing a hash tree structure. `hash_tree_root` is defined in the [SimpleSerialize spec](../simple-serialize.md#merkleization). +`def hash_tree_root(object: SSZSerializable) -> Hash` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SimpleSerialize spec](../simple-serialize.md#merkleization). ### `signing_root` -`def signing_root(object: Container) -> Hash` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. +`def signing_root(object: Container) -> Hash` is a function for computing signing messages, as defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers). ### `bls_domain` @@ -1046,15 +1046,15 @@ def get_churn_limit(state: BeaconState) -> int: ### `bls_verify` -`bls_verify` is a function for verifying a BLS signature, defined in the [BLS Signature spec](../bls_signature.md#bls_verify). +`bls_verify` is a function for verifying a BLS signature, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify). ### `bls_verify_multiple` -`bls_verify_multiple` is a function for verifying a BLS signature constructed from multiple messages, defined in the [BLS Signature spec](../bls_signature.md#bls_verify_multiple). +`bls_verify_multiple` is a function for verifying a BLS signature constructed from multiple messages, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify_multiple). ### `bls_aggregate_pubkeys` -`bls_aggregate_pubkeys` is a function for aggregating multiple BLS public keys into a single aggregate key, defined in the [BLS Signature spec](../bls_signature.md#bls_aggregate_pubkeys). +`bls_aggregate_pubkeys` is a function for aggregating multiple BLS public keys into a single aggregate key, as defined in the [BLS Signature spec](../bls_signature.md#bls_aggregate_pubkeys). ### Routines for updating validator status @@ -1119,7 +1119,7 @@ Before genesis has been triggered and whenever the deposit contract emits a `Dep * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log -When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: +When `is_genesis_trigger(deposits, timestamp) is True` for the first time, let: * `genesis_deposits = deposits` * `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` @@ -1128,7 +1128,7 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: * `genesis_eth1_data.deposit_root` is the deposit root for the last deposit in `deposits` * `genesis_eth1_data.deposit_count = len(genesis_deposits)` -*Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: +*Note*: The function `is_genesis_trigger` has yet to be agreed upon by the community, and can be updated as necessary. We define the following testing placeholder: ```python def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: @@ -1227,7 +1227,7 @@ def process_slot(state: BeaconState) -> None: ### Epoch processing -*Note*: the `# @LabelHere` lines below are placeholders to show that code will be inserted here in a future phase. +*Note*: The `# @LabelHere` lines below are placeholders to show that code will be inserted here in a future phase. ```python def process_epoch(state: BeaconState) -> None: @@ -1735,7 +1735,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Verify the deposit signature (proof of possession). # Invalid signatures are allowed by the deposit contract, # and hence included on-chain, but must not be processed. - # Note: deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` + # Note: Deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` if not bls_verify( pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT) ): diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index 338ece4875..d06dbaea14 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -61,4 +61,4 @@ Every Ethereum 1.0 deposit emits a `Deposit` log for consumption by the beacon c The deposit contract source code, written in Vyper, is available [here](https://github.com/ethereum/eth2.0-specs/blob/dev/deposit_contract/contracts/validator_registration.v.py). -*Note*: To save on gas the deposit contract uses a progressive Merkle root calculation algorithm that requires only O(log(n)) storage. See [here](https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py) for a Python implementation, and [here](https://github.com/runtimeverification/verified-smart-contracts/blob/master/deposit/formal-incremental-merkle-tree-algorithm.pdf) for a formal correctness proof. +*Note*: To save on gas, the deposit contract uses a progressive Merkle root calculation algorithm that requires only O(log(n)) storage. See [here](https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py) for a Python implementation, and [here](https://github.com/runtimeverification/verified-smart-contracts/blob/master/deposit/formal-incremental-merkle-tree-algorithm.pdf) for a formal correctness proof. diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 25b33ab304..f896caa3ba 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -8,7 +8,7 @@ - [Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice](#ethereum-20-phase-0----beacon-chain-fork-choice) - [Table of contents](#table-of-contents) - [Introduction](#introduction) - - [Constants](#constants) + - [Configuration](#configuration) - [Time parameters](#time-parameters) - [Fork choice](#fork-choice) - [Helpers](#helpers) @@ -39,7 +39,7 @@ This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0 ## Fork choice -The head block root associated with a `store` is defined as `get_head(store)`. At genesis let `store = get_genesis_store(genesis_state)` and update `store` by running: +The head block root associated with a `store` is defined as `get_head(store)`. At genesis, let `store = get_genesis_store(genesis_state)` and update `store` by running: * `on_tick(time)` whenever `time > store.time` where `time` is the current Unix time * `on_block(block)` whenever a block `block` is received @@ -49,7 +49,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A 1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. This is automatically handled by [UNIX time](https://en.wikipedia.org/wiki/Unix_time). 2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. -3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. +3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](../validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. 4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. 5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number of optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 2adff23888..3318fe45b5 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -70,17 +70,17 @@ The default value of a type upon initialization is recursively defined using `0` #### `is_empty` -An SSZ object is called empty (and thus `is_empty(object)` returns true) if it is equal to the default value for that type. +An SSZ object is called empty (and thus, `is_empty(object)` returns true) if it is equal to the default value for that type. ### Illegal types -Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `"null"` type is only legal as the first type in a union subtype (i.e., with type index zero). +Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `"null"` type is only legal as the first type in a union subtype (i.e. with type index zero). ## Serialization We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. -> *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signing_root`, `is_variable_size`, etc.) objects implicitly carry their type. +*Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signing_root`, `is_variable_size`, etc.) objects implicitly carry their type. ### `"uintN"` @@ -164,7 +164,7 @@ Let `value` be a self-signed container object. The convention is that the signat | Python | Ethereum 2.0 | Ethereum Foundation | [https://github.com/ethereum/py-ssz](https://github.com/ethereum/py-ssz) | | Rust | Lighthouse | Sigma Prime | [https://github.com/sigp/lighthouse/tree/master/eth2/utils/ssz](https://github.com/sigp/lighthouse/tree/master/eth2/utils/ssz) | | Nim | Nimbus | Status | [https://github.com/status-im/nim-beacon-chain/blob/master/beacon_chain/ssz.nim](https://github.com/status-im/nim-beacon-chain/blob/master/beacon_chain/ssz.nim) | -| Rust | Shasper | ParityTech | [https://github.com/paritytech/shasper/tree/master/util/ssz](https://github.com/paritytech/shasper/tree/master/util/ssz) | +| Rust | Shasper | ParityTech | [https://github.com/paritytech/shasper/tree/master/utils/ssz](https://github.com/paritytech/shasper/tree/master/util/ssz) | | TypeScript | Lodestar | ChainSafe Systems | [https://github.com/ChainSafe/ssz-js](https://github.com/ChainSafe/ssz-js) | | Java | Cava | ConsenSys | [https://www.github.com/ConsenSys/cava/tree/master/ssz](https://www.github.com/ConsenSys/cava/tree/master/ssz) | | Go | Prysm | Prysmatic Labs | [https://github.com/prysmaticlabs/go-ssz](https://github.com/prysmaticlabs/go-ssz) | diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ecbc5af277..aa8350b66c 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -57,7 +57,7 @@ This document represents the expected behavior of an "honest validator" with respect to Phase 0 of the Ethereum 2.0 protocol. This document does not distinguish between a "node" (i.e. the functionality of following and reading the beacon chain) and a "validator client" (i.e. the functionality of actively participating in consensus). The separation of concerns between these (potentially) two pieces of software is left as a design decision that is out of scope. -A validator is an entity that participates in the consensus of the Ethereum 2.0 protocol. This is an optional role for users in which they can post ETH as collateral and verify and attest to the validity of blocks to seek financial returns in exchange for building and securing the protocol. This is similar to proof of work networks in which a miner provides collateral in the form of hardware/hash-power to seek returns in exchange for building and securing the protocol. +A validator is an entity that participates in the consensus of the Ethereum 2.0 protocol. This is an optional role for users in which they can post ETH as collateral and verify and attest to the validity of blocks to seek financial returns in exchange for building and securing the protocol. This is similar to proof-of-work networks in which miners provide collateral in the form of hardware/hash-power to seek returns in exchange for building and securing the protocol. ## Prerequisites @@ -91,7 +91,7 @@ The validator constructs their `withdrawal_credentials` via the following: ### Submit deposit -In Phase 0, all incoming validator deposits originate from the Ethereum 1.0 PoW chain. Deposits are made to the [deposit contract](../core/0_deposit-contract.md) located at `DEPOSIT_CONTRACT_ADDRESS`. +In Phase 0, all incoming validator deposits originate from the Ethereum 1.0 proof-of-work chain. Deposits are made to the [deposit contract](../core/0_deposit-contract.md) located at `DEPOSIT_CONTRACT_ADDRESS`. To submit a deposit: @@ -113,7 +113,7 @@ Once a validator has been processed and added to the beacon state's `validators` ### Activation -In normal operation, the validator is quickly activated at which point the validator is added to the shuffling and begins validation after an additional `ACTIVATION_EXIT_DELAY` epochs (25.6 minutes). +In normal operation, the validator is quickly activated, at which point the validator is added to the shuffling and begins validation after an additional `ACTIVATION_EXIT_DELAY` epochs (25.6 minutes). The function [`is_active_validator`](../core/0_beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows: @@ -124,7 +124,7 @@ is_active = is_active_validator(validator, get_current_epoch(state)) Once a validator is activated, the validator is assigned [responsibilities](#beacon-chain-responsibilities) until exited. -*Note*: There is a maximum validator churn per finalized epoch so the delay until activation is variable depending upon finality, total active validator balance, and the number of validators in the queue to be activated. +*Note*: There is a maximum validator churn per finalized epoch, so the delay until activation is variable depending upon finality, total active validator balance, and the number of validators in the queue to be activated. ## Validator assignments @@ -183,7 +183,7 @@ A validator has two primary responsibilities to the beacon chain: [proposing blo A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator creates, signs, and broadcasts a `block` that is a child of `parent` that satisfies a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). -There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312500 validators = 10 million ETH, that's once per ~3 weeks). +There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312,500 validators = 10 million ETH, that's once per ~3 weeks). #### Block header @@ -255,15 +255,15 @@ block_signature = bls_sign( ##### Proposer slashings -Up to `MAX_PROPOSER_SLASHINGS` [`ProposerSlashing`](../core/0_beacon-chain.md#proposerslashing) objects can be included in the `block`. The proposer slashings must satisfy the verification conditions found in [proposer slashings processing](../core/0_beacon-chain.md#proposer-slashings). The validator receives a small "whistleblower" reward for each proposer slashing found and included. +Up to `MAX_PROPOSER_SLASHINGS`, [`ProposerSlashing`](../core/0_beacon-chain.md#proposerslashing) objects can be included in the `block`. The proposer slashings must satisfy the verification conditions found in [proposer slashings processing](../core/0_beacon-chain.md#proposer-slashings). The validator receives a small "whistleblower" reward for each proposer slashing found and included. ##### Attester slashings -Up to `MAX_ATTESTER_SLASHINGS` [`AttesterSlashing`](../core/0_beacon-chain.md#attesterslashing) objects can be included in the `block`. The attester slashings must satisfy the verification conditions found in [Attester slashings processing](../core/0_beacon-chain.md#attester-slashings). The validator receives a small "whistleblower" reward for each attester slashing found and included. +Up to `MAX_ATTESTER_SLASHINGS`, [`AttesterSlashing`](../core/0_beacon-chain.md#attesterslashing) objects can be included in the `block`. The attester slashings must satisfy the verification conditions found in [attester slashings processing](../core/0_beacon-chain.md#attester-slashings). The validator receives a small "whistleblower" reward for each attester slashing found and included. ##### Attestations -Up to `MAX_ATTESTATIONS` aggregate attestations can be included in the `block`. The attestations added must satisfy the verification conditions found in [attestation processing](../core/0_beacon-chain.md#attestations). To maximize profit, the validator should attempt to gather aggregate attestations that include singular attestations from the largest number of validators whose signatures from the same epoch have not previously been added on chain. +Up to `MAX_ATTESTATIONS`, aggregate attestations can be included in the `block`. The attestations added must satisfy the verification conditions found in [attestation processing](../core/0_beacon-chain.md#attestations). To maximize profit, the validator should attempt to gather aggregate attestations that include singular attestations from the largest number of validators whose signatures from the same epoch have not previously been added on chain. ##### Deposits @@ -273,17 +273,17 @@ The `proof` for each deposit must be constructed against the deposit root contai ##### Voluntary exits -Up to `MAX_VOLUNTARY_EXITS` [`VoluntaryExit`](../core/0_beacon-chain.md#voluntaryexit) objects can be included in the `block`. The exits must satisfy the verification conditions found in [exits processing](../core/0_beacon-chain.md#voluntary-exits). +Up to `MAX_VOLUNTARY_EXITS`, [`VoluntaryExit`](../core/0_beacon-chain.md#voluntaryexit) objects can be included in the `block`. The exits must satisfy the verification conditions found in [exits processing](../core/0_beacon-chain.md#voluntary-exits). ### Attestations -A validator is expected to create, sign, and broadcast an attestation during each epoch. The `committee`, assigned `shard`, and assigned `slot` for which the validator performs this role during an epoch is defined by `get_committee_assignment(state, epoch, validator_index)`. +A validator is expected to create, sign, and broadcast an attestation during each epoch. The `committee`, assigned `shard`, and assigned `slot` for which the validator performs this role during an epoch are defined by `get_committee_assignment(state, epoch, validator_index)`. -A validator should create and broadcast the attestation halfway through the `slot` during which the validator is assigned ― that is, `SECONDS_PER_SLOT * 0.5` seconds after the start of `slot`. +A validator should create and broadcast the attestation halfway through the `slot` during which the validator is assigned―that is, `SECONDS_PER_SLOT * 0.5` seconds after the start of `slot`. #### Attestation data -First the validator should construct `attestation_data`, an [`AttestationData`](../core/0_beacon-chain.md#attestationdata) object based upon the state at the assigned slot. +First, the validator should construct `attestation_data`, an [`AttestationData`](../core/0_beacon-chain.md#attestationdata) object based upon the state at the assigned slot. * Let `head_block` be the result of running the fork choice during the assigned slot. * Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. @@ -316,7 +316,7 @@ Construct `attestation_data.crosslink` via the following. #### Construct attestation -Next the validator creates `attestation`, an [`Attestation`](../core/0_beacon-chain.md#attestation) object. +Next, the validator creates `attestation`, an [`Attestation`](../core/0_beacon-chain.md#attestation) object. ##### Data @@ -362,9 +362,9 @@ signed_attestation_data = bls_sign( ## How to avoid slashing -"Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed -- [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. +"Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed: [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. -*Note*: Signed data must be within a sequential `Fork` context to conflict. Messages cannot be slashed across diverging forks. If the previous fork version is 1 and the chain splits into fork 2 and 102, messages from 1 can slashable against messages in forks 1, 2, and 102. Messages in 2 cannot be slashable against messages in 102 and vice versa. +*Note*: Signed data must be within a sequential `Fork` context to conflict. Messages cannot be slashed across diverging forks. If the previous fork version is 1 and the chain splits into fork 2 and 102, messages from 1 can slashable against messages in forks 1, 2, and 102. Messages in 2 cannot be slashable against messages in 102, and vice versa. ### Proposer slashing @@ -376,14 +376,14 @@ Specifically, when signing a `BeaconBlock`, a validator should perform the follo 1. Save a record to hard disk that a beacon block has been signed for the `epoch=slot_to_epoch(block.slot)`. 2. Generate and broadcast the block. -If the software crashes at some point within this routine, then when the validator comes back online the hard disk has the record of the *potentially* signed/broadcast block and can effectively avoid slashing. +If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast block and can effectively avoid slashing. ### Attester slashing To avoid "attester slashings", a validator must not sign two conflicting [`AttestationData`](../core/0_beacon-chain.md#attestationdata) objects, i.e. two attestations that satisfy [`is_slashable_attestation_data`](../core/0_beacon-chain.md#is_slashable_attestation_data). Specifically, when signing an `Attestation`, a validator should perform the following steps in the following order: -1. Save a record to hard disk that an attestation has been signed for source -- `attestation_data.source_epoch` -- and target -- `slot_to_epoch(attestation_data.slot)`. +1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source_epoch`) and target (i.e. `slot_to_epoch(attestation_data.slot)`). 2. Generate and broadcast attestation. -If the software crashes at some point within this routine, then when the validator comes back online the hard disk has the record of the *potentially* signed/broadcast attestation and can effectively avoid slashing. +If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast attestation and can effectively avoid slashing. diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index 2a5fe7fcda..20379eef03 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -1,28 +1,27 @@ # Ethereum 2.0 Phase 0 -- Beacon Node API for Validator -__NOTICE__: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- Honest Validator](0_beacon-chain-validator.md) that describes an API exposed by the beacon node, which enables the validator client to participate in the Ethereum 2.0 protocol. +**Notice**: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- Honest Validator](0_beacon-chain-validator.md) that describes an API exposed by the beacon node, which enables the validator client to participate in the Ethereum 2.0 protocol. ## Outline -This document outlines a minimal application programming interface (API) which is exposed by a beacon node for use by a validator client implementation which aims to facilitate [_phase 0_](../../README.md#phase-0) of Ethereum 2.0. +This document outlines a minimal application programming interface (API) which is exposed by a beacon node for use by a validator client implementation which aims to facilitate [Phase 0](../../README.md#phase-0) of Ethereum 2.0. -The API is a REST interface, accessed via HTTP, designed for use as a local communications protocol between binaries. The only supported return data type is currently JSON. +The API is a REST interface, accessed via HTTP, designed for use as a local communications protocol between binaries. Currently, the only supported return data type is JSON. -### Background -The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum Serenity network. Conceptually, it does not maintain keypairs that participate with the beacon chain. +## Background +The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum 2.0 network. Conceptually, it does not maintain keypairs that participate with the beacon chain. -The validator client is a conceptually separate entity which utilizes private keys to perform validator related tasks on the beacon chain, which we call validator "duties". These duties include the production of beacon blocks and signing of attestations. +The validator client is a conceptually separate entity which utilizes private keys to perform validator related tasks, called "duties", on the beacon chain. These duties include the production of beacon blocks and signing of attestations. Since it is recommended to separate these concerns in the client implementations, we must clearly define the communication between them. -The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects and to encourage innovation in validator client implementations, independently from beacon node development. For example, the validator client from Lighthouse could communicate with a running instance of the beacon node from Prysm, or a staking pool might create a decentrally managed validator client which utilises the same API. +The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects and to encourage innovation in validator client implementations, independently from beacon node development. For example, the validator client from [Lighthouse](https://github.com/sigp/lighthouse) could communicate with a running instance of the beacon node from [Prysm](https://github.com/prysmaticlabs/prysm), or a staking pool might create a decentrally managed validator client which utilizes the same API. -This specification is derived from a proposal and discussion on Issues [#1011](https://github.com/ethereum/eth2.0-specs/issues/1011) and [#1012](https://github.com/ethereum/eth2.0-specs/issues/1012) +This specification is derived from a proposal and discussion on Issues [#1011](https://github.com/ethereum/eth2.0-specs/issues/1011) and [#1012](https://github.com/ethereum/eth2.0-specs/issues/1012). ## Specification The API specification has been written in [OpenAPI 3.0](https://swagger.io/docs/specification/about/) and is provided in the [beacon_node_oapi.yaml](beacon_node_oapi.yaml) file alongside this document. -For convenience, this specification has been uploaded to [SwaggerHub](https://swagger.io/tools/swaggerhub/) at the following URL: -[https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator](https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator) +For convenience, this specification has been uploaded to SwaggerHub [here](https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator). From 4986311d5bb1e716e9dd6eea59864d44732b97ec Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 25 Jun 2019 10:24:14 -0600 Subject: [PATCH 184/405] Update 0_beacon-chain.md fix typo w/ refactor --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 99425c6c21..56a6fd06a4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1142,7 +1142,7 @@ def is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool: # Count active validators at genesis active_validator_count = 0 - for validator in state.validator_registry: + for validator in state.validators: if validator.effective_balance == MAX_EFFECTIVE_BALANCE: active_validator_count += 1 From 846ca649aafece56d7aaf09bb16aa3bc5a76725b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 25 Jun 2019 10:31:39 -0600 Subject: [PATCH 185/405] properly construct genesis latest block header in tests --- .../eth2spec/test/fork_choice/test_on_attestation.py | 12 ------------ .../eth2spec/test/fork_choice/test_on_block.py | 11 +---------- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 4 +++- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index cf0e7c9cb8..48f4bf46ff 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -1,5 +1,3 @@ -from eth2spec.utils.ssz.ssz_impl import hash_tree_root - from eth2spec.test.context import with_all_phases, with_state, bls_switch from eth2spec.test.helpers.block import build_empty_block_for_next_slot @@ -31,8 +29,6 @@ def run_on_attestation(spec, state, store, attestation, valid=True): @with_state @bls_switch def test_on_attestation(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 100 spec.on_tick(store, time) @@ -52,8 +48,6 @@ def test_on_attestation(spec, state): @with_state @bls_switch def test_on_attestation_target_not_in_store(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 100 spec.on_tick(store, time) @@ -74,8 +68,6 @@ def test_on_attestation_target_not_in_store(spec, state): @with_state @bls_switch def test_on_attestation_future_epoch(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 3 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) @@ -98,8 +90,6 @@ def test_on_attestation_future_epoch(spec, state): @with_state @bls_switch def test_on_attestation_same_slot(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 1 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) @@ -117,8 +107,6 @@ def test_on_attestation_same_slot(spec, state): @with_state @bls_switch def test_on_attestation_invalid_attestation(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 3 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py index b18752f8c6..158ee1a58e 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -1,7 +1,6 @@ -from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.test.context import with_all_phases, with_state, bls_switch - from eth2spec.test.helpers.block import build_empty_block_for_next_slot @@ -22,8 +21,6 @@ def run_on_block(spec, state, store, block, valid=True): @with_state @bls_switch def test_basic(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) time = 100 @@ -48,8 +45,6 @@ def test_basic(spec, state): @with_state @bls_switch def test_on_block_future_block(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) @@ -64,8 +59,6 @@ def test_on_block_future_block(spec, state): @with_state @bls_switch def test_on_block_bad_parent_root(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) time = 100 @@ -81,8 +74,6 @@ def test_on_block_bad_parent_root(spec, state): @with_state @bls_switch def test_on_block_before_finalized(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) time = 100 diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index d1d8189080..49e4feb7db 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -27,7 +27,9 @@ def create_genesis_state(spec, num_validators): deposit_root=deposit_root, deposit_count=num_validators, block_hash=spec.ZERO_HASH, - )) + ), + latest_block_header=spec.BeaconBlockHeader(body_root=spec.hash_tree_root(spec.BeaconBlockBody())), + ) # We "hack" in the initial validators, # as it is much faster than creating and processing genesis deposits for every single test case. From d90d56c610e8b58dc697c2dc92f9c1608c974520 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 18:42:34 +0200 Subject: [PATCH 186/405] Change uint aliases to just be subclasses, do not override init with no-op --- scripts/build_spec.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index d67c0e5c68..99c5cd69de 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -130,11 +130,7 @@ def objects_to_spec(functions: Dict[str, str], new_type_definitions = ( '\n\n'.join( [ - f"class {key}({value}):\n" - f" def __init__(self, _x: {value}) -> None:\n" - f" ...\n" - if value.startswith("uint") - else f"class {key}({value}):\n pass\n" + f"class {key}({value}):\n pass\n" for key, value in custom_types.items() ] ) From 3b5c7f243a4ec6555ea9f762411c8541f7c20e3a Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 19:32:49 +0200 Subject: [PATCH 187/405] type hint uint input --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6079f28662..58e66ca685 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -34,7 +34,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): class Bool(BasicValue): # can't subclass bool. byte_len = 1 - def __new__(cls, value, *args, **kwargs): + def __new__(cls, value: int): # int value, but can be any subclass of int (bool, Bit, Bool, etc...) if value < 0 or value > 1: raise ValueError(f"value {value} out of bounds for bit") return super().__new__(cls, value) @@ -54,7 +54,7 @@ class Bit(Bool): class uint(BasicValue, metaclass=BasicType): - def __new__(cls, value, *args, **kwargs): + def __new__(cls, value: int): if value < 0: raise ValueError("unsigned types must not be negative") if cls.byte_len and value.bit_length() > (cls.byte_len << 3): From 2c6f4f2597b3a6e821b03007de329058871f1d3f Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 19:33:24 +0200 Subject: [PATCH 188/405] update validator_registry -> validators missed case --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 99425c6c21..56a6fd06a4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1142,7 +1142,7 @@ def is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool: # Count active validators at genesis active_validator_count = 0 - for validator in state.validator_registry: + for validator in state.validators: if validator.effective_balance == MAX_EFFECTIVE_BALANCE: active_validator_count += 1 From 228195d89d464e5c4af7c4223808ffd33b96efda Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 25 Jun 2019 11:48:55 -0600 Subject: [PATCH 189/405] get head tests --- specs/core/0_fork-choice.md | 9 +- .../test/fork_choice/test_get_head.py | 118 ++++++++++++++++++ 2 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 71920d2bc8..f74f487a1d 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -91,7 +91,7 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: justified_checkpoint=justified_checkpoint, finalized_checkpoint=finalized_checkpoint, blocks={root: genesis_block}, - block_states={root: genesis_state}, + block_states={root: genesis_state.copy()}, checkpoint_states={justified_checkpoint: genesis_state.copy()}, ) ``` @@ -110,10 +110,11 @@ def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: ```python def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: state = store.checkpoint_states[store.justified_checkpoint] - active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) + active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( - state.validator_registry[i].effective_balance for i in active_indices - if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root + state.validators[i].effective_balance for i in active_indices + if (i in store.latest_targets and + get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root) )) ``` diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py new file mode 100644 index 0000000000..6ac46ba6c2 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -0,0 +1,118 @@ +from eth2spec.test.context import with_all_phases, with_state, bls_switch +from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.state import state_transition_and_sign_block + + +def add_block_to_store(spec, store, block): + pre_state = store.block_states[block.parent_root] + block_time = pre_state.genesis_time + block.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, block) + + +def add_attestation_to_store(spec, store, attestation): + parent_block = store.blocks[attestation.data.beacon_block_root] + pre_state = store.block_states[spec.signing_root(parent_block)] + block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT + next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT + + if store.time < next_epoch_time: + spec.on_tick(store, next_epoch_time) + + spec.on_attestation(store, attestation) + + +@with_all_phases +@with_state +@bls_switch +def test_genesis(spec, state): + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + +@with_all_phases +@with_state +@bls_switch +def test_chain_no_attestations(spec, state): + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + # On receiving a block of `GENESIS_SLOT + 1` slot + block_1 = build_empty_block_for_next_slot(spec, state) + state_transition_and_sign_block(spec, state, block_1) + add_block_to_store(spec, store, block_1) + + # On receiving a block of next epoch + block_2 = build_empty_block_for_next_slot(spec, state) + state_transition_and_sign_block(spec, state, block_2) + add_block_to_store(spec, store, block_2) + + assert spec.get_head(store) == spec.signing_root(block_2) + + +@with_all_phases +@with_state +@bls_switch +def test_split_tie_breaker_no_attestations(spec, state): + genesis_state = state.copy() + + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + # block at slot 1 + block_1_state = genesis_state.copy() + block_1 = build_empty_block_for_next_slot(spec, block_1_state) + state_transition_and_sign_block(spec, block_1_state, block_1) + add_block_to_store(spec, store, block_1) + + # additional block at slot 1 + block_2_state = genesis_state.copy() + block_2 = build_empty_block_for_next_slot(spec, block_2_state) + block_2.body.graffiti = b'\x42' * 32 + state_transition_and_sign_block(spec, block_2_state, block_2) + add_block_to_store(spec, store, block_2) + + highest_root = max(spec.signing_root(block_1), spec.signing_root(block_2)) + + assert spec.get_head(store) == highest_root + + +@with_all_phases +@with_state +@bls_switch +def test_shorter_chain_but_heavier_weight(spec, state): + genesis_state = state.copy() + + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + # build longer tree + long_state = genesis_state.copy() + for i in range(3): + long_block = build_empty_block_for_next_slot(spec, long_state) + state_transition_and_sign_block(spec, long_state, long_block) + add_block_to_store(spec, store, long_block) + + # build short tree + short_state = genesis_state.copy() + short_block = build_empty_block_for_next_slot(spec, short_state) + short_block.body.graffiti = b'\x42' * 32 + state_transition_and_sign_block(spec, short_state, short_block) + add_block_to_store(spec, store, short_block) + + short_attestation = get_valid_attestation(spec, short_state, short_block.slot, signed=True) + add_attestation_to_store(spec, store, short_attestation) + + assert spec.get_head(store) == spec.signing_root(short_block) From c64289677f0a5372dd1d8b6990ccc6510e0a14bc Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 25 Jun 2019 14:42:37 -0600 Subject: [PATCH 190/405] fix gethead tests --- specs/core/0_fork-choice.md | 22 ++++++++++++++----- .../test/fork_choice/test_on_attestation.py | 6 ++--- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index f74f487a1d..55ff91a81f 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -64,6 +64,16 @@ class Checkpoint(object): root: Hash ``` +#### `LatestMessage` + +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Hash +``` + + #### `Store` ```python @@ -75,7 +85,7 @@ class Store(object): blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) block_states: Dict[Hash, BeaconState] = field(default_factory=dict) checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_targets: Dict[ValidatorIndex, Checkpoint] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) ``` #### `get_genesis_store` @@ -113,8 +123,8 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices - if (i in store.latest_targets and - get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root) + if (i in store.latest_messages and + get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) )) ``` @@ -207,8 +217,8 @@ def on_attestation(store: Store, attestation: Attestation) -> None: indexed_attestation = convert_to_indexed(target_state, attestation) validate_indexed_attestation(target_state, indexed_attestation) - # Update latest targets + # Update latest messages for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: - if i not in store.latest_targets or target.epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = target + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root) ``` diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 48f4bf46ff..0e52e7fd05 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -17,10 +17,10 @@ def run_on_attestation(spec, state, store, attestation, valid=True): indexed_attestation = spec.convert_to_indexed(state, attestation) spec.on_attestation(store, attestation) assert ( - store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == - spec.Checkpoint( + store.latest_messages[indexed_attestation.custody_bit_0_indices[0]] == + spec.LatestMessage( epoch=attestation.data.target_epoch, - root=attestation.data.target_root, + root=attestation.data.beacon_block_root, ) ) From 8c34aa8c5f7fd7e1951843feae396ac66021e9db Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 26 Jun 2019 13:20:04 +0100 Subject: [PATCH 191/405] Initial draft --- specs/core/0_beacon-chain.md | 64 +++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 56a6fd06a4..6cf6db0cc3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -30,6 +30,7 @@ - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - [`IndexedAttestation`](#indexedattestation) - [`PendingAttestation`](#pendingattestation) + - [`CompactCommitteees`](#CompactCommitteees) - [`Eth1Data`](#eth1data) - [`HistoricalBatch`](#historicalbatch) - [`DepositData`](#depositdata) @@ -68,7 +69,7 @@ - [`get_block_root_at_slot`](#get_block_root_at_slot) - [`get_block_root`](#get_block_root) - [`get_randao_mix`](#get_randao_mix) - - [`get_active_index_root`](#get_active_index_root) + - [`get_compact_committee_root`](#get_compact_committee_root) - [`generate_seed`](#generate_seed) - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - [`verify_merkle_branch`](#verify_merkle_branch) @@ -187,7 +188,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `SHARD_COUNT` | `2**10` (= 1,024) | | `TARGET_COMMITTEE_SIZE` | `2**7` (= 128) | -| `MAX_INDICES_PER_ATTESTATION` | `2**12` (= 4,096) | +| `MAX_VALIDATORS_PER_COMMITTEE` | `2**12` (= 4,096) | | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | @@ -344,8 +345,8 @@ class AttestationDataAndCustodyBit(Container): ```python class IndexedAttestation(Container): - custody_bit_0_indices: List[ValidatorIndex, MAX_INDICES_PER_ATTESTATION] # Indices with custody bit equal to 0 - custody_bit_1_indices: List[ValidatorIndex, MAX_INDICES_PER_ATTESTATION] # Indices with custody bit equal to 1 + custody_bit_0_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] # Indices with custody bit equal to 0 + custody_bit_1_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] # Indices with custody bit equal to 1 data: AttestationData signature: BLSSignature ``` @@ -354,12 +355,22 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bytes[MAX_VALIDATORS_PER_COMMITTEE // 8] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex ``` +#### `CompactCommitteees` + +```python +class CompactCommitteees(Container): + data: Vector[Container( + pubkeys: List[Bytes48, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] + ), SHARD_COUNT] +``` + #### `Eth1Data` ```python @@ -421,9 +432,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bytes[MAX_VALIDATORS_PER_COMMITTEE // 8] data: AttestationData - custody_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + custody_bitfield: Bytes[MAX_VALIDATORS_PER_COMMITTEE // 8] signature: BLSSignature ``` @@ -511,7 +522,7 @@ class BeaconState(Container): # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] - active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Active registry digests for light clients + compact_committee_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Committee digests for light clients # Slashings slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of slashed effective balances # Attestations @@ -742,17 +753,23 @@ def get_randao_mix(state: BeaconState, return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` -### `get_active_index_root` +### `get_compact_committee_root` ```python -def get_active_index_root(state: BeaconState, - epoch: Epoch) -> Hash: +def get_compact_committee_root(state: BeaconState, epoch: Epoch) -> Hash: """ - Return the index root at a recent ``epoch``. - ``epoch`` expected to be between - (current_epoch - EPOCHS_PER_HISTORICAL_VECTOR + ACTIVATION_EXIT_DELAY, current_epoch + ACTIVATION_EXIT_DELAY]. + Return the compact committee root for the current epoch. """ - return state.active_index_roots[epoch % EPOCHS_PER_HISTORICAL_VECTOR] + committee_data = CompactCommitteees().data + for committee_number in range(get_epoch_committee_count(state, epoch)): + shard = (get_epoch_start_shard(state, epoch) + committee_number) % SHARD_COUNT + for index in get_crosslink_committee(state, epoch, shard): + validator = validators[index] + committee_data[shard].pubkeys.append(validator.pubkey) + # `index` (top 7 bytes) + `slashed` (8th bit) + `effective_balance` (bottom 7 bits) + compact_validator = index << 8 + validator.slashed << 7 + validator.effective_balance // GWEI_PER_ETH + committee_data[shard].compact_validators.append(compact_validator) + return hash_tree_root(committee_data) ``` ### `generate_seed` @@ -765,7 +782,7 @@ def generate_seed(state: BeaconState, """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + - get_active_index_root(state, epoch) + + get_compact_committee_root(state, epoch) + int_to_bytes(epoch, length=32) ) ``` @@ -973,7 +990,7 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify no index has custody bit equal to 1 [to be removed in phase 1] assert len(bit_1_indices) == 0 # Verify max number of indices - assert len(bit_0_indices) + len(bit_1_indices) <= MAX_INDICES_PER_ATTESTATION + assert len(bit_0_indices) + len(bit_1_indices) <= MAX_VALIDATORS_PER_COMMITTEE # Verify index sets are disjoint assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 # Verify indices are sorted @@ -1173,12 +1190,9 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots - genesis_active_index_root = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) - ) + # Populate compact_committee_roots for index in range(EPOCHS_PER_HISTORICAL_VECTOR): - state.active_index_roots[index] = genesis_active_index_root + state.compact_committee_roots[index] = get_compact_committee_root(state, GENESIS_EPOCH) return state ``` @@ -1532,11 +1546,7 @@ def process_final_updates(state: BeaconState) -> None: state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT) # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR - state.active_index_roots[index_root_position] = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT]( - get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) - ) - ) + state.compact_committee_roots[index_root_position] = get_compact_committee_root(state, next_epoch + ACTIVATION_EXIT_DELAY) # Set total slashed balances state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] From ccda5082540a4cb7b7ab46de290ce29cb9e60d0f Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 26 Jun 2019 15:34:54 +0100 Subject: [PATCH 192/405] Fix deposit root, add min_genesis_time, per-block processing --- specs/core/0_beacon-chain.md | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8aa307d666..588c2d1336 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -126,7 +126,7 @@ This document represents the specification for Phase 0 of Ethereum 2.0 -- The Beacon Chain. -At the core of Ethereum 2.0 is a system chain called the "beacon chain". The beacon chain stores and manages the registry of [validators](#dfn-validator). In the initial deployment phases of Ethereum 2.0, the only mechanism to become a [validator](#dfn-validator) is to make a one-way ETH transaction to a deposit contract on Ethereum 1.0. Activation as a [validator](#dfn-validator) happens when Ethereum 1.0 deposit receipts are processed by the beacon chain, the activation balance is reached, and a queuing process is completed. Exit is either voluntary or done forcibly as a penalty for misbehavior. +At the core of Ethereum 2.0 is a system chain called the "beacon chain". The beacon chain stores and manages the registry of [validators](#dfn-validator). In the initial deployment phases of Ethereum 2.0, the only mechanism to become a [validator](#dfn-validator) is to make a one-way ETH transaction to a deposit contract on Eth 1.0. Activation as a [validator](#dfn-validator) happens when Eth 1.0 deposit receipts are processed by the beacon chain, the activation balance is reached, and a queuing process is completed. Exit is either voluntary or done forcibly as a penalty for misbehavior. The primary source of load on the beacon chain is "attestations". Attestations are simultaneously availability votes for a shard block and proof-of-stake votes for a beacon block. A sufficient number of attestations for the same shard block create a "crosslink", confirming the shard segment up to that shard block into the beacon chain. Crosslinks also serve as infrastructure for asynchronous cross-shard communication. @@ -136,7 +136,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code. ## Terminology -* **Validator**—a registered participant in the beacon chain. You can become one by sending ether into the Ethereum 1.0 deposit contract. +* **Validator**—a registered participant in the beacon chain. You can become one by sending ether into the Eth 1.0 deposit contract. * **Active validator**—an active participant in the Ethereum 2.0 consensus invited to, among other things, propose and attest to blocks and vote for crosslinks. * **Committee**—a (pseudo-) randomly sampled subset of [active validators](#dfn-active-validator). When a committee is referred to collectively, as in "this committee attests to X", this is assumed to mean "some subset of that committee that contains enough [validators](#dfn-validator) that the protocol recognizes it as representing the committee". * **Proposer**—the [validator](#dfn-validator) that creates a beacon chain block. @@ -195,6 +195,7 @@ These configurations are updated for releases, but may be out of sync during `de | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | | `GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**16` (= 65,536) | +| `MIN_GENESIS_TIME` | `1578009600` (Jan 3, 2020) | * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -1130,31 +1131,34 @@ def slash_validator(state: BeaconState, ### Genesis trigger -Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool` where: +Before genesis has been triggered and for every Eth 1.0 block call `is_genesis_trigger(deposits, time)` where: -* `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log -* `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log +* `deposits` is the list of all deposits up to the Eth 1.0 block, ordered chronologically +* `time` is the Unix time of the Eth 1.0 block -When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: +When `is_genesis_trigger(deposits, time) is True` for the first time let: * `genesis_deposits = deposits` -* `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` +* `genesis_time = time - time % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` * `genesis_eth1_data` be the object of type `Eth1Data` where: * `genesis_eth1_data.deposit_root` is the deposit root for the last deposit in `deposits` * `genesis_eth1_data.deposit_count = len(genesis_deposits)` - * `genesis_eth1_data.block_hash` is the Ethereum 1.0 block hash that emitted the log for the last deposit in `deposits` + * `genesis_eth1_data.block_hash` is the Eth 1.0 block hash that emitted the log for the last deposit in `deposits` *Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: ```python -def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: +def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], time: uint64) -> bool: + # Do not deploy too early + if time - time % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY < MIN_GENESIS_TIME: + return False + # Initialize deposit root state = BeaconState() - for i in range(DEPOSIT_CONTRACT_TREE_DEPTH - 1): - state.eth1_data.deposit_root = hash(state.eth1_data.deposit_root + state.eth1_data.deposit_root) + state.eth1_data.deposit_root = hash_tree_root(map(deposits, lambda deposit: deposit.data)) # Process deposits - for deposit in deposits: + for deposit in deposits process_deposit(state, deposit) # Count active validators at genesis From d587c4fe619458b9e036ab44ecb8318cd91a0de6 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Wed, 26 Jun 2019 21:21:17 +0200 Subject: [PATCH 193/405] Critical fix: introduce back total-value check (#1220) This was dropped in a376b6607fe5e6406371f44254960e891ee5ee8d, as improvement in dust checking. Now that dust-checking is done, we still need to check if the sender has the minimum value, as decrease balance just clips to 0. See be86f966f87958856584b3f20c095abf910a3d0c for older dust-creation problem work around, which was dropped in the above. The bug enabled you to transfer your full balance to someone else, and pay the same amount in fee, possibly to a puppet proposer to collect back funds. Effectively enabling printing of money. Silly bug, good to fix and introduce tests for. --- specs/core/0_beacon-chain.md | 4 +- .../block_processing/test_process_transfer.py | 138 +++++++++++++++++- 2 files changed, 138 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 56a6fd06a4..264c6d23b8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1795,8 +1795,8 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: """ Process ``Transfer`` operation. """ - # Verify the amount and fee are not individually too big (for anti-overflow purposes) - assert state.balances[transfer.sender] >= max(transfer.amount, transfer.fee) + # Verify the balance the covers amount and fee (with overflow protection) + assert state.balances[transfer.sender] >= max(transfer.amount + transfer.fee, transfer.amount, transfer.fee) # A transfer is valid in only one slot assert state.slot == transfer.slot # Sender must satisfy at least one of the following conditions in the parenthesis: diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index e9d282b3a6..89246cc51a 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -114,7 +114,7 @@ def test_incorrect_slot(spec, state): @with_all_phases @spec_state_test -def test_insufficient_balance_for_fee(spec, state): +def test_insufficient_balance_for_fee_result_dust(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -127,7 +127,20 @@ def test_insufficient_balance_for_fee(spec, state): @with_all_phases @spec_state_test -def test_insufficient_balance(spec, state): +def test_insufficient_balance_for_fee_result_full(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=0, fee=state.balances[sender_index] + 1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_amount_result_dust(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -138,6 +151,127 @@ def test_insufficient_balance(spec, state): yield from run_transfer_processing(spec, state, transfer, False) +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_amount_result_full(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=state.balances[sender_index] + 1, fee=0, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_combined_result_dust(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee without dust, and amount without dust, but not both. + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT + 1 + transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_combined_result_full(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT * 2 + 1 + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=spec.MIN_DEPOSIT_AMOUNT + 1, + fee=spec.MIN_DEPOSIT_AMOUNT + 1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_combined_big_amount(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + # Try to create a dust balance (off by 1) with combination of fee and amount. + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT * 2 + 1 + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=spec.MIN_DEPOSIT_AMOUNT + 1, fee=1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_combined_big_fee(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + # Try to create a dust balance (off by 1) with combination of fee and amount. + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT * 2 + 1 + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=1, fee=spec.MIN_DEPOSIT_AMOUNT + 1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_off_by_1_fee(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + # Try to print money by using the full balance as amount, plus 1 for fee. + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=state.balances[sender_index], fee=1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_off_by_1_amount(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + # Try to print money by using the full balance as fee, plus 1 for amount. + transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, + fee=state.balances[sender_index], signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_duplicate_as_fee_and_amount(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + # Try to print money by using the full balance, twice. + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=state.balances[sender_index], + fee=state.balances[sender_index], signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + @with_all_phases @spec_state_test def test_no_dust_sender(spec, state): From b133dedeaf9d48695d407d9800a49d3b886ba8aa Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 26 Jun 2019 22:11:40 +0200 Subject: [PATCH 194/405] Eth1 data test --- .../eth2spec/test/sanity/test_blocks.py | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 286c0150ce..27f0a884f1 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -5,7 +5,7 @@ from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block # from eth2spec.test.helpers.transfers import get_valid_transfer -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing @@ -172,8 +172,6 @@ def test_attester_slashing(spec, state): ) -# TODO update functions below to be like above, i.e. with @spec_state_test and yielding data to put into the test vector - @with_all_phases @spec_state_test def test_deposit_in_block(spec, state): @@ -386,29 +384,43 @@ def test_historical_batch(spec, state): assert len(state.historical_roots) == pre_historical_roots_len + 1 -# @with_all_phases -# @spec_state_test -# def test_eth1_data_votes(spec, state): -# yield 'pre', state +@with_all_phases +@spec_state_test +def test_eth1_data_votes_success(spec, state): + # Don't run when it will take very, very long to simulate. Minimal configuration suffices. + if spec.SLOTS_PER_ETH1_VOTING_PERIOD > 16: + return -# expected_votes = 0 -# assert len(state.eth1_data_votes) == expected_votes + offset_block = build_empty_block(spec, state, slot=spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1) + state_transition_and_sign_block(spec, state, offset_block) + yield 'pre', state -# blocks = [] -# for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): -# block = build_empty_block_for_next_slot(spec, state) -# state_transition_and_sign_block(spec, state, block) -# expected_votes += 1 -# assert len(state.eth1_data_votes) == expected_votes -# blocks.append(block) + a = b'\xaa' * 32 + b = b'\xbb' * 32 + c = b'\xcc' * 32 -# block = build_empty_block_for_next_slot(spec, state) -# blocks.append(block) + blocks = [] -# state_transition_and_sign_block(spec, state, block) + for i in range(0, spec.SLOTS_PER_ETH1_VOTING_PERIOD): + block = build_empty_block_for_next_slot(spec, state) + # wait for over 50% for A, then start voting B + block.body.eth1_data.block_hash = b if i * 2 > spec.SLOTS_PER_ETH1_VOTING_PERIOD else a + state_transition_and_sign_block(spec, state, block) + blocks.append(block) -# yield 'blocks', [block] -# yield 'post', state + assert len(state.eth1_data_votes) == spec.SLOTS_PER_ETH1_VOTING_PERIOD + assert state.eth1_data.block_hash == a + + # transition to next eth1 voting period + block = build_empty_block_for_next_slot(spec, state) + block.body.eth1_data.block_hash = c + state_transition_and_sign_block(spec, state, block) + blocks.append(block) + + yield 'blocks', blocks + yield 'post', state -# assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 -# assert len(state.eth1_data_votes) == 1 + assert state.eth1_data.block_hash == a + assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 + assert len(state.eth1_data_votes) == 1 + assert state.eth1_data_votes[0].block_hash == c From f54d1a56f7192be254a8d8d87b8f67a84758af7f Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 01:23:37 +0200 Subject: [PATCH 195/405] eth1 voting no consensus test --- .../eth2spec/test/sanity/test_blocks.py | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 27f0a884f1..b11e41d666 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -4,7 +4,6 @@ from eth2spec.utils.bls import bls_sign from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block -# from eth2spec.test.helpers.transfers import get_valid_transfer from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing @@ -386,7 +385,7 @@ def test_historical_batch(spec, state): @with_all_phases @spec_state_test -def test_eth1_data_votes_success(spec, state): +def test_eth1_data_votes_consensus(spec, state): # Don't run when it will take very, very long to simulate. Minimal configuration suffices. if spec.SLOTS_PER_ETH1_VOTING_PERIOD > 16: return @@ -424,3 +423,33 @@ def test_eth1_data_votes_success(spec, state): assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 assert len(state.eth1_data_votes) == 1 assert state.eth1_data_votes[0].block_hash == c + + +@with_all_phases +@spec_state_test +def test_eth1_data_votes_no_consensus(spec, state): + # Don't run when it will take very, very long to simulate. Minimal configuration suffices. + if spec.SLOTS_PER_ETH1_VOTING_PERIOD > 16: + return + + offset_block = build_empty_block(spec, state, slot=spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1) + state_transition_and_sign_block(spec, state, offset_block) + yield 'pre', state + + a = b'\xaa' * 32 + b = b'\xbb' * 32 + + blocks = [] + + for i in range(0, spec.SLOTS_PER_ETH1_VOTING_PERIOD): + block = build_empty_block_for_next_slot(spec, state) + # wait for precisely 50% for A, then start voting B for other 50% + block.body.eth1_data.block_hash = b if i * 2 >= spec.SLOTS_PER_ETH1_VOTING_PERIOD else a + state_transition_and_sign_block(spec, state, block) + blocks.append(block) + + assert len(state.eth1_data_votes) == spec.SLOTS_PER_ETH1_VOTING_PERIOD + assert state.eth1_data.block_hash == b'\x00' * 32 + + yield 'blocks', blocks + yield 'post', state From 13b67b4cde6ddb762927ee78645ce38bd8538a76 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 01:46:26 +0200 Subject: [PATCH 196/405] sign blocks in eth1 vote tests --- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index b11e41d666..34409986ed 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -373,6 +373,7 @@ def test_historical_batch(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state, signed=True) + sign_block(spec, state, block) state_transition_and_sign_block(spec, state, block) yield 'blocks', [block] @@ -391,6 +392,7 @@ def test_eth1_data_votes_consensus(spec, state): return offset_block = build_empty_block(spec, state, slot=spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1) + sign_block(spec, state, offset_block) state_transition_and_sign_block(spec, state, offset_block) yield 'pre', state @@ -404,6 +406,7 @@ def test_eth1_data_votes_consensus(spec, state): block = build_empty_block_for_next_slot(spec, state) # wait for over 50% for A, then start voting B block.body.eth1_data.block_hash = b if i * 2 > spec.SLOTS_PER_ETH1_VOTING_PERIOD else a + sign_block(spec, state, block) state_transition_and_sign_block(spec, state, block) blocks.append(block) @@ -413,6 +416,7 @@ def test_eth1_data_votes_consensus(spec, state): # transition to next eth1 voting period block = build_empty_block_for_next_slot(spec, state) block.body.eth1_data.block_hash = c + sign_block(spec, state, block) state_transition_and_sign_block(spec, state, block) blocks.append(block) @@ -433,6 +437,7 @@ def test_eth1_data_votes_no_consensus(spec, state): return offset_block = build_empty_block(spec, state, slot=spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1) + sign_block(spec, state, offset_block) state_transition_and_sign_block(spec, state, offset_block) yield 'pre', state @@ -445,6 +450,7 @@ def test_eth1_data_votes_no_consensus(spec, state): block = build_empty_block_for_next_slot(spec, state) # wait for precisely 50% for A, then start voting B for other 50% block.body.eth1_data.block_hash = b if i * 2 >= spec.SLOTS_PER_ETH1_VOTING_PERIOD else a + sign_block(spec, state, block) state_transition_and_sign_block(spec, state, block) blocks.append(block) From 327953852d2034ebf6acf7eb7d39af84314b91b2 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 01:47:53 +0200 Subject: [PATCH 197/405] test invalid shard in attestation --- .../block_processing/test_process_attestation.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 2b34ab405e..79af0b2029 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -148,6 +148,20 @@ def test_wrong_shard(spec, state): yield from run_attestation_processing(spec, state, attestation, False) +@with_all_phases +@spec_state_test +def test_invalid_shard(spec, state): + attestation = get_valid_attestation(spec, state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + # off by one (with respect to valid range) on purpose + attestation.data.crosslink.shard = spec.SHARD_COUNT + + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation, False) + + @with_all_phases @spec_state_test def test_new_source_epoch(spec, state): From f75e3dccb2041cec7884266a7a348decd1051074 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 02:03:43 +0200 Subject: [PATCH 198/405] test old and future target epoch in attestation --- .../test_process_attestation.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 79af0b2029..59e99ac0cf 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -162,6 +162,33 @@ def test_invalid_shard(spec, state): yield from run_attestation_processing(spec, state, attestation, False) +@with_all_phases +@spec_state_test +def test_old_target_epoch(spec, state): + assert spec.MIN_ATTESTATION_INCLUSION_DELAY < spec.SLOTS_PER_EPOCH * 2 + + attestation = get_valid_attestation(spec, state, signed=True) + + state.slot = spec.SLOTS_PER_EPOCH * 2 # target epoch will be too old to handle + + yield from run_attestation_processing(spec, state, attestation, False) + + +@with_all_phases +@spec_state_test +def test_future_target_epoch(spec, state): + assert spec.MIN_ATTESTATION_INCLUSION_DELAY < spec.SLOTS_PER_EPOCH * 2 + + attestation = get_valid_attestation(spec, state) + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.data.target_epoch = spec.get_current_epoch(state) + 1 # target epoch will be too new to handle + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation, False) + + @with_all_phases @spec_state_test def test_new_source_epoch(spec, state): From 64e15c524b569ce474406e1c6781d95495bcd53e Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 02:27:28 +0200 Subject: [PATCH 199/405] improve intersection test, just 1 index is enough. And add invalid att1/att2 tests --- .../test_process_attester_slashing.py | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index c51f5a8a90..2d8f9d31a2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -142,12 +142,39 @@ def test_participants_already_slashed(spec, state): @with_all_phases @spec_state_test -def test_custody_bit_0_and_1(spec, state): +def test_custody_bit_0_and_1_intersect(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - attester_slashing.attestation_1.custody_bit_1_indices = ( - attester_slashing.attestation_1.custody_bit_0_indices + attester_slashing.attestation_1.custody_bit_1_indices.append( + attester_slashing.attestation_1.custody_bit_0_indices[0] ) + + sign_indexed_attestation(spec, state, attester_slashing.attestation_1) + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@with_all_phases +@spec_state_test +def test_attester_slashing_invalid_att_1(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) + + indices = attester_slashing.attestation_1.custody_bit_0_indices + assert len(indices) >= 3 + indices[1], indices[2] = indices[2], indices[1] # unsort second and third index sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@with_all_phases +@spec_state_test +def test_attester_slashing_invalid_att_2(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) + + indices = attester_slashing.attestation_2.custody_bit_0_indices + assert len(indices) >= 3 + indices[1], indices[2] = indices[2], indices[1] # unsort second and third index + sign_indexed_attestation(spec, state, attester_slashing.attestation_2) + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) From 1d6b1cab13dc979f15e1e1bc5b50ae3c3585e795 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 02:37:20 +0200 Subject: [PATCH 200/405] expected deposit count test --- .../eth2spec/test/sanity/test_blocks.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 34409986ed..bb6a6dc06e 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -171,6 +171,28 @@ def test_attester_slashing(spec, state): ) +@with_all_phases +@spec_state_test +def test_expected_deposit_in_block(spec, state): + # Make the state expect a deposit, then don't provide it. + state.eth1_data.deposit_count += 1 + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + sign_block(spec, state, block) + bad = False + try: + state_transition_and_sign_block(spec, state, block) + bad = True + except AssertionError: + pass + if bad: + raise AssertionError("expected deposit was not enforced") + + yield 'blocks', [block] + yield 'post', None + + @with_all_phases @spec_state_test def test_deposit_in_block(spec, state): From 55d86b4f13afacf5fec4e48cb544cfd003c7cd31 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 02:48:40 +0200 Subject: [PATCH 201/405] effective balance testing in deposits --- .../block_processing/test_process_deposit.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 8b3d7b413e..1d4b3a107c 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -49,6 +49,11 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef assert len(state.balances) == pre_validator_count + 1 assert get_balance(state, validator_index) == pre_balance + deposit.data.amount + effective = min(spec.MAX_EFFECTIVE_BALANCE, + pre_balance + deposit.data.amount) + effective -= effective % spec.EFFECTIVE_BALANCE_INCREMENT + assert state.validators[validator_index].effective_balance == effective + assert state.eth1_deposit_index == state.eth1_data.deposit_count @@ -57,7 +62,20 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef def test_new_deposit(spec, state): # fresh deposit = next validator index = validator appended to registry validator_index = len(state.validators) - amount = spec.MAX_EFFECTIVE_BALANCE + # effective balance will be 1 EFFECTIVE_BALANCE_INCREMENT smaller because of this small decrement. + amount = spec.MAX_EFFECTIVE_BALANCE - 1 + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + +@with_all_phases +@spec_state_test +def test_new_deposit_maxed_out(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + # just 1 over the limit, effective balance should be set MAX_EFFECTIVE_BALANCE during processing + amount = spec.MAX_EFFECTIVE_BALANCE + 1 deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) yield from run_deposit_processing(spec, state, deposit, validator_index) From 063d94b9c7da65d076f563e243584fa0b8b9ab3d Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 04:13:48 +0200 Subject: [PATCH 202/405] Bugfix transfer tests --- .../block_processing/test_process_transfer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 89246cc51a..45c1612147 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -107,7 +107,7 @@ def test_active_but_transfer_past_effective_balance(spec, state): def test_incorrect_slot(spec, state): transfer = get_valid_transfer(spec, state, slot=state.slot + 1, signed=True) # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -120,7 +120,7 @@ def test_insufficient_balance_for_fee_result_dust(spec, state): transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -146,7 +146,7 @@ def test_insufficient_balance_for_amount_result_dust(spec, state): transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -287,7 +287,7 @@ def test_no_dust_sender(spec, state): ) # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -301,7 +301,7 @@ def test_no_dust_recipient(spec, state): state.balances[transfer.recipient] = 0 # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -313,6 +313,6 @@ def test_invalid_pubkey(spec, state): state.validators[transfer.sender].withdrawal_credentials = spec.ZERO_HASH # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) From e79b47e3c3a24d54f341d3defb691707c0acfb73 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 03:54:07 +0200 Subject: [PATCH 203/405] non-existent transfer participants tests --- .../pyspec/eth2spec/test/helpers/transfers.py | 6 ++-- .../block_processing/test_process_transfer.py | 34 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index acc6a35c53..fa01a30880 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -4,13 +4,15 @@ from eth2spec.utils.ssz.ssz_impl import signing_root -def get_valid_transfer(spec, state, slot=None, sender_index=None, amount=None, fee=None, signed=False): +def get_valid_transfer(spec, state, slot=None, sender_index=None, + recipient_index=None, amount=None, fee=None, signed=False): if slot is None: slot = state.slot current_epoch = spec.get_current_epoch(state) if sender_index is None: sender_index = spec.get_active_validator_indices(state, current_epoch)[-1] - recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] + if recipient_index is None: + recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] transfer_pubkey = pubkeys[-1] transfer_privkey = privkeys[-1] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 45c1612147..ad7e6ff60e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -1,7 +1,7 @@ from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.block import apply_empty_block -from eth2spec.test.helpers.transfers import get_valid_transfer +from eth2spec.test.helpers.transfers import get_valid_transfer, sign_transfer def run_transfer_processing(spec, state, transfer, valid=True): @@ -13,11 +13,6 @@ def run_transfer_processing(spec, state, transfer, valid=True): If ``valid == False``, run expecting ``AssertionError`` """ - proposer_index = spec.get_beacon_proposer_index(state) - pre_transfer_sender_balance = state.balances[transfer.sender] - pre_transfer_recipient_balance = state.balances[transfer.recipient] - pre_transfer_proposer_balance = state.balances[proposer_index] - yield 'pre', state yield 'transfer', transfer @@ -26,6 +21,11 @@ def run_transfer_processing(spec, state, transfer, valid=True): yield 'post', None return + proposer_index = spec.get_beacon_proposer_index(state) + pre_transfer_sender_balance = state.balances[transfer.sender] + pre_transfer_recipient_balance = state.balances[transfer.recipient] + pre_transfer_proposer_balance = state.balances[proposer_index] + spec.process_transfer(state, transfer) yield 'post', state @@ -306,6 +306,28 @@ def test_no_dust_recipient(spec, state): yield from run_transfer_processing(spec, state, transfer, False) +@with_all_phases +@spec_state_test +def test_non_existent_sender(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0) + transfer.sender = len(state.validators) + sign_transfer(spec, state, transfer, 42) # mostly valid signature, but sender won't exist, use bogus key. + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_non_existent_recipient(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + recipient_index=len(state.validators), amount=1, fee=0, signed=True) + + yield from run_transfer_processing(spec, state, transfer, False) + + @with_all_phases @spec_state_test def test_invalid_pubkey(spec, state): From 6266133572494b606e12db1326fa2ac597c215af Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Tue, 25 Jun 2019 03:53:40 +0200 Subject: [PATCH 204/405] rename test methods based on suggestion Co-Authored-By: Danny Ryan --- .../block_processing/test_process_attester_slashing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 2d8f9d31a2..86b6811a2b 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -156,7 +156,7 @@ def test_custody_bit_0_and_1_intersect(spec, state): @with_all_phases @spec_state_test -def test_attester_slashing_invalid_att_1(spec, state): +def test_unsorted_att_1(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) indices = attester_slashing.attestation_1.custody_bit_0_indices @@ -169,7 +169,7 @@ def test_attester_slashing_invalid_att_1(spec, state): @with_all_phases @spec_state_test -def test_attester_slashing_invalid_att_2(spec, state): +def test_unsorted_att_2(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) indices = attester_slashing.attestation_2.custody_bit_0_indices From c4b88e68e1772bd2a88987f030113309dd4a9c4d Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 04:19:02 +0200 Subject: [PATCH 205/405] different new-deposit tests --- .../block_processing/test_process_deposit.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 1d4b3a107c..0f94bd26cc 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -59,10 +59,10 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef @with_all_phases @spec_state_test -def test_new_deposit(spec, state): +def test_new_deposit_under_max(spec, state): # fresh deposit = next validator index = validator appended to registry validator_index = len(state.validators) - # effective balance will be 1 EFFECTIVE_BALANCE_INCREMENT smaller because of this small decrement. + # effective balance will be 1 EFFECTIVE_BALANCE_INCREMENT smaller because of this small decrement. amount = spec.MAX_EFFECTIVE_BALANCE - 1 deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) @@ -71,7 +71,19 @@ def test_new_deposit(spec, state): @with_all_phases @spec_state_test -def test_new_deposit_maxed_out(spec, state): +def test_new_deposit_max(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + # effective balance will be exactly the same as balance. + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + +@with_all_phases +@spec_state_test +def test_new_deposit_over_max(spec, state): # fresh deposit = next validator index = validator appended to registry validator_index = len(state.validators) # just 1 over the limit, effective balance should be set MAX_EFFECTIVE_BALANCE during processing From b4b4e9571da32e6ff13cd550cb1308cfdf703919 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 04:45:53 +0200 Subject: [PATCH 206/405] test activation queue --- .../test_process_registry_updates.py | 51 +++++++++++++++---- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index d92220910e..6c1319a5b6 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -30,18 +30,20 @@ def run_process_registry_updates(spec, state, valid=True): yield 'post', state -@with_all_phases -@spec_state_test -def test_activation(spec, state): - index = 0 +def mock_deposit(spec, state, index): assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) - - # Mock a new deposit state.validators[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH state.validators[index].activation_epoch = spec.FAR_FUTURE_EPOCH state.validators[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE assert not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) + +@with_all_phases +@spec_state_test +def test_activation(spec, state): + index = 0 + mock_deposit(spec, state, index) + for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): next_epoch(spec, state) @@ -49,10 +51,39 @@ def test_activation(spec, state): assert state.validators[index].activation_eligibility_epoch != spec.FAR_FUTURE_EPOCH assert state.validators[index].activation_epoch != spec.FAR_FUTURE_EPOCH - assert spec.is_active_validator( - state.validators[index], - spec.get_current_epoch(state), - ) + assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) + + +@with_all_phases +@spec_state_test +def test_activation_queue_sorting(spec, state): + mock_activations = 10 + + epoch = spec.get_current_epoch(state) + for i in range(mock_activations): + mock_deposit(spec, state, i) + state.validators[i].activation_eligibility_epoch = epoch + 1 + + # give the last priority over the others + state.validators[mock_activations - 1].activation_eligibility_epoch = epoch + + # make sure we are hitting the churn + churn_limit = spec.get_churn_limit(state) + assert mock_activations > churn_limit + + yield from run_process_registry_updates(spec, state) + + # the first got in as second + assert state.validators[0].activation_epoch != spec.FAR_FUTURE_EPOCH + # the prioritized got in as first + assert state.validators[mock_activations - 1].activation_epoch != spec.FAR_FUTURE_EPOCH + # the second last is at the end of the queue, and did not make the churn, + # hence is not assigned an activation_epoch yet. + assert state.validators[mock_activations - 2].activation_epoch == spec.FAR_FUTURE_EPOCH + # the one at churn_limit - 1 did not make it, it was out-prioritized + assert state.validators[churn_limit - 1].activation_epoch == spec.FAR_FUTURE_EPOCH + # but the the one in front of the above did + assert state.validators[churn_limit - 2].activation_epoch != spec.FAR_FUTURE_EPOCH @with_all_phases From 0e3c2cef5ca56e011ea1ec776e8eac6aa0884f3e Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 15:45:57 +0200 Subject: [PATCH 207/405] fix transfer tests, add 2 new tests --- .../block_processing/test_process_transfer.py | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index ad7e6ff60e..6903f06666 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -114,9 +114,37 @@ def test_incorrect_slot(spec, state): @with_all_phases @spec_state_test -def test_insufficient_balance_for_fee_result_dust(spec, state): +def test_transfer_clean(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=spec.MIN_DEPOSIT_AMOUNT, fee=0, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer) + + +@with_all_phases +@spec_state_test +def test_transfer_clean_split_to_fee(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=spec.MIN_DEPOSIT_AMOUNT // 2, fee=spec.MIN_DEPOSIT_AMOUNT // 2, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_fee(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) # un-activate so validator can transfer @@ -142,7 +170,7 @@ def test_insufficient_balance_for_fee_result_full(spec, state): @spec_state_test def test_insufficient_balance_for_amount_result_dust(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) # un-activate so validator can transfer From b2034a54a07264553f7aad38b838cedd98ff64ec Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 22:22:17 +0200 Subject: [PATCH 208/405] generalize epoch processing testing, add final-processing tests --- .../run_epoch_process_base.py | 44 +++++++++++++++ .../test_process_crosslinks.py | 30 ++--------- .../test_process_final_updates.py | 53 +++++++++++++++++++ .../test_process_registry_updates.py | 31 ++--------- 4 files changed, 105 insertions(+), 53 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py create mode 100644 test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py new file mode 100644 index 0000000000..13e1c3f3bf --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py @@ -0,0 +1,44 @@ +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block +from eth2spec.test.helpers.state import state_transition_and_sign_block + + +process_calls = ( + 'process_justification_and_finalization' + 'process_crosslinks' + 'process_rewards_and_penalties' + 'process_registry_updates' + 'process_reveal_deadlines' + 'process_challenge_deadlines' + 'process_slashings' + 'process_final_updates' + 'after_process_final_updates' +) + + +def run_epoch_processing_to(spec, state, process_name: str): + """ + Run the epoch processing functions up to ``process_name`` (incl.), yielding: + - pre-state ('pre'), state before calling ``process_name`` + - post-state ('post'), state after calling ``process_name`` + """ + # transition state to slot before state transition + slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 + block = build_empty_block_for_next_slot(spec, state) + block.slot = slot + sign_block(spec, state, block) + state_transition_and_sign_block(spec, state, block) + + # cache state before epoch transition + spec.process_slot(state) + + # process components of epoch transition before final-updates + for name in process_calls: + if name == process_name: + break + # only run when present. Later phases introduce more to the epoch-processing. + if hasattr(spec, name): + getattr(spec, name)(state) + + yield 'pre', state + getattr(spec, process_name)(state) + yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index d51191efba..a38435b967 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -4,41 +4,19 @@ from eth2spec.test.helpers.state import ( next_epoch, next_slot, - state_transition_and_sign_block, ) -from eth2spec.test.helpers.block import apply_empty_block, sign_block +from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.attestations import ( add_attestation_to_state, - build_empty_block_for_next_slot, fill_aggregate_attestation, get_valid_attestation, sign_attestation, ) +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to -def run_process_crosslinks(spec, state, valid=True): - """ - Run ``process_crosslinks``, yielding: - - pre-state ('pre') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - # transition state to slot before state transition - slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(spec, state) - block.slot = slot - sign_block(spec, state, block) - state_transition_and_sign_block(spec, state, block) - - # cache state before epoch transition - spec.process_slot(state) - - # process components of epoch transition before processing crosslinks - spec.process_justification_and_finalization(state) - - yield 'pre', state - spec.process_crosslinks(state) - yield 'post', state +def run_process_crosslinks(spec, state): + yield from run_epoch_processing_to(spec, state, 'process_crosslinks') @with_all_phases diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py new file mode 100644 index 0000000000..ca2f9eb100 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py @@ -0,0 +1,53 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to + + +def run_process_final_updates(spec, state): + yield from run_epoch_processing_to(spec, state, 'process_final_updates') + + +@with_all_phases +@spec_state_test +def test_eth1_vote_reset(spec, state): + # skip ahead to near the end of the voting period + state.slot = spec.SLOTS_PER_ETH1_VOTING_PERIOD - 2 + for i in range(state.slot + 1): # add a vote for each skipped slot. + state.eth1_data_votes.append( + spec.Eth1Data(deposit_root=b'\xaa' * 32, + deposit_count=state.eth1_deposit_index, + block_hash=b'\xbb' * 32)) + + yield from run_process_final_updates(spec, state) + + assert len(state.eth1_data_votes) == 0 + + +@with_all_phases +@spec_state_test +def test_effective_balance_hysteresis(spec, state): + # Set some edge cases for balances + max = spec.MAX_EFFECTIVE_BALANCE + min = spec.EJECTION_BALANCE + inc = spec.EFFECTIVE_BALANCE_INCREMENT + half_inc = inc // 2 + cases = [ + (max, max, max), # as is + (max, max - 1, max - inc), # round down, step lower + (max, max + 1, max), # round down + (max, max - inc, max - inc), # exactly 1 step lower + (max, max - inc - 1, max - (2 * inc)), # just 1 over 1 step lower + (max, max - inc + 1, max - inc), # close to 1 step lower + (min, min + (half_inc * 3), min), # bigger balance, but not high enough + (min, min + (half_inc * 3) + 1, min + inc), # bigger balance, high enough, but small step + (min, min + (half_inc * 4) - 1, min + inc), # bigger balance, high enough, close to double step + (min, min + (half_inc * 4), min + (2 * inc)), # exact two step balance increment + (min, min + (half_inc * 4) + 1, min + (2 * inc)), # over two steps, round down + ] + for i, (pre_eff, bal, _) in enumerate(cases): + state.validators[i].effective_balance = pre_eff + state.balances[i] = bal + + yield from run_process_final_updates(spec, state) + + for i, (_, _, post_eff) in enumerate(cases): + assert state.validators[i].effective_balance == post_eff diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index 6c1319a5b6..ae76e95c20 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -1,33 +1,10 @@ -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block -from eth2spec.test.helpers.state import next_epoch, state_transition_and_sign_block +from eth2spec.test.helpers.state import next_epoch from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to -def run_process_registry_updates(spec, state, valid=True): - """ - Run ``process_crosslinks``, yielding: - - pre-state ('pre') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - # transition state to slot before state transition - slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(spec, state) - block.slot = slot - sign_block(spec, state, block) - state_transition_and_sign_block(spec, state, block) - - # cache state before epoch transition - spec.process_slot(state) - - # process components of epoch transition before registry update - spec.process_justification_and_finalization(state) - spec.process_crosslinks(state) - spec.process_rewards_and_penalties(state) - - yield 'pre', state - spec.process_registry_updates(state) - yield 'post', state +def run_process_registry_updates(spec, state): + yield from run_epoch_processing_to(spec, state, 'process_registry_updates') def mock_deposit(spec, state, index): From c4c9bd32e2af615bae95220c0c9e2492dc5ab67b Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 22:26:35 +0200 Subject: [PATCH 209/405] test_eth1_vote_no_reset --- .../test_process_final_updates.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py index ca2f9eb100..0c19f8e31f 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py @@ -6,6 +6,23 @@ def run_process_final_updates(spec, state): yield from run_epoch_processing_to(spec, state, 'process_final_updates') +@with_all_phases +@spec_state_test +def test_eth1_vote_no_reset(spec, state): + assert spec.SLOTS_PER_ETH1_VOTING_PERIOD > spec.SLOTS_PER_EPOCH + # skip ahead to near the end of the epoch + state.slot = spec.SLOTS_PER_EPOCH - 2 + for i in range(state.slot + 1): # add a vote for each skipped slot. + state.eth1_data_votes.append( + spec.Eth1Data(deposit_root=b'\xaa' * 32, + deposit_count=state.eth1_deposit_index, + block_hash=b'\xbb' * 32)) + + yield from run_process_final_updates(spec, state) + + assert len(state.eth1_data_votes) == spec.SLOTS_PER_EPOCH + + @with_all_phases @spec_state_test def test_eth1_vote_reset(spec, state): From aedd281edbcacfa68ff7ec91d4f3f2b0031a61c7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 23:18:19 +0200 Subject: [PATCH 210/405] clean up epoch processing testing --- .../run_epoch_process_base.py | 49 ++++++++--------- .../test_process_final_updates.py | 55 +++++++++++++------ 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py index 13e1c3f3bf..7e0cffc798 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py @@ -1,35 +1,29 @@ -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block -from eth2spec.test.helpers.state import state_transition_and_sign_block +process_calls = [ + 'process_justification_and_finalization', + 'process_crosslinks', + 'process_rewards_and_penalties', + 'process_registry_updates', + 'process_reveal_deadlines', + 'process_challenge_deadlines', + 'process_slashings', + 'process_final_updates', + 'after_process_final_updates', +] -process_calls = ( - 'process_justification_and_finalization' - 'process_crosslinks' - 'process_rewards_and_penalties' - 'process_registry_updates' - 'process_reveal_deadlines' - 'process_challenge_deadlines' - 'process_slashings' - 'process_final_updates' - 'after_process_final_updates' -) - -def run_epoch_processing_to(spec, state, process_name: str): +def run_epoch_processing_to(spec, state, process_name: str, exclusive=False): """ - Run the epoch processing functions up to ``process_name`` (incl.), yielding: + Run the epoch processing functions up to ``process_name``. + If ``exclusive`` is True, the process itself will not be ran. + If ``exclusive`` is False (default), this function yields: - pre-state ('pre'), state before calling ``process_name`` - post-state ('post'), state after calling ``process_name`` """ - # transition state to slot before state transition - slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(spec, state) - block.slot = slot - sign_block(spec, state, block) - state_transition_and_sign_block(spec, state, block) + slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - # cache state before epoch transition - spec.process_slot(state) + # transition state to slot before epoch state transition + spec.process_slots(state, slot) # process components of epoch transition before final-updates for name in process_calls: @@ -39,6 +33,7 @@ def run_epoch_processing_to(spec, state, process_name: str): if hasattr(spec, name): getattr(spec, name)(state) - yield 'pre', state - getattr(spec, process_name)(state) - yield 'post', state + if not exclusive: + yield 'pre', state + getattr(spec, process_name)(state) + yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py index 0c19f8e31f..d1af7d396e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py @@ -10,8 +10,8 @@ def run_process_final_updates(spec, state): @spec_state_test def test_eth1_vote_no_reset(spec, state): assert spec.SLOTS_PER_ETH1_VOTING_PERIOD > spec.SLOTS_PER_EPOCH - # skip ahead to near the end of the epoch - state.slot = spec.SLOTS_PER_EPOCH - 2 + # skip ahead to the end of the epoch + state.slot = spec.SLOTS_PER_EPOCH - 1 for i in range(state.slot + 1): # add a vote for each skipped slot. state.eth1_data_votes.append( spec.Eth1Data(deposit_root=b'\xaa' * 32, @@ -26,8 +26,8 @@ def test_eth1_vote_no_reset(spec, state): @with_all_phases @spec_state_test def test_eth1_vote_reset(spec, state): - # skip ahead to near the end of the voting period - state.slot = spec.SLOTS_PER_ETH1_VOTING_PERIOD - 2 + # skip ahead to the end of the voting period + state.slot = spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1 for i in range(state.slot + 1): # add a vote for each skipped slot. state.eth1_data_votes.append( spec.Eth1Data(deposit_root=b'\xaa' * 32, @@ -42,29 +42,48 @@ def test_eth1_vote_reset(spec, state): @with_all_phases @spec_state_test def test_effective_balance_hysteresis(spec, state): + # Prepare state up to the final-updates. + # Then overwrite the balances, we only want to focus to be on the hysteresis based changes. + run_epoch_processing_to(spec, state, 'process_final_updates') # Set some edge cases for balances max = spec.MAX_EFFECTIVE_BALANCE min = spec.EJECTION_BALANCE inc = spec.EFFECTIVE_BALANCE_INCREMENT half_inc = inc // 2 cases = [ - (max, max, max), # as is - (max, max - 1, max - inc), # round down, step lower - (max, max + 1, max), # round down - (max, max - inc, max - inc), # exactly 1 step lower - (max, max - inc - 1, max - (2 * inc)), # just 1 over 1 step lower - (max, max - inc + 1, max - inc), # close to 1 step lower - (min, min + (half_inc * 3), min), # bigger balance, but not high enough - (min, min + (half_inc * 3) + 1, min + inc), # bigger balance, high enough, but small step - (min, min + (half_inc * 4) - 1, min + inc), # bigger balance, high enough, close to double step - (min, min + (half_inc * 4), min + (2 * inc)), # exact two step balance increment - (min, min + (half_inc * 4) + 1, min + (2 * inc)), # over two steps, round down + (max, max, max, "as-is"), + (max, max - 1, max - inc, "round down, step lower"), + (max, max + 1, max, "round down"), + (max, max - inc, max - inc, "exactly 1 step lower"), + (max, max - inc - 1, max - (2 * inc), "just 1 over 1 step lower"), + (max, max - inc + 1, max - inc, "close to 1 step lower"), + (min, min + (half_inc * 3), min, "bigger balance, but not high enough"), + (min, min + (half_inc * 3) + 1, min + inc, "bigger balance, high enough, but small step"), + (min, min + (half_inc * 4) - 1, min + inc, "bigger balance, high enough, close to double step"), + (min, min + (half_inc * 4), min + (2 * inc), "exact two step balance increment"), + (min, min + (half_inc * 4) + 1, min + (2 * inc), "over two steps, round down"), ] - for i, (pre_eff, bal, _) in enumerate(cases): + current_epoch = spec.get_current_epoch(state) + for i, (pre_eff, bal, _, _) in enumerate(cases): + assert spec.is_active_validator(state.validators[i], current_epoch) state.validators[i].effective_balance = pre_eff state.balances[i] = bal + yield 'pre', state + spec.process_final_updates(state) + yield 'post', state + + for i, (_, _, post_eff, name) in enumerate(cases): + assert state.validators[i].effective_balance == post_eff, name + + +@with_all_phases +@spec_state_test +def test_historical_root_accumulator(spec, state): + # skip ahead to near the end of the historical roots period (excl block before epoch processing) + state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1 + history_len = len(state.historical_roots) + yield from run_process_final_updates(spec, state) - for i, (_, _, post_eff) in enumerate(cases): - assert state.validators[i].effective_balance == post_eff + assert len(state.historical_roots) == history_len + 1 From c66031f55cf4b24b832f917778730c0aaf8e99dd Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 26 Jun 2019 00:01:21 +0200 Subject: [PATCH 211/405] fix crosslink tests, fix generalization of epoch processing --- .../epoch_processing/run_epoch_process_base.py | 5 ++++- .../epoch_processing/test_process_crosslinks.py | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py index 7e0cffc798..c69160fb07 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py @@ -23,7 +23,10 @@ def run_epoch_processing_to(spec, state, process_name: str, exclusive=False): slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) # transition state to slot before epoch state transition - spec.process_slots(state, slot) + spec.process_slots(state, slot - 1) + + # start transitioning, do one slot update before the epoch itself. + spec.process_slot(state) # process components of epoch transition before final-updates for name in process_calls: diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index a38435b967..7e93675e8e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -4,8 +4,8 @@ from eth2spec.test.helpers.state import ( next_epoch, next_slot, -) -from eth2spec.test.helpers.block import apply_empty_block + state_transition_and_sign_block) +from eth2spec.test.helpers.block import apply_empty_block, build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.attestations import ( add_attestation_to_state, fill_aggregate_attestation, @@ -28,6 +28,14 @@ def test_no_attestations(spec, state): assert state.previous_crosslinks[shard] == state.current_crosslinks[shard] +def add_block_to_end_of_epoch(spec, state): + slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 + block = build_empty_block_for_next_slot(spec, state) + block.slot = slot + sign_block(spec, state, block) + state_transition_and_sign_block(spec, state, block) + + @with_all_phases @spec_state_test def test_single_crosslink_update_from_current_epoch(spec, state): @@ -43,6 +51,7 @@ def test_single_crosslink_update_from_current_epoch(spec, state): shard = attestation.data.crosslink.shard pre_crosslink = deepcopy(state.current_crosslinks[shard]) + add_block_to_end_of_epoch(spec, state) yield from run_process_crosslinks(spec, state) assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] @@ -66,6 +75,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): crosslink_deltas = spec.get_crosslink_deltas(state) + add_block_to_end_of_epoch(spec, state) yield from run_process_crosslinks(spec, state) assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] From 46dc3f39bb4c7d01c4e059c69de437983ac20262 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 26 Jun 2019 00:22:24 +0200 Subject: [PATCH 212/405] detach crosslink tests from extra block --- .../epoch_processing/test_process_crosslinks.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index 7e93675e8e..058c93733d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -3,9 +3,9 @@ from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.helpers.state import ( next_epoch, - next_slot, - state_transition_and_sign_block) -from eth2spec.test.helpers.block import apply_empty_block, build_empty_block_for_next_slot, sign_block + next_slot +) +from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.attestations import ( add_attestation_to_state, fill_aggregate_attestation, @@ -28,14 +28,6 @@ def test_no_attestations(spec, state): assert state.previous_crosslinks[shard] == state.current_crosslinks[shard] -def add_block_to_end_of_epoch(spec, state): - slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(spec, state) - block.slot = slot - sign_block(spec, state, block) - state_transition_and_sign_block(spec, state, block) - - @with_all_phases @spec_state_test def test_single_crosslink_update_from_current_epoch(spec, state): @@ -51,7 +43,6 @@ def test_single_crosslink_update_from_current_epoch(spec, state): shard = attestation.data.crosslink.shard pre_crosslink = deepcopy(state.current_crosslinks[shard]) - add_block_to_end_of_epoch(spec, state) yield from run_process_crosslinks(spec, state) assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] @@ -75,7 +66,6 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): crosslink_deltas = spec.get_crosslink_deltas(state) - add_block_to_end_of_epoch(spec, state) yield from run_process_crosslinks(spec, state) assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] From 24aa0646c095cb2ef6060345a2813991be9e94cc Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 26 Jun 2019 03:17:46 +0200 Subject: [PATCH 213/405] new process-slashings tests, and epoch processing bugfix with transition-to-excl not working when not yielded from --- .../run_epoch_process_base.py | 23 ++-- .../test_process_crosslinks.py | 4 +- .../test_process_final_updates.py | 6 +- .../test_process_registry_updates.py | 4 +- .../test_process_slashings.py | 121 ++++++++++++++++++ 5 files changed, 142 insertions(+), 16 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py index c69160fb07..5b2a2ece49 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py @@ -12,13 +12,9 @@ ] -def run_epoch_processing_to(spec, state, process_name: str, exclusive=False): +def run_epoch_processing_to(spec, state, process_name: str): """ - Run the epoch processing functions up to ``process_name``. - If ``exclusive`` is True, the process itself will not be ran. - If ``exclusive`` is False (default), this function yields: - - pre-state ('pre'), state before calling ``process_name`` - - post-state ('post'), state after calling ``process_name`` + Processes to the next epoch transition, up to, but not including, the sub-transition named ``process_name`` """ slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) @@ -36,7 +32,14 @@ def run_epoch_processing_to(spec, state, process_name: str, exclusive=False): if hasattr(spec, name): getattr(spec, name)(state) - if not exclusive: - yield 'pre', state - getattr(spec, process_name)(state) - yield 'post', state + +def run_epoch_processing_with(spec, state, process_name: str): + """ + Processes to the next epoch transition, up to and including the sub-transition named ``process_name`` + - pre-state ('pre'), state before calling ``process_name`` + - post-state ('post'), state after calling ``process_name`` + """ + run_epoch_processing_to(spec, state, process_name) + yield 'pre', state + getattr(spec, process_name)(state) + yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index 058c93733d..599fde8e74 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -12,11 +12,11 @@ get_valid_attestation, sign_attestation, ) -from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with def run_process_crosslinks(spec, state): - yield from run_epoch_processing_to(spec, state, 'process_crosslinks') + yield from run_epoch_processing_with(spec, state, 'process_crosslinks') @with_all_phases diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py index d1af7d396e..58882a44f8 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py @@ -1,9 +1,11 @@ from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( + run_epoch_processing_with, run_epoch_processing_to +) def run_process_final_updates(spec, state): - yield from run_epoch_processing_to(spec, state, 'process_final_updates') + yield from run_epoch_processing_with(spec, state, 'process_final_updates') @with_all_phases diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index ae76e95c20..19500d4ab7 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -1,10 +1,10 @@ from eth2spec.test.helpers.state import next_epoch from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with def run_process_registry_updates(spec, state): - yield from run_epoch_processing_to(spec, state, 'process_registry_updates') + yield from run_epoch_processing_with(spec, state, 'process_registry_updates') def mock_deposit(spec, state, index): diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py new file mode 100644 index 0000000000..f1a23326b3 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py @@ -0,0 +1,121 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( + run_epoch_processing_with, run_epoch_processing_to +) + + +def run_process_slashings(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_slashings') + + +def slash_validators(spec, state, indices, out_epochs): + total_slashed_balance = 0 + for i, out_epoch in zip(indices, out_epochs): + v = state.validators[i] + v.slashed = True + spec.initiate_validator_exit(state, i) + v.withdrawable_epoch = out_epoch + total_slashed_balance += v.effective_balance + + state.slashed_balances[ + spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR + ] = total_slashed_balance + + +@with_all_phases +@spec_state_test +def test_max_penalties(spec, state): + slashed_count = (len(state.validators) // 3) + 1 + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + + slashed_indices = list(range(slashed_count)) + slash_validators(spec, state, slashed_indices, [out_epoch] * slashed_count) + + total_balance = spec.get_total_active_balance(state) + total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] + + assert total_balance // 3 <= total_penalties + + yield from run_process_slashings(spec, state) + + for i in slashed_indices: + assert state.balances[i] == 0 + + +@with_all_phases +@spec_state_test +def test_min_penalties(spec, state): + # run_epoch_processing_to(spec, state, 'process_slashings', exclusive=True) + + # Just the bare minimum for this one validator + pre_balance = state.balances[0] = state.validators[0].effective_balance = spec.EJECTION_BALANCE + # All the other validators get the maximum. + for i in range(1, len(state.validators)): + state.validators[i].effective_balance = state.balances[i] = spec.MAX_EFFECTIVE_BALANCE + + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + + slash_validators(spec, state, [0], [out_epoch]) + + total_balance = spec.get_total_active_balance(state) + total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] + + # we are testing the minimum here, i.e. get slashed (effective_balance / MIN_SLASHING_PENALTY_QUOTIENT) + assert total_penalties * 3 / total_balance < 1 / spec.MIN_SLASHING_PENALTY_QUOTIENT + + yield from run_process_slashings(spec, state) + + assert state.balances[0] == pre_balance - (pre_balance // spec.MIN_SLASHING_PENALTY_QUOTIENT) + + +@with_all_phases +@spec_state_test +def test_scaled_penalties(spec, state): + # skip to next epoch + state.slot = spec.SLOTS_PER_EPOCH + + # Also mock some previous slashings, so that we test to have the delta in the penalties computation. + for i in range(spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR): + state.slashed_balances[i] = spec.MAX_EFFECTIVE_BALANCE * 3 + + # Mock the very last one (which is to be used for the delta balance computation) to be different. + # To enforce the client test runner to correctly get this one from the array, not the others. + prev_penalties = state.slashed_balances[ + (spec.get_current_epoch(state) + 1) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR + ] = spec.MAX_EFFECTIVE_BALANCE * 2 + + slashed_count = len(state.validators) // 4 + + assert slashed_count > 10 + + # make the balances non-uniform. + # Otherwise it would just be a simple 3/4 balance slashing. Test the per-validator scaled penalties. + for i in range(10): + state.validators[i].effective_balance += spec.EFFECTIVE_BALANCE_INCREMENT * 4 + state.balances[i] += spec.EFFECTIVE_BALANCE_INCREMENT * 4 + + total_balance = spec.get_total_active_balance(state) + + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + + slashed_indices = list(range(slashed_count)) + + # Process up to the sub-transition, then Hi-jack and get the balances. + # We just want to test the slashings. + # But we are not interested in the other balance changes during the same epoch transition. + run_epoch_processing_to(spec, state, 'process_slashings') + pre_slash_balances = list(state.balances) + + slash_validators(spec, state, slashed_indices, [out_epoch] * slashed_count) + + yield 'pre', state + spec.process_slashings(state) + yield 'post', state + + total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] + total_penalties -= prev_penalties + + for i in slashed_indices: + v = state.validators[i] + penalty = v.effective_balance * total_penalties * 3 // total_balance + assert state.balances[i] == pre_slash_balances[i] - penalty From 7a418ed682fc3edd4d2693c4084f0e0eb2ea5130 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 26 Jun 2019 23:40:56 +0200 Subject: [PATCH 214/405] test messed up indices in attester slashings --- .../test_process_attester_slashing.py | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 86b6811a2b..226d4a5610 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -154,9 +154,73 @@ def test_custody_bit_0_and_1_intersect(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) +@always_bls +@with_all_phases +@spec_state_test +def test_att1_bad_extra_index(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + + indices = attester_slashing.attestation_1.custody_bit_0_indices + options = list(set(range(len(state.validators))) - set(indices)) + indices.append(options[len(options) // 2]) # add random index, not previously in attestation. + attester_slashing.attestation_1.custody_bit_0_indices = sorted(indices) + # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), + # see if the bad extra index is spotted, and slashing is aborted. + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@always_bls @with_all_phases @spec_state_test -def test_unsorted_att_1(spec, state): +def test_att1_bad_replaced_index(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + + indices = attester_slashing.attestation_1.custody_bit_0_indices + options = list(set(range(len(state.validators))) - set(indices)) + indices[3] = options[len(options) // 2] # replace with random index, not previously in attestation. + attester_slashing.attestation_1.custody_bit_0_indices = sorted(indices) + # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), + # see if the bad replaced index is spotted, and slashing is aborted. + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@always_bls +@with_all_phases +@spec_state_test +def test_att2_bad_extra_index(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + + indices = attester_slashing.attestation_2.custody_bit_0_indices + options = list(set(range(len(state.validators))) - set(indices)) + indices.append(options[len(options) // 2]) # add random index, not previously in attestation. + attester_slashing.attestation_2.custody_bit_0_indices = sorted(indices) + # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), + # see if the bad extra index is spotted, and slashing is aborted. + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@always_bls +@with_all_phases +@spec_state_test +def test_att2_bad_replaced_index(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + + indices = attester_slashing.attestation_2.custody_bit_0_indices + options = list(set(range(len(state.validators))) - set(indices)) + indices[3] = options[len(options) // 2] # replace with random index, not previously in attestation. + attester_slashing.attestation_2.custody_bit_0_indices = sorted(indices) + # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), + # see if the bad replaced index is spotted, and slashing is aborted. + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@with_all_phases +@spec_state_test +def test_unsorted_att_1_bit0(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) indices = attester_slashing.attestation_1.custody_bit_0_indices @@ -169,7 +233,7 @@ def test_unsorted_att_1(spec, state): @with_all_phases @spec_state_test -def test_unsorted_att_2(spec, state): +def test_unsorted_att_2_bit0(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) indices = attester_slashing.attestation_2.custody_bit_0_indices @@ -178,3 +242,7 @@ def test_unsorted_att_2(spec, state): sign_indexed_attestation(spec, state, attester_slashing.attestation_2) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +# note: unsorted indices for custody bit 0 are to be introduced in phase 1 testing. + From 10e257490fc57e3fd335ac352d62abbd7a98ce68 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 26 Jun 2019 16:33:23 -0600 Subject: [PATCH 215/405] Clarify `get_randao_mix` accessor We avoid a genesis underflow by taking the randao epoch in `generate_seed` to be `+ EPOCHS_PER_HISTORICAL_VECTOR`. This conflicts with the expected epoch bounds noted in `get_randao_mix` and this PR attempts to clarify the situation by leaving a note. --- specs/core/0_beacon-chain.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 264c6d23b8..31fb7c1db5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -737,7 +737,7 @@ def get_randao_mix(state: BeaconState, epoch: Epoch) -> Hash: """ Return the randao mix at a recent ``epoch``. - ``epoch`` expected to be between (current_epoch - EPOCHS_PER_HISTORICAL_VECTOR, current_epoch]. + ``epoch`` expected to be between (current_epoch - EPOCHS_PER_HISTORICAL_VECTOR, current_epoch], unless otherwise noted at a call site. """ return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` @@ -762,6 +762,8 @@ def generate_seed(state: BeaconState, epoch: Epoch) -> Hash: """ Generate a seed for the given ``epoch``. + + Note that avoiding the underflow on ``get_randao_mix`` here violates the epoch validity condition given in that function's comment. """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + From ff2d711d51ccccff9ad60d6ef6f54c9a379d0d84 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 00:35:01 +0200 Subject: [PATCH 216/405] test block application on same and on previous slot state --- .../eth2spec/test/sanity/test_blocks.py | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index bb6a6dc06e..8150b06f6c 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -11,7 +11,39 @@ from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit -from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.context import spec_state_test, with_all_phases, expect_assertion_error + + +@with_all_phases +@spec_state_test +def test_prev_slot_block_transition(spec, state): + # Go to clean slot + spec.process_slots(state, state.slot + 1) + # Make a block for it + block = build_empty_block(spec, state, slot=state.slot, signed=True) + # Transition to next slot, above block will not be invalid on top of new state. + spec.process_slots(state, state.slot + 1) + + yield 'pre', state + expect_assertion_error(lambda: state_transition_and_sign_block(spec, state, block)) + yield 'blocks', [block] + yield 'post', None + + +@with_all_phases +@spec_state_test +def test_same_slot_block_transition(spec, state): + # Same slot on top of pre-state, but move out of slot 0 first. + spec.process_slots(state, state.slot + 1) + + block = build_empty_block(spec, state, slot=state.slot, signed=True) + + yield 'pre', state + + state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [block] + yield 'post', state @with_all_phases From 8445d1d90c05a29bcc8158ca6c7e9c7754b2df2c Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 00:37:32 +0200 Subject: [PATCH 217/405] fix formatting for lint --- .../phase_0/block_processing/test_process_attester_slashing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 226d4a5610..7fcbd2da5a 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -245,4 +245,3 @@ def test_unsorted_att_2_bit0(spec, state): # note: unsorted indices for custody bit 0 are to be introduced in phase 1 testing. - From f7b3c87715dae19119ba6b2e6d27dcc8b54d9af8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 00:43:50 +0200 Subject: [PATCH 218/405] check invalid state root --- .../pyspec/eth2spec/test/sanity/test_blocks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 8150b06f6c..0a879dac6c 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -66,6 +66,22 @@ def test_empty_block_transition(spec, state): assert spec.get_randao_mix(state, spec.get_current_epoch(state)) != spec.ZERO_HASH +@with_all_phases +@spec_state_test +def test_invalid_state_root(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.state_root = b"\xaa" * 32 + sign_block(spec, state, block) + + expect_assertion_error( + lambda: spec.state_transition(state, block, validate_state_root=True)) + + yield 'blocks', [block] + yield 'post', None + + @with_all_phases @spec_state_test def test_skipped_slots(spec, state): From 235c3d6841d5a29895ea0c9465fd447f5338b92e Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 00:58:25 +0200 Subject: [PATCH 219/405] re-enable test_empty_epoch_transition_not_finalizing for minimal config --- .../eth2spec/test/sanity/test_blocks.py | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 0a879dac6c..e56baee8c4 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -123,26 +123,29 @@ def test_empty_epoch_transition(spec, state): assert spec.get_block_root_at_slot(state, slot) == block.parent_root -# @with_all_phases -# @spec_state_test -# def test_empty_epoch_transition_not_finalizing(spec, state): -# # copy for later balance lookups. -# pre_state = deepcopy(state) -# yield 'pre', state +@with_all_phases +@spec_state_test +def test_empty_epoch_transition_not_finalizing(spec, state): + # Don't run for non-minimal configs, it takes very long, and the effect + # of calling finalization/justifcation is just the same as with the minimal configuration. + if spec.SLOTS_PER_EPOCH > 8: + return -# block = build_empty_block_for_next_slot(spec, state) -# block.slot += spec.SLOTS_PER_EPOCH * 5 -# sign_block(spec, state, block, proposer_index=0) + # copy for later balance lookups. + pre_balances = list(state.balances) + yield 'pre', state -# state_transition_and_sign_block(spec, state, block) + spec.process_slots(state, state.slot + (spec.SLOTS_PER_EPOCH * 5)) + block = build_empty_block_for_next_slot(spec, state, signed=True) + state_transition_and_sign_block(spec, state, block) -# yield 'blocks', [block] -# yield 'post', state + yield 'blocks', [block] + yield 'post', state -# assert state.slot == block.slot -# assert state.finalized_epoch < spec.get_current_epoch(state) - 4 -# for index in range(len(state.validators)): -# assert get_balance(state, index) < get_balance(pre_state, index) + assert state.slot == block.slot + assert state.finalized_epoch < spec.get_current_epoch(state) - 4 + for index in range(len(state.validators)): + assert state.balances[index] < pre_balances[index] @with_all_phases From 23909ca72760b002c25278de9817ae26164c67d4 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 26 Jun 2019 17:06:34 -0600 Subject: [PATCH 220/405] Fix line lengths --- specs/core/0_beacon-chain.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 31fb7c1db5..6f7a7a059e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -737,7 +737,8 @@ def get_randao_mix(state: BeaconState, epoch: Epoch) -> Hash: """ Return the randao mix at a recent ``epoch``. - ``epoch`` expected to be between (current_epoch - EPOCHS_PER_HISTORICAL_VECTOR, current_epoch], unless otherwise noted at a call site. + ``epoch`` expected to be between (current_epoch - EPOCHS_PER_HISTORICAL_VECTOR, current_epoch], unless + otherwise noted at a call site. """ return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` @@ -763,7 +764,8 @@ def generate_seed(state: BeaconState, """ Generate a seed for the given ``epoch``. - Note that avoiding the underflow on ``get_randao_mix`` here violates the epoch validity condition given in that function's comment. + Note that avoiding the underflow on ``get_randao_mix`` here violates + the epoch validity condition given in that function's comment. """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + From dbb697dadd3b027979baaffaafc62d5932b895a0 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 26 Jun 2019 19:40:11 -0400 Subject: [PATCH 221/405] Small update to typing in BLS spec file [uint384] -> Tuple[uint384, uint384] --- specs/bls_signature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/bls_signature.md b/specs/bls_signature.md index 3fe1bcc0eb..b901b9345f 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -71,7 +71,7 @@ We require: G2_cofactor = 305502333931268344200999753193121504214466019254188142667664032982267604182971884026507427359259977847832272839041616661285803823378372096355777062779109 q = 4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787 -def hash_to_G2(message_hash: Bytes32, domain: uint64) -> [uint384]: +def hash_to_G2(message_hash: Bytes32, domain: uint64) -> Tuple[uint384, uint384]: # Initial candidate x coordinate x_re = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x01'), 'big') x_im = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x02'), 'big') From aecaed7a662ffc7ad73b098b1834840b725fe76d Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 01:47:15 +0200 Subject: [PATCH 222/405] move genesis tests, structure properly, add yield keys for future test-vec generator --- .../pyspec/eth2spec/test/genesis/__init__.py | 0 .../eth2spec/test/genesis/test_genesis.py | 37 ++++++++ .../test/genesis/test_genesis_trigger.py | 54 ++++++++++++ .../eth2spec/test/sanity/test_genesis.py | 87 ------------------- 4 files changed, 91 insertions(+), 87 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/genesis/__init__.py create mode 100644 test_libs/pyspec/eth2spec/test/genesis/test_genesis.py create mode 100644 test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py delete mode 100644 test_libs/pyspec/eth2spec/test/sanity/test_genesis.py diff --git a/test_libs/pyspec/eth2spec/test/genesis/__init__.py b/test_libs/pyspec/eth2spec/test/genesis/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py b/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py new file mode 100644 index 0000000000..a89ec8793f --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py @@ -0,0 +1,37 @@ +from eth2spec.test.context import with_phases, spectest_with_bls_switch +from eth2spec.test.helpers.deposits import ( + prepare_genesis_deposits, +) + + +@with_phases(['phase0']) +@spectest_with_bls_switch +def test_genesis(spec): + deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT + genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + genesis_time = 1546300800 + block_hash = b'\x12' * 32 + + yield "deposits", genesis_deposits + yield "time", genesis_time + + genesis_eth1_data = spec.Eth1Data( + deposit_root=deposit_root, + deposit_count=deposit_count, + block_hash=block_hash, + ) + + yield "eth1_data", genesis_eth1_data + genesis_state = spec.get_genesis_beacon_state( + genesis_deposits, + genesis_time, + genesis_eth1_data, + ) + + assert genesis_state.genesis_time == genesis_time + assert len(genesis_state.validators) == deposit_count + assert genesis_state.eth1_data.deposit_root == deposit_root + assert genesis_state.eth1_data.deposit_count == deposit_count + assert genesis_state.eth1_data.block_hash == block_hash + + yield "state", genesis_state diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py b/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py new file mode 100644 index 0000000000..873638dc95 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py @@ -0,0 +1,54 @@ +from eth2spec.test.context import with_phases, spectest_with_bls_switch +from eth2spec.test.helpers.deposits import ( + prepare_genesis_deposits, +) + + +@with_phases(['phase0']) +@spectest_with_bls_switch +def test_is_genesis_trigger_false(spec): + deposit_count = 2 + genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + genesis_time = 1546300800 + + yield "deposits", genesis_deposits + yield "time", genesis_time + yield "deposit_root", deposit_root + + is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time) + assert is_triggered is False + + yield "is_triggered", is_triggered + + +@with_phases(['phase0']) +@spectest_with_bls_switch +def test_is_genesis_trigger_true(spec): + deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT + genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + genesis_time = 1546300800 + + yield "deposits", genesis_deposits + yield "time", genesis_time + yield "deposit_root", deposit_root + + is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time) + assert is_triggered is True + + yield "is_triggered", is_triggered + + +@with_phases(['phase0']) +@spectest_with_bls_switch +def test_is_genesis_trigger_not_enough_balance(spec): + deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT + genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE - 1) + genesis_time = 1546300800 + yield "deposits", genesis_deposits + yield "time", genesis_time + yield "deposit_root", deposit_root + + is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time) + assert is_triggered is False + + yield "is_triggered", is_triggered diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py b/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py deleted file mode 100644 index 780d039ebe..0000000000 --- a/test_libs/pyspec/eth2spec/test/sanity/test_genesis.py +++ /dev/null @@ -1,87 +0,0 @@ -from eth2spec.test.context import with_phases, spectest_with_bls_switch -from eth2spec.test.helpers.deposits import ( - prepare_genesis_deposits, -) - - -@with_phases(['phase0']) -@spectest_with_bls_switch -def test_is_genesis_trigger_false(spec): - deposit_count = 2 - genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - genesis_time = 1546300800 - - yield genesis_deposits - yield genesis_time - yield deposit_root - - is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time, deposit_root) - assert is_triggered is False - - yield is_triggered - - -@with_phases(['phase0']) -@spectest_with_bls_switch -def test_is_genesis_trigger_true(spec): - deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT - genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - genesis_time = 1546300800 - - yield genesis_deposits - yield genesis_time - yield deposit_root - - is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time, deposit_root) - assert is_triggered is True - - yield is_triggered - - -@with_phases(['phase0']) -@spectest_with_bls_switch -def test_is_genesis_trigger_not_enough_balance(spec): - deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT - genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE - 1) - genesis_time = 1546300800 - yield genesis_deposits - yield genesis_time - yield deposit_root - - is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time, deposit_root) - assert is_triggered is False - - yield is_triggered - - -@with_phases(['phase0']) -@spectest_with_bls_switch -def test_genesis(spec): - deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT - genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - genesis_time = 1546300800 - block_hash = b'\x12' * 32 - - yield genesis_deposits - yield genesis_time - - genesis_eth1_data = spec.Eth1Data( - deposit_root=deposit_root, - deposit_count=deposit_count, - block_hash=block_hash, - ) - - yield genesis_eth1_data - genesis_state = spec.get_genesis_beacon_state( - genesis_deposits, - genesis_time, - genesis_eth1_data, - ) - - assert genesis_state.genesis_time == genesis_time - assert len(genesis_state.validators) == deposit_count - assert genesis_state.eth1_data.deposit_root == deposit_root - assert genesis_state.eth1_data.deposit_count == deposit_count - assert genesis_state.eth1_data.block_hash == block_hash - - yield genesis_state From a0c2f5c6b5578d3ee09b155fb487ab739e22195a Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 01:57:30 +0200 Subject: [PATCH 223/405] fix genesis testing code, add missing constant temporarily, fix py Generator/map/list problems --- specs/core/0_beacon-chain.md | 6 ++++-- .../pyspec/eth2spec/test/genesis/test_genesis_trigger.py | 3 ++- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b0490cba54..1ff1d58b39 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1137,14 +1137,16 @@ When `is_genesis_trigger(deposits, time) is True` for the first time let: ```python def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], time: uint64) -> bool: + SECONDS_PER_DAY = 86400 # Do not deploy too early if time - time % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY < MIN_GENESIS_TIME: return False # Initialize deposit root state = BeaconState() - state.eth1_data.deposit_root = hash_tree_root(map(deposits, lambda deposit: deposit.data)) - + state.eth1_data.deposit_root = hash_tree_root( + Vector[DepositData, len(deposits)](list(map(lambda deposit: deposit.data, deposits))) + ) # Process deposits for deposit in deposits: process_deposit(state, deposit) diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py b/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py index 873638dc95..9425a1750d 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py @@ -26,7 +26,8 @@ def test_is_genesis_trigger_false(spec): def test_is_genesis_trigger_true(spec): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - genesis_time = 1546300800 + SECONDS_PER_DAY = 86400 + genesis_time = 1578009600 - 2 * SECONDS_PER_DAY yield "deposits", genesis_deposits yield "time", genesis_time diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index eea019e8b9..bda4f1699c 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -82,7 +82,7 @@ def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) root = get_merkle_root((tuple(deposit_data_leaves))) - genesis_deposits = ( + genesis_deposits = list( spec.Deposit(proof=list(get_merkle_proof(tree, item_index=index)), data=deposit_data) for index, deposit_data in enumerate(genesis_deposit_data_list) ) From e49519a53b5c1492fc39978316edafeef88769d3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 02:50:49 +0200 Subject: [PATCH 224/405] wrong end epoch test --- .../test_process_attestation.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 59e99ac0cf..daac7efed2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -85,6 +85,29 @@ def test_success_since_max_epochs_per_crosslink(spec, state): yield from run_attestation_processing(spec, state, attestation) +@with_all_phases +@spec_state_test +def test_wrong_end_epoch_with_max_epochs_per_crosslink(spec, state): + for _ in range(spec.MAX_EPOCHS_PER_CROSSLINK + 2): + next_epoch(spec, state) + apply_empty_block(spec, state) + + attestation = get_valid_attestation(spec, state) + data = attestation.data + # test logic sanity check: make sure the attestation only includes MAX_EPOCHS_PER_CROSSLINK epochs + assert data.crosslink.end_epoch - data.crosslink.start_epoch == spec.MAX_EPOCHS_PER_CROSSLINK + # Now change it to be different + data.crosslink.end_epoch += 1 + + sign_attestation(spec, state, attestation) + + for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(spec, state) + apply_empty_block(spec, state) + + yield from run_attestation_processing(spec, state, attestation, False) + + @with_all_phases @always_bls @spec_state_test From bcfe383e2555ee14261c995044ee3cf503432297 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 27 Jun 2019 08:44:44 +0100 Subject: [PATCH 225/405] WIP --- configs/constant_presets/mainnet.yaml | 2 +- configs/constant_presets/minimal.yaml | 2 +- specs/core/0_beacon-chain.md | 37 +++++++++++-------- specs/light_client/sync_protocol.md | 4 +- .../pyspec/eth2spec/test/helpers/genesis.py | 4 +- 5 files changed, 27 insertions(+), 22 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 9f7ca950f1..2aa45cde39 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -10,7 +10,7 @@ SHARD_COUNT: 1024 # 2**7 (= 128) TARGET_COMMITTEE_SIZE: 128 # 2**12 (= 4,096) -MAX_INDICES_PER_ATTESTATION: 4096 +MAX_VALIDATORS_PER_COMMITTEE: 4096 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**16 (= 65,536) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 3e3f7ccb48..417f11c942 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -9,7 +9,7 @@ SHARD_COUNT: 8 # [customized] unsecure, but fast TARGET_COMMITTEE_SIZE: 4 # 2**12 (= 4,096) -MAX_INDICES_PER_ATTESTATION: 4096 +MAX_VALIDATORS_PER_COMMITTEE: 4096 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**16 (= 65,536) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6cf6db0cc3..a7d47e108c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -30,7 +30,7 @@ - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - [`IndexedAttestation`](#indexedattestation) - [`PendingAttestation`](#pendingattestation) - - [`CompactCommitteees`](#CompactCommitteees) + - [`CompactCommittees`](#CompactCommittees) - [`Eth1Data`](#eth1data) - [`HistoricalBatch`](#historicalbatch) - [`DepositData`](#depositdata) @@ -69,7 +69,7 @@ - [`get_block_root_at_slot`](#get_block_root_at_slot) - [`get_block_root`](#get_block_root) - [`get_randao_mix`](#get_randao_mix) - - [`get_compact_committee_root`](#get_compact_committee_root) + - [`get_compact_committees_root`](#get_compact_committees_root) - [`generate_seed`](#generate_seed) - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - [`verify_merkle_branch`](#verify_merkle_branch) @@ -361,14 +361,19 @@ class PendingAttestation(Container): proposer_index: ValidatorIndex ``` -#### `CompactCommitteees` +#### `CompactCommittee` ```python -class CompactCommitteees(Container): - data: Vector[Container( - pubkeys: List[Bytes48, MAX_VALIDATORS_PER_COMMITTEE] - compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] - ), SHARD_COUNT] +class CompactCommittee(Container): + pubkeys: List[Bytes48, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] +``` + +#### `CompactCommittees` + +```python +class CompactCommittees(Container): + data: Vector[CompactCommittee, SHARD_COUNT] ``` #### `Eth1Data` @@ -522,7 +527,7 @@ class BeaconState(Container): # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] - compact_committee_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Committee digests for light clients + compact_committees_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Committee digests for light clients # Slashings slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of slashed effective balances # Attestations @@ -753,14 +758,14 @@ def get_randao_mix(state: BeaconState, return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` -### `get_compact_committee_root` +### `get_compact_committees_root` ```python -def get_compact_committee_root(state: BeaconState, epoch: Epoch) -> Hash: +def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: """ Return the compact committee root for the current epoch. """ - committee_data = CompactCommitteees().data + committee_data = CompactCommittees().data for committee_number in range(get_epoch_committee_count(state, epoch)): shard = (get_epoch_start_shard(state, epoch) + committee_number) % SHARD_COUNT for index in get_crosslink_committee(state, epoch, shard): @@ -782,7 +787,7 @@ def generate_seed(state: BeaconState, """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + - get_compact_committee_root(state, epoch) + + get_compact_committees_root(state, epoch) + int_to_bytes(epoch, length=32) ) ``` @@ -1190,9 +1195,9 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate compact_committee_roots + # Populate compact_committees_roots for index in range(EPOCHS_PER_HISTORICAL_VECTOR): - state.compact_committee_roots[index] = get_compact_committee_root(state, GENESIS_EPOCH) + state.compact_committees_roots[index] = get_compact_committees_root(state, GENESIS_EPOCH) return state ``` @@ -1546,7 +1551,7 @@ def process_final_updates(state: BeaconState) -> None: state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT) # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR - state.compact_committee_roots[index_root_position] = get_compact_committee_root(state, next_epoch + ACTIVATION_EXIT_DELAY) + state.compact_committees_roots[index_root_position] = get_compact_committees_root(state, next_epoch + ACTIVATION_EXIT_DELAY) # Set total slashed balances state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 045bf56086..a29ed05c8f 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -31,7 +31,7 @@ We define an "expansion" of an object as an object where a field in an object th We define two expansions: -* `ExtendedBeaconState`, which is identical to a `BeaconState` except `active_index_roots: List[Bytes32]` is replaced by `active_indices: List[List[ValidatorIndex]]`, where `BeaconState.active_index_roots[i] = hash_tree_root(ExtendedBeaconState.active_indices[i])`. +* `ExtendedBeaconState`, which is identical to a `BeaconState` except `compact_committees_roots: List[Bytes32]` is replaced by `active_indices: List[List[ValidatorIndex]]`, where `BeaconState.compact_committees_roots[i] = hash_tree_root(ExtendedBeaconState.active_indices[i])`. * `ExtendedBeaconBlock`, which is identical to a `BeaconBlock` except `state_root` is replaced with the corresponding `state: ExtendedBeaconState`. ### `get_active_validator_indices` @@ -40,7 +40,7 @@ Note that there is now a new way to compute `get_active_validator_indices`: ```python def get_active_validator_indices(state: ExtendedBeaconState, epoch: Epoch) -> List[ValidatorIndex]: - return state.active_indices[epoch % ACTIVE_INDEX_ROOTS_LENGTH] + return state.active_indices[epoch % compact_committees_rootS_LENGTH] ``` Note that it takes `state` instead of `state.validators` as an argument. This does not affect its use in `get_shuffled_committee`, because `get_shuffled_committee` has access to the full `state` as one of its arguments. diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index ce0be19bbd..c793254c8d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -41,9 +41,9 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_LIMIT]( + genesis_compact_committees_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_LIMIT]( spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): - state.active_index_roots[index] = genesis_active_index_root + state.compact_committees_roots[index] = genesis_compact_committees_root return state From 02f6ba36f048d4b79b556df69eed612f660d8e42 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 09:51:06 +0100 Subject: [PATCH 226/405] Add Bitvector and Bitlist Bool, Bit -> boolean, bit Fix simple-serialize.md --- scripts/build_spec.py | 6 +- specs/core/0_beacon-chain.md | 4 +- specs/simple-serialize.md | 87 ++++++++++++++----- test_libs/pyspec/eth2spec/debug/decode.py | 4 +- test_libs/pyspec/eth2spec/debug/encode.py | 4 +- .../pyspec/eth2spec/debug/random_value.py | 8 +- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 4 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 19 +++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 22 ++++- .../eth2spec/utils/ssz/test_ssz_impl.py | 10 +-- .../eth2spec/utils/ssz/test_ssz_typing.py | 18 ++-- 11 files changed, 127 insertions(+), 59 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 99c5cd69de..0b1512111e 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -52,7 +52,7 @@ is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -174,7 +174,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ - 'Bit', 'Bool', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' + 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 56a6fd06a4..7b9aeee4b2 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -297,7 +297,7 @@ class Validator(Container): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers effective_balance: Gwei # Balance at stake - slashed: Bool + slashed: boolean # Status epochs activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch @@ -337,7 +337,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): data: AttestationData - custody_bit: Bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data + custody_bit: bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data ``` #### `IndexedAttestation` diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 3318fe45b5..53c5649ed8 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -15,9 +15,9 @@ - [Default values](#default-values) - [Illegal types](#illegal-types) - [Serialization](#serialization) - - [`"uintN"`](#uintn) - - [`"bool"`](#bool) - - [`"null`](#null) + - [`uintN`](#uintn) + - [`boolean`](#boolean) + - [`null`](#null) - [Vectors, containers, lists, unions](#vectors-containers-lists-unions) - [Deserialization](#deserialization) - [Merkleization](#merkleization) @@ -37,36 +37,45 @@ ## Typing ### Basic types -* `"uintN"`: `N`-bit unsigned integer (where `N in [8, 16, 32, 64, 128, 256]`) -* `"bool"`: `True` or `False` +* `uintN`: `N`-bit unsigned integer (where `N in [8, 16, 32, 64, 128, 256]`) +* `boolean`: `True` or `False` ### Composite types * **container**: ordered heterogeneous collection of values - * key-pair curly bracket notation `{}`, e.g. `{"foo": "uint64", "bar": "bool"}` + * python dataclass notation with key-type pairs, e.g. +```python +class ContainerExample(Container): + foo: uint64 + bar: boolean +``` * **vector**: ordered fixed-length homogeneous collection of values - * angle bracket notation `[type, N]`, e.g. `["uint64", N]` -* **list**: ordered variable-length homogeneous collection of values - * angle bracket notation `[type]`, e.g. `["uint64"]` + * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` +* **list**: ordered variable-length homogeneous collection of values, with maximum length `N` + * notation `List[type, N]`, e.g. `List[uint64, N]` * **union**: union type containing one of the given subtypes - * round bracket notation `(type_1, type_2, ...)`, e.g. `("null", "uint64")` + * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` +* **Bitvector**: a fixed-length list of `boolean` values + * notation `Bitvector[N]` +* **Bitlist**: a variable-length list of `boolean` values with maximum length `N` + * notation `Bitlist[N]` ### Variable-size and fixed-size -We recursively define "variable-size" types to be lists and unions and all types that contain a variable-size type. All other types are said to be "fixed-size". +We recursively define "variable-size" types to be lists, unions, `Bitlist` and all types that contain a variable-size type. All other types are said to be "fixed-size". ### Aliases For convenience we alias: -* `"byte"` to `"uint8"` (this is a basic type) -* `"bytes"` to `["byte"]` (this is *not* a basic type) -* `"bytesN"` to `["byte", N]` (this is *not* a basic type) -* `"null"`: `{}`, i.e. the empty container +* `bit` to `boolean` +* `byte` to `uint8` (this is a basic type) +* `BytesN` to `Vector[byte, N]` (this is *not* a basic type) +* `null`: `{}`, i.e. the empty container ### Default values -The default value of a type upon initialization is recursively defined using `0` for `"uintN"`, `False` for `"bool"`, and `[]` for lists. Unions default to the first type in the union (with type index zero), which is `"null"` if present in the union. +The default value of a type upon initialization is recursively defined using `0` for `uintN`, `False` for `boolean` and the elements of `Bitvector`, and `[]` for lists and `Bitlist`. Unions default to the first type in the union (with type index zero), which is `null` if present in the union. #### `is_empty` @@ -74,34 +83,50 @@ An SSZ object is called empty (and thus, `is_empty(object)` returns true) if it ### Illegal types -Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `"null"` type is only legal as the first type in a union subtype (i.e. with type index zero). +Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `null` type is only legal as the first type in a union subtype (i.e. with type index zero). ## Serialization -We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. +We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `bytes`. *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signing_root`, `is_variable_size`, etc.) objects implicitly carry their type. -### `"uintN"` +### `uintN` ```python assert N in [8, 16, 32, 64, 128, 256] return value.to_bytes(N // 8, "little") ``` -### `"bool"` +### `boolean` ```python assert value in (True, False) return b"\x01" if value is True else b"\x00" ``` -### `"null"` +### `null` ```python return b"" ``` +### `Bitvector[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return as_integer.to_bytes((N + 7) // 8, "little") +``` + +### `Bitlist[N]` + +Note that from the offset coding, the length (in bytes) of the bitlist is known. An additional leading `1` bit is added so that the length in bits will also be known. + +```python +as_integer = (1 << len(value)) + sum([value[i] << i for i in range(len(value))]) +return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") +``` + ### Vectors, containers, lists, unions ```python @@ -142,17 +167,33 @@ We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. * `merkleize`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. Note that `merkleize` on a single chunk is simply that chunk, i.e. the identity when the number of chunks is one. +* `pad`: given a list `l` and a length `N`, adds `N-len(l)` empty objects to the end of the list (the type of the empty object is implicit in the list type) * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(value)), len(value))` if `value` is a list of basic objects +* `mix_in_length(merkleize(pack(pad(value, N))), len(value))` if `value` is a list of basic objects * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in value]), len(value))` if `value` is a list of composite objects +* `mix_in_length(merkleize([hash_tree_root(element) for element in pad(value, N)]), len(value))` if `value` is a list of composite objects * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type +### Merkleization of `Bitvector[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return merkleize(as_integer.to_bytes((N + 7) // 8, "little")) +``` + +### `Bitlist[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return mix_in_length(merkleize(as_integer.to_bytes((N + 7) // 8, "little")), len(value)) +``` + + ## Self-signed containers Let `value` be a self-signed container object. The convention is that the signature (e.g. a `"bytes96"` BLS12-381 signature) be the last field of `value`. Further, the signed message for `value` is `signing_root(value) = hash_tree_root(truncate_last(value))` where `truncate_last` truncates the last element of `value`. diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index c0b53b0efb..c0b977ab34 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,13 +1,13 @@ from typing import Any from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, uint, Container, Bytes, List, Bool, + SSZType, SSZValue, uint, Container, Bytes, List, boolean, Vector, BytesN ) def decode(data: Any, typ: SSZType) -> SSZValue: - if issubclass(typ, (uint, Bool)): + if issubclass(typ, (uint, boolean)): return typ(data) elif issubclass(typ, (List, Vector)): return typ(decode(element, typ.elem_type) for element in data) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 02814e441a..670f580b2b 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZValue, uint, Container, Bool + SSZValue, uint, Container, boolean ) @@ -10,7 +10,7 @@ def encode(value: SSZValue, include_hash_tree_roots=False): if value.type().byte_len > 8: return str(int(value)) return int(value) - elif isinstance(value, Bool): + elif isinstance(value, boolean): return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index c6efb722b9..cdcba343a9 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,7 +2,7 @@ from enum import Enum from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bool, + SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, boolean, Vector, BytesN ) @@ -118,7 +118,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(rng.choice((True, False))) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -128,7 +128,7 @@ def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: def get_min_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(False) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -138,7 +138,7 @@ def get_min_basic_value(typ: BasicType) -> BasicValue: def get_max_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(True) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index e533ca5c21..1309562352 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -19,7 +19,7 @@ def translate_typ(typ) -> ssz.BaseSedes: return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.List): return ssz.List(translate_typ(typ.elem_type)) - elif issubclass(typ, spec_ssz.Bool): + elif issubclass(typ, spec_ssz.boolean): return ssz.boolean elif issubclass(typ, spec_ssz.uint): if typ.byte_len == 1: @@ -64,7 +64,7 @@ def translate_value(value, typ): raise TypeError("invalid uint size") elif issubclass(typ, spec_ssz.List): return [translate_value(elem, typ.elem_type) for elem in value] - elif issubclass(typ, spec_ssz.Bool): + elif issubclass(typ, spec_ssz.boolean): return value elif issubclass(typ, spec_ssz.Vector): return typ(*(translate_value(elem, typ.elem_type) for elem in value)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index b9c7b6d384..a7f6f9da1f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,8 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bool, Container, List, Bytes, uint, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + Bitlist, Bitvector, uint, ) # SSZ Serialization @@ -13,7 +14,7 @@ def serialize_basic(value: SSZValue): if isinstance(value, uint): return value.to_bytes(value.type().byte_len, 'little') - elif isinstance(value, Bool): + elif isinstance(value, boolean): if value: return b'\x01' else: @@ -39,6 +40,12 @@ def is_empty(obj: SSZValue): def serialize(obj: SSZValue): if isinstance(obj, BasicValue): return serialize_basic(obj) + elif isinstance(obj, Bitvector): + as_integer = sum([obj[i] << i for i in range(len(obj))]) + return as_integer.to_bytes((len(obj) + 7) // 8, "little") + elif isinstance(obj, Bitlist): + as_integer = (1 << len(obj)) + sum([obj[i] << i for i in range(len(obj))]) + return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") elif isinstance(obj, Series): return encode_series(obj) else: @@ -85,6 +92,12 @@ def encode_series(values: Series): def pack(values: Series): if isinstance(values, bytes): # Bytes and BytesN are already packed return values + elif isinstance(values, Bitvector): + as_integer = sum([values[i] << i for i in range(len(values))]) + return as_integer.to_bytes((values.length + 7) // 8, "little") + elif isinstance(values, Bitlist): + as_integer = (1 << len(values)) + sum([values[i] << i for i in range(len(values))]) + return as_integer.to_bytes((values.length + 7) // 8, "little") return b''.join([serialize_basic(value) for value in values]) @@ -134,7 +147,7 @@ def hash_tree_root(obj: SSZValue): else: raise Exception(f"Type not supported: {type(obj)}") - if isinstance(obj, (List, Bytes)): + if isinstance(obj, (List, Bytes, Bitlist)): return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(obj.type())), len(obj)) else: return merkleize_chunks(leaves) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 58e66ca685..ea07359b2f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -31,7 +31,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): pass -class Bool(BasicValue): # can't subclass bool. +class boolean(BasicValue): # can't subclass bool. byte_len = 1 def __new__(cls, value: int): # int value, but can be any subclass of int (bool, Bit, Bool, etc...) @@ -48,7 +48,7 @@ def __bool__(self): # Alias for Bool -class Bit(Bool): +class bit(boolean): pass @@ -233,7 +233,7 @@ def __str__(self): return f"{self.__name__}~{self.__class__.__name__}" def __repr__(self): - return self, self.__class__ + return f"{self.__name__}~{self.__class__.__name__}" def attr_from_params(self, p): # single key params are valid too. Wrap them in a tuple. @@ -280,11 +280,12 @@ class ElementsType(ParamsMeta): elem_type: SSZType length: int +class BitElementsType(ElementsType): + elem_type = boolean class Elements(ParamsBase, metaclass=ElementsType): pass - class BaseList(list, Elements): def __init__(self, *args): @@ -310,6 +311,10 @@ def __str__(self): cls = self.__class__ return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" + def __repr__(self): + cls = self.__class__ + return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" + def __getitem__(self, k) -> SSZValue: if isinstance(k, int): # check if we are just doing a lookup, and not slicing if k < 0: @@ -337,6 +342,15 @@ def last(self): # be explict about getting the last item, for the non-python readers, and negative-index safety return self[len(self) - 1] +class BaseBitfield(BaseList, metaclass=BitElementsType): + elem_type = bool + +class Bitlist(BaseBitfield): + pass + +class Bitvector(BaseBitfield): + pass + class List(BaseList): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 82fb4ec684..33badcf4a4 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,7 +1,7 @@ from typing import Iterable from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, BytesN, + bit, boolean, Container, List, Vector, Bytes, BytesN, uint8, uint16, uint32, uint64, uint256, byte ) from ..hash_function import hash as bytes_hash @@ -74,10 +74,10 @@ def merge(a: str, branch: Iterable[str]) -> str: test_data = [ - ("bit F", Bit(False), "00", chunk("00")), - ("bit T", Bit(True), "01", chunk("01")), - ("bool F", Bool(False), "00", chunk("00")), - ("bool T", Bool(True), "01", chunk("01")), + ("bit F", bit(False), "00", chunk("00")), + ("bit T", bit(True), "01", chunk("01")), + ("boolean F", boolean(False), "00", chunk("00")), + ("boolean T", boolean(True), "01", chunk("01")), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 2af7423604..f746a29c9b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,6 +1,6 @@ from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, - Elements, Bit, Bool, Container, List, Vector, Bytes, BytesN, + Elements, bit, boolean, Container, List, Vector, Bytes, BytesN, byte, uint, uint8, uint16, uint32, uint64, uint128, uint256, Bytes32, Bytes48 ) @@ -22,8 +22,8 @@ def test_subclasses(): assert issubclass(u, SSZValue) assert isinstance(u, SSZType) assert isinstance(u, BasicType) - assert issubclass(Bool, BasicValue) - assert isinstance(Bool, BasicType) + assert issubclass(boolean, BasicValue) + assert isinstance(boolean, BasicType) for c in [Container, List, Vector, Bytes, BytesN]: assert issubclass(c, Series) @@ -45,16 +45,16 @@ def test_basic_instances(): assert isinstance(v, BasicValue) assert isinstance(v, SSZValue) - assert isinstance(Bool(True), BasicValue) - assert isinstance(Bool(False), BasicValue) - assert isinstance(Bit(True), Bool) - assert isinstance(Bit(False), Bool) + assert isinstance(boolean(True), BasicValue) + assert isinstance(boolean(False), BasicValue) + assert isinstance(bit(True), boolean) + assert isinstance(bit(False), boolean) def test_basic_value_bounds(): max = { - Bool: 2 ** 1, - Bit: 2 ** 1, + boolean: 2 ** 1, + bit: 2 ** 1, uint8: 2 ** (8 * 1), byte: 2 ** (8 * 1), uint16: 2 ** (8 * 2), From 23c743570e5b42119f7e7fdb9ffcf8584e98c1d1 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 10:26:45 +0100 Subject: [PATCH 227/405] Add some tests and fix pack --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index a7f6f9da1f..d6689892db 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -96,7 +96,7 @@ def pack(values: Series): as_integer = sum([values[i] << i for i in range(len(values))]) return as_integer.to_bytes((values.length + 7) // 8, "little") elif isinstance(values, Bitlist): - as_integer = (1 << len(values)) + sum([values[i] << i for i in range(len(values))]) + as_integer = sum([values[i] << i for i in range(len(values))]) return as_integer.to_bytes((values.length + 7) // 8, "little") return b''.join([serialize_basic(value) for value in values]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 33badcf4a4..1522ce2009 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -2,6 +2,7 @@ from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, BytesN, + Bitlist, Bitvector, uint8, uint16, uint32, uint64, uint256, byte ) from ..hash_function import hash as bytes_hash @@ -78,6 +79,10 @@ def merge(a: str, branch: Iterable[str]) -> str: ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), + ("bitvector TFTFFFTTFT", Bitvector[10](1,0,1,0,0,0,1,1,0,1), "c502", chunk("c502")), + ("bitlist TFTFFFTTFT", Bitlist[16](1,0,1,0,0,0,1,1,0,1), "c506", h(chunk("c502"), chunk("0A"))), + ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c2", chunk("c5c2")), + ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c201", h(chunk("c5c2"), chunk("10"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From 494984f7d3129d8d3c4798d285824b59dc7f68b2 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 10:42:14 +0100 Subject: [PATCH 228/405] Fix linting errors --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 ++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 ++++++ test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 10 ++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index d6689892db..7298fb3ca5 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -26,7 +26,7 @@ def serialize_basic(value: SSZValue): def deserialize_basic(value, typ: BasicType): if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif issubclass(typ, Bool): + elif issubclass(typ, boolean): assert value in (b'\x00', b'\x01') return typ(value == b'\x01') else: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ea07359b2f..047abd4fe0 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -280,12 +280,15 @@ class ElementsType(ParamsMeta): elem_type: SSZType length: int + class BitElementsType(ElementsType): elem_type = boolean + class Elements(ParamsBase, metaclass=ElementsType): pass + class BaseList(list, Elements): def __init__(self, *args): @@ -342,12 +345,15 @@ def last(self): # be explict about getting the last item, for the non-python readers, and negative-index safety return self[len(self) - 1] + class BaseBitfield(BaseList, metaclass=BitElementsType): elem_type = bool + class Bitlist(BaseBitfield): pass + class Bitvector(BaseBitfield): pass diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 1522ce2009..63f0c835de 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -79,10 +79,12 @@ def merge(a: str, branch: Iterable[str]) -> str: ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), - ("bitvector TFTFFFTTFT", Bitvector[10](1,0,1,0,0,0,1,1,0,1), "c502", chunk("c502")), - ("bitlist TFTFFFTTFT", Bitlist[16](1,0,1,0,0,0,1,1,0,1), "c506", h(chunk("c502"), chunk("0A"))), - ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c2", chunk("c5c2")), - ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c201", h(chunk("c5c2"), chunk("10"))), + ("bitvector TFTFFFTTFT", Bitvector[10](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c502", chunk("c502")), + ("bitlist TFTFFFTTFT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c506", h(chunk("c502"), chunk("0A"))), + ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), + "c5c2", chunk("c5c2")), + ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), + "c5c201", h(chunk("c5c2"), chunk("10"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From ee712ecdde94bf00a0df44e412bd4ede2637e967 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 27 Jun 2019 11:21:45 +0100 Subject: [PATCH 229/405] Make HW happy :) --- specs/core/0_beacon-chain.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1ff1d58b39..ea99bb4730 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1142,13 +1142,12 @@ def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], if time - time % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY < MIN_GENESIS_TIME: return False - # Initialize deposit root - state = BeaconState() - state.eth1_data.deposit_root = hash_tree_root( - Vector[DepositData, len(deposits)](list(map(lambda deposit: deposit.data, deposits))) - ) # Process deposits - for deposit in deposits: + state = BeaconState() + for i, deposit in enumerate(deposits): + state.eth1_data.deposit_root = hash_tree_root( + Vector[DepositData, len(deposits)](list(map(lambda deposit: deposit.data, deposits[:i]))) + ) process_deposit(state, deposit) # Count active validators at genesis @@ -1183,7 +1182,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots + # Populate active_index_roots genesis_active_index_root = hash_tree_root( List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) ) From d641e941519efc8cacd5e2ddabcaaf1722aa0797 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 27 Jun 2019 11:30:23 +0100 Subject: [PATCH 230/405] Cleanups --- specs/simple-serialize.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 53c5649ed8..97b1d560c6 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -43,7 +43,7 @@ ### Composite types * **container**: ordered heterogeneous collection of values - * python dataclass notation with key-type pairs, e.g. + * python dataclass notation with key-type pairs, e.g. ```python class ContainerExample(Container): foo: uint64 @@ -53,12 +53,12 @@ class ContainerExample(Container): * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` * **list**: ordered variable-length homogeneous collection of values, with maximum length `N` * notation `List[type, N]`, e.g. `List[uint64, N]` -* **union**: union type containing one of the given subtypes - * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` -* **Bitvector**: a fixed-length list of `boolean` values +* **bitvector**: ordered fixed-length collection of `boolean` values * notation `Bitvector[N]` -* **Bitlist**: a variable-length list of `boolean` values with maximum length `N` +* **bitlist**: ordered variable-length collection of `boolean` values, with maximum length `N` * notation `Bitlist[N]` +* **union**: union type containing one of the given subtypes + * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` ### Variable-size and fixed-size @@ -193,7 +193,6 @@ as_integer = sum([value[i] << i for i in range(len(value))]) return mix_in_length(merkleize(as_integer.to_bytes((N + 7) // 8, "little")), len(value)) ``` - ## Self-signed containers Let `value` be a self-signed container object. The convention is that the signature (e.g. a `"bytes96"` BLS12-381 signature) be the last field of `value`. Further, the signed message for `value` is `signing_root(value) = hash_tree_root(truncate_last(value))` where `truncate_last` truncates the last element of `value`. From 67c50cb197824a015ff9946fb6fd5c684e9e9639 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 11:45:35 +0100 Subject: [PATCH 231/405] Changed attestation and custody bitfields --- scripts/build_spec.py | 6 +-- specs/core/0_beacon-chain.md | 41 +++---------------- .../pyspec/eth2spec/debug/random_value.py | 6 +-- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 11 +++++ .../eth2spec/test/helpers/attestations.py | 9 ++-- .../pyspec/eth2spec/test/helpers/bitfields.py | 11 ----- .../test_process_attestation.py | 8 +++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 8 +++- 8 files changed, 38 insertions(+), 62 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/helpers/bitfields.py diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0b1512111e..0a41fe5c01 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -26,7 +26,7 @@ ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, + Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -53,7 +53,7 @@ ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, + Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -175,7 +175,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' - 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', + 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc ] diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7b9aeee4b2..13ab4cdae6 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -80,8 +80,6 @@ - [`bytes_to_int`](#bytes_to_int) - [`get_total_balance`](#get_total_balance) - [`get_domain`](#get_domain) - - [`get_bitfield_bit`](#get_bitfield_bit) - - [`verify_bitfield`](#verify_bitfield) - [`convert_to_indexed`](#convert_to_indexed) - [`validate_indexed_attestation`](#validate_indexed_attestation) - [`is_slashable_attestation_data`](#is_slashable_attestation_data) @@ -354,7 +352,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -421,9 +419,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData - custody_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + custody_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] signature: BLSSignature ``` @@ -865,13 +863,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S ```python def get_attesting_indices(state: BeaconState, attestation_data: AttestationData, - bitfield: bytes) -> Sequence[ValidatorIndex]: + bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: """ Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. """ committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) - assert verify_bitfield(bitfield, len(committee)) - return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) + return sorted([index for i, index in enumerate(committee) if bitfield[i]]) ``` ### `int_to_bytes` @@ -912,34 +909,6 @@ def get_domain(state: BeaconState, return bls_domain(domain_type, fork_version) ``` -### `get_bitfield_bit` - -```python -def get_bitfield_bit(bitfield: bytes, i: int) -> int: - """ - Extract the bit in ``bitfield`` at position ``i``. - """ - return (bitfield[i // 8] >> (i % 8)) % 2 -``` - -### `verify_bitfield` - -```python -def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: - """ - Verify ``bitfield`` against the ``committee_size``. - """ - if len(bitfield) != (committee_size + 7) // 8: - return False - - # Check `bitfield` is padded with zero bits only - for i in range(committee_size, len(bitfield) * 8): - if get_bitfield_bit(bitfield, i) == 0b1: - return False - - return True -``` - ### `convert_to_indexed` ```python diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index cdcba343a9..95a3ae9707 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -3,7 +3,7 @@ from eth2spec.utils.ssz.ssz_typing import ( SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, boolean, - Vector, BytesN + Vector, BytesN, Bitlist, Bitvector ) # in bytes @@ -83,12 +83,12 @@ def get_random_ssz_object(rng: Random, return get_max_basic_value(typ) else: return get_random_basic_value(rng, typ) - elif issubclass(typ, Vector): + elif issubclass(typ, Vector) or issubclass(typ, Bitvector): return typ( get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(typ.length) ) - elif issubclass(typ, List): + elif issubclass(typ, List) or issubclass(typ, Bitlist): length = rng.randint(0, min(typ.length, max_list_length)) if mode == RandomizationMode.mode_one_count: length = 1 diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index 1309562352..ccca17385d 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -18,7 +18,14 @@ def translate_typ(typ) -> ssz.BaseSedes: elif issubclass(typ, spec_ssz.Vector): return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.List): + # TODO: Make py-ssz List support the new fixed length list return ssz.List(translate_typ(typ.elem_type)) + elif issubclass(typ, spec_ssz.Bitlist): + # TODO: Once Bitlist implemented in py-ssz, use appropriate type + return ssz.List(translate_typ(typ.elem_type)) + elif issubclass(typ, spec_ssz.Bitvector): + # TODO: Once Bitvector implemented in py-ssz, use appropriate type + return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.boolean): return ssz.boolean elif issubclass(typ, spec_ssz.uint): @@ -68,6 +75,10 @@ def translate_value(value, typ): return value elif issubclass(typ, spec_ssz.Vector): return typ(*(translate_value(elem, typ.elem_type) for elem in value)) + elif issubclass(typ, spec_ssz.Bitlist): + return typ(value) + elif issubclass(typ, spec_ssz.Bitvector): + return typ(value) elif issubclass(typ, spec_ssz.BytesN): return typ(value) elif issubclass(typ, spec_ssz.Bytes): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 4c8b5c7ebb..2e637d42fa 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -1,10 +1,10 @@ from typing import List -from eth2spec.test.helpers.bitfields import set_bitfield_bit from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import Bitlist def build_attestation_data(spec, state, slot, shard): @@ -69,9 +69,8 @@ def get_valid_attestation(spec, state, slot=None, signed=False): ) committee_size = len(crosslink_committee) - bitfield_length = (committee_size + 7) // 8 - aggregation_bitfield = b'\x00' * bitfield_length - custody_bitfield = b'\x00' * bitfield_length + aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + custody_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) attestation = spec.Attestation( aggregation_bitfield=aggregation_bitfield, data=attestation_data, @@ -138,7 +137,7 @@ def fill_aggregate_attestation(spec, state, attestation): attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): - attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) + attestation.aggregation_bitfield[i] = True def add_attestation_to_state(spec, state, attestation, slot): diff --git a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py deleted file mode 100644 index 50e5b6cbad..0000000000 --- a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py +++ /dev/null @@ -1,11 +0,0 @@ -def set_bitfield_bit(bitfield, i): - """ - Set the bit in ``bitfield`` at position ``i`` to ``1``. - """ - byte_index = i // 8 - bit_index = i % 8 - return ( - bitfield[:byte_index] + - bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index + 1:] - ) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 2b34ab405e..ee76ab23d5 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -10,6 +10,7 @@ next_slot, ) from eth2spec.test.helpers.block import apply_empty_block +from eth2spec.utils.ssz.ssz_typing import Bitlist def run_attestation_processing(spec, state, attestation, valid=True): @@ -281,7 +282,10 @@ def test_inconsistent_bitfields(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + b'\x00' + custody_bitfield = deepcopy(attestation.aggregation_bitfield) + custody_bitfield.append(False) + + attestation.custody_bitfield = custody_bitfield sign_attestation(spec, state, attestation) @@ -307,7 +311,7 @@ def test_empty_aggregation_bitfield(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) + attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * len(attestation.aggregation_bitfield))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 047abd4fe0..d0aef1b44c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -351,11 +351,15 @@ class BaseBitfield(BaseList, metaclass=BitElementsType): class Bitlist(BaseBitfield): - pass + @classmethod + def is_fixed_size(cls): + return False class Bitvector(BaseBitfield): - pass + @classmethod + def is_fixed_size(cls): + return True class List(BaseList): From becb7a032ab4ef1cbc2fe0e3d12ae381250f6175 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 12:14:36 +0100 Subject: [PATCH 232/405] justification_bitfield -> Bitvector[4] --- specs/core/0_beacon-chain.md | 16 ++++++++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 9 ++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 13ab4cdae6..c1f1a7bcf1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -523,7 +523,7 @@ class BeaconState(Container): previous_justified_root: Hash # Previous epoch snapshot current_justified_epoch: Epoch current_justified_root: Hash - justification_bitfield: uint64 # Bit set for every recent justified epoch + justification_bitfield: Bitvector[4] # Bit set for every recent justified epoch # Finality finalized_epoch: Epoch finalized_root: Hash @@ -1291,38 +1291,38 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 + state.justification_bitfield = Bitvector[4](*([0] + state.justification_bitfield[0:3])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = previous_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield |= (1 << 1) + state.justification_bitfield[1] = True current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = current_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield |= (1 << 0) + state.justification_bitfield[0] = True # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: + if all(bitfield[1:4]) and old_previous_justified_epoch + 3 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: + if all(bitfield[1:3]) and old_previous_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: + if all(bitfield[0:3]) and old_current_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: + if all(bitfield[0:2]) and old_current_justified_epoch + 1 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) ``` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index d0aef1b44c..6ce2b1538b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -355,12 +355,19 @@ class Bitlist(BaseBitfield): def is_fixed_size(cls): return False + @classmethod + def default(cls): + return cls() + class Bitvector(BaseBitfield): @classmethod def is_fixed_size(cls): return True - + + @classmethod + def default(cls): + return cls(0 for _ in range(cls.length)) class List(BaseList): From 80c680e6147ffc3971856802b4e54d2b776c6862 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 12:41:22 +0100 Subject: [PATCH 233/405] Phase 1 to Bitvector/Bitlist --- specs/core/1_custody-game.md | 12 +++++++++++- specs/core/1_shard-data-chains.md | 5 ++--- specs/light_client/sync_protocol.md | 6 +++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 07f6ec6982..17f599fcd3 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -272,6 +272,16 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int: return crosslink_length * chunks_per_epoch ``` +### `get_bitfield_bit` + +```python +def get_bitfield_bit(bitfield: bytes, i: int) -> int: + """ + Extract the bit in ``bitfield`` at position ``i``. + """ + return (bitfield[i // 8] >> (i % 8)) % 2 +``` + ### `get_custody_chunk_bit` ```python @@ -566,7 +576,7 @@ def process_bit_challenge(state: BeaconState, chunk_count = get_custody_chunk_count(attestation.data.crosslink) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(challenge.responder_index)) + custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index d1c86eca26..b2a5ea9ea6 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -92,7 +92,7 @@ class ShardAttestation(Container): slot: Slot shard: Shard shard_block_root: Bytes32 - aggregation_bitfield: Bytes[PLACEHOLDER] + aggregation_bitfield: Bitlist[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -230,10 +230,9 @@ def verify_shard_attestation_signature(state: BeaconState, attestation: ShardAttestation) -> None: data = attestation.data persistent_committee = get_persistent_committee(state, data.shard, data.slot) - assert verify_bitfield(attestation.aggregation_bitfield, len(persistent_committee)) pubkeys = [] for i, index in enumerate(persistent_committee): - if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b1: + if attestation.aggregation_bitfield[i]: validator = state.validators[index] assert is_active_validator(validator, get_current_epoch(state)) pubkeys.append(validator.pubkey) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 045bf56086..a1b5777cfe 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -168,7 +168,7 @@ If a client wants to update its `finalized_header` it asks the network for a `Bl { 'header': BeaconBlockHeader, 'shard_aggregate_signature': BLSSignature, - 'shard_bitfield': 'bytes', + 'shard_bitfield': Bitlist[PLACEHOLDER], 'shard_parent_block': ShardBlock, } ``` @@ -180,13 +180,13 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header) committee = compute_committee(proof.header, validator_memory) # Verify that we have >=50% support - support_balance = sum([v.effective_balance for i, v in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True]) + support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bitfield[i]]) total_balance = sum([v.effective_balance for i, v in enumerate(committee)]) assert support_balance * 2 > total_balance # Verify shard attestations group_public_key = bls_aggregate_pubkeys([ v.pubkey for v, index in enumerate(committee) - if get_bitfield_bit(proof.shard_bitfield, index) is True + if proof.shard_bitfield[index] ]) assert bls_verify( pubkey=group_public_key, From f57387cc83f16e2ec6b19ee544964254786fa7fa Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 13:09:48 +0100 Subject: [PATCH 234/405] Justification bitvector length to constant --- specs/core/0_beacon-chain.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c1f1a7bcf1..4d3fe9773c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -189,6 +189,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | +| `JUSTIFICATION_BITVECTOR_LENGTH` | `4` | * For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -523,7 +524,7 @@ class BeaconState(Container): previous_justified_root: Hash # Previous epoch snapshot current_justified_epoch: Epoch current_justified_root: Hash - justification_bitfield: Bitvector[4] # Bit set for every recent justified epoch + justification_bitfield: Bitvector[JUSTIFICATION_BITVECTOR_LENGTH] # Bit set for every recent justified epoch # Finality finalized_epoch: Epoch finalized_root: Hash @@ -1291,21 +1292,21 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[4](*([0] + state.justification_bitfield[0:3])) + state.justification_bitfield = Bitvector[4](*([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = previous_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield[1] = True + state.justification_bitfield[1] = 0b1 current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = current_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield[0] = True + state.justification_bitfield[0] = 0b1 # Process finalizations bitfield = state.justification_bitfield From a5154da1ff76e8b768e64987471820f958c391d1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 15:40:40 +0200 Subject: [PATCH 235/405] suggestion to implement bitfield like --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 ++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 33 ++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 7298fb3ca5..f0ee944bd0 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bitfield, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -128,6 +128,8 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 + elif issubclass(typ, Bitfield): + return (typ.length + 7) // 8 // 32 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 elif issubclass(typ, Container): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6ce2b1538b..53ab427438 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -281,10 +281,6 @@ class ElementsType(ParamsMeta): length: int -class BitElementsType(ElementsType): - elem_type = boolean - - class Elements(ParamsBase, metaclass=ElementsType): pass @@ -346,11 +342,16 @@ def last(self): return self[len(self) - 1] -class BaseBitfield(BaseList, metaclass=BitElementsType): - elem_type = bool +class BitElementsType(ElementsType): + elem_type: SSZType = boolean + length: int + + +class Bitfield(BaseList, metaclass=BitElementsType): + pass -class Bitlist(BaseBitfield): +class Bitlist(Bitfield): @classmethod def is_fixed_size(cls): return False @@ -360,15 +361,29 @@ def default(cls): return cls() -class Bitvector(BaseBitfield): +class Bitvector(Bitfield): + + @classmethod + def extract_args(cls, *args): + if len(args) == 0: + return cls.default() + else: + return super().extract_args(*args) + + @classmethod + def value_check(cls, value): + # check length limit strictly + return len(value) == cls.length and super().value_check(value) + @classmethod def is_fixed_size(cls): return True - + @classmethod def default(cls): return cls(0 for _ in range(cls.length)) + class List(BaseList): @classmethod From b574a581094b8a00b360de573bee6a949b44650a Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 14:45:17 +0100 Subject: [PATCH 236/405] Remove not working py-ssz decoder tests --- test_libs/pyspec/eth2spec/fuzzing/test_decoder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index 26ee6e913a..b362d503b9 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,9 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - for typ in [spec.BeaconBlock, spec.BeaconState, spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: + # TODO: Once has Bitfields and Bitvectors, add back + # spec.BeaconState and spec.BeaconBlock + for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value original = random_value.get_random_ssz_object(rng, typ, 100, 10, mode=random_value.RandomizationMode.mode_random, From 8ed638bb84ae3dd88e6658f7bfd5eeda874b97f9 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 15:21:04 +0100 Subject: [PATCH 237/405] Linter fixes --- test_libs/pyspec/eth2spec/fuzzing/test_decoder.py | 2 +- .../test/phase_0/block_processing/test_process_attestation.py | 3 ++- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index b362d503b9..c707c840af 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,7 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - # TODO: Once has Bitfields and Bitvectors, add back + # TODO: Once has Bitfields and Bitvectors, add back # spec.BeaconState and spec.BeaconBlock for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index ee76ab23d5..d17d93e6d0 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -311,7 +311,8 @@ def test_empty_aggregation_bitfield(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * len(attestation.aggregation_bitfield))) + attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( + *([0b0] * len(attestation.aggregation_bitfield))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6ce2b1538b..85f5e0bdbf 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -364,11 +364,12 @@ class Bitvector(BaseBitfield): @classmethod def is_fixed_size(cls): return True - + @classmethod def default(cls): return cls(0 for _ in range(cls.length)) + class List(BaseList): @classmethod From afd86f71de09e556a7a4cc5d82a38b8d514e0e93 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 16:31:33 +0100 Subject: [PATCH 238/405] Fixes in ssz impl --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index f0ee944bd0..53075845bc 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -129,7 +129,7 @@ def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 elif issubclass(typ, Bitfield): - return (typ.length + 7) // 8 // 32 + return (typ.length + 255) // 256 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 elif issubclass(typ, Container): From 93ce1688629f45f92f7a261a958e99351299606a Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 16:47:48 +0100 Subject: [PATCH 239/405] More linting fixes --- scripts/build_spec.py | 2 +- specs/core/0_beacon-chain.md | 3 ++- specs/core/1_custody-game.md | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0a41fe5c01..1f5fe1ee61 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - bit, boolean, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, uint64, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4d3fe9773c..f596da5204 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1292,7 +1292,8 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[4](*([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) + state.justification_bitfield = Bitvector[JUSTIFICATION_BITVECTOR_LENGTH]( + *([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 17f599fcd3..9ad00516e2 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -574,7 +574,6 @@ def process_bit_challenge(state: BeaconState, # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) - assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) From 853f5fc3f0a8ff671ec9bf946d5cf2a84e1c4f9e Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 27 Jun 2019 19:05:27 +0100 Subject: [PATCH 240/405] Apply Danny's suggestions --- specs/core/0_beacon-chain.md | 45 ++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a7d47e108c..ba2cf771b6 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -30,10 +30,10 @@ - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - [`IndexedAttestation`](#indexedattestation) - [`PendingAttestation`](#pendingattestation) - - [`CompactCommittees`](#CompactCommittees) - [`Eth1Data`](#eth1data) - [`HistoricalBatch`](#historicalbatch) - [`DepositData`](#depositdata) + - [`CompactCommittee`](#compactcommittee) - [`BeaconBlockHeader`](#beaconblockheader) - [Beacon operations](#beacon-operations) - [`ProposerSlashing`](#proposerslashing) @@ -361,21 +361,6 @@ class PendingAttestation(Container): proposer_index: ValidatorIndex ``` -#### `CompactCommittee` - -```python -class CompactCommittee(Container): - pubkeys: List[Bytes48, MAX_VALIDATORS_PER_COMMITTEE] - compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] -``` - -#### `CompactCommittees` - -```python -class CompactCommittees(Container): - data: Vector[CompactCommittee, SHARD_COUNT] -``` - #### `Eth1Data` ```python @@ -403,6 +388,14 @@ class DepositData(Container): signature: BLSSignature ``` +#### `CompactCommittee` + +```python +class CompactCommittee(Container): + pubkeys: List[Bytes48, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] +``` + #### `BeaconBlockHeader` ```python @@ -765,16 +758,18 @@ def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: """ Return the compact committee root for the current epoch. """ - committee_data = CompactCommittees().data + committees = Vector[CompactCommittee, SHARD_COUNT]() + start_shard = get_epoch_start_shard(state, epoch) for committee_number in range(get_epoch_committee_count(state, epoch)): - shard = (get_epoch_start_shard(state, epoch) + committee_number) % SHARD_COUNT + shard = (start_shard + committee_number) % SHARD_COUNT for index in get_crosslink_committee(state, epoch, shard): - validator = validators[index] - committee_data[shard].pubkeys.append(validator.pubkey) - # `index` (top 7 bytes) + `slashed` (8th bit) + `effective_balance` (bottom 7 bits) - compact_validator = index << 8 + validator.slashed << 7 + validator.effective_balance // GWEI_PER_ETH - committee_data[shard].compact_validators.append(compact_validator) - return hash_tree_root(committee_data) + validator = state.validators[index] + committees[shard].pubkeys.append(validator.pubkey) + compact_balance = validator.effective_balance // EFFECTIVE_BALANCE_INCREMENT + # `index` (top 6 bytes) + `slashed` (16th bit) + `compact_balance` (bottom 15 bits) + compact_validator = uint64(index << 16 + validator.slashed << 15 + compact_balance) + committees[shard].compact_validators.append(compact_validator) + return hash_tree_root(committees) ``` ### `generate_seed` @@ -787,7 +782,7 @@ def generate_seed(state: BeaconState, """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + - get_compact_committees_root(state, epoch) + + state.compact_committees_roots[epoch] + int_to_bytes(epoch, length=32) ) ``` From 7adf07ea5f1fca3696cc52cfa6e7127aa7e80883 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 22:58:44 +0100 Subject: [PATCH 241/405] A few more tests for Bitvector/Bitlist --- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 63f0c835de..88ccc838c0 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -85,6 +85,16 @@ def merge(a: str, branch: Iterable[str]) -> str: "c5c2", chunk("c5c2")), ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), "c5c201", h(chunk("c5c2"), chunk("10"))), + ("long bitvector", Bitvector[512](1 for i in range(512)), + "ff" * 64, h("ff" * 32, "ff" * 32)), + ("long bitlist", Bitlist[512](1), + "03", h(h(chunk("01"), chunk("")), chunk("01"))), + ("long bitlist", Bitlist[512](1 for i in range(512)), + "ff" * 64 + "01", h(h("ff" * 32, "ff" * 32), chunk("0002"))), + ("odd bitvector", Bitvector[513](1 for i in range(513)), + "ff" * 64 + "01", h(h("ff" * 32, "ff" * 32), h(chunk("01"), chunk("")))), + ("odd bitlist", Bitlist[513](1 for i in range(513)), + "ff" * 64 + "03", h(h(h("ff" * 32, "ff" * 32), h(chunk("01"), chunk(""))), chunk("0102"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From 384fa8854a92666694aabd5b4c923d2968438148 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 00:19:55 +0200 Subject: [PATCH 242/405] justification/finalization testing groundwork --- ..._process_justification_and_finalization.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py new file mode 100644 index 0000000000..95f00edd4e --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -0,0 +1,74 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( + run_epoch_processing_with +) + + +def run_process_just_and_fin(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_justification_and_finalization') + + +def get_committee_size(spec, state, slot): + epoch = spec.slot_to_epoch(slot) + epoch_start_shard = spec.get_epoch_start_shard(state, epoch) + committees_per_slot = spec.get_epoch_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH + shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT + committee_index = (shard + spec.SHARD_COUNT - spec.get_epoch_start_shard(state, epoch)) % spec.SHARD_COUNT + committee_count = spec.get_epoch_committee_count(state, epoch) + indices = spec.get_active_validator_indices(state, epoch) + start = (len(indices) * committee_index) // committee_count + end = (len(indices) * (committee_index + 1)) // committee_count + size = end - start + return size + + +def add_mock_attestations(spec, state, target_epoch, att_count, att_ratio): + # we must be at the end of the epoch + assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 + + previous_epoch = spec.get_previous_epoch(state) + current_epoch = spec.get_current_epoch(state) + + if current_epoch == target_epoch: + attestations = state.current_epoch_attestations + elif previous_epoch == target_epoch: + attestations = state.previous_epoch_attestations + else: + raise Exception(f"cannot target epoch ${target_epoch} from epoch ${current_epoch}") + + total = 0 + while total < att_count: + for i in range(spec.SLOTS_PER_EPOCH): + size = get_committee_size(spec, state, state.slot + i) + # Create a bitfield filled with the given count per attestation, + # exactly on the right-most part of the committee field. + attesting_count = int(size * att_ratio) + aggregation_bitfield = ((1 << attesting_count) - 1).to_bytes(length=((size + 7) // 8), byteorder='big') + + attestations.append(spec.PendingAttestation( + aggregation_bitfield=aggregation_bitfield, + data=spec.AttestationData( + beacon_block_root=b'\xaa' * 32, + source_epoch=0, + source_root=b'\xbb' * 32, + target_root=b'\xbb' * 32, + crosslink=spec.Crosslink() + ), + inclusion_delay=0, + )) + total += 1 + + +@with_all_phases +@spec_state_test +def test_rule_1(spec, state): + previous_epoch = spec.get_previous_epoch(state) + current_epoch = spec.get_current_epoch(state) + + # TODO + # add_mock_attestations(spec, state, ...) + # get indices attesting e.g. current_epoch_attestations + # set their balances + # yield from run_process_just_and_fin(spec, state) + # check finalization + From 990cc55db74628c045ba9919a65c2d9f5d8d0692 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 27 Jun 2019 16:32:10 -0600 Subject: [PATCH 243/405] fix committee typing error --- motes.md | 42 ++++++++++++++++++++++++++++++++++++ notes.txt | 1 + specs/core/0_beacon-chain.md | 4 ++-- 3 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 motes.md create mode 100644 notes.txt diff --git a/motes.md b/motes.md new file mode 100644 index 0000000000..e01ba19b77 --- /dev/null +++ b/motes.md @@ -0,0 +1,42 @@ +* `BLS_WITHDRAWAL_PREFIX` + * Why int rather than bytes? +* `MIN_SEED_LOOKAHEAD` + * Is this actually tunable? + * If so, what are the reprecussions? +* `ACTIVATION_EXIT_DELAY` + * Reaquaint with purpose +* AttesterSlashings + * `MAX_ATTESTER_SLASHINGS` is 1. + * Are there scenarios in which validators can create more effective slashable + messages than can be included on chain? For example, Validators split up to + create double attestations for checkpoints but different (junk) crosslink + data to prevent them from being aggregatable to the fullest + * Max is for block size, no? +* Signature domains + * Max 4byte ints +* `Version` not defined in one of the lists of custom types (2!!). ensure in spec +* `PendingAttestation` + * Don't think `proposer_index` is actually necessary here because effective + balance is stable until end of epoch so can do dynamic lookups +* is_genesis_trigger + * only run at ends of blocks to preserve invariant that eth1data.deposit_root + is the deposit root at the _end_ of an eth1 block +* `Attestation` + * why bitfields not together? +* `Transfer` + * replay mechanism... say the slot gets missed and you sign another transfer + * in a fork you could include both transfers +* `get_previous_epoch` + * do a once over on the genesis stuff +* `get_epoch_start_shard` + * checking next hinges upon the fact that the validator set for the next + epoch is 100% known at the current epoch. Ensure this is the case +* `get_block_root_at_slot` .. `generate_seed` can be bade into one line + function signatures +* `get_shuffled_index` + * I think it should be maybe `assert index_count <= VALIDATOR_REGISTRY_LIMIT` + * is the `2**40` special for security of alg? probably. + + +pubkey/privkey g1 vs g2 + diff --git a/notes.txt b/notes.txt new file mode 100644 index 0000000000..421ad77509 --- /dev/null +++ b/notes.txt @@ -0,0 +1 @@ +* `BLS_WITHDRAWAL_PREFIX` -- diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ba2cf771b6..842b16b986 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -758,7 +758,7 @@ def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: """ Return the compact committee root for the current epoch. """ - committees = Vector[CompactCommittee, SHARD_COUNT]() + committees = [CompactCommittee() for _ in range(SHARD_COUNT)] start_shard = get_epoch_start_shard(state, epoch) for committee_number in range(get_epoch_committee_count(state, epoch)): shard = (start_shard + committee_number) % SHARD_COUNT @@ -769,7 +769,7 @@ def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: # `index` (top 6 bytes) + `slashed` (16th bit) + `compact_balance` (bottom 15 bits) compact_validator = uint64(index << 16 + validator.slashed << 15 + compact_balance) committees[shard].compact_validators.append(compact_validator) - return hash_tree_root(committees) + return hash_tree_root(Vector[CompactCommittee, SHARD_COUNT](committees)) ``` ### `generate_seed` From 237b41df5b41a37972c66c51d5acd551134189b5 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 00:18:54 +0100 Subject: [PATCH 244/405] Slice notation for justification_bitfield shift --- specs/core/0_beacon-chain.md | 5 +++-- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f596da5204..0ea24a2556 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1292,8 +1292,9 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[JUSTIFICATION_BITVECTOR_LENGTH]( - *([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) + state.justification_bitfield[1:JUSTIFICATION_BITVECTOR_LENGTH] = \ + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1] + state.justification_bitfield[0] = 0b0 previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 53ab427438..ba773b4439 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -324,12 +324,18 @@ def __getitem__(self, k) -> SSZValue: return super().__getitem__(k) def __setitem__(self, k, v): - if k < 0: - raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") - if k > len(self): - raise IndexError(f"cannot set item in type {self.__class__}" - f" at out of bounds index {k} (to {v}, bound: {len(self)})") - super().__setitem__(k, coerce_type_maybe(v, self.__class__.elem_type, strict=True)) + if type(k) == slice: + if k.start < 0 or k.stop > len(self): + raise IndexError(f"cannot set item in type {self.__class__}" + f" at out of bounds slice {k} (to {v}, bound: {len(self)})") + super().__setitem__(k, [coerce_type_maybe(x, self.__class__.elem_type) for x in v]) + else: + if k < 0: + raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") + if k > len(self): + raise IndexError(f"cannot set item in type {self.__class__}" + f" at out of bounds index {k} (to {v}, bound: {len(self)})") + super().__setitem__(k, coerce_type_maybe(v, self.__class__.elem_type, strict=True)) def append(self, v): super().append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) From 2677d233a830b07a7f6f60392a1a54b888d3ac60 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 00:31:37 +0100 Subject: [PATCH 245/405] Some more (shorter) Bitvector and Bitlist tests --- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 88ccc838c0..637d9c5c4a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -79,6 +79,12 @@ def merge(a: str, branch: Iterable[str]) -> str: ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), + ("bitvector TTFTFTFF", Bitvector[8](1, 1, 0, 1, 0, 1, 0, 0), "2b", chunk("2b")), + ("bitlist TTFTFTFF", Bitlist[8](1, 1, 0, 1, 0, 1, 0, 0), "2b01", h(chunk("2b"), chunk("08"))), + ("bitvector FTFT", Bitvector[4](0, 1, 0, 1), "0a", chunk("0a")), + ("bitlist FTFT", Bitlist[4](0, 1, 0, 1), "1a", h(chunk("0a"), chunk("04"))), + ("bitvector FTF", Bitvector[3](0, 1, 0), "02", chunk("02")), + ("bitlist FTF", Bitlist[3](0, 1, 0), "0a", h(chunk("02"), chunk("03"))), ("bitvector TFTFFFTTFT", Bitvector[10](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c502", chunk("c502")), ("bitlist TFTFFFTTFT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c506", h(chunk("c502"), chunk("0A"))), ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), From 2252142e01b9431cedbd5f1a2824a41db4e430dc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Jun 2019 10:24:16 +0800 Subject: [PATCH 246/405] padding version normal merkle tree --- .../pyspec/eth2spec/utils/merkle_minimal.py | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 038b555cf8..e3e81f3a55 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -1,4 +1,5 @@ from .hash_function import hash +from math import log2 ZERO_BYTES32 = b'\x00' * 32 @@ -8,11 +9,10 @@ zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1])) -# Compute a Merkle root of a right-zerobyte-padded 2**32 sized tree -def calc_merkle_tree_from_leaves(values): +def calc_merkle_tree_from_leaves(values, layer_count=32): values = list(values) tree = [values[::]] - for h in range(32): + for h in range(layer_count): if len(values) % 2 == 1: values.append(zerohashes[h]) values = [hash(values[i] + values[i + 1]) for i in range(0, len(values), 2)] @@ -20,8 +20,11 @@ def calc_merkle_tree_from_leaves(values): return tree -def get_merkle_root(values): - return calc_merkle_tree_from_leaves(values)[-1][0] +def get_merkle_root(values, pad_to=1): + layer_count = int(log2(pad_to)) + if len(values) == 0: + return zerohashes[layer_count] + return calc_merkle_tree_from_leaves(values, layer_count)[-1][0] def get_merkle_proof(tree, item_index): @@ -32,19 +35,7 @@ def get_merkle_proof(tree, item_index): return proof -def next_power_of_two(v: int) -> int: - """ - Get the next power of 2. (for 64 bit range ints). - 0 is a special case, to have non-empty defaults. - Examples: - 0 -> 1, 1 -> 1, 2 -> 2, 3 -> 4, 32 -> 32, 33 -> 64 - """ - if v == 0: - return 1 - return 1 << (v - 1).bit_length() - - -def merkleize_chunks(chunks, pad_to: int = 1): +def merkleize_chunks(chunks, pad_to: int=1): count = len(chunks) depth = max(count - 1, 0).bit_length() max_depth = max(depth, (pad_to - 1).bit_length()) From 196ac4202595a716a7acba91e7a437d80cc1a4ad Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:23:22 +0100 Subject: [PATCH 247/405] Cleanup naming --- specs/core/0_beacon-chain.md | 49 +++++++++---------- specs/core/1_custody-game.md | 26 +++++----- specs/core/1_shard-data-chains.md | 4 +- specs/light_client/sync_protocol.md | 8 +-- specs/validator/0_beacon-chain-validator.md | 20 ++++---- specs/validator/beacon_node_oapi.yaml | 8 +-- .../pyspec/eth2spec/fuzzing/test_decoder.py | 2 +- .../test/fork_choice/test_on_attestation.py | 2 +- .../eth2spec/test/helpers/attestations.py | 12 ++--- .../test_process_attestation.py | 18 +++---- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 +-- 12 files changed, 79 insertions(+), 80 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9ce7976b52..d411734248 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -190,7 +190,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | -| `JUSTIFICATION_BITVECTOR_LENGTH` | `4` | +| `JUSTIFICATION_BITS_LENGTH` | `4` | * For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -360,7 +360,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + aggregation_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -427,9 +427,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + aggregation_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData - custody_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + custody_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] signature: BLSSignature ``` @@ -527,7 +527,7 @@ class BeaconState(Container): previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] # Finality - justification_bitfield: Bitvector[JUSTIFICATION_BITVECTOR_LENGTH] # Bit set for every recent justified epoch + justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint @@ -867,12 +867,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S ```python def get_attesting_indices(state: BeaconState, data: AttestationData, - bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: + bits: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: """ - Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. + Return the sorted attesting indices corresponding to ``data`` and ``bits``. """ committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) - return sorted([index for i, index in enumerate(committee) if bitfield[i]]) + return sorted([index for i, index in enumerate(committee) if bits[i]]) ``` ### `int_to_bytes` @@ -920,8 +920,8 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedA """ Convert ``attestation`` to (almost) indexed-verifiable form. """ - attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) - custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bitfield) + attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bits) assert set(custody_bit_1_indices).issubset(attesting_indices) custody_bit_0_indices = [index for index in attesting_indices if index not in custody_bit_1_indices] @@ -1146,7 +1146,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots + # Populate active_index_roots genesis_active_index_root = hash_tree_root( List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) ) @@ -1254,7 +1254,7 @@ def get_unslashed_attesting_indices(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: output = set() # type: Set[ValidatorIndex] for a in attestations: - output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) + output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) return set(filter(lambda index: not state.validators[index].slashed, list(output))) ``` @@ -1294,36 +1294,35 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bitfield[1:JUSTIFICATION_BITVECTOR_LENGTH] = \ - state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1] - state.justification_bitfield[0] = 0b0 + state.justification_bits[1:JUSTIFICATION_BITS_LENGTH] = state.justification_bits[0:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) - state.justification_bitfield[1] = 0b1 + state.justification_bits[1] = 0b1 current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) - state.justification_bitfield[0] = 0b1 + state.justification_bits[0] = 0b1 # Process finalizations - bitfield = state.justification_bitfield + bits = state.justification_bits # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if all(bitfield[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: + if all(bits[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if all(bitfield[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: + if all(bits[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if all(bitfield[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: + if all(bits[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if all(bitfield[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: + if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint ``` @@ -1379,7 +1378,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence index = ValidatorIndex(index) attestation = min([ a for a in matching_source_attestations - if index in get_attesting_indices(state, a.data, a.aggregation_bitfield) + if index in get_attesting_indices(state, a.data, a.aggregation_bits) ], key=lambda a: a.inclusion_delay) proposer_reward = Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) rewards[attestation.proposer_index] += proposer_reward @@ -1661,7 +1660,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: pending_attestation = PendingAttestation( data=data, - aggregation_bitfield=attestation.aggregation_bitfield, + aggregation_bits=attestation.aggregation_bits, inclusion_delay=state.slot - attestation_slot, proposer_index=get_beacon_proposer_index(state), ) @@ -1680,7 +1679,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.crosslink.start_epoch == parent_crosslink.end_epoch assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] - + # Check signature validate_indexed_attestation(state, convert_to_indexed(state, attestation)) ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 9ad00516e2..883476c58a 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -272,14 +272,14 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int: return crosslink_length * chunks_per_epoch ``` -### `get_bitfield_bit` +### `get_bit` ```python -def get_bitfield_bit(bitfield: bytes, i: int) -> int: +def get_bit(serialization: bytes, i: int) -> int: """ - Extract the bit in ``bitfield`` at position ``i``. + Extract the bit in ``serialization`` at position ``i``. """ - return (bitfield[i // 8] >> (i % 8)) % 2 + return (serialization[i // 8] >> (i % 8)) % 2 ``` ### `get_custody_chunk_bit` @@ -287,17 +287,17 @@ def get_bitfield_bit(bitfield: bytes, i: int) -> int: ```python def get_custody_chunk_bit(key: BLSSignature, chunk: bytes) -> bool: # TODO: Replace with something MPC-friendly, e.g. the Legendre symbol - return bool(get_bitfield_bit(hash(key + chunk), 0)) + return bool(get_bit(hash(key + chunk), 0)) ``` ### `get_chunk_bits_root` ```python -def get_chunk_bits_root(chunk_bitfield: bytes) -> Bytes32: +def get_chunk_bits_root(chunk_bits: bytes) -> Bytes32: aggregated_bits = bytearray([0] * 32) - for i in range(0, len(chunk_bitfield), 32): + for i in range(0, len(chunk_bits), 32): for j in range(32): - aggregated_bits[j] ^= chunk_bitfield[i + j] + aggregated_bits[j] ^= chunk_bits[i + j] return hash(aggregated_bits) ``` @@ -489,7 +489,7 @@ def process_chunk_challenge(state: BeaconState, responder = state.validators[challenge.responder_index] assert responder.exit_epoch >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bitfield) + attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bits) assert challenge.responder_index in attesters # Verify the challenge is not a duplicate for record in state.custody_chunk_challenge_records: @@ -546,7 +546,7 @@ def process_bit_challenge(state: BeaconState, get_validators_custody_reveal_period(state, challenge.responder_index)) # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) + attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) assert challenge.responder_index in attesters # A validator can be the challenger for at most one challenge at a time @@ -575,8 +575,8 @@ def process_bit_challenge(state: BeaconState, # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] - assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) + custody_bit = attestation.custody_bits[attesters.index(challenge.responder_index)] + assert custody_bit != get_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( challenge_index=state.custody_challenge_index, @@ -670,7 +670,7 @@ def process_bit_challenge_response(state: BeaconState, ) # Verify the chunk bit does not match the challenge chunk bit assert (get_custody_chunk_bit(challenge.responder_key, response.chunk) - != get_bitfield_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) + != get_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) # Clear the challenge records = state.custody_bit_challenge_records records[records.index(challenge)] = CustodyBitChallengeRecord() diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index b2a5ea9ea6..432820b7cf 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -92,7 +92,7 @@ class ShardAttestation(Container): slot: Slot shard: Shard shard_block_root: Bytes32 - aggregation_bitfield: Bitlist[PLACEHOLDER] + aggregation_bits: Bitlist[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -232,7 +232,7 @@ def verify_shard_attestation_signature(state: BeaconState, persistent_committee = get_persistent_committee(state, data.shard, data.slot) pubkeys = [] for i, index in enumerate(persistent_committee): - if attestation.aggregation_bitfield[i]: + if attestation.aggregation_bits[i]: validator = state.validators[index] assert is_active_validator(validator, get_current_epoch(state)) pubkeys.append(validator.pubkey) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index a1b5777cfe..3f5a372891 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -168,7 +168,7 @@ If a client wants to update its `finalized_header` it asks the network for a `Bl { 'header': BeaconBlockHeader, 'shard_aggregate_signature': BLSSignature, - 'shard_bitfield': Bitlist[PLACEHOLDER], + 'shard_bits': Bitlist[PLACEHOLDER], 'shard_parent_block': ShardBlock, } ``` @@ -180,13 +180,13 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header) committee = compute_committee(proof.header, validator_memory) # Verify that we have >=50% support - support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bitfield[i]]) + support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bits[i]]) total_balance = sum([v.effective_balance for i, v in enumerate(committee)]) assert support_balance * 2 > total_balance # Verify shard attestations group_public_key = bls_aggregate_pubkeys([ v.pubkey for v, index in enumerate(committee) - if proof.shard_bitfield[index] + if proof.shard_bits[index] ]) assert bls_verify( pubkey=group_public_key, @@ -196,4 +196,4 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val ) ``` -The size of this proof is only 200 (header) + 96 (signature) + 16 (bitfield) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_root, ShardBlock)`, which would cut off ~220 bytes. +The size of this proof is only 200 (header) + 96 (signature) + 16 (bits) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_root, ShardBlock)`, which would cut off ~220 bytes. diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index aa8350b66c..813b8bcfe0 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -44,8 +44,8 @@ - [Crosslink vote](#crosslink-vote) - [Construct attestation](#construct-attestation) - [Data](#data) - - [Aggregation bitfield](#aggregation-bitfield) - - [Custody bitfield](#custody-bitfield) + - [Aggregation bits](#aggregation-bits) + - [Custody bits](#custody-bits) - [Aggregate signature](#aggregate-signature) - [How to avoid slashing](#how-to-avoid-slashing) - [Proposer slashing](#proposer-slashing) @@ -322,19 +322,19 @@ Next, the validator creates `attestation`, an [`Attestation`](../core/0_beacon-c Set `attestation.data = attestation_data` where `attestation_data` is the `AttestationData` object defined in the previous section, [attestation data](#attestation-data). -##### Aggregation bitfield +##### Aggregation bits -* Let `aggregation_bitfield` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. +* Let `aggregation_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. * Let `index_into_committee` be the index into the validator's `committee` at which `validator_index` is located. -* Set `aggregation_bitfield[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. -* Set `attestation.aggregation_bitfield = aggregation_bitfield`. +* Set `aggregation_bits[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. +* Set `attestation.aggregation_bits = aggregation_bits`. -*Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield)` should return a list of length equal to 1, containing `validator_index`. +*Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. -##### Custody bitfield +##### Custody bits -* Let `custody_bitfield` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Set `attestation.custody_bitfield = custody_bitfield`. +* Let `custody_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. +* Set `attestation.custody_bits = custody_bits`. *Note*: This is a stub for Phase 0. diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 74be21fac8..4da8f7933e 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -415,16 +415,16 @@ components: type: object description: "The [`Attestation`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestation) object from the Eth2.0 spec." properties: - aggregation_bitfield: + aggregation_bits: type: string format: byte pattern: "^0x[a-fA-F0-9]+$" - description: "Attester aggregation bitfield." - custody_bitfield: + description: "Attester aggregation bits." + custody_bits: type: string format: byte pattern: "^0x[a-fA-F0-9]+$" - description: "Custody bitfield." + description: "Custody bits." signature: type: string format: byte diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index c707c840af..ea1f1d47f8 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,7 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - # TODO: Once has Bitfields and Bitvectors, add back + # TODO: Once has Bitlists and Bitvectors, add back # spec.BeaconState and spec.BeaconBlock for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 1a67e01d53..5adb022a66 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -118,5 +118,5 @@ def test_on_attestation_invalid_attestation(spec, state): attestation = get_valid_attestation(spec, state, slot=block.slot) # make attestation invalid - attestation.custody_bitfield[0:8] = [0, 0, 0, 0, 1, 1, 1, 1] + attestation.custody_bits[0:8] = [0, 0, 0, 0, 1, 1, 1, 1] run_on_attestation(spec, state, store, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index a184a5f5ee..6fdf1538b1 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -67,12 +67,12 @@ def get_valid_attestation(spec, state, slot=None, signed=False): ) committee_size = len(crosslink_committee) - aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) - custody_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + aggregation_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + custody_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) attestation = spec.Attestation( - aggregation_bitfield=aggregation_bitfield, + aggregation_bits=aggregation_bits, data=attestation_data, - custody_bitfield=custody_bitfield, + custody_bits=custody_bits, ) fill_aggregate_attestation(spec, state, attestation) if signed: @@ -105,7 +105,7 @@ def sign_attestation(spec, state, attestation): participants = spec.get_attesting_indices( state, attestation.data, - attestation.aggregation_bitfield, + attestation.aggregation_bits, ) attestation.signature = sign_aggregate_attestation(spec, state, attestation.data, participants) @@ -135,7 +135,7 @@ def fill_aggregate_attestation(spec, state, attestation): attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): - attestation.aggregation_bitfield[i] = True + attestation.aggregation_bits[i] = True def add_attestation_to_state(spec, state, attestation, slot): diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 1ea7c92cfc..9167240016 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -275,14 +275,14 @@ def test_bad_crosslink_end_epoch(spec, state): @with_all_phases @spec_state_test -def test_inconsistent_bitfields(spec, state): +def test_inconsistent_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - custody_bitfield = deepcopy(attestation.aggregation_bitfield) - custody_bitfield.append(False) + custody_bits = deepcopy(attestation.aggregation_bits) + custody_bits.append(False) - attestation.custody_bitfield = custody_bitfield + attestation.custody_bits = custody_bits sign_attestation(spec, state, attestation) @@ -291,11 +291,11 @@ def test_inconsistent_bitfields(spec, state): @with_phases(['phase0']) @spec_state_test -def test_non_empty_custody_bitfield(spec, state): +def test_non_empty_custody_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + attestation.custody_bits = deepcopy(attestation.aggregation_bits) sign_attestation(spec, state, attestation) @@ -304,12 +304,12 @@ def test_non_empty_custody_bitfield(spec, state): @with_all_phases @spec_state_test -def test_empty_aggregation_bitfield(spec, state): +def test_empty_aggregation_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( - *([0b0] * len(attestation.aggregation_bitfield))) + attestation.aggregation_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( + *([0b0] * len(attestation.aggregation_bits))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 53075845bc..d5855a755f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bitfield, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bits, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -128,7 +128,7 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 - elif issubclass(typ, Bitfield): + elif issubclass(typ, Bits): return (typ.length + 255) // 256 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ba773b4439..91d3ff4f2f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -353,11 +353,11 @@ class BitElementsType(ElementsType): length: int -class Bitfield(BaseList, metaclass=BitElementsType): +class Bits(BaseList, metaclass=BitElementsType): pass -class Bitlist(Bitfield): +class Bitlist(Bits): @classmethod def is_fixed_size(cls): return False @@ -367,7 +367,7 @@ def default(cls): return cls() -class Bitvector(Bitfield): +class Bitvector(Bits): @classmethod def extract_args(cls, *args): From 6f9d37485d624cb96188714db495cdb07ace9aa6 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:34:01 +0100 Subject: [PATCH 248/405] Cleanups --- specs/core/0_beacon-chain.md | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d411734248..9de0249095 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,20 +1294,14 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:JUSTIFICATION_BITS_LENGTH] = state.justification_bits[0:JUSTIFICATION_BITS_LENGTH - 1] - state.justification_bits[0] = 0b0 - previous_epoch_matching_target_balance = get_attesting_balance( - state, get_matching_target_attestations(state, previous_epoch) - ) - if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, - root=get_block_root(state, previous_epoch)) + state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch + if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 - current_epoch_matching_target_balance = get_attesting_balance( - state, get_matching_target_attestations(state, current_epoch) - ) - if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) + matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch + if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From e36593b155d3fdffdb8ebe451236d5dfb54e22c9 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:35:50 +0100 Subject: [PATCH 249/405] Add comment --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9de0249095..96b8981fe5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,7 +1294,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] # Bitshift matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) From c764202a5768003bfaf354e90e591f66c48c9679 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 28 Jun 2019 21:35:26 +0800 Subject: [PATCH 250/405] Slashing penalty calculation change (#1217) If the exit queue is very long, then a validator may take many months to exit. With the code as currently written, however, self-slashing is a potentially lucrative route to get one's money out faster, because one can exit in 36 days. This PR changes it so that slashing can only extend your withdrawal time, not contract it. Also, instead of the slashed balances used to calculate one's slashing penalty being those in `[withdrawal - 54 days ... withdrawal - 18 days]`, we now run the penalization algorithm once every 36 days that a validator is slashed but not withdrawn, so that it covers the 36-day period where the validator was actually slashed. It also moves the minimum slashing penalty to the `slash_validator` function so that it is only applied once. We also simplify the `slashed_balances` logic to be per-epoch. --- configs/constant_presets/mainnet.yaml | 4 +- configs/constant_presets/minimal.yaml | 4 +- specs/core/0_beacon-chain.md | 46 ++++------ specs/core/1_custody-game.md | 2 +- .../test_process_attester_slashing.py | 86 +++++++++++++++---- 5 files changed, 94 insertions(+), 48 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 9f7ca950f1..38f10c8fca 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -76,7 +76,7 @@ MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 # 2**16 (= 65,536) epochs ~0.8 years EPOCHS_PER_HISTORICAL_VECTOR: 65536 # 2**13 (= 8,192) epochs ~36 days -EPOCHS_PER_SLASHED_BALANCES_VECTOR: 8192 +EPOCHS_PER_SLASHINGS_VECTOR: 8192 # 2**24 (= 16,777,216) historical roots, ~26,131 years HISTORICAL_ROOTS_LIMIT: 16777216 # 2**40 (= 1,099,511,627,776) validator spots @@ -88,7 +88,7 @@ VALIDATOR_REGISTRY_LIMIT: 1099511627776 # 2**5 (= 32) BASE_REWARD_FACTOR: 32 # 2**9 (= 512) -WHISTLEBLOWING_REWARD_QUOTIENT: 512 +WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 # 2**25 (= 33,554,432) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 3e3f7ccb48..3aa4a6b719 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -77,7 +77,7 @@ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 # [customized] smaller state EPOCHS_PER_HISTORICAL_VECTOR: 64 # [customized] smaller state -EPOCHS_PER_SLASHED_BALANCES_VECTOR: 64 +EPOCHS_PER_SLASHINGS_VECTOR: 64 # 2**24 (= 16,777,216) historical roots HISTORICAL_ROOTS_LIMIT: 16777216 # 2**40 (= 1,099,511,627,776) validator spots @@ -89,7 +89,7 @@ VALIDATOR_REGISTRY_LIMIT: 1099511627776 # 2**5 (= 32) BASE_REWARD_FACTOR: 32 # 2**9 (= 512) -WHISTLEBLOWING_REWARD_QUOTIENT: 512 +WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 # 2**25 (= 33,554,432) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1b7cda0a4d..61ef8b7421 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -234,7 +234,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | -| `EPOCHS_PER_SLASHED_BALANCES_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | +| `EPOCHS_PER_SLASHINGS_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | | `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | | `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validator spots | | @@ -243,7 +243,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | | `BASE_REWARD_FACTOR` | `2**6` (= 64) | -| `WHISTLEBLOWING_REWARD_QUOTIENT` | `2**9` (= 512) | +| `WHISTLEBLOWER_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | @@ -520,7 +520,7 @@ class BeaconState(Container): randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Active registry digests for light clients # Slashings - slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of slashed effective balances + slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Attestations previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] @@ -1097,21 +1097,22 @@ def slash_validator(state: BeaconState, """ Slash the validator with index ``slashed_index``. """ - current_epoch = get_current_epoch(state) + epoch = get_current_epoch(state) initiate_validator_exit(state, slashed_index) - state.validators[slashed_index].slashed = True - state.validators[slashed_index].withdrawable_epoch = Epoch(current_epoch + EPOCHS_PER_SLASHED_BALANCES_VECTOR) - slashed_balance = state.validators[slashed_index].effective_balance - state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] += slashed_balance + validator = state.validators[slashed_index] + validator.slashed = True + validator.withdrawable_epoch = max(validator.withdrawable_epoch, Epoch(epoch + EPOCHS_PER_SLASHINGS_VECTOR)) + state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effective_balance + decrease_balance(state, slashed_index, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT) + # Apply proposer and whistleblower rewards proposer_index = get_beacon_proposer_index(state) if whistleblower_index is None: whistleblower_index = proposer_index - whistleblowing_reward = Gwei(slashed_balance // WHISTLEBLOWING_REWARD_QUOTIENT) - proposer_reward = Gwei(whistleblowing_reward // PROPOSER_REWARD_QUOTIENT) + whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) + proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) - increase_balance(state, whistleblower_index, whistleblowing_reward - proposer_reward) - decrease_balance(state, slashed_index, whistleblowing_reward) + increase_balance(state, whistleblower_index, whistleblower_reward - proposer_reward) ``` ## Genesis @@ -1174,7 +1175,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots + # Populate active_index_roots genesis_active_index_root = hash_tree_root( List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) ) @@ -1493,18 +1494,9 @@ def process_registry_updates(state: BeaconState) -> None: def process_slashings(state: BeaconState) -> None: epoch = get_current_epoch(state) total_balance = get_total_active_balance(state) - - # Compute slashed balances in the current epoch - total_at_start = state.slashed_balances[(epoch + 1) % EPOCHS_PER_SLASHED_BALANCES_VECTOR] - total_at_end = state.slashed_balances[epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] - total_penalties = total_at_end - total_at_start - for index, validator in enumerate(state.validators): - if validator.slashed and epoch + EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2 == validator.withdrawable_epoch: - penalty = max( - validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, - validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT - ) + if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: + penalty = validator.effective_balance * min(sum(state.slashings) * 3, total_balance) // total_balance decrease_balance(state, ValidatorIndex(index), penalty) ``` @@ -1532,10 +1524,8 @@ def process_final_updates(state: BeaconState) -> None: get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) ) ) - # Set total slashed balances - state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( - state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] - ) + # Reset slashings + state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) # Set randao mix state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) # Set historical root accumulator diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 47d1578c56..c29033fe3c 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -453,7 +453,7 @@ def process_early_derived_secret_reveal(state: BeaconState, # Apply penalty proposer_index = get_beacon_proposer_index(state) whistleblower_index = reveal.masker_index - whistleblowing_reward = Gwei(penalty // WHISTLEBLOWING_REWARD_QUOTIENT) + whistleblowing_reward = Gwei(penalty // WHISTLEBLOWER_REWARD_QUOTIENT) proposer_reward = Gwei(whistleblowing_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) increase_balance(state, whistleblower_index, whistleblowing_reward - proposer_reward) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index e2b50ea0b2..e78e1a8669 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -25,31 +25,56 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) yield 'post', None return - slashed_index = attester_slashing.attestation_1.custody_bit_0_indices[0] - pre_slashed_balance = get_balance(state, slashed_index) + slashed_indices = ( + attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices + ) proposer_index = spec.get_beacon_proposer_index(state) pre_proposer_balance = get_balance(state, proposer_index) + pre_slashings = {slashed_index: get_balance(state, slashed_index) for slashed_index in slashed_indices} + pre_withdrawalable_epochs = { + slashed_index: state.validators[slashed_index].withdrawable_epoch + for slashed_index in slashed_indices + } + + total_proposer_rewards = sum( + balance // spec.WHISTLEBLOWER_REWARD_QUOTIENT + for balance in pre_slashings.values() + ) # Process slashing spec.process_attester_slashing(state, attester_slashing) - slashed_validator = state.validators[slashed_index] - - # Check slashing - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - - if slashed_index != proposer_index: - # lost whistleblower reward - assert get_balance(state, slashed_index) < pre_slashed_balance + for slashed_index in slashed_indices: + pre_withdrawalable_epoch = pre_withdrawalable_epochs[slashed_index] + slashed_validator = state.validators[slashed_index] + + # Check slashing + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + if pre_withdrawalable_epoch < spec.FAR_FUTURE_EPOCH: + expected_withdrawable_epoch = max( + pre_withdrawalable_epoch, + spec.get_current_epoch(state) + spec.EPOCHS_PER_SLASHINGS_VECTOR + ) + assert slashed_validator.withdrawable_epoch == expected_withdrawable_epoch + else: + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + assert get_balance(state, slashed_index) < pre_slashings[slashed_index] + + if proposer_index not in slashed_indices: # gained whistleblower reward - assert get_balance(state, proposer_index) > pre_proposer_balance + assert get_balance(state, proposer_index) == pre_proposer_balance + total_proposer_rewards else: # gained rewards for all slashings, which may include others. And only lost that of themselves. - # Netto at least 0, if more people where slashed, a balance increase. - assert get_balance(state, slashed_index) >= pre_slashed_balance + expected_balance = ( + pre_proposer_balance + + total_proposer_rewards + - pre_slashings[proposer_index] // spec.MIN_SLASHING_PENALTY_QUOTIENT + ) + + assert get_balance(state, proposer_index) == expected_balance yield 'post', state @@ -82,6 +107,37 @@ def test_success_surround(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing) +@with_all_phases +@always_bls +@spec_state_test +def test_success_already_exited_recent(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + slashed_indices = ( + attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices + ) + for index in slashed_indices: + spec.initiate_validator_exit(state, index) + + yield from run_attester_slashing_processing(spec, state, attester_slashing) + + +@with_all_phases +@always_bls +@spec_state_test +def test_success_already_exited_long_ago(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + slashed_indices = ( + attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices + ) + for index in slashed_indices: + spec.initiate_validator_exit(state, index) + state.validators[index].withdrawable_epoch = spec.get_current_epoch(state) + 2 + + yield from run_attester_slashing_processing(spec, state, attester_slashing) + + @with_all_phases @always_bls @spec_state_test From 2739767a7147f6b2efd6537189f3f3437b885058 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 28 Jun 2019 14:43:44 +0100 Subject: [PATCH 251/405] Hardened Eth 1.0 voting strategy (#1218) --- specs/validator/0_beacon-chain-validator.md | 33 +++++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index aa8350b66c..44b5e2a3cf 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -221,19 +221,26 @@ epoch_signature = bls_sign( ##### Eth1 Data -`block.eth1_data` is a mechanism used by block proposers vote on a recent Ethereum 1.0 block hash and an associated deposit root found in the Ethereum 1.0 deposit contract. When consensus is formed, `state.eth1_data` is updated, and validator deposits up to this root can be processed. The deposit root can be calculated by calling the `get_deposit_root()` function of the deposit contract using the post-state of the block hash. - -* Let `D` be the list of `Eth1DataVote` objects `vote` in `state.eth1_data_votes` where: - * `vote.eth1_data.block_hash` is the hash of an Eth 1.0 block that is (i) part of the canonical chain, (ii) >= `ETH1_FOLLOW_DISTANCE` blocks behind the head, and (iii) newer than `state.eth1_data.block_hash`. - * `vote.eth1_data.deposit_count` is the deposit count of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. - * `vote.eth1_data.deposit_root` is the deposit root of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. -* If `D` is empty: - * Let `block_hash` be the block hash of the `ETH1_FOLLOW_DISTANCE`'th ancestor of the head of the canonical Eth 1.0 chain. - * Let `deposit_root` and `deposit_count` be the deposit root and deposit count of the Eth 1.0 deposit contract in the post-state of the block referenced by `block_hash` - * Let `best_vote_data = Eth1Data(block_hash=block_hash, deposit_root=deposit_root, deposit_count=deposit_count)`. -* If `D` is nonempty: - * Let `best_vote_data` be the `eth1_data` member of `D` that has the highest vote count (`D.count(eth1_data)`), breaking ties by favoring block hashes with higher associated block height. -* Set `block.eth1_data = best_vote_data`. +The `block.eth1_data` field is for block proposers to vote on recent Eth 1.0 data. This recent data contains an Eth 1.0 block hash as well as the associated deposit root (as calculated by the `get_deposit_root()` method of the deposit contract) and deposit count after execution of the corresponding Eth 1.0 block. If over half of the block proposers in the current Eth 1.0 voting period vote for the same `eth1_data` then `state.eth1_data` updates at the end of the voting period. Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. + +Let `get_eth1_data(distance: int) -> Eth1Data` be the (subjective) function that returns the Eth 1.0 data at distance `distance` relative to the Eth 1.0 head at the start of the current Eth 1.0 voting period. Let `previous_eth1_distance` be the distance relative to the Eth 1.0 block corresponding to `state.eth1_data.block_hash` at the start of the current Eth 1.0 voting period. An honest block proposer sets `block.eth1_data = get_eth1_vote(state, previous_eth1_distance)` where: + +```python +def get_eth1_vote(state: BeaconState, previous_eth1_distance: uint64) -> Eth1Data: + new_eth1_data = [get_eth1_data(distance) for distance in range(ETH1_FOLLOW_DISTANCE, 2 * ETH1_FOLLOW_DISTANCE)] + all_eth1_data = [get_eth1_data(distance) for distance in range(ETH1_FOLLOW_DISTANCE, previous_eth1_distance)] + + valid_votes = [] + for slot, vote in enumerate(state.eth1_data_votes): + period_tail = slot % SLOTS_PER_ETH1_VOTING_PERIOD >= integer_square_root(SLOTS_PER_ETH1_VOTING_PERIOD) + if vote in new_eth1_data or (period_tail and vote in all_eth1_data): + valid_votes.append(vote) + + return max(valid_votes, + key=lambda v: (valid_votes.count(v), -all_eth1_data.index(v)), # Tiebreak by smallest distance + default=get_eth1_data(ETH1_FOLLOW_DISTANCE), + ) +``` ##### Signature From 05842f83718644184ce997277e7817bada07d568 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 28 Jun 2019 15:26:02 +0100 Subject: [PATCH 252/405] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 96b8981fe5..b9c3f5f0c5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,14 +1294,15 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] # Bitshift + state.justification_bits[1:] == state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From 5ff13dd81a95d7852c2afa65dd3851d5642c17dc Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:07:36 +0200 Subject: [PATCH 253/405] be explicit about limiting for HTR and chunk padding --- specs/simple-serialize.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 97b1d560c6..3fba59fbad 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -49,13 +49,13 @@ class ContainerExample(Container): foo: uint64 bar: boolean ``` -* **vector**: ordered fixed-length homogeneous collection of values +* **vector**: ordered fixed-length homogeneous collection, with `N` values * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` -* **list**: ordered variable-length homogeneous collection of values, with maximum length `N` +* **list**: ordered variable-length homogeneous collection, limited to `N` values * notation `List[type, N]`, e.g. `List[uint64, N]` -* **bitvector**: ordered fixed-length collection of `boolean` values +* **bitvector**: ordered fixed-length collection of `boolean` values, with `N` bits * notation `Bitvector[N]` -* **bitlist**: ordered variable-length collection of `boolean` values, with maximum length `N` +* **bitlist**: ordered variable-length collection of `boolean` values, limited to `N` bits * notation `Bitlist[N]` * **union**: union type containing one of the given subtypes * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` @@ -161,22 +161,31 @@ return serialized_type_index + serialized_bytes Because serialization is an injective function (i.e. two distinct objects of the same type will serialize to different values) any bytestring has at most one object it could deserialize to. Efficient algorithms for computing this object can be found in [the implementations](#implementations). +Note that deserialization requires hardening against invalid inputs. A non-exhaustive list: +- Offsets: out of order, out of range, mismatching minimum element size +- Scope: Extra unused bytes, not aligned with element size. +- More elements than a list limit allows. Part of enforcing consensus. + ## Merkleization We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. -* `merkleize`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. Note that `merkleize` on a single chunk is simply that chunk, i.e. the identity when the number of chunks is one. -* `pad`: given a list `l` and a length `N`, adds `N-len(l)` empty objects to the end of the list (the type of the empty object is implicit in the list type) +* `merkleize(data, pad_to)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. + There merkleization depends on the input length: + - `0` chunks: A chunk filled with zeroes. + - `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. + - `1+` chunks: pad to the next power of 2, merkleize as binary tree. + - with `pad_to` set: pad the `data` with zeroed chunks to this power of two (virtually for memory efficiency). * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(pad(value, N))), len(value))` if `value` is a list of basic objects +* `mix_in_length(merkleize(pack(value), pad_to=N), len(value))` if `value` is a list of basic objects. `N` is the amount of chunks needed for the list limit (*this depends on element-type size*). * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in pad(value, N)]), len(value))` if `value` is a list of composite objects +* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_to=N), len(value))` if `value` is a list of composite objects. `N` is the list limit (*1 chunk per element*). * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type ### Merkleization of `Bitvector[N]` From dcb0244a4f99f50dfa82884a81bd14e1eabb316e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 10:19:59 -0500 Subject: [PATCH 254/405] get_attesting_indices set instead of sorted (#1225) --- specs/core/0_beacon-chain.md | 14 +++--- specs/core/1_custody-game.md | 94 ++++++++++++------------------------ 2 files changed, 39 insertions(+), 69 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 61ef8b7421..abca72ff84 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -866,13 +866,13 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S ### `get_attesting_indices` ```python -def get_attesting_indices(state: BeaconState, data: AttestationData, bitfield: bytes) -> Sequence[ValidatorIndex]: +def get_attesting_indices(state: BeaconState, data: AttestationData, bitfield: bytes) -> Set[ValidatorIndex]: """ - Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. + Return the set of attesting indices corresponding to ``data`` and ``bitfield``. """ committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) - return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) + return set(index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1) ``` ### `int_to_bytes` @@ -950,12 +950,12 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedA """ attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bitfield) - assert set(custody_bit_1_indices).issubset(attesting_indices) - custody_bit_0_indices = [index for index in attesting_indices if index not in custody_bit_1_indices] + assert custody_bit_1_indices.issubset(attesting_indices) + custody_bit_0_indices = attesting_indices.difference(custody_bit_1_indices) return IndexedAttestation( - custody_bit_0_indices=custody_bit_0_indices, - custody_bit_1_indices=custody_bit_1_indices, + custody_bit_0_indices=sorted(custody_bit_0_indices), + custody_bit_1_indices=sorted(custody_bit_1_indices), data=attestation.data, signature=attestation.signature, ) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index c29033fe3c..9b17b39ed6 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -36,7 +36,7 @@ - [`get_custody_chunk_bit`](#get_custody_chunk_bit) - [`get_chunk_bits_root`](#get_chunk_bits_root) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) - - [`get_validators_custody_reveal_period`](#get_validators_custody_reveal_period) + - [`get_reveal_period`](#get_reveal_period) - [`replace_empty_or_append`](#replace_empty_or_append) - [Per-block processing](#per-block-processing) - [Operations](#operations) @@ -224,7 +224,7 @@ Add the following fields to the end of the specified container objects. Fields w class Validator(Container): # next_custody_reveal_period is initialised to the custody period # (of the particular validator) in which the validator is activated - # = get_validators_custody_reveal_period(...) + # = get_reveal_period(...) next_custody_reveal_period: uint64 max_reveal_lateness: uint64 ``` @@ -299,17 +299,12 @@ def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorI return Epoch(next_period_start + CUSTODY_PERIOD_TO_RANDAO_PADDING) ``` -### `get_validators_custody_reveal_period` +### `get_reveal_period` ```python -def get_validators_custody_reveal_period(state: BeaconState, - validator_index: ValidatorIndex, - epoch: Epoch=None) -> int: +def get_reveal_period(state: BeaconState, validator_index: ValidatorIndex, epoch: Epoch=None) -> int: ''' - This function returns the reveal period for a given validator. - If no epoch is supplied, the current epoch is assumed. - Note: This function implicitly requires that validators are not removed from the - validator set in fewer than EPOCHS_PER_CUSTODY_PERIOD epochs + Return the reveal period for a given validator. ''' epoch = get_current_epoch(state) if epoch is None else epoch return (epoch + validator_index % EPOCHS_PER_CUSTODY_PERIOD) // EPOCHS_PER_CUSTODY_PERIOD @@ -340,17 +335,15 @@ Verify that `len(block.body.custody_key_reveals) <= MAX_CUSTODY_KEY_REVEALS`. For each `reveal` in `block.body.custody_key_reveals`, run the following function: ```python -def process_custody_key_reveal(state: BeaconState, - reveal: CustodyKeyReveal) -> None: +def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> None: """ Process ``CustodyKeyReveal`` operation. Note that this function mutates ``state``. """ - revealer = state.validators[reveal.revealer_index] epoch_to_sign = get_randao_epoch_for_custody_period(revealer.next_custody_reveal_period, reveal.revealed_index) - assert revealer.next_custody_reveal_period < get_validators_custody_reveal_period(state, reveal.revealed_index) + assert revealer.next_custody_reveal_period < get_reveal_period(state, reveal.revealed_index) # Revealed validator is active or exited, but not withdrawn assert is_slashable_validator(revealer, get_current_epoch(state)) @@ -368,11 +361,11 @@ def process_custody_key_reveal(state: BeaconState, ) # Decrement max reveal lateness if response is timely - if revealer.next_custody_reveal_period == get_validators_custody_reveal_period(state, reveal.revealer_index) - 2: + if revealer.next_custody_reveal_period == get_reveal_period(state, reveal.revealer_index) - 2: revealer.max_reveal_lateness -= MAX_REVEAL_LATENESS_DECREMENT revealer.max_reveal_lateness = max( revealer.max_reveal_lateness, - get_validators_custody_reveal_period(state, reveal.revealed_index) - revealer.next_custody_reveal_period + get_reveal_period(state, reveal.revealed_index) - revealer.next_custody_reveal_period ) # Process reveal @@ -394,13 +387,11 @@ Verify that `len(block.body.early_derived_secret_reveals) <= MAX_EARLY_DERIVED_S For each `reveal` in `block.body.early_derived_secret_reveals`, run the following function: ```python -def process_early_derived_secret_reveal(state: BeaconState, - reveal: EarlyDerivedSecretReveal) -> None: +def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerivedSecretReveal) -> None: """ Process ``EarlyDerivedSecretReveal`` operation. Note that this function mutates ``state``. """ - revealed_validator = state.validators[reveal.revealed_index] derived_secret_location = reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS @@ -470,8 +461,7 @@ Verify that `len(block.body.custody_chunk_challenges) <= MAX_CUSTODY_CHUNK_CHALL For each `challenge` in `block.body.custody_chunk_challenges`, run the following function: ```python -def process_chunk_challenge(state: BeaconState, - challenge: CustodyChunkChallenge) -> None: +def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: # Verify the attestation validate_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) # Verify it is not too late to challenge @@ -514,59 +504,41 @@ Verify that `len(block.body.custody_bit_challenges) <= MAX_CUSTODY_BIT_CHALLENGE For each `challenge` in `block.body.custody_bit_challenges`, run the following function: ```python -def process_bit_challenge(state: BeaconState, - challenge: CustodyBitChallenge) -> None: +def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> None: + attestation = challenge.attestation + epoch = slot_to_epoch(attestation.data.slot) + shard = attestation.data.crosslink.shard # Verify challenge signature challenger = state.validators[challenge.challenger_index] - assert bls_verify( - pubkey=challenger.pubkey, - message_hash=signing_root(challenge), - signature=challenge.signature, - domain=get_domain(state, DOMAIN_CUSTODY_BIT_CHALLENGE, get_current_epoch(state)), - ) + domain = get_domain(state, DOMAIN_CUSTODY_BIT_CHALLENGE, get_current_epoch(state)) + assert bls_verify(challenger.pubkey, signing_root(challenge), challenge.signature, domain) + # Verify challenger is slashable assert is_slashable_validator(challenger, get_current_epoch(state)) - - # Verify the attestation - attestation = challenge.attestation + # Verify attestation validate_indexed_attestation(state, convert_to_indexed(state, attestation)) - # Verify the attestation is eligible for challenging + # Verify attestation is eligible for challenging responder = state.validators[challenge.responder_index] - assert (slot_to_epoch(attestation.data.slot) + responder.max_reveal_lateness <= - get_validators_custody_reveal_period(state, challenge.responder_index)) - - # Verify the responder participated in the attestation + assert epoch + responder.max_reveal_lateness <= get_reveal_period(state, challenge.responder_index) + # Verify responder participated in the attestation attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) assert challenge.responder_index in attesters - - # A validator can be the challenger for at most one challenge at a time + # Verifier challenger is not already challenging for record in state.custody_bit_challenge_records: assert record.challenger_index != challenge.challenger_index - - # Verify the responder is a valid custody key + # Verify the responder custody key epoch_to_sign = get_randao_epoch_for_custody_period( - get_validators_custody_reveal_period( - state, - challenge.responder_index, - epoch=slot_to_epoch(attestation.data.slot)), - challenge.responder_index + get_reveal_period(state, challenge.responder_index, epoch), + challenge.responder_index, ) - assert bls_verify( - pubkey=responder.pubkey, - message_hash=hash_tree_root(epoch_to_sign), - signature=challenge.responder_key, - domain=get_domain( - state=state, - domain_type=DOMAIN_RANDAO, - message_epoch=epoch_to_sign, - ), - ) - + domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) + assert bls_verify(responder.pubkey, hash_tree_root(epoch_to_sign), challenge.responder_key, domain) # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(challenge.responder_index)) + committee = get_crosslink_committee(state, epoch, shard) + custody_bit = get_bitfield_bit(attestation.custody_bitfield, committee.index(challenge.responder_index)) assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( @@ -581,7 +553,6 @@ def process_bit_challenge(state: BeaconState, ) replace_empty_or_append(state.custody_bit_challenge_records, new_record) state.custody_challenge_index += 1 - # Postpone responder withdrawability responder.withdrawable_epoch = FAR_FUTURE_EPOCH ``` @@ -593,8 +564,7 @@ Verify that `len(block.body.custody_responses) <= MAX_CUSTODY_RESPONSES`. For each `response` in `block.body.custody_responses`, run the following function: ```python -def process_custody_response(state: BeaconState, - response: CustodyResponse) -> None: +def process_custody_response(state: BeaconState, response: CustodyResponse) -> None: chunk_challenge = next((record for record in state.custody_chunk_challenge_records if record.challenge_index == response.challenge_index), None) if chunk_challenge is not None: @@ -682,7 +652,7 @@ Run `process_reveal_deadlines(state)` immediately after `process_registry_update def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validators): deadline = validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) - if get_validators_custody_reveal_period(state, ValidatorIndex(index)) > deadline: + if get_reveal_period(state, ValidatorIndex(index)) > deadline: slash_validator(state, ValidatorIndex(index)) ``` From 128bbbc665c8eabe10640c019ccb0765c5be8f11 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:27:59 +0200 Subject: [PATCH 255/405] fix slicing, and support partial slice bounds --- specs/core/0_beacon-chain.md | 2 +- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b9c3f5f0c5..887ae68020 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,7 +1294,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:] == state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 91d3ff4f2f..d87a223994 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -325,7 +325,7 @@ def __getitem__(self, k) -> SSZValue: def __setitem__(self, k, v): if type(k) == slice: - if k.start < 0 or k.stop > len(self): + if (k.start is not None and k.start < 0) or (k.stop is not None and k.stop > len(self)): raise IndexError(f"cannot set item in type {self.__class__}" f" at out of bounds slice {k} (to {v}, bound: {len(self)})") super().__setitem__(k, [coerce_type_maybe(x, self.__class__.elem_type) for x in v]) From 25db39755047ce6ae28332d90080c502ce899423 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:34:31 +0200 Subject: [PATCH 256/405] fix line length lint problem in checkpoint --- specs/core/0_beacon-chain.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 887ae68020..ba5b591d1f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1298,11 +1298,13 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, + root=get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From 5a8f3e495a8be99d6f6ae821c141e1b881d44d92 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 11:10:17 -0600 Subject: [PATCH 257/405] set committees root for next epoch rather tahn ACTIVaTION_EXIT_DELAY in the future --- specs/core/0_beacon-chain.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 842b16b986..8f55addbc0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -698,6 +698,9 @@ def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: ```python def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: + """ + Return the start shard of the 0th committee in an epoch. + """ assert epoch <= get_current_epoch(state) + 1 check_epoch = Epoch(get_current_epoch(state) + 1) shard = Shard((state.start_shard + get_shard_delta(state, get_current_epoch(state))) % SHARD_COUNT) @@ -767,7 +770,7 @@ def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: committees[shard].pubkeys.append(validator.pubkey) compact_balance = validator.effective_balance // EFFECTIVE_BALANCE_INCREMENT # `index` (top 6 bytes) + `slashed` (16th bit) + `compact_balance` (bottom 15 bits) - compact_validator = uint64(index << 16 + validator.slashed << 15 + compact_balance) + compact_validator = uint64((index << 16) + (validator.slashed << 15) + compact_balance) committees[shard].compact_validators.append(compact_validator) return hash_tree_root(Vector[CompactCommittee, SHARD_COUNT](committees)) ``` @@ -782,7 +785,7 @@ def generate_seed(state: BeaconState, """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + - state.compact_committees_roots[epoch] + + state.compact_committees_roots[epoch % EPOCHS_PER_HISTORICAL_VECTOR] + int_to_bytes(epoch, length=32) ) ``` @@ -1546,7 +1549,7 @@ def process_final_updates(state: BeaconState) -> None: state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT) # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR - state.compact_committees_roots[index_root_position] = get_compact_committees_root(state, next_epoch + ACTIVATION_EXIT_DELAY) + state.compact_committees_roots[index_root_position] = get_compact_committees_root(state, next_epoch) # Set total slashed balances state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] From b40e2284a0f601dd6f6ac80f833d4036c244ecfb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 11:20:24 -0600 Subject: [PATCH 258/405] use active index root for generate seed mix in --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8f55addbc0..80977c8f75 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -785,7 +785,7 @@ def generate_seed(state: BeaconState, """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + - state.compact_committees_roots[epoch % EPOCHS_PER_HISTORICAL_VECTOR] + + hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, epoch))) + int_to_bytes(epoch, length=32) ) ``` From 9993a2879629b7e27f97086ff3d5d70cc3da6a2f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 11:26:05 -0600 Subject: [PATCH 259/405] lint --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 80977c8f75..490ac0c96f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -764,7 +764,7 @@ def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: committees = [CompactCommittee() for _ in range(SHARD_COUNT)] start_shard = get_epoch_start_shard(state, epoch) for committee_number in range(get_epoch_committee_count(state, epoch)): - shard = (start_shard + committee_number) % SHARD_COUNT + shard = Shard((start_shard + committee_number) % SHARD_COUNT) for index in get_crosslink_committee(state, epoch, shard): validator = state.validators[index] committees[shard].pubkeys.append(validator.pubkey) @@ -1535,7 +1535,7 @@ def process_slashings(state: BeaconState) -> None: ```python def process_final_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) - next_epoch = current_epoch + 1 + next_epoch = Shard(current_epoch + 1) # Reset eth1 data votes if (state.slot + 1) % SLOTS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] From 4dc526fbb7d619d0675950b703dcb30c5eb22836 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 29 Jun 2019 02:16:16 +0800 Subject: [PATCH 260/405] In the end, `get_merkle_root` is back --- scripts/build_spec.py | 6 ++- specs/core/0_beacon-chain.md | 44 ++++++++++++++++--- .../test/genesis/test_genesis_trigger.py | 9 ++-- .../pyspec/eth2spec/test/helpers/deposits.py | 17 +++---- .../pyspec/eth2spec/utils/merkle_minimal.py | 2 +- .../eth2spec/utils/test_merkle_minimal.py | 5 ++- 6 files changed, 59 insertions(+), 24 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 99c5cd69de..1bdebd1302 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -11,7 +11,8 @@ ) -PHASE0_IMPORTS = '''from typing import ( +PHASE0_IMPORTS = '''from math import log2 +from typing import ( Any, Callable, Dict, Set, Sequence, Tuple, ) @@ -36,7 +37,8 @@ from eth2spec.utils.hash_function import hash ''' -PHASE1_IMPORTS = '''from typing import ( +PHASE1_IMPORTS = '''from math import log2 +from typing import ( Any, Callable, Dict, Optional, Set, Sequence, MutableSequence, Tuple, ) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ea99bb4730..25a7340d3f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -51,6 +51,8 @@ - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) - [`signing_root`](#signing_root) + - [`calc_merkle_tree_from_leaves`](#calc_merkle_tree_from_leaves) + - [`get_merkle_root`](#get_merkle_root) - [`bls_domain`](#bls_domain) - [`slot_to_epoch`](#slot_to_epoch) - [`get_previous_epoch`](#get_previous_epoch) @@ -558,6 +560,33 @@ The `hash` function is SHA256. `def signing_root(object: Container) -> Hash` is a function for computing signing messages, as defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers). +### `calc_merkle_tree_from_leaves` + +```python +zerohashes = [ZERO_HASH] +for layer in range(1, 100): + zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1])) +def calc_merkle_tree_from_leaves(values: Sequence[Hash], layer_count: int=32) -> Sequence[Sequence[Hash]]: + values = list(values) + tree = [values[::]] + for h in range(layer_count): + if len(values) % 2 == 1: + values.append(zerohashes[h]) + values = [hash(values[i] + values[i + 1]) for i in range(0, len(values), 2)] + tree.append(values[::]) + return tree +``` + +### `get_merkle_root` + +```python +def get_merkle_root(values: Sequence[Hash], pad_to: int=1) -> Hash: + layer_count = int(log2(pad_to)) + if len(values) == 0: + return zerohashes[layer_count] + return calc_merkle_tree_from_leaves(values, layer_count)[-1][0] +``` + ### `bls_domain` ```python @@ -1144,10 +1173,10 @@ def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], # Process deposits state = BeaconState() - for i, deposit in enumerate(deposits): - state.eth1_data.deposit_root = hash_tree_root( - Vector[DepositData, len(deposits)](list(map(lambda deposit: deposit.data, deposits[:i]))) - ) + for deposit_index, deposit in enumerate(deposits): + leaves = [hash_tree_root(d.data) for d in deposits[:deposit_index + 1]] + state.eth1_data.deposit_root = get_merkle_root(leaves, 2**DEPOSIT_CONTRACT_TREE_DEPTH) + state.eth1_deposit_index = deposit_index process_deposit(state, deposit) # Count active validators at genesis @@ -1173,9 +1202,14 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth ) # Process genesis deposits - for deposit in deposits: + for deposit_index, deposit in enumerate(deposits): + leaves = [hash_tree_root(d.data) for d in deposits[:deposit_index + 1]] + state.eth1_data.deposit_root = get_merkle_root(leaves, 2**DEPOSIT_CONTRACT_TREE_DEPTH) + state.eth1_deposit_index = deposit_index process_deposit(state, deposit) + assert state.eth1_data.deposit_root == eth1_data.deposit_root + # Process genesis activations for validator in state.validators: if validator.effective_balance == MAX_EFFECTIVE_BALANCE: diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py b/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py index 9425a1750d..783def3bad 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py @@ -8,12 +8,11 @@ @spectest_with_bls_switch def test_is_genesis_trigger_false(spec): deposit_count = 2 - genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + genesis_deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) genesis_time = 1546300800 yield "deposits", genesis_deposits yield "time", genesis_time - yield "deposit_root", deposit_root is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time) assert is_triggered is False @@ -25,13 +24,12 @@ def test_is_genesis_trigger_false(spec): @spectest_with_bls_switch def test_is_genesis_trigger_true(spec): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT - genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + genesis_deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) SECONDS_PER_DAY = 86400 genesis_time = 1578009600 - 2 * SECONDS_PER_DAY yield "deposits", genesis_deposits yield "time", genesis_time - yield "deposit_root", deposit_root is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time) assert is_triggered is True @@ -43,11 +41,10 @@ def test_is_genesis_trigger_true(spec): @spectest_with_bls_switch def test_is_genesis_trigger_not_enough_balance(spec): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT - genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE - 1) + genesis_deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE - 1) genesis_time = 1546300800 yield "deposits", genesis_deposits yield "time", genesis_time - yield "deposit_root", deposit_root is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time) assert is_triggered is False diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index bda4f1699c..60ff2a3db3 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -41,7 +41,9 @@ def build_deposit(spec, amount, withdrawal_credentials, signed): - deposit_data = build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=state, signed=signed) + deposit_data = build_deposit_data( + spec, pubkey, privkey, amount, withdrawal_credentials, state=state, signed=signed, + ) item = deposit_data.hash_tree_root() index = len(deposit_data_leaves) @@ -63,6 +65,7 @@ def build_deposit(spec, def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False): genesis_deposit_data_list = [] deposit_data_leaves = [] + genesis_deposits = [] for validator_index in range(genesis_validator_count): pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] @@ -79,13 +82,11 @@ def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False deposit_data_leaves.append(item) genesis_deposit_data_list.append(deposit_data) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - - genesis_deposits = list( - spec.Deposit(proof=list(get_merkle_proof(tree, item_index=index)), data=deposit_data) - for index, deposit_data in enumerate(genesis_deposit_data_list) - ) + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) + root = get_merkle_root((tuple(deposit_data_leaves)), 2**32) + genesis_deposits.append( + spec.Deposit(proof=list(get_merkle_proof(tree, item_index=validator_index)), data=deposit_data) + ) return genesis_deposits, root diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index e3e81f3a55..e9416ea051 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -1,5 +1,5 @@ from .hash_function import hash -from math import log2 +from math import log2 ZERO_BYTES32 = b'\x00' * 32 diff --git a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py index 1c72a649b6..f1ed768e63 100644 --- a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py @@ -1,5 +1,5 @@ import pytest -from .merkle_minimal import zerohashes, merkleize_chunks +from .merkle_minimal import zerohashes, merkleize_chunks, get_merkle_root from .hash_function import hash @@ -53,6 +53,7 @@ def z(i: int) -> bytes: 'depth,count,pow2,value', cases, ) -def test_merkleize_chunks(depth, count, pow2, value): +def test_merkleize_chunks_and_get_merkle_root(depth, count, pow2, value): chunks = [e(i) for i in range(count)] assert merkleize_chunks(chunks, pad_to=pow2) == value + assert get_merkle_root(chunks, pad_to=pow2) == value From cb2bfd67dc18d19f69f293c424370437ec08ae0e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 29 Jun 2019 02:27:48 +0800 Subject: [PATCH 261/405] Apply Proto's feedback of list(map(...)) --- specs/core/0_beacon-chain.md | 8 ++++---- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 25a7340d3f..9e59212fd0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1173,9 +1173,9 @@ def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], # Process deposits state = BeaconState() + leaves = list(map(lambda deposit: hash_tree_root(deposit.data), deposits)) for deposit_index, deposit in enumerate(deposits): - leaves = [hash_tree_root(d.data) for d in deposits[:deposit_index + 1]] - state.eth1_data.deposit_root = get_merkle_root(leaves, 2**DEPOSIT_CONTRACT_TREE_DEPTH) + state.eth1_data.deposit_root = get_merkle_root(leaves[:deposit_index + 1], 2**DEPOSIT_CONTRACT_TREE_DEPTH) state.eth1_deposit_index = deposit_index process_deposit(state, deposit) @@ -1202,9 +1202,9 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth ) # Process genesis deposits + leaves = list(map(lambda deposit: hash_tree_root(deposit.data), deposits)) for deposit_index, deposit in enumerate(deposits): - leaves = [hash_tree_root(d.data) for d in deposits[:deposit_index + 1]] - state.eth1_data.deposit_root = get_merkle_root(leaves, 2**DEPOSIT_CONTRACT_TREE_DEPTH) + state.eth1_data.deposit_root = get_merkle_root(leaves[:deposit_index + 1], 2**DEPOSIT_CONTRACT_TREE_DEPTH) state.eth1_deposit_index = deposit_index process_deposit(state, deposit) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 60ff2a3db3..529dd8c128 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -51,8 +51,6 @@ def build_deposit(spec, tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) root = get_merkle_root((tuple(deposit_data_leaves))) proof = list(get_merkle_proof(tree, item_index=index)) - assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) - deposit = spec.Deposit( proof=list(proof), index=index, From 7bc6a46eaaecfe73dcdbc99067f8f7ae77d7c220 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 29 Jun 2019 02:51:24 +0800 Subject: [PATCH 262/405] Fix `build_deposit` --- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 529dd8c128..d7df156a37 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -48,20 +48,20 @@ def build_deposit(spec, item = deposit_data.hash_tree_root() index = len(deposit_data_leaves) deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves), spec.DEPOSIT_CONTRACT_TREE_DEPTH) + root = get_merkle_root((tuple(deposit_data_leaves)), 2**spec.DEPOSIT_CONTRACT_TREE_DEPTH) proof = list(get_merkle_proof(tree, item_index=index)) deposit = spec.Deposit( proof=list(proof), index=index, data=deposit_data, ) + assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) return deposit, root, deposit_data_leaves def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False): - genesis_deposit_data_list = [] deposit_data_leaves = [] genesis_deposits = [] for validator_index in range(genesis_validator_count): @@ -78,10 +78,9 @@ def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False sign_deposit_data(spec, deposit_data, privkey) # state=None item = deposit_data.hash_tree_root() deposit_data_leaves.append(item) - genesis_deposit_data_list.append(deposit_data) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves)), 2**32) + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves), spec.DEPOSIT_CONTRACT_TREE_DEPTH) + root = get_merkle_root((tuple(deposit_data_leaves)), 2**spec.DEPOSIT_CONTRACT_TREE_DEPTH) genesis_deposits.append( spec.Deposit(proof=list(get_merkle_proof(tree, item_index=validator_index)), data=deposit_data) ) From fa84c496592d722b544c775065b9258cd22dbd73 Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:23:34 +0100 Subject: [PATCH 263/405] Update specs/core/0_beacon-chain.md Co-Authored-By: Danny Ryan --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cb50e2f864..6f28c0e3d3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1295,7 +1295,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[1:] = state.justification_bits[:-1] state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: From 6a2d2c84a8f3fa3ea846d17dff5d10ebfb3508e6 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 20:49:57 +0100 Subject: [PATCH 264/405] Bitlist for attestation doc --- specs/validator/0_beacon-chain-validator.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ad01476e98..7d5630c7be 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -331,17 +331,13 @@ Set `attestation.data = attestation_data` where `attestation_data` is the `Attes ##### Aggregation bits -* Let `aggregation_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Let `index_into_committee` be the index into the validator's `committee` at which `validator_index` is located. -* Set `aggregation_bits[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. -* Set `attestation.aggregation_bits = aggregation_bits`. +* Let `attestation.aggregation_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` where the bits at the index in the aggregated validator's `committee` is set to `0b1`. *Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. ##### Custody bits -* Let `custody_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Set `attestation.custody_bits = custody_bits`. +* Let `attestation.custody_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` filled with zeros of length `len(committee)`. *Note*: This is a stub for Phase 0. From 4dcb47e393ca68ab9e32af8ba913eaf829d6cb69 Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:52:06 +0100 Subject: [PATCH 265/405] Update test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py Co-Authored-By: Danny Ryan --- .../test/phase_0/block_processing/test_process_attestation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 9167240016..b114c85e9e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -279,7 +279,7 @@ def test_inconsistent_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - custody_bits = deepcopy(attestation.aggregation_bits) + custody_bits = attestation.aggregation_bits[:] custody_bits.append(False) attestation.custody_bits = custody_bits From be04eb2673563b6250cc5637777badc76913ec5f Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:52:16 +0100 Subject: [PATCH 266/405] Change copy style, and remove deepcopy import Update test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py Co-Authored-By: Danny Ryan --- .../test/phase_0/block_processing/test_process_attestation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index b114c85e9e..41207fdf4b 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -1,5 +1,3 @@ -from copy import deepcopy - from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases from eth2spec.test.helpers.attestations import ( get_valid_attestation, @@ -295,7 +293,7 @@ def test_non_empty_custody_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bits = deepcopy(attestation.aggregation_bits) + attestation.custody_bits = attestation.aggregation_bits[:] sign_attestation(spec, state, attestation) From 4f31207b7f2cd4ba2bf21976b6e8bc6644e24202 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 22:45:20 +0200 Subject: [PATCH 267/405] reword merkleize with limit / length --- specs/simple-serialize.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 3fba59fbad..7076b6410f 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -171,21 +171,21 @@ Note that deserialization requires hardening against invalid inputs. A non-exhau We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. -* `merkleize(data, pad_to)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. - There merkleization depends on the input length: - - `0` chunks: A chunk filled with zeroes. - - `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. - - `1+` chunks: pad to the next power of 2, merkleize as binary tree. - - with `pad_to` set: pad the `data` with zeroed chunks to this power of two (virtually for memory efficiency). +* `next_pow_of_two(i)`: get the next power of 2 of `i`, if not already a power of 2, with 0 mapping to 1. Examples: `0->1, 1->1, 2->2, 3->4, 4->4, 6->8, 9->16` +* `merkleize(data, pad_for)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. + The merkleization depends on the effective input, which can be padded: if `pad_for=L`, then pad the `data` with zeroed chunks to `next_pow_of_two(L)` (virtually for memory efficiency). + Then, merkleize the chunks (empty input is padded to 1 zero chunk): + - If `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. + - If `> 1` chunks: pad to `next_pow_of_two(len(chunks))`, merkleize as binary tree. * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(value), pad_to=N), len(value))` if `value` is a list of basic objects. `N` is the amount of chunks needed for the list limit (*this depends on element-type size*). +* `mix_in_length(merkleize(pack(value), pad_for=(N * elem_size / BYTES_PER_CHUNK)), len(value))` if `value` is a list of basic objects. * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_to=N), len(value))` if `value` is a list of composite objects. `N` is the list limit (*1 chunk per element*). +* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_for=N), len(value))` if `value` is a list of composite objects. * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type ### Merkleization of `Bitvector[N]` From d5c2ecb6f73a3c2ea673332d45739704b6af9b29 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 15:44:26 -0600 Subject: [PATCH 268/405] remove local notes files --- motes.md | 42 ------------------------------------------ notes.txt | 1 - 2 files changed, 43 deletions(-) delete mode 100644 motes.md delete mode 100644 notes.txt diff --git a/motes.md b/motes.md deleted file mode 100644 index e01ba19b77..0000000000 --- a/motes.md +++ /dev/null @@ -1,42 +0,0 @@ -* `BLS_WITHDRAWAL_PREFIX` - * Why int rather than bytes? -* `MIN_SEED_LOOKAHEAD` - * Is this actually tunable? - * If so, what are the reprecussions? -* `ACTIVATION_EXIT_DELAY` - * Reaquaint with purpose -* AttesterSlashings - * `MAX_ATTESTER_SLASHINGS` is 1. - * Are there scenarios in which validators can create more effective slashable - messages than can be included on chain? For example, Validators split up to - create double attestations for checkpoints but different (junk) crosslink - data to prevent them from being aggregatable to the fullest - * Max is for block size, no? -* Signature domains - * Max 4byte ints -* `Version` not defined in one of the lists of custom types (2!!). ensure in spec -* `PendingAttestation` - * Don't think `proposer_index` is actually necessary here because effective - balance is stable until end of epoch so can do dynamic lookups -* is_genesis_trigger - * only run at ends of blocks to preserve invariant that eth1data.deposit_root - is the deposit root at the _end_ of an eth1 block -* `Attestation` - * why bitfields not together? -* `Transfer` - * replay mechanism... say the slot gets missed and you sign another transfer - * in a fork you could include both transfers -* `get_previous_epoch` - * do a once over on the genesis stuff -* `get_epoch_start_shard` - * checking next hinges upon the fact that the validator set for the next - epoch is 100% known at the current epoch. Ensure this is the case -* `get_block_root_at_slot` .. `generate_seed` can be bade into one line - function signatures -* `get_shuffled_index` - * I think it should be maybe `assert index_count <= VALIDATOR_REGISTRY_LIMIT` - * is the `2**40` special for security of alg? probably. - - -pubkey/privkey g1 vs g2 - diff --git a/notes.txt b/notes.txt deleted file mode 100644 index 421ad77509..0000000000 --- a/notes.txt +++ /dev/null @@ -1 +0,0 @@ -* `BLS_WITHDRAWAL_PREFIX` -- From 3a60f64b9275f0dea931bf240eba2d1f083a22e8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 01:22:29 +0200 Subject: [PATCH 269/405] refactor finalization test helper func --- ..._process_justification_and_finalization.py | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 95f00edd4e..f0fe35e8bb 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -1,3 +1,4 @@ +import math from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( run_epoch_processing_with @@ -8,41 +9,46 @@ def run_process_just_and_fin(spec, state): yield from run_epoch_processing_with(spec, state, 'process_justification_and_finalization') -def get_committee_size(spec, state, slot): +def get_shards_for_slot(spec, state, slot): epoch = spec.slot_to_epoch(slot) epoch_start_shard = spec.get_epoch_start_shard(state, epoch) committees_per_slot = spec.get_epoch_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT - committee_index = (shard + spec.SHARD_COUNT - spec.get_epoch_start_shard(state, epoch)) % spec.SHARD_COUNT - committee_count = spec.get_epoch_committee_count(state, epoch) - indices = spec.get_active_validator_indices(state, epoch) + return [shard + i for i in range(committees_per_slot)] + + +def get_committee_size(spec, epoch_start_shard, shard, committee_count, indices): + committee_index = (shard + spec.SHARD_COUNT - epoch_start_shard) % spec.SHARD_COUNT start = (len(indices) * committee_index) // committee_count end = (len(indices) * (committee_index + 1)) // committee_count size = end - start - return size + return size, -def add_mock_attestations(spec, state, target_epoch, att_count, att_ratio): +def add_mock_attestations(spec, state, epoch, att_count, att_ratio): # we must be at the end of the epoch assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 previous_epoch = spec.get_previous_epoch(state) current_epoch = spec.get_current_epoch(state) - if current_epoch == target_epoch: + if current_epoch == epoch: attestations = state.current_epoch_attestations - elif previous_epoch == target_epoch: + elif previous_epoch == epoch: attestations = state.previous_epoch_attestations else: - raise Exception(f"cannot target epoch ${target_epoch} from epoch ${current_epoch}") + raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") + committee_count = spec.get_epoch_committee_count(state, epoch) + indices = spec.get_active_validator_indices(state, epoch) + epoch_start_shard = spec.get_epoch_start_shard(state, epoch) total = 0 - while total < att_count: - for i in range(spec.SLOTS_PER_EPOCH): - size = get_committee_size(spec, state, state.slot + i) + for i in range(spec.SLOTS_PER_EPOCH): + for shard in get_shards_for_slot(spec, state, state.slot + i): + size = get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. - attesting_count = int(size * att_ratio) + attesting_count = math.ceil(size * att_ratio) aggregation_bitfield = ((1 << attesting_count) - 1).to_bytes(length=((size + 7) // 8), byteorder='big') attestations.append(spec.PendingAttestation( @@ -54,16 +60,20 @@ def add_mock_attestations(spec, state, target_epoch, att_count, att_ratio): target_root=b'\xbb' * 32, crosslink=spec.Crosslink() ), - inclusion_delay=0, + inclusion_delay=1, )) total += 1 + if total >= att_count: + return + + raise Exception(f"could not fill state with {att_count} attestations for epoch {epoch}") @with_all_phases @spec_state_test def test_rule_1(spec, state): - previous_epoch = spec.get_previous_epoch(state) - current_epoch = spec.get_current_epoch(state) + # previous_epoch = spec.get_previous_epoch(state) + # current_epoch = spec.get_current_epoch(state) # TODO # add_mock_attestations(spec, state, ...) @@ -71,4 +81,4 @@ def test_rule_1(spec, state): # set their balances # yield from run_process_just_and_fin(spec, state) # check finalization - + pass From 518db42de75b9714347cd4ad2d549691f28afe6b Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 03:19:30 +0200 Subject: [PATCH 270/405] fix attestation tests to work with checkpoints --- .../test/phase_0/block_processing/test_process_attestation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index a1f87f059c..f083a07f49 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -205,7 +205,7 @@ def test_future_target_epoch(spec, state): state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.target_epoch = spec.get_current_epoch(state) + 1 # target epoch will be too new to handle + attestation.data.target.epoch = spec.get_current_epoch(state) + 1 # target epoch will be too new to handle sign_attestation(spec, state, attestation) yield from run_attestation_processing(spec, state, attestation, False) From efd9d173d786c87ad4daf4710615cd739ee2d584 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 03:19:57 +0200 Subject: [PATCH 271/405] update attester slashings processing tests --- .../test_process_slashings.py | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py index f1a23326b3..7be23a04d0 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py @@ -17,8 +17,8 @@ def slash_validators(spec, state, indices, out_epochs): v.withdrawable_epoch = out_epoch total_slashed_balance += v.effective_balance - state.slashed_balances[ - spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR + state.slashings[ + spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHINGS_VECTOR ] = total_slashed_balance @@ -26,13 +26,13 @@ def slash_validators(spec, state, indices, out_epochs): @spec_state_test def test_max_penalties(spec, state): slashed_count = (len(state.validators) // 3) + 1 - out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHINGS_VECTOR // 2) slashed_indices = list(range(slashed_count)) slash_validators(spec, state, slashed_indices, [out_epoch] * slashed_count) total_balance = spec.get_total_active_balance(state) - total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] + total_penalties = sum(state.slashings) assert total_balance // 3 <= total_penalties @@ -44,28 +44,30 @@ def test_max_penalties(spec, state): @with_all_phases @spec_state_test -def test_min_penalties(spec, state): - # run_epoch_processing_to(spec, state, 'process_slashings', exclusive=True) - +def test_small_penalty(spec, state): # Just the bare minimum for this one validator - pre_balance = state.balances[0] = state.validators[0].effective_balance = spec.EJECTION_BALANCE + state.balances[0] = state.validators[0].effective_balance = spec.EJECTION_BALANCE # All the other validators get the maximum. for i in range(1, len(state.validators)): state.validators[i].effective_balance = state.balances[i] = spec.MAX_EFFECTIVE_BALANCE - out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHINGS_VECTOR // 2) slash_validators(spec, state, [0], [out_epoch]) total_balance = spec.get_total_active_balance(state) - total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] + total_penalties = sum(state.slashings) - # we are testing the minimum here, i.e. get slashed (effective_balance / MIN_SLASHING_PENALTY_QUOTIENT) - assert total_penalties * 3 / total_balance < 1 / spec.MIN_SLASHING_PENALTY_QUOTIENT + assert total_balance // 3 > total_penalties - yield from run_process_slashings(spec, state) + run_epoch_processing_to(spec, state, 'process_slashings') + pre_slash_balances = list(state.balances) + yield 'pre', state + spec.process_slashings(state) + yield 'post', state - assert state.balances[0] == pre_balance - (pre_balance // spec.MIN_SLASHING_PENALTY_QUOTIENT) + assert state.balances[0] == pre_slash_balances[0] - (state.validators[0].effective_balance + * 3 * total_penalties // total_balance) @with_all_phases @@ -75,14 +77,13 @@ def test_scaled_penalties(spec, state): state.slot = spec.SLOTS_PER_EPOCH # Also mock some previous slashings, so that we test to have the delta in the penalties computation. - for i in range(spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR): - state.slashed_balances[i] = spec.MAX_EFFECTIVE_BALANCE * 3 - - # Mock the very last one (which is to be used for the delta balance computation) to be different. - # To enforce the client test runner to correctly get this one from the array, not the others. - prev_penalties = state.slashed_balances[ - (spec.get_current_epoch(state) + 1) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR - ] = spec.MAX_EFFECTIVE_BALANCE * 2 + base = spec.EJECTION_BALANCE + incr = spec.EFFECTIVE_BALANCE_INCREMENT + # Just add some random slashings. non-zero slashings are at least the minimal effective balance. + state.slashings[0] = base + (incr * 12) + state.slashings[4] = base + (incr * 3) + state.slashings[5] = base + (incr * 6) + state.slashings[spec.EPOCHS_PER_SLASHINGS_VECTOR - 1] = base + (incr * 7) slashed_count = len(state.validators) // 4 @@ -90,13 +91,17 @@ def test_scaled_penalties(spec, state): # make the balances non-uniform. # Otherwise it would just be a simple 3/4 balance slashing. Test the per-validator scaled penalties. + diff = spec.MAX_EFFECTIVE_BALANCE - base + increments = diff // incr for i in range(10): - state.validators[i].effective_balance += spec.EFFECTIVE_BALANCE_INCREMENT * 4 - state.balances[i] += spec.EFFECTIVE_BALANCE_INCREMENT * 4 + state.validators[i].effective_balance = base + (incr * (i % increments)) + assert state.validators[i].effective_balance <= spec.MAX_EFFECTIVE_BALANCE + # add/remove some, see if balances different than the effective balances are picked up + state.balances[i] = state.validators[i].effective_balance + i - 5 total_balance = spec.get_total_active_balance(state) - out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHINGS_VECTOR // 2) slashed_indices = list(range(slashed_count)) @@ -112,8 +117,7 @@ def test_scaled_penalties(spec, state): spec.process_slashings(state) yield 'post', state - total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] - total_penalties -= prev_penalties + total_penalties = sum(state.slashings) for i in slashed_indices: v = state.validators[i] From afb34c71e6aeaa3b088b46b685f03a9dfaba8a9b Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 03:22:14 +0200 Subject: [PATCH 272/405] fix broken block test for checkpoint use --- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index e56baee8c4..019563978d 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -143,7 +143,7 @@ def test_empty_epoch_transition_not_finalizing(spec, state): yield 'post', state assert state.slot == block.slot - assert state.finalized_epoch < spec.get_current_epoch(state) - 4 + assert state.finalized_checkpoint.epoch < spec.get_current_epoch(state) - 4 for index in range(len(state.validators)): assert state.balances[index] < pre_balances[index] From cfbdee709bd408572d10c70f2a7adb3dd72220aa Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 05:04:25 +0200 Subject: [PATCH 273/405] finalization testing --- ..._process_justification_and_finalization.py | 228 ++++++++++++++++-- 1 file changed, 205 insertions(+), 23 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index f0fe35e8bb..0afa4f307a 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -22,10 +22,10 @@ def get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) start = (len(indices) * committee_index) // committee_count end = (len(indices) * (committee_index + 1)) // committee_count size = end - start - return size, + return size -def add_mock_attestations(spec, state, epoch, att_count, att_ratio): +def add_mock_attestations(spec, state, epoch, att_ratio, source, target): # we must be at the end of the epoch assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 @@ -42,9 +42,9 @@ def add_mock_attestations(spec, state, epoch, att_count, att_ratio): committee_count = spec.get_epoch_committee_count(state, epoch) indices = spec.get_active_validator_indices(state, epoch) epoch_start_shard = spec.get_epoch_start_shard(state, epoch) - total = 0 - for i in range(spec.SLOTS_PER_EPOCH): - for shard in get_shards_for_slot(spec, state, state.slot + i): + epoch_start_slot = spec.get_epoch_start_slot(epoch) + for slot in range(epoch_start_slot, epoch_start_slot + spec.SLOTS_PER_EPOCH): + for shard in get_shards_for_slot(spec, state, slot): size = get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. @@ -55,30 +55,212 @@ def add_mock_attestations(spec, state, epoch, att_count, att_ratio): aggregation_bitfield=aggregation_bitfield, data=spec.AttestationData( beacon_block_root=b'\xaa' * 32, - source_epoch=0, - source_root=b'\xbb' * 32, - target_root=b'\xbb' * 32, + source=source, + target=source, crosslink=spec.Crosslink() ), inclusion_delay=1, )) - total += 1 - if total >= att_count: - return - raise Exception(f"could not fill state with {att_count} attestations for epoch {epoch}") + +def finalize_on_234(spec, state, epoch, support): + assert epoch > 4 + state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch + + # 43210 -- epochs ago + # 3210x -- justification bitfield indices + # 11*0. -- justification bitfield contents, . = this epoch, * is being justified now + # checkpoints for the epochs ago: + c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + # c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + + old_finalized = state.finalized_checkpoint + state.previous_justified_checkpoint = c4 + state.current_justified_checkpoint = c3 + bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + bits[3:4] = [1, 1] # mock 3rd and 4th latest epochs as justified + # mock the 2nd latest epoch as justifiable, with 4th as source + add_mock_attestations(spec, state, + epoch=epoch - 2, + att_ratio=support, + source=c4, + target=c2) + + # process! + yield from run_process_just_and_fin(spec, state) + + if support >= (2 / 3): + assert state.previous_justified_checkpoint == c3 # changed to old current + assert state.current_justified_checkpoint == c2 # changed to 2nd latest + assert state.finalized_checkpoint == c4 # finalized old previous justified epoch + else: + assert state.previous_justified_checkpoint == c3 # changed to old current + assert state.current_justified_checkpoint == c3 # still old current + assert state.finalized_checkpoint == old_finalized # no new finalized + + +def finalize_on_23(spec, state, epoch, support): + assert epoch > 3 + state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch + + # 43210 -- epochs ago + # 3210x -- justification bitfield indices + # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now + # checkpoints for the epochs ago: + # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + # c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + + old_finalized = state.finalized_checkpoint + state.previous_justified_checkpoint = c3 + state.current_justified_checkpoint = c3 + bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + bits[2] = 1 # mock 3rd latest epoch as justified + # mock the 2nd latest epoch as justifiable, with 3rd as source + add_mock_attestations(spec, state, + epoch=epoch - 2, + att_ratio=support, + source=c3, + target=c2) + + # process! + yield from run_process_just_and_fin(spec, state) + + if support >= (2 / 3): + assert state.previous_justified_checkpoint == c3 # changed to old current + assert state.current_justified_checkpoint == c2 # changed to 2nd latest + assert state.finalized_checkpoint == c3 # finalized old previous justified epoch + else: + assert state.previous_justified_checkpoint == c3 # changed to old current + assert state.current_justified_checkpoint == c3 # still old current + assert state.finalized_checkpoint == old_finalized # no new finalized + + +def finalize_on_123(spec, state, epoch, support): + assert epoch > 3 + state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch + + # 43210 -- epochs ago + # 3210x -- justification bitfield indices + # 011*. -- justification bitfield contents, . = this epoch, * is being justified now + # checkpoints for the epochs ago: + # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + + old_finalized = state.finalized_checkpoint + state.previous_justified_checkpoint = c3 + state.current_justified_checkpoint = c2 + bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + bits[1:2] = [1, 1] # mock 2rd and 3th latest epochs as justified + # mock the 1st latest epoch as justifiable, with 3rd as source + add_mock_attestations(spec, state, + epoch=epoch - 1, + att_ratio=support, + source=c3, + target=c1) + + # process! + yield from run_process_just_and_fin(spec, state) + + if support >= (2 / 3): + assert state.previous_justified_checkpoint == c2 # changed to old current + assert state.current_justified_checkpoint == c1 # changed to 1st latest + assert state.finalized_checkpoint == c2 # finalized old current + else: + assert state.previous_justified_checkpoint == c2 # changed to old current + assert state.current_justified_checkpoint == c2 # still old current + assert state.finalized_checkpoint == old_finalized # no new finalized + + +def finalize_on_12(spec, state, epoch, support): + assert epoch > 2 + state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch + + # 43210 -- epochs ago + # 3210 -- justification bitfield indices + # 001*. -- justification bitfield contents, . = this epoch, * is being justified now + # checkpoints for the epochs ago: + # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) + # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + + old_finalized = state.finalized_checkpoint + state.previous_justified_checkpoint = c2 + state.current_justified_checkpoint = c2 + bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + bits[3] = 1 # mock 3rd latest epoch as justified + # mock the 1st latest epoch as justifiable, with 2nd as source + add_mock_attestations(spec, state, + epoch=epoch - 1, + att_ratio=support, + source=c2, + target=c1) + + # process! + yield from run_process_just_and_fin(spec, state) + + if support >= (2 / 3): + assert state.previous_justified_checkpoint == c2 # changed to old current + assert state.current_justified_checkpoint == c1 # changed to 1st latest + assert state.finalized_checkpoint == c2 # finalized previous justified epoch + else: + assert state.previous_justified_checkpoint == c2 # changed to old current + assert state.current_justified_checkpoint == c2 # still old current + assert state.finalized_checkpoint == old_finalized # no new finalized @with_all_phases @spec_state_test -def test_rule_1(spec, state): - # previous_epoch = spec.get_previous_epoch(state) - # current_epoch = spec.get_current_epoch(state) - - # TODO - # add_mock_attestations(spec, state, ...) - # get indices attesting e.g. current_epoch_attestations - # set their balances - # yield from run_process_just_and_fin(spec, state) - # check finalization - pass +def test_234_ok_support(spec, state): + yield from finalize_on_234(spec, state, 5, 1.0) + + +@with_all_phases +@spec_state_test +def test_234_poor_support(spec, state): + yield from finalize_on_234(spec, state, 5, 0.6) + + +@with_all_phases +@spec_state_test +def test_23_ok_support(spec, state): + yield from finalize_on_23(spec, state, 4, 1.0) + + +@with_all_phases +@spec_state_test +def test_23_poor_support(spec, state): + yield from finalize_on_23(spec, state, 4, 0.6) + + +@with_all_phases +@spec_state_test +def test_123_ok_support(spec, state): + yield from finalize_on_123(spec, state, 4, 1.0) + + +@with_all_phases +@spec_state_test +def test_123_poor_support(spec, state): + yield from finalize_on_123(spec, state, 4, 0.6) + + +@with_all_phases +@spec_state_test +def test_12_ok_support(spec, state): + yield from finalize_on_12(spec, state, 3, 1.0) + + +@with_all_phases +@spec_state_test +def test_12_poor_support(spec, state): + yield from finalize_on_12(spec, state, 3, 0.6) + + +# TODO: bring ratios closer to 2/3 for edge case testing. From f484b1e98c44564c33672d49da23a42ed38ccc22 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:18:22 -0600 Subject: [PATCH 274/405] some fixes to finality_12 --- .../test_process_justification_and_finalization.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 0afa4f307a..b1d94c4fb4 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -189,12 +189,14 @@ def finalize_on_12(spec, state, epoch, support): # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root + state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c2 state.current_justified_checkpoint = c2 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[3] = 1 # mock 3rd latest epoch as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[0] = 1 # mock latest epoch as justified # mock the 1st latest epoch as justifiable, with 2nd as source add_mock_attestations(spec, state, epoch=epoch - 1, From 1885e285e3b57a51450910e77e19cf6d505b34c6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:28:36 -0600 Subject: [PATCH 275/405] fix source/target epochs in test_12 --- .../test_process_justification_and_finalization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index b1d94c4fb4..afb25fdaed 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -187,8 +187,8 @@ def finalize_on_12(spec, state, epoch, support): # checkpoints for the epochs ago: # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) - c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + c2 = spec.Checkpoint(epoch=epoch - 1, root=b'\xbb' * 32) + c1 = spec.Checkpoint(epoch=epoch, root=b'\xcc' * 32) state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root From bc5e947efc01141909aaf952167d6b20e6eb9d63 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:38:14 -0600 Subject: [PATCH 276/405] aggregation_bitfield - bits --- .../test_process_justification_and_finalization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index afb25fdaed..879c982f0f 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -49,10 +49,10 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. attesting_count = math.ceil(size * att_ratio) - aggregation_bitfield = ((1 << attesting_count) - 1).to_bytes(length=((size + 7) // 8), byteorder='big') + aggregation_bits = [i < attesting_count for i in range(size)] attestations.append(spec.PendingAttestation( - aggregation_bitfield=aggregation_bitfield, + aggregation_bits=aggregation_bits, data=spec.AttestationData( beacon_block_root=b'\xaa' * 32, source=source, From 022f1e7108a2d6971c147b22a3fef2b0aacebbc9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:59:20 -0600 Subject: [PATCH 277/405] fix source/target --- .../test_process_justification_and_finalization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 879c982f0f..246f20c9e9 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -56,7 +56,7 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): data=spec.AttestationData( beacon_block_root=b'\xaa' * 32, source=source, - target=source, + target=target, crosslink=spec.Crosslink() ), inclusion_delay=1, @@ -187,8 +187,8 @@ def finalize_on_12(spec, state, epoch, support): # checkpoints for the epochs ago: # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) - c2 = spec.Checkpoint(epoch=epoch - 1, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch, root=b'\xcc' * 32) + c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root From 129fd6297e1218f0655dda2bf8eb2b49a75bd387 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 29 Jun 2019 00:03:06 -0600 Subject: [PATCH 278/405] add shard to mock crosslink to separate attestations from eachother --- .../test_process_justification_and_finalization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 246f20c9e9..adcf6ea1d5 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -57,7 +57,7 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): beacon_block_root=b'\xaa' * 32, source=source, target=target, - crosslink=spec.Crosslink() + crosslink=spec.Crosslink(shard=shard) ), inclusion_delay=1, )) From ad943bbd06def7e25918ffaa3c39793175741176 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 08:39:27 +0100 Subject: [PATCH 279/405] Make deposit root the root of an SSZ vector --- deposit_contract/contracts/validator_registration.v.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index 14f1005209..0c129dac45 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -43,7 +43,8 @@ def to_little_endian_64(value: uint256) -> bytes[8]: @public @constant def get_deposit_root() -> bytes32: - node: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000 + zero_bytes32: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000 + node: bytes32 = zero_bytes32 size: uint256 = self.deposit_count for height in range(DEPOSIT_CONTRACT_TREE_DEPTH): if bitwise_and(size, 1) == 1: # More gas efficient than `size % 2 == 1` @@ -51,7 +52,7 @@ def get_deposit_root() -> bytes32: else: node = sha256(concat(node, self.zero_hashes[height])) size /= 2 - return node + return sha256(concat(node, slice(zero_bytes32, start=0, len=24), self.to_little_endian_64(deposit_amount)) @public From d820dbd7b1b402076ff7195db43fe826db19ebb0 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 09:01:05 +0100 Subject: [PATCH 280/405] Compile contract --- deposit_contract/contracts/validator_registration.json | 2 +- deposit_contract/contracts/validator_registration.v.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index 1988b28c09..5f154fa4c7 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334417}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b50506111c656600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63c5f2892f60005114156103cf57341561029057600080fd5b6000610140526001546101605261018060006020818352015b600160016101605116141561032a57600061018051602081106102cb57600080fd5b600060c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af161031c57600080fd5b60c051905061014052610398565b6000610140516020826101a0010152602081019050610180516020811061035057600080fd5b600260c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161038e57600080fd5b60c0519050610140525b61016060026103a657600080fd5b60028151048152505b81516001018083528114156102a9575b50506101405160005260206000f3005b63621fd13060005114156104e15734156103e857600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f8201039050610220511015156104335761044c565b610220516101e001526102205160200161022052610411565b6101c0805160200180610280828460006004600a8704601201f161046f57600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156104a1576104bd565b60006102e0516102a001535b8151600101808352811415610491575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d600051141561103b57606060046101403760506004356004016101a037603060043560040135111561051757600080fd5b604060243560040161022037602060243560040135111561053757600080fd5b608060443560040161028037606060443560040135111561055757600080fd5b63ffffffff6001541061056957600080fd5b633b9aca00610340526103405161057f57600080fd5b61034051340461032052633b9aca0061032051101561059d57600080fd5b60306101a051146105ad57600080fd5b602061022051146105bd57600080fd5b606061028051146105cd57600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a05110151561065d57610676565b6104a05161046001526104a0516020016104a05261063b565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f16106dd57600080fd5b50506101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a05163806732896104c0526001546104e0526104e0516006580161009b565b506105405260006105a0525b6105405160206001820306601f82010390506105a05110151561078e576107a7565b6105a05161056001526105a0516020016105a05261076c565b6104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526105408051602001806105c0828460006004600a8704601201f161082e57600080fd5b505060a06106405261064051610680526101a08051602001806106405161068001828460006004600a8704601201f161086657600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516040818352015b83610620511015156108a4576108c1565b6000610620516020850101535b8151600101808352811415610893575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106a0526102208051602001806106405161068001828460006004600a8704601201f161091857600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b836106205110151561095657610973565b6000610620516020850101535b8151600101808352811415610945575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106c0526103608051602001806106405161068001828460006004600a8704601201f16109ca57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610a0857610a25565b6000610620516020850101535b81516001018083528114156109f7575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106e0526102808051602001806106405161068001828460006004600a8704601201f1610a7c57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516060818352015b8361062051101515610aba57610ad7565b6000610620516020850101535b8151600101808352811415610aa9575b50505050602061064051610680015160206001820306601f82010390506106405101016106405261064051610700526105c08051602001806106405161068001828460006004600a8704601201f1610b2e57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610b6c57610b89565b6000610620516020850101535b8151600101808352811415610b5b575b50505050602061064051610680015160206001820306601f8201039050610640510101610640527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce61064051610680a160006107205260006101a06030806020846107e001018260208501600060046016f150508051820191505060006010602082066107600160208284011115610c2057600080fd5b60208061078082610720600060046015f15050818152809050905090506010806020846107e001018260208501600060046013f1505080518201915050806107e0526107e09050602060c0825160208401600060025af1610c8057600080fd5b60c0519050610740526000600060406020820661088001610280518284011115610ca957600080fd5b6060806108a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af1610ce957600080fd5b60c0519050602082610a800101526020810190506000604060206020820661094001610280518284011115610d1d57600080fd5b606080610960826020602088068803016102800160006004601bf1505081815280905090509050602080602084610a0001018260208501600060046015f150508051820191505061072051602082610a0001015260208101905080610a0052610a009050602060c0825160208401600060025af1610d9a57600080fd5b60c0519050602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610dd157600080fd5b60c0519050610860526000600061074051602082610b20010152602081019050610220602080602084610b2001018260208501600060046015f150508051820191505080610b2052610b209050602060c0825160208401600060025af1610e3757600080fd5b60c0519050602082610ca00101526020810190506000610360600880602084610c2001018260208501600060046012f15050805182019150506000601860208206610ba00160208284011115610e8c57600080fd5b602080610bc082610720600060046015f1505081815280905090509050601880602084610c2001018260208501600060046014f150508051820191505061086051602082610c2001015260208101905080610c2052610c209050602060c0825160208401600060025af1610eff57600080fd5b60c0519050602082610ca001015260208101905080610ca052610ca09050602060c0825160208401600060025af1610f3657600080fd5b60c0519050610b00526001805460018254011015610f5357600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d2051161415610fa357610b0051610d405160208110610f9257600080fd5b600060c052602060c0200155611037565b6000610d405160208110610fb657600080fd5b600060c052602060c0200154602082610d60010152602081019050610b0051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af161100757600080fd5b60c0519050610b0052610d20600261101e57600080fd5b60028151048152505b8151600101808352811415610f6e575b5050005b60006000fd5b6101856111c6036101856000396101856111c6036000f3"} \ No newline at end of file +{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 91830}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334417}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b505061136756600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63c5f2892f600051141561057057341561029057600080fd5b6000610140526101405161016052600154610180526101a060006020818352015b60016001610180511614156103325760006101a051602081106102d357600080fd5b600060c052602060c02001546020826102400101526020810190506101605160208261024001015260208101905080610240526102409050602060c0825160208401600060025af161032457600080fd5b60c0519050610160526103a0565b6000610160516020826101c00101526020810190506101a0516020811061035857600080fd5b600260c052602060c02001546020826101c0010152602081019050806101c0526101c09050602060c0825160208401600060025af161039657600080fd5b60c0519050610160525b61018060026103ae57600080fd5b60028151048152505b81516001018083528114156102b1575b505060006101605160208261044001015260208101905060006018602082066102c001602082840111156103fa57600080fd5b6020806102e082610140600060046015f150508181528090509050905060188060208461044001018260208501600060046014f15050805182019150506101405161016051610180516101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e051610300516103205163806732896103405260015461036052610360516006580161009b565b506103c0526000610420525b6103c05160206001820306601f8201039050610420511015156104c2576104db565b610420516103e0015261042051602001610420526104a0565b61032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526101805261016052610140526103c060088060208461044001018260208501600060046012f150508051820191505080610440526104409050602060c0825160208401600060025af161056157600080fd5b60c051905060005260206000f3005b63621fd130600051141561068257341561058957600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f8201039050610220511015156105d4576105ed565b610220516101e0015261022051602001610220526105b2565b6101c0805160200180610280828460006004600a8704601201f161061057600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156106425761065e565b60006102e0516102a001535b8151600101808352811415610632575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d60005114156111dc57606060046101403760506004356004016101a03760306004356004013511156106b857600080fd5b60406024356004016102203760206024356004013511156106d857600080fd5b60806044356004016102803760606044356004013511156106f857600080fd5b63ffffffff6001541061070a57600080fd5b633b9aca00610340526103405161072057600080fd5b61034051340461032052633b9aca0061032051101561073e57600080fd5b60306101a0511461074e57600080fd5b6020610220511461075e57600080fd5b6060610280511461076e57600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a0511015156107fe57610817565b6104a05161046001526104a0516020016104a0526107dc565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f161087e57600080fd5b50506101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a05163806732896104c0526001546104e0526104e0516006580161009b565b506105405260006105a0525b6105405160206001820306601f82010390506105a05110151561092f57610948565b6105a05161056001526105a0516020016105a05261090d565b6104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526105408051602001806105c0828460006004600a8704601201f16109cf57600080fd5b505060a06106405261064051610680526101a08051602001806106405161068001828460006004600a8704601201f1610a0757600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516040818352015b8361062051101515610a4557610a62565b6000610620516020850101535b8151600101808352811415610a34575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106a0526102208051602001806106405161068001828460006004600a8704601201f1610ab957600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610af757610b14565b6000610620516020850101535b8151600101808352811415610ae6575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106c0526103608051602001806106405161068001828460006004600a8704601201f1610b6b57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610ba957610bc6565b6000610620516020850101535b8151600101808352811415610b98575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106e0526102808051602001806106405161068001828460006004600a8704601201f1610c1d57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516060818352015b8361062051101515610c5b57610c78565b6000610620516020850101535b8151600101808352811415610c4a575b50505050602061064051610680015160206001820306601f82010390506106405101016106405261064051610700526105c08051602001806106405161068001828460006004600a8704601201f1610ccf57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610d0d57610d2a565b6000610620516020850101535b8151600101808352811415610cfc575b50505050602061064051610680015160206001820306601f8201039050610640510101610640527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce61064051610680a160006107205260006101a06030806020846107e001018260208501600060046016f150508051820191505060006010602082066107600160208284011115610dc157600080fd5b60208061078082610720600060046015f15050818152809050905090506010806020846107e001018260208501600060046013f1505080518201915050806107e0526107e09050602060c0825160208401600060025af1610e2157600080fd5b60c0519050610740526000600060406020820661088001610280518284011115610e4a57600080fd5b6060806108a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af1610e8a57600080fd5b60c0519050602082610a800101526020810190506000604060206020820661094001610280518284011115610ebe57600080fd5b606080610960826020602088068803016102800160006004601bf1505081815280905090509050602080602084610a0001018260208501600060046015f150508051820191505061072051602082610a0001015260208101905080610a0052610a009050602060c0825160208401600060025af1610f3b57600080fd5b60c0519050602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610f7257600080fd5b60c0519050610860526000600061074051602082610b20010152602081019050610220602080602084610b2001018260208501600060046015f150508051820191505080610b2052610b209050602060c0825160208401600060025af1610fd857600080fd5b60c0519050602082610ca00101526020810190506000610360600880602084610c2001018260208501600060046012f15050805182019150506000601860208206610ba0016020828401111561102d57600080fd5b602080610bc082610720600060046015f1505081815280905090509050601880602084610c2001018260208501600060046014f150508051820191505061086051602082610c2001015260208101905080610c2052610c209050602060c0825160208401600060025af16110a057600080fd5b60c0519050602082610ca001015260208101905080610ca052610ca09050602060c0825160208401600060025af16110d757600080fd5b60c0519050610b005260018054600182540110156110f457600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d205116141561114457610b0051610d40516020811061113357600080fd5b600060c052602060c02001556111d8565b6000610d40516020811061115757600080fd5b600060c052602060c0200154602082610d60010152602081019050610b0051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af16111a857600080fd5b60c0519050610b0052610d2060026111bf57600080fd5b60028151048152505b815160010180835281141561110f575b5050005b60006000fd5b61018561136703610185600039610185611367036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index 0c129dac45..c1f3c6c79a 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -52,7 +52,7 @@ def get_deposit_root() -> bytes32: else: node = sha256(concat(node, self.zero_hashes[height])) size /= 2 - return sha256(concat(node, slice(zero_bytes32, start=0, len=24), self.to_little_endian_64(deposit_amount)) + return sha256(concat(node, slice(zero_bytes32, start=0, len=24), self.to_little_endian_64(self.deposit_count))) @public From 34b8d8ab335c31b451d0cfb46fc78cdab723b4f5 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 09:13:42 +0100 Subject: [PATCH 281/405] Start modifying test --- deposit_contract/tests/contracts/test_deposit.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 783af33567..80c3b11c1a 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -15,6 +15,7 @@ DepositData, ) from eth2spec.utils.hash_function import hash +from eth2spec.utils.ssz.ssz_typing import List from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, ) @@ -137,7 +138,7 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input ) deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(10)] - leaf_nodes = [] + deposit_data_list = List[DepositData, 2**32]() for i in range(0, 10): tx_hash = registration_contract.functions.deposit( *deposit_input, @@ -151,13 +152,11 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input assert log["index"] == i.to_bytes(8, 'little') - deposit_data = DepositData( + deposit_data_list[i] = DepositData( pubkey=deposit_input[0], withdrawal_credentials=deposit_input[1], amount=deposit_amount_list[i], signature=deposit_input[2], ) - hash_tree_root_result = hash_tree_root(deposit_data) - leaf_nodes.append(hash_tree_root_result) - root = compute_merkle_root(leaf_nodes) + root = hash_tree_root(deposit_data_list) assert root == registration_contract.functions.get_deposit_root().call() From d5d3e49c5f7958a717740c8a13538116a3cd952f Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 09:25:19 +0100 Subject: [PATCH 282/405] Tests pass --- .../contracts/validator_registration.json | 2 +- .../contracts/validator_registration.v.py | 2 +- .../tests/contracts/test_deposit.py | 214 ++++++++---------- 3 files changed, 102 insertions(+), 116 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index 5f154fa4c7..a3d3092696 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 91830}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334417}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b505061136756600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63c5f2892f600051141561057057341561029057600080fd5b6000610140526101405161016052600154610180526101a060006020818352015b60016001610180511614156103325760006101a051602081106102d357600080fd5b600060c052602060c02001546020826102400101526020810190506101605160208261024001015260208101905080610240526102409050602060c0825160208401600060025af161032457600080fd5b60c0519050610160526103a0565b6000610160516020826101c00101526020810190506101a0516020811061035857600080fd5b600260c052602060c02001546020826101c0010152602081019050806101c0526101c09050602060c0825160208401600060025af161039657600080fd5b60c0519050610160525b61018060026103ae57600080fd5b60028151048152505b81516001018083528114156102b1575b505060006101605160208261044001015260208101905060006018602082066102c001602082840111156103fa57600080fd5b6020806102e082610140600060046015f150508181528090509050905060188060208461044001018260208501600060046014f15050805182019150506101405161016051610180516101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e051610300516103205163806732896103405260015461036052610360516006580161009b565b506103c0526000610420525b6103c05160206001820306601f8201039050610420511015156104c2576104db565b610420516103e0015261042051602001610420526104a0565b61032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526101805261016052610140526103c060088060208461044001018260208501600060046012f150508051820191505080610440526104409050602060c0825160208401600060025af161056157600080fd5b60c051905060005260206000f3005b63621fd130600051141561068257341561058957600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f8201039050610220511015156105d4576105ed565b610220516101e0015261022051602001610220526105b2565b6101c0805160200180610280828460006004600a8704601201f161061057600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156106425761065e565b60006102e0516102a001535b8151600101808352811415610632575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d60005114156111dc57606060046101403760506004356004016101a03760306004356004013511156106b857600080fd5b60406024356004016102203760206024356004013511156106d857600080fd5b60806044356004016102803760606044356004013511156106f857600080fd5b63ffffffff6001541061070a57600080fd5b633b9aca00610340526103405161072057600080fd5b61034051340461032052633b9aca0061032051101561073e57600080fd5b60306101a0511461074e57600080fd5b6020610220511461075e57600080fd5b6060610280511461076e57600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a0511015156107fe57610817565b6104a05161046001526104a0516020016104a0526107dc565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f161087e57600080fd5b50506101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a05163806732896104c0526001546104e0526104e0516006580161009b565b506105405260006105a0525b6105405160206001820306601f82010390506105a05110151561092f57610948565b6105a05161056001526105a0516020016105a05261090d565b6104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526105408051602001806105c0828460006004600a8704601201f16109cf57600080fd5b505060a06106405261064051610680526101a08051602001806106405161068001828460006004600a8704601201f1610a0757600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516040818352015b8361062051101515610a4557610a62565b6000610620516020850101535b8151600101808352811415610a34575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106a0526102208051602001806106405161068001828460006004600a8704601201f1610ab957600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610af757610b14565b6000610620516020850101535b8151600101808352811415610ae6575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106c0526103608051602001806106405161068001828460006004600a8704601201f1610b6b57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610ba957610bc6565b6000610620516020850101535b8151600101808352811415610b98575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106e0526102808051602001806106405161068001828460006004600a8704601201f1610c1d57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516060818352015b8361062051101515610c5b57610c78565b6000610620516020850101535b8151600101808352811415610c4a575b50505050602061064051610680015160206001820306601f82010390506106405101016106405261064051610700526105c08051602001806106405161068001828460006004600a8704601201f1610ccf57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610d0d57610d2a565b6000610620516020850101535b8151600101808352811415610cfc575b50505050602061064051610680015160206001820306601f8201039050610640510101610640527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce61064051610680a160006107205260006101a06030806020846107e001018260208501600060046016f150508051820191505060006010602082066107600160208284011115610dc157600080fd5b60208061078082610720600060046015f15050818152809050905090506010806020846107e001018260208501600060046013f1505080518201915050806107e0526107e09050602060c0825160208401600060025af1610e2157600080fd5b60c0519050610740526000600060406020820661088001610280518284011115610e4a57600080fd5b6060806108a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af1610e8a57600080fd5b60c0519050602082610a800101526020810190506000604060206020820661094001610280518284011115610ebe57600080fd5b606080610960826020602088068803016102800160006004601bf1505081815280905090509050602080602084610a0001018260208501600060046015f150508051820191505061072051602082610a0001015260208101905080610a0052610a009050602060c0825160208401600060025af1610f3b57600080fd5b60c0519050602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610f7257600080fd5b60c0519050610860526000600061074051602082610b20010152602081019050610220602080602084610b2001018260208501600060046015f150508051820191505080610b2052610b209050602060c0825160208401600060025af1610fd857600080fd5b60c0519050602082610ca00101526020810190506000610360600880602084610c2001018260208501600060046012f15050805182019150506000601860208206610ba0016020828401111561102d57600080fd5b602080610bc082610720600060046015f1505081815280905090509050601880602084610c2001018260208501600060046014f150508051820191505061086051602082610c2001015260208101905080610c2052610c209050602060c0825160208401600060025af16110a057600080fd5b60c0519050602082610ca001015260208101905080610ca052610ca09050602060c0825160208401600060025af16110d757600080fd5b60c0519050610b005260018054600182540110156110f457600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d205116141561114457610b0051610d40516020811061113357600080fd5b600060c052602060c02001556111d8565b6000610d40516020811061115757600080fd5b600060c052602060c0200154602082610d60010152602081019050610b0051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af16111a857600080fd5b60c0519050610b0052610d2060026111bf57600080fd5b60028151048152505b815160010180835281141561110f575b5050005b60006000fd5b61018561136703610185600039610185611367036000f3"} \ No newline at end of file +{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 91674}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334417}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b50506112ff56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63c5f2892f600051141561050857341561029057600080fd5b6000610140526101405161016052600154610180526101a060006020818352015b60016001610180511614156103325760006101a051602081106102d357600080fd5b600060c052602060c02001546020826102400101526020810190506101605160208261024001015260208101905080610240526102409050602060c0825160208401600060025af161032457600080fd5b60c0519050610160526103a0565b6000610160516020826101c00101526020810190506101a0516020811061035857600080fd5b600260c052602060c02001546020826101c0010152602081019050806101c0526101c09050602060c0825160208401600060025af161039657600080fd5b60c0519050610160525b61018060026103ae57600080fd5b60028151048152505b81516001018083528114156102b1575b505060006101605160208261044001015260208101905061014051610160516101805163806732896102c0526001546102e0526102e0516006580161009b565b506103405260006103a0525b6103405160206001820306601f82010390506103a0511015156104355761044e565b6103a05161036001526103a0516020016103a052610413565b61018052610160526101405261034060088060208461044001018260208501600060046012f150508051820191505060006018602082066103c0016020828401111561049957600080fd5b6020806103e082610140600060046015f150508181528090509050905060188060208461044001018260208501600060046014f150508051820191505080610440526104409050602060c0825160208401600060025af16104f957600080fd5b60c051905060005260206000f3005b63621fd130600051141561061a57341561052157600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f82010390506102205110151561056c57610585565b610220516101e00152610220516020016102205261054a565b6101c0805160200180610280828460006004600a8704601201f16105a857600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156105da576105f6565b60006102e0516102a001535b81516001018083528114156105ca575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d600051141561117457606060046101403760506004356004016101a037603060043560040135111561065057600080fd5b604060243560040161022037602060243560040135111561067057600080fd5b608060443560040161028037606060443560040135111561069057600080fd5b63ffffffff600154106106a257600080fd5b633b9aca0061034052610340516106b857600080fd5b61034051340461032052633b9aca006103205110156106d657600080fd5b60306101a051146106e657600080fd5b602061022051146106f657600080fd5b6060610280511461070657600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a051101515610796576107af565b6104a05161046001526104a0516020016104a052610774565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f161081657600080fd5b50506101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a05163806732896104c0526001546104e0526104e0516006580161009b565b506105405260006105a0525b6105405160206001820306601f82010390506105a0511015156108c7576108e0565b6105a05161056001526105a0516020016105a0526108a5565b6104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526105408051602001806105c0828460006004600a8704601201f161096757600080fd5b505060a06106405261064051610680526101a08051602001806106405161068001828460006004600a8704601201f161099f57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516040818352015b83610620511015156109dd576109fa565b6000610620516020850101535b81516001018083528114156109cc575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106a0526102208051602001806106405161068001828460006004600a8704601201f1610a5157600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610a8f57610aac565b6000610620516020850101535b8151600101808352811415610a7e575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106c0526103608051602001806106405161068001828460006004600a8704601201f1610b0357600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610b4157610b5e565b6000610620516020850101535b8151600101808352811415610b30575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106e0526102808051602001806106405161068001828460006004600a8704601201f1610bb557600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516060818352015b8361062051101515610bf357610c10565b6000610620516020850101535b8151600101808352811415610be2575b50505050602061064051610680015160206001820306601f82010390506106405101016106405261064051610700526105c08051602001806106405161068001828460006004600a8704601201f1610c6757600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610ca557610cc2565b6000610620516020850101535b8151600101808352811415610c94575b50505050602061064051610680015160206001820306601f8201039050610640510101610640527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce61064051610680a160006107205260006101a06030806020846107e001018260208501600060046016f150508051820191505060006010602082066107600160208284011115610d5957600080fd5b60208061078082610720600060046015f15050818152809050905090506010806020846107e001018260208501600060046013f1505080518201915050806107e0526107e09050602060c0825160208401600060025af1610db957600080fd5b60c0519050610740526000600060406020820661088001610280518284011115610de257600080fd5b6060806108a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af1610e2257600080fd5b60c0519050602082610a800101526020810190506000604060206020820661094001610280518284011115610e5657600080fd5b606080610960826020602088068803016102800160006004601bf1505081815280905090509050602080602084610a0001018260208501600060046015f150508051820191505061072051602082610a0001015260208101905080610a0052610a009050602060c0825160208401600060025af1610ed357600080fd5b60c0519050602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610f0a57600080fd5b60c0519050610860526000600061074051602082610b20010152602081019050610220602080602084610b2001018260208501600060046015f150508051820191505080610b2052610b209050602060c0825160208401600060025af1610f7057600080fd5b60c0519050602082610ca00101526020810190506000610360600880602084610c2001018260208501600060046012f15050805182019150506000601860208206610ba00160208284011115610fc557600080fd5b602080610bc082610720600060046015f1505081815280905090509050601880602084610c2001018260208501600060046014f150508051820191505061086051602082610c2001015260208101905080610c2052610c209050602060c0825160208401600060025af161103857600080fd5b60c0519050602082610ca001015260208101905080610ca052610ca09050602060c0825160208401600060025af161106f57600080fd5b60c0519050610b0052600180546001825401101561108c57600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d20511614156110dc57610b0051610d4051602081106110cb57600080fd5b600060c052602060c0200155611170565b6000610d4051602081106110ef57600080fd5b600060c052602060c0200154602082610d60010152602081019050610b0051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af161114057600080fd5b60c0519050610b0052610d20600261115757600080fd5b60028151048152505b81516001018083528114156110a7575b5050005b60006000fd5b6101856112ff036101856000396101856112ff036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index c1f3c6c79a..b49cc45e1a 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -52,7 +52,7 @@ def get_deposit_root() -> bytes32: else: node = sha256(concat(node, self.zero_hashes[height])) size /= 2 - return sha256(concat(node, slice(zero_bytes32, start=0, len=24), self.to_little_endian_64(self.deposit_count))) + return sha256(concat(node, self.to_little_endian_64(self.deposit_count), slice(zero_bytes32, start=0, len=24))) @public diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 80c3b11c1a..58e4158a35 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -21,116 +21,101 @@ ) -def compute_merkle_root(leaf_nodes): - assert len(leaf_nodes) >= 1 - empty_node = b'\x00' * 32 - child_nodes = leaf_nodes[:] - for _ in range(DEPOSIT_CONTRACT_TREE_DEPTH): - parent_nodes = [] - if len(child_nodes) % 2 == 1: - child_nodes.append(empty_node) - for j in range(0, len(child_nodes), 2): - parent_nodes.append(hash(child_nodes[j] + child_nodes[j + 1])) - child_nodes = parent_nodes - empty_node = hash(empty_node + empty_node) - return child_nodes[0] - - -@pytest.fixture -def deposit_input(): - """ - pubkey: bytes[48] - withdrawal_credentials: bytes[32] - signature: bytes[96] - """ - return ( - b'\x11' * 48, - b'\x22' * 32, - b'\x33' * 96, - ) - - -@pytest.mark.parametrize( - 'success,deposit_amount', - [ - (True, FULL_DEPOSIT_AMOUNT), - (True, MIN_DEPOSIT_AMOUNT), - (False, MIN_DEPOSIT_AMOUNT - 1), - (True, FULL_DEPOSIT_AMOUNT + 1) - ] -) -def test_deposit_amount(registration_contract, - w3, - success, - deposit_amount, - assert_tx_failed, - deposit_input): - call = registration_contract.functions.deposit(*deposit_input) - if success: - assert call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) - else: - assert_tx_failed( - lambda: call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) - ) - - -@pytest.mark.parametrize( - 'invalid_pubkey,invalid_withdrawal_credentials,invalid_signature,success', - [ - (False, False, False, True), - (True, False, False, False), - (False, True, False, False), - (False, False, True, False), - ] -) -def test_deposit_inputs(registration_contract, - w3, - assert_tx_failed, - deposit_input, - invalid_pubkey, - invalid_withdrawal_credentials, - invalid_signature, - success): - pubkey = deposit_input[0][2:] if invalid_pubkey else deposit_input[0] - if invalid_withdrawal_credentials: # this one is different to satisfy linter - withdrawal_credentials = deposit_input[1][2:] - else: - withdrawal_credentials = deposit_input[1] - signature = deposit_input[2][2:] if invalid_signature else deposit_input[2] - - call = registration_contract.functions.deposit( - pubkey, - withdrawal_credentials, - signature, - ) - if success: - assert call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) - else: - assert_tx_failed( - lambda: call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) - ) - - -def test_deposit_log(registration_contract, a0, w3, deposit_input): - log_filter = registration_contract.events.Deposit.createFilter( - fromBlock='latest', - ) - - deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(3)] - for i in range(3): - registration_contract.functions.deposit( - *deposit_input, - ).transact({"value": deposit_amount_list[i] * eth_utils.denoms.gwei}) - - logs = log_filter.get_new_entries() - assert len(logs) == 1 - log = logs[0]['args'] - - assert log['pubkey'] == deposit_input[0] - assert log['withdrawal_credentials'] == deposit_input[1] - assert log['amount'] == deposit_amount_list[i].to_bytes(8, 'little') - assert log['signature'] == deposit_input[2] - assert log['index'] == i.to_bytes(8, 'little') +# @pytest.fixture +# def deposit_input(): +# """ +# pubkey: bytes[48] +# withdrawal_credentials: bytes[32] +# signature: bytes[96] +# """ +# return ( +# b'\x11' * 48, +# b'\x22' * 32, +# b'\x33' * 96, +# ) + + +# @pytest.mark.parametrize( +# 'success,deposit_amount', +# [ +# (True, FULL_DEPOSIT_AMOUNT), +# (True, MIN_DEPOSIT_AMOUNT), +# (False, MIN_DEPOSIT_AMOUNT - 1), +# (True, FULL_DEPOSIT_AMOUNT + 1) +# ] +# ) +# def test_deposit_amount(registration_contract, +# w3, +# success, +# deposit_amount, +# assert_tx_failed, +# deposit_input): +# call = registration_contract.functions.deposit(*deposit_input) +# if success: +# assert call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) +# else: +# assert_tx_failed( +# lambda: call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) +# ) + + +# @pytest.mark.parametrize( +# 'invalid_pubkey,invalid_withdrawal_credentials,invalid_signature,success', +# [ +# (False, False, False, True), +# (True, False, False, False), +# (False, True, False, False), +# (False, False, True, False), +# ] +# ) +# def test_deposit_inputs(registration_contract, +# w3, +# assert_tx_failed, +# deposit_input, +# invalid_pubkey, +# invalid_withdrawal_credentials, +# invalid_signature, +# success): +# pubkey = deposit_input[0][2:] if invalid_pubkey else deposit_input[0] +# if invalid_withdrawal_credentials: # this one is different to satisfy linter +# withdrawal_credentials = deposit_input[1][2:] +# else: +# withdrawal_credentials = deposit_input[1] +# signature = deposit_input[2][2:] if invalid_signature else deposit_input[2] + +# call = registration_contract.functions.deposit( +# pubkey, +# withdrawal_credentials, +# signature, +# ) +# if success: +# assert call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) +# else: +# assert_tx_failed( +# lambda: call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) +# ) + + +# def test_deposit_log(registration_contract, a0, w3, deposit_input): +# log_filter = registration_contract.events.Deposit.createFilter( +# fromBlock='latest', +# ) + +# deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(3)] +# for i in range(3): +# registration_contract.functions.deposit( +# *deposit_input, +# ).transact({"value": deposit_amount_list[i] * eth_utils.denoms.gwei}) + +# logs = log_filter.get_new_entries() +# assert len(logs) == 1 +# log = logs[0]['args'] + +# assert log['pubkey'] == deposit_input[0] +# assert log['withdrawal_credentials'] == deposit_input[1] +# assert log['amount'] == deposit_amount_list[i].to_bytes(8, 'little') +# assert log['signature'] == deposit_input[2] +# assert log['index'] == i.to_bytes(8, 'little') def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input): log_filter = registration_contract.events.Deposit.createFilter( @@ -138,7 +123,7 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input ) deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(10)] - deposit_data_list = List[DepositData, 2**32]() + deposit_data_list = [] for i in range(0, 10): tx_hash = registration_contract.functions.deposit( *deposit_input, @@ -152,11 +137,12 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input assert log["index"] == i.to_bytes(8, 'little') - deposit_data_list[i] = DepositData( + deposit_data_list.append(DepositData( pubkey=deposit_input[0], withdrawal_credentials=deposit_input[1], amount=deposit_amount_list[i], signature=deposit_input[2], - ) - root = hash_tree_root(deposit_data_list) + )) + + root = hash_tree_root(List[DepositData, 2**32](*(tuple(deposit_data_list)))) assert root == registration_contract.functions.get_deposit_root().call() From a7ceec1e8a6a68a4ba6bf96be4f22f91be0ceffa Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 09:28:46 +0100 Subject: [PATCH 283/405] Uncomment tests --- .../tests/contracts/test_deposit.py | 191 +++++++++--------- 1 file changed, 96 insertions(+), 95 deletions(-) diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 58e4158a35..94c7b29d1a 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -21,101 +21,102 @@ ) -# @pytest.fixture -# def deposit_input(): -# """ -# pubkey: bytes[48] -# withdrawal_credentials: bytes[32] -# signature: bytes[96] -# """ -# return ( -# b'\x11' * 48, -# b'\x22' * 32, -# b'\x33' * 96, -# ) - - -# @pytest.mark.parametrize( -# 'success,deposit_amount', -# [ -# (True, FULL_DEPOSIT_AMOUNT), -# (True, MIN_DEPOSIT_AMOUNT), -# (False, MIN_DEPOSIT_AMOUNT - 1), -# (True, FULL_DEPOSIT_AMOUNT + 1) -# ] -# ) -# def test_deposit_amount(registration_contract, -# w3, -# success, -# deposit_amount, -# assert_tx_failed, -# deposit_input): -# call = registration_contract.functions.deposit(*deposit_input) -# if success: -# assert call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) -# else: -# assert_tx_failed( -# lambda: call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) -# ) - - -# @pytest.mark.parametrize( -# 'invalid_pubkey,invalid_withdrawal_credentials,invalid_signature,success', -# [ -# (False, False, False, True), -# (True, False, False, False), -# (False, True, False, False), -# (False, False, True, False), -# ] -# ) -# def test_deposit_inputs(registration_contract, -# w3, -# assert_tx_failed, -# deposit_input, -# invalid_pubkey, -# invalid_withdrawal_credentials, -# invalid_signature, -# success): -# pubkey = deposit_input[0][2:] if invalid_pubkey else deposit_input[0] -# if invalid_withdrawal_credentials: # this one is different to satisfy linter -# withdrawal_credentials = deposit_input[1][2:] -# else: -# withdrawal_credentials = deposit_input[1] -# signature = deposit_input[2][2:] if invalid_signature else deposit_input[2] - -# call = registration_contract.functions.deposit( -# pubkey, -# withdrawal_credentials, -# signature, -# ) -# if success: -# assert call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) -# else: -# assert_tx_failed( -# lambda: call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) -# ) - - -# def test_deposit_log(registration_contract, a0, w3, deposit_input): -# log_filter = registration_contract.events.Deposit.createFilter( -# fromBlock='latest', -# ) - -# deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(3)] -# for i in range(3): -# registration_contract.functions.deposit( -# *deposit_input, -# ).transact({"value": deposit_amount_list[i] * eth_utils.denoms.gwei}) - -# logs = log_filter.get_new_entries() -# assert len(logs) == 1 -# log = logs[0]['args'] - -# assert log['pubkey'] == deposit_input[0] -# assert log['withdrawal_credentials'] == deposit_input[1] -# assert log['amount'] == deposit_amount_list[i].to_bytes(8, 'little') -# assert log['signature'] == deposit_input[2] -# assert log['index'] == i.to_bytes(8, 'little') +@pytest.fixture +def deposit_input(): + """ + pubkey: bytes[48] + withdrawal_credentials: bytes[32] + signature: bytes[96] + """ + return ( + b'\x11' * 48, + b'\x22' * 32, + b'\x33' * 96, + ) + + +@pytest.mark.parametrize( + 'success,deposit_amount', + [ + (True, FULL_DEPOSIT_AMOUNT), + (True, MIN_DEPOSIT_AMOUNT), + (False, MIN_DEPOSIT_AMOUNT - 1), + (True, FULL_DEPOSIT_AMOUNT + 1) + ] +) +def test_deposit_amount(registration_contract, + w3, + success, + deposit_amount, + assert_tx_failed, + deposit_input): + call = registration_contract.functions.deposit(*deposit_input) + if success: + assert call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) + else: + assert_tx_failed( + lambda: call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) + ) + + +@pytest.mark.parametrize( + 'invalid_pubkey,invalid_withdrawal_credentials,invalid_signature,success', + [ + (False, False, False, True), + (True, False, False, False), + (False, True, False, False), + (False, False, True, False), + ] +) +def test_deposit_inputs(registration_contract, + w3, + assert_tx_failed, + deposit_input, + invalid_pubkey, + invalid_withdrawal_credentials, + invalid_signature, + success): + pubkey = deposit_input[0][2:] if invalid_pubkey else deposit_input[0] + if invalid_withdrawal_credentials: # this one is different to satisfy linter + withdrawal_credentials = deposit_input[1][2:] + else: + withdrawal_credentials = deposit_input[1] + signature = deposit_input[2][2:] if invalid_signature else deposit_input[2] + + call = registration_contract.functions.deposit( + pubkey, + withdrawal_credentials, + signature, + ) + if success: + assert call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) + else: + assert_tx_failed( + lambda: call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) + ) + + +def test_deposit_log(registration_contract, a0, w3, deposit_input): + log_filter = registration_contract.events.Deposit.createFilter( + fromBlock='latest', + ) + + deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(3)] + for i in range(3): + registration_contract.functions.deposit( + *deposit_input, + ).transact({"value": deposit_amount_list[i] * eth_utils.denoms.gwei}) + + logs = log_filter.get_new_entries() + assert len(logs) == 1 + log = logs[0]['args'] + + assert log['pubkey'] == deposit_input[0] + assert log['withdrawal_credentials'] == deposit_input[1] + assert log['amount'] == deposit_amount_list[i].to_bytes(8, 'little') + assert log['signature'] == deposit_input[2] + assert log['index'] == i.to_bytes(8, 'little') + def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input): log_filter = registration_contract.events.Deposit.createFilter( From 7fdf59d012065e59c4d9e771ce03a9d10cf85530 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 29 Jun 2019 18:09:05 +0800 Subject: [PATCH 284/405] `active_validator_count >= GENESIS_ACTIVE_VALIDATOR_COUNT`: per Eth1 block --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9e59212fd0..7cd5725b10 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1186,7 +1186,7 @@ def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], active_validator_count += 1 # Check effective balance to trigger genesis - return active_validator_count == GENESIS_ACTIVE_VALIDATOR_COUNT + return active_validator_count >= GENESIS_ACTIVE_VALIDATOR_COUNT ``` ### Genesis state From b05bebf45c1bfe31fdf9ee195283827be4f145e0 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 12:23:03 +0200 Subject: [PATCH 285/405] Fix list slicing --- ..._process_justification_and_finalization.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index adcf6ea1d5..3e2f6e9677 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -74,13 +74,15 @@ def finalize_on_234(spec, state, epoch, support): c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - # c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c4.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c4.root + state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c3 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[3:4] = [1, 1] # mock 3rd and 4th latest epochs as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[1:3] = [1, 1] # mock 3rd and 4th latest epochs as justified # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -109,16 +111,16 @@ def finalize_on_23(spec, state, epoch, support): # 3210x -- justification bitfield indices # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - # c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c3 state.current_justified_checkpoint = c3 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[2] = 1 # mock 3rd latest epoch as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[1] = 1 # mock 2nd latest epoch as justified # mock the 2nd latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -147,16 +149,18 @@ def finalize_on_123(spec, state, epoch, support): # 3210x -- justification bitfield indices # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root + state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c3 state.current_justified_checkpoint = c2 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[1:2] = [1, 1] # mock 2rd and 3th latest epochs as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[0:2] = [1, 1] # mock 1st and 2nd latest epochs as justified # mock the 1st latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 1, @@ -185,8 +189,6 @@ def finalize_on_12(spec, state, epoch, support): # 3210 -- justification bitfield indices # 001*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) - # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root From f9ca7c97c9bc4d994ba88ee721afcfc04336673b Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 13:01:59 +0200 Subject: [PATCH 286/405] Fix 123 finalisation --- .../test_process_justification_and_finalization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 3e2f6e9677..42ef73084b 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -149,6 +149,7 @@ def finalize_on_123(spec, state, epoch, support): # 3210x -- justification bitfield indices # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: + c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) @@ -157,7 +158,7 @@ def finalize_on_123(spec, state, epoch, support): state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint - state.previous_justified_checkpoint = c3 + state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c2 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() state.justification_bits[0:2] = [1, 1] # mock 1st and 2nd latest epochs as justified From 2eca6ef09d6d94656863907ba2fed5213788f213 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 13:10:09 +0200 Subject: [PATCH 287/405] Corrects justification comments --- .../test_process_justification_and_finalization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 42ef73084b..02439f6649 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -82,7 +82,7 @@ def finalize_on_234(spec, state, epoch, support): state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[1:3] = [1, 1] # mock 3rd and 4th latest epochs as justified + state.justification_bits[1:3] = [1, 1] # mock 2nd and 3rd latest epochs as justified # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, From 0680d8cc536f72a863d9e9ec1975a2fd5ff1729c Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 14:35:07 +0200 Subject: [PATCH 288/405] Makes justification ratios more marginal --- ..._process_justification_and_finalization.py | 56 +++++++++---------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 02439f6649..406bdbf7ff 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -25,7 +25,7 @@ def get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) return size -def add_mock_attestations(spec, state, epoch, att_ratio, source, target): +def add_mock_attestations(spec, state, epoch, source, target, sufficient_support=False): # we must be at the end of the epoch assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 @@ -48,7 +48,8 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): size = get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. - attesting_count = math.ceil(size * att_ratio) + attesting_count = math.ceil(size * 2 /3) + attesting_count = attesting_count if sufficient_support else attesting_count - 1 aggregation_bits = [i < attesting_count for i in range(size)] attestations.append(spec.PendingAttestation( @@ -63,7 +64,7 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): )) -def finalize_on_234(spec, state, epoch, support): +def finalize_on_234(spec, state, epoch, sufficient_support): assert epoch > 4 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -86,14 +87,14 @@ def finalize_on_234(spec, state, epoch, support): # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, - att_ratio=support, source=c4, - target=c2) + target=c2, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c4 # finalized old previous justified epoch @@ -103,7 +104,7 @@ def finalize_on_234(spec, state, epoch, support): assert state.finalized_checkpoint == old_finalized # no new finalized -def finalize_on_23(spec, state, epoch, support): +def finalize_on_23(spec, state, epoch, sufficient_support): assert epoch > 3 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -124,14 +125,14 @@ def finalize_on_23(spec, state, epoch, support): # mock the 2nd latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 2, - att_ratio=support, source=c3, - target=c2) + target=c2, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c3 # finalized old previous justified epoch @@ -141,7 +142,7 @@ def finalize_on_23(spec, state, epoch, support): assert state.finalized_checkpoint == old_finalized # no new finalized -def finalize_on_123(spec, state, epoch, support): +def finalize_on_123(spec, state, epoch, sufficient_support): assert epoch > 3 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -165,14 +166,14 @@ def finalize_on_123(spec, state, epoch, support): # mock the 1st latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 1, - att_ratio=support, source=c3, - target=c1) + target=c1, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized old current @@ -182,7 +183,7 @@ def finalize_on_123(spec, state, epoch, support): assert state.finalized_checkpoint == old_finalized # no new finalized -def finalize_on_12(spec, state, epoch, support): +def finalize_on_12(spec, state, epoch, sufficient_support): assert epoch > 2 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -203,14 +204,14 @@ def finalize_on_12(spec, state, epoch, support): # mock the 1st latest epoch as justifiable, with 2nd as source add_mock_attestations(spec, state, epoch=epoch - 1, - att_ratio=support, source=c2, - target=c1) + target=c1, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized previous justified epoch @@ -223,49 +224,46 @@ def finalize_on_12(spec, state, epoch, support): @with_all_phases @spec_state_test def test_234_ok_support(spec, state): - yield from finalize_on_234(spec, state, 5, 1.0) + yield from finalize_on_234(spec, state, 5, True) @with_all_phases @spec_state_test def test_234_poor_support(spec, state): - yield from finalize_on_234(spec, state, 5, 0.6) + yield from finalize_on_234(spec, state, 5, False) @with_all_phases @spec_state_test def test_23_ok_support(spec, state): - yield from finalize_on_23(spec, state, 4, 1.0) + yield from finalize_on_23(spec, state, 4, True) @with_all_phases @spec_state_test def test_23_poor_support(spec, state): - yield from finalize_on_23(spec, state, 4, 0.6) + yield from finalize_on_23(spec, state, 4, False) @with_all_phases @spec_state_test def test_123_ok_support(spec, state): - yield from finalize_on_123(spec, state, 4, 1.0) + yield from finalize_on_123(spec, state, 4, True) @with_all_phases @spec_state_test def test_123_poor_support(spec, state): - yield from finalize_on_123(spec, state, 4, 0.6) + yield from finalize_on_123(spec, state, 4, False) @with_all_phases @spec_state_test def test_12_ok_support(spec, state): - yield from finalize_on_12(spec, state, 3, 1.0) + yield from finalize_on_12(spec, state, 3, True) @with_all_phases @spec_state_test def test_12_poor_support(spec, state): - yield from finalize_on_12(spec, state, 3, 0.6) - - -# TODO: bring ratios closer to 2/3 for edge case testing. + yield from finalize_on_12(spec, state, 3, False) From 4ed7af7bacb2602d97ea67596b8b21d4b019b150 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 15:48:39 +0200 Subject: [PATCH 289/405] mock attestation refactor --- ..._process_justification_and_finalization.py | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 406bdbf7ff..66e2095b49 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -1,4 +1,3 @@ -import math from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( run_epoch_processing_with @@ -39,29 +38,37 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support else: raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") - committee_count = spec.get_epoch_committee_count(state, epoch) - indices = spec.get_active_validator_indices(state, epoch) - epoch_start_shard = spec.get_epoch_start_shard(state, epoch) + total_balance = spec.get_total_active_balance(state) + remaining_balance = total_balance * 2 // 3 + epoch_start_slot = spec.get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + spec.SLOTS_PER_EPOCH): for shard in get_shards_for_slot(spec, state, slot): - size = get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) + committee = spec.get_crosslink_committee(state, spec.slot_to_epoch(slot), shard) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. - attesting_count = math.ceil(size * 2 /3) - attesting_count = attesting_count if sufficient_support else attesting_count - 1 - aggregation_bits = [i < attesting_count for i in range(size)] - - attestations.append(spec.PendingAttestation( - aggregation_bits=aggregation_bits, - data=spec.AttestationData( - beacon_block_root=b'\xaa' * 32, - source=source, - target=target, - crosslink=spec.Crosslink(shard=shard) - ), - inclusion_delay=1, - )) + + aggregation_bits = [0] * len(committee) + for v in range(len(committee) * 2 // 3 + 1): + if remaining_balance > 0: + remaining_balance -= state.validators[v].effective_balance + aggregation_bits[v] = 1 + elif not sufficient_support: + aggregation_bits[v - 1] = 0 + break + else: + break + + attestations.append(spec.PendingAttestation( + aggregation_bits=aggregation_bits, + data=spec.AttestationData( + beacon_block_root=b'\xaa' * 32, + source=source, + target=target, + crosslink=spec.Crosslink(shard=shard) + ), + inclusion_delay=1, + )) def finalize_on_234(spec, state, epoch, sufficient_support): From b162a8ff8a40f372c1dcfc36b6030b4c9038f225 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 16:48:08 +0200 Subject: [PATCH 290/405] simplify list creation --- deposit_contract/tests/contracts/test_deposit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 94c7b29d1a..8c165f0f9b 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -145,5 +145,5 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input signature=deposit_input[2], )) - root = hash_tree_root(List[DepositData, 2**32](*(tuple(deposit_data_list)))) + root = hash_tree_root(List[DepositData, 2**32](*deposit_data_list)) assert root == registration_contract.functions.get_deposit_root().call() From 0c29c5125f0e890e5bdd7d9e14afd166d0b6b260 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 17:10:12 +0200 Subject: [PATCH 291/405] Finnish refactor --- .../test_process_justification_and_finalization.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 66e2095b49..50a0a0cab1 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -16,14 +16,6 @@ def get_shards_for_slot(spec, state, slot): return [shard + i for i in range(committees_per_slot)] -def get_committee_size(spec, epoch_start_shard, shard, committee_count, indices): - committee_index = (shard + spec.SHARD_COUNT - epoch_start_shard) % spec.SHARD_COUNT - start = (len(indices) * committee_index) // committee_count - end = (len(indices) * (committee_index + 1)) // committee_count - size = end - start - return size - - def add_mock_attestations(spec, state, epoch, source, target, sufficient_support=False): # we must be at the end of the epoch assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 From 5d633bfdf36c5b4a9e069e302cecd70bbd782b13 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 18:14:17 +0200 Subject: [PATCH 292/405] bugfix attestation creation so that it works on mainnet with multiple committees per slot, and improve bitfield index descriptions --- ..._process_justification_and_finalization.py | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 50a0a0cab1..d043d8d97e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -36,6 +36,11 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support epoch_start_slot = spec.get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + spec.SLOTS_PER_EPOCH): for shard in get_shards_for_slot(spec, state, slot): + # Check if we already have had sufficient balance. (and undone if we don't want it). + # If so, do not create more attestations. (we do not have empty pending attestations normally anyway) + if remaining_balance < 0: + return + committee = spec.get_crosslink_committee(state, spec.slot_to_epoch(slot), shard) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. @@ -45,22 +50,23 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support if remaining_balance > 0: remaining_balance -= state.validators[v].effective_balance aggregation_bits[v] = 1 - elif not sufficient_support: - aggregation_bits[v - 1] = 0 - break else: break - attestations.append(spec.PendingAttestation( - aggregation_bits=aggregation_bits, - data=spec.AttestationData( - beacon_block_root=b'\xaa' * 32, - source=source, - target=target, - crosslink=spec.Crosslink(shard=shard) - ), - inclusion_delay=1, - )) + # remove just one attester to make the marginal support insufficient + if not sufficient_support: + aggregation_bits[aggregation_bits.index(1)] = 0 + + attestations.append(spec.PendingAttestation( + aggregation_bits=aggregation_bits, + data=spec.AttestationData( + beacon_block_root=b'\xaa' * 32, + source=source, + target=target, + crosslink=spec.Crosslink(shard=shard) + ), + inclusion_delay=1, + )) def finalize_on_234(spec, state, epoch, sufficient_support): @@ -82,7 +88,7 @@ def finalize_on_234(spec, state, epoch, sufficient_support): state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[1:3] = [1, 1] # mock 2nd and 3rd latest epochs as justified + state.justification_bits[1:3] = [1, 1] # mock 3rd and 4th latest epochs as justified (indices are pre-shift) # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -93,12 +99,11 @@ def finalize_on_234(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c3 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c4 # finalized old previous justified epoch else: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c3 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized @@ -108,7 +113,8 @@ def finalize_on_23(spec, state, epoch, sufficient_support): state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch # 43210 -- epochs ago - # 3210x -- justification bitfield indices + # 210xx -- justification bitfield indices (pre shift) + # 3210x -- justification bitfield indices (post shift) # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) @@ -120,7 +126,7 @@ def finalize_on_23(spec, state, epoch, sufficient_support): state.previous_justified_checkpoint = c3 state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[1] = 1 # mock 2nd latest epoch as justified + state.justification_bits[1] = 1 # mock 3rd latest epoch as justified (indices are pre-shift) # mock the 2nd latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -131,12 +137,11 @@ def finalize_on_23(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c3 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c3 # finalized old previous justified epoch else: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c3 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized @@ -146,7 +151,8 @@ def finalize_on_123(spec, state, epoch, sufficient_support): state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch # 43210 -- epochs ago - # 3210x -- justification bitfield indices + # 210xx -- justification bitfield indices (pre shift) + # 3210x -- justification bitfield indices (post shift) # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) @@ -158,10 +164,10 @@ def finalize_on_123(spec, state, epoch, sufficient_support): state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint - state.previous_justified_checkpoint = c4 + state.previous_justified_checkpoint = c4 # not c3, otherwise finalize 23 would trigger. state.current_justified_checkpoint = c2 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[0:2] = [1, 1] # mock 1st and 2nd latest epochs as justified + state.justification_bits[0:2] = [1, 1] # mock 2nd and 3rd latest epochs as justified (indices are pre-shift) # mock the 1st latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 1, @@ -172,12 +178,11 @@ def finalize_on_123(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c2 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized old current else: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c2 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized @@ -187,7 +192,8 @@ def finalize_on_12(spec, state, epoch, sufficient_support): state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch # 43210 -- epochs ago - # 3210 -- justification bitfield indices + # 210xx -- justification bitfield indices (pre shift) + # 3210x -- justification bitfield indices (post shift) # 001*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) @@ -199,7 +205,7 @@ def finalize_on_12(spec, state, epoch, sufficient_support): state.previous_justified_checkpoint = c2 state.current_justified_checkpoint = c2 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[0] = 1 # mock latest epoch as justified + state.justification_bits[0] = 1 # mock 2nd latest epoch as justified (this is pre-shift) # mock the 1st latest epoch as justifiable, with 2nd as source add_mock_attestations(spec, state, epoch=epoch - 1, @@ -210,12 +216,11 @@ def finalize_on_12(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c2 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized previous justified epoch else: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c2 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized From f0a8e392434fa06e96b84249c4dd9533d8012cfb Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 17:29:21 +0100 Subject: [PATCH 293/405] WIP --- specs/core/0_beacon-chain.md | 4 +- .../pyspec/eth2spec/test/helpers/deposits.py | 40 ++++++++----------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6f28c0e3d3..138d3fd2ae 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -437,7 +437,7 @@ class Attestation(Container): ```python class Deposit(Container): - proof: Vector[Hash, DEPOSIT_CONTRACT_TREE_DEPTH] # Merkle path to deposit root + proof: Vector[Hash, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to deposit root data: DepositData ``` @@ -1682,7 +1682,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: assert verify_merkle_branch( leaf=hash_tree_root(deposit.data), proof=deposit.proof, - depth=DEPOSIT_CONTRACT_TREE_DEPTH, + depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, index=state.eth1_deposit_index, root=state.eth1_data.deposit_root, ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 20ff7440f7..5a0909185c 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -1,11 +1,13 @@ from eth2spec.test.helpers.keys import pubkeys, privkeys from eth2spec.utils.bls import bls_sign from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_root, get_merkle_proof -from eth2spec.utils.ssz.ssz_impl import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root +from eth2spec.utils.ssz.ssz_typing import List +from eth2spec.phase0.spec import DepositData def build_deposit_data(spec, state, pubkey, privkey, amount, withdrawal_credentials, signed=False): - deposit_data = spec.DepositData( + deposit_data = DepositData( pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, amount=amount, @@ -29,27 +31,21 @@ def sign_deposit_data(spec, state, deposit_data, privkey): def build_deposit(spec, state, - deposit_data_leaves, + deposit_data_list, pubkey, privkey, amount, withdrawal_credentials, signed): deposit_data = build_deposit_data(spec, state, pubkey, privkey, amount, withdrawal_credentials, signed) - - item = deposit_data.hash_tree_root() - index = len(deposit_data_leaves) - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=index)) - assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) - - deposit = spec.Deposit( - proof=list(proof), - index=index, - data=deposit_data, - ) + deposit_data_list.append(deposit_data) + index = len(deposit_data_list) + root = hash_tree_root(List[DepositData, 2**32](*deposit_data_list)) + tree = calc_merkle_tree_from_leaves(tuple([d.hash_tree_root() for d in deposit_data_list])) + proof = list(get_merkle_proof(tree, item_index=index)) + [index.to_bytes(32, 'little')] + leaf = deposit_data.hash_tree_root() + assert spec.verify_merkle_branch(leaf, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH + 1, index, root) + deposit = spec.Deposit(proof, index, deposit_data) return deposit, root, deposit_data_leaves @@ -58,9 +54,7 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c """ Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. """ - pre_validator_count = len(state.validators) - # fill previous deposits with zero-hash - deposit_data_leaves = [spec.ZERO_HASH] * pre_validator_count + deposit_data_list = [] pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] @@ -69,10 +63,10 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c if withdrawal_credentials is None: withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(pubkey)[1:] - deposit, root, deposit_data_leaves = build_deposit( + deposit, root, deposit_data_list = build_deposit( spec, state, - deposit_data_leaves, + deposit_data_list, pubkey, privkey, amount, @@ -81,5 +75,5 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c ) state.eth1_data.deposit_root = root - state.eth1_data.deposit_count = len(deposit_data_leaves) + state.eth1_data.deposit_count = len(deposit_data_list) return deposit From bc8df3cba3ccef7ce28b202720d6f698971e5126 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 29 Jun 2019 12:04:56 -0500 Subject: [PATCH 294/405] minor typo Co-Authored-By: Alex Stokes --- specs/light_client/sync_protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index a29ed05c8f..cb7b6ce109 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -40,7 +40,7 @@ Note that there is now a new way to compute `get_active_validator_indices`: ```python def get_active_validator_indices(state: ExtendedBeaconState, epoch: Epoch) -> List[ValidatorIndex]: - return state.active_indices[epoch % compact_committees_rootS_LENGTH] + return state.active_indices[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` Note that it takes `state` instead of `state.validators` as an argument. This does not affect its use in `get_shuffled_committee`, because `get_shuffled_committee` has access to the full `state` as one of its arguments. From d0009b09807b11b970866cd4cffea075e602fee8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 30 Jun 2019 01:12:25 +0800 Subject: [PATCH 295/405] Pass `genesis_eth1_block_hash` instead of `genesis_eth1_data` --- specs/core/0_beacon-chain.md | 15 ++++++--------- .../pyspec/eth2spec/test/genesis/test_genesis.py | 16 +++++----------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7cd5725b10..1ddf58b056 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1157,10 +1157,7 @@ When `is_genesis_trigger(deposits, time) is True` for the first time let: * `genesis_deposits = deposits` * `genesis_time = time - time % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` -* `genesis_eth1_data` be the object of type `Eth1Data` where: - * `genesis_eth1_data.deposit_root` is the deposit root for the last deposit in `deposits` - * `genesis_eth1_data.deposit_count = len(genesis_deposits)` - * `genesis_eth1_data.block_hash` is the Eth 1.0 block hash that emitted the log for the last deposit in `deposits` +* `genesis_eth1_block_hash` is the Eth 1.0 block hash that emitted the log for the last deposit in `deposits` *Note*: The function `is_genesis_trigger` has yet to be agreed upon by the community, and can be updated as necessary. We define the following testing placeholder: @@ -1176,6 +1173,7 @@ def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], leaves = list(map(lambda deposit: hash_tree_root(deposit.data), deposits)) for deposit_index, deposit in enumerate(deposits): state.eth1_data.deposit_root = get_merkle_root(leaves[:deposit_index + 1], 2**DEPOSIT_CONTRACT_TREE_DEPTH) + state.eth1_data.deposit_count = deposit_index + 1 state.eth1_deposit_index = deposit_index process_deposit(state, deposit) @@ -1191,13 +1189,13 @@ def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], ### Genesis state -Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_data)`. +Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_block_hash)`. ```python -def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: +def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, genesis_eth1_block_hash: Hash) -> BeaconState: state = BeaconState( genesis_time=genesis_time, - eth1_data=eth1_data, + eth1_data=Eth1Data(block_hash=genesis_eth1_block_hash), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) @@ -1205,11 +1203,10 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth leaves = list(map(lambda deposit: hash_tree_root(deposit.data), deposits)) for deposit_index, deposit in enumerate(deposits): state.eth1_data.deposit_root = get_merkle_root(leaves[:deposit_index + 1], 2**DEPOSIT_CONTRACT_TREE_DEPTH) + state.eth1_data.deposit_count = deposit_index + 1 state.eth1_deposit_index = deposit_index process_deposit(state, deposit) - assert state.eth1_data.deposit_root == eth1_data.deposit_root - # Process genesis activations for validator in state.validators: if validator.effective_balance == MAX_EFFECTIVE_BALANCE: diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py b/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py index a89ec8793f..0d2b4fbac0 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py @@ -10,28 +10,22 @@ def test_genesis(spec): deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) genesis_time = 1546300800 - block_hash = b'\x12' * 32 + genesis_eth1_block_hash = b'\x12' * 32 yield "deposits", genesis_deposits - yield "time", genesis_time + yield "genesis_time", genesis_time - genesis_eth1_data = spec.Eth1Data( - deposit_root=deposit_root, - deposit_count=deposit_count, - block_hash=block_hash, - ) - - yield "eth1_data", genesis_eth1_data + yield "genesis_eth1_block_hash", genesis_eth1_block_hash genesis_state = spec.get_genesis_beacon_state( genesis_deposits, genesis_time, - genesis_eth1_data, + genesis_eth1_block_hash, ) assert genesis_state.genesis_time == genesis_time assert len(genesis_state.validators) == deposit_count assert genesis_state.eth1_data.deposit_root == deposit_root assert genesis_state.eth1_data.deposit_count == deposit_count - assert genesis_state.eth1_data.block_hash == block_hash + assert genesis_state.eth1_data.block_hash == genesis_eth1_block_hash yield "state", genesis_state From 1b66323806505a465236f3adb924ea3a777e02cb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 30 Jun 2019 01:44:17 +0800 Subject: [PATCH 296/405] head to 1229 --- scripts/build_spec.py | 6 ++-- specs/core/0_beacon-chain.md | 33 ++----------------- .../pyspec/eth2spec/test/helpers/deposits.py | 25 +++++++------- 3 files changed, 17 insertions(+), 47 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index e4e95c7ec4..1f5fe1ee61 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -11,8 +11,7 @@ ) -PHASE0_IMPORTS = '''from math import log2 -from typing import ( +PHASE0_IMPORTS = '''from typing import ( Any, Callable, Dict, Set, Sequence, Tuple, ) @@ -37,8 +36,7 @@ from eth2spec.utils.hash_function import hash ''' -PHASE1_IMPORTS = '''from math import log2 -from typing import ( +PHASE1_IMPORTS = '''from typing import ( Any, Callable, Dict, Optional, Set, Sequence, MutableSequence, Tuple, ) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f9d529e0c9..e51fbde3ee 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -52,8 +52,6 @@ - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) - [`signing_root`](#signing_root) - - [`calc_merkle_tree_from_leaves`](#calc_merkle_tree_from_leaves) - - [`get_merkle_root`](#get_merkle_root) - [`bls_domain`](#bls_domain) - [`slot_to_epoch`](#slot_to_epoch) - [`get_previous_epoch`](#get_previous_epoch) @@ -562,33 +560,6 @@ The `hash` function is SHA256. `def signing_root(object: Container) -> Hash` is a function for computing signing messages, as defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers). -### `calc_merkle_tree_from_leaves` - -```python -zerohashes = [ZERO_HASH] -for layer in range(1, 100): - zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1])) -def calc_merkle_tree_from_leaves(values: Sequence[Hash], layer_count: int=32) -> Sequence[Sequence[Hash]]: - values = list(values) - tree = [values[::]] - for h in range(layer_count): - if len(values) % 2 == 1: - values.append(zerohashes[h]) - values = [hash(values[i] + values[i + 1]) for i in range(0, len(values), 2)] - tree.append(values[::]) - return tree -``` - -### `get_merkle_root` - -```python -def get_merkle_root(values: Sequence[Hash], pad_to: int=1) -> Hash: - layer_count = int(log2(pad_to)) - if len(values) == 0: - return zerohashes[layer_count] - return calc_merkle_tree_from_leaves(values, layer_count)[-1][0] -``` - ### `bls_domain` ```python @@ -1146,7 +1117,7 @@ def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], state = BeaconState() leaves = list(map(lambda deposit: hash_tree_root(deposit.data), deposits)) for deposit_index, deposit in enumerate(deposits): - state.eth1_data.deposit_root = get_merkle_root(leaves[:deposit_index + 1], 2**DEPOSIT_CONTRACT_TREE_DEPTH) + state.eth1_data.deposit_root = hash_tree_root(leaves) state.eth1_data.deposit_count = deposit_index + 1 state.eth1_deposit_index = deposit_index process_deposit(state, deposit) @@ -1176,7 +1147,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, gen # Process genesis deposits leaves = list(map(lambda deposit: hash_tree_root(deposit.data), deposits)) for deposit_index, deposit in enumerate(deposits): - state.eth1_data.deposit_root = get_merkle_root(leaves[:deposit_index + 1], 2**DEPOSIT_CONTRACT_TREE_DEPTH) + state.eth1_data.deposit_root = hash_tree_root(leaves) state.eth1_data.deposit_count = deposit_index + 1 state.eth1_deposit_index = deposit_index process_deposit(state, deposit) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index ac842405ee..a0d59b9a69 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -47,10 +47,10 @@ def build_deposit(spec, spec, pubkey, privkey, amount, withdrawal_credentials, state=state, signed=signed, ) - deposit_data = build_deposit_data(spec, state, pubkey, privkey, amount, withdrawal_credentials, signed) + deposit_data = build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=state, signed=signed) deposit_data_list.append(deposit_data) index = len(deposit_data_list) - root = hash_tree_root(List[DepositData, 2**32](*deposit_data_list)) + root = hash_tree_root(List[DepositData, 2**spec.DEPOSIT_CONTRACT_TREE_DEPTH](*deposit_data_list)) tree = calc_merkle_tree_from_leaves(tuple([d.hash_tree_root() for d in deposit_data_list])) proof = list(get_merkle_proof(tree, item_index=index)) + [index.to_bytes(32, 'little')] leaf = deposit_data.hash_tree_root() @@ -61,7 +61,7 @@ def build_deposit(spec, def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False): - deposit_data_leaves = [] + deposit_data_list = [] genesis_deposits = [] for validator_index in range(genesis_validator_count): pubkey = pubkeys[validator_index] @@ -73,16 +73,17 @@ def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False withdrawal_credentials=withdrawal_credentials, amount=amount, ) - if signed: - sign_deposit_data(spec, deposit_data, privkey) # state=None - item = deposit_data.hash_tree_root() - deposit_data_leaves.append(item) - - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves), spec.DEPOSIT_CONTRACT_TREE_DEPTH) - root = get_merkle_root((tuple(deposit_data_leaves)), 2**spec.DEPOSIT_CONTRACT_TREE_DEPTH) - genesis_deposits.append( - spec.Deposit(proof=list(get_merkle_proof(tree, item_index=validator_index)), data=deposit_data) + deposit, root, deposit_data_list = build_deposit( + spec, + None, + deposit_data_list, + pubkey, + privkey, + amount, + withdrawal_credentials, + signed, ) + genesis_deposits.append(deposit) return genesis_deposits, root From 6b81e747b25a0b478099c594abd640317d6f6767 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 29 Jun 2019 18:50:23 +0100 Subject: [PATCH 297/405] Fix comment --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 619bd3c01c..b5e36bcf0e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1167,7 +1167,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots + # Populate compact_committees_roots genesis_committee_root = get_compact_committees_root(state, GENESIS_EPOCH) for index in range(EPOCHS_PER_HISTORICAL_VECTOR): state.compact_committees_roots[index] = genesis_committee_root From c8dc30eec20bd8116e94ff8d38e1082ec650eb0f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 30 Jun 2019 01:56:10 +0800 Subject: [PATCH 298/405] Recover from auto-merge --- specs/core/0_beacon-chain.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e51fbde3ee..f62469d593 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -124,8 +124,7 @@ This document represents the specification for Phase 0 of Ethereum 2.0 -- The Beacon Chain. -At the core of Ethereum 2.0 is a system chain called the "beacon chain". The beacon chain stores and manages the registry of [validators](#dfn-validator). In the initial deployment phases of Ethereum 2.0, the only mechanism to become a [validator](#dfn-validator) is to make a one-way ETH transaction to a deposit contract on Eth 1.0. Activation as a [validator](#dfn-validator) happens when Eth 1.0 deposit receipts are processed by the beacon chain, the activation balance is reached, and a queuing process is completed. Exit is either voluntary or done forcibly as a penalty for misbehavior. - +At the core of Ethereum 2.0 is a system chain called the "beacon chain". The beacon chain stores and manages the registry of [validators](#dfn-validator). In the initial deployment phases of Ethereum 2.0, the only mechanism to become a [validator](#dfn-validator) is to make a one-way ETH transaction to a deposit contract on Ethereum 1.0. Activation as a [validator](#dfn-validator) happens when Ethereum 1.0 deposit receipts are processed by the beacon chain, the activation balance is reached, and a queuing process is completed. Exit is either voluntary or done forcibly as a penalty for misbehavior. The primary source of load on the beacon chain is "attestations". Attestations are simultaneously availability votes for a shard block and proof-of-stake votes for a beacon block. A sufficient number of attestations for the same shard block create a "crosslink", confirming the shard segment up to that shard block into the beacon chain. Crosslinks also serve as infrastructure for asynchronous cross-shard communication. ## Notation @@ -134,7 +133,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code. ## Terminology -* **Validator**—a registered participant in the beacon chain. You can become one by sending ether into the Eth 1.0 deposit contract. +* **Validator**—a registered participant in the beacon chain. You can become one by sending ether into the Ethereum 1.0 deposit contract. * **Active validator**—an active participant in the Ethereum 2.0 consensus invited to, among other things, propose and attest to blocks and vote for crosslinks. * **Committee**—a (pseudo-) randomly sampled subset of [active validators](#dfn-active-validator). When a committee is referred to collectively, as in "this committee attests to X", this is assumed to mean "some subset of that committee that contains enough [validators](#dfn-validator) that the protocol recognizes it as representing the committee". * **Proposer**—the [validator](#dfn-validator) that creates a beacon chain block. @@ -194,7 +193,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_GENESIS_TIME` | `1578009600` (Jan 3, 2020) | | `JUSTIFICATION_BITS_LENGTH` | `4` | -* For the safety of cr>>osslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) +* For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) ### Gwei values @@ -1095,14 +1094,14 @@ def slash_validator(state: BeaconState, Before genesis has been triggered and for every Eth 1.0 block call `is_genesis_trigger(deposits, time)` where: -* `deposits` is the list of all deposits up to the Eth 1.0 block, ordered chronologically -* `time` is the Unix time of the Eth 1.0 block +* `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log +* `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log -When `is_genesis_trigger(deposits, time) is True` for the first time let: +When `is_genesis_trigger(deposits, timestamp) is True` for the first time, let: * `genesis_deposits = deposits` -* `genesis_time = time - time % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` -* `genesis_eth1_block_hash` is the Eth 1.0 block hash that emitted the log for the last deposit in `deposits` +* `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` +* `genesis_eth1_block_hash` is the Ethereum 1.0 block hash that emitted the log for the last deposit in `deposits` *Note*: The function `is_genesis_trigger` has yet to be agreed upon by the community, and can be updated as necessary. We define the following testing placeholder: From 354cd1c1e19fa1008e43b3861da308d9472a5e12 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 30 Jun 2019 02:02:05 +0800 Subject: [PATCH 299/405] Clean up leftover and linter --- specs/core/0_beacon-chain.md | 10 ++++++---- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 7 +------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f62469d593..5aa57c2942 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1092,7 +1092,7 @@ def slash_validator(state: BeaconState, ### Genesis trigger -Before genesis has been triggered and for every Eth 1.0 block call `is_genesis_trigger(deposits, time)` where: +Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool` where: * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log @@ -1106,10 +1106,10 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time, let: *Note*: The function `is_genesis_trigger` has yet to be agreed upon by the community, and can be updated as necessary. We define the following testing placeholder: ```python -def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], time: uint64) -> bool: +def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], timestamp: uint64) -> bool: SECONDS_PER_DAY = 86400 # Do not deploy too early - if time - time % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY < MIN_GENESIS_TIME: + if timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY < MIN_GENESIS_TIME: return False # Process deposits @@ -1136,7 +1136,9 @@ def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_block_hash)`. ```python -def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, genesis_eth1_block_hash: Hash) -> BeaconState: +def get_genesis_beacon_state(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], + genesis_time: int, + genesis_eth1_block_hash: Hash) -> BeaconState: state = BeaconState( genesis_time=genesis_time, eth1_data=Eth1Data(block_hash=genesis_eth1_block_hash), diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index a0d59b9a69..39ecbd2d77 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -1,6 +1,6 @@ from eth2spec.test.helpers.keys import pubkeys, privkeys from eth2spec.utils.bls import bls_sign -from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_root, get_merkle_proof +from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_proof from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root from eth2spec.utils.ssz.ssz_typing import List from eth2spec.phase0.spec import DepositData @@ -68,11 +68,6 @@ def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False privkey = privkeys[validator_index] # insecurely use pubkey as withdrawal key if no credentials provided withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(pubkey)[1:] - deposit_data = spec.DepositData( - pubkey=pubkey, - withdrawal_credentials=withdrawal_credentials, - amount=amount, - ) deposit, root, deposit_data_list = build_deposit( spec, None, From e4eebef3a4098e49123895532226fcabe87b768d Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 20:09:43 +0200 Subject: [PATCH 300/405] fix broken deposits test, now same error as others --- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 9 ++------- .../phase_0/block_processing/test_process_deposit.py | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 39ecbd2d77..7f5975dd4a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -3,11 +3,10 @@ from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_proof from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root from eth2spec.utils.ssz.ssz_typing import List -from eth2spec.phase0.spec import DepositData def build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=None, signed=False): - deposit_data = DepositData( + deposit_data = spec.DepositData( pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, amount=amount, @@ -43,14 +42,10 @@ def build_deposit(spec, amount, withdrawal_credentials, signed): - deposit_data = build_deposit_data( - spec, pubkey, privkey, amount, withdrawal_credentials, state=state, signed=signed, - ) - deposit_data = build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=state, signed=signed) deposit_data_list.append(deposit_data) index = len(deposit_data_list) - root = hash_tree_root(List[DepositData, 2**spec.DEPOSIT_CONTRACT_TREE_DEPTH](*deposit_data_list)) + root = hash_tree_root(List[spec.DepositData, 2**spec.DEPOSIT_CONTRACT_TREE_DEPTH](*deposit_data_list)) tree = calc_merkle_tree_from_leaves(tuple([d.hash_tree_root() for d in deposit_data_list])) proof = list(get_merkle_proof(tree, item_index=index)) + [index.to_bytes(32, 'little')] leaf = deposit_data.hash_tree_root() diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index d0a62ff6bf..d596523ec7 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -117,7 +117,7 @@ def test_invalid_withdrawal_credentials_top_up(spec, state): @with_all_phases @spec_state_test def test_wrong_deposit_for_deposit_count(spec, state): - deposit_data_leaves = [spec.ZERO_HASH] * len(state.validators) + deposit_data_leaves = [spec.DepositData() for _ in range(len(state.validators))] # build root for deposit_1 index_1 = len(deposit_data_leaves) From fb165dc14e11cafb273f96f8e75352cc73df73da Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 20:33:11 +0200 Subject: [PATCH 301/405] fixes a few tests, not all --- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 7f5975dd4a..99c39211e7 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -43,14 +43,14 @@ def build_deposit(spec, withdrawal_credentials, signed): deposit_data = build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=state, signed=signed) - deposit_data_list.append(deposit_data) index = len(deposit_data_list) + deposit_data_list.append(deposit_data) root = hash_tree_root(List[spec.DepositData, 2**spec.DEPOSIT_CONTRACT_TREE_DEPTH](*deposit_data_list)) tree = calc_merkle_tree_from_leaves(tuple([d.hash_tree_root() for d in deposit_data_list])) - proof = list(get_merkle_proof(tree, item_index=index)) + [index.to_bytes(32, 'little')] + proof = list(get_merkle_proof(tree, item_index=index)) + [(index + 1).to_bytes(32, 'little')] leaf = deposit_data.hash_tree_root() assert spec.verify_merkle_branch(leaf, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH + 1, index, root) - deposit = spec.Deposit(proof, index, deposit_data) + deposit = spec.Deposit(proof=proof, data=deposit_data) return deposit, root, deposit_data_list From ff185c348697c024f1ebc7e2706cd3b1717eec36 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 20:40:25 +0200 Subject: [PATCH 302/405] fix deposit state mocking for tests --- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 99c39211e7..b46363e62a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -102,6 +102,7 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c signed, ) + state.eth1_deposit_index = 0 state.eth1_data.deposit_root = root state.eth1_data.deposit_count = len(deposit_data_list) return deposit From 125660c5af055209c8bdeb422947f85e4b518626 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 30 Jun 2019 03:20:11 +0800 Subject: [PATCH 303/405] Update input `deposits` type from `Sequence[Deposit]` to `List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH` and fix tests --- specs/core/0_beacon-chain.md | 17 ++++++++++------- .../pyspec/eth2spec/test/helpers/deposits.py | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 5aa57c2942..a77ec77a34 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1092,9 +1092,8 @@ def slash_validator(state: BeaconState, ### Genesis trigger -Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool` where: - -* `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log +Before genesis has been triggered and for every Ethereum 1.0 block call `is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool` where: +* `deposits` is the SSZ list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log When `is_genesis_trigger(deposits, timestamp) is True` for the first time, let: @@ -1114,9 +1113,11 @@ def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], # Process deposits state = BeaconState() - leaves = list(map(lambda deposit: hash_tree_root(deposit.data), deposits)) + leaves = list(map(lambda deposit: deposit.data, deposits)) for deposit_index, deposit in enumerate(deposits): - state.eth1_data.deposit_root = hash_tree_root(leaves) + state.eth1_data.deposit_root = hash_tree_root( + List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:deposit_index + 1]) + ) state.eth1_data.deposit_count = deposit_index + 1 state.eth1_deposit_index = deposit_index process_deposit(state, deposit) @@ -1146,9 +1147,11 @@ def get_genesis_beacon_state(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DE ) # Process genesis deposits - leaves = list(map(lambda deposit: hash_tree_root(deposit.data), deposits)) + leaves = list(map(lambda deposit: deposit.data, deposits)) for deposit_index, deposit in enumerate(deposits): - state.eth1_data.deposit_root = hash_tree_root(leaves) + state.eth1_data.deposit_root = hash_tree_root( + List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:deposit_index + 1]) + ) state.eth1_data.deposit_count = deposit_index + 1 state.eth1_deposit_index = deposit_index process_deposit(state, deposit) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index b46363e62a..f79dcf3ffe 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -75,7 +75,7 @@ def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False ) genesis_deposits.append(deposit) - return genesis_deposits, root + return List[spec.Deposit, 2**spec.DEPOSIT_CONTRACT_TREE_DEPTH](*genesis_deposits), root def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_credentials=None, signed=False): From 56caa483144d26f0f5cd09b5128fad2c259e9e93 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 30 Jun 2019 03:27:25 +0800 Subject: [PATCH 304/405] Should use Sequence --- specs/core/0_beacon-chain.md | 6 +++--- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a77ec77a34..c70fc006ed 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1093,7 +1093,7 @@ def slash_validator(state: BeaconState, ### Genesis trigger Before genesis has been triggered and for every Ethereum 1.0 block call `is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool` where: -* `deposits` is the SSZ list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log +* `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log When `is_genesis_trigger(deposits, timestamp) is True` for the first time, let: @@ -1105,7 +1105,7 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time, let: *Note*: The function `is_genesis_trigger` has yet to be agreed upon by the community, and can be updated as necessary. We define the following testing placeholder: ```python -def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], timestamp: uint64) -> bool: +def is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool: SECONDS_PER_DAY = 86400 # Do not deploy too early if timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY < MIN_GENESIS_TIME: @@ -1137,7 +1137,7 @@ def is_genesis_trigger(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_block_hash)`. ```python -def get_genesis_beacon_state(deposits: List[Deposit, 2**DEPOSIT_CONTRACT_TREE_DEPTH], +def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, genesis_eth1_block_hash: Hash) -> BeaconState: state = BeaconState( diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index f79dcf3ffe..b46363e62a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -75,7 +75,7 @@ def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False ) genesis_deposits.append(deposit) - return List[spec.Deposit, 2**spec.DEPOSIT_CONTRACT_TREE_DEPTH](*genesis_deposits), root + return genesis_deposits, root def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_credentials=None, signed=False): From 302b3afe2ae2dcb0287096dec8b8121dca5d248f Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 21:37:54 +0200 Subject: [PATCH 305/405] rename/fix roots in justification tests for consistency --- .../test_process_justification_and_finalization.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index d043d8d97e..c76692a955 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -77,8 +77,8 @@ def finalize_on_234(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices # 11*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xdd' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) state.block_roots[spec.get_epoch_start_slot(c4.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c4.root state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root @@ -117,7 +117,7 @@ def finalize_on_23(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices (post shift) # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root @@ -155,10 +155,10 @@ def finalize_on_123(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices (post shift) # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xdd' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xaa' * 32) state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root @@ -197,7 +197,7 @@ def finalize_on_12(spec, state, epoch, sufficient_support): # 001*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xaa' * 32) state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root From afb33ddc5bff4a8d2ad4b456fd33587838054ba4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 21:39:10 +0200 Subject: [PATCH 306/405] fix typo in justification wording --- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 019563978d..b2eb192442 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -127,7 +127,7 @@ def test_empty_epoch_transition(spec, state): @spec_state_test def test_empty_epoch_transition_not_finalizing(spec, state): # Don't run for non-minimal configs, it takes very long, and the effect - # of calling finalization/justifcation is just the same as with the minimal configuration. + # of calling finalization/justification is just the same as with the minimal configuration. if spec.SLOTS_PER_EPOCH > 8: return From 12dff5349df00d22af0f8897bca1bd114f65cdbd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 30 Jun 2019 03:42:00 +0800 Subject: [PATCH 307/405] Renames: 1. `Deposit` log -> `DepositEvent` log 2. `get_deposit_root` -> `get_hash_tree_root` --- .../contracts/validator_registration.json | 2 +- .../contracts/validator_registration.v.py | 12 ++++++------ deposit_contract/tests/contracts/test_deposit.py | 8 ++++---- specs/core/0_beacon-chain.md | 2 +- specs/core/0_deposit-contract.md | 6 +++--- specs/validator/0_beacon-chain-validator.md | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index a3d3092696..527ce60871 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 91674}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334417}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b50506112ff56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63c5f2892f600051141561050857341561029057600080fd5b6000610140526101405161016052600154610180526101a060006020818352015b60016001610180511614156103325760006101a051602081106102d357600080fd5b600060c052602060c02001546020826102400101526020810190506101605160208261024001015260208101905080610240526102409050602060c0825160208401600060025af161032457600080fd5b60c0519050610160526103a0565b6000610160516020826101c00101526020810190506101a0516020811061035857600080fd5b600260c052602060c02001546020826101c0010152602081019050806101c0526101c09050602060c0825160208401600060025af161039657600080fd5b60c0519050610160525b61018060026103ae57600080fd5b60028151048152505b81516001018083528114156102b1575b505060006101605160208261044001015260208101905061014051610160516101805163806732896102c0526001546102e0526102e0516006580161009b565b506103405260006103a0525b6103405160206001820306601f82010390506103a0511015156104355761044e565b6103a05161036001526103a0516020016103a052610413565b61018052610160526101405261034060088060208461044001018260208501600060046012f150508051820191505060006018602082066103c0016020828401111561049957600080fd5b6020806103e082610140600060046015f150508181528090509050905060188060208461044001018260208501600060046014f150508051820191505080610440526104409050602060c0825160208401600060025af16104f957600080fd5b60c051905060005260206000f3005b63621fd130600051141561061a57341561052157600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f82010390506102205110151561056c57610585565b610220516101e00152610220516020016102205261054a565b6101c0805160200180610280828460006004600a8704601201f16105a857600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156105da576105f6565b60006102e0516102a001535b81516001018083528114156105ca575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d600051141561117457606060046101403760506004356004016101a037603060043560040135111561065057600080fd5b604060243560040161022037602060243560040135111561067057600080fd5b608060443560040161028037606060443560040135111561069057600080fd5b63ffffffff600154106106a257600080fd5b633b9aca0061034052610340516106b857600080fd5b61034051340461032052633b9aca006103205110156106d657600080fd5b60306101a051146106e657600080fd5b602061022051146106f657600080fd5b6060610280511461070657600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a051101515610796576107af565b6104a05161046001526104a0516020016104a052610774565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f161081657600080fd5b50506101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a05163806732896104c0526001546104e0526104e0516006580161009b565b506105405260006105a0525b6105405160206001820306601f82010390506105a0511015156108c7576108e0565b6105a05161056001526105a0516020016105a0526108a5565b6104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526105408051602001806105c0828460006004600a8704601201f161096757600080fd5b505060a06106405261064051610680526101a08051602001806106405161068001828460006004600a8704601201f161099f57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516040818352015b83610620511015156109dd576109fa565b6000610620516020850101535b81516001018083528114156109cc575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106a0526102208051602001806106405161068001828460006004600a8704601201f1610a5157600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610a8f57610aac565b6000610620516020850101535b8151600101808352811415610a7e575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106c0526103608051602001806106405161068001828460006004600a8704601201f1610b0357600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610b4157610b5e565b6000610620516020850101535b8151600101808352811415610b30575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106e0526102808051602001806106405161068001828460006004600a8704601201f1610bb557600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516060818352015b8361062051101515610bf357610c10565b6000610620516020850101535b8151600101808352811415610be2575b50505050602061064051610680015160206001820306601f82010390506106405101016106405261064051610700526105c08051602001806106405161068001828460006004600a8704601201f1610c6757600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610ca557610cc2565b6000610620516020850101535b8151600101808352811415610c94575b50505050602061064051610680015160206001820306601f8201039050610640510101610640527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce61064051610680a160006107205260006101a06030806020846107e001018260208501600060046016f150508051820191505060006010602082066107600160208284011115610d5957600080fd5b60208061078082610720600060046015f15050818152809050905090506010806020846107e001018260208501600060046013f1505080518201915050806107e0526107e09050602060c0825160208401600060025af1610db957600080fd5b60c0519050610740526000600060406020820661088001610280518284011115610de257600080fd5b6060806108a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af1610e2257600080fd5b60c0519050602082610a800101526020810190506000604060206020820661094001610280518284011115610e5657600080fd5b606080610960826020602088068803016102800160006004601bf1505081815280905090509050602080602084610a0001018260208501600060046015f150508051820191505061072051602082610a0001015260208101905080610a0052610a009050602060c0825160208401600060025af1610ed357600080fd5b60c0519050602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610f0a57600080fd5b60c0519050610860526000600061074051602082610b20010152602081019050610220602080602084610b2001018260208501600060046015f150508051820191505080610b2052610b209050602060c0825160208401600060025af1610f7057600080fd5b60c0519050602082610ca00101526020810190506000610360600880602084610c2001018260208501600060046012f15050805182019150506000601860208206610ba00160208284011115610fc557600080fd5b602080610bc082610720600060046015f1505081815280905090509050601880602084610c2001018260208501600060046014f150508051820191505061086051602082610c2001015260208101905080610c2052610c209050602060c0825160208401600060025af161103857600080fd5b60c0519050602082610ca001015260208101905080610ca052610ca09050602060c0825160208401600060025af161106f57600080fd5b60c0519050610b0052600180546001825401101561108c57600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d20511614156110dc57610b0051610d4051602081106110cb57600080fd5b600060c052602060c0200155611170565b6000610d4051602081106110ef57600080fd5b600060c052602060c0200154602082610d60010152602081019050610b0051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af161114057600080fd5b60c0519050610b0052610d20600261115757600080fd5b60028151048152505b81516001018083528114156110a7575b5050005b60006000fd5b6101856112ff036101856000396101856112ff036000f3"} \ No newline at end of file +{"abi": [{"name": "DepositEvent", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_hash_tree_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 91674}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334417}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b50506112ff56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63863a311b600051141561050857341561029057600080fd5b6000610140526101405161016052600154610180526101a060006020818352015b60016001610180511614156103325760006101a051602081106102d357600080fd5b600060c052602060c02001546020826102400101526020810190506101605160208261024001015260208101905080610240526102409050602060c0825160208401600060025af161032457600080fd5b60c0519050610160526103a0565b6000610160516020826101c00101526020810190506101a0516020811061035857600080fd5b600260c052602060c02001546020826101c0010152602081019050806101c0526101c09050602060c0825160208401600060025af161039657600080fd5b60c0519050610160525b61018060026103ae57600080fd5b60028151048152505b81516001018083528114156102b1575b505060006101605160208261044001015260208101905061014051610160516101805163806732896102c0526001546102e0526102e0516006580161009b565b506103405260006103a0525b6103405160206001820306601f82010390506103a0511015156104355761044e565b6103a05161036001526103a0516020016103a052610413565b61018052610160526101405261034060088060208461044001018260208501600060046012f150508051820191505060006018602082066103c0016020828401111561049957600080fd5b6020806103e082610140600060046015f150508181528090509050905060188060208461044001018260208501600060046014f150508051820191505080610440526104409050602060c0825160208401600060025af16104f957600080fd5b60c051905060005260206000f3005b63621fd130600051141561061a57341561052157600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f82010390506102205110151561056c57610585565b610220516101e00152610220516020016102205261054a565b6101c0805160200180610280828460006004600a8704601201f16105a857600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156105da576105f6565b60006102e0516102a001535b81516001018083528114156105ca575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d600051141561117457606060046101403760506004356004016101a037603060043560040135111561065057600080fd5b604060243560040161022037602060243560040135111561067057600080fd5b608060443560040161028037606060443560040135111561069057600080fd5b63ffffffff600154106106a257600080fd5b633b9aca0061034052610340516106b857600080fd5b61034051340461032052633b9aca006103205110156106d657600080fd5b60306101a051146106e657600080fd5b602061022051146106f657600080fd5b6060610280511461070657600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a051101515610796576107af565b6104a05161046001526104a0516020016104a052610774565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f161081657600080fd5b50506101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a05163806732896104c0526001546104e0526104e0516006580161009b565b506105405260006105a0525b6105405160206001820306601f82010390506105a0511015156108c7576108e0565b6105a05161056001526105a0516020016105a0526108a5565b6104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526105408051602001806105c0828460006004600a8704601201f161096757600080fd5b505060a06106405261064051610680526101a08051602001806106405161068001828460006004600a8704601201f161099f57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516040818352015b83610620511015156109dd576109fa565b6000610620516020850101535b81516001018083528114156109cc575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106a0526102208051602001806106405161068001828460006004600a8704601201f1610a5157600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610a8f57610aac565b6000610620516020850101535b8151600101808352811415610a7e575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106c0526103608051602001806106405161068001828460006004600a8704601201f1610b0357600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610b4157610b5e565b6000610620516020850101535b8151600101808352811415610b30575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106e0526102808051602001806106405161068001828460006004600a8704601201f1610bb557600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516060818352015b8361062051101515610bf357610c10565b6000610620516020850101535b8151600101808352811415610be2575b50505050602061064051610680015160206001820306601f82010390506106405101016106405261064051610700526105c08051602001806106405161068001828460006004600a8704601201f1610c6757600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610ca557610cc2565b6000610620516020850101535b8151600101808352811415610c94575b50505050602061064051610680015160206001820306601f8201039050610640510101610640527f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c561064051610680a160006107205260006101a06030806020846107e001018260208501600060046016f150508051820191505060006010602082066107600160208284011115610d5957600080fd5b60208061078082610720600060046015f15050818152809050905090506010806020846107e001018260208501600060046013f1505080518201915050806107e0526107e09050602060c0825160208401600060025af1610db957600080fd5b60c0519050610740526000600060406020820661088001610280518284011115610de257600080fd5b6060806108a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af1610e2257600080fd5b60c0519050602082610a800101526020810190506000604060206020820661094001610280518284011115610e5657600080fd5b606080610960826020602088068803016102800160006004601bf1505081815280905090509050602080602084610a0001018260208501600060046015f150508051820191505061072051602082610a0001015260208101905080610a0052610a009050602060c0825160208401600060025af1610ed357600080fd5b60c0519050602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610f0a57600080fd5b60c0519050610860526000600061074051602082610b20010152602081019050610220602080602084610b2001018260208501600060046015f150508051820191505080610b2052610b209050602060c0825160208401600060025af1610f7057600080fd5b60c0519050602082610ca00101526020810190506000610360600880602084610c2001018260208501600060046012f15050805182019150506000601860208206610ba00160208284011115610fc557600080fd5b602080610bc082610720600060046015f1505081815280905090509050601880602084610c2001018260208501600060046014f150508051820191505061086051602082610c2001015260208101905080610c2052610c209050602060c0825160208401600060025af161103857600080fd5b60c0519050602082610ca001015260208101905080610ca052610ca09050602060c0825160208401600060025af161106f57600080fd5b60c0519050610b0052600180546001825401101561108c57600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d20511614156110dc57610b0051610d4051602081106110cb57600080fd5b600060c052602060c0200155611170565b6000610d4051602081106110ef57600080fd5b600060c052602060c0200154602082610d60010152602081019050610b0051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af161114057600080fd5b60c0519050610b0052610d20600261115757600080fd5b60028151048152505b81516001018083528114156110a7575b5050005b60006000fd5b6101856112ff036101856000396101856112ff036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index b49cc45e1a..bad619b076 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -6,7 +6,7 @@ AMOUNT_LENGTH: constant(uint256) = 8 # bytes SIGNATURE_LENGTH: constant(uint256) = 96 # bytes -Deposit: event({ +DepositEvent: event({ pubkey: bytes[48], withdrawal_credentials: bytes[32], amount: bytes[8], @@ -42,7 +42,7 @@ def to_little_endian_64(value: uint256) -> bytes[8]: @public @constant -def get_deposit_root() -> bytes32: +def get_hash_tree_root() -> bytes32: zero_bytes32: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000 node: bytes32 = zero_bytes32 size: uint256 = self.deposit_count @@ -76,11 +76,11 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], assert len(withdrawal_credentials) == WITHDRAWAL_CREDENTIALS_LENGTH assert len(signature) == SIGNATURE_LENGTH - # Emit `Deposit` log + # Emit `DepositEvent` log amount: bytes[8] = self.to_little_endian_64(deposit_amount) - log.Deposit(pubkey, withdrawal_credentials, amount, signature, self.to_little_endian_64(self.deposit_count)) + log.DepositEvent(pubkey, withdrawal_credentials, amount, signature, self.to_little_endian_64(self.deposit_count)) - # Compute `DepositData` root + # Compute `DepositData` hash tree root zero_bytes32: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000 pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes32, start=0, len=64 - PUBKEY_LENGTH))) signature_root: bytes32 = sha256(concat( @@ -92,7 +92,7 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], sha256(concat(amount, slice(zero_bytes32, start=0, len=32 - AMOUNT_LENGTH), signature_root)), )) - # Add `DepositData` root to Merkle tree (update a single `branch` node) + # Add `DepositData` hash tree root to Merkle tree (update a single `branch` node) self.deposit_count += 1 size: uint256 = self.deposit_count for height in range(DEPOSIT_CONTRACT_TREE_DEPTH): diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 8c165f0f9b..1c96d074ec 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -96,8 +96,8 @@ def test_deposit_inputs(registration_contract, ) -def test_deposit_log(registration_contract, a0, w3, deposit_input): - log_filter = registration_contract.events.Deposit.createFilter( +def test_deposit_event_log(registration_contract, a0, w3, deposit_input): + log_filter = registration_contract.events.DepositEvent.createFilter( fromBlock='latest', ) @@ -119,7 +119,7 @@ def test_deposit_log(registration_contract, a0, w3, deposit_input): def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input): - log_filter = registration_contract.events.Deposit.createFilter( + log_filter = registration_contract.events.DepositEvent.createFilter( fromBlock='latest', ) @@ -146,4 +146,4 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input )) root = hash_tree_root(List[DepositData, 2**32](*deposit_data_list)) - assert root == registration_contract.functions.get_deposit_root().call() + assert root == registration_contract.functions.get_hash_tree_root().call() diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c70fc006ed..088619202c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -438,7 +438,7 @@ class Attestation(Container): ```python class Deposit(Container): - proof: Vector[Hash, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to deposit root + proof: Vector[Hash, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to SSZ deposit data list hash tree root data: DepositData ``` diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index d06dbaea14..0efc15f250 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -14,7 +14,7 @@ - [`deposit` function](#deposit-function) - [Deposit amount](#deposit-amount) - [Withdrawal credentials](#withdrawal-credentials) - - [`Deposit` log](#deposit-log) + - [`DepositEvent` log](#depositevent-log) - [Vyper code](#vyper-code) @@ -53,9 +53,9 @@ One of the `DepositData` fields is `withdrawal_credentials`. It is a commitment The private key corresponding to `withdrawal_pubkey` will be required to initiate a withdrawal. It can be stored separately until a withdrawal is required, e.g. in cold storage. -#### `Deposit` log +#### `DepositEvent` log -Every Ethereum 1.0 deposit emits a `Deposit` log for consumption by the beacon chain. The deposit contract does little validation, pushing most of the validator onboarding logic to the beacon chain. In particular, the proof of possession (a BLS12-381 signature) is not verified by the deposit contract. +Every Ethereum 1.0 deposit emits a `DepositEvent` log for consumption by the beacon chain. The deposit contract does little validation, pushing most of the validator onboarding logic to the beacon chain. In particular, the proof of possession (a BLS12-381 signature) is not verified by the deposit contract. ## Vyper code diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 7d5630c7be..1b103b218d 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -221,7 +221,7 @@ epoch_signature = bls_sign( ##### Eth1 Data -The `block.eth1_data` field is for block proposers to vote on recent Eth 1.0 data. This recent data contains an Eth 1.0 block hash as well as the associated deposit root (as calculated by the `get_deposit_root()` method of the deposit contract) and deposit count after execution of the corresponding Eth 1.0 block. If over half of the block proposers in the current Eth 1.0 voting period vote for the same `eth1_data` then `state.eth1_data` updates at the end of the voting period. Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. +The `block.eth1_data` field is for block proposers to vote on recent Eth 1.0 data. This recent data contains an Eth 1.0 block hash as well as the associated deposit root (as calculated by the `get_hash_tree_root()` method of the deposit contract) and deposit count after execution of the corresponding Eth 1.0 block. If over half of the block proposers in the current Eth 1.0 voting period vote for the same `eth1_data` then `state.eth1_data` updates at the end of the voting period. Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. Let `get_eth1_data(distance: int) -> Eth1Data` be the (subjective) function that returns the Eth 1.0 data at distance `distance` relative to the Eth 1.0 head at the start of the current Eth 1.0 voting period. Let `previous_eth1_distance` be the distance relative to the Eth 1.0 block corresponding to `state.eth1_data.block_hash` at the start of the current Eth 1.0 voting period. An honest block proposer sets `block.eth1_data = get_eth1_vote(state, previous_eth1_distance)` where: From 22476420f3922f741dd24294eafb5cd181cad4a7 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 21:03:15 +0100 Subject: [PATCH 308/405] Cleanups; think about merging is_genesis_trigger into get_genesis_state --- specs/core/0_beacon-chain.md | 47 ++++++++++++++---------------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 088619202c..f8768935b9 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -174,6 +174,7 @@ The following values are (non-configurable) constants used throughout the specif | `ZERO_HASH` | `Hash(b'\x00' * 32)` | | `BASE_REWARDS_PER_EPOCH` | `5` | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | +| `SECONDS_PER_DAY` | `86400` | ## Configuration @@ -438,7 +439,7 @@ class Attestation(Container): ```python class Deposit(Container): - proof: Vector[Hash, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to SSZ deposit data list hash tree root + proof: Vector[Hash, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to deposit data list root data: DepositData ``` @@ -1093,43 +1094,45 @@ def slash_validator(state: BeaconState, ### Genesis trigger Before genesis has been triggered and for every Ethereum 1.0 block call `is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool` where: -* `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log + +* `deposits` is the sequence of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log When `is_genesis_trigger(deposits, timestamp) is True` for the first time, let: * `genesis_deposits = deposits` -* `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` +* `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` * `genesis_eth1_block_hash` is the Ethereum 1.0 block hash that emitted the log for the last deposit in `deposits` *Note*: The function `is_genesis_trigger` has yet to be agreed upon by the community, and can be updated as necessary. We define the following testing placeholder: ```python def is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool: - SECONDS_PER_DAY = 86400 # Do not deploy too early if timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY < MIN_GENESIS_TIME: return False - - # Process deposits + # Process genesis deposits state = BeaconState() + process_genesis_deposits(state, deposits) + # Check active validator count + return len(get_active_validator_indices(state, GENESIS_EPOCH)) >= GENESIS_ACTIVE_VALIDATOR_COUNT +``` + +```python +def process_genesis_deposits(state: BeaconState, deposits: Sequence[Deposit]) -> None: leaves = list(map(lambda deposit: deposit.data, deposits)) for deposit_index, deposit in enumerate(deposits): state.eth1_data.deposit_root = hash_tree_root( List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:deposit_index + 1]) ) state.eth1_data.deposit_count = deposit_index + 1 - state.eth1_deposit_index = deposit_index process_deposit(state, deposit) - # Count active validators at genesis - active_validator_count = 0 + # Process genesis activations for validator in state.validators: if validator.effective_balance == MAX_EFFECTIVE_BALANCE: - active_validator_count += 1 - - # Check effective balance to trigger genesis - return active_validator_count >= GENESIS_ACTIVE_VALIDATOR_COUNT + validator.activation_eligibility_epoch = GENESIS_EPOCH + validator.activation_epoch = GENESIS_EPOCH ``` ### Genesis state @@ -1140,27 +1143,13 @@ Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, ge def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, genesis_eth1_block_hash: Hash) -> BeaconState: + # Process genesis deposits state = BeaconState( genesis_time=genesis_time, eth1_data=Eth1Data(block_hash=genesis_eth1_block_hash), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) - - # Process genesis deposits - leaves = list(map(lambda deposit: deposit.data, deposits)) - for deposit_index, deposit in enumerate(deposits): - state.eth1_data.deposit_root = hash_tree_root( - List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:deposit_index + 1]) - ) - state.eth1_data.deposit_count = deposit_index + 1 - state.eth1_deposit_index = deposit_index - process_deposit(state, deposit) - - # Process genesis activations - for validator in state.validators: - if validator.effective_balance == MAX_EFFECTIVE_BALANCE: - validator.activation_eligibility_epoch = GENESIS_EPOCH - validator.activation_epoch = GENESIS_EPOCH + process_genesis_deposits(state, deposits) # Populate active_index_roots genesis_active_index_root = hash_tree_root( From d00b5b9ea0d3f22a8a58a7db45c9879cf99c0736 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 21:18:18 +0100 Subject: [PATCH 309/405] Merge is_genesis_trigger into get_genesis_state --- specs/core/0_beacon-chain.md | 58 +++++++++++------------------------- 1 file changed, 17 insertions(+), 41 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f8768935b9..ba7f1d06d7 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -94,7 +94,6 @@ - [`initiate_validator_exit`](#initiate_validator_exit) - [`slash_validator`](#slash_validator) - [Genesis](#genesis) - - [Genesis trigger](#genesis-trigger) - [Genesis state](#genesis-state) - [Genesis block](#genesis-block) - [Beacon chain state transition function](#beacon-chain-state-transition-function) @@ -190,7 +189,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | -| `GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**16` (= 65,536) | +| `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**16` (= 65,536) | | `MIN_GENESIS_TIME` | `1578009600` (Jan 3, 2020) | | `JUSTIFICATION_BITS_LENGTH` | `4` | @@ -1091,35 +1090,28 @@ def slash_validator(state: BeaconState, ## Genesis -### Genesis trigger - -Before genesis has been triggered and for every Ethereum 1.0 block call `is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool` where: +### Genesis state -* `deposits` is the sequence of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log -* `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log +Before the Ethereum 2.0 genesis has been triggered and for every Ethereum 1.0 block, call `get_genesis_beacon_state(eth1_block_hash, eth1_timestamp, deposits)` where: -When `is_genesis_trigger(deposits, timestamp) is True` for the first time, let: +* `eth1_block_hash` is the hash of the Ethereum 1.0 block +* `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` +* `deposits` is the sequence of all deposits, ordered chronologically, up to the block with hash `eth1_block_hash` -* `genesis_deposits = deposits` -* `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` -* `genesis_eth1_block_hash` is the Ethereum 1.0 block hash that emitted the log for the last deposit in `deposits` +The genesis state `genesis_state` is defined as the return value of the first call to `get_genesis_beacon_state` that does not trigger an `assert`. -*Note*: The function `is_genesis_trigger` has yet to be agreed upon by the community, and can be updated as necessary. We define the following testing placeholder: +*Note*: The two constants `MIN_GENESIS_TIME` and `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` in `get_genesis_beacon_state` have yet to be agreed upon by the community, and can be updated as necessary. ```python -def is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool: - # Do not deploy too early - if timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY < MIN_GENESIS_TIME: - return False - # Process genesis deposits - state = BeaconState() - process_genesis_deposits(state, deposits) - # Check active validator count - return len(get_active_validator_indices(state, GENESIS_EPOCH)) >= GENESIS_ACTIVE_VALIDATOR_COUNT -``` +def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: int, deposits: Sequence[Deposit]) -> BeaconState: + state = BeaconState( + genesis_time=timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY, + eth1_data=Eth1Data(block_hash=genesis_eth1_block_hash), + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + ) + assert state.genesis_time >= MIN_GENESIS_TIME -```python -def process_genesis_deposits(state: BeaconState, deposits: Sequence[Deposit]) -> None: + # Process deposits leaves = list(map(lambda deposit: deposit.data, deposits)) for deposit_index, deposit in enumerate(deposits): state.eth1_data.deposit_root = hash_tree_root( @@ -1133,23 +1125,7 @@ def process_genesis_deposits(state: BeaconState, deposits: Sequence[Deposit]) -> if validator.effective_balance == MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH -``` - -### Genesis state - -Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_block_hash)`. - -```python -def get_genesis_beacon_state(deposits: Sequence[Deposit], - genesis_time: int, - genesis_eth1_block_hash: Hash) -> BeaconState: - # Process genesis deposits - state = BeaconState( - genesis_time=genesis_time, - eth1_data=Eth1Data(block_hash=genesis_eth1_block_hash), - latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), - ) - process_genesis_deposits(state, deposits) + assert len(get_active_validator_indices(state, GENESIS_EPOCH)) >= MIN_GENESIS_ACTIVE_VALIDATOR_COUNT # Populate active_index_roots genesis_active_index_root = hash_tree_root( From 36dd977b85ef1c21e35f691cd071b30900709759 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 22:27:05 +0200 Subject: [PATCH 310/405] fix finalize on double justification in 123 rule --- ..._process_justification_and_finalization.py | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index c76692a955..1744d388ca 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -60,7 +60,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support attestations.append(spec.PendingAttestation( aggregation_bits=aggregation_bits, data=spec.AttestationData( - beacon_block_root=b'\xaa' * 32, + beacon_block_root=b'\xff' * 32, # irrelevant to testing source=source, target=target, crosslink=spec.Crosslink(shard=shard) @@ -69,6 +69,20 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support )) +def get_checkpoints(spec, epoch): + c1 = None if epoch < 1 else spec.Checkpoint(epoch=epoch - 1, root=b'\xaa' * 32) + c2 = None if epoch < 2 else spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + c3 = None if epoch < 3 else spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) + c4 = None if epoch < 4 else spec.Checkpoint(epoch=epoch - 4, root=b'\xdd' * 32) + c5 = None if epoch < 5 else spec.Checkpoint(epoch=epoch - 5, root=b'\xee' * 32) + return c1, c2, c3, c4, c5 + + +def put_checkpoints_in_block_roots(spec, state, checkpoints): + for c in checkpoints: + state.block_roots[spec.get_epoch_start_slot(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root + + def finalize_on_234(spec, state, epoch, sufficient_support): assert epoch > 4 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -77,12 +91,8 @@ def finalize_on_234(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices # 11*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xdd' * 32) - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) - c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - state.block_roots[spec.get_epoch_start_slot(c4.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c4.root - state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root - state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root + c1, c2, c3, c4, _ = get_checkpoints(spec, epoch) + put_checkpoints_in_block_roots(spec, state, [c1, c2, c3, c4]) old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c4 @@ -117,16 +127,14 @@ def finalize_on_23(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices (post shift) # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) - c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root - state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root + c1, c2, c3, _, _ = get_checkpoints(spec, epoch) + put_checkpoints_in_block_roots(spec, state, [c1, c2, c3]) old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c3 state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[1] = 1 # mock 3rd latest epoch as justified (indices are pre-shift) + state.justification_bits[1] = 1 # mock 3rd latest epoch as justified (index is pre-shift) # mock the 2nd latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -147,7 +155,7 @@ def finalize_on_23(spec, state, epoch, sufficient_support): def finalize_on_123(spec, state, epoch, sufficient_support): - assert epoch > 3 + assert epoch > 5 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch # 43210 -- epochs ago @@ -155,19 +163,20 @@ def finalize_on_123(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices (post shift) # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xdd' * 32) - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) - c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xaa' * 32) - state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root - state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root - state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root + c1, c2, c3, c4, c5 = get_checkpoints(spec, epoch) + put_checkpoints_in_block_roots(spec, state, [c1, c2, c3, c4, c5]) old_finalized = state.finalized_checkpoint - state.previous_justified_checkpoint = c4 # not c3, otherwise finalize 23 would trigger. - state.current_justified_checkpoint = c2 + state.previous_justified_checkpoint = c5 + state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[0:2] = [1, 1] # mock 2nd and 3rd latest epochs as justified (indices are pre-shift) + state.justification_bits[1] = 1 # mock 3rd latest epochs as justified (index is pre-shift) + # mock the 2nd latest epoch as justifiable, with 5th as source + add_mock_attestations(spec, state, + epoch=epoch - 2, + source=c5, + target=c2, + sufficient_support=sufficient_support) # mock the 1st latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 1, @@ -178,12 +187,12 @@ def finalize_on_123(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) - assert state.previous_justified_checkpoint == c2 # changed to old current + assert state.previous_justified_checkpoint == c3 # changed to old current if sufficient_support: assert state.current_justified_checkpoint == c1 # changed to 1st latest - assert state.finalized_checkpoint == c2 # finalized old current + assert state.finalized_checkpoint == c3 # finalized old current else: - assert state.current_justified_checkpoint == c2 # still old current + assert state.current_justified_checkpoint == c3 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized @@ -196,10 +205,8 @@ def finalize_on_12(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices (post shift) # 001*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xaa' * 32) - state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root - state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root + c1, c2, _, _, _ = get_checkpoints(spec, epoch) + put_checkpoints_in_block_roots(spec, state, [c1, c2]) old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c2 @@ -252,13 +259,13 @@ def test_23_poor_support(spec, state): @with_all_phases @spec_state_test def test_123_ok_support(spec, state): - yield from finalize_on_123(spec, state, 4, True) + yield from finalize_on_123(spec, state, 6, True) @with_all_phases @spec_state_test def test_123_poor_support(spec, state): - yield from finalize_on_123(spec, state, 4, False) + yield from finalize_on_123(spec, state, 6, False) @with_all_phases From 3a5872f8e45ba2c23790007071a711ff322a379c Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 21:29:10 +0100 Subject: [PATCH 311/405] Cleanups and fixes --- configs/constant_presets/mainnet.yaml | 2 +- configs/constant_presets/minimal.yaml | 2 +- specs/core/0_beacon-chain.md | 19 +++++++++---------- .../eth2spec/test/genesis/test_genesis.py | 2 +- .../test/genesis/test_genesis_trigger.py | 4 ++-- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index d7a3b5efe5..1ebae3c87b 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -18,7 +18,7 @@ CHURN_LIMIT_QUOTIENT: 65536 # See issue 563 SHUFFLE_ROUND_COUNT: 90 # `2**16` (= 65,536) -GENESIS_ACTIVE_VALIDATOR_COUNT: 65536 +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 65536 # Deposit contract diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index e8dc370bef..c17c5806f6 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -17,7 +17,7 @@ CHURN_LIMIT_QUOTIENT: 65536 # [customized] Faster, but unsecure. SHUFFLE_ROUND_COUNT: 10 # [customized] -GENESIS_ACTIVE_VALIDATOR_COUNT: 100 +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 100 # Deposit contract diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ba7f1d06d7..9df0e2f5aa 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -189,7 +189,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | -| `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**16` (= 65,536) | +| `MIN_MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**16` (= 65,536) | | `MIN_GENESIS_TIME` | `1578009600` (Jan 3, 2020) | | `JUSTIFICATION_BITS_LENGTH` | `4` | @@ -1092,40 +1092,39 @@ def slash_validator(state: BeaconState, ### Genesis state -Before the Ethereum 2.0 genesis has been triggered and for every Ethereum 1.0 block, call `get_genesis_beacon_state(eth1_block_hash, eth1_timestamp, deposits)` where: +Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 block, call `get_genesis_beacon_state(eth1_block_hash, eth1_timestamp, deposits)` where: * `eth1_block_hash` is the hash of the Ethereum 1.0 block * `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` * `deposits` is the sequence of all deposits, ordered chronologically, up to the block with hash `eth1_block_hash` -The genesis state `genesis_state` is defined as the return value of the first call to `get_genesis_beacon_state` that does not trigger an `assert`. +The genesis state `genesis_state` is the return value of the first call to `get_genesis_beacon_state` that does not trigger an `assert`. -*Note*: The two constants `MIN_GENESIS_TIME` and `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` in `get_genesis_beacon_state` have yet to be agreed upon by the community, and can be updated as necessary. +*Note*: The two constants `MIN_GENESIS_TIME` and `MIN_MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` have yet to be agreed upon by the community, and can be updated as necessary. ```python def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: int, deposits: Sequence[Deposit]) -> BeaconState: state = BeaconState( genesis_time=timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY, - eth1_data=Eth1Data(block_hash=genesis_eth1_block_hash), + eth1_data=Eth1Data(block_hash=genesis_eth1_block_hash, deposit_count=len(deposits)), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) assert state.genesis_time >= MIN_GENESIS_TIME # Process deposits leaves = list(map(lambda deposit: deposit.data, deposits)) - for deposit_index, deposit in enumerate(deposits): + for index, deposit in enumerate(deposits): state.eth1_data.deposit_root = hash_tree_root( - List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:deposit_index + 1]) + List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1]) ) - state.eth1_data.deposit_count = deposit_index + 1 process_deposit(state, deposit) - # Process genesis activations + # Process activations for validator in state.validators: if validator.effective_balance == MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - assert len(get_active_validator_indices(state, GENESIS_EPOCH)) >= MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + assert len(get_active_validator_indices(state, GENESIS_EPOCH)) >= MIN_MIN_GENESIS_ACTIVE_VALIDATOR_COUNT # Populate active_index_roots genesis_active_index_root = hash_tree_root( diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py b/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py index 0d2b4fbac0..21fcaca40e 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py @@ -7,7 +7,7 @@ @with_phases(['phase0']) @spectest_with_bls_switch def test_genesis(spec): - deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT + deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) genesis_time = 1546300800 genesis_eth1_block_hash = b'\x12' * 32 diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py b/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py index 783def3bad..998189e15e 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py @@ -23,7 +23,7 @@ def test_is_genesis_trigger_false(spec): @with_phases(['phase0']) @spectest_with_bls_switch def test_is_genesis_trigger_true(spec): - deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT + deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) SECONDS_PER_DAY = 86400 genesis_time = 1578009600 - 2 * SECONDS_PER_DAY @@ -40,7 +40,7 @@ def test_is_genesis_trigger_true(spec): @with_phases(['phase0']) @spectest_with_bls_switch def test_is_genesis_trigger_not_enough_balance(spec): - deposit_count = spec.GENESIS_ACTIVE_VALIDATOR_COUNT + deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT genesis_deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE - 1) genesis_time = 1546300800 yield "deposits", genesis_deposits From ada3cb2ae126653ce64d240b78a55e7eff46ac96 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 21:36:27 +0100 Subject: [PATCH 312/405] Fix genesis balance bug (git add -u) --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9df0e2f5aa..7a79dde014 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1120,8 +1120,8 @@ def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: int, deposit process_deposit(state, deposit) # Process activations - for validator in state.validators: - if validator.effective_balance == MAX_EFFECTIVE_BALANCE: + for index, validator in enumerate(state.validators): + if balances[index] >= MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH assert len(get_active_validator_indices(state, GENESIS_EPOCH)) >= MIN_MIN_GENESIS_ACTIVE_VALIDATOR_COUNT From 43beb743e6bc7eab133bcf07fdfb8313fdd1b20d Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 21:39:55 +0100 Subject: [PATCH 313/405] typo --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7a79dde014..a0c3d85c0d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1121,7 +1121,7 @@ def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: int, deposit # Process activations for index, validator in enumerate(state.validators): - if balances[index] >= MAX_EFFECTIVE_BALANCE: + if state.balances[index] >= MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH assert len(get_active_validator_indices(state, GENESIS_EPOCH)) >= MIN_MIN_GENESIS_ACTIVE_VALIDATOR_COUNT From a356fc9fc8073794ec2d7ea7fa71689e2d8e5a79 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 21:53:04 +0100 Subject: [PATCH 314/405] Fixes --- specs/core/0_beacon-chain.md | 8 ++++---- test_libs/pyspec/eth2spec/test/genesis/test_genesis.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a0c3d85c0d..3de90a87ca 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -189,7 +189,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | -| `MIN_MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**16` (= 65,536) | +| `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**16` (= 65,536) | | `MIN_GENESIS_TIME` | `1578009600` (Jan 3, 2020) | | `JUSTIFICATION_BITS_LENGTH` | `4` | @@ -1100,12 +1100,12 @@ Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 b The genesis state `genesis_state` is the return value of the first call to `get_genesis_beacon_state` that does not trigger an `assert`. -*Note*: The two constants `MIN_GENESIS_TIME` and `MIN_MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` have yet to be agreed upon by the community, and can be updated as necessary. +*Note*: The two constants `MIN_GENESIS_TIME` and `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` have yet to be agreed upon by the community, and can be updated as necessary. ```python def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: int, deposits: Sequence[Deposit]) -> BeaconState: state = BeaconState( - genesis_time=timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY, + genesis_time=eth1_timestamp - eth1_timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY, eth1_data=Eth1Data(block_hash=genesis_eth1_block_hash, deposit_count=len(deposits)), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) @@ -1124,7 +1124,7 @@ def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: int, deposit if state.balances[index] >= MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - assert len(get_active_validator_indices(state, GENESIS_EPOCH)) >= MIN_MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + assert len(get_active_validator_indices(state, GENESIS_EPOCH)) >= MIN_GENESIS_ACTIVE_VALIDATOR_COUNT # Populate active_index_roots genesis_active_index_root = hash_tree_root( diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py b/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py index 21fcaca40e..0023fbf5ea 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py @@ -17,9 +17,9 @@ def test_genesis(spec): yield "genesis_eth1_block_hash", genesis_eth1_block_hash genesis_state = spec.get_genesis_beacon_state( - genesis_deposits, - genesis_time, genesis_eth1_block_hash, + genesis_time, + genesis_deposits, ) assert genesis_state.genesis_time == genesis_time From 6d455136059f2c92a650ab3567dcaf52333bdd25 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 29 Jun 2019 21:56:07 +0100 Subject: [PATCH 315/405] Make timestamp a uint64 --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3de90a87ca..4d8f6332e0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1103,7 +1103,7 @@ The genesis state `genesis_state` is the return value of the first call to `get_ *Note*: The two constants `MIN_GENESIS_TIME` and `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` have yet to be agreed upon by the community, and can be updated as necessary. ```python -def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: int, deposits: Sequence[Deposit]) -> BeaconState: +def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: uint64, deposits: Sequence[Deposit]) -> BeaconState: state = BeaconState( genesis_time=eth1_timestamp - eth1_timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY, eth1_data=Eth1Data(block_hash=genesis_eth1_block_hash, deposit_count=len(deposits)), From 03a243e96cbfa77c67d6a3ccb2c630ea7405b549 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 30 Jun 2019 05:06:17 +0800 Subject: [PATCH 316/405] fix basic test --- specs/core/0_beacon-chain.md | 2 +- .../eth2spec/test/genesis/test_genesis.py | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4d8f6332e0..7e39a52842 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1106,7 +1106,7 @@ The genesis state `genesis_state` is the return value of the first call to `get_ def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: uint64, deposits: Sequence[Deposit]) -> BeaconState: state = BeaconState( genesis_time=eth1_timestamp - eth1_timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY, - eth1_data=Eth1Data(block_hash=genesis_eth1_block_hash, deposit_count=len(deposits)), + eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=len(deposits)), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) assert state.genesis_time >= MIN_GENESIS_TIME diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py b/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py index 0023fbf5ea..9c546b172a 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py @@ -6,26 +6,26 @@ @with_phases(['phase0']) @spectest_with_bls_switch -def test_genesis(spec): +def test_get_genesis_beacon_state_success(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - genesis_deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - genesis_time = 1546300800 - genesis_eth1_block_hash = b'\x12' * 32 + deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - yield "deposits", genesis_deposits - yield "genesis_time", genesis_time + eth1_block_hash = b'\x12' * 32 + eth1_timestamp = spec.MIN_GENESIS_TIME - yield "genesis_eth1_block_hash", genesis_eth1_block_hash + yield "eth1_block_hash", eth1_block_hash + yield "eth1_timestamp", eth1_timestamp + yield "deposits", deposits genesis_state = spec.get_genesis_beacon_state( - genesis_eth1_block_hash, - genesis_time, - genesis_deposits, + eth1_block_hash, + eth1_timestamp, + deposits, ) - assert genesis_state.genesis_time == genesis_time + assert genesis_state.genesis_time == eth1_timestamp - eth1_timestamp % spec.SECONDS_PER_DAY + 2 * spec.SECONDS_PER_DAY assert len(genesis_state.validators) == deposit_count assert genesis_state.eth1_data.deposit_root == deposit_root assert genesis_state.eth1_data.deposit_count == deposit_count - assert genesis_state.eth1_data.block_hash == genesis_eth1_block_hash + assert genesis_state.eth1_data.block_hash == eth1_block_hash yield "state", genesis_state From ded936ebad1e2486a6918ada4a955d06b3e50e55 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 29 Jun 2019 15:49:11 -0600 Subject: [PATCH 317/405] quick comment on avoiding underflow --- specs/core/0_beacon-chain.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6f7a7a059e..0070795558 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -737,8 +737,7 @@ def get_randao_mix(state: BeaconState, epoch: Epoch) -> Hash: """ Return the randao mix at a recent ``epoch``. - ``epoch`` expected to be between (current_epoch - EPOCHS_PER_HISTORICAL_VECTOR, current_epoch], unless - otherwise noted at a call site. + ``epoch`` expected to be between (current_epoch - EPOCHS_PER_HISTORICAL_VECTOR, current_epoch]. """ return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` @@ -763,12 +762,9 @@ def generate_seed(state: BeaconState, epoch: Epoch) -> Hash: """ Generate a seed for the given ``epoch``. - - Note that avoiding the underflow on ``get_randao_mix`` here violates - the epoch validity condition given in that function's comment. """ return hash( - get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + + get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + #avoid underflow get_active_index_root(state, epoch) + int_to_bytes(epoch, length=32) ) From 2a2bd72425f564bff9dde87a6071228c9841fe65 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 29 Jun 2019 22:52:01 +0100 Subject: [PATCH 318/405] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 77e9055ffc..f372c457a1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -786,7 +786,7 @@ def generate_seed(state: BeaconState, Generate a seed for the given ``epoch``. """ return hash( - get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + # avoid underflow + get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + # Avoid underflow hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, epoch))) + int_to_bytes(epoch, length=32) ) From 47cdae42926fc87084805c2e89b153e4f2f0b58a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 30 Jun 2019 06:38:30 +0800 Subject: [PATCH 319/405] Refactor 1. Rename the current `get_genesis_beacon_state(...)` to `initialize_beacon_state_from_eth1(...)` 2. Extract `is_valid_genesis_state(state: BeaconState) -> bool` from `initialize_beacon_state_from_eth1(...)` --- specs/core/0_beacon-chain.md | 20 +++-- .../eth2spec/test/genesis/test_genesis.py | 31 ------- .../test/genesis/test_genesis_trigger.py | 52 ----------- .../test_initialize_beacon_state_from_eth1.py | 39 +++++++++ .../genesis/test_is_valid_genesis_state.py | 86 +++++++++++++++++++ 5 files changed, 140 insertions(+), 88 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/genesis/test_genesis.py delete mode 100644 test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py create mode 100644 test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py create mode 100644 test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7e39a52842..1009da909e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1092,24 +1092,25 @@ def slash_validator(state: BeaconState, ### Genesis state -Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 block, call `get_genesis_beacon_state(eth1_block_hash, eth1_timestamp, deposits)` where: +Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 block, call `initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)` where: * `eth1_block_hash` is the hash of the Ethereum 1.0 block * `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` * `deposits` is the sequence of all deposits, ordered chronologically, up to the block with hash `eth1_block_hash` -The genesis state `genesis_state` is the return value of the first call to `get_genesis_beacon_state` that does not trigger an `assert`. +The genesis state `genesis_state` is the return value of calling `initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)` only if `is_valid_genesis_state(genesis_state) is True`. *Note*: The two constants `MIN_GENESIS_TIME` and `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` have yet to be agreed upon by the community, and can be updated as necessary. ```python -def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: uint64, deposits: Sequence[Deposit]) -> BeaconState: +def initialize_beacon_state_from_eth1(eth1_block_hash: Hash, + eth1_timestamp: uint64, + deposits: Sequence[Deposit]) -> BeaconState: state = BeaconState( genesis_time=eth1_timestamp - eth1_timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY, eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=len(deposits)), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) - assert state.genesis_time >= MIN_GENESIS_TIME # Process deposits leaves = list(map(lambda deposit: deposit.data, deposits)) @@ -1124,7 +1125,6 @@ def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: uint64, depo if state.balances[index] >= MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - assert len(get_active_validator_indices(state, GENESIS_EPOCH)) >= MIN_GENESIS_ACTIVE_VALIDATOR_COUNT # Populate active_index_roots genesis_active_index_root = hash_tree_root( @@ -1136,6 +1136,16 @@ def get_genesis_beacon_state(eth1_block_hash: Hash, eth1_timestamp: uint64, depo return state ``` +```python +def is_valid_genesis_state(state: BeaconState) -> bool: + if state.genesis_time < MIN_GENESIS_TIME: + return False + elif len(get_active_validator_indices(state, GENESIS_EPOCH)) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: + return False + else: + return True +``` + ### Genesis block Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py b/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py deleted file mode 100644 index 9c546b172a..0000000000 --- a/test_libs/pyspec/eth2spec/test/genesis/test_genesis.py +++ /dev/null @@ -1,31 +0,0 @@ -from eth2spec.test.context import with_phases, spectest_with_bls_switch -from eth2spec.test.helpers.deposits import ( - prepare_genesis_deposits, -) - - -@with_phases(['phase0']) -@spectest_with_bls_switch -def test_get_genesis_beacon_state_success(spec): - deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - - eth1_block_hash = b'\x12' * 32 - eth1_timestamp = spec.MIN_GENESIS_TIME - - yield "eth1_block_hash", eth1_block_hash - yield "eth1_timestamp", eth1_timestamp - yield "deposits", deposits - genesis_state = spec.get_genesis_beacon_state( - eth1_block_hash, - eth1_timestamp, - deposits, - ) - - assert genesis_state.genesis_time == eth1_timestamp - eth1_timestamp % spec.SECONDS_PER_DAY + 2 * spec.SECONDS_PER_DAY - assert len(genesis_state.validators) == deposit_count - assert genesis_state.eth1_data.deposit_root == deposit_root - assert genesis_state.eth1_data.deposit_count == deposit_count - assert genesis_state.eth1_data.block_hash == eth1_block_hash - - yield "state", genesis_state diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py b/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py deleted file mode 100644 index 998189e15e..0000000000 --- a/test_libs/pyspec/eth2spec/test/genesis/test_genesis_trigger.py +++ /dev/null @@ -1,52 +0,0 @@ -from eth2spec.test.context import with_phases, spectest_with_bls_switch -from eth2spec.test.helpers.deposits import ( - prepare_genesis_deposits, -) - - -@with_phases(['phase0']) -@spectest_with_bls_switch -def test_is_genesis_trigger_false(spec): - deposit_count = 2 - genesis_deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - genesis_time = 1546300800 - - yield "deposits", genesis_deposits - yield "time", genesis_time - - is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time) - assert is_triggered is False - - yield "is_triggered", is_triggered - - -@with_phases(['phase0']) -@spectest_with_bls_switch -def test_is_genesis_trigger_true(spec): - deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - genesis_deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - SECONDS_PER_DAY = 86400 - genesis_time = 1578009600 - 2 * SECONDS_PER_DAY - - yield "deposits", genesis_deposits - yield "time", genesis_time - - is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time) - assert is_triggered is True - - yield "is_triggered", is_triggered - - -@with_phases(['phase0']) -@spectest_with_bls_switch -def test_is_genesis_trigger_not_enough_balance(spec): - deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - genesis_deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE - 1) - genesis_time = 1546300800 - yield "deposits", genesis_deposits - yield "time", genesis_time - - is_triggered = spec.is_genesis_trigger(genesis_deposits, genesis_time) - assert is_triggered is False - - yield "is_triggered", is_triggered diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py b/test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py new file mode 100644 index 0000000000..86f845f748 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py @@ -0,0 +1,39 @@ +from eth2spec.test.context import spectest_with_bls_switch, with_phases +from eth2spec.test.helpers.deposits import ( + prepare_genesis_deposits, +) + + +def create_valid_beacon_state(spec): + deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + + eth1_block_hash = b'\x12' * 32 + eth1_timestamp = spec.MIN_GENESIS_TIME + return spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) + + +@with_phases(['phase0']) +@spectest_with_bls_switch +def test_initialize_beacon_state_from_eth1(spec): + deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + + eth1_block_hash = b'\x12' * 32 + eth1_timestamp = spec.MIN_GENESIS_TIME + + yield 'eth1_block_hash', eth1_block_hash + yield 'eth1_timestamp', eth1_timestamp + yield 'deposits', deposits + + # initialize beacon_state + state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) + + assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.SECONDS_PER_DAY + 2 * spec.SECONDS_PER_DAY + assert len(state.validators) == deposit_count + assert state.eth1_data.deposit_root == deposit_root + assert state.eth1_data.deposit_count == deposit_count + assert state.eth1_data.block_hash == eth1_block_hash + + # yield state + yield 'state', state diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py new file mode 100644 index 0000000000..8b41e5ac4b --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py @@ -0,0 +1,86 @@ +from eth2spec.test.context import spectest_with_bls_switch, with_phases +from eth2spec.test.helpers.deposits import ( + prepare_genesis_deposits, +) + + +def create_valid_beacon_state(spec): + deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + + eth1_block_hash = b'\x12' * 32 + eth1_timestamp = spec.MIN_GENESIS_TIME + return spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) + + +def run_is_valid_genesis_state(spec, state, valid=True): + """ + Run ``is_valid_genesis_state``, yielding: + - state ('state') + - is_valid ('is_valid') + If ``valid == False``, run expecting ``AssertionError`` + """ + yield state + is_valid = spec.is_valid_genesis_state(state) + yield 'is_valid', is_valid + + +@with_phases(['phase0']) +@spectest_with_bls_switch +def test_is_valid_genesis_state_true(spec): + state = create_valid_beacon_state(spec) + + yield from run_is_valid_genesis_state(spec, state, valid=True) + + +@with_phases(['phase0']) +@spectest_with_bls_switch +def test_is_valid_genesis_state_false_invalid_timestamp(spec): + state = create_valid_beacon_state(spec) + state.genesis_time = spec.MIN_GENESIS_TIME - 1 + + yield from run_is_valid_genesis_state(spec, state, valid=True) + + +@with_phases(['phase0']) +@spectest_with_bls_switch +def test_is_valid_genesis_state_true_more_balance(spec): + state = create_valid_beacon_state(spec) + state.validators[0].effective_balance = spec.MAX_EFFECTIVE_BALANCE + 1 + + yield from run_is_valid_genesis_state(spec, state, valid=True) + + +@with_phases(['phase0']) +@spectest_with_bls_switch +def test_is_valid_genesis_state_false_not_enough_balance(spec): + state = create_valid_beacon_state(spec) + state.validators[0].effective_balance = spec.MAX_EFFECTIVE_BALANCE - 1 + + yield from run_is_valid_genesis_state(spec, state, valid=False) + + +@with_phases(['phase0']) +@spectest_with_bls_switch +def test_is_valid_genesis_state_true_one_more_validator(spec): + deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + 1 + deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + + eth1_block_hash = b'\x12' * 32 + eth1_timestamp = spec.MIN_GENESIS_TIME + state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) + + yield from run_is_valid_genesis_state(spec, state, valid=True) + + +@with_phases(['phase0']) +@spectest_with_bls_switch +def test_is_valid_genesis_state_flase_not_enough_validator(spec): + deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - 1 + deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + + eth1_block_hash = b'\x12' * 32 + eth1_timestamp = spec.MIN_GENESIS_TIME + state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) + + yield from run_is_valid_genesis_state(spec, state, valid=False) From 8d2cbc9722bbb264ced4bae1c67d940acc86e217 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 02:08:34 +0200 Subject: [PATCH 320/405] add comments to make variations in genesis initialization clear, and about the mix-in in verification --- specs/core/0_beacon-chain.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1009da909e..957239f586 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1100,6 +1100,10 @@ Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 b The genesis state `genesis_state` is the return value of calling `initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)` only if `is_valid_genesis_state(genesis_state) is True`. +Implementations can choose to support different (more optimized) variations of the below initialization approach: + - Build the `genesis_state` from a stream of deposits by incrementally updating the `state.eth1_data.deposit_root`. + - Compute deposit proofs for the final `state.eth1_data.deposit_root`, and process as a pre-determined collection. + *Note*: The two constants `MIN_GENESIS_TIME` and `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` have yet to be agreed upon by the community, and can be updated as necessary. ```python @@ -1671,7 +1675,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: assert verify_merkle_branch( leaf=hash_tree_root(deposit.data), proof=deposit.proof, - depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, + depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # add 1 for the SSZ length mix-in index=state.eth1_deposit_index, root=state.eth1_data.deposit_root, ) From 4d5d5971e2b6fef4dba37c4854feec77e1411a05 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 02:10:18 +0200 Subject: [PATCH 321/405] fix genesis tests; sign deposit-datas, and lower min validator count for testing --- configs/constant_presets/minimal.yaml | 2 +- .../genesis/test_initialize_beacon_state_from_eth1.py | 11 +---------- .../test/genesis/test_is_valid_genesis_state.py | 6 +++--- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index c17c5806f6..50339cd929 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -17,7 +17,7 @@ CHURN_LIMIT_QUOTIENT: 65536 # [customized] Faster, but unsecure. SHUFFLE_ROUND_COUNT: 10 # [customized] -MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 100 +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 64 # Deposit contract diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py b/test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py index 86f845f748..b95b70feff 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py @@ -4,20 +4,11 @@ ) -def create_valid_beacon_state(spec): - deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) - - eth1_block_hash = b'\x12' * 32 - eth1_timestamp = spec.MIN_GENESIS_TIME - return spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) - - @with_phases(['phase0']) @spectest_with_bls_switch def test_initialize_beacon_state_from_eth1(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) eth1_block_hash = b'\x12' * 32 eth1_timestamp = spec.MIN_GENESIS_TIME diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py index 8b41e5ac4b..828c62e90d 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py @@ -6,7 +6,7 @@ def create_valid_beacon_state(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) eth1_block_hash = b'\x12' * 32 eth1_timestamp = spec.MIN_GENESIS_TIME @@ -64,7 +64,7 @@ def test_is_valid_genesis_state_false_not_enough_balance(spec): @spectest_with_bls_switch def test_is_valid_genesis_state_true_one_more_validator(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + 1 - deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) eth1_block_hash = b'\x12' * 32 eth1_timestamp = spec.MIN_GENESIS_TIME @@ -77,7 +77,7 @@ def test_is_valid_genesis_state_true_one_more_validator(spec): @spectest_with_bls_switch def test_is_valid_genesis_state_flase_not_enough_validator(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - 1 - deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE) + deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) eth1_block_hash = b'\x12' * 32 eth1_timestamp = spec.MIN_GENESIS_TIME From f1749df58723214f4a2440974a1b8fe3eaed2278 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 29 Jun 2019 22:25:51 -0600 Subject: [PATCH 322/405] minor typo --- .../pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py index 828c62e90d..4ad5092002 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py @@ -75,7 +75,7 @@ def test_is_valid_genesis_state_true_one_more_validator(spec): @with_phases(['phase0']) @spectest_with_bls_switch -def test_is_valid_genesis_state_flase_not_enough_validator(spec): +def test_is_valid_genesis_state_false_not_enough_validator(spec): deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - 1 deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True) From 6cfd3b5047ec54e0a4250a2b262d71a87e99830c Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 30 Jun 2019 08:10:23 +0100 Subject: [PATCH 323/405] Fix typo Thanks @NIC619 https://github.com/ethereum/eth2.0-specs/pull/1219#discussion_r298818138 --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3f834c19c0..6ee3fd801e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1485,7 +1485,7 @@ def process_slashings(state: BeaconState) -> None: ```python def process_final_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) - next_epoch = Shard(current_epoch + 1) + next_epoch = Epoch(current_epoch + 1) # Reset eth1 data votes if (state.slot + 1) % SLOTS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] From ae888e148c57c471f916f45f0696717f54699b2a Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 30 Jun 2019 08:13:07 +0100 Subject: [PATCH 324/405] Minor copyedit Can the linter catch non-capitalised comments? --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3f834c19c0..2e1b5d1389 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1688,7 +1688,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: assert verify_merkle_branch( leaf=hash_tree_root(deposit.data), proof=deposit.proof, - depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # add 1 for the SSZ length mix-in + depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the `List` length mix-in index=state.eth1_deposit_index, root=state.eth1_data.deposit_root, ) From 1d9fe90d97440f8c95e74017941a360bb94df0bb Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 30 Jun 2019 10:58:04 +0200 Subject: [PATCH 325/405] Makes v-guide executable --- Makefile | 14 +++- scripts/build_spec.py | 23 ++++-- specs/validator/0_beacon-chain-validator.md | 86 +++++++++++---------- 3 files changed, 74 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index 9bcbb3b8cd..826c4a41e3 100644 --- a/Makefile +++ b/Makefile @@ -29,16 +29,22 @@ COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) -clean: +# deletes everything excpet the venvs +partial_clean: rm -rf $(YAML_TEST_DIR) rm -rf $(GENERATOR_VENVS) - rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache + rm -rf $(PY_SPEC_DIR)/.pytest_cache rm -rf $(PY_SPEC_ALL_TARGETS) - rm -rf $(DEPOSIT_CONTRACT_DIR)/venv $(DEPOSIT_CONTRACT_DIR)/.pytest_cache + rm -rf $(DEPOSIT_CONTRACT_DIR)/.pytest_cache rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT) rm -rf $(PY_SPEC_DIR)/.coverage rm -rf $(PY_SPEC_DIR)/test-reports +clean: + partial_clean + rm -rf $(PY_SPEC_DIR)/venv + rm -rf $(DEPOSIT_CONTRACT_DIR)/venv + # "make gen_yaml_tests" to run generators gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) @@ -78,7 +84,7 @@ test_deposit_contract: pyspec: $(PY_SPEC_ALL_TARGETS) $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@ diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 1f5fe1ee61..cbebb0c6ec 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -32,6 +32,7 @@ bls_aggregate_pubkeys, bls_verify, bls_verify_multiple, + bls_sign, ) from eth2spec.utils.hash_function import hash @@ -69,6 +70,10 @@ hash_cache: Dict[bytes, Hash] = {} +def get_eth1_data(distance: uint64) -> Hash: + return hash(distance) + + def hash(x: bytes) -> Hash: if x not in hash_cache: hash_cache[x] = Hash(_hash(x)) @@ -238,10 +243,14 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: return functions, custom_types, constants, ssz_objects, inserts -def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]: +def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, + v_guide_sourcefile: str, outfile: str=None) -> Optional[str]: phase0_spec = get_spec(phase0_sourcefile) fork_choice_spec = get_spec(fork_choice_sourcefile) - spec_objects = combine_spec_objects(phase0_spec, fork_choice_spec) + v_guide = get_spec(v_guide_sourcefile) + spec_objects = phase0_spec + for value in [fork_choice_spec, v_guide]: + spec_objects = combine_spec_objects(spec_objects, value) spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: @@ -274,7 +283,8 @@ def build_phase1_spec(phase0_sourcefile: str, If building phase 0: 1st argument is input spec.md 2nd argument is input fork_choice.md - 3rd argument is output spec.py + 3rd argument is input validator_guide.md + 4th argument is output spec.py If building phase 1: 1st argument is input spec_phase0.md @@ -289,14 +299,15 @@ def build_phase1_spec(phase0_sourcefile: str, args = parser.parse_args() if args.phase == 0: - if len(args.files) == 3: + if len(args.files) == 4: build_phase0_spec(*args.files) else: - print(" Phase 0 requires an output as well as spec and forkchoice files.") + print(" Phase 0 requires spec, forkchoice, and v-guide inputs as well as an output file.") elif args.phase == 1: if len(args.files) == 5: build_phase1_spec(*args.files) else: - print(" Phase 1 requires an output as well as 4 input files (phase0.md and phase1.md, phase1.md, fork_choice.md)") + print(" Phase 1 requires 4 input files as well as an output file: " + + "(phase0.md and phase1.md, phase1.md, fork_choice.md, output.py)") else: print("Invalid phase: {0}".format(args.phase)) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 7d5630c7be..0b92928590 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -118,8 +118,9 @@ In normal operation, the validator is quickly activated, at which point the vali The function [`is_active_validator`](../core/0_beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows: ```python -validator = state.validators[validator_index] -is_active = is_active_validator(validator, get_current_epoch(state)) +def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool: + validator = state.validators[validator_index] + return is_active_validator(validator, get_current_epoch(state)) ``` Once a validator is activated, the validator is assigned [responsibilities](#beacon-chain-responsibilities) until exited. @@ -151,10 +152,14 @@ def get_committee_assignment( offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) slot_start_shard = (get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT for i in range(committees_per_slot): - shard = (slot_start_shard + i) % SHARD_COUNT + shard = Shard((slot_start_shard + i) % SHARD_COUNT) committee = get_crosslink_committee(state, epoch, shard) if validator_index in committee: - return committee, shard, slot + break + else: + continue + break + return committee, shard, Slot(slot) ``` A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run with a `state` of the slot in question. Proposer selection is only stable within the context of the current epoch. @@ -205,18 +210,19 @@ Set `block.state_root = hash_tree_root(state)` of the resulting `state` of the ` ##### Randao reveal -Set `block.randao_reveal = epoch_signature` where `epoch_signature` is defined as: +Set `block.randao_reveal = epoch_signature` where `epoch_signature` is obtained from: ```python -epoch_signature = bls_sign( - privkey=validator.privkey, # privkey stored locally, not in state - message_hash=hash_tree_root(slot_to_epoch(block.slot)), - domain=get_domain( - fork=fork, # `fork` is the fork object at the slot `block.slot` - epoch=slot_to_epoch(block.slot), - domain_type=DOMAIN_RANDAO, +def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + return bls_sign( + privkey=privkey, # privkey stored locally + message_hash=hash_tree_root(slot_to_epoch(block.slot)), + domain=get_domain( + state=state, + domain_type=DOMAIN_RANDAO, + message_epoch=slot_to_epoch(block.slot), + ) ) -) ``` ##### Eth1 Data @@ -232,11 +238,12 @@ def get_eth1_vote(state: BeaconState, previous_eth1_distance: uint64) -> Eth1Dat valid_votes = [] for slot, vote in enumerate(state.eth1_data_votes): - period_tail = slot % SLOTS_PER_ETH1_VOTING_PERIOD >= integer_square_root(SLOTS_PER_ETH1_VOTING_PERIOD) + period_tail = slot % SLOTS_PER_ETH1_VOTING_PERIOD >= integer_squareroot(SLOTS_PER_ETH1_VOTING_PERIOD) if vote in new_eth1_data or (period_tail and vote in all_eth1_data): valid_votes.append(vote) - return max(valid_votes, + return max( + valid_votes, key=lambda v: (valid_votes.count(v), -all_eth1_data.index(v)), # Tiebreak by smallest distance default=get_eth1_data(ETH1_FOLLOW_DISTANCE), ) @@ -244,18 +251,19 @@ def get_eth1_vote(state: BeaconState, previous_eth1_distance: uint64) -> Eth1Dat ##### Signature -Set `block.signature = block_signature` where `block_signature` is defined as: +Set `header.signature = block_signature` where `block_signature` is obtained from: ```python -block_signature = bls_sign( - privkey=validator.privkey, # privkey store locally, not in state - message_hash=signing_root(block), - domain=get_domain( - fork=fork, # `fork` is the fork object at the slot `block.slot` - epoch=slot_to_epoch(block.slot), - domain_type=DOMAIN_BEACON_BLOCK, +def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature: + return bls_sign( + privkey=privkey, # privkey stored locally + message_hash=signing_root(header), + domain=get_domain( + state=state, + domain_type=DOMAIN_BEACON_PROPOSER, + message_epoch=slot_to_epoch(header.slot), + ) ) -) ``` #### Block body @@ -343,24 +351,24 @@ Set `attestation.data = attestation_data` where `attestation_data` is the `Attes ##### Aggregate signature -Set `attestation.aggregate_signature = signed_attestation_data` where `signed_attestation_data` is defined as: +Set `attestation.signature = signed_attestation_data` where `signed_attestation_data` is obtained from: ```python -attestation_data_and_custody_bit = AttestationDataAndCustodyBit( - data=attestation.data, - custody_bit=0b0, -) -attestation_message = hash_tree_root(attestation_data_and_custody_bit) - -signed_attestation_data = bls_sign( - privkey=validator.privkey, # privkey stored locally, not in state - message_hash=attestation_message, - domain=get_domain( - fork=fork, # `fork` is the fork object at the slot, `attestation_data.slot` - epoch=slot_to_epoch(attestation_data.slot), - domain_type=DOMAIN_ATTESTATION, +def get_signed_attestation_data(state: BeaconState, attestation: IndexedAttestation, privkey: int) -> BLSSignature: + attestation_data_and_custody_bit = AttestationDataAndCustodyBit( + data=attestation.data, + custody_bit=0b0, + ) + + return bls_sign( + privkey=privkey, # privkey stored locally + message_hash=hash_tree_root(attestation_data_and_custody_bit), + domain=get_domain( + state=state, + domain_type=DOMAIN_ATTESTATION, + message_epoch=attestation.data.target.epoch, + ) ) -) ``` ## How to avoid slashing From d0acd81157adb01dfd0b5ebf417d8c43758769f1 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 10:02:18 +0100 Subject: [PATCH 326/405] WIP --- specs/core/0_beacon-chain.md | 614 +++++++++--------- specs/core/0_fork-choice.md | 10 +- specs/core/1_custody-game.md | 10 +- specs/core/1_shard-data-chains.md | 4 +- specs/light_client/sync_protocol.md | 2 +- specs/validator/0_beacon-chain-validator.md | 12 +- .../test/fork_choice/test_on_attestation.py | 2 +- .../eth2spec/test/helpers/attestations.py | 6 +- .../test/helpers/attester_slashings.py | 4 +- .../test_process_crosslinks.py | 2 +- ..._process_justification_and_finalization.py | 10 +- .../pyspec/eth2spec/test/test_finality.py | 2 +- 12 files changed, 348 insertions(+), 330 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3f834c19c0..c82524eb52 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -49,48 +49,52 @@ - [Beacon state](#beacon-state) - [`BeaconState`](#beaconstate) - [Helper functions](#helper-functions) - - [`xor`](#xor) - - [`hash`](#hash) - - [`hash_tree_root`](#hash_tree_root) - - [`signing_root`](#signing_root) - - [`bls_domain`](#bls_domain) - - [`slot_to_epoch`](#slot_to_epoch) - - [`get_previous_epoch`](#get_previous_epoch) - - [`get_current_epoch`](#get_current_epoch) - - [`get_epoch_start_slot`](#get_epoch_start_slot) - - [`is_active_validator`](#is_active_validator) - - [`is_slashable_validator`](#is_slashable_validator) - - [`get_active_validator_indices`](#get_active_validator_indices) - - [`increase_balance`](#increase_balance) - - [`decrease_balance`](#decrease_balance) - - [`get_epoch_committee_count`](#get_epoch_committee_count) - - [`get_shard_delta`](#get_shard_delta) - - [`get_epoch_start_shard`](#get_epoch_start_shard) - - [`get_attestation_data_slot`](#get_attestation_data_slot) - - [`get_block_root_at_slot`](#get_block_root_at_slot) - - [`get_block_root`](#get_block_root) - - [`get_randao_mix`](#get_randao_mix) - - [`get_compact_committees_root`](#get_compact_committees_root) - - [`generate_seed`](#generate_seed) - - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - - [`verify_merkle_branch`](#verify_merkle_branch) - - [`get_shuffled_index`](#get_shuffled_index) - - [`compute_committee`](#compute_committee) - - [`get_crosslink_committee`](#get_crosslink_committee) - - [`get_attesting_indices`](#get_attesting_indices) - - [`int_to_bytes`](#int_to_bytes) - - [`bytes_to_int`](#bytes_to_int) - - [`get_total_balance`](#get_total_balance) - - [`get_domain`](#get_domain) - - [`convert_to_indexed`](#convert_to_indexed) - - [`validate_indexed_attestation`](#validate_indexed_attestation) - - [`is_slashable_attestation_data`](#is_slashable_attestation_data) - - [`integer_squareroot`](#integer_squareroot) - - [`get_delayed_activation_exit_epoch`](#get_delayed_activation_exit_epoch) - - [`get_churn_limit`](#get_churn_limit) - - [`bls_verify`](#bls_verify) - - [`bls_verify_multiple`](#bls_verify_multiple) - - [`bls_aggregate_pubkeys`](#bls_aggregate_pubkeys) + - [Math and crypto](#math-and-crypto) + - [`int_to_bytes`](#int_to_bytes) + - [`bytes_to_int`](#bytes_to_int) + - [`integer_squareroot`](#integer_squareroot) + - [`xor`](#xor) + - [`hash`](#hash) + - [`hash_tree_root`](#hash_tree_root) + - [`signing_root`](#signing_root) + - [`bls_verify`](#bls_verify) + - [`bls_verify_multiple`](#bls_verify_multiple) + - [`bls_aggregate_pubkeys`](#bls_aggregate_pubkeys) + - [Beacon state getters](#beacon-state-getters) + - [`get_current_epoch`](#get_current_epoch) + - [`get_previous_epoch`](#get_previous_epoch) + - [`get_committee_count`](#get_committee_count) + - [`get_start_shard`](#get_start_shard) + - [`get_active_validator_indices`](#get_active_validator_indices) + + + - [`bls_domain`](#bls_domain) + - [`slot_to_epoch`](#slot_to_epoch) + - [`epoch_start_slot`](#epoch_start_slot) + - [`is_active_validator`](#is_active_validator) + - [`is_slashable_validator`](#is_slashable_validator) + - [`increase_balance`](#increase_balance) + - [`decrease_balance`](#decrease_balance) + - [`get_shard_delta`](#get_shard_delta) + - [`get_attestation_data_slot`](#get_attestation_data_slot) + - [`get_block_root_at_slot`](#get_block_root_at_slot) + - [`get_block_root`](#get_block_root) + - [`get_randao_mix`](#get_randao_mix) + - [`get_compact_committees_root`](#get_compact_committees_root) + - [`get_seed`](#get_seed) + - [`get_beacon_proposer_index`](#get_beacon_proposer_index) + - [`verify_merkle_branch`](#verify_merkle_branch) + - [`get_shuffled_index`](#get_shuffled_index) + - [`compute_committee`](#compute_committee) + - [`get_crosslink_committee`](#get_crosslink_committee) + - [`get_attesting_indices`](#get_attesting_indices) + - [`get_total_balance`](#get_total_balance) + - [`get_domain`](#get_domain) + - [`get_indexed_attestation`](#get_indexed_attestation) + - [`validate_indexed_attestation`](#validate_indexed_attestation) + - [`is_slashable_attestation_data`](#is_slashable_attestation_data) + - [`get_delayed_activation_exit_epoch`](#get_delayed_activation_exit_epoch) + - [`get_churn_limit`](#get_churn_limit) - [Routines for updating validator status](#routines-for-updating-validator-status) - [`initiate_validator_exit`](#initiate_validator_exit) - [`slash_validator`](#slash_validator) @@ -547,133 +551,154 @@ class BeaconState(Container): *Note*: The definitions below are for specification purposes and are not necessarily optimal implementations. -### `xor` +### Math + +#### `int_to_bytes` + +```python +def int_to_bytes(integer: int, length: int) -> bytes: + return integer.to_bytes(length, 'little') +``` + +#### `bytes_to_int` + +```python +def bytes_to_int(data: bytes) -> int: + return int.from_bytes(data, 'little') +``` + +#### `integer_squareroot` + +```python +def integer_squareroot(n: int) -> int: + """ + Return the largest integer ``x`` such that ``x**2 <= n``. + """ + assert n >= 0 + x = n + y = (x + 1) // 2 + while y < x: + x = y + y = (x + n // x) // 2 + return x +``` + +#### `xor` ```python def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: + """ + Return the exclusive or of two 32-byte strings. + """ return Bytes32(a ^ b for a, b in zip(bytes1, bytes2)) ``` -### `hash` +### Crypto + +#### `hash` The `hash` function is SHA256. -*Note*: We aim to migrate to a S[T/N]ARK-friendly hash function in a future Ethereum 2.0 deployment phase. +#### `hash_tree_root` + +`def hash_tree_root(object: SSZSerializable) -> Hash` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SSZ spec](../simple-serialize.md#merkleization). -### `hash_tree_root` +#### `signing_root` -`def hash_tree_root(object: SSZSerializable) -> Hash` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SimpleSerialize spec](../simple-serialize.md#merkleization). +`def signing_root(object: Container) -> Hash` is a function for computing signing messages, as defined in the [SSZ spec](../simple-serialize.md#self-signed-containers). -### `signing_root` +#### `bls_verify` -`def signing_root(object: Container) -> Hash` is a function for computing signing messages, as defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers). +`bls_verify` is a function for verifying a BLS signature, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify). -### `bls_domain` +#### `bls_verify_multiple` -```python -def bls_domain(domain_type: int, fork_version: bytes=b'\x00\x00\x00\x00') -> int: - """ - Return the bls domain given by the ``domain_type`` and optional 4 byte ``fork_version`` (defaults to zero). - """ - return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) -``` +`bls_verify_multiple` is a function for verifying a BLS signature constructed from multiple messages, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify_multiple). + +#### `bls_aggregate_pubkeys` -### `slot_to_epoch` +`bls_aggregate_pubkeys` is a function for aggregating multiple BLS public keys into a single aggregate key, as defined in the [BLS Signature spec](../bls_signature.md#bls_aggregate_pubkeys). + +### Beacon state getters + +#### `get_current_epoch` ```python -def slot_to_epoch(slot: Slot) -> Epoch: +def get_current_epoch(state: BeaconState) -> Epoch: """ - Return the epoch number of the given ``slot``. + Return the current epoch. """ - return Epoch(slot // SLOTS_PER_EPOCH) + return slot_to_epoch(state.slot) ``` -### `get_previous_epoch` +#### `get_previous_epoch` ```python def get_previous_epoch(state: BeaconState) -> Epoch: """` - Return the previous epoch of the given ``state``. - Return the current epoch if it's genesis epoch. + Return the previous epoch (current epoch if at ``GENESIS_EPOCH``). """ current_epoch = get_current_epoch(state) return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else Epoch(current_epoch - 1) ``` -### `get_current_epoch` +#### `get_block_root` ```python -def get_current_epoch(state: BeaconState) -> Epoch: +def get_block_root(state: BeaconState, epoch: Epoch) -> Hash: """ - Return the current epoch of the given ``state``. - """ - return slot_to_epoch(state.slot) -``` - -### `get_epoch_start_slot` - -```python -def get_epoch_start_slot(epoch: Epoch) -> Slot: - """ - Return the starting slot of the given ``epoch``. + Return the block root at a recent ``epoch``. """ - return Slot(epoch * SLOTS_PER_EPOCH) + return get_block_root_at_slot(state, epoch_start_slot(epoch)) ``` -### `is_active_validator` +#### `get_block_root_at_slot` ```python -def is_active_validator(validator: Validator, epoch: Epoch) -> bool: +def get_block_root_at_slot(state: BeaconState, slot: Slot) -> Hash: """ - Check if ``validator`` is active. + Return the block root at a recent ``slot``. """ - return validator.activation_epoch <= epoch < validator.exit_epoch + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] ``` -### `is_slashable_validator` +#### `get_randao_mix` ```python -def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: +def get_randao_mix(state: BeaconState, epoch: Epoch) -> Hash: """ - Check if ``validator`` is slashable. + Return the randao mix at a recent ``epoch``. """ - return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) + return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` -### `get_active_validator_indices` +#### `get_active_validator_indices` ```python def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ - Get active validator indices at ``epoch``. + Return the sequence of active validator indices at ``epoch``. """ return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] ``` -### `increase_balance` +#### `get_churn_limit` ```python -def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: +def get_churn_limit(state: BeaconState) -> int: """ - Increase validator balance by ``delta``. + Return the validator churn limit for the current epoch. """ - state.balances[index] += delta + active_validator_indices = get_active_validator_indices(state, get_current_epoch(state)) + return max(MIN_PER_EPOCH_CHURN_LIMIT, len(active_validator_indices) // CHURN_LIMIT_QUOTIENT) ``` -### `decrease_balance` - -```python -def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: - """ - Decrease validator balance by ``delta`` with underflow protection. - """ - state.balances[index] = 0 if delta > state.balances[index] else state.balances[index] - delta -``` -### `get_epoch_committee_count` +#### `get_committee_count` ```python -def get_epoch_committee_count(state: BeaconState, epoch: Epoch) -> int: +def get_committee_count(state: BeaconState, epoch: Epoch) -> int: """ Return the number of committees at ``epoch``. """ @@ -687,22 +712,41 @@ def get_epoch_committee_count(state: BeaconState, epoch: Epoch) -> int: ) * SLOTS_PER_EPOCH ``` -### `get_shard_delta` +#### `get_seed` ```python -def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: +def get_seed(state: BeaconState, epoch: Epoch) -> Hash: """ - Return the number of shards to increment ``state.start_shard`` during ``epoch``. + Return the seed at ``epoch``. """ - return min(get_epoch_committee_count(state, epoch), SHARD_COUNT - SHARD_COUNT // SLOTS_PER_EPOCH) + return hash( + get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + # Avoid underflow + hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, epoch))) + + int_to_bytes(epoch, length=32) + ) ``` -### `get_epoch_start_shard` +#### `get_crosslink_committee` ```python -def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: +def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: """ - Return the start shard of the 0th committee in an epoch. + Return the crosslink committee at ``epoch`` for ``shard``. + """ + return compute_committee( + indices=get_active_validator_indices(state, epoch), + seed=get_seed(state, epoch), + index=(shard + SHARD_COUNT - get_start_shard(state, epoch)) % SHARD_COUNT, + count=get_committee_count(state, epoch), + ) +``` + +#### `get_start_shard` + +```python +def get_start_shard(state: BeaconState, epoch: Epoch) -> Shard: + """ + Return the start shard of the 0th committee at ``epoch``. """ assert epoch <= get_current_epoch(state) + 1 check_epoch = Epoch(get_current_epoch(state) + 1) @@ -713,60 +757,62 @@ def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: return shard ``` -### `get_attestation_data_slot` - -```python -def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot: - committee_count = get_epoch_committee_count(state, data.target.epoch) - offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, data.target.epoch)) % SHARD_COUNT - return Slot(get_epoch_start_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) -``` - -### `get_block_root_at_slot` +#### `get_shard_delta` ```python -def get_block_root_at_slot(state: BeaconState, - slot: Slot) -> Hash: +def get_shard_delta(state: BeaconState, epoch: Epoch) -> Shard: """ - Return the block root at a recent ``slot``. + Return the number of shards to increment ``state.start_shard`` at ``epoch``. """ - assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT - return state.block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] + return Shard(min(get_committee_count(state, epoch), SHARD_COUNT - SHARD_COUNT // SLOTS_PER_EPOCH)) ``` -### `get_block_root` +#### `get_beacon_proposer_index` ```python -def get_block_root(state: BeaconState, - epoch: Epoch) -> Hash: +def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: """ - Return the block root at a recent ``epoch``. + Return the beacon proposer index at the current epoch. """ - return get_block_root_at_slot(state, get_epoch_start_slot(epoch)) + epoch = get_current_epoch(state) + committees_per_slot = get_committee_count(state, epoch) // SLOTS_PER_EPOCH + offset = committees_per_slot * (state.slot % SLOTS_PER_EPOCH) + shard = Shard((get_start_shard(state, epoch) + offset) % SHARD_COUNT) + first_committee = get_crosslink_committee(state, epoch, shard) + MAX_RANDOM_BYTE = 2**8 - 1 + seed = get_seed(state, epoch) + i = 0 + while True: + candidate_index = first_committee[(epoch + i) % len(first_committee)] + random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] + effective_balance = state.validators[candidate_index].effective_balance + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + return ValidatorIndex(candidate_index) + i += 1 ``` -### `get_randao_mix` +#### `get_attestation_data_slot` ```python -def get_randao_mix(state: BeaconState, - epoch: Epoch) -> Hash: +def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot: """ - Return the randao mix at a recent ``epoch``. - ``epoch`` expected to be between (current_epoch - EPOCHS_PER_HISTORICAL_VECTOR, current_epoch]. + Return the slot corresponding to the attestation ``data``. """ - return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] + committee_count = get_committee_count(state, data.target.epoch) + offset = (data.crosslink.shard + SHARD_COUNT - get_start_shard(state, data.target.epoch)) % SHARD_COUNT + return Slot(epoch_start_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` -### `get_compact_committees_root` +#### `get_compact_committees_root` ```python def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: """ - Return the compact committee root for the current epoch. + Return the compact committee root at ``epoch``. """ committees = [CompactCommittee() for _ in range(SHARD_COUNT)] - start_shard = get_epoch_start_shard(state, epoch) - for committee_number in range(get_epoch_committee_count(state, epoch)): + start_shard = get_start_shard(state, epoch) + for committee_number in range(get_committee_count(state, epoch)): shard = Shard((start_shard + committee_number) % SHARD_COUNT) for index in get_crosslink_committee(state, epoch, shard): validator = state.validators[index] @@ -778,183 +824,196 @@ def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: return hash_tree_root(Vector[CompactCommittee, SHARD_COUNT](committees)) ``` -### `generate_seed` +#### `get_total_balance` ```python -def generate_seed(state: BeaconState, - epoch: Epoch) -> Hash: +def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: """ - Generate a seed for the given ``epoch``. + Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ - return hash( - get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + # Avoid underflow - hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, epoch))) + - int_to_bytes(epoch, length=32) + return Gwei(max(sum([state.validators[index].effective_balance for index in indices]), 1)) +``` + +#### `get_domain` + +```python +def get_domain(state: BeaconState, domain_type: int, message_epoch: Epoch=None) -> int: + """ + Return the signature domain (fork version concatenated with domain type) of a message. + """ + epoch = get_current_epoch(state) if message_epoch is None else message_epoch + fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version + return bls_domain(domain_type, fork_version) +``` + +#### `get_indexed_attestation` + +```python +def get_indexed_attestation(state: BeaconState, attestation: Attestation) -> IndexedAttestation: + """ + Return the indexed attestation corresponding to ``attestation``. + """ + attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bits) + assert custody_bit_1_indices.issubset(attesting_indices) + custody_bit_0_indices = attesting_indices.difference(custody_bit_1_indices) + + return IndexedAttestation( + custody_bit_0_indices=sorted(custody_bit_0_indices), + custody_bit_1_indices=sorted(custody_bit_1_indices), + data=attestation.data, + signature=attestation.signature, ) ``` -### `get_beacon_proposer_index` +#### `get_attesting_indices` ```python -def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: +def get_attesting_indices(state: BeaconState, + data: AttestationData, + bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> Set[ValidatorIndex]: """ - Return the current beacon proposer index. + Return the set of attesting indices corresponding to ``data`` and ``bits``. """ - epoch = get_current_epoch(state) - committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH - offset = committees_per_slot * (state.slot % SLOTS_PER_EPOCH) - shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) - first_committee = get_crosslink_committee(state, epoch, shard) - MAX_RANDOM_BYTE = 2**8 - 1 - seed = generate_seed(state, epoch) - i = 0 - while True: - candidate_index = first_committee[(epoch + i) % len(first_committee)] - random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] - effective_balance = state.validators[candidate_index].effective_balance - if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - return ValidatorIndex(candidate_index) - i += 1 + committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) + return set(index for i, index in enumerate(committee) if bits[i]) ``` -### `verify_merkle_branch` + + + + + + + + + + +#### `verify_merkle_branch` ```python -def verify_merkle_branch(leaf: Hash, proof: Sequence[Hash], depth: int, index: int, root: Hash) -> bool: +def verify_merkle_branch(leaf: Hash, branch: Sequence[Hash], depth: int, index: int, root: Hash) -> bool: """ - Verify that the given ``leaf`` is on the merkle branch ``proof`` - starting with the given ``root``. + Verify that the ``leaf`` at ``index`` verifies against ``root`` on the Merkle ``branch``. """ value = leaf for i in range(depth): if index // (2**i) % 2: - value = hash(proof[i] + value) + value = hash(branch[i] + value) else: - value = hash(value + proof[i]) + value = hash(value + branch[i]) return value == root ``` -### `get_shuffled_index` + +#### `bls_domain` ```python -def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Hash) -> ValidatorIndex: +def bls_domain(domain_type: int, fork_version: bytes=b'\x00\x00\x00\x00') -> int: """ - Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). + Return the bls domain given by the ``domain_type`` and optional 4 byte ``fork_version`` (defaults to zero). """ - assert index < index_count - assert index_count <= 2**40 - - # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) - # See the 'generalized domain' algorithm on page 3 - for current_round in range(SHUFFLE_ROUND_COUNT): - pivot = bytes_to_int(hash(seed + int_to_bytes(current_round, length=1))[0:8]) % index_count - flip = ValidatorIndex((pivot + index_count - index) % index_count) - position = max(index, flip) - source = hash( - seed + int_to_bytes(current_round, length=1) + - int_to_bytes(position // 256, length=4) - ) - byte = source[(position % 256) // 8] - bit = (byte >> (position % 8)) % 2 - index = flip if bit else index - - return ValidatorIndex(index) + return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) ``` -### `compute_committee` +#### `slot_to_epoch` ```python -def compute_committee(indices: Sequence[ValidatorIndex], - seed: Hash, index: int, count: int) -> Sequence[ValidatorIndex]: - start = (len(indices) * index) // count - end = (len(indices) * (index + 1)) // count - return [indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] +def slot_to_epoch(slot: Slot) -> Epoch: + """ + Return the epoch number of the given ``slot``. + """ + return Epoch(slot // SLOTS_PER_EPOCH) ``` -### `get_crosslink_committee` +#### `epoch_start_slot` ```python -def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: - return compute_committee( - indices=get_active_validator_indices(state, epoch), - seed=generate_seed(state, epoch), - index=(shard + SHARD_COUNT - get_epoch_start_shard(state, epoch)) % SHARD_COUNT, - count=get_epoch_committee_count(state, epoch), - ) +def epoch_start_slot(epoch: Epoch) -> Slot: + """ + Return the starting slot of the given ``epoch``. + """ + return Slot(epoch * SLOTS_PER_EPOCH) ``` -### `get_attesting_indices` +#### `is_active_validator` ```python -def get_attesting_indices(state: BeaconState, - data: AttestationData, - bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> Set[ValidatorIndex]: +def is_active_validator(validator: Validator, epoch: Epoch) -> bool: """ - Return the set of attesting indices corresponding to ``data`` and ``bitfield``. + Check if ``validator`` is active. """ - committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) - return set(index for i, index in enumerate(committee) if bits[i]) + return validator.activation_epoch <= epoch < validator.exit_epoch ``` -### `int_to_bytes` +#### `is_slashable_validator` ```python -def int_to_bytes(integer: int, length: int) -> bytes: - return integer.to_bytes(length, 'little') +def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is slashable. + """ + return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) ``` -### `bytes_to_int` +#### `increase_balance` ```python -def bytes_to_int(data: bytes) -> int: - return int.from_bytes(data, 'little') +def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: + """ + Increase validator balance by ``delta``. + """ + state.balances[index] += delta ``` -### `get_total_balance` +#### `decrease_balance` ```python -def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: +def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: """ - Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) + Decrease validator balance by ``delta`` with underflow protection. """ - return Gwei(max(sum([state.validators[index].effective_balance for index in indices]), 1)) + state.balances[index] = 0 if delta > state.balances[index] else state.balances[index] - delta ``` -### `get_domain` +#### `get_shuffled_index` ```python -def get_domain(state: BeaconState, - domain_type: int, - message_epoch: Epoch=None) -> int: +def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Hash) -> ValidatorIndex: """ - Return the signature domain (fork version concatenated with domain type) of a message. + Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). """ - epoch = get_current_epoch(state) if message_epoch is None else message_epoch - fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version - return bls_domain(domain_type, fork_version) + assert index < index_count + assert index_count <= 2**40 + + # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) + # See the 'generalized domain' algorithm on page 3 + for current_round in range(SHUFFLE_ROUND_COUNT): + pivot = bytes_to_int(hash(seed + int_to_bytes(current_round, length=1))[0:8]) % index_count + flip = ValidatorIndex((pivot + index_count - index) % index_count) + position = max(index, flip) + source = hash( + seed + int_to_bytes(current_round, length=1) + + int_to_bytes(position // 256, length=4) + ) + byte = source[(position % 256) // 8] + bit = (byte >> (position % 8)) % 2 + index = flip if bit else index + + return ValidatorIndex(index) ``` -### `convert_to_indexed` +#### `compute_committee` ```python -def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedAttestation: - """ - Convert ``attestation`` to (almost) indexed-verifiable form. - """ - attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) - custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bits) - assert custody_bit_1_indices.issubset(attesting_indices) - custody_bit_0_indices = attesting_indices.difference(custody_bit_1_indices) - - return IndexedAttestation( - custody_bit_0_indices=sorted(custody_bit_0_indices), - custody_bit_1_indices=sorted(custody_bit_1_indices), - data=attestation.data, - signature=attestation.signature, - ) +def compute_committee(indices: Sequence[ValidatorIndex], + seed: Hash, index: int, count: int) -> Sequence[ValidatorIndex]: + start = (len(indices) * index) // count + end = (len(indices) * (index + 1)) // count + return [indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] ``` -### `validate_indexed_attestation` +#### `validate_indexed_attestation` ```python def validate_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> None: @@ -987,7 +1046,7 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe ) ``` -### `is_slashable_attestation_data` +#### `is_slashable_attestation_data` ```python def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationData) -> bool: @@ -1002,23 +1061,7 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa ) ``` -### `integer_squareroot` - -```python -def integer_squareroot(n: int) -> int: - """ - The largest integer ``x`` such that ``x**2`` is less than or equal to ``n``. - """ - assert n >= 0 - x = n - y = (x + 1) // 2 - while y < x: - x = y - y = (x + n // x) // 2 - return x -``` - -### `get_delayed_activation_exit_epoch` +#### `get_delayed_activation_exit_epoch` ```python def get_delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: @@ -1028,31 +1071,6 @@ def get_delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: return Epoch(epoch + 1 + ACTIVATION_EXIT_DELAY) ``` -### `get_churn_limit` - -```python -def get_churn_limit(state: BeaconState) -> int: - """ - Return the churn limit based on the active validator count. - """ - return max( - MIN_PER_EPOCH_CHURN_LIMIT, - len(get_active_validator_indices(state, get_current_epoch(state))) // CHURN_LIMIT_QUOTIENT - ) -``` - -### `bls_verify` - -`bls_verify` is a function for verifying a BLS signature, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify). - -### `bls_verify_multiple` - -`bls_verify_multiple` is a function for verifying a BLS signature constructed from multiple messages, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify_multiple). - -### `bls_aggregate_pubkeys` - -`bls_aggregate_pubkeys` is a function for aggregating multiple BLS public keys into a single aggregate key, as defined in the [BLS Signature spec](../bls_signature.md#bls_aggregate_pubkeys). - ### Routines for updating validator status *Note*: All functions in this section mutate `state`. @@ -1340,8 +1358,8 @@ def process_justification_and_finalization(state: BeaconState) -> None: def process_crosslinks(state: BeaconState) -> None: state.previous_crosslinks = [c for c in state.current_crosslinks] for epoch in (get_previous_epoch(state), get_current_epoch(state)): - for offset in range(get_epoch_committee_count(state, epoch)): - shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) + for offset in range(get_committee_count(state, epoch)): + shard = Shard((get_start_shard(state, epoch) + offset) % SHARD_COUNT) crosslink_committee = set(get_crosslink_committee(state, epoch, shard)) winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) if 3 * get_total_balance(state, attesting_indices) >= 2 * get_total_balance(state, crosslink_committee): @@ -1413,8 +1431,8 @@ def get_crosslink_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[G rewards = [Gwei(0) for _ in range(len(state.validators))] penalties = [Gwei(0) for _ in range(len(state.validators))] epoch = get_previous_epoch(state) - for offset in range(get_epoch_committee_count(state, epoch)): - shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) + for offset in range(get_committee_count(state, epoch)): + shard = Shard((get_start_shard(state, epoch) + offset) % SHARD_COUNT) crosslink_committee = set(get_crosslink_committee(state, epoch, shard)) winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) attesting_balance = get_total_balance(state, attesting_indices) @@ -1674,7 +1692,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] # Check signature - validate_indexed_attestation(state, convert_to_indexed(state, attestation)) + validate_indexed_attestation(state, get_indexed_attestation(state, attestation)) ``` ##### Deposits @@ -1687,7 +1705,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Verify the Merkle branch assert verify_merkle_branch( leaf=hash_tree_root(deposit.data), - proof=deposit.proof, + branch=deposit.proof, depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # add 1 for the SSZ length mix-in index=state.eth1_deposit_index, root=state.eth1_data.deposit_root, diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index c54824f3bf..64afb3f6b1 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -124,7 +124,7 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice head = store.justified_checkpoint.root - justified_slot = get_epoch_start_slot(store.justified_checkpoint.epoch) + justified_slot = epoch_start_slot(store.justified_checkpoint.epoch) while True: children = [ root for root in store.blocks.keys() @@ -162,7 +162,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.finalized_checkpoint.root ) # Check that block is later than the finalized epoch slot - assert block.slot > get_epoch_start_slot(store.finalized_checkpoint.epoch) + assert block.slot > epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state for this block to the store @@ -190,11 +190,11 @@ def on_attestation(store: Store, attestation: Attestation) -> None: # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrivesr base_state = store.block_states[target.root].copy() - assert store.time >= base_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT + assert store.time >= base_state.genesis_time + epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - process_slots(base_state, get_epoch_start_slot(target.epoch)) + process_slots(base_state, epoch_start_slot(target.epoch)) store.checkpoint_states[target] = base_state target_state = store.checkpoint_states[target] @@ -204,7 +204,7 @@ def on_attestation(store: Store, attestation: Attestation) -> None: assert store.time >= (attestation_slot + 1) * SECONDS_PER_SLOT # Get state at the `target` to validate attestation and calculate the committees - indexed_attestation = convert_to_indexed(target_state, attestation) + indexed_attestation = get_indexed_attestation(target_state, attestation) validate_indexed_attestation(target_state, indexed_attestation) # Update latest messages diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 24076fef65..1fb5b4d3e4 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -473,7 +473,7 @@ For each `challenge` in `block.body.custody_chunk_challenges`, run the following ```python def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: # Verify the attestation - validate_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) + validate_indexed_attestation(state, get_indexed_attestation(state, challenge.attestation)) # Verify it is not too late to challenge assert slot_to_epoch(challenge.attestation.data.slot) >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY responder = state.validators[challenge.responder_index] @@ -526,7 +526,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> # Verify challenger is slashable assert is_slashable_validator(challenger, get_current_epoch(state)) # Verify attestation - validate_indexed_attestation(state, convert_to_indexed(state, attestation)) + validate_indexed_attestation(state, get_indexed_attestation(state, attestation)) # Verify attestation is eligible for challenging responder = state.validators[challenge.responder_index] assert epoch + responder.max_reveal_lateness <= get_reveal_period(state, challenge.responder_index) @@ -601,7 +601,7 @@ def process_chunk_challenge_response(state: BeaconState, # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), - proof=response.data_branch, + branch=response.data_branch, depth=challenge.depth, index=response.chunk_index, root=challenge.data_root, @@ -626,7 +626,7 @@ def process_bit_challenge_response(state: BeaconState, # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), - proof=response.data_branch, + branch=response.data_branch, depth=ceillog2(challenge.chunk_count), index=response.chunk_index, root=challenge.data_root, @@ -634,7 +634,7 @@ def process_bit_challenge_response(state: BeaconState, # Verify the chunk bit leaf matches the challenge data assert verify_merkle_branch( leaf=response.chunk_bits_leaf, - proof=response.chunk_bits_branch, + branch=response.chunk_bits_branch, depth=ceillog2(challenge.chunk_count) >> 8, index=response.chunk_index // 256, root=challenge.chunk_bits_merkle_root diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 432820b7cf..eb20abe7f5 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -139,7 +139,7 @@ def get_period_committee(state: BeaconState, """ return compute_committee( indices=get_active_validator_indices(state, epoch), - seed=generate_seed(state, epoch), + seed=get_seed(state, epoch), index=shard * count + index, count=SHARD_COUNT * count, ) @@ -150,7 +150,7 @@ def get_period_committee(state: BeaconState, ```python def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex) -> int: earlier_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2) - return (bytes_to_int(hash(generate_seed(state, earlier_start_epoch) + int_to_bytes(index, length=3)[0:8])) + return (bytes_to_int(hash(get_seed(state, earlier_start_epoch) + int_to_bytes(index, length=3)[0:8])) % PERSISTENT_COMMITTEE_PERIOD) ``` diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 9a089720b5..2b1703f217 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -84,7 +84,7 @@ def get_period_data(block: ExtendedBeaconBlock, shard_id: Shard, later: bool) -> indices = get_period_committee(block.state, shard_id, period_start, 0, committee_count) return PeriodData( validator_count, - generate_seed(block.state, period_start), + get_seed(block.state, period_start), [block.state.validators[i] for i in indices], ) ``` diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 1b103b218d..f4a83b8335 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -145,11 +145,11 @@ def get_committee_assignment( next_epoch = get_current_epoch(state) + 1 assert epoch <= next_epoch - committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH - epoch_start_slot = get_epoch_start_slot(epoch) - for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH): + committees_per_slot = get_committee_count(state, epoch) // SLOTS_PER_EPOCH + start_slot = epoch_start_slot(epoch) + for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) - slot_start_shard = (get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT + slot_start_shard = (get_start_shard(state, epoch) + offset) % SHARD_COUNT for i in range(committees_per_slot): shard = (slot_start_shard + i) % SHARD_COUNT committee = get_crosslink_committee(state, epoch, shard) @@ -307,8 +307,8 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. * Set `attestation_data.target_root = epoch_boundary_block_root` where `epoch_boundary_block_root` is the root of block at the most recent epoch boundary. *Note*: `epoch_boundary_block_root` can be looked up in the state using: -* Let `epoch_start_slot = get_epoch_start_slot(get_current_epoch(head_state))`. -* Let `epoch_boundary_block_root = signing_root(head_block) if epoch_start_slot == head_state.slot else get_block_root(state, epoch_start_slot)`. +* Let `start_slot = epoch_start_slot(get_current_epoch(head_state))`. +* Let `epoch_boundary_block_root = signing_root(head_block) if start_slot == head_state.slot else get_block_root(state, start_slot)`. ##### Crosslink vote diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 5adb022a66..4006758880 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -14,7 +14,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): else: assert False - indexed_attestation = spec.convert_to_indexed(state, attestation) + indexed_attestation = spec.get_indexed_attestation(state, attestation) spec.on_attestation(store, attestation) assert ( store.latest_messages[indexed_attestation.custody_bit_0_indices[0]] == diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index ffd8d1afa6..38635bafe6 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -15,7 +15,7 @@ def build_attestation_data(spec, state, slot, shard): else: block_root = spec.get_block_root_at_slot(state, slot) - current_epoch_start_slot = spec.get_epoch_start_slot(spec.get_current_epoch(state)) + current_epoch_start_slot = spec.epoch_start_slot(spec.get_current_epoch(state)) if slot < current_epoch_start_slot: epoch_boundary_root = spec.get_block_root(state, spec.get_previous_epoch(state)) elif slot == current_epoch_start_slot: @@ -54,8 +54,8 @@ def get_valid_attestation(spec, state, slot=None, signed=False): slot = state.slot epoch = spec.slot_to_epoch(slot) - epoch_start_shard = spec.get_epoch_start_shard(state, epoch) - committees_per_slot = spec.get_epoch_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH + epoch_start_shard = spec.get_start_shard(state, epoch) + committees_per_slot = spec.get_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT attestation_data = build_attestation_data(spec, state, slot, shard) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index 9c68b7bbe4..20abcacfb8 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -13,6 +13,6 @@ def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): sign_attestation(spec, state, attestation_2) return spec.AttesterSlashing( - attestation_1=spec.convert_to_indexed(state, attestation_1), - attestation_2=spec.convert_to_indexed(state, attestation_2), + attestation_1=spec.get_indexed_attestation(state, attestation_1), + attestation_2=spec.get_indexed_attestation(state, attestation_2), ) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index 59d47f0ad5..41d784c50c 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -83,7 +83,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): @with_all_phases @spec_state_test def test_double_late_crosslink(spec, state): - if spec.get_epoch_committee_count(state, spec.get_current_epoch(state)) < spec.SHARD_COUNT: + if spec.get_committee_count(state, spec.get_current_epoch(state)) < spec.SHARD_COUNT: print("warning: ignoring test, test-assumptions are incompatible with configuration") return diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 1744d388ca..f59da1fc5c 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -10,8 +10,8 @@ def run_process_just_and_fin(spec, state): def get_shards_for_slot(spec, state, slot): epoch = spec.slot_to_epoch(slot) - epoch_start_shard = spec.get_epoch_start_shard(state, epoch) - committees_per_slot = spec.get_epoch_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH + epoch_start_shard = spec.get_start_shard(state, epoch) + committees_per_slot = spec.get_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT return [shard + i for i in range(committees_per_slot)] @@ -33,8 +33,8 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support total_balance = spec.get_total_active_balance(state) remaining_balance = total_balance * 2 // 3 - epoch_start_slot = spec.get_epoch_start_slot(epoch) - for slot in range(epoch_start_slot, epoch_start_slot + spec.SLOTS_PER_EPOCH): + start_slot = spec.epoch_start_slot(epoch) + for slot in range(start_slot, start_slot + spec.SLOTS_PER_EPOCH): for shard in get_shards_for_slot(spec, state, slot): # Check if we already have had sufficient balance. (and undone if we don't want it). # If so, do not create more attestations. (we do not have empty pending attestations normally anyway) @@ -80,7 +80,7 @@ def get_checkpoints(spec, epoch): def put_checkpoints_in_block_roots(spec, state, checkpoints): for c in checkpoints: - state.block_roots[spec.get_epoch_start_slot(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root + state.block_roots[spec.epoch_start_slot(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root def finalize_on_234(spec, state, epoch, sufficient_support): diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 2d86ef523c..5ff0ed1efb 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -43,7 +43,7 @@ def next_epoch_with_attestations(spec, block = build_empty_block_for_next_slot(spec, post_state) if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): + if slot_to_attest >= spec.epoch_start_slot(spec.get_current_epoch(post_state)): cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) block.body.attestations.append(cur_attestation) From 369c457d76efecd92a6771fefb95792625995107 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 10:11:23 +0100 Subject: [PATCH 327/405] WIP2 --- specs/core/0_beacon-chain.md | 144 +++++++++--------- specs/core/1_custody-game.md | 6 +- test_generators/shuffling/main.py | 2 +- .../pyspec/eth2spec/test/helpers/deposits.py | 2 +- 4 files changed, 80 insertions(+), 74 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c82524eb52..698f026c98 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -49,17 +49,23 @@ - [Beacon state](#beacon-state) - [`BeaconState`](#beaconstate) - [Helper functions](#helper-functions) - - [Math and crypto](#math-and-crypto) + - [Math](#math) - [`int_to_bytes`](#int_to_bytes) - [`bytes_to_int`](#bytes_to_int) - [`integer_squareroot`](#integer_squareroot) - [`xor`](#xor) + - [Crypto](#crypto) - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) - [`signing_root`](#signing_root) - [`bls_verify`](#bls_verify) - [`bls_verify_multiple`](#bls_verify_multiple) - [`bls_aggregate_pubkeys`](#bls_aggregate_pubkeys) + - [Predicates](#predicates) + - [`is_active_validator`](#is_active_validator) + - [`is_slashable_validator`](#is_slashable_validator) + - [`is_valid_merkle_branch`](#is_valid_merkle_branch) + - [`is_slashable_attestation_data`](#is_slashable_attestation_data) - [Beacon state getters](#beacon-state-getters) - [`get_current_epoch`](#get_current_epoch) - [`get_previous_epoch`](#get_previous_epoch) @@ -71,8 +77,7 @@ - [`bls_domain`](#bls_domain) - [`slot_to_epoch`](#slot_to_epoch) - [`epoch_start_slot`](#epoch_start_slot) - - [`is_active_validator`](#is_active_validator) - - [`is_slashable_validator`](#is_slashable_validator) + - [`increase_balance`](#increase_balance) - [`decrease_balance`](#decrease_balance) - [`get_shard_delta`](#get_shard_delta) @@ -83,8 +88,8 @@ - [`get_compact_committees_root`](#get_compact_committees_root) - [`get_seed`](#get_seed) - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - - [`verify_merkle_branch`](#verify_merkle_branch) - - [`get_shuffled_index`](#get_shuffled_index) + + - [`shuffle_index`](#shuffle_index) - [`compute_committee`](#compute_committee) - [`get_crosslink_committee`](#get_crosslink_committee) - [`get_attesting_indices`](#get_attesting_indices) @@ -92,8 +97,7 @@ - [`get_domain`](#get_domain) - [`get_indexed_attestation`](#get_indexed_attestation) - [`validate_indexed_attestation`](#validate_indexed_attestation) - - [`is_slashable_attestation_data`](#is_slashable_attestation_data) - - [`get_delayed_activation_exit_epoch`](#get_delayed_activation_exit_epoch) + - [`delayed_activation_exit_epoch`](#delayed_activation_exit_epoch) - [`get_churn_limit`](#get_churn_limit) - [Routines for updating validator status](#routines-for-updating-validator-status) - [`initiate_validator_exit`](#initiate_validator_exit) @@ -619,6 +623,59 @@ The `hash` function is SHA256. `bls_aggregate_pubkeys` is a function for aggregating multiple BLS public keys into a single aggregate key, as defined in the [BLS Signature spec](../bls_signature.md#bls_aggregate_pubkeys). +### Predicates + +#### `is_valid_merkle_branch` + +```python +def is_valid_merkle_branch(leaf: Hash, branch: Sequence[Hash], depth: int, index: int, root: Hash) -> bool: + """ + Check if ``leaf`` at ``index`` verifies against the Merkle ``root`` and ``branch``. + """ + value = leaf + for i in range(depth): + if index // (2**i) % 2: + value = hash(branch[i] + value) + else: + value = hash(value + branch[i]) + return value == root +``` + +#### `is_active_validator` + +```python +def is_active_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is active. + """ + return validator.activation_epoch <= epoch < validator.exit_epoch +``` + +#### `is_slashable_validator` + +```python +def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is slashable. + """ + return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) +``` + +#### `is_slashable_attestation_data` + +```python +def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationData) -> bool: + """ + Check if ``data_1`` and ``data_2`` are slashable according to Casper FFG rules. + """ + return ( + # Double vote + (data_1 != data_2 and data_1.target.epoch == data_2.target.epoch) or + # Surround vote + (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch) + ) +``` + ### Beacon state getters #### `get_current_epoch` @@ -889,22 +946,6 @@ def get_attesting_indices(state: BeaconState, -#### `verify_merkle_branch` - -```python -def verify_merkle_branch(leaf: Hash, branch: Sequence[Hash], depth: int, index: int, root: Hash) -> bool: - """ - Verify that the ``leaf`` at ``index`` verifies against ``root`` on the Merkle ``branch``. - """ - value = leaf - for i in range(depth): - if index // (2**i) % 2: - value = hash(branch[i] + value) - else: - value = hash(value + branch[i]) - return value == root -``` - #### `bls_domain` @@ -936,24 +977,14 @@ def epoch_start_slot(epoch: Epoch) -> Slot: return Slot(epoch * SLOTS_PER_EPOCH) ``` -#### `is_active_validator` +#### `delayed_activation_exit_epoch` ```python -def is_active_validator(validator: Validator, epoch: Epoch) -> bool: - """ - Check if ``validator`` is active. - """ - return validator.activation_epoch <= epoch < validator.exit_epoch -``` - -#### `is_slashable_validator` - -```python -def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: +def delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: """ - Check if ``validator`` is slashable. + Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. """ - return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) + return Epoch(epoch + 1 + ACTIVATION_EXIT_DELAY) ``` #### `increase_balance` @@ -976,10 +1007,10 @@ def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> state.balances[index] = 0 if delta > state.balances[index] else state.balances[index] - delta ``` -#### `get_shuffled_index` +#### `shuffle_index` ```python -def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Hash) -> ValidatorIndex: +def shuffle_index(index: ValidatorIndex, index_count: int, seed: Hash) -> ValidatorIndex: """ Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). """ @@ -1010,7 +1041,7 @@ def compute_committee(indices: Sequence[ValidatorIndex], seed: Hash, index: int, count: int) -> Sequence[ValidatorIndex]: start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return [indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] + return [indices[shuffle_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] ``` #### `validate_indexed_attestation` @@ -1046,31 +1077,6 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe ) ``` -#### `is_slashable_attestation_data` - -```python -def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationData) -> bool: - """ - Check if ``data_1`` and ``data_2`` are slashable according to Casper FFG rules. - """ - return ( - # Double vote - (data_1 != data_2 and data_1.target.epoch == data_2.target.epoch) or - # Surround vote - (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch) - ) -``` - -#### `get_delayed_activation_exit_epoch` - -```python -def get_delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: - """ - Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. - """ - return Epoch(epoch + 1 + ACTIVATION_EXIT_DELAY) -``` - ### Routines for updating validator status *Note*: All functions in this section mutate `state`. @@ -1089,7 +1095,7 @@ def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: # Compute exit queue epoch exit_epochs = [v.exit_epoch for v in state.validators if v.exit_epoch != FAR_FUTURE_EPOCH] - exit_queue_epoch = max(exit_epochs + [get_delayed_activation_exit_epoch(get_current_epoch(state))]) + exit_queue_epoch = max(exit_epochs + [delayed_activation_exit_epoch(get_current_epoch(state))]) exit_queue_churn = len([v for v in state.validators if v.exit_epoch == exit_queue_epoch]) if exit_queue_churn >= get_churn_limit(state): exit_queue_epoch += Epoch(1) @@ -1477,13 +1483,13 @@ def process_registry_updates(state: BeaconState) -> None: activation_queue = sorted([ index for index, validator in enumerate(state.validators) if validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH and - validator.activation_epoch >= get_delayed_activation_exit_epoch(state.finalized_checkpoint.epoch) + validator.activation_epoch >= delayed_activation_exit_epoch(state.finalized_checkpoint.epoch) ], key=lambda index: state.validators[index].activation_eligibility_epoch) # Dequeued validators for activation up to churn limit (without resetting activation epoch) for index in activation_queue[:get_churn_limit(state)]: validator = state.validators[index] if validator.activation_epoch == FAR_FUTURE_EPOCH: - validator.activation_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) + validator.activation_epoch = delayed_activation_exit_epoch(get_current_epoch(state)) ``` #### Slashings @@ -1703,7 +1709,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: Process an Eth1 deposit, registering a validator or increasing its balance. """ # Verify the Merkle branch - assert verify_merkle_branch( + assert is_valid_merkle_branch( leaf=hash_tree_root(deposit.data), branch=deposit.proof, depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # add 1 for the SSZ length mix-in diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 1fb5b4d3e4..e588e9fdf4 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -599,7 +599,7 @@ def process_chunk_challenge_response(state: BeaconState, # Verify minimum delay assert get_current_epoch(state) >= challenge.inclusion_epoch + ACTIVATION_EXIT_DELAY # Verify the chunk matches the crosslink data root - assert verify_merkle_branch( + assert is_valid_merkle_branch( leaf=hash_tree_root(response.chunk), branch=response.data_branch, depth=challenge.depth, @@ -624,7 +624,7 @@ def process_bit_challenge_response(state: BeaconState, responder = state.validators[challenge.responder_index] assert not responder.slashed # Verify the chunk matches the crosslink data root - assert verify_merkle_branch( + assert is_valid_merkle_branch( leaf=hash_tree_root(response.chunk), branch=response.data_branch, depth=ceillog2(challenge.chunk_count), @@ -632,7 +632,7 @@ def process_bit_challenge_response(state: BeaconState, root=challenge.data_root, ) # Verify the chunk bit leaf matches the challenge data - assert verify_merkle_branch( + assert is_valid_merkle_branch( leaf=response.chunk_bits_leaf, branch=response.chunk_bits_branch, depth=ceillog2(challenge.chunk_count) >> 8, diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index 291aa2c47d..e40dda5209 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -10,7 +10,7 @@ def shuffling_case(seed, count): yield 'seed', '0x' + seed.hex() yield 'count', count - yield 'shuffled', [spec.get_shuffled_index(i, count, seed) for i in range(count)] + yield 'shuffled', [spec.shuffle_index(i, count, seed) for i in range(count)] @to_tuple diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index b46363e62a..4f099be9d1 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -49,7 +49,7 @@ def build_deposit(spec, tree = calc_merkle_tree_from_leaves(tuple([d.hash_tree_root() for d in deposit_data_list])) proof = list(get_merkle_proof(tree, item_index=index)) + [(index + 1).to_bytes(32, 'little')] leaf = deposit_data.hash_tree_root() - assert spec.verify_merkle_branch(leaf, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH + 1, index, root) + assert spec.is_valid_merkle_branch(leaf, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH + 1, index, root) deposit = spec.Deposit(proof=proof, data=deposit_data) return deposit, root, deposit_data_list From f6322e199b8ead54a094f9ac3ca856dfb4697727 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 30 Jun 2019 11:18:36 +0200 Subject: [PATCH 328/405] Fixes typing and Makefile --- Makefile | 3 +-- specs/validator/0_beacon-chain-validator.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 826c4a41e3..617eff293c 100644 --- a/Makefile +++ b/Makefile @@ -40,8 +40,7 @@ partial_clean: rm -rf $(PY_SPEC_DIR)/.coverage rm -rf $(PY_SPEC_DIR)/test-reports -clean: - partial_clean +clean: partial_clean rm -rf $(PY_SPEC_DIR)/venv rm -rf $(DEPOSIT_CONTRACT_DIR)/venv diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 0b92928590..294d2a5d58 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -135,7 +135,7 @@ A validator can get committee assignments for a given epoch using the following def get_committee_assignment( state: BeaconState, epoch: Epoch, - validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]: + validator_index: ValidatorIndex) -> Tuple[Sequence[ValidatorIndex], Shard, Slot]: """ Return the committee assignment in the ``epoch`` for ``validator_index``. ``assignment`` returned is a tuple of the following form: From e8532ced79877847a40a46e597a5e64d3f292d1d Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 10:56:14 +0100 Subject: [PATCH 329/405] WIP --- specs/core/0_beacon-chain.md | 425 ++++++++---------- specs/core/0_fork-choice.md | 8 +- specs/core/1_custody-game.md | 8 +- specs/validator/0_beacon-chain-validator.md | 6 +- .../eth2spec/test/helpers/attestations.py | 8 +- .../pyspec/eth2spec/test/helpers/block.py | 4 +- .../test_process_attester_slashing.py | 2 +- .../test_process_block_header.py | 2 +- .../block_processing/test_process_transfer.py | 2 +- ..._process_justification_and_finalization.py | 4 +- .../eth2spec/test/sanity/test_blocks.py | 2 +- .../pyspec/eth2spec/test/test_finality.py | 2 +- 12 files changed, 223 insertions(+), 250 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 698f026c98..1d4a486a1c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -50,10 +50,10 @@ - [`BeaconState`](#beaconstate) - [Helper functions](#helper-functions) - [Math](#math) - - [`int_to_bytes`](#int_to_bytes) - - [`bytes_to_int`](#bytes_to_int) - [`integer_squareroot`](#integer_squareroot) - [`xor`](#xor) + - [`int_to_bytes`](#int_to_bytes) + - [`bytes_to_int`](#bytes_to_int) - [Crypto](#crypto) - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) @@ -64,42 +64,39 @@ - [Predicates](#predicates) - [`is_active_validator`](#is_active_validator) - [`is_slashable_validator`](#is_slashable_validator) - - [`is_valid_merkle_branch`](#is_valid_merkle_branch) - [`is_slashable_attestation_data`](#is_slashable_attestation_data) + - [`is_valid_merkle_branch`](#is_valid_merkle_branch) + - [Misc](#misc) + - [`shuffle_index`](#shuffle_index) + - [`compute_committee`](#compute_committee) + - [`validate_indexed_attestation`](#validate_indexed_attestation) + - [`slot_to_epoch`](#slot_to_epoch) + - [`epoch_first_slot`](#epoch_first_slot) + - [`delayed_activation_exit_epoch`](#delayed_activation_exit_epoch) + - [`bls_domain`](#bls_domain) - [Beacon state getters](#beacon-state-getters) - [`get_current_epoch`](#get_current_epoch) - [`get_previous_epoch`](#get_previous_epoch) + - [`get_block_root`](#get_block_root) + - [`get_block_root_at_slot`](#get_block_root_at_slot) + - [`get_randao_mix`](#get_randao_mix) + - [`get_active_validator_indices`](#get_active_validator_indices) + - [`get_churn_limit`](#get_churn_limit) - [`get_committee_count`](#get_committee_count) + - [`get_seed`](#get_seed) + - [`get_crosslink_committee`](#get_crosslink_committee) - [`get_start_shard`](#get_start_shard) - - [`get_active_validator_indices`](#get_active_validator_indices) - - - - [`bls_domain`](#bls_domain) - - [`slot_to_epoch`](#slot_to_epoch) - - [`epoch_start_slot`](#epoch_start_slot) - - - [`increase_balance`](#increase_balance) - - [`decrease_balance`](#decrease_balance) - [`get_shard_delta`](#get_shard_delta) + - [`get_proposer_index`](#get_proposer_index) - [`get_attestation_data_slot`](#get_attestation_data_slot) - - [`get_block_root_at_slot`](#get_block_root_at_slot) - - [`get_block_root`](#get_block_root) - - [`get_randao_mix`](#get_randao_mix) - [`get_compact_committees_root`](#get_compact_committees_root) - - [`get_seed`](#get_seed) - - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - - - [`shuffle_index`](#shuffle_index) - - [`compute_committee`](#compute_committee) - - [`get_crosslink_committee`](#get_crosslink_committee) - - [`get_attesting_indices`](#get_attesting_indices) - [`get_total_balance`](#get_total_balance) - [`get_domain`](#get_domain) - [`get_indexed_attestation`](#get_indexed_attestation) - - [`validate_indexed_attestation`](#validate_indexed_attestation) - - [`delayed_activation_exit_epoch`](#delayed_activation_exit_epoch) - - [`get_churn_limit`](#get_churn_limit) - - [Routines for updating validator status](#routines-for-updating-validator-status) + - [`get_attesting_indices`](#get_attesting_indices) + - [State mutators](#state-mutators) + - [`increase_balance`](#increase_balance) + - [`decrease_balance`](#decrease_balance) - [`initiate_validator_exit`](#initiate_validator_exit) - [`slash_validator`](#slash_validator) - [Genesis](#genesis) @@ -201,6 +198,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**16` (= 65,536) | | `MIN_GENESIS_TIME` | `1578009600` (Jan 3, 2020) | | `JUSTIFICATION_BITS_LENGTH` | `4` | +| `ENDIANNESS` | `'little'` | * For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -559,18 +557,6 @@ class BeaconState(Container): #### `int_to_bytes` -```python -def int_to_bytes(integer: int, length: int) -> bytes: - return integer.to_bytes(length, 'little') -``` - -#### `bytes_to_int` - -```python -def bytes_to_int(data: bytes) -> int: - return int.from_bytes(data, 'little') -``` - #### `integer_squareroot` ```python @@ -597,11 +583,29 @@ def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: return Bytes32(a ^ b for a, b in zip(bytes1, bytes2)) ``` +```python +def int_to_bytes(integer: int, length: int) -> bytes: + """ + Return the ``length``-byte serialization of ``integer``. + """ + return integer.to_bytes(length, ENDIANNESS) +``` + +#### `bytes_to_int` + +```python +def bytes_to_int(data: bytes) -> int: + """ + Return the integer deserialization of ``data``. + """ + return int.from_bytes(data, ENDIANNESS) +``` + ### Crypto #### `hash` -The `hash` function is SHA256. +`def hash(data: bytes) -> Hash` is SHA256. #### `hash_tree_root` @@ -625,22 +629,6 @@ The `hash` function is SHA256. ### Predicates -#### `is_valid_merkle_branch` - -```python -def is_valid_merkle_branch(leaf: Hash, branch: Sequence[Hash], depth: int, index: int, root: Hash) -> bool: - """ - Check if ``leaf`` at ``index`` verifies against the Merkle ``root`` and ``branch``. - """ - value = leaf - for i in range(depth): - if index // (2**i) % 2: - value = hash(branch[i] + value) - else: - value = hash(value + branch[i]) - return value == root -``` - #### `is_active_validator` ```python @@ -676,6 +664,137 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa ) ``` +#### `is_valid_merkle_branch` + +```python +def is_valid_merkle_branch(leaf: Hash, branch: Sequence[Hash], depth: int, index: int, root: Hash) -> bool: + """ + Check if ``leaf`` at ``index`` verifies against the Merkle ``root`` and ``branch``. + """ + value = leaf + for i in range(depth): + if index // (2**i) % 2: + value = hash(branch[i] + value) + else: + value = hash(value + branch[i]) + return value == root +``` + +### Misc + +#### `shuffle_index` + +```python +def shuffle_index(index: ValidatorIndex, index_count: int, seed: Hash) -> ValidatorIndex: + """ + Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). + """ + assert index < index_count + assert index_count <= 2**40 + + # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) + # See the 'generalized domain' algorithm on page 3 + for current_round in range(SHUFFLE_ROUND_COUNT): + pivot = bytes_to_int(hash(seed + int_to_bytes(current_round, length=1))[0:8]) % index_count + flip = ValidatorIndex((pivot + index_count - index) % index_count) + position = max(index, flip) + source = hash( + seed + int_to_bytes(current_round, length=1) + + int_to_bytes(position // 256, length=4) + ) + byte = source[(position % 256) // 8] + bit = (byte >> (position % 8)) % 2 + index = flip if bit else index + + return ValidatorIndex(index) +``` + +#### `compute_committee` + +```python +def compute_committee(indices: Sequence[ValidatorIndex], + seed: Hash, index: int, count: int) -> Sequence[ValidatorIndex]: + """ + Return the committee corresponding to ``indices``, ``seed``, ``index`` and committee ``count``. + """ + start = (len(indices) * index) // count + end = (len(indices) * (index + 1)) // count + return [indices[shuffle_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] +``` + +#### `validate_indexed_attestation` + +```python +def validate_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> None: + """ + Verify validity of ``indexed_attestation``. + """ + bit_0_indices = indexed_attestation.custody_bit_0_indices + bit_1_indices = indexed_attestation.custody_bit_1_indices + + # Verify no index has custody bit equal to 1 [to be removed in phase 1] + assert len(bit_1_indices) == 0 + # Verify max number of indices + assert len(bit_0_indices) + len(bit_1_indices) <= MAX_VALIDATORS_PER_COMMITTEE + # Verify index sets are disjoint + assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 + # Verify indices are sorted + assert bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices) + # Verify aggregate signature + assert bls_verify_multiple( + pubkeys=[ + bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_0_indices]), + bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_1_indices]), + ], + message_hashes=[ + hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), + hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), + ], + signature=indexed_attestation.signature, + domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target.epoch), + ) +``` + +#### `slot_to_epoch` + +```python +def slot_to_epoch(slot: Slot) -> Epoch: + """ + Return the epoch number of ``slot``. + """ + return Epoch(slot // SLOTS_PER_EPOCH) +``` + +#### `epoch_first_slot` + +```python +def epoch_first_slot(epoch: Epoch) -> Slot: + """ + Return the first slot of ``epoch``. + """ + return Slot(epoch * SLOTS_PER_EPOCH) +``` + +#### `delayed_activation_exit_epoch` + +```python +def delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: + """ + Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. + """ + return Epoch(epoch + 1 + ACTIVATION_EXIT_DELAY) +``` + +#### `bls_domain` + +```python +def bls_domain(domain_type: int, fork_version: bytes=b'\x00' * 4) -> int: + """ + Return the BLS domain for the ``domain_type`` and ``fork_version``. + """ + return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) +``` + ### Beacon state getters #### `get_current_epoch` @@ -706,7 +825,7 @@ def get_block_root(state: BeaconState, epoch: Epoch) -> Hash: """ Return the block root at a recent ``epoch``. """ - return get_block_root_at_slot(state, epoch_start_slot(epoch)) + return get_block_root_at_slot(state, epoch_first_slot(epoch)) ``` #### `get_block_root_at_slot` @@ -824,10 +943,10 @@ def get_shard_delta(state: BeaconState, epoch: Epoch) -> Shard: return Shard(min(get_committee_count(state, epoch), SHARD_COUNT - SHARD_COUNT // SLOTS_PER_EPOCH)) ``` -#### `get_beacon_proposer_index` +#### `get_proposer_index` ```python -def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: +def get_proposer_index(state: BeaconState) -> ValidatorIndex: """ Return the beacon proposer index at the current epoch. """ @@ -857,7 +976,7 @@ def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot """ committee_count = get_committee_count(state, data.target.epoch) offset = (data.crosslink.shard + SHARD_COUNT - get_start_shard(state, data.target.epoch)) % SHARD_COUNT - return Slot(epoch_start_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) + return Slot(epoch_first_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` #### `get_compact_committees_root` @@ -936,63 +1055,14 @@ def get_attesting_indices(state: BeaconState, return set(index for i, index in enumerate(committee) if bits[i]) ``` - - - - - - - - - - - -#### `bls_domain` - -```python -def bls_domain(domain_type: int, fork_version: bytes=b'\x00\x00\x00\x00') -> int: - """ - Return the bls domain given by the ``domain_type`` and optional 4 byte ``fork_version`` (defaults to zero). - """ - return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) -``` - -#### `slot_to_epoch` - -```python -def slot_to_epoch(slot: Slot) -> Epoch: - """ - Return the epoch number of the given ``slot``. - """ - return Epoch(slot // SLOTS_PER_EPOCH) -``` - -#### `epoch_start_slot` - -```python -def epoch_start_slot(epoch: Epoch) -> Slot: - """ - Return the starting slot of the given ``epoch``. - """ - return Slot(epoch * SLOTS_PER_EPOCH) -``` - -#### `delayed_activation_exit_epoch` - -```python -def delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: - """ - Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. - """ - return Epoch(epoch + 1 + ACTIVATION_EXIT_DELAY) -``` +### State mutators #### `increase_balance` ```python def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: """ - Increase validator balance by ``delta``. + Increase the validator balance at index ``index`` by ``delta``. """ state.balances[index] += delta ``` @@ -1002,91 +1072,17 @@ def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> ```python def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: """ - Decrease validator balance by ``delta`` with underflow protection. + Decrease the validator balance at index ``index`` by ``delta``, with underflow protection. """ state.balances[index] = 0 if delta > state.balances[index] else state.balances[index] - delta ``` -#### `shuffle_index` - -```python -def shuffle_index(index: ValidatorIndex, index_count: int, seed: Hash) -> ValidatorIndex: - """ - Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). - """ - assert index < index_count - assert index_count <= 2**40 - - # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) - # See the 'generalized domain' algorithm on page 3 - for current_round in range(SHUFFLE_ROUND_COUNT): - pivot = bytes_to_int(hash(seed + int_to_bytes(current_round, length=1))[0:8]) % index_count - flip = ValidatorIndex((pivot + index_count - index) % index_count) - position = max(index, flip) - source = hash( - seed + int_to_bytes(current_round, length=1) + - int_to_bytes(position // 256, length=4) - ) - byte = source[(position % 256) // 8] - bit = (byte >> (position % 8)) % 2 - index = flip if bit else index - - return ValidatorIndex(index) -``` - -#### `compute_committee` - -```python -def compute_committee(indices: Sequence[ValidatorIndex], - seed: Hash, index: int, count: int) -> Sequence[ValidatorIndex]: - start = (len(indices) * index) // count - end = (len(indices) * (index + 1)) // count - return [indices[shuffle_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] -``` - -#### `validate_indexed_attestation` - -```python -def validate_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> None: - """ - Verify validity of ``indexed_attestation``. - """ - bit_0_indices = indexed_attestation.custody_bit_0_indices - bit_1_indices = indexed_attestation.custody_bit_1_indices - - # Verify no index has custody bit equal to 1 [to be removed in phase 1] - assert len(bit_1_indices) == 0 - # Verify max number of indices - assert len(bit_0_indices) + len(bit_1_indices) <= MAX_VALIDATORS_PER_COMMITTEE - # Verify index sets are disjoint - assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 - # Verify indices are sorted - assert bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices) - # Verify aggregate signature - assert bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_0_indices]), - bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_1_indices]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), - ], - signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target.epoch), - ) -``` - -### Routines for updating validator status - -*Note*: All functions in this section mutate `state`. - #### `initiate_validator_exit` ```python def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: """ - Initiate the exit of the validator of the given ``index``. + Initiate the exit of the validator with index ``index``. """ # Return if validator already initiated exit validator = state.validators[index] @@ -1123,7 +1119,7 @@ def slash_validator(state: BeaconState, decrease_balance(state, slashed_index, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT) # Apply proposer and whistleblower rewards - proposer_index = get_beacon_proposer_index(state) + proposer_index = get_proposer_index(state) if whistleblower_index is None: whistleblower_index = proposer_index whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) @@ -1134,22 +1130,12 @@ def slash_validator(state: BeaconState, ## Genesis -### Genesis state - -Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 block, call `initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)` where: +Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 block, let `candidate_state = initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)` where: * `eth1_block_hash` is the hash of the Ethereum 1.0 block * `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` * `deposits` is the sequence of all deposits, ordered chronologically, up to the block with hash `eth1_block_hash` -The genesis state `genesis_state` is the return value of calling `initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)` only if `is_valid_genesis_state(genesis_state) is True`. - -Implementations can choose to support different (more optimized) variations of the below initialization approach: - - Build the `genesis_state` from a stream of deposits by incrementally updating the `state.eth1_data.deposit_root`. - - Compute deposit proofs for the final `state.eth1_data.deposit_root`, and process as a pre-determined collection. - -*Note*: The two constants `MIN_GENESIS_TIME` and `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` have yet to be agreed upon by the community, and can be updated as necessary. - ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Hash, eth1_timestamp: uint64, @@ -1175,22 +1161,27 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Hash, validator.activation_epoch = GENESIS_EPOCH # Populate compact_committees_roots - genesis_committee_root = get_compact_committees_root(state, GENESIS_EPOCH) + committee_root = get_compact_committees_root(state, GENESIS_EPOCH) for index in range(EPOCHS_PER_HISTORICAL_VECTOR): - state.compact_committees_roots[index] = genesis_committee_root + state.compact_committees_roots[index] = committee_root return state ``` +### Genesis state + +Let `genesis_state = candidate_state` whenever `is_valid_genesis_state(candidate_state) is True` for the first time. + ```python def is_valid_genesis_state(state: BeaconState) -> bool: if state.genesis_time < MIN_GENESIS_TIME: return False - elif len(get_active_validator_indices(state, GENESIS_EPOCH)) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: + if len(get_active_validator_indices(state, GENESIS_EPOCH)) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: return False - else: - return True + return True ``` +*Note*: The `is_valid_genesis_state` function (including `MIN_GENESIS_TIME` and `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT`) is a placeholder for testing. It has yet to be finalized by the community, and can be updated as necessary. + ### Genesis block Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. @@ -1287,7 +1278,7 @@ def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Sequence ```python def get_unslashed_attesting_indices(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: - output = set() # type: Set[ValidatorIndex] + output = set() # Type: Set[ValidatorIndex] for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) return set(filter(lambda index: not state.validators[index].slashed, list(output))) @@ -1566,7 +1557,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: body_root=hash_tree_root(block.body), ) # Verify proposer is not slashed - proposer = state.validators[get_beacon_proposer_index(state)] + proposer = state.validators[get_proposer_index(state)] assert not proposer.slashed # Verify proposer signature assert bls_verify(proposer.pubkey, signing_root(block), block.signature, get_domain(state, DOMAIN_BEACON_PROPOSER)) @@ -1578,7 +1569,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: epoch = get_current_epoch(state) # Verify RANDAO reveal - proposer = state.validators[get_beacon_proposer_index(state)] + proposer = state.validators[get_proposer_index(state)] assert bls_verify(proposer.pubkey, hash_tree_root(epoch), body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) # Mix in RANDAO reveal mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) @@ -1620,9 +1611,6 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: ```python def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: - """ - Process ``ProposerSlashing`` operation. - """ proposer = state.validators[proposer_slashing.proposer_index] # Verify that the epoch is the same assert slot_to_epoch(proposer_slashing.header_1.slot) == slot_to_epoch(proposer_slashing.header_2.slot) @@ -1642,9 +1630,6 @@ def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSla ```python def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: - """ - Process ``AttesterSlashing`` operation. - """ attestation_1 = attester_slashing.attestation_1 attestation_2 = attester_slashing.attestation_2 assert is_slashable_attestation_data(attestation_1.data, attestation_2.data) @@ -1665,9 +1650,6 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla ```python def process_attestation(state: BeaconState, attestation: Attestation) -> None: - """ - Process ``Attestation`` operation. - """ data = attestation.data assert data.crosslink.shard < SHARD_COUNT assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) @@ -1679,7 +1661,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: data=data, aggregation_bits=attestation.aggregation_bits, inclusion_delay=state.slot - attestation_slot, - proposer_index=get_beacon_proposer_index(state), + proposer_index=get_proposer_index(state), ) if data.target.epoch == get_current_epoch(state): @@ -1705,9 +1687,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ```python def process_deposit(state: BeaconState, deposit: Deposit) -> None: - """ - Process an Eth1 deposit, registering a validator or increasing its balance. - """ # Verify the Merkle branch assert is_valid_merkle_branch( leaf=hash_tree_root(deposit.data), @@ -1754,9 +1733,6 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: ```python def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: - """ - Process ``VoluntaryExit`` operation. - """ validator = state.validators[exit.validator_index] # Verify the validator is active assert is_active_validator(validator, get_current_epoch(state)) @@ -1777,9 +1753,6 @@ def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: ```python def process_transfer(state: BeaconState, transfer: Transfer) -> None: - """ - Process ``Transfer`` operation. - """ # Verify the balance the covers amount and fee (with overflow protection) assert state.balances[transfer.sender] >= max(transfer.amount + transfer.fee, transfer.amount, transfer.fee) # A transfer is valid in only one slot @@ -1803,7 +1776,7 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: # Process the transfer decrease_balance(state, transfer.sender, transfer.amount + transfer.fee) increase_balance(state, transfer.recipient, transfer.amount) - increase_balance(state, get_beacon_proposer_index(state), transfer.fee) + increase_balance(state, get_proposer_index(state), transfer.fee) # Verify balances are not dust assert not (0 < state.balances[transfer.sender] < MIN_DEPOSIT_AMOUNT) assert not (0 < state.balances[transfer.recipient] < MIN_DEPOSIT_AMOUNT) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 64afb3f6b1..b2ae89e9f1 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -124,7 +124,7 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice head = store.justified_checkpoint.root - justified_slot = epoch_start_slot(store.justified_checkpoint.epoch) + justified_slot = epoch_first_slot(store.justified_checkpoint.epoch) while True: children = [ root for root in store.blocks.keys() @@ -162,7 +162,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.finalized_checkpoint.root ) # Check that block is later than the finalized epoch slot - assert block.slot > epoch_start_slot(store.finalized_checkpoint.epoch) + assert block.slot > epoch_first_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state for this block to the store @@ -190,11 +190,11 @@ def on_attestation(store: Store, attestation: Attestation) -> None: # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrivesr base_state = store.block_states[target.root].copy() - assert store.time >= base_state.genesis_time + epoch_start_slot(target.epoch) * SECONDS_PER_SLOT + assert store.time >= base_state.genesis_time + epoch_first_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - process_slots(base_state, epoch_start_slot(target.epoch)) + process_slots(base_state, epoch_first_slot(target.epoch)) store.checkpoint_states[target] = base_state target_state = store.checkpoint_states[target] diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index e588e9fdf4..993a45b0fd 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -382,7 +382,7 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> revealer.next_custody_reveal_period += 1 # Reward Block Preposer - proposer_index = get_beacon_proposer_index(state) + proposer_index = get_proposer_index(state) increase_balance( state, proposer_index, @@ -452,7 +452,7 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived ) # Apply penalty - proposer_index = get_beacon_proposer_index(state) + proposer_index = get_proposer_index(state) whistleblower_index = reveal.masker_index whistleblowing_reward = Gwei(penalty // WHISTLEBLOWER_REWARD_QUOTIENT) proposer_reward = Gwei(whistleblowing_reward // PROPOSER_REWARD_QUOTIENT) @@ -493,7 +493,7 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( challenge_index=state.custody_challenge_index, - challenger_index=get_beacon_proposer_index(state), + challenger_index=get_proposer_index(state), responder_index=challenge.responder_index, inclusion_epoch=get_current_epoch(state), data_root=challenge.attestation.data.crosslink.data_root, @@ -610,7 +610,7 @@ def process_chunk_challenge_response(state: BeaconState, records = state.custody_chunk_challenge_records records[records.index(challenge)] = CustodyChunkChallengeRecord() # Reward the proposer - proposer_index = get_beacon_proposer_index(state) + proposer_index = get_proposer_index(state) increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) ``` diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index f4a83b8335..9921bf0348 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -146,7 +146,7 @@ def get_committee_assignment( assert epoch <= next_epoch committees_per_slot = get_committee_count(state, epoch) // SLOTS_PER_EPOCH - start_slot = epoch_start_slot(epoch) + start_slot = epoch_first_slot(epoch) for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) slot_start_shard = (get_start_shard(state, epoch) + offset) % SHARD_COUNT @@ -162,7 +162,7 @@ A validator can use the following function to see if they are supposed to propos ```python def is_proposer(state: BeaconState, validator_index: ValidatorIndex) -> bool: - return get_beacon_proposer_index(state) == validator_index + return get_proposer_index(state) == validator_index ``` *Note*: To see if a validator is assigned to propose during the slot, the beacon state must be in the epoch in question. At the epoch boundaries, the validator must run an epoch transition into the epoch to successfully check the proposal assignment of the first slot. @@ -307,7 +307,7 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. * Set `attestation_data.target_root = epoch_boundary_block_root` where `epoch_boundary_block_root` is the root of block at the most recent epoch boundary. *Note*: `epoch_boundary_block_root` can be looked up in the state using: -* Let `start_slot = epoch_start_slot(get_current_epoch(head_state))`. +* Let `start_slot = epoch_first_slot(get_current_epoch(head_state))`. * Let `epoch_boundary_block_root = signing_root(head_block) if start_slot == head_state.slot else get_block_root(state, start_slot)`. ##### Crosslink vote diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 38635bafe6..053446a2db 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -15,15 +15,15 @@ def build_attestation_data(spec, state, slot, shard): else: block_root = spec.get_block_root_at_slot(state, slot) - current_epoch_start_slot = spec.epoch_start_slot(spec.get_current_epoch(state)) - if slot < current_epoch_start_slot: + current_epoch_first_slot = spec.epoch_first_slot(spec.get_current_epoch(state)) + if slot < current_epoch_first_slot: epoch_boundary_root = spec.get_block_root(state, spec.get_previous_epoch(state)) - elif slot == current_epoch_start_slot: + elif slot == current_epoch_first_slot: epoch_boundary_root = block_root else: epoch_boundary_root = spec.get_block_root(state, spec.get_current_epoch(state)) - if slot < current_epoch_start_slot: + if slot < current_epoch_first_slot: source_epoch = state.previous_justified_checkpoint.epoch source_root = state.previous_justified_checkpoint.root else: diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 16249fe93d..324c5bbe7d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -12,7 +12,7 @@ def sign_block(spec, state, block, proposer_index=None): if proposer_index is None: if block.slot == state.slot: - proposer_index = spec.get_beacon_proposer_index(state) + proposer_index = spec.get_proposer_index(state) else: if spec.slot_to_epoch(state.slot) + 1 > spec.slot_to_epoch(block.slot): print("warning: block slot far away, and no proposer index manually given." @@ -20,7 +20,7 @@ def sign_block(spec, state, block, proposer_index=None): # use stub state to get proposer index of future slot stub_state = deepcopy(state) spec.process_slots(stub_state, block.slot) - proposer_index = spec.get_beacon_proposer_index(stub_state) + proposer_index = spec.get_proposer_index(stub_state) privkey = privkeys[proposer_index] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 7a60301577..7f539e3b95 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -30,7 +30,7 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) + attester_slashing.attestation_1.custody_bit_1_indices ) - proposer_index = spec.get_beacon_proposer_index(state) + proposer_index = spec.get_proposer_index(state) pre_proposer_balance = get_balance(state, proposer_index) pre_slashings = {slashed_index: get_balance(state, slashed_index) for slashed_index in slashed_indices} pre_withdrawalable_epochs = { diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py index a2306ef4d9..c348e57e14 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py @@ -75,7 +75,7 @@ def test_proposer_slashed(spec, state): # use stub state to get proposer index of next slot stub_state = deepcopy(state) next_slot(spec, stub_state) - proposer_index = spec.get_beacon_proposer_index(stub_state) + proposer_index = spec.get_proposer_index(stub_state) # set proposer to slashed state.validators[proposer_index].slashed = True diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 6903f06666..5ea8cbacf8 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -21,7 +21,7 @@ def run_transfer_processing(spec, state, transfer, valid=True): yield 'post', None return - proposer_index = spec.get_beacon_proposer_index(state) + proposer_index = spec.get_proposer_index(state) pre_transfer_sender_balance = state.balances[transfer.sender] pre_transfer_recipient_balance = state.balances[transfer.recipient] pre_transfer_proposer_balance = state.balances[proposer_index] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index f59da1fc5c..a308fe3d2d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -33,7 +33,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support total_balance = spec.get_total_active_balance(state) remaining_balance = total_balance * 2 // 3 - start_slot = spec.epoch_start_slot(epoch) + start_slot = spec.epoch_first_slot(epoch) for slot in range(start_slot, start_slot + spec.SLOTS_PER_EPOCH): for shard in get_shards_for_slot(spec, state, slot): # Check if we already have had sufficient balance. (and undone if we don't want it). @@ -80,7 +80,7 @@ def get_checkpoints(spec, epoch): def put_checkpoints_in_block_roots(spec, state, checkpoints): for c in checkpoints: - state.block_roots[spec.epoch_start_slot(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root + state.block_roots[spec.epoch_first_slot(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root def finalize_on_234(spec, state, epoch, sufficient_support): diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index b2eb192442..086d141a57 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -214,7 +214,7 @@ def test_attester_slashing(spec, state): # lost whistleblower reward assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) - proposer_index = spec.get_beacon_proposer_index(state) + proposer_index = spec.get_proposer_index(state) # gained whistleblower reward assert ( get_balance(state, proposer_index) > diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 5ff0ed1efb..d572b05fb5 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -43,7 +43,7 @@ def next_epoch_with_attestations(spec, block = build_empty_block_for_next_slot(spec, post_state) if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= spec.epoch_start_slot(spec.get_current_epoch(post_state)): + if slot_to_attest >= spec.epoch_first_slot(spec.get_current_epoch(post_state)): cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) block.body.attestations.append(cur_attestation) From 58c864ddf4a88b0ba85e2f709917cca3ba5b834e Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 11:01:54 +0100 Subject: [PATCH 330/405] Push draft --- specs/core/0_beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1d4a486a1c..971ea54802 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -94,7 +94,7 @@ - [`get_domain`](#get_domain) - [`get_indexed_attestation`](#get_indexed_attestation) - [`get_attesting_indices`](#get_attesting_indices) - - [State mutators](#state-mutators) + - [Beacon state mutators](#beacon-state-mutators) - [`increase_balance`](#increase_balance) - [`decrease_balance`](#decrease_balance) - [`initiate_validator_exit`](#initiate_validator_exit) @@ -1055,7 +1055,7 @@ def get_attesting_indices(state: BeaconState, return set(index for i, index in enumerate(committee) if bits[i]) ``` -### State mutators +### Beacon state mutators #### `increase_balance` @@ -1500,7 +1500,7 @@ def process_slashings(state: BeaconState) -> None: ```python def process_final_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) - next_epoch = Shard(current_epoch + 1) + next_epoch = Epoch(current_epoch + 1) # Reset eth1 data votes if (state.slot + 1) % SLOTS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] @@ -1691,7 +1691,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: assert is_valid_merkle_branch( leaf=hash_tree_root(deposit.data), branch=deposit.proof, - depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # add 1 for the SSZ length mix-in + depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the `List` length mix-in index=state.eth1_deposit_index, root=state.eth1_data.deposit_root, ) From abdcbc4972682644eb3f607d07c37be386fd1a66 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 14:32:38 +0200 Subject: [PATCH 331/405] update epoch processing tests generation and format --- specs/test_formats/epoch_processing/README.md | 14 +++++++++----- test_generators/epoch_processing/main.py | 13 ++++++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/specs/test_formats/epoch_processing/README.md b/specs/test_formats/epoch_processing/README.md index 6384a0eda9..6998184f6d 100644 --- a/specs/test_formats/epoch_processing/README.md +++ b/specs/test_formats/epoch_processing/README.md @@ -17,13 +17,17 @@ post: BeaconState -- state after applying the epoch sub-transition. ## Condition A handler of the `epoch_processing` test-runner should process these cases, - calling the corresponding processing implementation. + calling the corresponding processing implementation (same name, prefixed with `process_`). +This excludes the other parts of the epoch-transition. +The provided pre-state is already transitioned to just before the specific sub-transition of focus of the handler. Sub-transitions: -| *`sub-transition-name`* | *`processing call`* | -|-------------------------|-----------------------------------| -| `crosslinks` | `process_crosslinks(state)` | -| `registry_updates` | `process_registry_updates(state)` | +- `justification_and_finalization` +- `crosslinks` +- *`justification_and_finalization` - planned testing extension* +- `registry_updates` +- `slashings` +- `final_updates` The resulting state should match the expected `post` state. diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py index 846f463a18..6a578c598e 100644 --- a/test_generators/epoch_processing/main.py +++ b/test_generators/epoch_processing/main.py @@ -4,7 +4,10 @@ from eth2spec.phase1 import spec as spec_phase1 from eth2spec.test.phase_0.epoch_processing import ( test_process_crosslinks, - test_process_registry_updates + test_process_final_updates, + test_process_justification_and_finalization, + test_process_registry_updates, + test_process_slashings ) from gen_base import gen_runner, gen_suite, gen_typing from gen_from_tests.gen import generate_from_tests @@ -35,8 +38,16 @@ def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: gen_runner.run_generator("epoch_processing", [ create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), + create_suite('final_updates', 'minimal', lambda: generate_from_tests(test_process_final_updates, 'phase0')), + create_suite('final_updates', 'mainnet', lambda: generate_from_tests(test_process_final_updates, 'phase0')), + create_suite('justification_and_finalization', 'minimal', + lambda: generate_from_tests(test_process_justification_and_finalization, 'phase0')), + create_suite('justification_and_finalization', 'mainnet', + lambda: generate_from_tests(test_process_justification_and_finalization, 'phase0')), create_suite('registry_updates', 'minimal', lambda: generate_from_tests(test_process_registry_updates, 'phase0')), create_suite('registry_updates', 'mainnet', lambda: generate_from_tests(test_process_registry_updates, 'phase0')), + create_suite('slashings', 'minimal', lambda: generate_from_tests(test_process_slashings, 'phase0')), + create_suite('slashings', 'mainnet', lambda: generate_from_tests(test_process_slashings, 'phase0')), ]) From 992a51b5876f27d3bae23067ebd5b7c38bdb559d Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 14:35:07 +0200 Subject: [PATCH 332/405] add note to block operatiosn test format --- specs/test_formats/operations/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md index cc7e43f3f2..37c5df498b 100644 --- a/specs/test_formats/operations/README.md +++ b/specs/test_formats/operations/README.md @@ -16,13 +16,14 @@ post: BeaconState -- state after applying the operation. No A handler of the `operations` test-runner should process these cases, calling the corresponding processing implementation. +This excludes the other parts of the block-transition. Operations: | *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* | |-------------------------|----------------------|----------------------|--------------------------------------------------------| -| `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` | -| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | +| `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` | +| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | | `block_header` | `Block` | `block` | `process_block_header(state, block)` | | `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | | `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | From da090b67f69a570d3db9a435301e22f07734b508 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 14:58:53 +0200 Subject: [PATCH 333/405] document genesis test vector format, fix missing label and assertion in tests --- specs/test_formats/genesis/README.md | 8 +++++++ specs/test_formats/genesis/initialization.md | 22 +++++++++++++++++++ specs/test_formats/genesis/validity.md | 19 ++++++++++++++++ .../genesis/test_is_valid_genesis_state.py | 6 ++--- 4 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 specs/test_formats/genesis/README.md create mode 100644 specs/test_formats/genesis/initialization.md create mode 100644 specs/test_formats/genesis/validity.md diff --git a/specs/test_formats/genesis/README.md b/specs/test_formats/genesis/README.md new file mode 100644 index 0000000000..25761e2f6a --- /dev/null +++ b/specs/test_formats/genesis/README.md @@ -0,0 +1,8 @@ +# Genesis tests + +The aim of the genesis tests is to provide a baseline to test genesis-state initialization and test + if the proposed genesis-validity conditions are working. + +There are two handlers, documented individually: +- [`validity`](./validity.md): Tests if a genesis state is valid, i.e. if it counts as trigger to launch. +- [`initialization`](./initialization.md): Tests the initialization of a genesis state based on Eth1 data. diff --git a/specs/test_formats/genesis/initialization.md b/specs/test_formats/genesis/initialization.md new file mode 100644 index 0000000000..437dd91a33 --- /dev/null +++ b/specs/test_formats/genesis/initialization.md @@ -0,0 +1,22 @@ +# Genesis creation testing + +Tests the initialization of a genesis state based on Eth1 data. + +## Test case format + +```yaml +description: string -- description of test case, purely for debugging purposes +bls_setting: int -- see general test-format spec. +eth1_block_hash: Bytes32 -- the root of the Eth-1 block, hex encoded, with prefix 0x +eth1_timestamp: int -- the timestamp of the block, in seconds. +deposits: [Deposit] -- list of deposits to build the genesis state with +state: BeaconState -- the expected genesis state. +``` + +To process this test, build a genesis state with the provided `eth1_block_hash`, `eth1_timestamp` and `deposits`: +`initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)`, + as described in the Beacon Chain specification. + +## Condition + +The resulting state should match the expected `state`. diff --git a/specs/test_formats/genesis/validity.md b/specs/test_formats/genesis/validity.md new file mode 100644 index 0000000000..792923e3a0 --- /dev/null +++ b/specs/test_formats/genesis/validity.md @@ -0,0 +1,19 @@ +# Genesis validity testing + +Tests if a genesis state is valid, i.e. if it counts as trigger to launch. + +## Test case format + +```yaml +description: string -- description of test case, purely for debugging purposes +bls_setting: int -- see general test-format spec. +genesis: BeaconState -- state to validate. +is_valid: bool -- true if the genesis state is deemed valid as to launch with, false otherwise. +``` + +To process the data, call `is_valid_genesis_state(genesis)`. + + +## Condition + +The result of calling `is_valid_genesis_state(genesis)` should match the expected `is_valid` boolean. diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py index 4ad5092002..2dc9eb7fd2 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py @@ -16,13 +16,13 @@ def create_valid_beacon_state(spec): def run_is_valid_genesis_state(spec, state, valid=True): """ Run ``is_valid_genesis_state``, yielding: - - state ('state') + - genesis ('state') - is_valid ('is_valid') - If ``valid == False``, run expecting ``AssertionError`` """ - yield state + yield 'genesis', state is_valid = spec.is_valid_genesis_state(state) yield 'is_valid', is_valid + assert is_valid == valid @with_phases(['phase0']) From 64b02ebecc912ad8b54cea30c02cbd60e0e507e8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 15:01:30 +0200 Subject: [PATCH 334/405] fix genesis tests --- .../test/genesis/test_is_valid_genesis_state.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py index 2dc9eb7fd2..bb95bb2b0f 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py @@ -39,7 +39,7 @@ def test_is_valid_genesis_state_false_invalid_timestamp(spec): state = create_valid_beacon_state(spec) state.genesis_time = spec.MIN_GENESIS_TIME - 1 - yield from run_is_valid_genesis_state(spec, state, valid=True) + yield from run_is_valid_genesis_state(spec, state, valid=False) @with_phases(['phase0']) @@ -51,13 +51,14 @@ def test_is_valid_genesis_state_true_more_balance(spec): yield from run_is_valid_genesis_state(spec, state, valid=True) -@with_phases(['phase0']) -@spectest_with_bls_switch -def test_is_valid_genesis_state_false_not_enough_balance(spec): - state = create_valid_beacon_state(spec) - state.validators[0].effective_balance = spec.MAX_EFFECTIVE_BALANCE - 1 - - yield from run_is_valid_genesis_state(spec, state, valid=False) +# TODO: not part of the genesis function yet. Erroneously merged. +# @with_phases(['phase0']) +# @spectest_with_bls_switch +# def test_is_valid_genesis_state_false_not_enough_balance(spec): +# state = create_valid_beacon_state(spec) +# state.validators[0].effective_balance = spec.MAX_EFFECTIVE_BALANCE - 1 +# +# yield from run_is_valid_genesis_state(spec, state, valid=False) @with_phases(['phase0']) From 83d91f81db8530da9faffb23b0910c4ebfd20957 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 15:03:50 +0200 Subject: [PATCH 335/405] update sanity slots description to match new transition naming --- specs/test_formats/sanity/slots.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/test_formats/sanity/slots.md b/specs/test_formats/sanity/slots.md index eb0f336b4e..04fecd1867 100644 --- a/specs/test_formats/sanity/slots.md +++ b/specs/test_formats/sanity/slots.md @@ -12,11 +12,10 @@ slots: N -- amount of slots to process, N being a positive number. post: BeaconState -- state after applying all the transitions. ``` -The transition with pure time, no blocks, is known as `state_transition_to(state, slot)` in the spec. +The transition with pure time, no blocks, is known as `process_slots(state, slot)` in the spec. This runs state-caching (pure slot transition) and epoch processing (every E slots). -To process the data, call `state_transition_to(pre, pre.slot + N)`. And see if `pre` mutated into the equivalent of `post`. - +To process the data, call `process_slots(pre, pre.slot + N)`. ## Condition From 94404a58562adf9693007aa7d7860b2d49093f50 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 14:11:46 +0100 Subject: [PATCH 336/405] A few more cleanups --- specs/core/0_beacon-chain.md | 88 +++++++++---------- .../test_process_voluntary_exit.py | 2 +- .../test_process_registry_updates.py | 2 +- 3 files changed, 45 insertions(+), 47 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 971ea54802..99de628876 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -81,9 +81,9 @@ - [`get_block_root_at_slot`](#get_block_root_at_slot) - [`get_randao_mix`](#get_randao_mix) - [`get_active_validator_indices`](#get_active_validator_indices) - - [`get_churn_limit`](#get_churn_limit) - - [`get_committee_count`](#get_committee_count) + - [`get_validator_churn_limit`](#get_validator_churn_limit) - [`get_seed`](#get_seed) + - [`get_committee_count`](#get_committee_count) - [`get_crosslink_committee`](#get_crosslink_committee) - [`get_start_shard`](#get_start_shard) - [`get_shard_delta`](#get_shard_delta) @@ -713,9 +713,11 @@ def shuffle_index(index: ValidatorIndex, index_count: int, seed: Hash) -> Valida ```python def compute_committee(indices: Sequence[ValidatorIndex], - seed: Hash, index: int, count: int) -> Sequence[ValidatorIndex]: + seed: Hash, + index: int, + count: int) -> Sequence[ValidatorIndex]: """ - Return the committee corresponding to ``indices``, ``seed``, ``index`` and committee ``count``. + Return the committee corresponding to ``indices``, ``seed``, ``index``, and committee ``count``. """ start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count @@ -780,7 +782,7 @@ def epoch_first_slot(epoch: Epoch) -> Slot: ```python def delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: """ - Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. + Return the epoch where validator activations and exits initiated in ``epoch`` take effect. """ return Epoch(epoch + 1 + ACTIVATION_EXIT_DELAY) ``` @@ -812,7 +814,7 @@ def get_current_epoch(state: BeaconState) -> Epoch: ```python def get_previous_epoch(state: BeaconState) -> Epoch: """` - Return the previous epoch (current epoch if at ``GENESIS_EPOCH``). + Return the previous epoch (unless the current epoch is ``GENESIS_EPOCH``). """ current_epoch = get_current_epoch(state) return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else Epoch(current_epoch - 1) @@ -859,10 +861,10 @@ def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[V return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] ``` -#### `get_churn_limit` +#### `get_validator_churn_limit` ```python -def get_churn_limit(state: BeaconState) -> int: +def get_validator_churn_limit(state: BeaconState) -> int: """ Return the validator churn limit for the current epoch. """ @@ -870,6 +872,17 @@ def get_churn_limit(state: BeaconState) -> int: return max(MIN_PER_EPOCH_CHURN_LIMIT, len(active_validator_indices) // CHURN_LIMIT_QUOTIENT) ``` +#### `get_seed` + +```python +def get_seed(state: BeaconState, epoch: Epoch) -> Hash: + """ + Return the seed at ``epoch``. + """ + randao_mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) # Avoid underflow + active_indices_root = hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, epoch))) + return hash(randao_mix + active_indices_root + int_to_bytes(epoch, length=32)) +``` #### `get_committee_count` @@ -878,30 +891,15 @@ def get_committee_count(state: BeaconState, epoch: Epoch) -> int: """ Return the number of committees at ``epoch``. """ - active_validator_indices = get_active_validator_indices(state, epoch) return max( 1, min( SHARD_COUNT // SLOTS_PER_EPOCH, - len(active_validator_indices) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, + len(get_active_validator_indices(state, epoch)) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, ) ) * SLOTS_PER_EPOCH ``` -#### `get_seed` - -```python -def get_seed(state: BeaconState, epoch: Epoch) -> Hash: - """ - Return the seed at ``epoch``. - """ - return hash( - get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + # Avoid underflow - hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, epoch))) + - int_to_bytes(epoch, length=32) - ) -``` - #### `get_crosslink_committee` ```python @@ -1010,6 +1008,16 @@ def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: return Gwei(max(sum([state.validators[index].effective_balance for index in indices]), 1)) ``` +#### `get_total_active_balance` + +```python +def get_total_active_balance(state: BeaconState) -> Gwei: + """ + Return the combined effective balance of the active validators. + """ + return get_total_balance(state, set(get_active_validator_indices(state, get_current_epoch(state)))) +``` + #### `get_domain` ```python @@ -1093,7 +1101,7 @@ def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: exit_epochs = [v.exit_epoch for v in state.validators if v.exit_epoch != FAR_FUTURE_EPOCH] exit_queue_epoch = max(exit_epochs + [delayed_activation_exit_epoch(get_current_epoch(state))]) exit_queue_churn = len([v for v in state.validators if v.exit_epoch == exit_queue_epoch]) - if exit_queue_churn >= get_churn_limit(state): + if exit_queue_churn >= get_validator_churn_limit(state): exit_queue_epoch += Epoch(1) # Set validator exit epoch and withdrawable epoch @@ -1219,11 +1227,9 @@ def process_slot(state: BeaconState) -> None: # Cache state root previous_state_root = hash_tree_root(state) state.state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root - # Cache latest block header state root if state.latest_block_header.state_root == ZERO_HASH: state.latest_block_header.state_root = previous_state_root - # Cache block root previous_block_root = signing_root(state.latest_block_header) state.block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root @@ -1248,11 +1254,6 @@ def process_epoch(state: BeaconState) -> None: #### Helper functions -```python -def get_total_active_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, set(get_active_validator_indices(state, get_current_epoch(state)))) -``` - ```python def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: assert epoch in (get_previous_epoch(state), get_current_epoch(state)) @@ -1278,7 +1279,7 @@ def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Sequence ```python def get_unslashed_attesting_indices(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: - output = set() # Type: Set[ValidatorIndex] + output = set() # type: Set[ValidatorIndex] for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) return set(filter(lambda index: not state.validators[index].slashed, list(output))) @@ -1477,7 +1478,7 @@ def process_registry_updates(state: BeaconState) -> None: validator.activation_epoch >= delayed_activation_exit_epoch(state.finalized_checkpoint.epoch) ], key=lambda index: state.validators[index].activation_eligibility_epoch) # Dequeued validators for activation up to churn limit (without resetting activation epoch) - for index in activation_queue[:get_churn_limit(state)]: + for index in activation_queue[:get_validator_churn_limit(state)]: validator = state.validators[index] if validator.activation_epoch == FAR_FUTURE_EPOCH: validator.activation_epoch = delayed_activation_exit_epoch(get_current_epoch(state)) @@ -1703,13 +1704,10 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: amount = deposit.data.amount validator_pubkeys = [v.pubkey for v in state.validators] if pubkey not in validator_pubkeys: - # Verify the deposit signature (proof of possession). - # Invalid signatures are allowed by the deposit contract, - # and hence included on-chain, but must not be processed. + # Verify the deposit signature (proof of possession) for new validators. + # Note: The deposit contract does not check signatures. # Note: Deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` - if not bls_verify( - pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT) - ): + if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT)): return # Add validator and balance entries @@ -1757,14 +1755,14 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: assert state.balances[transfer.sender] >= max(transfer.amount + transfer.fee, transfer.amount, transfer.fee) # A transfer is valid in only one slot assert state.slot == transfer.slot - # Sender must satisfy at least one of the following conditions in the parenthesis: + # Sender must satisfy at least one of the following: assert ( - # * Has not been activated + # 1) Never have been eligible for activation state.validators[transfer.sender].activation_eligibility_epoch == FAR_FUTURE_EPOCH or - # * Is withdrawable + # 2) Be withdrawable get_current_epoch(state) >= state.validators[transfer.sender].withdrawable_epoch or - # * Balance after transfer is more than the effective balance threshold - transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE <= state.balances[transfer.sender] + # 3) Have a balance of at least MAX_EFFECTIVE_BALANCE after the transfer + state.balances[transfer.sender] >= transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE ) # Verify that the pubkey is valid assert ( diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index 33cacc4e20..6c9298eccc 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -71,7 +71,7 @@ def test_success_exit_queue(spec, state): current_epoch = spec.get_current_epoch(state) # exit `MAX_EXITS_PER_EPOCH` - initial_indices = spec.get_active_validator_indices(state, current_epoch)[:spec.get_churn_limit(state)] + initial_indices = spec.get_active_validator_indices(state, current_epoch)[:spec.get_validator_churn_limit(state)] # Prepare a bunch of exits, based on the current state exit_queue = [] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index 19500d4ab7..ab6a74a704 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -45,7 +45,7 @@ def test_activation_queue_sorting(spec, state): state.validators[mock_activations - 1].activation_eligibility_epoch = epoch # make sure we are hitting the churn - churn_limit = spec.get_churn_limit(state) + churn_limit = spec.get_validator_churn_limit(state) assert mock_activations > churn_limit yield from run_process_registry_updates(spec, state) From c1317640c4111cfcb4e8ae6e294e2d7920fb2278 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 15:26:54 +0200 Subject: [PATCH 337/405] genesis tests generator --- test_generators/genesis/README.md | 8 +++++ test_generators/genesis/main.py | 33 +++++++++++++++++++ test_generators/genesis/requirements.txt | 4 +++ ...te_from_eth1.py => test_initialization.py} | 0 ...alid_genesis_state.py => test_validity.py} | 0 5 files changed, 45 insertions(+) create mode 100644 test_generators/genesis/README.md create mode 100644 test_generators/genesis/main.py create mode 100644 test_generators/genesis/requirements.txt rename test_libs/pyspec/eth2spec/test/genesis/{test_initialize_beacon_state_from_eth1.py => test_initialization.py} (100%) rename test_libs/pyspec/eth2spec/test/genesis/{test_is_valid_genesis_state.py => test_validity.py} (100%) diff --git a/test_generators/genesis/README.md b/test_generators/genesis/README.md new file mode 100644 index 0000000000..8a2b01c626 --- /dev/null +++ b/test_generators/genesis/README.md @@ -0,0 +1,8 @@ +# Genesis test generator + +Genesis tests cover the initialization and validity-based launch trigger for the Beacon Chain genesis state. + +Information on the format of the tests can be found in the [genesis test formats documentation](../../specs/test_formats/genesis/README.md). + + + diff --git a/test_generators/genesis/main.py b/test_generators/genesis/main.py new file mode 100644 index 0000000000..82899b967f --- /dev/null +++ b/test_generators/genesis/main.py @@ -0,0 +1,33 @@ +from typing import Callable, Iterable + +from eth2spec.test.genesis import test_initialization, test_validity + +from gen_base import gen_runner, gen_suite, gen_typing +from gen_from_tests.gen import generate_from_tests +from preset_loader import loader +from eth2spec.phase0 import spec as spec + + +def create_suite(handler_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]]) \ + -> Callable[[str], gen_typing.TestSuiteOutput]: + def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: + presets = loader.load_presets(configs_path, config_name) + spec.apply_constants_preset(presets) + + return ("genesis_%s_%s" % (handler_name, config_name), handler_name, gen_suite.render_suite( + title="genesis testing", + summary="Genesis test suite, %s type, generated from pytests" % handler_name, + forks_timeline="testing", + forks=["phase0"], + config=config_name, + runner="genesis", + handler=handler_name, + test_cases=get_cases())) + return suite_definition + + +if __name__ == "__main__": + gen_runner.run_generator("genesis", [ + create_suite('initialization', 'minimal', lambda: generate_from_tests(test_initialization, 'phase0')), + create_suite('validity', 'minimal', lambda: generate_from_tests(test_validity, 'phase0')), + ]) diff --git a/test_generators/genesis/requirements.txt b/test_generators/genesis/requirements.txt new file mode 100644 index 0000000000..595cee69cd --- /dev/null +++ b/test_generators/genesis/requirements.txt @@ -0,0 +1,4 @@ +eth-utils==1.6.0 +../../test_libs/gen_helpers +../../test_libs/config_helpers +../../test_libs/pyspec \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py b/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/genesis/test_initialize_beacon_state_from_eth1.py rename to test_libs/pyspec/eth2spec/test/genesis/test_initialization.py diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py b/test_libs/pyspec/eth2spec/test/genesis/test_validity.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/genesis/test_is_valid_genesis_state.py rename to test_libs/pyspec/eth2spec/test/genesis/test_validity.py From b38802ced06aab88e919f66adecafa13e3c58112 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 15:27:31 +0200 Subject: [PATCH 338/405] accept yielded lists, encode per item --- test_libs/pyspec/eth2spec/test/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/utils.py b/test_libs/pyspec/eth2spec/test/utils.py index 817c952b7e..bc91053e8f 100644 --- a/test_libs/pyspec/eth2spec/test/utils.py +++ b/test_libs/pyspec/eth2spec/test/utils.py @@ -1,6 +1,6 @@ from typing import Dict, Any, Callable, Iterable from eth2spec.debug.encode import encode -from eth2spec.utils.ssz.ssz_typing import Container +from eth2spec.utils.ssz.ssz_typing import SSZValue def spectest(description: str = None): @@ -31,8 +31,10 @@ def entry(*args, **kw): else: # Otherwise, try to infer the type, but keep it as-is if it's not a SSZ container. (key, value) = data - if isinstance(value, Container): + if isinstance(value, SSZValue): out[key] = encode(value, value.__class__) + elif isinstance(value, list) and all([isinstance(el, SSZValue) for el in value]): + out[key] = [encode(el, el.__class__) for el in value] else: # not a ssz value. # It could be vector or bytes still, but it is a rare case, From e873bb6e591b9b98a599fa304c502859ada2063a Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 14:32:50 +0100 Subject: [PATCH 339/405] ready for review --- specs/core/0_beacon-chain.md | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 99de628876..96d827bbf1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -698,10 +698,7 @@ def shuffle_index(index: ValidatorIndex, index_count: int, seed: Hash) -> Valida pivot = bytes_to_int(hash(seed + int_to_bytes(current_round, length=1))[0:8]) % index_count flip = ValidatorIndex((pivot + index_count - index) % index_count) position = max(index, flip) - source = hash( - seed + int_to_bytes(current_round, length=1) + - int_to_bytes(position // 256, length=4) - ) + source = hash(seed + int_to_bytes(current_round, length=1) + int_to_bytes(position // 256, length=4)) byte = source[(position % 256) // 8] bit = (byte >> (position % 8)) % 2 index = flip if bit else index @@ -879,9 +876,10 @@ def get_seed(state: BeaconState, epoch: Epoch) -> Hash: """ Return the seed at ``epoch``. """ - randao_mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) # Avoid underflow - active_indices_root = hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, epoch))) - return hash(randao_mix + active_indices_root + int_to_bytes(epoch, length=32)) + mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) # Avoid underflow + active_indices = get_active_validator_indices(state, epoch) + active_indices_root = hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](active_indices)) + return hash(mix + active_indices_root + int_to_bytes(epoch, length=32)) ``` #### `get_committee_count` @@ -891,13 +889,10 @@ def get_committee_count(state: BeaconState, epoch: Epoch) -> int: """ Return the number of committees at ``epoch``. """ - return max( - 1, - min( - SHARD_COUNT // SLOTS_PER_EPOCH, - len(get_active_validator_indices(state, epoch)) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, - ) - ) * SLOTS_PER_EPOCH + return max(1, min( + SHARD_COUNT // SLOTS_PER_EPOCH, + len(get_active_validator_indices(state, epoch)) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, + )) * SLOTS_PER_EPOCH ``` #### `get_crosslink_committee` @@ -1157,9 +1152,8 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Hash, # Process deposits leaves = list(map(lambda deposit: deposit.data, deposits)) for index, deposit in enumerate(deposits): - state.eth1_data.deposit_root = hash_tree_root( - List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1]) - ) + deposit_data_list = List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1]) + state.eth1_data.deposit_root = hash_tree_root(deposit_data_list) process_deposit(state, deposit) # Process activations @@ -1522,10 +1516,7 @@ def process_final_updates(state: BeaconState) -> None: state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) # Set historical root accumulator if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: - historical_batch = HistoricalBatch( - block_roots=state.block_roots, - state_roots=state.state_roots, - ) + historical_batch = HistoricalBatch(state.block_roots, state.state_roots) state.historical_roots.append(hash_tree_root(historical_batch)) # Rotate current/previous epoch attestations state.previous_epoch_attestations = state.current_epoch_attestations @@ -1554,7 +1545,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: state.latest_block_header = BeaconBlockHeader( slot=block.slot, parent_root=block.parent_root, - state_root=ZERO_HASH, # Overwritten in next `process_slot` call + state_root=ZERO_HASH, # Overwritten in the next `process_slot` call body_root=hash_tree_root(block.body), ) # Verify proposer is not slashed From bf618f8d282dd76e226fdd261dc0745ca7f8ad90 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 15:36:54 +0200 Subject: [PATCH 340/405] fix encoder to also encode bytes nicely --- test_libs/pyspec/eth2spec/debug/encode.py | 4 ++-- test_libs/pyspec/eth2spec/test/utils.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 670f580b2b..9080bb573e 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,10 +1,10 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZValue, uint, Container, boolean + uint, Container, boolean ) -def encode(value: SSZValue, include_hash_tree_roots=False): +def encode(value, include_hash_tree_roots=False): if isinstance(value, uint): # Larger uints are boxed and the class declares their byte length if value.type().byte_len > 8: diff --git a/test_libs/pyspec/eth2spec/test/utils.py b/test_libs/pyspec/eth2spec/test/utils.py index bc91053e8f..253691764f 100644 --- a/test_libs/pyspec/eth2spec/test/utils.py +++ b/test_libs/pyspec/eth2spec/test/utils.py @@ -29,12 +29,12 @@ def entry(*args, **kw): (key, value, typ) = data out[key] = encode(value, typ) else: - # Otherwise, try to infer the type, but keep it as-is if it's not a SSZ container. + # Otherwise, try to infer the type, but keep it as-is if it's not a SSZ type or bytes. (key, value) = data - if isinstance(value, SSZValue): - out[key] = encode(value, value.__class__) - elif isinstance(value, list) and all([isinstance(el, SSZValue) for el in value]): - out[key] = [encode(el, el.__class__) for el in value] + if isinstance(value, (SSZValue, bytes)): + out[key] = encode(value) + elif isinstance(value, list) and all([isinstance(el, (SSZValue, bytes)) for el in value]): + out[key] = [encode(el) for el in value] else: # not a ssz value. # It could be vector or bytes still, but it is a rare case, From 423d6dcb29af2fc2025363445e242dba0aff0362 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 14:45:36 +0100 Subject: [PATCH 341/405] Fix bug --- specs/core/0_beacon-chain.md | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 96d827bbf1..a5aa113672 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -9,7 +9,6 @@ - [Table of contents](#table-of-contents) - [Introduction](#introduction) - [Notation](#notation) - - [Terminology](#terminology) - [Custom types](#custom-types) - [Constants](#constants) - [Configuration](#configuration) @@ -129,29 +128,12 @@ This document represents the specification for Phase 0 of Ethereum 2.0 -- The Beacon Chain. -At the core of Ethereum 2.0 is a system chain called the "beacon chain". The beacon chain stores and manages the registry of [validators](#dfn-validator). In the initial deployment phases of Ethereum 2.0, the only mechanism to become a [validator](#dfn-validator) is to make a one-way ETH transaction to a deposit contract on Ethereum 1.0. Activation as a [validator](#dfn-validator) happens when Ethereum 1.0 deposit receipts are processed by the beacon chain, the activation balance is reached, and a queuing process is completed. Exit is either voluntary or done forcibly as a penalty for misbehavior. +At the core of Ethereum 2.0 is a system chain called the "beacon chain". The beacon chain stores and manages the registry of validators. In the initial deployment phases of Ethereum 2.0, the only mechanism to become a validator is to make a one-way ETH transaction to a deposit contract on Ethereum 1.0. Activation as a validator happens when Ethereum 1.0 deposit receipts are processed by the beacon chain, the activation balance is reached, and a queuing process is completed. Exit is either voluntary or done forcibly as a penalty for misbehavior. The primary source of load on the beacon chain is "attestations". Attestations are simultaneously availability votes for a shard block and proof-of-stake votes for a beacon block. A sufficient number of attestations for the same shard block create a "crosslink", confirming the shard segment up to that shard block into the beacon chain. Crosslinks also serve as infrastructure for asynchronous cross-shard communication. ## Notation -Code snippets appearing in `this style` are to be interpreted as Python code. - -## Terminology - -* **Validator**—a registered participant in the beacon chain. You can become one by sending ether into the Ethereum 1.0 deposit contract. -* **Active validator**—an active participant in the Ethereum 2.0 consensus invited to, among other things, propose and attest to blocks and vote for crosslinks. -* **Committee**—a (pseudo-) randomly sampled subset of [active validators](#dfn-active-validator). When a committee is referred to collectively, as in "this committee attests to X", this is assumed to mean "some subset of that committee that contains enough [validators](#dfn-validator) that the protocol recognizes it as representing the committee". -* **Proposer**—the [validator](#dfn-validator) that creates a beacon chain block. -* **Attester**—a [validator](#dfn-validator) that is part of a committee that needs to sign off on a beacon chain block while simultaneously creating a link (crosslink) to a recent shard block on a particular shard chain. -* **Beacon chain**—the central proof-of-stake chain that is the base of the sharding system. -* **Shard chain**—one of the chains on which user transactions take place and account data is stored. -* **Block root**—a 32-byte Merkle root of a beacon chain block or shard chain block. Previously called "block hash". -* **Crosslink**—a set of signatures from a committee attesting to a block in a shard chain that can be included into the beacon chain. Crosslinks are the main means by which the beacon chain "learns about" the updated state of shard chains. -* **Slot**—a period during which one proposer has the ability to create a beacon chain block and some attesters have the ability to make attestations. -* **Epoch**—an aligned span of slots during which all [validators](#dfn-validator) get exactly one chance to make an attestation. -* **Finalized**, **justified**—see the [Casper FFG paper](https://arxiv.org/abs/1710.09437). -* **Withdrawal period**—the number of slots between a [validator](#dfn-validator) exit and the [validator](#dfn-validator) balance being withdrawable. -* **Genesis time**—the Unix time of the genesis beacon chain block at slot 0. +Code snippets appearing in `this style` are to be interpreted as Python 3 code. ## Custom types @@ -255,7 +237,7 @@ The following values are (non-configurable) constants used throughout the specif | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -* The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. +* The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block @@ -1516,7 +1498,7 @@ def process_final_updates(state: BeaconState) -> None: state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) # Set historical root accumulator if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: - historical_batch = HistoricalBatch(state.block_roots, state.state_roots) + historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) state.historical_roots.append(hash_tree_root(historical_batch)) # Rotate current/previous epoch attestations state.previous_epoch_attestations = state.current_epoch_attestations From 91f55f55b532eaa422459702153876b961844dfb Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 16:07:54 +0200 Subject: [PATCH 342/405] make BLS test format and output consistent with spec --- specs/test_formats/bls/msg_hash_g2_compressed.md | 2 +- specs/test_formats/bls/msg_hash_g2_uncompressed.md | 4 ++-- specs/test_formats/bls/sign_msg.md | 2 +- test_generators/bls/main.py | 9 +++------ 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/specs/test_formats/bls/msg_hash_g2_compressed.md b/specs/test_formats/bls/msg_hash_g2_compressed.md index 2feeb92ba5..44ea8ded77 100644 --- a/specs/test_formats/bls/msg_hash_g2_compressed.md +++ b/specs/test_formats/bls/msg_hash_g2_compressed.md @@ -7,7 +7,7 @@ A BLS compressed-hash to G2. ```yaml input: message: bytes32, - domain: bytes -- any number + domain: uint64 -- the BLS domain output: List[bytes48] -- length of two ``` diff --git a/specs/test_formats/bls/msg_hash_g2_uncompressed.md b/specs/test_formats/bls/msg_hash_g2_uncompressed.md index 792fe1f038..847b5f61d4 100644 --- a/specs/test_formats/bls/msg_hash_g2_uncompressed.md +++ b/specs/test_formats/bls/msg_hash_g2_uncompressed.md @@ -6,8 +6,8 @@ A BLS uncompressed-hash to G2. ```yaml input: - message: bytes32, - domain: bytes -- any number + message: bytes32 + domain: uint64 -- the BLS domain output: List[List[bytes48]] -- 3 lists, each a length of two ``` diff --git a/specs/test_formats/bls/sign_msg.md b/specs/test_formats/bls/sign_msg.md index 9916f2cc2c..b17e6246de 100644 --- a/specs/test_formats/bls/sign_msg.md +++ b/specs/test_formats/bls/sign_msg.md @@ -8,7 +8,7 @@ Message signing with BLS should produce a signature. input: privkey: bytes32 -- the private key used for signing message: bytes32 -- input message to sign (a hash) - domain: bytes -- BLS domain + domain: uint64 -- the BLS domain output: bytes96 -- expected signature ``` diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py index 8a6a7dafec..a792dda9ad 100644 --- a/test_generators/bls/main.py +++ b/test_generators/bls/main.py @@ -27,9 +27,6 @@ def hex_to_int(x: str) -> int: return int(x, 16) -# Note: even though a domain is only an uint64, -# To avoid issues with YAML parsers that are limited to 53-bit (JS language limit) -# It is serialized as an hex string as well. DOMAINS = [ 0, 1, @@ -92,7 +89,7 @@ def case01_message_hash_G2_uncompressed(): yield { 'input': { 'message': '0x' + msg.hex(), - 'domain': int_to_hex(domain) + 'domain': domain }, 'output': hash_message(msg, domain) } @@ -104,7 +101,7 @@ def case02_message_hash_G2_compressed(): yield { 'input': { 'message': '0x' + msg.hex(), - 'domain': int_to_hex(domain) + 'domain': domain }, 'output': hash_message_compressed(msg, domain) } @@ -129,7 +126,7 @@ def case04_sign_messages(): 'input': { 'privkey': int_to_hex(privkey), 'message': '0x' + message.hex(), - 'domain': int_to_hex(domain) + 'domain': domain }, 'output': '0x' + sig.hex() } From 7b91c289d4166fdb5ccea93173cdace7d6ddde93 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 16:46:40 +0200 Subject: [PATCH 343/405] fix shuffling generator, convert ValidatorIndex output to normal int --- test_generators/shuffling/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index 291aa2c47d..7e7b3f94b5 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -10,7 +10,7 @@ def shuffling_case(seed, count): yield 'seed', '0x' + seed.hex() yield 'count', count - yield 'shuffled', [spec.get_shuffled_index(i, count, seed) for i in range(count)] + yield 'shuffled', [int(spec.get_shuffled_index(i, count, seed)) for i in range(count)] @to_tuple From 0be2b8e6205f6f0312bb2dcddb7eb74ce57c2c1b Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 16:10:22 +0100 Subject: [PATCH 344/405] Cleanup --- scripts/build_spec.py | 4 ++-- specs/core/0_beacon-chain.md | 17 +++++++---------- specs/core/1_custody-game.md | 2 +- specs/core/1_shard-data-chains.md | 6 +++--- .../eth2spec/test/helpers/attestations.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/block.py | 2 +- .../pyspec/eth2spec/test/helpers/genesis.py | 2 +- .../block_processing/test_process_deposit.py | 2 +- .../block_processing/test_process_transfer.py | 2 +- .../pyspec/eth2spec/test/sanity/test_blocks.py | 4 ++-- 10 files changed, 20 insertions(+), 23 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 1f5fe1ee61..fe66a72d57 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,7 +12,7 @@ PHASE0_IMPORTS = '''from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, + Any, Dict, Set, Sequence, Tuple, ) from dataclasses import ( @@ -37,7 +37,7 @@ from eth2spec.utils.hash_function import hash ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Sequence, MutableSequence, Tuple, + Any, Dict, Optional, Set, Sequence, MutableSequence, Tuple, ) from dataclasses import ( diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a5aa113672..cd943187f4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -147,7 +147,7 @@ We define the following Python custom types for type hinting and readability: | `ValidatorIndex` | `uint64` | a validator registry index | | `Gwei` | `uint64` | an amount in Gwei | | `Version` | `Bytes4` | a fork version number | -| `Hash` | `Bytes32` | a hashed result | +| `Hash` | `Bytes32` | a hash | | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | @@ -158,7 +158,6 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | | `FAR_FUTURE_EPOCH` | `Epoch(2**64 - 1)` | -| `ZERO_HASH` | `Hash(b'\x00' * 32)` | | `BASE_REWARDS_PER_EPOCH` | `5` | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | | `SECONDS_PER_DAY` | `86400` | @@ -542,11 +541,10 @@ class BeaconState(Container): #### `integer_squareroot` ```python -def integer_squareroot(n: int) -> int: +def integer_squareroot(n: uint64) -> uint64: """ Return the largest integer ``x`` such that ``x**2 <= n``. """ - assert n >= 0 x = n y = (x + 1) // 2 while y < x: @@ -1204,7 +1202,7 @@ def process_slot(state: BeaconState) -> None: previous_state_root = hash_tree_root(state) state.state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root # Cache latest block header state root - if state.latest_block_header.state_root == ZERO_HASH: + if state.latest_block_header.state_root == Hash(): state.latest_block_header.state_root = previous_state_root # Cache block root previous_block_root = signing_root(state.latest_block_header) @@ -1527,7 +1525,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: state.latest_block_header = BeaconBlockHeader( slot=block.slot, parent_root=block.parent_root, - state_root=ZERO_HASH, # Overwritten in the next `process_slot` call + state_root=Hash(), # Overwritten in the next `process_slot` call body_root=hash_tree_root(block.body), ) # Verify proposer is not slashed @@ -1568,15 +1566,14 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) - all_operations = ( + for operations, function in ( (body.proposer_slashings, process_proposer_slashing), (body.attester_slashings, process_attester_slashing), (body.attestations, process_attestation), (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), - ) # type: Sequence[Tuple[List, Callable]] - for operations, function in all_operations: + ): for operation in operations: function(state, operation) ``` @@ -1651,7 +1648,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.start_epoch == parent_crosslink.end_epoch assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) - assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] + assert data.crosslink.data_root == Hash() # [to be removed in phase 1] # Check signature validate_indexed_attestation(state, get_indexed_attestation(state, attestation)) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 993a45b0fd..bf221ee3ec 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -595,7 +595,7 @@ def process_chunk_challenge_response(state: BeaconState, # Verify chunk index assert response.chunk_index == challenge.chunk_index # Verify bit challenge data is null - assert response.chunk_bits_branch == [] and response.chunk_bits_leaf == ZERO_HASH + assert response.chunk_bits_branch == [] and response.chunk_bits_leaf == Hash() # Verify minimum delay assert get_current_epoch(state) >= challenge.inclusion_epoch + ACTIVATION_EXIT_DELAY # Verify the chunk matches the crosslink data root diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index eb20abe7f5..82843b7a6b 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -309,11 +309,11 @@ def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], assert beacon_block.slot <= candidate.slot # Check state root - assert candidate.state_root == ZERO_HASH # [to be removed in phase 2] + assert candidate.state_root == Hash() # [to be removed in phase 2] # Check parent block if candidate.slot == PHASE_1_FORK_SLOT: - assert candidate.parent_root == ZERO_HASH + assert candidate.parent_root == Hash() else: parent_block = next( (block for block in valid_shard_blocks if signing_root(block) == candidate.parent_root), @@ -395,7 +395,7 @@ def is_valid_beacon_attestation(shard: Shard, # Check previous attestation if candidate.data.previous_crosslink.epoch <= PHASE_1_FORK_EPOCH: - assert candidate.data.previous_crosslink.data_root == ZERO_HASH + assert candidate.data.previous_crosslink.data_root == Hash() else: previous_attestation = next( (attestation for attestation in valid_attestations if diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 053446a2db..c47245bb9c 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -43,7 +43,7 @@ def build_attestation_data(spec, state, slot, shard): shard=shard, start_epoch=parent_crosslink.end_epoch, end_epoch=min(spec.slot_to_epoch(slot), parent_crosslink.end_epoch + spec.MAX_EPOCHS_PER_CROSSLINK), - data_root=spec.ZERO_HASH, + data_root=spec.Hash(), parent_root=hash_tree_root(parent_crosslink), ), ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 324c5bbe7d..befeec7d3c 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -59,7 +59,7 @@ def build_empty_block(spec, state, slot=None, signed=False): empty_block.slot = slot empty_block.body.eth1_data.deposit_count = state.eth1_deposit_index previous_block_header = deepcopy(state.latest_block_header) - if previous_block_header.state_root == spec.ZERO_HASH: + if previous_block_header.state_root == spec.Hash(): previous_block_header.state_root = state.hash_tree_root() empty_block.parent_root = signing_root(previous_block_header) diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 7ffacd7d0b..a5a816977a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -27,7 +27,7 @@ def create_genesis_state(spec, num_validators): eth1_data=spec.Eth1Data( deposit_root=deposit_root, deposit_count=num_validators, - block_hash=spec.ZERO_HASH, + block_hash=spec.Hash(), ), latest_block_header=spec.BeaconBlockHeader(body_root=spec.hash_tree_root(spec.BeaconBlockBody())), ) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 685eb6ebba..54a284a6e9 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -195,7 +195,7 @@ def test_bad_merkle_proof(spec, state): deposit = prepare_state_and_deposit(spec, state, validator_index, amount) # mess up merkle branch - deposit.proof[5] = spec.ZERO_HASH + deposit.proof[5] = spec.Hash() sign_deposit_data(spec, deposit.data, privkeys[validator_index], state=state) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 5ea8cbacf8..436b36e5b9 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -360,7 +360,7 @@ def test_non_existent_recipient(spec, state): @spec_state_test def test_invalid_pubkey(spec, state): transfer = get_valid_transfer(spec, state, signed=True) - state.validators[transfer.sender].withdrawal_credentials = spec.ZERO_HASH + state.validators[transfer.sender].withdrawal_credentials = spec.Hash() # un-activate so validator can transfer state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 086d141a57..fa7da930d2 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -63,7 +63,7 @@ def test_empty_block_transition(spec, state): assert len(state.eth1_data_votes) == pre_eth1_votes + 1 assert spec.get_block_root_at_slot(state, pre_slot) == block.parent_root - assert spec.get_randao_mix(state, spec.get_current_epoch(state)) != spec.ZERO_HASH + assert spec.get_randao_mix(state, spec.get_current_epoch(state)) != spec.Hash() @with_all_phases @@ -98,7 +98,7 @@ def test_skipped_slots(spec, state): yield 'post', state assert state.slot == block.slot - assert spec.get_randao_mix(state, spec.get_current_epoch(state)) != spec.ZERO_HASH + assert spec.get_randao_mix(state, spec.get_current_epoch(state)) != spec.Hash() for slot in range(pre_slot, state.slot): assert spec.get_block_root_at_slot(state, slot) == block.parent_root From 90bba0348c7fcda4caec75a6bb8669dcd95af63a Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 30 Jun 2019 17:53:21 +0200 Subject: [PATCH 345/405] Fix spelling error Co-Authored-By: Diederik Loerakker --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 617eff293c..318056689d 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) -# deletes everything excpet the venvs +# deletes everything except the venvs partial_clean: rm -rf $(YAML_TEST_DIR) rm -rf $(GENERATOR_VENVS) From 4b93f5d9211c81e40c6f03dcc357156de4da43bf Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 18:15:36 +0200 Subject: [PATCH 346/405] disable some super-long tests, and fix a few attestation signatures --- .../block_processing/test_process_attestation.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index e7c753f402..af55c3ec68 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -68,6 +68,9 @@ def test_success_previous_epoch(spec, state): @with_all_phases @spec_state_test def test_success_since_max_epochs_per_crosslink(spec, state): + # Do not run mainnet (64 epochs), that would mean the equivalent of ~7 hours chain simulation. + if spec.MAX_EPOCHS_PER_CROSSLINK > 4: + return for _ in range(spec.MAX_EPOCHS_PER_CROSSLINK + 2): next_epoch(spec, state) apply_empty_block(spec, state) @@ -87,6 +90,9 @@ def test_success_since_max_epochs_per_crosslink(spec, state): @with_all_phases @spec_state_test def test_wrong_end_epoch_with_max_epochs_per_crosslink(spec, state): + # Do not run mainnet (64 epochs), that would mean the equivalent of ~7 hours chain simulation. + if spec.MAX_EPOCHS_PER_CROSSLINK > 4: + return for _ in range(spec.MAX_EPOCHS_PER_CROSSLINK + 2): next_epoch(spec, state) apply_empty_block(spec, state) @@ -293,12 +299,13 @@ def test_bad_parent_crosslink(spec, state): next_epoch(spec, state) apply_empty_block(spec, state) - attestation = get_valid_attestation(spec, state, signed=True) + attestation = get_valid_attestation(spec, state, signed=False) for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(spec, state) apply_empty_block(spec, state) attestation.data.crosslink.parent_root = b'\x27' * 32 + sign_attestation(spec, state, attestation) yield from run_attestation_processing(spec, state, attestation, False) @@ -309,12 +316,13 @@ def test_bad_crosslink_start_epoch(spec, state): next_epoch(spec, state) apply_empty_block(spec, state) - attestation = get_valid_attestation(spec, state, signed=True) + attestation = get_valid_attestation(spec, state, signed=False) for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(spec, state) apply_empty_block(spec, state) attestation.data.crosslink.start_epoch += 1 + sign_attestation(spec, state, attestation) yield from run_attestation_processing(spec, state, attestation, False) @@ -325,12 +333,13 @@ def test_bad_crosslink_end_epoch(spec, state): next_epoch(spec, state) apply_empty_block(spec, state) - attestation = get_valid_attestation(spec, state, signed=True) + attestation = get_valid_attestation(spec, state, signed=False) for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(spec, state) apply_empty_block(spec, state) attestation.data.crosslink.end_epoch += 1 + sign_attestation(spec, state, attestation) yield from run_attestation_processing(spec, state, attestation, False) From cb01f3ccd9ff1b3b9c8e5bd8a0d5b823d2dafcc2 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 18:40:43 +0200 Subject: [PATCH 347/405] speed up remaining attestation tests by mocking slots --- .../phase_0/block_processing/test_process_attestation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index af55c3ec68..cb49a2baff 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -59,6 +59,7 @@ def test_success(spec, state): @spec_state_test def test_success_previous_epoch(spec, state): attestation = get_valid_attestation(spec, state, signed=True) + state.slot = spec.SLOTS_PER_EPOCH - 1 next_epoch(spec, state) apply_empty_block(spec, state) @@ -136,8 +137,9 @@ def test_before_inclusion_delay(spec, state): @spec_state_test def test_after_epoch_slots(spec, state): attestation = get_valid_attestation(spec, state, signed=True) + state.slot = spec.SLOTS_PER_EPOCH - 1 # increment past latest inclusion slot - spec.process_slots(state, state.slot + spec.SLOTS_PER_EPOCH + 1) + spec.process_slots(state, state.slot + 2) apply_empty_block(spec, state) yield from run_attestation_processing(spec, state, attestation, False) @@ -296,6 +298,7 @@ def test_non_zero_crosslink_data_root(spec, state): @with_all_phases @spec_state_test def test_bad_parent_crosslink(spec, state): + state.slot = spec.SLOTS_PER_EPOCH - 1 next_epoch(spec, state) apply_empty_block(spec, state) @@ -313,6 +316,7 @@ def test_bad_parent_crosslink(spec, state): @with_all_phases @spec_state_test def test_bad_crosslink_start_epoch(spec, state): + state.slot = spec.SLOTS_PER_EPOCH - 1 next_epoch(spec, state) apply_empty_block(spec, state) @@ -330,6 +334,7 @@ def test_bad_crosslink_start_epoch(spec, state): @with_all_phases @spec_state_test def test_bad_crosslink_end_epoch(spec, state): + state.slot = spec.SLOTS_PER_EPOCH - 1 next_epoch(spec, state) apply_empty_block(spec, state) From 8415e6c79d81538f2715479ba1f29dc322214b63 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 18:42:54 +0200 Subject: [PATCH 348/405] mark incompatible mainnet tests --- .../phase_0/block_processing/test_process_attestation.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index cb49a2baff..761fed6332 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -207,6 +207,12 @@ def test_old_target_epoch(spec, state): @with_all_phases @spec_state_test def test_future_target_epoch(spec, state): + # TODO: fails in mainnet, + # something with committee indices resulting in out of range validator indices (we only have 512 test validators). + # Disabled in mainnet for now. + if spec.SHARD_COUNT > 8: + return + assert spec.MIN_ATTESTATION_INCLUSION_DELAY < spec.SLOTS_PER_EPOCH * 2 attestation = get_valid_attestation(spec, state) From 6d5e4da3e0d57afdf1c5b9a09250a1fe76f53b11 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 12:42:24 -0500 Subject: [PATCH 349/405] PR feedback --- specs/core/0_beacon-chain.md | 49 ++++++++++--------- specs/core/0_fork-choice.md | 8 +-- specs/core/1_custody-game.md | 8 +-- specs/validator/0_beacon-chain-validator.md | 6 +-- .../eth2spec/test/helpers/attestations.py | 8 +-- .../pyspec/eth2spec/test/helpers/block.py | 4 +- .../test_process_attester_slashing.py | 2 +- .../test_process_block_header.py | 2 +- .../block_processing/test_process_transfer.py | 2 +- ..._process_justification_and_finalization.py | 4 +- .../eth2spec/test/sanity/test_blocks.py | 2 +- .../pyspec/eth2spec/test/test_finality.py | 2 +- 12 files changed, 49 insertions(+), 48 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cd943187f4..4753510c23 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -70,7 +70,7 @@ - [`compute_committee`](#compute_committee) - [`validate_indexed_attestation`](#validate_indexed_attestation) - [`slot_to_epoch`](#slot_to_epoch) - - [`epoch_first_slot`](#epoch_first_slot) + - [`epoch_start_slot`](#epoch_start_slot) - [`delayed_activation_exit_epoch`](#delayed_activation_exit_epoch) - [`bls_domain`](#bls_domain) - [Beacon state getters](#beacon-state-getters) @@ -86,7 +86,7 @@ - [`get_crosslink_committee`](#get_crosslink_committee) - [`get_start_shard`](#get_start_shard) - [`get_shard_delta`](#get_shard_delta) - - [`get_proposer_index`](#get_proposer_index) + - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - [`get_attestation_data_slot`](#get_attestation_data_slot) - [`get_compact_committees_root`](#get_compact_committees_root) - [`get_total_balance`](#get_total_balance) @@ -541,7 +541,7 @@ class BeaconState(Container): #### `integer_squareroot` ```python -def integer_squareroot(n: uint64) -> uint64: +def integer_squareroot(n: int) -> int: """ Return the largest integer ``x`` such that ``x**2 <= n``. """ @@ -558,7 +558,7 @@ def integer_squareroot(n: uint64) -> uint64: ```python def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: """ - Return the exclusive or of two 32-byte strings. + Return the exclusive-or of two 32-byte strings. """ return Bytes32(a ^ b for a, b in zip(bytes1, bytes2)) ``` @@ -744,12 +744,12 @@ def slot_to_epoch(slot: Slot) -> Epoch: return Epoch(slot // SLOTS_PER_EPOCH) ``` -#### `epoch_first_slot` +#### `epoch_start_slot` ```python -def epoch_first_slot(epoch: Epoch) -> Slot: +def epoch_start_slot(epoch: Epoch) -> Slot: """ - Return the first slot of ``epoch``. + Return the start slot of ``epoch``. """ return Slot(epoch * SLOTS_PER_EPOCH) ``` @@ -759,7 +759,7 @@ def epoch_first_slot(epoch: Epoch) -> Slot: ```python def delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: """ - Return the epoch where validator activations and exits initiated in ``epoch`` take effect. + Return the epoch during which validator activations and exits initiated in ``epoch`` take effect. """ return Epoch(epoch + 1 + ACTIVATION_EXIT_DELAY) ``` @@ -802,9 +802,9 @@ def get_previous_epoch(state: BeaconState) -> Epoch: ```python def get_block_root(state: BeaconState, epoch: Epoch) -> Hash: """ - Return the block root at a recent ``epoch``. + Return the block root at the start of a recent ``epoch``. """ - return get_block_root_at_slot(state, epoch_first_slot(epoch)) + return get_block_root_at_slot(state, epoch_start_slot(epoch)) ``` #### `get_block_root_at_slot` @@ -869,10 +869,11 @@ def get_committee_count(state: BeaconState, epoch: Epoch) -> int: """ Return the number of committees at ``epoch``. """ - return max(1, min( + committees_per_slot = max(1, min( SHARD_COUNT // SLOTS_PER_EPOCH, len(get_active_validator_indices(state, epoch)) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, - )) * SLOTS_PER_EPOCH + )) + return committees_per_slot * SLOTS_PER_EPOCH ``` #### `get_crosslink_committee` @@ -909,19 +910,19 @@ def get_start_shard(state: BeaconState, epoch: Epoch) -> Shard: #### `get_shard_delta` ```python -def get_shard_delta(state: BeaconState, epoch: Epoch) -> Shard: +def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: """ Return the number of shards to increment ``state.start_shard`` at ``epoch``. """ - return Shard(min(get_committee_count(state, epoch), SHARD_COUNT - SHARD_COUNT // SLOTS_PER_EPOCH)) + return min(get_committee_count(state, epoch), SHARD_COUNT - SHARD_COUNT // SLOTS_PER_EPOCH) ``` -#### `get_proposer_index` +#### `get_beacon_proposer_index` ```python -def get_proposer_index(state: BeaconState) -> ValidatorIndex: +def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: """ - Return the beacon proposer index at the current epoch. + Return the beacon proposer index at the current slot. """ epoch = get_current_epoch(state) committees_per_slot = get_committee_count(state, epoch) // SLOTS_PER_EPOCH @@ -949,7 +950,7 @@ def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot """ committee_count = get_committee_count(state, data.target.epoch) offset = (data.crosslink.shard + SHARD_COUNT - get_start_shard(state, data.target.epoch)) % SHARD_COUNT - return Slot(epoch_first_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) + return Slot(epoch_start_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` #### `get_compact_committees_root` @@ -1102,7 +1103,7 @@ def slash_validator(state: BeaconState, decrease_balance(state, slashed_index, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT) # Apply proposer and whistleblower rewards - proposer_index = get_proposer_index(state) + proposer_index = get_beacon_proposer_index(state) if whistleblower_index is None: whistleblower_index = proposer_index whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) @@ -1190,7 +1191,7 @@ def process_slots(state: BeaconState, slot: Slot) -> None: assert state.slot <= slot while state.slot < slot: process_slot(state) - # Process epoch on the first slot of the next epoch + # Process epoch on the start slot of the next epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) state.slot += Slot(1) @@ -1529,7 +1530,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: body_root=hash_tree_root(block.body), ) # Verify proposer is not slashed - proposer = state.validators[get_proposer_index(state)] + proposer = state.validators[get_beacon_proposer_index(state)] assert not proposer.slashed # Verify proposer signature assert bls_verify(proposer.pubkey, signing_root(block), block.signature, get_domain(state, DOMAIN_BEACON_PROPOSER)) @@ -1541,7 +1542,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: epoch = get_current_epoch(state) # Verify RANDAO reveal - proposer = state.validators[get_proposer_index(state)] + proposer = state.validators[get_beacon_proposer_index(state)] assert bls_verify(proposer.pubkey, hash_tree_root(epoch), body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) # Mix in RANDAO reveal mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) @@ -1632,7 +1633,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: data=data, aggregation_bits=attestation.aggregation_bits, inclusion_delay=state.slot - attestation_slot, - proposer_index=get_proposer_index(state), + proposer_index=get_beacon_proposer_index(state), ) if data.target.epoch == get_current_epoch(state): @@ -1744,7 +1745,7 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: # Process the transfer decrease_balance(state, transfer.sender, transfer.amount + transfer.fee) increase_balance(state, transfer.recipient, transfer.amount) - increase_balance(state, get_proposer_index(state), transfer.fee) + increase_balance(state, get_beacon_proposer_index(state), transfer.fee) # Verify balances are not dust assert not (0 < state.balances[transfer.sender] < MIN_DEPOSIT_AMOUNT) assert not (0 < state.balances[transfer.recipient] < MIN_DEPOSIT_AMOUNT) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index b2ae89e9f1..64afb3f6b1 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -124,7 +124,7 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice head = store.justified_checkpoint.root - justified_slot = epoch_first_slot(store.justified_checkpoint.epoch) + justified_slot = epoch_start_slot(store.justified_checkpoint.epoch) while True: children = [ root for root in store.blocks.keys() @@ -162,7 +162,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.finalized_checkpoint.root ) # Check that block is later than the finalized epoch slot - assert block.slot > epoch_first_slot(store.finalized_checkpoint.epoch) + assert block.slot > epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state for this block to the store @@ -190,11 +190,11 @@ def on_attestation(store: Store, attestation: Attestation) -> None: # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrivesr base_state = store.block_states[target.root].copy() - assert store.time >= base_state.genesis_time + epoch_first_slot(target.epoch) * SECONDS_PER_SLOT + assert store.time >= base_state.genesis_time + epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - process_slots(base_state, epoch_first_slot(target.epoch)) + process_slots(base_state, epoch_start_slot(target.epoch)) store.checkpoint_states[target] = base_state target_state = store.checkpoint_states[target] diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index bf221ee3ec..5e8be7b390 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -382,7 +382,7 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> revealer.next_custody_reveal_period += 1 # Reward Block Preposer - proposer_index = get_proposer_index(state) + proposer_index = get_beacon_proposer_index(state) increase_balance( state, proposer_index, @@ -452,7 +452,7 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived ) # Apply penalty - proposer_index = get_proposer_index(state) + proposer_index = get_beacon_proposer_index(state) whistleblower_index = reveal.masker_index whistleblowing_reward = Gwei(penalty // WHISTLEBLOWER_REWARD_QUOTIENT) proposer_reward = Gwei(whistleblowing_reward // PROPOSER_REWARD_QUOTIENT) @@ -493,7 +493,7 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( challenge_index=state.custody_challenge_index, - challenger_index=get_proposer_index(state), + challenger_index=get_beacon_proposer_index(state), responder_index=challenge.responder_index, inclusion_epoch=get_current_epoch(state), data_root=challenge.attestation.data.crosslink.data_root, @@ -610,7 +610,7 @@ def process_chunk_challenge_response(state: BeaconState, records = state.custody_chunk_challenge_records records[records.index(challenge)] = CustodyChunkChallengeRecord() # Reward the proposer - proposer_index = get_proposer_index(state) + proposer_index = get_beacon_proposer_index(state) increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) ``` diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 9921bf0348..f4a83b8335 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -146,7 +146,7 @@ def get_committee_assignment( assert epoch <= next_epoch committees_per_slot = get_committee_count(state, epoch) // SLOTS_PER_EPOCH - start_slot = epoch_first_slot(epoch) + start_slot = epoch_start_slot(epoch) for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) slot_start_shard = (get_start_shard(state, epoch) + offset) % SHARD_COUNT @@ -162,7 +162,7 @@ A validator can use the following function to see if they are supposed to propos ```python def is_proposer(state: BeaconState, validator_index: ValidatorIndex) -> bool: - return get_proposer_index(state) == validator_index + return get_beacon_proposer_index(state) == validator_index ``` *Note*: To see if a validator is assigned to propose during the slot, the beacon state must be in the epoch in question. At the epoch boundaries, the validator must run an epoch transition into the epoch to successfully check the proposal assignment of the first slot. @@ -307,7 +307,7 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. * Set `attestation_data.target_root = epoch_boundary_block_root` where `epoch_boundary_block_root` is the root of block at the most recent epoch boundary. *Note*: `epoch_boundary_block_root` can be looked up in the state using: -* Let `start_slot = epoch_first_slot(get_current_epoch(head_state))`. +* Let `start_slot = epoch_start_slot(get_current_epoch(head_state))`. * Let `epoch_boundary_block_root = signing_root(head_block) if start_slot == head_state.slot else get_block_root(state, start_slot)`. ##### Crosslink vote diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index c47245bb9c..92c4b843dc 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -15,15 +15,15 @@ def build_attestation_data(spec, state, slot, shard): else: block_root = spec.get_block_root_at_slot(state, slot) - current_epoch_first_slot = spec.epoch_first_slot(spec.get_current_epoch(state)) - if slot < current_epoch_first_slot: + current_epoch_start_slot = spec.epoch_start_slot(spec.get_current_epoch(state)) + if slot < current_epoch_start_slot: epoch_boundary_root = spec.get_block_root(state, spec.get_previous_epoch(state)) - elif slot == current_epoch_first_slot: + elif slot == current_epoch_start_slot: epoch_boundary_root = block_root else: epoch_boundary_root = spec.get_block_root(state, spec.get_current_epoch(state)) - if slot < current_epoch_first_slot: + if slot < current_epoch_start_slot: source_epoch = state.previous_justified_checkpoint.epoch source_root = state.previous_justified_checkpoint.root else: diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index befeec7d3c..fdbc5ef55d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -12,7 +12,7 @@ def sign_block(spec, state, block, proposer_index=None): if proposer_index is None: if block.slot == state.slot: - proposer_index = spec.get_proposer_index(state) + proposer_index = spec.get_beacon_proposer_index(state) else: if spec.slot_to_epoch(state.slot) + 1 > spec.slot_to_epoch(block.slot): print("warning: block slot far away, and no proposer index manually given." @@ -20,7 +20,7 @@ def sign_block(spec, state, block, proposer_index=None): # use stub state to get proposer index of future slot stub_state = deepcopy(state) spec.process_slots(stub_state, block.slot) - proposer_index = spec.get_proposer_index(stub_state) + proposer_index = spec.get_beacon_proposer_index(stub_state) privkey = privkeys[proposer_index] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 7f539e3b95..7a60301577 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -30,7 +30,7 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) + attester_slashing.attestation_1.custody_bit_1_indices ) - proposer_index = spec.get_proposer_index(state) + proposer_index = spec.get_beacon_proposer_index(state) pre_proposer_balance = get_balance(state, proposer_index) pre_slashings = {slashed_index: get_balance(state, slashed_index) for slashed_index in slashed_indices} pre_withdrawalable_epochs = { diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py index c348e57e14..a2306ef4d9 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py @@ -75,7 +75,7 @@ def test_proposer_slashed(spec, state): # use stub state to get proposer index of next slot stub_state = deepcopy(state) next_slot(spec, stub_state) - proposer_index = spec.get_proposer_index(stub_state) + proposer_index = spec.get_beacon_proposer_index(stub_state) # set proposer to slashed state.validators[proposer_index].slashed = True diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 436b36e5b9..f079ff5781 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -21,7 +21,7 @@ def run_transfer_processing(spec, state, transfer, valid=True): yield 'post', None return - proposer_index = spec.get_proposer_index(state) + proposer_index = spec.get_beacon_proposer_index(state) pre_transfer_sender_balance = state.balances[transfer.sender] pre_transfer_recipient_balance = state.balances[transfer.recipient] pre_transfer_proposer_balance = state.balances[proposer_index] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index a308fe3d2d..f59da1fc5c 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -33,7 +33,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support total_balance = spec.get_total_active_balance(state) remaining_balance = total_balance * 2 // 3 - start_slot = spec.epoch_first_slot(epoch) + start_slot = spec.epoch_start_slot(epoch) for slot in range(start_slot, start_slot + spec.SLOTS_PER_EPOCH): for shard in get_shards_for_slot(spec, state, slot): # Check if we already have had sufficient balance. (and undone if we don't want it). @@ -80,7 +80,7 @@ def get_checkpoints(spec, epoch): def put_checkpoints_in_block_roots(spec, state, checkpoints): for c in checkpoints: - state.block_roots[spec.epoch_first_slot(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root + state.block_roots[spec.epoch_start_slot(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root def finalize_on_234(spec, state, epoch, sufficient_support): diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index fa7da930d2..b6f0ecba2b 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -214,7 +214,7 @@ def test_attester_slashing(spec, state): # lost whistleblower reward assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) - proposer_index = spec.get_proposer_index(state) + proposer_index = spec.get_beacon_proposer_index(state) # gained whistleblower reward assert ( get_balance(state, proposer_index) > diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index d572b05fb5..5ff0ed1efb 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -43,7 +43,7 @@ def next_epoch_with_attestations(spec, block = build_empty_block_for_next_slot(spec, post_state) if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= spec.epoch_first_slot(spec.get_current_epoch(post_state)): + if slot_to_attest >= spec.epoch_start_slot(spec.get_current_epoch(post_state)): cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) block.body.attestations.append(cur_attestation) From ed182e5e9d3036a59f6683a046bfe66fc1ade6a2 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 19:44:35 +0200 Subject: [PATCH 350/405] disable mainnet transfer test output --- test_generators/operations/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index 38fa42f689..b61e98526f 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -49,7 +49,9 @@ def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: create_suite('proposer_slashing', 'minimal', lambda: generate_from_tests(test_process_proposer_slashing, 'phase0')), create_suite('proposer_slashing', 'mainnet', lambda: generate_from_tests(test_process_proposer_slashing, 'phase0')), create_suite('transfer', 'minimal', lambda: generate_from_tests(test_process_transfer, 'phase0')), - create_suite('transfer', 'mainnet', lambda: generate_from_tests(test_process_transfer, 'phase0')), + # Disabled, due to the high amount of different transfer tests, this produces a shocking size of tests. + # Unnecessarily, as transfer are disabled currently, so not a priority. + # create_suite('transfer', 'mainnet', lambda: generate_from_tests(test_process_transfer, 'phase0')), create_suite('voluntary_exit', 'minimal', lambda: generate_from_tests(test_process_voluntary_exit, 'phase0')), create_suite('voluntary_exit', 'mainnet', lambda: generate_from_tests(test_process_voluntary_exit, 'phase0')), ]) From 3ae077a09b3e374f1e45461f1be535671c5b0476 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 12:51:13 -0500 Subject: [PATCH 351/405] 'beacon state getters' -> ' beacon state accessors' --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4753510c23..7bc0c5377a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -73,7 +73,7 @@ - [`epoch_start_slot`](#epoch_start_slot) - [`delayed_activation_exit_epoch`](#delayed_activation_exit_epoch) - [`bls_domain`](#bls_domain) - - [Beacon state getters](#beacon-state-getters) + - [Beacon state accessors](#beacon-state-accessors) - [`get_current_epoch`](#get_current_epoch) - [`get_previous_epoch`](#get_previous_epoch) - [`get_block_root`](#get_block_root) @@ -774,7 +774,7 @@ def bls_domain(domain_type: int, fork_version: bytes=b'\x00' * 4) -> int: return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) ``` -### Beacon state getters +### Beacon state accessors #### `get_current_epoch` From d1bc2f03acc9cff67dcecdd5945aa9c15e14f582 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 13:00:22 -0500 Subject: [PATCH 352/405] remove 'int' types from spec typehints. replace with uint64 --- specs/core/0_beacon-chain.md | 16 ++++++++-------- specs/core/0_fork-choice.md | 4 ++-- specs/core/1_custody-game.md | 6 +++--- specs/core/1_shard-data-chains.md | 8 ++++---- specs/validator/0_beacon-chain-validator.md | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7bc0c5377a..fb388e4389 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -541,7 +541,7 @@ class BeaconState(Container): #### `integer_squareroot` ```python -def integer_squareroot(n: int) -> int: +def integer_squareroot(n: uint64) -> int: """ Return the largest integer ``x`` such that ``x**2 <= n``. """ @@ -564,7 +564,7 @@ def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: ``` ```python -def int_to_bytes(integer: int, length: int) -> bytes: +def int_to_bytes(integer: uint64, length: uint64) -> bytes: """ Return the ``length``-byte serialization of ``integer``. """ @@ -647,7 +647,7 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa #### `is_valid_merkle_branch` ```python -def is_valid_merkle_branch(leaf: Hash, branch: Sequence[Hash], depth: int, index: int, root: Hash) -> bool: +def is_valid_merkle_branch(leaf: Hash, branch: Sequence[Hash], depth: uint64, index: uint64, root: Hash) -> bool: """ Check if ``leaf`` at ``index`` verifies against the Merkle ``root`` and ``branch``. """ @@ -665,7 +665,7 @@ def is_valid_merkle_branch(leaf: Hash, branch: Sequence[Hash], depth: int, index #### `shuffle_index` ```python -def shuffle_index(index: ValidatorIndex, index_count: int, seed: Hash) -> ValidatorIndex: +def shuffle_index(index: ValidatorIndex, index_count: uint64, seed: Hash) -> ValidatorIndex: """ Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). """ @@ -691,8 +691,8 @@ def shuffle_index(index: ValidatorIndex, index_count: int, seed: Hash) -> Valida ```python def compute_committee(indices: Sequence[ValidatorIndex], seed: Hash, - index: int, - count: int) -> Sequence[ValidatorIndex]: + index: uint64, + count: uint64) -> Sequence[ValidatorIndex]: """ Return the committee corresponding to ``indices``, ``seed``, ``index``, and committee ``count``. """ @@ -767,7 +767,7 @@ def delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: #### `bls_domain` ```python -def bls_domain(domain_type: int, fork_version: bytes=b'\x00' * 4) -> int: +def bls_domain(domain_type: uint64, fork_version: bytes=b'\x00' * 4) -> int: """ Return the BLS domain for the ``domain_type`` and ``fork_version``. """ @@ -997,7 +997,7 @@ def get_total_active_balance(state: BeaconState) -> Gwei: #### `get_domain` ```python -def get_domain(state: BeaconState, domain_type: int, message_epoch: Epoch=None) -> int: +def get_domain(state: BeaconState, domain_type: uint64, message_epoch: Epoch=None) -> int: """ Return the signature domain (fork version concatenated with domain type) of a message. """ diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 64afb3f6b1..a45d79fbec 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -69,7 +69,7 @@ class LatestMessage(object): ```python @dataclass class Store(object): - time: int + time: uint64 justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) @@ -141,7 +141,7 @@ def get_head(store: Store) -> Hash: #### `on_tick` ```python -def on_tick(store: Store, time: int) -> None: +def on_tick(store: Store, time: uint64) -> None: store.time = time ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 5e8be7b390..54a1ca34be 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -259,7 +259,7 @@ class BeaconBlockBody(Container): ### `ceillog2` ```python -def ceillog2(x: int) -> int: +def ceillog2(x: uint64) -> int: return x.bit_length() ``` @@ -275,7 +275,7 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int: ### `get_bit` ```python -def get_bit(serialization: bytes, i: int) -> int: +def get_bit(serialization: bytes, i: uint64) -> int: """ Extract the bit in ``serialization`` at position ``i``. """ @@ -304,7 +304,7 @@ def get_chunk_bits_root(chunk_bits: bytes) -> Bytes32: ### `get_randao_epoch_for_custody_period` ```python -def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorIndex) -> Epoch: +def get_randao_epoch_for_custody_period(period: uint64, validator_index: ValidatorIndex) -> Epoch: next_period_start = (period + 1) * EPOCHS_PER_CUSTODY_PERIOD - validator_index % EPOCHS_PER_CUSTODY_PERIOD return Epoch(next_period_start + CUSTODY_PERIOD_TO_RANDAO_PADDING) ``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 82843b7a6b..fd0e41b4fe 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -132,8 +132,8 @@ class ShardBlockHeader(Container): def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard, - index: int, - count: int) -> Sequence[ValidatorIndex]: + index: uint64, + count: uint64) -> Sequence[ValidatorIndex]: """ Return committee for a period. Used to construct persistent committees. """ @@ -248,7 +248,7 @@ def verify_shard_attestation_signature(state: BeaconState, ```python def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: - def is_power_of_two(value: int) -> bool: + def is_power_of_two(value: uint64) -> bool: return (value > 0) and (value & (value - 1) == 0) def pad_to_power_of_2(values: MutableSequence[bytes]) -> Sequence[bytes]: @@ -259,7 +259,7 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: def hash_tree_root_of_bytes(data: bytes) -> bytes: return hash_tree_root([data[i:i + 32] for i in range(0, len(data), 32)]) - def zpad(data: bytes, length: int) -> bytes: + def zpad(data: bytes, length: uint64) -> bytes: return data + b'\x00' * (length - len(data)) return hash( diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index f4a83b8335..0b6857a72d 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -223,7 +223,7 @@ epoch_signature = bls_sign( The `block.eth1_data` field is for block proposers to vote on recent Eth 1.0 data. This recent data contains an Eth 1.0 block hash as well as the associated deposit root (as calculated by the `get_hash_tree_root()` method of the deposit contract) and deposit count after execution of the corresponding Eth 1.0 block. If over half of the block proposers in the current Eth 1.0 voting period vote for the same `eth1_data` then `state.eth1_data` updates at the end of the voting period. Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. -Let `get_eth1_data(distance: int) -> Eth1Data` be the (subjective) function that returns the Eth 1.0 data at distance `distance` relative to the Eth 1.0 head at the start of the current Eth 1.0 voting period. Let `previous_eth1_distance` be the distance relative to the Eth 1.0 block corresponding to `state.eth1_data.block_hash` at the start of the current Eth 1.0 voting period. An honest block proposer sets `block.eth1_data = get_eth1_vote(state, previous_eth1_distance)` where: +Let `get_eth1_data(distance: uint64) -> Eth1Data` be the (subjective) function that returns the Eth 1.0 data at distance `distance` relative to the Eth 1.0 head at the start of the current Eth 1.0 voting period. Let `previous_eth1_distance` be the distance relative to the Eth 1.0 block corresponding to `state.eth1_data.block_hash` at the start of the current Eth 1.0 voting period. An honest block proposer sets `block.eth1_data = get_eth1_vote(state, previous_eth1_distance)` where: ```python def get_eth1_vote(state: BeaconState, previous_eth1_distance: uint64) -> Eth1Data: From 6be1404b66c1238ed515e5718c146f757dca7d68 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 13:16:56 -0500 Subject: [PATCH 353/405] delayed_activation_exit_epoch -> compute_activation_exit_epoch --- specs/core/0_beacon-chain.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index fb388e4389..68915177c5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -71,7 +71,7 @@ - [`validate_indexed_attestation`](#validate_indexed_attestation) - [`slot_to_epoch`](#slot_to_epoch) - [`epoch_start_slot`](#epoch_start_slot) - - [`delayed_activation_exit_epoch`](#delayed_activation_exit_epoch) + - [`compute_activation_exit_epoch`](#compute_activation_exit_epoch) - [`bls_domain`](#bls_domain) - [Beacon state accessors](#beacon-state-accessors) - [`get_current_epoch`](#get_current_epoch) @@ -754,10 +754,10 @@ def epoch_start_slot(epoch: Epoch) -> Slot: return Slot(epoch * SLOTS_PER_EPOCH) ``` -#### `delayed_activation_exit_epoch` +#### `compute_activation_exit_epoch` ```python -def delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: +def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: """ Return the epoch during which validator activations and exits initiated in ``epoch`` take effect. """ @@ -1075,7 +1075,7 @@ def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: # Compute exit queue epoch exit_epochs = [v.exit_epoch for v in state.validators if v.exit_epoch != FAR_FUTURE_EPOCH] - exit_queue_epoch = max(exit_epochs + [delayed_activation_exit_epoch(get_current_epoch(state))]) + exit_queue_epoch = max(exit_epochs + [compute_activation_exit_epoch(get_current_epoch(state))]) exit_queue_churn = len([v for v in state.validators if v.exit_epoch == exit_queue_epoch]) if exit_queue_churn >= get_validator_churn_limit(state): exit_queue_epoch += Epoch(1) @@ -1450,13 +1450,13 @@ def process_registry_updates(state: BeaconState) -> None: activation_queue = sorted([ index for index, validator in enumerate(state.validators) if validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH and - validator.activation_epoch >= delayed_activation_exit_epoch(state.finalized_checkpoint.epoch) + validator.activation_epoch >= compute_activation_exit_epoch(state.finalized_checkpoint.epoch) ], key=lambda index: state.validators[index].activation_eligibility_epoch) # Dequeued validators for activation up to churn limit (without resetting activation epoch) for index in activation_queue[:get_validator_churn_limit(state)]: validator = state.validators[index] if validator.activation_epoch == FAR_FUTURE_EPOCH: - validator.activation_epoch = delayed_activation_exit_epoch(get_current_epoch(state)) + validator.activation_epoch = compute_activation_exit_epoch(get_current_epoch(state)) ``` #### Slashings From 4aa676bae7ed01a6deaf34d5ad1f6d8484a3bbb0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 13:52:35 -0500 Subject: [PATCH 354/405] fix mainnet attestation test --- .../test_process_attestation.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 761fed6332..ab46a0d8ce 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -1,6 +1,7 @@ from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases from eth2spec.test.helpers.attestations import ( get_valid_attestation, + sign_aggregate_attestation, sign_attestation, ) from eth2spec.test.helpers.state import ( @@ -207,20 +208,21 @@ def test_old_target_epoch(spec, state): @with_all_phases @spec_state_test def test_future_target_epoch(spec, state): - # TODO: fails in mainnet, - # something with committee indices resulting in out of range validator indices (we only have 512 test validators). - # Disabled in mainnet for now. - if spec.SHARD_COUNT > 8: - return - assert spec.MIN_ATTESTATION_INCLUSION_DELAY < spec.SLOTS_PER_EPOCH * 2 attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - + participants = spec.get_attesting_indices( + state, + attestation.data, + attestation.aggregation_bits + ) attestation.data.target.epoch = spec.get_current_epoch(state) + 1 # target epoch will be too new to handle - sign_attestation(spec, state, attestation) + + # manually add signature for correct participants + attestation.signature = sign_aggregate_attestation(spec, state, attestation.data, participants) + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY yield from run_attestation_processing(spec, state, attestation, False) From 0fd1d384179048cebfd56423dbd47c4d84958407 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 30 Jun 2019 20:58:02 +0200 Subject: [PATCH 355/405] `Misc` function rename --- scripts/build_spec.py | 2 +- specs/core/0_beacon-chain.md | 111 +++++++++--------- specs/core/0_fork-choice.md | 8 +- specs/core/1_custody-game.md | 4 +- specs/core/1_shard-data-chains.md | 11 +- specs/light_client/sync_protocol.md | 4 +- specs/validator/0_beacon-chain-validator.md | 18 +-- test_generators/shuffling/main.py | 2 +- .../eth2spec/test/helpers/attestations.py | 10 +- .../pyspec/eth2spec/test/helpers/block.py | 8 +- .../pyspec/eth2spec/test/helpers/deposits.py | 2 +- ..._process_justification_and_finalization.py | 8 +- .../pyspec/eth2spec/test/test_finality.py | 2 +- 13 files changed, 96 insertions(+), 94 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index fe66a72d57..2a556cb87c 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -98,7 +98,7 @@ def apply_constants_preset(preset: Dict[str, Any]) -> None: global_vars[k] = v # Deal with derived constants - global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) + global_vars['GENESIS_EPOCH'] = compute_slot_epoch(GENESIS_SLOT) # Initialize SSZ types again, to account for changed lengths init_SSZ_types() diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 68915177c5..75721ec1a4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -66,13 +66,13 @@ - [`is_slashable_attestation_data`](#is_slashable_attestation_data) - [`is_valid_merkle_branch`](#is_valid_merkle_branch) - [Misc](#misc) - - [`shuffle_index`](#shuffle_index) + - [`compute_shuffle_index`](#compute_shuffle_index) - [`compute_committee`](#compute_committee) - - [`validate_indexed_attestation`](#validate_indexed_attestation) - - [`slot_to_epoch`](#slot_to_epoch) - - [`epoch_start_slot`](#epoch_start_slot) + - [`compute_slot_epoch`](#compute_slot_epoch) + - [`compute_epoch_start_slot`](#compute_epoch_start_slot) - [`compute_activation_exit_epoch`](#compute_activation_exit_epoch) - - [`bls_domain`](#bls_domain) + - [`compute_bls_domain`](#compute_bls_domain) + - [`validate_indexed_attestation`](#validate_indexed_attestation) - [Beacon state accessors](#beacon-state-accessors) - [`get_current_epoch`](#get_current_epoch) - [`get_previous_epoch`](#get_previous_epoch) @@ -662,10 +662,10 @@ def is_valid_merkle_branch(leaf: Hash, branch: Sequence[Hash], depth: uint64, in ### Misc -#### `shuffle_index` +#### `compute_shuffle_index` ```python -def shuffle_index(index: ValidatorIndex, index_count: uint64, seed: Hash) -> ValidatorIndex: +def compute_shuffle_index(index: ValidatorIndex, index_count: uint64, seed: Hash) -> ValidatorIndex: """ Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). """ @@ -698,56 +698,23 @@ def compute_committee(indices: Sequence[ValidatorIndex], """ start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return [indices[shuffle_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] -``` - -#### `validate_indexed_attestation` - -```python -def validate_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> None: - """ - Verify validity of ``indexed_attestation``. - """ - bit_0_indices = indexed_attestation.custody_bit_0_indices - bit_1_indices = indexed_attestation.custody_bit_1_indices - - # Verify no index has custody bit equal to 1 [to be removed in phase 1] - assert len(bit_1_indices) == 0 - # Verify max number of indices - assert len(bit_0_indices) + len(bit_1_indices) <= MAX_VALIDATORS_PER_COMMITTEE - # Verify index sets are disjoint - assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 - # Verify indices are sorted - assert bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices) - # Verify aggregate signature - assert bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_0_indices]), - bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_1_indices]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), - ], - signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target.epoch), - ) + return [indices[compute_shuffle_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] ``` -#### `slot_to_epoch` +#### `compute_slot_epoch` ```python -def slot_to_epoch(slot: Slot) -> Epoch: +def compute_slot_epoch(slot: Slot) -> Epoch: """ Return the epoch number of ``slot``. """ return Epoch(slot // SLOTS_PER_EPOCH) ``` -#### `epoch_start_slot` +#### `compute_epoch_start_slot` ```python -def epoch_start_slot(epoch: Epoch) -> Slot: +def compute_epoch_start_slot(epoch: Epoch) -> Slot: """ Return the start slot of ``epoch``. """ @@ -764,16 +731,49 @@ def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: return Epoch(epoch + 1 + ACTIVATION_EXIT_DELAY) ``` -#### `bls_domain` +#### `compute_bls_domain` ```python -def bls_domain(domain_type: uint64, fork_version: bytes=b'\x00' * 4) -> int: +def compute_bls_domain(domain_type: uint64, fork_version: bytes=b'\x00' * 4) -> int: """ Return the BLS domain for the ``domain_type`` and ``fork_version``. """ return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) ``` +#### `validate_indexed_attestation` + +```python +def validate_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> None: + """ + Verify validity of ``indexed_attestation``. + """ + bit_0_indices = indexed_attestation.custody_bit_0_indices + bit_1_indices = indexed_attestation.custody_bit_1_indices + + # Verify no index has custody bit equal to 1 [to be removed in phase 1] + assert len(bit_1_indices) == 0 + # Verify max number of indices + assert len(bit_0_indices) + len(bit_1_indices) <= MAX_VALIDATORS_PER_COMMITTEE + # Verify index sets are disjoint + assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 + # Verify indices are sorted + assert bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices) + # Verify aggregate signature + assert bls_verify_multiple( + pubkeys=[ + bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_0_indices]), + bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_1_indices]), + ], + message_hashes=[ + hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), + hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), + ], + signature=indexed_attestation.signature, + domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target.epoch), + ) +``` + ### Beacon state accessors #### `get_current_epoch` @@ -783,7 +783,7 @@ def get_current_epoch(state: BeaconState) -> Epoch: """ Return the current epoch. """ - return slot_to_epoch(state.slot) + return compute_slot_epoch(state.slot) ``` #### `get_previous_epoch` @@ -804,7 +804,7 @@ def get_block_root(state: BeaconState, epoch: Epoch) -> Hash: """ Return the block root at the start of a recent ``epoch``. """ - return get_block_root_at_slot(state, epoch_start_slot(epoch)) + return get_block_root_at_slot(state, compute_epoch_start_slot(epoch)) ``` #### `get_block_root_at_slot` @@ -950,7 +950,7 @@ def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot """ committee_count = get_committee_count(state, data.target.epoch) offset = (data.crosslink.shard + SHARD_COUNT - get_start_shard(state, data.target.epoch)) % SHARD_COUNT - return Slot(epoch_start_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) + return Slot(compute_epoch_start_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` #### `get_compact_committees_root` @@ -1003,7 +1003,7 @@ def get_domain(state: BeaconState, domain_type: uint64, message_epoch: Epoch=Non """ epoch = get_current_epoch(state) if message_epoch is None else message_epoch fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version - return bls_domain(domain_type, fork_version) + return compute_bls_domain(domain_type, fork_version) ``` #### `get_indexed_attestation` @@ -1585,14 +1585,14 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: proposer = state.validators[proposer_slashing.proposer_index] # Verify that the epoch is the same - assert slot_to_epoch(proposer_slashing.header_1.slot) == slot_to_epoch(proposer_slashing.header_2.slot) + assert compute_slot_epoch(proposer_slashing.header_1.slot) == compute_slot_epoch(proposer_slashing.header_2.slot) # But the headers are different assert proposer_slashing.header_1 != proposer_slashing.header_2 # Check proposer is slashable assert is_slashable_validator(proposer, get_current_epoch(state)) # Signatures are valid for header in (proposer_slashing.header_1, proposer_slashing.header_2): - domain = get_domain(state, DOMAIN_BEACON_PROPOSER, slot_to_epoch(header.slot)) + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_slot_epoch(header.slot)) assert bls_verify(proposer.pubkey, signing_root(header), header.signature, domain) slash_validator(state, proposer_slashing.proposer_index) @@ -1677,8 +1677,9 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) for new validators. # Note: The deposit contract does not check signatures. - # Note: Deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` - if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT)): + # Note: Deposits are valid across forks. thus the deposit domain is retrieved directly from `compute_bls_domain` + if not bls_verify(pubkey, signing_root(deposit.data), + deposit.data.signature, compute_bls_domain(DOMAIN_DEPOSIT)): return # Add validator and balance entries diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index a45d79fbec..a386e162db 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -124,7 +124,7 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice head = store.justified_checkpoint.root - justified_slot = epoch_start_slot(store.justified_checkpoint.epoch) + justified_slot = compute_epoch_start_slot(store.justified_checkpoint.epoch) while True: children = [ root for root in store.blocks.keys() @@ -162,7 +162,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.finalized_checkpoint.root ) # Check that block is later than the finalized epoch slot - assert block.slot > epoch_start_slot(store.finalized_checkpoint.epoch) + assert block.slot > compute_epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state for this block to the store @@ -190,11 +190,11 @@ def on_attestation(store: Store, attestation: Attestation) -> None: # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrivesr base_state = store.block_states[target.root].copy() - assert store.time >= base_state.genesis_time + epoch_start_slot(target.epoch) * SECONDS_PER_SLOT + assert store.time >= base_state.genesis_time + compute_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - process_slots(base_state, epoch_start_slot(target.epoch)) + process_slots(base_state, compute_epoch_start_slot(target.epoch)) store.checkpoint_states[target] = base_state target_state = store.checkpoint_states[target] diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 54a1ca34be..149386519d 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -475,7 +475,7 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge # Verify the attestation validate_indexed_attestation(state, get_indexed_attestation(state, challenge.attestation)) # Verify it is not too late to challenge - assert slot_to_epoch(challenge.attestation.data.slot) >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY + assert compute_slot_epoch(challenge.attestation.data.slot) >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY responder = state.validators[challenge.responder_index] assert responder.exit_epoch >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY # Verify the responder participated in the attestation @@ -516,7 +516,7 @@ For each `challenge` in `block.body.custody_bit_challenges`, run the following f ```python def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> None: attestation = challenge.attestation - epoch = slot_to_epoch(attestation.data.slot) + epoch = compute_slot_epoch(attestation.data.slot) shard = attestation.data.crosslink.shard # Verify challenge signature diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index fd0e41b4fe..5dc6e698f3 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -163,7 +163,7 @@ def get_persistent_committee(state: BeaconState, """ Return the persistent committee for the given ``shard`` at the given ``slot``. """ - epoch = slot_to_epoch(slot) + epoch = compute_slot_epoch(slot) earlier_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2) later_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD) @@ -240,7 +240,7 @@ def verify_shard_attestation_signature(state: BeaconState, pubkey=bls_aggregate_pubkeys(pubkeys), message_hash=data.shard_block_root, signature=attestation.aggregate_signature, - domain=get_domain(state, DOMAIN_SHARD_ATTESTER, slot_to_epoch(data.slot)) + domain=get_domain(state, DOMAIN_SHARD_ATTESTER, compute_slot_epoch(data.slot)) ) ``` @@ -339,7 +339,7 @@ def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], pubkey=beacon_state.validators[proposer_index].pubkey, message_hash=signing_root(candidate), signature=candidate.signature, - domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, slot_to_epoch(candidate.slot)), + domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_slot_epoch(candidate.slot)), ) return True @@ -403,11 +403,12 @@ def is_valid_beacon_attestation(shard: Shard, None, ) assert previous_attestation is not None - assert candidate.data.previous_attestation.epoch < slot_to_epoch(candidate.data.slot) + assert candidate.data.previous_attestation.epoch < compute_slot_epoch(candidate.data.slot) # Check crosslink data root start_epoch = beacon_state.crosslinks[shard].epoch - end_epoch = min(slot_to_epoch(candidate.data.slot) - CROSSLINK_LOOKBACK, start_epoch + MAX_EPOCHS_PER_CROSSLINK) + end_epoch = min(compute_slot_epoch(candidate.data.slot) - CROSSLINK_LOOKBACK, + start_epoch + MAX_EPOCHS_PER_CROSSLINK) blocks = [] for slot in range(start_epoch * SLOTS_PER_EPOCH, end_epoch * SLOTS_PER_EPOCH): blocks.append(shard_blocks[slot]) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 2b1703f217..f9e347d807 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -124,7 +124,7 @@ def compute_committee(header: BeaconBlockHeader, maximal_later_committee = validator_memory.later_period_data.committee earlier_start_epoch = get_earlier_start_epoch(header.slot) later_start_epoch = get_later_start_epoch(header.slot) - epoch = slot_to_epoch(header.slot) + epoch = compute_slot_epoch(header.slot) committee_count = max( earlier_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), @@ -192,7 +192,7 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val pubkey=group_public_key, message_hash=hash_tree_root(shard_parent_block), signature=proof.shard_aggregate_signature, - domain=get_domain(state, slot_to_epoch(shard_block.slot), DOMAIN_SHARD_ATTESTER), + domain=get_domain(state, compute_slot_epoch(shard_block.slot), DOMAIN_SHARD_ATTESTER), ) ``` diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 0b6857a72d..d9906827d2 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -98,7 +98,7 @@ To submit a deposit: * Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. * Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. * Set `deposit_data.amount = amount`. -* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=bls_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `bls_domain` will default to zeroes there). +* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=compute_bls_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `compute_bls_domain` will default to zeroes there). * Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. *Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validators` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. @@ -146,7 +146,7 @@ def get_committee_assignment( assert epoch <= next_epoch committees_per_slot = get_committee_count(state, epoch) // SLOTS_PER_EPOCH - start_slot = epoch_start_slot(epoch) + start_slot = compute_epoch_start_slot(epoch) for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) slot_start_shard = (get_start_shard(state, epoch) + offset) % SHARD_COUNT @@ -210,10 +210,10 @@ Set `block.randao_reveal = epoch_signature` where `epoch_signature` is defined a ```python epoch_signature = bls_sign( privkey=validator.privkey, # privkey stored locally, not in state - message_hash=hash_tree_root(slot_to_epoch(block.slot)), + message_hash=hash_tree_root(compute_slot_epoch(block.slot)), domain=get_domain( fork=fork, # `fork` is the fork object at the slot `block.slot` - epoch=slot_to_epoch(block.slot), + epoch=compute_slot_epoch(block.slot), domain_type=DOMAIN_RANDAO, ) ) @@ -252,7 +252,7 @@ block_signature = bls_sign( message_hash=signing_root(block), domain=get_domain( fork=fork, # `fork` is the fork object at the slot `block.slot` - epoch=slot_to_epoch(block.slot), + epoch=compute_slot_epoch(block.slot), domain_type=DOMAIN_BEACON_BLOCK, ) ) @@ -307,7 +307,7 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. * Set `attestation_data.target_root = epoch_boundary_block_root` where `epoch_boundary_block_root` is the root of block at the most recent epoch boundary. *Note*: `epoch_boundary_block_root` can be looked up in the state using: -* Let `start_slot = epoch_start_slot(get_current_epoch(head_state))`. +* Let `start_slot = compute_epoch_start_slot(get_current_epoch(head_state))`. * Let `epoch_boundary_block_root = signing_root(head_block) if start_slot == head_state.slot else get_block_root(state, start_slot)`. ##### Crosslink vote @@ -357,7 +357,7 @@ signed_attestation_data = bls_sign( message_hash=attestation_message, domain=get_domain( fork=fork, # `fork` is the fork object at the slot, `attestation_data.slot` - epoch=slot_to_epoch(attestation_data.slot), + epoch=compute_slot_epoch(attestation_data.slot), domain_type=DOMAIN_ATTESTATION, ) ) @@ -376,7 +376,7 @@ To avoid "proposer slashings", a validator must not sign two conflicting [`Beaco *In Phase 0, as long as the validator does not sign two different beacon blocks for the same epoch, the validator is safe against proposer slashings.* Specifically, when signing a `BeaconBlock`, a validator should perform the following steps in the following order: -1. Save a record to hard disk that a beacon block has been signed for the `epoch=slot_to_epoch(block.slot)`. +1. Save a record to hard disk that a beacon block has been signed for the `epoch=compute_slot_epoch(block.slot)`. 2. Generate and broadcast the block. If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast block and can effectively avoid slashing. @@ -386,7 +386,7 @@ If the software crashes at some point within this routine, then when the validat To avoid "attester slashings", a validator must not sign two conflicting [`AttestationData`](../core/0_beacon-chain.md#attestationdata) objects, i.e. two attestations that satisfy [`is_slashable_attestation_data`](../core/0_beacon-chain.md#is_slashable_attestation_data). Specifically, when signing an `Attestation`, a validator should perform the following steps in the following order: -1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source_epoch`) and target (i.e. `slot_to_epoch(attestation_data.slot)`). +1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source_epoch`) and target (i.e. `compute_slot_epoch(attestation_data.slot)`). 2. Generate and broadcast attestation. If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast attestation and can effectively avoid slashing. diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index e40dda5209..bc7907242c 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -10,7 +10,7 @@ def shuffling_case(seed, count): yield 'seed', '0x' + seed.hex() yield 'count', count - yield 'shuffled', [spec.shuffle_index(i, count, seed) for i in range(count)] + yield 'shuffled', [spec.compute_shuffle_index(i, count, seed) for i in range(count)] @to_tuple diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 92c4b843dc..ba2bd3ca00 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -15,7 +15,7 @@ def build_attestation_data(spec, state, slot, shard): else: block_root = spec.get_block_root_at_slot(state, slot) - current_epoch_start_slot = spec.epoch_start_slot(spec.get_current_epoch(state)) + current_epoch_start_slot = spec.compute_epoch_start_slot(spec.get_current_epoch(state)) if slot < current_epoch_start_slot: epoch_boundary_root = spec.get_block_root(state, spec.get_previous_epoch(state)) elif slot == current_epoch_start_slot: @@ -30,7 +30,7 @@ def build_attestation_data(spec, state, slot, shard): source_epoch = state.current_justified_checkpoint.epoch source_root = state.current_justified_checkpoint.root - if spec.slot_to_epoch(slot) == spec.get_current_epoch(state): + if spec.compute_slot_epoch(slot) == spec.get_current_epoch(state): parent_crosslink = state.current_crosslinks[shard] else: parent_crosslink = state.previous_crosslinks[shard] @@ -38,11 +38,11 @@ def build_attestation_data(spec, state, slot, shard): return spec.AttestationData( beacon_block_root=block_root, source=spec.Checkpoint(epoch=source_epoch, root=source_root), - target=spec.Checkpoint(epoch=spec.slot_to_epoch(slot), root=epoch_boundary_root), + target=spec.Checkpoint(epoch=spec.compute_slot_epoch(slot), root=epoch_boundary_root), crosslink=spec.Crosslink( shard=shard, start_epoch=parent_crosslink.end_epoch, - end_epoch=min(spec.slot_to_epoch(slot), parent_crosslink.end_epoch + spec.MAX_EPOCHS_PER_CROSSLINK), + end_epoch=min(spec.compute_slot_epoch(slot), parent_crosslink.end_epoch + spec.MAX_EPOCHS_PER_CROSSLINK), data_root=spec.Hash(), parent_root=hash_tree_root(parent_crosslink), ), @@ -53,7 +53,7 @@ def get_valid_attestation(spec, state, slot=None, signed=False): if slot is None: slot = state.slot - epoch = spec.slot_to_epoch(slot) + epoch = spec.compute_slot_epoch(slot) epoch_start_shard = spec.get_start_shard(state, epoch) committees_per_slot = spec.get_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index fdbc5ef55d..830a1e9ec3 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -14,7 +14,7 @@ def sign_block(spec, state, block, proposer_index=None): if block.slot == state.slot: proposer_index = spec.get_beacon_proposer_index(state) else: - if spec.slot_to_epoch(state.slot) + 1 > spec.slot_to_epoch(block.slot): + if spec.compute_slot_epoch(state.slot) + 1 > spec.compute_slot_epoch(block.slot): print("warning: block slot far away, and no proposer index manually given." " Signing block is slow due to transition for proposer index calculation.") # use stub state to get proposer index of future slot @@ -26,10 +26,10 @@ def sign_block(spec, state, block, proposer_index=None): block.body.randao_reveal = bls_sign( privkey=privkey, - message_hash=hash_tree_root(spec.slot_to_epoch(block.slot)), + message_hash=hash_tree_root(spec.compute_slot_epoch(block.slot)), domain=spec.get_domain( state, - message_epoch=spec.slot_to_epoch(block.slot), + message_epoch=spec.compute_slot_epoch(block.slot), domain_type=spec.DOMAIN_RANDAO, ) ) @@ -39,7 +39,7 @@ def sign_block(spec, state, block, proposer_index=None): domain=spec.get_domain( state, spec.DOMAIN_BEACON_PROPOSER, - spec.slot_to_epoch(block.slot))) + spec.compute_slot_epoch(block.slot))) def apply_empty_block(spec, state): diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 4f099be9d1..276fa617bf 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -19,7 +19,7 @@ def build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, st def sign_deposit_data(spec, deposit_data, privkey, state=None): if state is None: # Genesis - domain = spec.bls_domain(spec.DOMAIN_DEPOSIT) + domain = spec.compute_bls_domain(spec.DOMAIN_DEPOSIT) else: domain = spec.get_domain( state, diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index f59da1fc5c..b7a34b5ee2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -9,7 +9,7 @@ def run_process_just_and_fin(spec, state): def get_shards_for_slot(spec, state, slot): - epoch = spec.slot_to_epoch(slot) + epoch = spec.compute_slot_epoch(slot) epoch_start_shard = spec.get_start_shard(state, epoch) committees_per_slot = spec.get_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT @@ -33,7 +33,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support total_balance = spec.get_total_active_balance(state) remaining_balance = total_balance * 2 // 3 - start_slot = spec.epoch_start_slot(epoch) + start_slot = spec.compute_epoch_start_slot(epoch) for slot in range(start_slot, start_slot + spec.SLOTS_PER_EPOCH): for shard in get_shards_for_slot(spec, state, slot): # Check if we already have had sufficient balance. (and undone if we don't want it). @@ -41,7 +41,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support if remaining_balance < 0: return - committee = spec.get_crosslink_committee(state, spec.slot_to_epoch(slot), shard) + committee = spec.get_crosslink_committee(state, spec.compute_slot_epoch(slot), shard) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. @@ -80,7 +80,7 @@ def get_checkpoints(spec, epoch): def put_checkpoints_in_block_roots(spec, state, checkpoints): for c in checkpoints: - state.block_roots[spec.epoch_start_slot(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root + state.block_roots[spec.compute_epoch_start_slot(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root def finalize_on_234(spec, state, epoch, sufficient_support): diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 5ff0ed1efb..cdeee23516 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -43,7 +43,7 @@ def next_epoch_with_attestations(spec, block = build_empty_block_for_next_slot(spec, post_state) if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= spec.epoch_start_slot(spec.get_current_epoch(post_state)): + if slot_to_attest >= spec.compute_epoch_start_slot(spec.get_current_epoch(post_state)): cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) block.body.attestations.append(cur_attestation) From 1ba955b5f0fa1832f582741bcbd946e0a42068d3 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Sun, 30 Jun 2019 21:19:19 +0200 Subject: [PATCH 356/405] Update specs/test_formats/epoch_processing/README.md Co-Authored-By: Danny Ryan --- specs/test_formats/epoch_processing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/test_formats/epoch_processing/README.md b/specs/test_formats/epoch_processing/README.md index 6998184f6d..dbd4ca639f 100644 --- a/specs/test_formats/epoch_processing/README.md +++ b/specs/test_formats/epoch_processing/README.md @@ -25,7 +25,7 @@ Sub-transitions: - `justification_and_finalization` - `crosslinks` -- *`justification_and_finalization` - planned testing extension* +- *`rewards_and_penalties` - planned testing extension* - `registry_updates` - `slashings` - `final_updates` From e0a0f6e1569950a020fa9def1d733f5b8abe9338 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 30 Jun 2019 21:25:58 +0200 Subject: [PATCH 357/405] Makes markdown comply with linter --- scripts/build_spec.py | 3 +- specs/core/0_beacon-chain.md | 14 +++--- specs/core/0_deposit-contract.md | 4 +- specs/core/0_fork-choice.md | 8 +-- specs/core/1_custody-game.md | 25 +++++----- specs/core/1_shard-data-chains.md | 24 ++++----- specs/validator/0_beacon-chain-validator.md | 50 ++++++++++--------- .../validator/0_beacon-node-validator-api.md | 6 +-- 8 files changed, 68 insertions(+), 66 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index fe66a72d57..1542973f29 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -6,7 +6,6 @@ from argparse import ArgumentParser from typing import ( Dict, - List, Optional, ) @@ -106,7 +105,7 @@ def apply_constants_preset(preset: Dict[str, Any]) -> None: def strip_comments(raw: str) -> str: - comment_line_regex = re.compile('^\s+# ') + comment_line_regex = re.compile(r'^\s+# ') lines = raw.split('\n') out = [] for line in lines: diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 68915177c5..7dd6a71bfa 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -181,7 +181,7 @@ The following values are (non-configurable) constants used throughout the specif | `JUSTIFICATION_BITS_LENGTH` | `4` | | `ENDIANNESS` | `'little'` | -* For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) +- For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) ### Gwei values @@ -215,7 +215,7 @@ The following values are (non-configurable) constants used throughout the specif | `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | | `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes | -* `MAX_EPOCHS_PER_CROSSLINK` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH`. +- `MAX_EPOCHS_PER_CROSSLINK` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH`. ### State list lengths @@ -236,7 +236,7 @@ The following values are (non-configurable) constants used throughout the specif | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -* The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. +- The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block @@ -1116,9 +1116,9 @@ def slash_validator(state: BeaconState, Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 block, let `candidate_state = initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)` where: -* `eth1_block_hash` is the hash of the Ethereum 1.0 block -* `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` -* `deposits` is the sequence of all deposits, ordered chronologically, up to the block with hash `eth1_block_hash` +- `eth1_block_hash` is the hash of the Ethereum 1.0 block +- `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` +- `deposits` is the sequence of all deposits, ordered chronologically, up to the block with hash `eth1_block_hash` ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Hash, @@ -1171,7 +1171,7 @@ Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled excpetion (e.g. a failed `assert` or an out-of-range list access) are considered invalid. +The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python def state_transition(state: BeaconState, block: BeaconBlock, validate_state_root: bool=False) -> BeaconState: diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index 0efc15f250..b95748ae6b 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -48,8 +48,8 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac One of the `DepositData` fields is `withdrawal_credentials`. It is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). The first byte of `withdrawal_credentials` is a version number. As of now, the only expected format is as follows: -* `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX_BYTE` -* `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]` where `withdrawal_pubkey` is a BLS pubkey +- `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX_BYTE` +- `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]` where `withdrawal_pubkey` is a BLS pubkey The private key corresponding to `withdrawal_pubkey` will be required to initiate a withdrawal. It can be stored separately until a withdrawal is required, e.g. in cold storage. diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index a45d79fbec..89eecff02d 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -41,9 +41,9 @@ This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0 The head block root associated with a `store` is defined as `get_head(store)`. At genesis, let `store = get_genesis_store(genesis_state)` and update `store` by running: -* `on_tick(time)` whenever `time > store.time` where `time` is the current Unix time -* `on_block(block)` whenever a block `block` is received -* `on_attestation(attestation)` whenever an attestation `attestation` is received +- `on_tick(time)` whenever `time > store.time` where `time` is the current Unix time +- `on_block(block)` whenever a block `block` is received +- `on_attestation(attestation)` whenever an attestation `attestation` is received *Notes*: @@ -188,7 +188,7 @@ def on_attestation(store: Store, attestation: Attestation) -> None: # Cannot calculate the current shuffling if have not seen the target assert target.root in store.blocks - # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrivesr + # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives base_state = store.block_states[target.root].copy() assert store.time >= base_state.genesis_time + epoch_start_slot(target.epoch) * SECONDS_PER_SLOT diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 54a1ca34be..babe4f991e 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -56,18 +56,18 @@ This document details the beacon chain additions and changes in Phase 1 of Ether ## Terminology -* **Custody game**— -* **Custody period**— -* **Custody chunk**— -* **Custody chunk bit**— -* **Custody chunk challenge**— -* **Custody bit**— -* **Custody bit challenge**— -* **Custody key**— -* **Custody key reveal**— -* **Custody key mask**— -* **Custody response**— -* **Custody response deadline**— +- **Custody game**— +- **Custody period**— +- **Custody chunk**— +- **Custody chunk bit**— +- **Custody chunk challenge**— +- **Custody bit**— +- **Custody bit challenge**— +- **Custody key**— +- **Custody key reveal**— +- **Custody key mask**— +- **Custody response**— +- **Custody response deadline**— ## Constants @@ -113,7 +113,6 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | - | - | | `DOMAIN_CUSTODY_BIT_CHALLENGE` | `6` | - ### TODO PLACEHOLDER | Name | Value | diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index fd0e41b4fe..745d437c20 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -282,10 +282,10 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: Let: -* `beacon_blocks` be the `BeaconBlock` list such that `beacon_blocks[slot]` is the canonical `BeaconBlock` at slot `slot` -* `beacon_state` be the canonical `BeaconState` after processing `beacon_blocks[-1]` -* `valid_shard_blocks` be the list of valid `ShardBlock`, recursively defined -* `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block` +- `beacon_blocks` be the `BeaconBlock` list such that `beacon_blocks[slot]` is the canonical `BeaconBlock` at slot `slot` +- `beacon_state` be the canonical `BeaconState` after processing `beacon_blocks[-1]` +- `valid_shard_blocks` be the list of valid `ShardBlock`, recursively defined +- `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block` ```python def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], @@ -349,9 +349,9 @@ def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], Let: -* `valid_shard_blocks` be the list of valid `ShardBlock` -* `beacon_state` be the canonical `BeaconState` -* `candidate` be a candidate `ShardAttestation` for which validity is to be determined by running `is_valid_shard_attestation` +- `valid_shard_blocks` be the list of valid `ShardBlock` +- `beacon_state` be the canonical `BeaconState` +- `candidate` be a candidate `ShardAttestation` for which validity is to be determined by running `is_valid_shard_attestation` ```python def is_valid_shard_attestation(valid_shard_blocks: Sequence[ShardBlock], @@ -376,11 +376,11 @@ def is_valid_shard_attestation(valid_shard_blocks: Sequence[ShardBlock], Let: -* `shard` be a valid `Shard` -* `shard_blocks` be the `ShardBlock` list such that `shard_blocks[slot]` is the canonical `ShardBlock` for shard `shard` at slot `slot` -* `beacon_state` be the canonical `BeaconState` -* `valid_attestations` be the set of valid `Attestation` objects, recursively defined -* `candidate` be a candidate `Attestation` which is valid under Phase 0 rules, and for which validity is to be determined under Phase 1 rules by running `is_valid_beacon_attestation` +- `shard` be a valid `Shard` +- `shard_blocks` be the `ShardBlock` list such that `shard_blocks[slot]` is the canonical `ShardBlock` for shard `shard` at slot `slot` +- `beacon_state` be the canonical `BeaconState` +- `valid_attestations` be the set of valid `Attestation` objects, recursively defined +- `candidate` be a candidate `Attestation` which is valid under Phase 0 rules, and for which validity is to be determined under Phase 1 rules by running `is_valid_beacon_attestation` ```python def is_valid_beacon_attestation(shard: Shard, diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 0b6857a72d..e354345faa 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -86,8 +86,9 @@ Validator public keys are [G1 points](../bls_signature.md#g1-points) on the [BLS A secondary withdrawal private key, `withdrawal_privkey`, must also be securely generated along with the resultant `withdrawal_pubkey`. This `withdrawal_privkey` does not have to be available for signing during the normal lifetime of a validator and can live in "cold storage". The validator constructs their `withdrawal_credentials` via the following: -* Set `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX_BYTE`. -* Set `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]`. + +- Set `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX_BYTE`. +- Set `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]`. ### Submit deposit @@ -95,11 +96,11 @@ In Phase 0, all incoming validator deposits originate from the Ethereum 1.0 proo To submit a deposit: -* Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. -* Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. -* Set `deposit_data.amount = amount`. -* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=bls_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `bls_domain` will default to zeroes there). -* Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. +- Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. +- Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. +- Set `deposit_data.amount = amount`. +- Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=bls_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `bls_domain` will default to zeroes there). +- Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. *Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validators` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. @@ -292,8 +293,8 @@ A validator should create and broadcast the attestation halfway through the `slo First, the validator should construct `attestation_data`, an [`AttestationData`](../core/0_beacon-chain.md#attestationdata) object based upon the state at the assigned slot. -* Let `head_block` be the result of running the fork choice during the assigned slot. -* Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. +- Let `head_block` be the result of running the fork choice during the assigned slot. +- Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. ##### LMD GHOST vote @@ -301,25 +302,26 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. ##### FFG vote -* Set `attestation_data.source_epoch = head_state.current_justified_epoch`. -* Set `attestation_data.source_root = head_state.current_justified_root`. -* Set `attestation_data.target_epoch = get_current_epoch(head_state)` -* Set `attestation_data.target_root = epoch_boundary_block_root` where `epoch_boundary_block_root` is the root of block at the most recent epoch boundary. +- Set `attestation_data.source_epoch = head_state.current_justified_epoch`. +- Set `attestation_data.source_root = head_state.current_justified_root`. +- Set `attestation_data.target_epoch = get_current_epoch(head_state)` +- Set `attestation_data.target_root = epoch_boundary_block_root` where `epoch_boundary_block_root` is the root of block at the most recent epoch boundary. *Note*: `epoch_boundary_block_root` can be looked up in the state using: -* Let `start_slot = epoch_start_slot(get_current_epoch(head_state))`. -* Let `epoch_boundary_block_root = signing_root(head_block) if start_slot == head_state.slot else get_block_root(state, start_slot)`. + +- Let `start_slot = epoch_start_slot(get_current_epoch(head_state))`. +- Let `epoch_boundary_block_root = signing_root(head_block) if start_slot == head_state.slot else get_block_root(state, start_slot)`. ##### Crosslink vote Construct `attestation_data.crosslink` via the following. -* Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee. -* Let `parent_crosslink = head_state.current_crosslinks[shard]`. -* Set `attestation_data.crosslink.start_epoch = parent_crosslink.end_epoch`. -* Set `attestation_data.crosslink.end_epoch = min(attestation_data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK)`. -* Set `attestation_data.crosslink.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. -* Set `attestation_data.crosslink.data_root = ZERO_HASH`. *Note*: This is a stub for Phase 0. +- Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee. +- Let `parent_crosslink = head_state.current_crosslinks[shard]`. +- Set `attestation_data.crosslink.start_epoch = parent_crosslink.end_epoch`. +- Set `attestation_data.crosslink.end_epoch = min(attestation_data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK)`. +- Set `attestation_data.crosslink.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. +- Set `attestation_data.crosslink.data_root = ZERO_HASH`. *Note*: This is a stub for Phase 0. #### Construct attestation @@ -331,13 +333,13 @@ Set `attestation.data = attestation_data` where `attestation_data` is the `Attes ##### Aggregation bits -* Let `attestation.aggregation_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` where the bits at the index in the aggregated validator's `committee` is set to `0b1`. +- Let `attestation.aggregation_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` where the bits at the index in the aggregated validator's `committee` is set to `0b1`. *Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. ##### Custody bits -* Let `attestation.custody_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` filled with zeros of length `len(committee)`. +- Let `attestation.custody_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` filled with zeros of length `len(committee)`. *Note*: This is a stub for Phase 0. @@ -376,6 +378,7 @@ To avoid "proposer slashings", a validator must not sign two conflicting [`Beaco *In Phase 0, as long as the validator does not sign two different beacon blocks for the same epoch, the validator is safe against proposer slashings.* Specifically, when signing a `BeaconBlock`, a validator should perform the following steps in the following order: + 1. Save a record to hard disk that a beacon block has been signed for the `epoch=slot_to_epoch(block.slot)`. 2. Generate and broadcast the block. @@ -386,6 +389,7 @@ If the software crashes at some point within this routine, then when the validat To avoid "attester slashings", a validator must not sign two conflicting [`AttestationData`](../core/0_beacon-chain.md#attestationdata) objects, i.e. two attestations that satisfy [`is_slashable_attestation_data`](../core/0_beacon-chain.md#is_slashable_attestation_data). Specifically, when signing an `Attestation`, a validator should perform the following steps in the following order: + 1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source_epoch`) and target (i.e. `slot_to_epoch(attestation_data.slot)`). 2. Generate and broadcast attestation. diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index 20379eef03..e87d367019 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -8,7 +8,8 @@ This document outlines a minimal application programming interface (API) which i The API is a REST interface, accessed via HTTP, designed for use as a local communications protocol between binaries. Currently, the only supported return data type is JSON. -## Background +## Background + The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum 2.0 network. Conceptually, it does not maintain keypairs that participate with the beacon chain. The validator client is a conceptually separate entity which utilizes private keys to perform validator related tasks, called "duties", on the beacon chain. These duties include the production of beacon blocks and signing of attestations. @@ -19,8 +20,7 @@ The goal of this specification is to promote interoperability between beacon nod This specification is derived from a proposal and discussion on Issues [#1011](https://github.com/ethereum/eth2.0-specs/issues/1011) and [#1012](https://github.com/ethereum/eth2.0-specs/issues/1012). - -## Specification +## Specification The API specification has been written in [OpenAPI 3.0](https://swagger.io/docs/specification/about/) and is provided in the [beacon_node_oapi.yaml](beacon_node_oapi.yaml) file alongside this document. From c2a4838f5af373bf00edf8c3ed4cf56877632e8d Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 30 Jun 2019 21:49:29 +0200 Subject: [PATCH 358/405] Implments suggestions from review --- specs/core/0_beacon-chain.md | 14 +++++++------- test_generators/shuffling/main.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 75721ec1a4..4554c31955 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -66,7 +66,7 @@ - [`is_slashable_attestation_data`](#is_slashable_attestation_data) - [`is_valid_merkle_branch`](#is_valid_merkle_branch) - [Misc](#misc) - - [`compute_shuffle_index`](#compute_shuffle_index) + - [`compute_shuffled_index`](#compute_shuffled_index) - [`compute_committee`](#compute_committee) - [`compute_slot_epoch`](#compute_slot_epoch) - [`compute_epoch_start_slot`](#compute_epoch_start_slot) @@ -662,10 +662,10 @@ def is_valid_merkle_branch(leaf: Hash, branch: Sequence[Hash], depth: uint64, in ### Misc -#### `compute_shuffle_index` +#### `compute_shuffled_index` ```python -def compute_shuffle_index(index: ValidatorIndex, index_count: uint64, seed: Hash) -> ValidatorIndex: +def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Hash) -> ValidatorIndex: """ Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). """ @@ -698,7 +698,7 @@ def compute_committee(indices: Sequence[ValidatorIndex], """ start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return [indices[compute_shuffle_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] + return [indices[compute_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] ``` #### `compute_slot_epoch` @@ -1677,9 +1677,9 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) for new validators. # Note: The deposit contract does not check signatures. - # Note: Deposits are valid across forks. thus the deposit domain is retrieved directly from `compute_bls_domain` - if not bls_verify(pubkey, signing_root(deposit.data), - deposit.data.signature, compute_bls_domain(DOMAIN_DEPOSIT)): + # Note: Deposits are valid across forks, thus the deposit domain is retrieved directly from `compute_bls_domain` + domain = compute_bls_domain(DOMAIN_DEPOSIT) + if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, domain): return # Add validator and balance entries diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index bc7907242c..8e6125e6fd 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -10,7 +10,7 @@ def shuffling_case(seed, count): yield 'seed', '0x' + seed.hex() yield 'count', count - yield 'shuffled', [spec.compute_shuffle_index(i, count, seed) for i in range(count)] + yield 'shuffled', [spec.compute_shuffled_index(i, count, seed) for i in range(count)] @to_tuple From 9b5e18b61c22e3f3356468cc0ba2191d46aba1c3 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 20:51:10 +0100 Subject: [PATCH 359/405] Last minute cleanups --- scripts/build_spec.py | 6 +- specs/core/0_beacon-chain.md | 58 +++++++++---------- specs/core/0_deposit-contract.md | 2 +- specs/validator/0_beacon-chain-validator.md | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 3 +- 5 files changed, 35 insertions(+), 36 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index fe66a72d57..f0c849e85a 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -26,7 +26,7 @@ ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -53,7 +53,7 @@ ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -175,7 +175,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' - 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', + 'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc ] diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 68915177c5..b0857b213d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -161,6 +161,7 @@ The following values are (non-configurable) constants used throughout the specif | `BASE_REWARDS_PER_EPOCH` | `5` | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | | `SECONDS_PER_DAY` | `86400` | +| `JUSTIFICATION_BITS_LENGTH` | `4` | ## Configuration @@ -176,10 +177,9 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | +| `ENDIANNESS` | `'little'` | | `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**16` (= 65,536) | | `MIN_GENESIS_TIME` | `1578009600` (Jan 3, 2020) | -| `JUSTIFICATION_BITS_LENGTH` | `4` | -| `ENDIANNESS` | `'little'` | * For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -202,28 +202,26 @@ The following values are (non-configurable) constants used throughout the specif ### Time parameters -| Name | Value | Unit | Duration | +| Name | Value | Duration | | - | - | :-: | :-: | -| `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 6 seconds | -| `SLOTS_PER_EPOCH` | `2**6` (= 64) | slots | 6.4 minutes | -| `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | -| `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | epochs | 25.6 minutes | -| `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~1.7 hours | -| `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | -| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | -| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | -| `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | -| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes | - -* `MAX_EPOCHS_PER_CROSSLINK` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH`. +| `MIN_ATTESTATION_INCLUSION_DELAY` | `Slot(2**0)` (= 1) | 6 seconds | +| `SLOTS_PER_EPOCH` | `Slot(2**6)` (= 64) | 6.4 minutes | +| `MIN_SEED_LOOKAHEAD` | `Epoch(2**0)` (= 1) | 6.4 minutes | +| `ACTIVATION_EXIT_DELAY` | `Epoch(2**2)` (= 4) | 25.6 minutes | +| `SLOTS_PER_ETH1_VOTING_PERIOD` | `Slot(2**10)` (= 1,024) | ~1.7 hours | +| `SLOTS_PER_HISTORICAL_ROOT` | `Slot(2**13)` (= 8,192) | ~13 hours | +| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `Epoch(2**8)` (= 256) | ~27 hours | +| `PERSISTENT_COMMITTEE_PERIOD` | `Epoch(2**11)` (= 2,048) | 9 days | +| `MAX_EPOCHS_PER_CROSSLINK` | `Epoch(2**6)` (= 64) | ~7 hours | +| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `Epoch(2**2)` (= 4) | 25.6 minutes | ### State list lengths -| Name | Value | Unit | Duration | +| Name | Value | Duration | | - | - | :-: | :-: | -| `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | -| `EPOCHS_PER_SLASHINGS_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | -| `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | +| `EPOCHS_PER_HISTORICAL_VECTOR` | `Epoch(2**16)` (= 65,536) | ~0.8 years | +| `EPOCHS_PER_SLASHINGS_VECTOR` | `Epoch(2**13)` (= 8,192) | ~36 days | +| `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | ~26,131 years | | `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validator spots | | ### Rewards and penalties @@ -541,7 +539,7 @@ class BeaconState(Container): #### `integer_squareroot` ```python -def integer_squareroot(n: uint64) -> int: +def integer_squareroot(n: uint64) -> uint64: """ Return the largest integer ``x`` such that ``x**2 <= n``. """ @@ -564,17 +562,17 @@ def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: ``` ```python -def int_to_bytes(integer: uint64, length: uint64) -> bytes: +def int_to_bytes(n: uint64, length: uint64) -> bytes: """ - Return the ``length``-byte serialization of ``integer``. + Return the ``length``-byte serialization of ``n``. """ - return integer.to_bytes(length, ENDIANNESS) + return n.to_bytes(length, ENDIANNESS) ``` #### `bytes_to_int` ```python -def bytes_to_int(data: bytes) -> int: +def bytes_to_int(data: bytes) -> uint64: """ Return the integer deserialization of ``data``. """ @@ -767,7 +765,7 @@ def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: #### `bls_domain` ```python -def bls_domain(domain_type: uint64, fork_version: bytes=b'\x00' * 4) -> int: +def bls_domain(domain_type: uint64, fork_version: Version=Version()) -> int: """ Return the BLS domain for the ``domain_type`` and ``fork_version``. """ @@ -841,7 +839,7 @@ def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[V #### `get_validator_churn_limit` ```python -def get_validator_churn_limit(state: BeaconState) -> int: +def get_validator_churn_limit(state: BeaconState) -> uint64: """ Return the validator churn limit for the current epoch. """ @@ -865,7 +863,7 @@ def get_seed(state: BeaconState, epoch: Epoch) -> Hash: #### `get_committee_count` ```python -def get_committee_count(state: BeaconState, epoch: Epoch) -> int: +def get_committee_count(state: BeaconState, epoch: Epoch) -> uint64: """ Return the number of committees at ``epoch``. """ @@ -910,7 +908,7 @@ def get_start_shard(state: BeaconState, epoch: Epoch) -> Shard: #### `get_shard_delta` ```python -def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: +def get_shard_delta(state: BeaconState, epoch: Epoch) -> uint64: """ Return the number of shards to increment ``state.start_shard`` at ``epoch``. """ @@ -1675,8 +1673,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: amount = deposit.data.amount validator_pubkeys = [v.pubkey for v in state.validators] if pubkey not in validator_pubkeys: - # Verify the deposit signature (proof of possession) for new validators. - # Note: The deposit contract does not check signatures. + # Verify the deposit signature (proof of possession) for new validators + # Note: The deposit contract does not check signatures # Note: Deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT)): return diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index 0efc15f250..7dee6a9980 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -48,7 +48,7 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac One of the `DepositData` fields is `withdrawal_credentials`. It is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). The first byte of `withdrawal_credentials` is a version number. As of now, the only expected format is as follows: -* `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX_BYTE` +* `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]` where `withdrawal_pubkey` is a BLS pubkey The private key corresponding to `withdrawal_pubkey` will be required to initiate a withdrawal. It can be stored separately until a withdrawal is required, e.g. in cold storage. diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 0b6857a72d..66abe5e2e1 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -86,7 +86,7 @@ Validator public keys are [G1 points](../bls_signature.md#g1-points) on the [BLS A secondary withdrawal private key, `withdrawal_privkey`, must also be securely generated along with the resultant `withdrawal_pubkey`. This `withdrawal_privkey` does not have to be available for signing during the normal lifetime of a validator and can live in "cold storage". The validator constructs their `withdrawal_credentials` via the following: -* Set `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX_BYTE`. +* Set `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX`. * Set `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]`. ### Submit deposit diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index d87a223994..cec43fc6d1 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -495,7 +495,8 @@ def is_fixed_size(cls): return True -# Helpers for common BytesN types. +# Helpers for common BytesN types +Bytes1: BytesType = BytesN[1] Bytes4: BytesType = BytesN[4] Bytes32: BytesType = BytesN[32] Bytes48: BytesType = BytesN[48] From e4a225ba68f08b73a1ccab2ec572cacd467a78ed Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 20:51:44 +0100 Subject: [PATCH 360/405] BLS_WITHDRAWAL_PREFIX a Bytes1 --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b0857b213d..81b02c9ab7 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -198,7 +198,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_SLOT` | `Slot(0)` | | `GENESIS_EPOCH` | `Epoch(0)` | -| `BLS_WITHDRAWAL_PREFIX` | `0` | +| `BLS_WITHDRAWAL_PREFIX` | `Bytes1()` | ### Time parameters From f1931c030fe4d5cb712b6c075d86b199627c8e21 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 20:56:55 +0100 Subject: [PATCH 361/405] Fix --- specs/core/0_beacon-chain.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 81b02c9ab7..bcd0737c37 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1734,10 +1734,7 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: state.balances[transfer.sender] >= transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE ) # Verify that the pubkey is valid - assert ( - state.validators[transfer.sender].withdrawal_credentials == - int_to_bytes(BLS_WITHDRAWAL_PREFIX, length=1) + hash(transfer.pubkey)[1:] - ) + assert state.validators[transfer.sender].withdrawal_credentials == BLS_WITHDRAWAL_PREFIX + hash(transfer.pubkey)[1:] # Verify that the signature is valid assert bls_verify(transfer.pubkey, signing_root(transfer), transfer.signature, get_domain(state, DOMAIN_TRANSFER)) # Process the transfer From fb598bc70c53debaf869c6b6bee11cca0f79a8d5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 04:11:17 +0800 Subject: [PATCH 362/405] Update ToC --- specs/core/0_beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7dd6a71bfa..e10d0c0115 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -49,9 +49,9 @@ - [`BeaconState`](#beaconstate) - [Helper functions](#helper-functions) - [Math](#math) + - [`int_to_bytes`](#int_to_bytes) - [`integer_squareroot`](#integer_squareroot) - [`xor`](#xor) - - [`int_to_bytes`](#int_to_bytes) - [`bytes_to_int`](#bytes_to_int) - [Crypto](#crypto) - [`hash`](#hash) @@ -65,7 +65,7 @@ - [`is_slashable_validator`](#is_slashable_validator) - [`is_slashable_attestation_data`](#is_slashable_attestation_data) - [`is_valid_merkle_branch`](#is_valid_merkle_branch) - - [Misc](#misc) + - [Misc](#misc-1) - [`shuffle_index`](#shuffle_index) - [`compute_committee`](#compute_committee) - [`validate_indexed_attestation`](#validate_indexed_attestation) @@ -90,6 +90,7 @@ - [`get_attestation_data_slot`](#get_attestation_data_slot) - [`get_compact_committees_root`](#get_compact_committees_root) - [`get_total_balance`](#get_total_balance) + - [`get_total_active_balance`](#get_total_active_balance) - [`get_domain`](#get_domain) - [`get_indexed_attestation`](#get_indexed_attestation) - [`get_attesting_indices`](#get_attesting_indices) From e53063c08d5cab0c2304213b9cb57b9fa2218e4c Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 22:12:02 +0200 Subject: [PATCH 363/405] type BLS domain and domain type --- configs/constant_presets/mainnet.yaml | 12 ++++++------ configs/constant_presets/minimal.yaml | 12 ++++++------ scripts/build_spec.py | 12 +++++++++--- specs/bls_signature.md | 10 +++++----- specs/core/0_beacon-chain.md | 12 ++++++++---- specs/core/1_custody-game.md | 4 +++- specs/core/1_shard-data-chains.md | 4 +++- test_libs/pyspec/eth2spec/utils/bls.py | 6 ++++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 5 +++++ 9 files changed, 49 insertions(+), 28 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 629badcee1..269e96b23b 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -117,9 +117,9 @@ MAX_TRANSFERS: 0 # Signature domains # --------------------------------------------------------------- -DOMAIN_BEACON_PROPOSER: 0 -DOMAIN_RANDAO: 1 -DOMAIN_ATTESTATION: 2 -DOMAIN_DEPOSIT: 3 -DOMAIN_VOLUNTARY_EXIT: 4 -DOMAIN_TRANSFER: 5 +DOMAIN_BEACON_PROPOSER: 0x00000000 +DOMAIN_RANDAO: 0x01000000 +DOMAIN_ATTESTATION: 0x02000000 +DOMAIN_DEPOSIT: 0x03000000 +DOMAIN_VOLUNTARY_EXIT: 0x04000000 +DOMAIN_TRANSFER: 0x05000000 diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 1d0ad015db..d0fb988b81 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -118,9 +118,9 @@ MAX_TRANSFERS: 0 # Signature domains # --------------------------------------------------------------- -DOMAIN_BEACON_PROPOSER: 0 -DOMAIN_RANDAO: 1 -DOMAIN_ATTESTATION: 2 -DOMAIN_DEPOSIT: 3 -DOMAIN_VOLUNTARY_EXIT: 4 -DOMAIN_TRANSFER: 5 +DOMAIN_BEACON_PROPOSER: 0x00000000 +DOMAIN_RANDAO: 0x01000000 +DOMAIN_ATTESTATION: 0x02000000 +DOMAIN_DEPOSIT: 0x03000000 +DOMAIN_VOLUNTARY_EXIT: 0x04000000 +DOMAIN_TRANSFER: 0x05000000 \ No newline at end of file diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 1542973f29..d5b0c9aaf6 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -52,7 +52,7 @@ ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -94,7 +94,10 @@ def compute_committee(indices: Sequence[ValidatorIndex], # type: ignore def apply_constants_preset(preset: Dict[str, Any]) -> None: global_vars = globals() for k, v in preset.items(): - global_vars[k] = v + if k.startswith('DOMAIN_'): + global_vars[k] = DomainType(v) # domain types are defined as bytes in the configs + else: + global_vars[k] = v # Deal with derived constants global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) @@ -135,6 +138,9 @@ def objects_to_spec(functions: Dict[str, str], ) ) functions_spec = '\n\n'.join(functions.values()) + for k in list(constants.keys()): + if k.startswith('DOMAIN_'): + constants[k] = f"DomainType(({constants[k]}).to_bytes(length=4, byteorder='little'))" constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants)) ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) ssz_objects_reinitialization_spec = ( diff --git a/specs/bls_signature.md b/specs/bls_signature.md index b901b9345f..652279cd7f 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -71,10 +71,10 @@ We require: G2_cofactor = 305502333931268344200999753193121504214466019254188142667664032982267604182971884026507427359259977847832272839041616661285803823378372096355777062779109 q = 4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787 -def hash_to_G2(message_hash: Bytes32, domain: uint64) -> Tuple[uint384, uint384]: +def hash_to_G2(message_hash: Bytes32, domain: Bytes8) -> Tuple[uint384, uint384]: # Initial candidate x coordinate - x_re = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x01'), 'big') - x_im = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x02'), 'big') + x_re = int.from_bytes(hash(message_hash + domain + b'\x01'), 'big') + x_im = int.from_bytes(hash(message_hash + domain + b'\x02'), 'big') x_coordinate = Fq2([x_re, x_im]) # x = x_re + i * x_im # Test candidate y coordinates until a one is found @@ -130,7 +130,7 @@ g = Fq2([g_x, g_y]) ### `bls_verify` -Let `bls_verify(pubkey: Bytes48, message_hash: Bytes32, signature: Bytes96, domain: uint64) -> bool`: +Let `bls_verify(pubkey: Bytes48, message_hash: Bytes32, signature: Bytes96, domain: Bytes8) -> bool`: * Verify that `pubkey` is a valid G1 point. * Verify that `signature` is a valid G2 point. @@ -138,7 +138,7 @@ Let `bls_verify(pubkey: Bytes48, message_hash: Bytes32, signature: Bytes96, doma ### `bls_verify_multiple` -Let `bls_verify_multiple(pubkeys: List[Bytes48], message_hashes: List[Bytes32], signature: Bytes96, domain: uint64) -> bool`: +Let `bls_verify_multiple(pubkeys: List[Bytes48], message_hashes: List[Bytes32], signature: Bytes96, domain: Bytes8) -> bool`: * Verify that each `pubkey` in `pubkeys` is a valid G1 point. * Verify that `signature` is a valid G2 point. diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7dd6a71bfa..3918fda06e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -147,6 +147,8 @@ We define the following Python custom types for type hinting and readability: | `ValidatorIndex` | `uint64` | a validator registry index | | `Gwei` | `uint64` | an amount in Gwei | | `Version` | `Bytes4` | a fork version number | +| `DomainType` | `Bytes4` | a signature domain type | +| `Domain` | `Bytes8` | a signature domain | | `Hash` | `Bytes32` | a hash | | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | @@ -249,7 +251,9 @@ The following values are (non-configurable) constants used throughout the specif | `MAX_VOLUNTARY_EXITS` | `2**4` (= 16) | | `MAX_TRANSFERS` | `0` | -### Signature domains +### Signature domain types + +The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | @@ -767,11 +771,11 @@ def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: #### `bls_domain` ```python -def bls_domain(domain_type: uint64, fork_version: bytes=b'\x00' * 4) -> int: +def bls_domain(domain_type: DomainType, fork_version: bytes=b'\x00' * 4) -> Domain: """ Return the BLS domain for the ``domain_type`` and ``fork_version``. """ - return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) + return Domain(domain_type + fork_version) ``` ### Beacon state accessors @@ -997,7 +1001,7 @@ def get_total_active_balance(state: BeaconState) -> Gwei: #### `get_domain` ```python -def get_domain(state: BeaconState, domain_type: uint64, message_epoch: Epoch=None) -> int: +def get_domain(state: BeaconState, domain_type: DomainType, message_epoch: Epoch=None) -> Domain: """ Return the signature domain (fork version concatenated with domain type) of a message. """ diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index babe4f991e..cdca249d9c 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -107,7 +107,9 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | - | - | | `EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE` | `2**1` (= 2) | -### Signature domains +### Signature domain types + +The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 745d437c20..e212c97a56 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -62,7 +62,9 @@ This document describes the shard data layer and the shard fork choice rule in P | `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.2 minutes | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | -### Signature domains +### Signature domain types + +The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index 52f1fed632..b687458666 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -23,7 +23,8 @@ def entry(*args, **kw): @only_with_bls(alt_return=True) def bls_verify(pubkey, message_hash, signature, domain): - return bls.verify(message_hash=message_hash, pubkey=pubkey, signature=signature, domain=domain) + return bls.verify(message_hash=message_hash, pubkey=pubkey, + signature=signature, domain=int.from_bytes(domain, byteorder='little')) @only_with_bls(alt_return=True) @@ -43,4 +44,5 @@ def bls_aggregate_signatures(signatures): @only_with_bls(alt_return=STUB_SIGNATURE) def bls_sign(message_hash, privkey, domain): - return bls.sign(message_hash=message_hash, privkey=privkey, domain=domain) + return bls.sign(message_hash=message_hash, privkey=privkey, + domain=int.from_bytes(domain, byteorder='little')) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index d87a223994..7b57985294 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -497,6 +497,11 @@ def is_fixed_size(cls): # Helpers for common BytesN types. Bytes4: BytesType = BytesN[4] +Bytes8: BytesType = BytesN[8] Bytes32: BytesType = BytesN[32] Bytes48: BytesType = BytesN[48] Bytes96: BytesType = BytesN[96] + + +class DomainType(Bytes4): + pass From d45b73389e5d9e4410907fdd9aee990eb862733c Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 30 Jun 2019 22:18:52 +0200 Subject: [PATCH 364/405] update BLS test generator and format to reflect spec change in BLS --- specs/test_formats/bls/msg_hash_g2_compressed.md | 4 ++-- specs/test_formats/bls/msg_hash_g2_uncompressed.md | 2 +- specs/test_formats/bls/sign_msg.md | 2 +- test_generators/bls/main.py | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/specs/test_formats/bls/msg_hash_g2_compressed.md b/specs/test_formats/bls/msg_hash_g2_compressed.md index 44ea8ded77..bbc1b82fef 100644 --- a/specs/test_formats/bls/msg_hash_g2_compressed.md +++ b/specs/test_formats/bls/msg_hash_g2_compressed.md @@ -6,8 +6,8 @@ A BLS compressed-hash to G2. ```yaml input: - message: bytes32, - domain: uint64 -- the BLS domain + message: bytes32 + domain: bytes8 -- the BLS domain output: List[bytes48] -- length of two ``` diff --git a/specs/test_formats/bls/msg_hash_g2_uncompressed.md b/specs/test_formats/bls/msg_hash_g2_uncompressed.md index 847b5f61d4..8337baaab4 100644 --- a/specs/test_formats/bls/msg_hash_g2_uncompressed.md +++ b/specs/test_formats/bls/msg_hash_g2_uncompressed.md @@ -7,7 +7,7 @@ A BLS uncompressed-hash to G2. ```yaml input: message: bytes32 - domain: uint64 -- the BLS domain + domain: bytes8 -- the BLS domain output: List[List[bytes48]] -- 3 lists, each a length of two ``` diff --git a/specs/test_formats/bls/sign_msg.md b/specs/test_formats/bls/sign_msg.md index b17e6246de..8c6b8cdd0f 100644 --- a/specs/test_formats/bls/sign_msg.md +++ b/specs/test_formats/bls/sign_msg.md @@ -8,7 +8,7 @@ Message signing with BLS should produce a signature. input: privkey: bytes32 -- the private key used for signing message: bytes32 -- input message to sign (a hash) - domain: uint64 -- the BLS domain + domain: bytes8 -- the BLS domain output: bytes96 -- expected signature ``` diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py index a792dda9ad..284cf68b0e 100644 --- a/test_generators/bls/main.py +++ b/test_generators/bls/main.py @@ -89,7 +89,7 @@ def case01_message_hash_G2_uncompressed(): yield { 'input': { 'message': '0x' + msg.hex(), - 'domain': domain + 'domain': int_to_hex(domain, byte_length=8) }, 'output': hash_message(msg, domain) } @@ -101,7 +101,7 @@ def case02_message_hash_G2_compressed(): yield { 'input': { 'message': '0x' + msg.hex(), - 'domain': domain + 'domain': int_to_hex(domain, byte_length=8) }, 'output': hash_message_compressed(msg, domain) } @@ -126,7 +126,7 @@ def case04_sign_messages(): 'input': { 'privkey': int_to_hex(privkey), 'message': '0x' + message.hex(), - 'domain': domain + 'domain': int_to_hex(domain, byte_length=8) }, 'output': '0x' + sig.hex() } From c42b26b0c40c22163e55aed3d3fadb46ce6369e4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 15:19:52 -0500 Subject: [PATCH 365/405] add back in active_index_roots --- specs/core/0_beacon-chain.md | 20 ++++++++++++++----- .../pyspec/eth2spec/test/helpers/genesis.py | 3 +++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7dd6a71bfa..82700f8743 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -514,6 +514,7 @@ class BeaconState(Container): # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] + active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Active index digests for light clients compact_committees_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Committee digests for light clients # Slashings slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances @@ -857,9 +858,8 @@ def get_seed(state: BeaconState, epoch: Epoch) -> Hash: Return the seed at ``epoch``. """ mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) # Avoid underflow - active_indices = get_active_validator_indices(state, epoch) - active_indices_root = hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](active_indices)) - return hash(mix + active_indices_root + int_to_bytes(epoch, length=32)) + active_index_root = state.active_index_roots[epoch % EPOCHS_PER_HISTORICAL_VECTOR] + return hash(mix + active_index_root + int_to_bytes(epoch, length=32)) ``` #### `get_committee_count` @@ -1143,9 +1143,13 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Hash, validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate compact_committees_roots + # Populate active_index_roots and compact_committees_roots + genesis_active_index_root = hash_tree_root( + List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) + ) committee_root = get_compact_committees_root(state, GENESIS_EPOCH) for index in range(EPOCHS_PER_HISTORICAL_VECTOR): + state.active_index_roots[index] = genesis_active_index_root state.compact_committees_roots[index] = committee_root return state ``` @@ -1489,7 +1493,13 @@ def process_final_updates(state: BeaconState) -> None: # Update start shard state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT) # Set active index root - committee_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR + index_epoch = Epoch(next_epoch + ACTIVATION_EXIT_DELAY) + index_root_position = index_epoch % EPOCHS_PER_HISTORICAL_VECTOR + state.active_index_roots[index_root_position] = hash_tree_root( + List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, index_epoch)) + ) + # Set committees root + committee_root_position = next_epoch % EPOCHS_PER_HISTORICAL_VECTOR state.compact_committees_roots[committee_root_position] = get_compact_committees_root(state, next_epoch) # Reset slashings state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index a5a816977a..42dc50a6d7 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -43,9 +43,12 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH + genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_LIMIT]( + spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) genesis_compact_committees_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_LIMIT]( spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): + state.active_index_roots[index] = genesis_active_index_root state.compact_committees_roots[index] = genesis_compact_committees_root return state From d329e90aa5daaf73285808807987476d5f1945cc Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 21:27:01 +0100 Subject: [PATCH 366/405] Fix effective balance --- specs/core/0_beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7dd6a71bfa..cd4d099033 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1139,7 +1139,9 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Hash, # Process activations for index, validator in enumerate(state.validators): - if state.balances[index] >= MAX_EFFECTIVE_BALANCE: + balance = state.balances[index] + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + if validator.effective_balance == MAX_EFFECTIVE_BALANCE: validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH @@ -1689,7 +1691,6 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) )) state.balances.append(amount) else: From 67b0b4ad333484e8f5a3948f8122d5b4c2136497 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 04:57:42 +0800 Subject: [PATCH 367/405] Update other spec files --- specs/core/0_fork-choice.md | 2 +- specs/core/1_custody-game.md | 2 ++ specs/core/1_shard-data-chains.md | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 89eecff02d..d28087a1ae 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -12,7 +12,7 @@ - [Time parameters](#time-parameters) - [Fork choice](#fork-choice) - [Helpers](#helpers) - - [`Checkpoint`](#checkpoint) + - [`LatestMessage`](#latestmessage) - [`Store`](#store) - [`get_genesis_store`](#get_genesis_store) - [`get_ancestor`](#get_ancestor) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index babe4f991e..5c05ae2fc1 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -16,6 +16,7 @@ - [Max operations per block](#max-operations-per-block) - [Reward and penalty quotients](#reward-and-penalty-quotients) - [Signature domains](#signature-domains) + - [TODO PLACEHOLDER](#todo-placeholder) - [Data structures](#data-structures) - [Custody objects](#custody-objects) - [`CustodyChunkChallenge`](#custodychunkchallenge) @@ -33,6 +34,7 @@ - [Helpers](#helpers) - [`ceillog2`](#ceillog2) - [`get_crosslink_chunk_count`](#get_crosslink_chunk_count) + - [`get_bit`](#get_bit) - [`get_custody_chunk_bit`](#get_custody_chunk_bit) - [`get_chunk_bits_root`](#get_chunk_bits_root) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 745d437c20..093e381cc7 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -14,6 +14,7 @@ - [Initial values](#initial-values) - [Time parameters](#time-parameters) - [Signature domains](#signature-domains) + - [TODO PLACEHOLDER](#todo-placeholder) - [Data structures](#data-structures) - [`ShardBlockBody`](#shardblockbody) - [`ShardAttestation`](#shardattestation) From fe68a8d1f0febab6524e232a50cac136b5b02af9 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 30 Jun 2019 22:59:12 +0200 Subject: [PATCH 368/405] Minor python style tweaks --- specs/core/0_beacon-chain.md | 10 +++++----- specs/core/0_fork-choice.md | 4 ++-- specs/core/1_shard-data-chains.md | 8 ++++---- specs/light_client/sync_protocol.md | 4 ++-- specs/validator/0_beacon-chain-validator.md | 7 +++---- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 4 ++-- 6 files changed, 18 insertions(+), 19 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7dd6a71bfa..9bb562bb6e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1438,8 +1438,8 @@ def process_registry_updates(state: BeaconState) -> None: # Process activation eligibility and ejections for index, validator in enumerate(state.validators): if ( - validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and - validator.effective_balance == MAX_EFFECTIVE_BALANCE + validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH + and validator.effective_balance == MAX_EFFECTIVE_BALANCE ): validator.activation_eligibility_epoch = get_current_epoch(state) @@ -1448,9 +1448,9 @@ def process_registry_updates(state: BeaconState) -> None: # Queue validators eligible for activation and not dequeued for activation prior to finalized epoch activation_queue = sorted([ - index for index, validator in enumerate(state.validators) if - validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH and - validator.activation_epoch >= compute_activation_exit_epoch(state.finalized_checkpoint.epoch) + index for index, validator in enumerate(state.validators) + if validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH + and validator.activation_epoch >= compute_activation_exit_epoch(state.finalized_checkpoint.epoch) ], key=lambda index: state.validators[index].activation_eligibility_epoch) # Dequeued validators for activation up to churn limit (without resetting activation epoch) for index in activation_queue[:get_validator_churn_limit(state)]: diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 89eecff02d..40ca305dc7 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -113,8 +113,8 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices - if (i in store.latest_messages and - get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) + if (i in store.latest_messages + and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) )) ``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 745d437c20..05f95e16a2 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -181,8 +181,8 @@ def get_persistent_committee(state: BeaconState, # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated return sorted(list(set( - [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(state, epoch, i)] + - [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(state, epoch, i)] + [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(state, epoch, i)] + + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(state, epoch, i)] ))) ``` @@ -398,8 +398,8 @@ def is_valid_beacon_attestation(shard: Shard, assert candidate.data.previous_crosslink.data_root == Hash() else: previous_attestation = next( - (attestation for attestation in valid_attestations if - attestation.data.crosslink.data_root == candidate.data.previous_crosslink.data_root), + (attestation for attestation in valid_attestations + if attestation.data.crosslink.data_root == candidate.data.previous_crosslink.data_root), None, ) assert previous_attestation is not None diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 2b1703f217..15f13b15b5 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -153,8 +153,8 @@ def compute_committee(header: BeaconBlockHeader, # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated return sorted(list(set( - [i for i in actual_earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(i)] + - [i for i in actual_later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] + [i for i in actual_earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(i)] + + [i for i in actual_later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] ))) ``` diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index e354345faa..2626b50aa0 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -132,10 +132,9 @@ Once a validator is activated, the validator is assigned [responsibilities](#bea A validator can get committee assignments for a given epoch using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `epoch <= next_epoch`. ```python -def get_committee_assignment( - state: BeaconState, - epoch: Epoch, - validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]: +def get_committee_assignment(state: BeaconState, + epoch: Epoch, + validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]: """ Return the committee assignment in the ``epoch`` for ``validator_index``. ``assignment`` returned is a tuple of the following form: diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index b6f0ecba2b..886f9bf6ab 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -188,8 +188,8 @@ def test_attester_slashing(spec, state): pre_state = deepcopy(state) attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - validator_index = (attester_slashing.attestation_1.custody_bit_0_indices + - attester_slashing.attestation_1.custody_bit_1_indices)[0] + validator_index = (attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices)[0] assert not state.validators[validator_index].slashed From 8b10ed598e9ce99d16181fcf3d7cff1f675fe678 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 30 Jun 2019 23:14:16 +0200 Subject: [PATCH 369/405] Adds a lone space --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 05f95e16a2..6213169d70 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -399,7 +399,7 @@ def is_valid_beacon_attestation(shard: Shard, else: previous_attestation = next( (attestation for attestation in valid_attestations - if attestation.data.crosslink.data_root == candidate.data.previous_crosslink.data_root), + if attestation.data.crosslink.data_root == candidate.data.previous_crosslink.data_root), None, ) assert previous_attestation is not None From 918192cdaba57868593b88c08d1f0b7ae9fe8307 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 30 Jun 2019 23:35:07 +0200 Subject: [PATCH 370/405] `compute_slot_epoch`->`compute_epoch_of_slot` --- scripts/build_spec.py | 2 +- specs/core/0_beacon-chain.md | 23 ++++++++++--------- specs/core/0_fork-choice.md | 8 +++---- specs/core/1_custody-game.md | 5 ++-- specs/core/1_shard-data-chains.md | 10 ++++---- specs/light_client/sync_protocol.md | 4 ++-- specs/validator/0_beacon-chain-validator.md | 16 ++++++------- .../eth2spec/test/helpers/attestations.py | 10 ++++---- .../pyspec/eth2spec/test/helpers/block.py | 8 +++---- ..._process_justification_and_finalization.py | 8 +++---- .../pyspec/eth2spec/test/test_finality.py | 2 +- 11 files changed, 49 insertions(+), 47 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index c9b421409f..43d8b51498 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -97,7 +97,7 @@ def apply_constants_preset(preset: Dict[str, Any]) -> None: global_vars[k] = v # Deal with derived constants - global_vars['GENESIS_EPOCH'] = compute_slot_epoch(GENESIS_SLOT) + global_vars['GENESIS_EPOCH'] = compute_epoch_of_slot(GENESIS_SLOT) # Initialize SSZ types again, to account for changed lengths init_SSZ_types() diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 600a246b4f..bdbd96f858 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -68,8 +68,8 @@ - [Misc](#misc) - [`compute_shuffled_index`](#compute_shuffled_index) - [`compute_committee`](#compute_committee) - - [`compute_slot_epoch`](#compute_slot_epoch) - - [`compute_epoch_start_slot`](#compute_epoch_start_slot) + - [`compute_epoch_of_slot`](#compute_epoch_of_slot) + - [`compute_start_slot_of_epoch`](#compute_start_slot_of_epoch) - [`compute_activation_exit_epoch`](#compute_activation_exit_epoch) - [`compute_bls_domain`](#compute_bls_domain) - [`validate_indexed_attestation`](#validate_indexed_attestation) @@ -701,20 +701,20 @@ def compute_committee(indices: Sequence[ValidatorIndex], return [indices[compute_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] ``` -#### `compute_slot_epoch` +#### `compute_epoch_of_slot` ```python -def compute_slot_epoch(slot: Slot) -> Epoch: +def compute_epoch_of_slot(slot: Slot) -> Epoch: """ Return the epoch number of ``slot``. """ return Epoch(slot // SLOTS_PER_EPOCH) ``` -#### `compute_epoch_start_slot` +#### `compute_start_slot_of_epoch` ```python -def compute_epoch_start_slot(epoch: Epoch) -> Slot: +def compute_start_slot_of_epoch(epoch: Epoch) -> Slot: """ Return the start slot of ``epoch``. """ @@ -783,7 +783,7 @@ def get_current_epoch(state: BeaconState) -> Epoch: """ Return the current epoch. """ - return compute_slot_epoch(state.slot) + return compute_epoch_of_slot(state.slot) ``` #### `get_previous_epoch` @@ -804,7 +804,7 @@ def get_block_root(state: BeaconState, epoch: Epoch) -> Hash: """ Return the block root at the start of a recent ``epoch``. """ - return get_block_root_at_slot(state, compute_epoch_start_slot(epoch)) + return get_block_root_at_slot(state, compute_start_slot_of_epoch(epoch)) ``` #### `get_block_root_at_slot` @@ -950,7 +950,7 @@ def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot """ committee_count = get_committee_count(state, data.target.epoch) offset = (data.crosslink.shard + SHARD_COUNT - get_start_shard(state, data.target.epoch)) % SHARD_COUNT - return Slot(compute_epoch_start_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) + return Slot(compute_start_slot_of_epoch(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` #### `get_compact_committees_root` @@ -1585,14 +1585,15 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: proposer = state.validators[proposer_slashing.proposer_index] # Verify that the epoch is the same - assert compute_slot_epoch(proposer_slashing.header_1.slot) == compute_slot_epoch(proposer_slashing.header_2.slot) + assert (compute_epoch_of_slot(proposer_slashing.header_1.slot) + == compute_epoch_of_slot(proposer_slashing.header_2.slot)) # But the headers are different assert proposer_slashing.header_1 != proposer_slashing.header_2 # Check proposer is slashable assert is_slashable_validator(proposer, get_current_epoch(state)) # Signatures are valid for header in (proposer_slashing.header_1, proposer_slashing.header_2): - domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_slot_epoch(header.slot)) + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_of_slot(header.slot)) assert bls_verify(proposer.pubkey, signing_root(header), header.signature, domain) slash_validator(state, proposer_slashing.proposer_index) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 28ff93e1c7..ef07619408 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -124,7 +124,7 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice head = store.justified_checkpoint.root - justified_slot = compute_epoch_start_slot(store.justified_checkpoint.epoch) + justified_slot = compute_start_slot_of_epoch(store.justified_checkpoint.epoch) while True: children = [ root for root in store.blocks.keys() @@ -162,7 +162,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.finalized_checkpoint.root ) # Check that block is later than the finalized epoch slot - assert block.slot > compute_epoch_start_slot(store.finalized_checkpoint.epoch) + assert block.slot > compute_start_slot_of_epoch(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state for this block to the store @@ -190,11 +190,11 @@ def on_attestation(store: Store, attestation: Attestation) -> None: # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives base_state = store.block_states[target.root].copy() - assert store.time >= base_state.genesis_time + compute_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT + assert store.time >= base_state.genesis_time + compute_start_slot_of_epoch(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - process_slots(base_state, compute_epoch_start_slot(target.epoch)) + process_slots(base_state, compute_start_slot_of_epoch(target.epoch)) store.checkpoint_states[target] = base_state target_state = store.checkpoint_states[target] diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index b79a203f8f..69931af04b 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -474,7 +474,8 @@ def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge # Verify the attestation validate_indexed_attestation(state, get_indexed_attestation(state, challenge.attestation)) # Verify it is not too late to challenge - assert compute_slot_epoch(challenge.attestation.data.slot) >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY + assert (compute_epoch_of_slot(challenge.attestation.data.slot) + >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY) responder = state.validators[challenge.responder_index] assert responder.exit_epoch >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY # Verify the responder participated in the attestation @@ -515,7 +516,7 @@ For each `challenge` in `block.body.custody_bit_challenges`, run the following f ```python def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> None: attestation = challenge.attestation - epoch = compute_slot_epoch(attestation.data.slot) + epoch = compute_epoch_of_slot(attestation.data.slot) shard = attestation.data.crosslink.shard # Verify challenge signature diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index e0f3648bec..1358a860f2 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -163,7 +163,7 @@ def get_persistent_committee(state: BeaconState, """ Return the persistent committee for the given ``shard`` at the given ``slot``. """ - epoch = compute_slot_epoch(slot) + epoch = compute_epoch_of_slot(slot) earlier_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2) later_start_epoch = Epoch(epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD) @@ -240,7 +240,7 @@ def verify_shard_attestation_signature(state: BeaconState, pubkey=bls_aggregate_pubkeys(pubkeys), message_hash=data.shard_block_root, signature=attestation.aggregate_signature, - domain=get_domain(state, DOMAIN_SHARD_ATTESTER, compute_slot_epoch(data.slot)) + domain=get_domain(state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_slot(data.slot)) ) ``` @@ -339,7 +339,7 @@ def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], pubkey=beacon_state.validators[proposer_index].pubkey, message_hash=signing_root(candidate), signature=candidate.signature, - domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_slot_epoch(candidate.slot)), + domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_slot(candidate.slot)), ) return True @@ -403,11 +403,11 @@ def is_valid_beacon_attestation(shard: Shard, None, ) assert previous_attestation is not None - assert candidate.data.previous_attestation.epoch < compute_slot_epoch(candidate.data.slot) + assert candidate.data.previous_attestation.epoch < compute_epoch_of_slot(candidate.data.slot) # Check crosslink data root start_epoch = beacon_state.crosslinks[shard].epoch - end_epoch = min(compute_slot_epoch(candidate.data.slot) - CROSSLINK_LOOKBACK, + end_epoch = min(compute_epoch_of_slot(candidate.data.slot) - CROSSLINK_LOOKBACK, start_epoch + MAX_EPOCHS_PER_CROSSLINK) blocks = [] for slot in range(start_epoch * SLOTS_PER_EPOCH, end_epoch * SLOTS_PER_EPOCH): diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index f9e347d807..41cd9dcb69 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -124,7 +124,7 @@ def compute_committee(header: BeaconBlockHeader, maximal_later_committee = validator_memory.later_period_data.committee earlier_start_epoch = get_earlier_start_epoch(header.slot) later_start_epoch = get_later_start_epoch(header.slot) - epoch = compute_slot_epoch(header.slot) + epoch = compute_epoch_of_slot(header.slot) committee_count = max( earlier_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), @@ -192,7 +192,7 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val pubkey=group_public_key, message_hash=hash_tree_root(shard_parent_block), signature=proof.shard_aggregate_signature, - domain=get_domain(state, compute_slot_epoch(shard_block.slot), DOMAIN_SHARD_ATTESTER), + domain=get_domain(state, compute_epoch_of_slot(shard_block.slot), DOMAIN_SHARD_ATTESTER), ) ``` diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 15eb4255f0..422ea089cb 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -147,7 +147,7 @@ def get_committee_assignment( assert epoch <= next_epoch committees_per_slot = get_committee_count(state, epoch) // SLOTS_PER_EPOCH - start_slot = compute_epoch_start_slot(epoch) + start_slot = compute_start_slot_of_epoch(epoch) for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) slot_start_shard = (get_start_shard(state, epoch) + offset) % SHARD_COUNT @@ -211,10 +211,10 @@ Set `block.randao_reveal = epoch_signature` where `epoch_signature` is defined a ```python epoch_signature = bls_sign( privkey=validator.privkey, # privkey stored locally, not in state - message_hash=hash_tree_root(compute_slot_epoch(block.slot)), + message_hash=hash_tree_root(compute_epoch_of_slot(block.slot)), domain=get_domain( fork=fork, # `fork` is the fork object at the slot `block.slot` - epoch=compute_slot_epoch(block.slot), + epoch=compute_epoch_of_slot(block.slot), domain_type=DOMAIN_RANDAO, ) ) @@ -253,7 +253,7 @@ block_signature = bls_sign( message_hash=signing_root(block), domain=get_domain( fork=fork, # `fork` is the fork object at the slot `block.slot` - epoch=compute_slot_epoch(block.slot), + epoch=compute_epoch_of_slot(block.slot), domain_type=DOMAIN_BEACON_BLOCK, ) ) @@ -309,7 +309,7 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. *Note*: `epoch_boundary_block_root` can be looked up in the state using: -- Let `start_slot = compute_epoch_start_slot(get_current_epoch(head_state))`. +- Let `start_slot = compute_start_slot_of_epoch(get_current_epoch(head_state))`. - Let `epoch_boundary_block_root = signing_root(head_block) if start_slot == head_state.slot else get_block_root(state, start_slot)`. ##### Crosslink vote @@ -359,7 +359,7 @@ signed_attestation_data = bls_sign( message_hash=attestation_message, domain=get_domain( fork=fork, # `fork` is the fork object at the slot, `attestation_data.slot` - epoch=compute_slot_epoch(attestation_data.slot), + epoch=compute_epoch_of_slot(attestation_data.slot), domain_type=DOMAIN_ATTESTATION, ) ) @@ -379,7 +379,7 @@ To avoid "proposer slashings", a validator must not sign two conflicting [`Beaco Specifically, when signing a `BeaconBlock`, a validator should perform the following steps in the following order: -1. Save a record to hard disk that a beacon block has been signed for the `epoch=compute_slot_epoch(block.slot)`. +1. Save a record to hard disk that a beacon block has been signed for the `epoch=compute_epoch_of_slot(block.slot)`. 2. Generate and broadcast the block. If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast block and can effectively avoid slashing. @@ -390,7 +390,7 @@ To avoid "attester slashings", a validator must not sign two conflicting [`Attes Specifically, when signing an `Attestation`, a validator should perform the following steps in the following order: -1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source_epoch`) and target (i.e. `compute_slot_epoch(attestation_data.slot)`). +1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source_epoch`) and target (i.e. `compute_epoch_of_slot(attestation_data.slot)`). 2. Generate and broadcast attestation. If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast attestation and can effectively avoid slashing. diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index ba2bd3ca00..8685170180 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -15,7 +15,7 @@ def build_attestation_data(spec, state, slot, shard): else: block_root = spec.get_block_root_at_slot(state, slot) - current_epoch_start_slot = spec.compute_epoch_start_slot(spec.get_current_epoch(state)) + current_epoch_start_slot = spec.compute_start_slot_of_epoch(spec.get_current_epoch(state)) if slot < current_epoch_start_slot: epoch_boundary_root = spec.get_block_root(state, spec.get_previous_epoch(state)) elif slot == current_epoch_start_slot: @@ -30,7 +30,7 @@ def build_attestation_data(spec, state, slot, shard): source_epoch = state.current_justified_checkpoint.epoch source_root = state.current_justified_checkpoint.root - if spec.compute_slot_epoch(slot) == spec.get_current_epoch(state): + if spec.compute_epoch_of_slot(slot) == spec.get_current_epoch(state): parent_crosslink = state.current_crosslinks[shard] else: parent_crosslink = state.previous_crosslinks[shard] @@ -38,11 +38,11 @@ def build_attestation_data(spec, state, slot, shard): return spec.AttestationData( beacon_block_root=block_root, source=spec.Checkpoint(epoch=source_epoch, root=source_root), - target=spec.Checkpoint(epoch=spec.compute_slot_epoch(slot), root=epoch_boundary_root), + target=spec.Checkpoint(epoch=spec.compute_epoch_of_slot(slot), root=epoch_boundary_root), crosslink=spec.Crosslink( shard=shard, start_epoch=parent_crosslink.end_epoch, - end_epoch=min(spec.compute_slot_epoch(slot), parent_crosslink.end_epoch + spec.MAX_EPOCHS_PER_CROSSLINK), + end_epoch=min(spec.compute_epoch_of_slot(slot), parent_crosslink.end_epoch + spec.MAX_EPOCHS_PER_CROSSLINK), data_root=spec.Hash(), parent_root=hash_tree_root(parent_crosslink), ), @@ -53,7 +53,7 @@ def get_valid_attestation(spec, state, slot=None, signed=False): if slot is None: slot = state.slot - epoch = spec.compute_slot_epoch(slot) + epoch = spec.compute_epoch_of_slot(slot) epoch_start_shard = spec.get_start_shard(state, epoch) committees_per_slot = spec.get_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 830a1e9ec3..2682a0c82a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -14,7 +14,7 @@ def sign_block(spec, state, block, proposer_index=None): if block.slot == state.slot: proposer_index = spec.get_beacon_proposer_index(state) else: - if spec.compute_slot_epoch(state.slot) + 1 > spec.compute_slot_epoch(block.slot): + if spec.compute_epoch_of_slot(state.slot) + 1 > spec.compute_epoch_of_slot(block.slot): print("warning: block slot far away, and no proposer index manually given." " Signing block is slow due to transition for proposer index calculation.") # use stub state to get proposer index of future slot @@ -26,10 +26,10 @@ def sign_block(spec, state, block, proposer_index=None): block.body.randao_reveal = bls_sign( privkey=privkey, - message_hash=hash_tree_root(spec.compute_slot_epoch(block.slot)), + message_hash=hash_tree_root(spec.compute_epoch_of_slot(block.slot)), domain=spec.get_domain( state, - message_epoch=spec.compute_slot_epoch(block.slot), + message_epoch=spec.compute_epoch_of_slot(block.slot), domain_type=spec.DOMAIN_RANDAO, ) ) @@ -39,7 +39,7 @@ def sign_block(spec, state, block, proposer_index=None): domain=spec.get_domain( state, spec.DOMAIN_BEACON_PROPOSER, - spec.compute_slot_epoch(block.slot))) + spec.compute_epoch_of_slot(block.slot))) def apply_empty_block(spec, state): diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index b7a34b5ee2..7dcdb42a4f 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -9,7 +9,7 @@ def run_process_just_and_fin(spec, state): def get_shards_for_slot(spec, state, slot): - epoch = spec.compute_slot_epoch(slot) + epoch = spec.compute_epoch_of_slot(slot) epoch_start_shard = spec.get_start_shard(state, epoch) committees_per_slot = spec.get_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT @@ -33,7 +33,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support total_balance = spec.get_total_active_balance(state) remaining_balance = total_balance * 2 // 3 - start_slot = spec.compute_epoch_start_slot(epoch) + start_slot = spec.compute_start_slot_of_epoch(epoch) for slot in range(start_slot, start_slot + spec.SLOTS_PER_EPOCH): for shard in get_shards_for_slot(spec, state, slot): # Check if we already have had sufficient balance. (and undone if we don't want it). @@ -41,7 +41,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support if remaining_balance < 0: return - committee = spec.get_crosslink_committee(state, spec.compute_slot_epoch(slot), shard) + committee = spec.get_crosslink_committee(state, spec.compute_epoch_of_slot(slot), shard) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. @@ -80,7 +80,7 @@ def get_checkpoints(spec, epoch): def put_checkpoints_in_block_roots(spec, state, checkpoints): for c in checkpoints: - state.block_roots[spec.compute_epoch_start_slot(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root + state.block_roots[spec.compute_start_slot_of_epoch(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root def finalize_on_234(spec, state, epoch, sufficient_support): diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index cdeee23516..7d0e5d3db8 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -43,7 +43,7 @@ def next_epoch_with_attestations(spec, block = build_empty_block_for_next_slot(spec, post_state) if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= spec.compute_epoch_start_slot(spec.get_current_epoch(post_state)): + if slot_to_attest >= spec.compute_start_slot_of_epoch(spec.get_current_epoch(post_state)): cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) block.body.attestations.append(cur_attestation) From cf59c303f022a01db5ffe7e2149259a7541e455f Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 22:50:01 +0100 Subject: [PATCH 371/405] Remove index_count --- specs/core/0_beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index bcd0737c37..8a90bc2f4b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -668,7 +668,6 @@ def shuffle_index(index: ValidatorIndex, index_count: uint64, seed: Hash) -> Val Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). """ assert index < index_count - assert index_count <= 2**40 # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) # See the 'generalized domain' algorithm on page 3 From 9df17f54941c10baca7599270aa034565705eb53 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 22:58:02 +0100 Subject: [PATCH 372/405] Explitcit zero byte --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8a90bc2f4b..71566baab2 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -198,7 +198,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_SLOT` | `Slot(0)` | | `GENESIS_EPOCH` | `Epoch(0)` | -| `BLS_WITHDRAWAL_PREFIX` | `Bytes1()` | +| `BLS_WITHDRAWAL_PREFIX` | `Bytes1(b'\x00')` | ### Time parameters From bde73b59fc89fad461fba55e7b3571d68a1bac98 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Jul 2019 00:00:54 +0200 Subject: [PATCH 373/405] fix bls_verify_multiple --- test_libs/pyspec/eth2spec/utils/bls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index b687458666..ab2327f438 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -29,7 +29,8 @@ def bls_verify(pubkey, message_hash, signature, domain): @only_with_bls(alt_return=True) def bls_verify_multiple(pubkeys, message_hashes, signature, domain): - return bls.verify_multiple(pubkeys, message_hashes, signature, domain) + return bls.verify_multiple(pubkeys=pubkeys, message_hashes=message_hashes, + signature=signature, domain=int.from_bytes(domain, byteorder='little')) @only_with_bls(alt_return=STUB_PUBKEY) From 7f2eb813cbcbfa10e3e2f47b205a73524dfbfb2e Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Jul 2019 00:05:40 +0200 Subject: [PATCH 374/405] minor BLS domain type related improvements --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3918fda06e..433b3d3d40 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -146,10 +146,10 @@ We define the following Python custom types for type hinting and readability: | `Shard` | `uint64` | a shard number | | `ValidatorIndex` | `uint64` | a validator registry index | | `Gwei` | `uint64` | an amount in Gwei | +| `Hash` | `Bytes32` | a hash | | `Version` | `Bytes4` | a fork version number | | `DomainType` | `Bytes4` | a signature domain type | | `Domain` | `Bytes8` | a signature domain | -| `Hash` | `Bytes32` | a hash | | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | @@ -771,7 +771,7 @@ def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: #### `bls_domain` ```python -def bls_domain(domain_type: DomainType, fork_version: bytes=b'\x00' * 4) -> Domain: +def bls_domain(domain_type: DomainType, fork_version: Version=Version()) -> Domain: """ Return the BLS domain for the ``domain_type`` and ``fork_version``. """ From 41897e779a33ae66f505a4d3f076e57004165495 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 1 Jul 2019 00:10:28 +0200 Subject: [PATCH 375/405] validate_indexed_attestation -> is_valid_indexed_attestation` --- specs/core/0_beacon-chain.md | 80 +++++++++++++++++++----------------- specs/core/0_fork-choice.md | 2 +- specs/core/1_custody-game.md | 4 +- 3 files changed, 46 insertions(+), 40 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index dcc3685f99..7104b78237 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -64,6 +64,7 @@ - [`is_active_validator`](#is_active_validator) - [`is_slashable_validator`](#is_slashable_validator) - [`is_slashable_attestation_data`](#is_slashable_attestation_data) + - [`is_valid_indexed_attestation`](#is_valid_indexed_attestation) - [`is_valid_merkle_branch`](#is_valid_merkle_branch) - [Misc](#misc-1) - [`compute_shuffled_index`](#compute_shuffled_index) @@ -72,7 +73,6 @@ - [`compute_start_slot_of_epoch`](#compute_start_slot_of_epoch) - [`compute_activation_exit_epoch`](#compute_activation_exit_epoch) - [`compute_bls_domain`](#compute_bls_domain) - - [`validate_indexed_attestation`](#validate_indexed_attestation) - [Beacon state accessors](#beacon-state-accessors) - [`get_current_epoch`](#get_current_epoch) - [`get_previous_epoch`](#get_previous_epoch) @@ -645,6 +645,45 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa ) ``` +#### `is_valid_indexed_attestation` + +```python +def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: + """ + Verify validity of ``indexed_attestation``. + """ + bit_0_indices = indexed_attestation.custody_bit_0_indices + bit_1_indices = indexed_attestation.custody_bit_1_indices + + # Verify no index has custody bit equal to 1 [to be removed in phase 1] + if not len(bit_1_indices) == 0: + return False + # Verify max number of indices + if not len(bit_0_indices) + len(bit_1_indices) <= MAX_VALIDATORS_PER_COMMITTEE: + return False + # Verify index sets are disjoint + if not len(set(bit_0_indices).intersection(bit_1_indices)) == 0: + return False + # Verify indices are sorted + if not bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices): + return False + # Verify aggregate signature + if not bls_verify_multiple( + pubkeys=[ + bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_0_indices]), + bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_1_indices]), + ], + message_hashes=[ + hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), + hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), + ], + signature=indexed_attestation.signature, + domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target.epoch), + ): + return False + return True +``` + #### `is_valid_merkle_branch` ```python @@ -742,39 +781,6 @@ def compute_bls_domain(domain_type: uint64, fork_version: bytes=b'\x00' * 4) -> return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) ``` -#### `validate_indexed_attestation` - -```python -def validate_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> None: - """ - Verify validity of ``indexed_attestation``. - """ - bit_0_indices = indexed_attestation.custody_bit_0_indices - bit_1_indices = indexed_attestation.custody_bit_1_indices - - # Verify no index has custody bit equal to 1 [to be removed in phase 1] - assert len(bit_1_indices) == 0 - # Verify max number of indices - assert len(bit_0_indices) + len(bit_1_indices) <= MAX_VALIDATORS_PER_COMMITTEE - # Verify index sets are disjoint - assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 - # Verify indices are sorted - assert bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices) - # Verify aggregate signature - assert bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_0_indices]), - bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_1_indices]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), - ], - signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target.epoch), - ) -``` - ### Beacon state accessors #### `get_current_epoch` @@ -1607,8 +1613,8 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla attestation_1 = attester_slashing.attestation_1 attestation_2 = attester_slashing.attestation_2 assert is_slashable_attestation_data(attestation_1.data, attestation_2.data) - validate_indexed_attestation(state, attestation_1) - validate_indexed_attestation(state, attestation_2) + assert is_valid_indexed_attestation(state, attestation_1) + assert is_valid_indexed_attestation(state, attestation_2) slashed_any = False attesting_indices_1 = attestation_1.custody_bit_0_indices + attestation_1.custody_bit_1_indices @@ -1654,7 +1660,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.crosslink.data_root == Hash() # [to be removed in phase 1] # Check signature - validate_indexed_attestation(state, get_indexed_attestation(state, attestation)) + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) ``` ##### Deposits diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 82defa9b11..5ffc0e0250 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -205,7 +205,7 @@ def on_attestation(store: Store, attestation: Attestation) -> None: # Get state at the `target` to validate attestation and calculate the committees indexed_attestation = get_indexed_attestation(target_state, attestation) - validate_indexed_attestation(target_state, indexed_attestation) + assert is_valid_indexed_attestation(target_state, indexed_attestation) # Update latest messages for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 8634970768..dab24d91c5 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -474,7 +474,7 @@ For each `challenge` in `block.body.custody_chunk_challenges`, run the following ```python def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: # Verify the attestation - validate_indexed_attestation(state, get_indexed_attestation(state, challenge.attestation)) + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, challenge.attestation)) # Verify it is not too late to challenge assert (compute_epoch_of_slot(challenge.attestation.data.slot) >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY) @@ -528,7 +528,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> # Verify challenger is slashable assert is_slashable_validator(challenger, get_current_epoch(state)) # Verify attestation - validate_indexed_attestation(state, get_indexed_attestation(state, attestation)) + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) # Verify attestation is eligible for challenging responder = state.validators[challenge.responder_index] assert epoch + responder.max_reveal_lateness <= get_reveal_period(state, challenge.responder_index) From 5398281a1fdc084f2851de3bb4a1f872e85471b3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Jul 2019 00:20:31 +0200 Subject: [PATCH 376/405] fix typed constants in spec func puller --- scripts/function_puller.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 36dd5b81f7..26671bafcf 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -74,15 +74,14 @@ def get_spec(file_name: str) -> SpecObject: row[i] = row[i].strip().strip('`') if '`' in row[i]: row[i] = row[i][:row[i].find('`')] - if row[1].startswith('uint') or row[1].startswith('Bytes'): + is_constant_def = True + if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': + is_constant_def = False + for c in row[0]: + if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': + is_constant_def = False + if is_constant_def: + constants[row[0]] = row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890') + elif row[1].startswith('uint') or row[1].startswith('Bytes'): custom_types[row[0]] = row[1] - else: - eligible = True - if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': - eligible = False - for c in row[0]: - if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': - eligible = False - if eligible: - constants[row[0]] = row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890') return functions, custom_types, constants, ssz_objects, inserts From 393c583d06a7f0d7e34135276506ac4551cc2636 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Jul 2019 00:21:19 +0200 Subject: [PATCH 377/405] move endianness to be a constant --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 71566baab2..c8e1b22b76 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -162,6 +162,7 @@ The following values are (non-configurable) constants used throughout the specif | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | | `SECONDS_PER_DAY` | `86400` | | `JUSTIFICATION_BITS_LENGTH` | `4` | +| `ENDIANNESS` | `'little'` | ## Configuration @@ -177,7 +178,6 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | -| `ENDIANNESS` | `'little'` | | `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**16` (= 65,536) | | `MIN_GENESIS_TIME` | `1578009600` (Jan 3, 2020) | From fdd1fd5b2c598298d20e56653174c5f5f1ef2f3a Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Jul 2019 00:26:46 +0200 Subject: [PATCH 378/405] make config values reflect the withdrawal prefix is a byte --- configs/constant_presets/mainnet.yaml | 2 +- configs/constant_presets/minimal.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 629badcee1..6ce6ec7f76 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -44,7 +44,7 @@ EFFECTIVE_BALANCE_INCREMENT: 1000000000 GENESIS_FORK_VERSION: 0x00000000 # 0, GENESIS_EPOCH is derived from this constant GENESIS_SLOT: 0 -BLS_WITHDRAWAL_PREFIX: 0 +BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 1d0ad015db..fd9684a0ca 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -43,7 +43,7 @@ EFFECTIVE_BALANCE_INCREMENT: 1000000000 GENESIS_FORK_VERSION: 0x00000000 # 0, GENESIS_EPOCH is derived from this constant GENESIS_SLOT: 0 -BLS_WITHDRAWAL_PREFIX: 0 +BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters From 84d8ca740c51e9c817534fa6c309481a123fb85f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 06:33:10 +0800 Subject: [PATCH 379/405] Improve fork choice rule test coverage --- specs/core/0_fork-choice.md | 2 -- .../test/fork_choice/test_on_block.py | 32 +++++++++++++++++ .../pyspec/eth2spec/test/helpers/state.py | 31 ++++++++++++++++- .../pyspec/eth2spec/test/test_finality.py | 34 ++----------------- 4 files changed, 64 insertions(+), 35 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 326a3c86ff..086f79707e 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -171,8 +171,6 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Update justified checkpoint if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: store.justified_checkpoint = state.current_justified_checkpoint - elif state.previous_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = state.previous_justified_checkpoint # Update finalized checkpoint if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py index 158ee1a58e..207341621d 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -2,6 +2,7 @@ from eth2spec.test.context import with_all_phases, with_state, bls_switch from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.state import next_epoch, next_epoch_with_attestations def run_on_block(spec, state, store, block, valid=True): @@ -41,6 +42,37 @@ def test_basic(spec, state): # TODO: add tests for justified_root and finalized_root +def apply_next_epoch_with_attestations(spec, state, store): + _, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + for block in new_blocks: + block_root = signing_root(block) + store.blocks[block_root] = block + store.block_states[block_root] = state + spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT) + return state, store + + +@with_all_phases +@with_state +@bls_switch +def test_on_block_justified_checkpoint(spec, state): + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + next_epoch(spec, state) + spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT) + state, store = apply_next_epoch_with_attestations(spec, state, store) + next_epoch(spec, state) + spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT) + + # On receiving a block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + # state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch + run_on_block(spec, state, store, block) + + @with_all_phases @with_state @bls_switch diff --git a/test_libs/pyspec/eth2spec/test/helpers/state.py b/test_libs/pyspec/eth2spec/test/helpers/state.py index 3708a354eb..ecad8af2e6 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/state.py +++ b/test_libs/pyspec/eth2spec/test/helpers/state.py @@ -1,4 +1,6 @@ -from eth2spec.test.helpers.block import sign_block +from copy import deepcopy +from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.block import sign_block, build_empty_block_for_next_slot def get_balance(state, index): @@ -36,3 +38,30 @@ def state_transition_and_sign_block(spec, state, block): spec.state_transition(state, block) block.state_root = state.hash_tree_root() sign_block(spec, state, block) + + +def next_epoch_with_attestations(spec, + state, + fill_cur_epoch, + fill_prev_epoch): + assert state.slot % spec.SLOTS_PER_EPOCH == 0 + + post_state = deepcopy(state) + blocks = [] + for _ in range(spec.SLOTS_PER_EPOCH): + block = build_empty_block_for_next_slot(spec, post_state) + if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: + slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 + if slot_to_attest >= spec.epoch_start_slot(spec.get_current_epoch(post_state)): + cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) + block.body.attestations.append(cur_attestation) + + if fill_prev_epoch: + slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 + prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest) + block.body.attestations.append(prev_attestation) + + state_transition_and_sign_block(spec, post_state, block) + blocks.append(block) + + return state, blocks, post_state diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 5ff0ed1efb..6250a685d7 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,9 +1,6 @@ -from copy import deepcopy - from eth2spec.test.context import spec_state_test, never_bls, with_all_phases -from eth2spec.test.helpers.state import next_epoch, state_transition_and_sign_block -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block -from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.state import next_epoch, next_epoch_with_attestations +from eth2spec.test.helpers.block import apply_empty_block def check_finality(spec, @@ -31,33 +28,6 @@ def check_finality(spec, assert state.finalized_checkpoint == prev_state.finalized_checkpoint -def next_epoch_with_attestations(spec, - state, - fill_cur_epoch, - fill_prev_epoch): - assert state.slot % spec.SLOTS_PER_EPOCH == 0 - - post_state = deepcopy(state) - blocks = [] - for _ in range(spec.SLOTS_PER_EPOCH): - block = build_empty_block_for_next_slot(spec, post_state) - if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: - slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= spec.epoch_start_slot(spec.get_current_epoch(post_state)): - cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) - block.body.attestations.append(cur_attestation) - - if fill_prev_epoch: - slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 - prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest) - block.body.attestations.append(prev_attestation) - - state_transition_and_sign_block(spec, post_state, block) - blocks.append(block) - - return state, blocks, post_state - - @with_all_phases @never_bls @spec_state_test From b7e87338127901755ebe6977f526fa76dcd2b00b Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Jul 2019 00:36:24 +0200 Subject: [PATCH 380/405] fix usage of bls withdrawal byte in tests --- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 4 ++-- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/transfers.py | 2 +- .../test/phase_0/block_processing/test_process_deposit.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 4f099be9d1..fccba9aa26 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -62,7 +62,7 @@ def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] # insecurely use pubkey as withdrawal key if no credentials provided - withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(pubkey)[1:] + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] deposit, root, deposit_data_list = build_deposit( spec, None, @@ -89,7 +89,7 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c # insecurely use pubkey as withdrawal key if no credentials provided if withdrawal_credentials is None: - withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(pubkey)[1:] + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] deposit, root, deposit_data_list = build_deposit( spec, diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index a5a816977a..1589b6ae86 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -6,7 +6,7 @@ def build_mock_validator(spec, i: int, balance: int): pubkey = pubkeys[i] # insecurely use pubkey as withdrawal key as well - withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(pubkey)[1:] + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] return spec.Validator( pubkey=pubkeys[i], withdrawal_credentials=withdrawal_credentials, diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index fa01a30880..3d3b0f4e37 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -34,7 +34,7 @@ def get_valid_transfer(spec, state, slot=None, sender_index=None, # ensure withdrawal_credentials reproducible state.validators[transfer.sender].withdrawal_credentials = ( - spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(transfer.pubkey)[1:] + spec.BLS_WITHDRAWAL_PREFIX + spec.hash(transfer.pubkey)[1:] ) return transfer diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 54a284a6e9..3dbbeedf02 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -131,7 +131,7 @@ def test_invalid_sig_top_up(spec, state): def test_invalid_withdrawal_credentials_top_up(spec, state): validator_index = 0 amount = spec.MAX_EFFECTIVE_BALANCE // 4 - withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(b"junk")[1:] + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(b"junk")[1:] deposit = prepare_state_and_deposit( spec, state, From 6a799903a97b1326d67b70ae2d44dc6f023278fc Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 23:36:26 +0100 Subject: [PATCH 381/405] Simplify --- specs/core/0_beacon-chain.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 82700f8743..d92f1cee67 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1144,12 +1144,11 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Hash, validator.activation_epoch = GENESIS_EPOCH # Populate active_index_roots and compact_committees_roots - genesis_active_index_root = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) - ) + indices_list = List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) + active_index_root = hash_tree_root(indices_list) committee_root = get_compact_committees_root(state, GENESIS_EPOCH) for index in range(EPOCHS_PER_HISTORICAL_VECTOR): - state.active_index_roots[index] = genesis_active_index_root + state.active_index_roots[index] = active_index_root state.compact_committees_roots[index] = committee_root return state ``` @@ -1495,9 +1494,8 @@ def process_final_updates(state: BeaconState) -> None: # Set active index root index_epoch = Epoch(next_epoch + ACTIVATION_EXIT_DELAY) index_root_position = index_epoch % EPOCHS_PER_HISTORICAL_VECTOR - state.active_index_roots[index_root_position] = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, index_epoch)) - ) + indices_list = List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, index_epoch)) + state.active_index_roots[index_root_position] = hash_tree_root(indices_list) # Set committees root committee_root_position = next_epoch % EPOCHS_PER_HISTORICAL_VECTOR state.compact_committees_roots[committee_root_position] = get_compact_committees_root(state, next_epoch) From b582afed84b2835b2db911dbfeee7e9b15621656 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 30 Jun 2019 23:49:53 +0100 Subject: [PATCH 382/405] Add back effective balance initialisation --- specs/core/0_beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cd4d099033..0bcc1174e9 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1691,6 +1691,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, + effective_balance=min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE), )) state.balances.append(amount) else: From 7ef98f7478ffe4d7e526364f90c83b43e0d2e9d9 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 1 Jul 2019 01:06:27 +0200 Subject: [PATCH 383/405] Compact `get_domain` calls --- specs/validator/0_beacon-chain-validator.md | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 6aa58c7e01..fa0dd1abb9 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -217,11 +217,7 @@ def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> return bls_sign( privkey=privkey, # privkey stored locally message_hash=hash_tree_root(slot_to_epoch(block.slot)), - domain=get_domain( - state=state, - domain_type=DOMAIN_RANDAO, - message_epoch=slot_to_epoch(block.slot), - ) + domain=get_domain(state, DOMAIN_RANDAO, slot_to_epoch(block.slot)) ) ``` @@ -258,11 +254,7 @@ def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: return bls_sign( privkey=privkey, # privkey stored locally message_hash=signing_root(header), - domain=get_domain( - state=state, - domain_type=DOMAIN_BEACON_PROPOSER, - message_epoch=slot_to_epoch(header.slot), - ) + domain=get_domain(state, DOMAIN_BEACON_PROPOSER, slot_to_epoch(header.slot)) ) ``` @@ -364,11 +356,7 @@ def get_signed_attestation_data(state: BeaconState, attestation: IndexedAttestat return bls_sign( privkey=privkey, # privkey stored locally message_hash=hash_tree_root(attestation_data_and_custody_bit), - domain=get_domain( - state=state, - domain_type=DOMAIN_ATTESTATION, - message_epoch=attestation.data.target.epoch, - ) + domain=get_domain(state, DOMAIN_ATTESTATION, attestation.data.target.epoch) ) ``` From 4f47e301440c84f6c1d1e8cb95935d2d5d3259b4 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 1 Jul 2019 01:22:58 +0200 Subject: [PATCH 384/405] separates out `domain` --- specs/validator/0_beacon-chain-validator.md | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index fa0dd1abb9..f335559ab9 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -214,11 +214,8 @@ Set `block.randao_reveal = epoch_signature` where `epoch_signature` is obtained ```python def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: - return bls_sign( - privkey=privkey, # privkey stored locally - message_hash=hash_tree_root(slot_to_epoch(block.slot)), - domain=get_domain(state, DOMAIN_RANDAO, slot_to_epoch(block.slot)) - ) + domain = get_domain(state, DOMAIN_RANDAO, slot_to_epoch(block.slot)) + return bls_sign(privkey, hash_tree_root(slot_to_epoch(block.slot)), domain) ``` ##### Eth1 Data @@ -251,11 +248,8 @@ Set `header.signature = block_signature` where `block_signature` is obtained fro ```python def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature: - return bls_sign( - privkey=privkey, # privkey stored locally - message_hash=signing_root(header), - domain=get_domain(state, DOMAIN_BEACON_PROPOSER, slot_to_epoch(header.slot)) - ) + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, slot_to_epoch(header.slot)) + return bls_sign(privkey, signing_root(header), domain) ``` #### Block body @@ -353,11 +347,8 @@ def get_signed_attestation_data(state: BeaconState, attestation: IndexedAttestat custody_bit=0b0, ) - return bls_sign( - privkey=privkey, # privkey stored locally - message_hash=hash_tree_root(attestation_data_and_custody_bit), - domain=get_domain(state, DOMAIN_ATTESTATION, attestation.data.target.epoch) - ) + domain = get_domain(state, DOMAIN_ATTESTATION, attestation.data.target.epoch) + return bls_sign(privkey, hash_tree_root(attestation_data_and_custody_bit), domain) ``` ## How to avoid slashing From aeba6ee8fb4377f262274f25fc2afddd617cff32 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 1 Jul 2019 00:35:09 +0100 Subject: [PATCH 385/405] compute_domain for consistency with get_domain --- specs/core/0_beacon-chain.md | 12 ++++++------ specs/validator/0_beacon-chain-validator.md | 2 +- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7104b78237..9415814ac1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -72,7 +72,7 @@ - [`compute_epoch_of_slot`](#compute_epoch_of_slot) - [`compute_start_slot_of_epoch`](#compute_start_slot_of_epoch) - [`compute_activation_exit_epoch`](#compute_activation_exit_epoch) - - [`compute_bls_domain`](#compute_bls_domain) + - [`compute_domain`](#compute_domain) - [Beacon state accessors](#beacon-state-accessors) - [`get_current_epoch`](#get_current_epoch) - [`get_previous_epoch`](#get_previous_epoch) @@ -771,10 +771,10 @@ def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: return Epoch(epoch + 1 + ACTIVATION_EXIT_DELAY) ``` -#### `compute_bls_domain` +#### `compute_domain` ```python -def compute_bls_domain(domain_type: uint64, fork_version: bytes=b'\x00' * 4) -> int: +def compute_domain(domain_type: uint64, fork_version: bytes=b'\x00' * 4) -> int: """ Return the BLS domain for the ``domain_type`` and ``fork_version``. """ @@ -1010,7 +1010,7 @@ def get_domain(state: BeaconState, domain_type: uint64, message_epoch: Epoch=Non """ epoch = get_current_epoch(state) if message_epoch is None else message_epoch fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version - return compute_bls_domain(domain_type, fork_version) + return compute_domain(domain_type, fork_version) ``` #### `get_indexed_attestation` @@ -1685,8 +1685,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) for new validators. # Note: The deposit contract does not check signatures. - # Note: Deposits are valid across forks, thus the deposit domain is retrieved directly from `compute_bls_domain` - domain = compute_bls_domain(DOMAIN_DEPOSIT) + # Note: Deposits are valid across forks, thus the deposit domain is retrieved directly from `compute_domain` + domain = compute_domain(DOMAIN_DEPOSIT) if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, domain): return diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index a7542878b9..0e9583a070 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -99,7 +99,7 @@ To submit a deposit: - Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. - Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. - Set `deposit_data.amount = amount`. -- Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=compute_bls_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `compute_bls_domain` will default to zeroes there). +- Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=compute_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `compute_domain` will default to zeroes there). - Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. *Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validators` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 276fa617bf..148bc95493 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -19,7 +19,7 @@ def build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, st def sign_deposit_data(spec, deposit_data, privkey, state=None): if state is None: # Genesis - domain = spec.compute_bls_domain(spec.DOMAIN_DEPOSIT) + domain = spec.compute_domain(spec.DOMAIN_DEPOSIT) else: domain = spec.get_domain( state, From 1698263621c476850e2eccd012b9ebf3812c84a0 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 1 Jul 2019 00:36:19 +0100 Subject: [PATCH 386/405] BLS domain => domain --- specs/core/0_beacon-chain.md | 2 +- specs/test_formats/bls/msg_hash_g2_compressed.md | 4 ++-- specs/test_formats/bls/msg_hash_g2_uncompressed.md | 6 +++--- specs/test_formats/bls/sign_msg.md | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9415814ac1..bf7aa4dd55 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -776,7 +776,7 @@ def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: ```python def compute_domain(domain_type: uint64, fork_version: bytes=b'\x00' * 4) -> int: """ - Return the BLS domain for the ``domain_type`` and ``fork_version``. + Return the domain for the ``domain_type`` and ``fork_version``. """ return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) ``` diff --git a/specs/test_formats/bls/msg_hash_g2_compressed.md b/specs/test_formats/bls/msg_hash_g2_compressed.md index 44ea8ded77..d82041633a 100644 --- a/specs/test_formats/bls/msg_hash_g2_compressed.md +++ b/specs/test_formats/bls/msg_hash_g2_compressed.md @@ -5,9 +5,9 @@ A BLS compressed-hash to G2. ## Test case format ```yaml -input: +input: message: bytes32, - domain: uint64 -- the BLS domain + domain: uint64 -- the domain output: List[bytes48] -- length of two ``` diff --git a/specs/test_formats/bls/msg_hash_g2_uncompressed.md b/specs/test_formats/bls/msg_hash_g2_uncompressed.md index 847b5f61d4..873c767d70 100644 --- a/specs/test_formats/bls/msg_hash_g2_uncompressed.md +++ b/specs/test_formats/bls/msg_hash_g2_uncompressed.md @@ -1,13 +1,13 @@ # Test format: BLS hash-uncompressed -A BLS uncompressed-hash to G2. +A BLS uncompressed-hash to G2. ## Test case format ```yaml -input: +input: message: bytes32 - domain: uint64 -- the BLS domain + domain: uint64 -- the domain output: List[List[bytes48]] -- 3 lists, each a length of two ``` diff --git a/specs/test_formats/bls/sign_msg.md b/specs/test_formats/bls/sign_msg.md index b17e6246de..94346edaa7 100644 --- a/specs/test_formats/bls/sign_msg.md +++ b/specs/test_formats/bls/sign_msg.md @@ -1,6 +1,6 @@ # Test format: BLS sign message -Message signing with BLS should produce a signature. +Message signing with BLS should produce a signature. ## Test case format @@ -8,7 +8,7 @@ Message signing with BLS should produce a signature. input: privkey: bytes32 -- the private key used for signing message: bytes32 -- input message to sign (a hash) - domain: uint64 -- the BLS domain + domain: uint64 -- the domain output: bytes96 -- expected signature ``` From 1869cfed2ec15091a2b3c365f0257f4e6d350fbf Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 07:48:20 +0800 Subject: [PATCH 387/405] test finalized_checkpoint update --- .../test/fork_choice/test_on_block.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py index 207341621d..90f161fa22 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -18,6 +18,17 @@ def run_on_block(spec, state, store, block, valid=True): assert store.blocks[signing_root(block)] == block +def apply_next_epoch_with_attestations(spec, state, store): + _, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + for block in new_blocks: + block_root = signing_root(block) + store.blocks[block_root] = block + store.block_states[block_root] = state + last_block = block + spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT) + return state, store, last_block + + @with_all_phases @with_state @bls_switch @@ -42,20 +53,10 @@ def test_basic(spec, state): # TODO: add tests for justified_root and finalized_root -def apply_next_epoch_with_attestations(spec, state, store): - _, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) - for block in new_blocks: - block_root = signing_root(block) - store.blocks[block_root] = block - store.block_states[block_root] = state - spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT) - return state, store - - @with_all_phases @with_state @bls_switch -def test_on_block_justified_checkpoint(spec, state): +def test_on_block_checkpoints(spec, state): # Initialization store = spec.get_genesis_store(state) time = 100 @@ -63,13 +64,18 @@ def test_on_block_justified_checkpoint(spec, state): next_epoch(spec, state) spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT) - state, store = apply_next_epoch_with_attestations(spec, state, store) + state, store, last_block = apply_next_epoch_with_attestations(spec, state, store) next_epoch(spec, state) spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT) + last_block_root = signing_root(last_block) + + # Mock the finalized_checkpoint + store.block_states[last_block_root].finalized_checkpoint = ( + store.block_states[last_block_root].current_justified_checkpoint + ) # On receiving a block of `GENESIS_SLOT + 1` slot block = build_empty_block_for_next_slot(spec, state) - # state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch run_on_block(spec, state, store, block) From 5b2b3cfe33a99f96d8f2eba15ecbda18597ba9d1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 09:03:35 +0800 Subject: [PATCH 388/405] class DomainType would be added in build_spec --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7b57985294..50b6a3386a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -501,7 +501,3 @@ def is_fixed_size(cls): Bytes32: BytesType = BytesN[32] Bytes48: BytesType = BytesN[48] Bytes96: BytesType = BytesN[96] - - -class DomainType(Bytes4): - pass From 8da447f607a1f08e85942142fa8d1d54f81a7da1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 09:07:33 +0800 Subject: [PATCH 389/405] Revert "5 slots is a `Slot`" --- specs/core/0_beacon-chain.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 82599be8d8..b05b263c05 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -205,16 +205,16 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Duration | | - | - | :-: | :-: | -| `MIN_ATTESTATION_INCLUSION_DELAY` | `Slot(2**0)` (= 1) | 6 seconds | -| `SLOTS_PER_EPOCH` | `Slot(2**6)` (= 64) | 6.4 minutes | -| `MIN_SEED_LOOKAHEAD` | `Epoch(2**0)` (= 1) | 6.4 minutes | -| `ACTIVATION_EXIT_DELAY` | `Epoch(2**2)` (= 4) | 25.6 minutes | -| `SLOTS_PER_ETH1_VOTING_PERIOD` | `Slot(2**10)` (= 1,024) | ~1.7 hours | -| `SLOTS_PER_HISTORICAL_ROOT` | `Slot(2**13)` (= 8,192) | ~13 hours | -| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `Epoch(2**8)` (= 256) | ~27 hours | -| `PERSISTENT_COMMITTEE_PERIOD` | `Epoch(2**11)` (= 2,048) | 9 days | -| `MAX_EPOCHS_PER_CROSSLINK` | `Epoch(2**6)` (= 64) | ~7 hours | -| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `Epoch(2**2)` (= 4) | 25.6 minutes | +| `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | 6 seconds | +| `SLOTS_PER_EPOCH` | `2**6` (= 64) | 6.4 minutes | +| `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | 6.4 minutes | +| `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | 25.6 minutes | +| `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | ~1.7 hours | +| `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | ~13 hours | +| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | ~27 hours | +| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | 9 days | +| `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | ~7 hours | +| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | 25.6 minutes | ### State list lengths From b08d1d8d73fe1a05fa8a9fa442815f1dea259780 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 20:13:27 -0500 Subject: [PATCH 390/405] fix associtivity in conditional --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index bf7aa4dd55..7d26c2dfec 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -665,7 +665,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe if not len(set(bit_0_indices).intersection(bit_1_indices)) == 0: return False # Verify indices are sorted - if not bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices): + if not (bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices)): return False # Verify aggregate signature if not bls_verify_multiple( From e061972eee18ea870855cd3ba7d64c1e4e992b0b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 09:17:03 +0800 Subject: [PATCH 391/405] Revert State list lengths --- specs/core/0_beacon-chain.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b05b263c05..daa8d335c2 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -218,12 +218,12 @@ The following values are (non-configurable) constants used throughout the specif ### State list lengths -| Name | Value | Duration | +| Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `EPOCHS_PER_HISTORICAL_VECTOR` | `Epoch(2**16)` (= 65,536) | ~0.8 years | -| `EPOCHS_PER_SLASHINGS_VECTOR` | `Epoch(2**13)` (= 8,192) | ~36 days | -| `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | ~26,131 years | -| `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validator spots | | +| `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | +| `EPOCHS_PER_SLASHINGS_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | +| `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | +| `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validator spots | ### Rewards and penalties From 8b65127c6800d616699fa75b321e41859b50a7e5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 09:33:28 +0800 Subject: [PATCH 392/405] fix --- test_libs/pyspec/eth2spec/test/helpers/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/state.py b/test_libs/pyspec/eth2spec/test/helpers/state.py index ecad8af2e6..e88fc6adee 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/state.py +++ b/test_libs/pyspec/eth2spec/test/helpers/state.py @@ -52,7 +52,7 @@ def next_epoch_with_attestations(spec, block = build_empty_block_for_next_slot(spec, post_state) if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= spec.epoch_start_slot(spec.get_current_epoch(post_state)): + if slot_to_attest >= spec.compute_start_slot_of_epoch(spec.get_current_epoch(post_state)): cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) block.body.attestations.append(cur_attestation) From f4907d67ee8ba171e1ea8aeeec4cf6613052425a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 20:44:02 -0500 Subject: [PATCH 393/405] clean up commitee assignment typing --- scripts/build_spec.py | 2 +- specs/validator/0_beacon-chain-validator.md | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index cf2a8f19a1..61fd6547bf 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -11,7 +11,7 @@ PHASE0_IMPORTS = '''from typing import ( - Any, Dict, Set, Sequence, Tuple, + Any, Dict, Set, Sequence, Tuple, Optional ) from dataclasses import ( diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index f335559ab9..1c2f081dab 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -135,13 +135,14 @@ A validator can get committee assignments for a given epoch using the following ```python def get_committee_assignment(state: BeaconState, epoch: Epoch, - validator_index: ValidatorIndex) -> Tuple[Sequence[ValidatorIndex], Shard, Slot]: + validator_index: ValidatorIndex) -> Optional[Tuple[Sequence[ValidatorIndex], Shard, Slot]]: """ Return the committee assignment in the ``epoch`` for ``validator_index``. ``assignment`` returned is a tuple of the following form: * ``assignment[0]`` is the list of validators in the committee * ``assignment[1]`` is the shard to which the committee is assigned * ``assignment[2]`` is the slot at which the committee is assigned + Return None if no assignment. """ next_epoch = get_current_epoch(state) + 1 assert epoch <= next_epoch @@ -155,11 +156,8 @@ def get_committee_assignment(state: BeaconState, shard = Shard((slot_start_shard + i) % SHARD_COUNT) committee = get_crosslink_committee(state, epoch, shard) if validator_index in committee: - break - else: - continue - break - return committee, shard, Slot(slot) + return committee, shard, Slot(slot) + return None ``` A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run with a `state` of the slot in question. Proposer selection is only stable within the context of the current epoch. From f0027b7b3f2b7e6557295cfe9f28f33b9eb52427 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 10:05:38 +0800 Subject: [PATCH 394/405] Use `VALIDATOR_REGISTRY_LIMIT` in `compute_shuffled_index` --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 799fa7a3ad..a439858a0f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -19,7 +19,7 @@ - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) - [Max operations per block](#max-operations-per-block) - - [Signature domains](#signature-domains) + - [Signature domain types](#signature-domain-types) - [Containers](#containers) - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) @@ -715,7 +715,7 @@ def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Has Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). """ assert index < index_count - assert index_count <= 2**40 + assert index_count <= VALIDATOR_REGISTRY_LIMIT # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) # See the 'generalized domain' algorithm on page 3 From f7ed1864471632b670e8454c8f9d43fb7c43f000 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 10:46:22 +0800 Subject: [PATCH 395/405] Use `Checkpoint` --- specs/validator/0_beacon-chain-validator.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 0e9583a070..aa41ca0ce6 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -301,10 +301,8 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. ##### FFG vote -- Set `attestation_data.source_epoch = head_state.current_justified_epoch`. -- Set `attestation_data.source_root = head_state.current_justified_root`. -- Set `attestation_data.target_epoch = get_current_epoch(head_state)` -- Set `attestation_data.target_root = epoch_boundary_block_root` where `epoch_boundary_block_root` is the root of block at the most recent epoch boundary. +- Set `attestation_data.source = head_state.current_justified_checkpoint`. +- Set `attestation_data.target = Checkpoint(epoch=get_current_epoch(head_state), root=epoch_boundary_block_root)` where `epoch_boundary_block_root` is the root of block at the most recent epoch boundary. *Note*: `epoch_boundary_block_root` can be looked up in the state using: @@ -318,7 +316,7 @@ Construct `attestation_data.crosslink` via the following. - Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee. - Let `parent_crosslink = head_state.current_crosslinks[shard]`. - Set `attestation_data.crosslink.start_epoch = parent_crosslink.end_epoch`. -- Set `attestation_data.crosslink.end_epoch = min(attestation_data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK)`. +- Set `attestation_data.crosslink.end_epoch = min(attestation_data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK)`. - Set `attestation_data.crosslink.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. - Set `attestation_data.crosslink.data_root = ZERO_HASH`. *Note*: This is a stub for Phase 0. @@ -389,7 +387,7 @@ To avoid "attester slashings", a validator must not sign two conflicting [`Attes Specifically, when signing an `Attestation`, a validator should perform the following steps in the following order: -1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source_epoch`) and target (i.e. `compute_epoch_of_slot(attestation_data.slot)`). +1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source.epoch`) and target (i.e. `compute_epoch_of_slot(attestation_data.slot)`). 2. Generate and broadcast attestation. If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast attestation and can effectively avoid slashing. From 4f360a58e2a17549b74a92b59a2245cf8138e9d7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 10:58:09 +0800 Subject: [PATCH 396/405] Fix outdated `fork.slot` and `attestation_data.slot` --- specs/validator/0_beacon-chain-validator.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index aa41ca0ce6..63ccf10bfc 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -212,7 +212,7 @@ epoch_signature = bls_sign( privkey=validator.privkey, # privkey stored locally, not in state message_hash=hash_tree_root(compute_epoch_of_slot(block.slot)), domain=get_domain( - fork=fork, # `fork` is the fork object at the slot `block.slot` + fork=fork, # `fork` is the fork object at the epoch `compute_epoch_of_slot(block.slot)` epoch=compute_epoch_of_slot(block.slot), domain_type=DOMAIN_RANDAO, ) @@ -251,7 +251,7 @@ block_signature = bls_sign( privkey=validator.privkey, # privkey store locally, not in state message_hash=signing_root(block), domain=get_domain( - fork=fork, # `fork` is the fork object at the slot `block.slot` + fork=fork, # `fork` is the fork object at the epoch `compute_epoch_of_slot(block.slot)` epoch=compute_epoch_of_slot(block.slot), domain_type=DOMAIN_BEACON_BLOCK, ) @@ -355,8 +355,8 @@ signed_attestation_data = bls_sign( privkey=validator.privkey, # privkey stored locally, not in state message_hash=attestation_message, domain=get_domain( - fork=fork, # `fork` is the fork object at the slot, `attestation_data.slot` - epoch=compute_epoch_of_slot(attestation_data.slot), + fork=fork, # `fork` is the fork object at epoch `target.epoch` + epoch=target.epoch, domain_type=DOMAIN_ATTESTATION, ) ) @@ -387,7 +387,7 @@ To avoid "attester slashings", a validator must not sign two conflicting [`Attes Specifically, when signing an `Attestation`, a validator should perform the following steps in the following order: -1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source.epoch`) and target (i.e. `compute_epoch_of_slot(attestation_data.slot)`). +1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source.epoch`) and target (i.e. `attestation_data.target.epoch`). 2. Generate and broadcast attestation. If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast attestation and can effectively avoid slashing. From 476dbdb15364bb89e3ff7a82eaf5d941a5fdcfa5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 22:13:41 -0500 Subject: [PATCH 397/405] add time param back tp units --- specs/core/0_beacon-chain.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a05a21b5de..e7ea49f4c5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -205,18 +205,18 @@ The following values are (non-configurable) constants used throughout the specif ### Time parameters -| Name | Value | Duration | +| Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | 6 seconds | -| `SLOTS_PER_EPOCH` | `2**6` (= 64) | 6.4 minutes | -| `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | 6.4 minutes | -| `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | 25.6 minutes | -| `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | ~1.7 hours | -| `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | ~13 hours | -| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | ~27 hours | -| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | 9 days | -| `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | ~7 hours | -| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | 25.6 minutes | +| `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 6 seconds | +| `SLOTS_PER_EPOCH` | `2**6` (= 64) | slots | 6.4 minutes | +| `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | +| `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | epochs | 25.6 minutes | +| `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~1.7 hours | +| `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | +| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | +| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | +| `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | +| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes | ### State list lengths From aba7ee66f2bf1ce8e9580040afd6210b761aa10e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 22:20:41 -0500 Subject: [PATCH 398/405] remove length check in compute shuffled index --- specs/core/0_beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a439858a0f..fe2f93760b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -715,7 +715,6 @@ def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Has Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). """ assert index < index_count - assert index_count <= VALIDATOR_REGISTRY_LIMIT # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) # See the 'generalized domain' algorithm on page 3 From 93fc81e761230fff048d5aa7a0b204f459353ffd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 11:22:18 +0800 Subject: [PATCH 399/405] Use relative inner repo link --- specs/core/0_deposit-contract.md | 2 +- test_generators/bls/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index b95748ae6b..ffef283e5f 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -59,6 +59,6 @@ Every Ethereum 1.0 deposit emits a `DepositEvent` log for consumption by the bea ## Vyper code -The deposit contract source code, written in Vyper, is available [here](https://github.com/ethereum/eth2.0-specs/blob/dev/deposit_contract/contracts/validator_registration.v.py). +The deposit contract source code, written in Vyper, is available [here](../../deposit_contract/contracts/validator_registration.v.py). *Note*: To save on gas, the deposit contract uses a progressive Merkle root calculation algorithm that requires only O(log(n)) storage. See [here](https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py) for a Python implementation, and [here](https://github.com/runtimeverification/verified-smart-contracts/blob/master/deposit/formal-incremental-merkle-tree-algorithm.pdf) for a formal correctness proof. diff --git a/test_generators/bls/README.md b/test_generators/bls/README.md index a21ad16d94..2bf46e9ea1 100644 --- a/test_generators/bls/README.md +++ b/test_generators/bls/README.md @@ -9,7 +9,7 @@ The base unit is bytes48 of which only 381 bits are used ## Resources -- [Eth2.0 spec](https://github.com/ethereum/eth2.0-specs/blob/master/specs/bls_signature.md) +- [Eth2.0 spec](../../specs/bls_signature.md) - [Finite Field Arithmetic](http://www.springeronline.com/sgw/cda/pageitems/document/cda_downloaddocument/0,11996,0-0-45-110359-0,00.pdf) - Chapter 2 of [Elliptic Curve Cryptography](http://cacr.uwaterloo.ca/ecc/). Darrel Hankerson, Alfred Menezes, and Scott Vanstone - [Zcash BLS parameters](https://github.com/zkcrypto/pairing/tree/master/src/bls12_381) From 63d465345319ad0808cb524ed884511a9e8cc8a4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 22:29:02 -0500 Subject: [PATCH 400/405] remove unnecessary casting --- specs/core/0_beacon-chain.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e7ea49f4c5..1014c9c6e8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1387,7 +1387,6 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence # Proposer and inclusion delay micro-rewards for index in get_unslashed_attesting_indices(state, matching_source_attestations): - index = ValidatorIndex(index) attestation = min([ a for a in matching_source_attestations if index in get_attesting_indices(state, a.data, a.aggregation_bits) @@ -1402,7 +1401,6 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in eligible_validator_indices: - index = ValidatorIndex(index) penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * get_base_reward(state, index)) if index not in matching_target_attesting_indices: penalties[index] += Gwei( From ac741c096d30b682ec2225ea45214dcbc26a2170 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 11:29:10 +0800 Subject: [PATCH 401/405] Fix ToC of SSZ spec --- specs/simple-serialize.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 7076b6410f..8d9c331033 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -11,16 +11,22 @@ - [Typing](#typing) - [Basic types](#basic-types) - [Composite types](#composite-types) + - [Variable-size and fixed-size](#variable-size-and-fixed-size) - [Aliases](#aliases) - [Default values](#default-values) + - [`is_empty`](#is_empty) - [Illegal types](#illegal-types) - [Serialization](#serialization) - [`uintN`](#uintn) - [`boolean`](#boolean) - [`null`](#null) + - [`Bitvector[N]`](#bitvectorn) + - [`Bitlist[N]`](#bitlistn) - [Vectors, containers, lists, unions](#vectors-containers-lists-unions) - [Deserialization](#deserialization) - [Merkleization](#merkleization) + - [Merkleization of `Bitvector[N]`](#merkleization-of-bitvectorn) + - [`Bitlist[N]`](#bitlistn-1) - [Self-signed containers](#self-signed-containers) - [Implementations](#implementations) From 126591b2f9b5d2d678e50c1d9698c8f038784046 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 22:34:35 -0500 Subject: [PATCH 402/405] scale inclusion reward properly --- specs/core/0_beacon-chain.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e7ea49f4c5..cfd2e61d30 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1395,7 +1395,11 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence proposer_reward = Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) rewards[attestation.proposer_index] += proposer_reward max_attester_reward = get_base_reward(state, index) - proposer_reward - rewards[index] += Gwei(max_attester_reward * MIN_ATTESTATION_INCLUSION_DELAY // attestation.inclusion_delay) + rewards[index] += Gwei( + max_attester_reward + * (SLOTS_PER_EPOCH + MIN_ATTESTATION_INCLUSION_DELAY - attestation.inclusion_delay) + // SLOTS_PER_EPOCH + ) # Inactivity penalty finality_delay = previous_epoch - state.finalized_checkpoint.epoch From 17b0bc35d8d1b41ec2b26d9778735ec7e12635b2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 22:04:47 -0600 Subject: [PATCH 403/405] add json encoding for bitlist and vector --- test_libs/pyspec/eth2spec/debug/encode.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 9080bb573e..ac4bd9df22 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,6 +1,7 @@ -from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root, serialize from eth2spec.utils.ssz.ssz_typing import ( - uint, Container, boolean + uint, boolean, + Bitlist, Bitvector, Container ) @@ -12,6 +13,8 @@ def encode(value, include_hash_tree_roots=False): return int(value) elif isinstance(value, boolean): return value == 1 + elif isinstance(value, (Bitlist, Bitvector)): + return '0x' + serialize(value).hex() elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] elif isinstance(value, bytes): # both bytes and BytesN From 27d615ca56e9e8cecc786377a6d8b8297ab32eb6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jul 2019 12:26:57 +0800 Subject: [PATCH 404/405] Bump vyper to `0.1.0b10` --- deposit_contract/contracts/validator_registration.json | 2 +- deposit_contract/requirements-testing.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index 527ce60871..3a6bfb2d86 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "DepositEvent", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_hash_tree_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 91674}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334417}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600260c052602060c02001555b81516001018083528114156100aa575b50506112ff56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63863a311b600051141561050857341561029057600080fd5b6000610140526101405161016052600154610180526101a060006020818352015b60016001610180511614156103325760006101a051602081106102d357600080fd5b600060c052602060c02001546020826102400101526020810190506101605160208261024001015260208101905080610240526102409050602060c0825160208401600060025af161032457600080fd5b60c0519050610160526103a0565b6000610160516020826101c00101526020810190506101a0516020811061035857600080fd5b600260c052602060c02001546020826101c0010152602081019050806101c0526101c09050602060c0825160208401600060025af161039657600080fd5b60c0519050610160525b61018060026103ae57600080fd5b60028151048152505b81516001018083528114156102b1575b505060006101605160208261044001015260208101905061014051610160516101805163806732896102c0526001546102e0526102e0516006580161009b565b506103405260006103a0525b6103405160206001820306601f82010390506103a0511015156104355761044e565b6103a05161036001526103a0516020016103a052610413565b61018052610160526101405261034060088060208461044001018260208501600060046012f150508051820191505060006018602082066103c0016020828401111561049957600080fd5b6020806103e082610140600060046015f150508181528090509050905060188060208461044001018260208501600060046014f150508051820191505080610440526104409050602060c0825160208401600060025af16104f957600080fd5b60c051905060005260206000f3005b63621fd130600051141561061a57341561052157600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f82010390506102205110151561056c57610585565b610220516101e00152610220516020016102205261054a565b6101c0805160200180610280828460006004600a8704601201f16105a857600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156105da576105f6565b60006102e0516102a001535b81516001018083528114156105ca575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d600051141561117457606060046101403760506004356004016101a037603060043560040135111561065057600080fd5b604060243560040161022037602060243560040135111561067057600080fd5b608060443560040161028037606060443560040135111561069057600080fd5b63ffffffff600154106106a257600080fd5b633b9aca0061034052610340516106b857600080fd5b61034051340461032052633b9aca006103205110156106d657600080fd5b60306101a051146106e657600080fd5b602061022051146106f657600080fd5b6060610280511461070657600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a051101515610796576107af565b6104a05161046001526104a0516020016104a052610774565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f161081657600080fd5b50506101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a05163806732896104c0526001546104e0526104e0516006580161009b565b506105405260006105a0525b6105405160206001820306601f82010390506105a0511015156108c7576108e0565b6105a05161056001526105a0516020016105a0526108a5565b6104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526105408051602001806105c0828460006004600a8704601201f161096757600080fd5b505060a06106405261064051610680526101a08051602001806106405161068001828460006004600a8704601201f161099f57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516040818352015b83610620511015156109dd576109fa565b6000610620516020850101535b81516001018083528114156109cc575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106a0526102208051602001806106405161068001828460006004600a8704601201f1610a5157600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610a8f57610aac565b6000610620516020850101535b8151600101808352811415610a7e575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106c0526103608051602001806106405161068001828460006004600a8704601201f1610b0357600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610b4157610b5e565b6000610620516020850101535b8151600101808352811415610b30575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106e0526102808051602001806106405161068001828460006004600a8704601201f1610bb557600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516060818352015b8361062051101515610bf357610c10565b6000610620516020850101535b8151600101808352811415610be2575b50505050602061064051610680015160206001820306601f82010390506106405101016106405261064051610700526105c08051602001806106405161068001828460006004600a8704601201f1610c6757600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610ca557610cc2565b6000610620516020850101535b8151600101808352811415610c94575b50505050602061064051610680015160206001820306601f8201039050610640510101610640527f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c561064051610680a160006107205260006101a06030806020846107e001018260208501600060046016f150508051820191505060006010602082066107600160208284011115610d5957600080fd5b60208061078082610720600060046015f15050818152809050905090506010806020846107e001018260208501600060046013f1505080518201915050806107e0526107e09050602060c0825160208401600060025af1610db957600080fd5b60c0519050610740526000600060406020820661088001610280518284011115610de257600080fd5b6060806108a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af1610e2257600080fd5b60c0519050602082610a800101526020810190506000604060206020820661094001610280518284011115610e5657600080fd5b606080610960826020602088068803016102800160006004601bf1505081815280905090509050602080602084610a0001018260208501600060046015f150508051820191505061072051602082610a0001015260208101905080610a0052610a009050602060c0825160208401600060025af1610ed357600080fd5b60c0519050602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610f0a57600080fd5b60c0519050610860526000600061074051602082610b20010152602081019050610220602080602084610b2001018260208501600060046015f150508051820191505080610b2052610b209050602060c0825160208401600060025af1610f7057600080fd5b60c0519050602082610ca00101526020810190506000610360600880602084610c2001018260208501600060046012f15050805182019150506000601860208206610ba00160208284011115610fc557600080fd5b602080610bc082610720600060046015f1505081815280905090509050601880602084610c2001018260208501600060046014f150508051820191505061086051602082610c2001015260208101905080610c2052610c209050602060c0825160208401600060025af161103857600080fd5b60c0519050602082610ca001015260208101905080610ca052610ca09050602060c0825160208401600060025af161106f57600080fd5b60c0519050610b0052600180546001825401101561108c57600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d20511614156110dc57610b0051610d4051602081106110cb57600080fd5b600060c052602060c0200155611170565b6000610d4051602081106110ef57600080fd5b600060c052602060c0200154602082610d60010152602081019050610b0051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af161114057600080fd5b60c0519050610b0052610d20600261115757600080fd5b60028151048152505b81516001018083528114156110a7575b5050005b60006000fd5b6101856112ff036101856000396101856112ff036000f3"} \ No newline at end of file +{"abi": [{"name": "DepositEvent", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_hash_tree_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 91674}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 10433}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 1334417}], "bytecode": "0x740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009857600080fd5b6101406000601f818352015b600061014051602081106100b757600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e357600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012157600080fd5b60c0519050606051600161014051018060405190131561014057600080fd5b809190121561014e57600080fd5b6020811061015b57600080fd5b600260c052602060c02001555b81516001018083528114156100a4575b50506112f956600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610277575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100da578060000360020a82046100e1565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561010c57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610155578060000360020a820461015c565b8060020a82025b905090506101a0525b81516001018083528114156100bd575b50506018600860208206610200016020828401111561019357600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101d057600080fd5b50506102c05160206001820306601f82010390506103206102c0516008818352015b826103205111156102025761021e565b6000610320516102e001535b81516001018083528114156101f2575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102535761026f565b602061028051036102a001516020610280510361028052610241565b610160515650005b63863a311b600051141561050857341561029057600080fd5b6000610140526101405161016052600154610180526101a060006020818352015b60016001610180511614156103325760006101a051602081106102d357600080fd5b600060c052602060c02001546020826102400101526020810190506101605160208261024001015260208101905080610240526102409050602060c0825160208401600060025af161032457600080fd5b60c0519050610160526103a0565b6000610160516020826101c00101526020810190506101a0516020811061035857600080fd5b600260c052602060c02001546020826101c0010152602081019050806101c0526101c09050602060c0825160208401600060025af161039657600080fd5b60c0519050610160525b61018060026103ae57600080fd5b60028151048152505b81516001018083528114156102b1575b505060006101605160208261044001015260208101905061014051610160516101805163806732896102c0526001546102e0526102e0516006580161009b565b506103405260006103a0525b6103405160206001820306601f82010390506103a0511015156104355761044e565b6103a05161036001526103a0516020016103a052610413565b61018052610160526101405261034060088060208461044001018260208501600060046012f150508051820191505060006018602082066103c0016020828401111561049957600080fd5b6020806103e082610140600060046015f150508181528090509050905060188060208461044001018260208501600060046014f150508051820191505080610440526104409050602060c0825160208401600060025af16104f957600080fd5b60c051905060005260206000f3005b63621fd130600051141561061a57341561052157600080fd5b63806732896101405260015461016052610160516006580161009b565b506101c0526000610220525b6101c05160206001820306601f82010390506102205110151561056c57610585565b610220516101e00152610220516020016102205261054a565b6101c0805160200180610280828460006004600a8704601201f16105a857600080fd5b50506102805160206001820306601f82010390506102e0610280516008818352015b826102e05111156105da576105f6565b60006102e0516102a001535b81516001018083528114156105ca575b5050506020610260526040610280510160206001820306601f8201039050610260f3005b63c47e300d600051141561117457606060046101403760506004356004016101a037603060043560040135111561065057600080fd5b604060243560040161022037602060243560040135111561067057600080fd5b608060443560040161028037606060443560040135111561069057600080fd5b63ffffffff600154106106a257600080fd5b633b9aca0061034052610340516106b857600080fd5b61034051340461032052633b9aca006103205110156106d657600080fd5b60306101a051146106e657600080fd5b602061022051146106f657600080fd5b6060610280511461070657600080fd5b6101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a05163806732896103c052610320516103e0526103e0516006580161009b565b506104405260006104a0525b6104405160206001820306601f82010390506104a051101515610796576107af565b6104a05161046001526104a0516020016104a052610774565b6103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a052610440805160200180610360828460006004600a8704601201f161081657600080fd5b50506101a0516101c0516101e05161020051610220516102405161026051610280516102a0516102c0516102e05161030051610320516103405161036051610380516103a0516103c0516103e05161040051610420516104405161046051610480516104a05163806732896104c0526001546104e0526104e0516006580161009b565b506105405260006105a0525b6105405160206001820306601f82010390506105a0511015156108c7576108e0565b6105a05161056001526105a0516020016105a0526108a5565b6104a05261048052610460526104405261042052610400526103e0526103c0526103a05261038052610360526103405261032052610300526102e0526102c0526102a05261028052610260526102405261022052610200526101e0526101c0526101a0526105408051602001806105c0828460006004600a8704601201f161096757600080fd5b505060a06106405261064051610680526101a08051602001806106405161068001828460006004600a8704601201f161099f57600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516040818352015b83610620511015156109dd576109fa565b6000610620516020850101535b81516001018083528114156109cc575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106a0526102208051602001806106405161068001828460006004600a8704601201f1610a5157600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610a8f57610aac565b6000610620516020850101535b8151600101808352811415610a7e575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106c0526103608051602001806106405161068001828460006004600a8704601201f1610b0357600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610b4157610b5e565b6000610620516020850101535b8151600101808352811415610b30575b50505050602061064051610680015160206001820306601f820103905061064051010161064052610640516106e0526102808051602001806106405161068001828460006004600a8704601201f1610bb557600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516060818352015b8361062051101515610bf357610c10565b6000610620516020850101535b8151600101808352811415610be2575b50505050602061064051610680015160206001820306601f82010390506106405101016106405261064051610700526105c08051602001806106405161068001828460006004600a8704601201f1610c6757600080fd5b505061064051610680015160206001820306601f8201039050610640516106800161062081516020818352015b8361062051101515610ca557610cc2565b6000610620516020850101535b8151600101808352811415610c94575b50505050602061064051610680015160206001820306601f8201039050610640510101610640527f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c561064051610680a160006107205260006101a06030806020846107e001018260208501600060046016f150508051820191505060006010602082066107600160208284011115610d5957600080fd5b60208061078082610720600060046015f15050818152809050905090506010806020846107e001018260208501600060046013f1505080518201915050806107e0526107e09050602060c0825160208401600060025af1610db957600080fd5b60c0519050610740526000600060406020820661088001610280518284011115610de257600080fd5b6060806108a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af1610e2257600080fd5b60c0519050602082610a800101526020810190506000604060206020820661094001610280518284011115610e5657600080fd5b606080610960826020602088068803016102800160006004601bf1505081815280905090509050602080602084610a0001018260208501600060046015f150508051820191505061072051602082610a0001015260208101905080610a0052610a009050602060c0825160208401600060025af1610ed357600080fd5b60c0519050602082610a8001015260208101905080610a8052610a809050602060c0825160208401600060025af1610f0a57600080fd5b60c0519050610860526000600061074051602082610b20010152602081019050610220602080602084610b2001018260208501600060046015f150508051820191505080610b2052610b209050602060c0825160208401600060025af1610f7057600080fd5b60c0519050602082610ca00101526020810190506000610360600880602084610c2001018260208501600060046012f15050805182019150506000601860208206610ba00160208284011115610fc557600080fd5b602080610bc082610720600060046015f1505081815280905090509050601880602084610c2001018260208501600060046014f150508051820191505061086051602082610c2001015260208101905080610c2052610c209050602060c0825160208401600060025af161103857600080fd5b60c0519050602082610ca001015260208101905080610ca052610ca09050602060c0825160208401600060025af161106f57600080fd5b60c0519050610b0052600180546001825401101561108c57600080fd5b6001815401815550600154610d2052610d4060006020818352015b60016001610d20511614156110dc57610b0051610d4051602081106110cb57600080fd5b600060c052602060c0200155611170565b6000610d4051602081106110ef57600080fd5b600060c052602060c0200154602082610d60010152602081019050610b0051602082610d6001015260208101905080610d6052610d609050602060c0825160208401600060025af161114057600080fd5b60c0519050610b0052610d20600261115757600080fd5b60028151048152505b81516001018083528114156110a7575b5050005b60006000fd5b61017f6112f90361017f60003961017f6112f9036000f3"} \ No newline at end of file diff --git a/deposit_contract/requirements-testing.txt b/deposit_contract/requirements-testing.txt index b3a90a88a7..280d7e5279 100644 --- a/deposit_contract/requirements-testing.txt +++ b/deposit_contract/requirements-testing.txt @@ -1,5 +1,5 @@ eth-tester[py-evm]==0.1.0b39 -vyper==0.1.0b9 +vyper==0.1.0b10 web3==5.0.0b2 pytest==3.6.1 ../test_libs/pyspec From ce336d4a391c0f5dc207d58bcf7b336293d79d74 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 30 Jun 2019 22:31:14 -0600 Subject: [PATCH 405/405] last few config fixes to match spec --- configs/constant_presets/mainnet.yaml | 15 ++++++++++----- configs/constant_presets/minimal.yaml | 12 +++++++----- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 55993ada1e..10ab26a00a 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -19,6 +19,9 @@ CHURN_LIMIT_QUOTIENT: 65536 SHUFFLE_ROUND_COUNT: 90 # `2**16` (= 65,536) MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 65536 +# Jan 3, 2020 +MIN_GENESIS_TIME: 1578009600 + # Deposit contract @@ -41,7 +44,6 @@ EFFECTIVE_BALANCE_INCREMENT: 1000000000 # Initial values # --------------------------------------------------------------- -GENESIS_FORK_VERSION: 0x00000000 # 0, GENESIS_EPOCH is derived from this constant GENESIS_SLOT: 0 BLS_WITHDRAWAL_PREFIX: 0x00 @@ -51,8 +53,8 @@ BLS_WITHDRAWAL_PREFIX: 0x00 # --------------------------------------------------------------- # 6 seconds 6 seconds SECONDS_PER_SLOT: 6 -# 2**2 (= 4) slots 24 seconds -MIN_ATTESTATION_INCLUSION_DELAY: 4 +# 2**0 (= 1) slots 6 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 # 2**6 (= 64) slots 6.4 minutes SLOTS_PER_EPOCH: 64 # 2**0 (= 1) epochs 6.4 minutes @@ -71,6 +73,9 @@ PERSISTENT_COMMITTEE_PERIOD: 2048 MAX_EPOCHS_PER_CROSSLINK: 64 # 2**2 (= 4) epochs 25.6 minutes MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 +# 2**14 (= 16,384) epochs ~73 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 16384 + # State vector lengths @@ -87,8 +92,8 @@ VALIDATOR_REGISTRY_LIMIT: 1099511627776 # Reward and penalty quotients # --------------------------------------------------------------- -# 2**5 (= 32) -BASE_REWARD_FACTOR: 32 +# 2**6 (= 64) +BASE_REWARD_FACTOR: 64 # 2**9 (= 512) WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 153b55a6cd..b030333ff0 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -18,6 +18,9 @@ CHURN_LIMIT_QUOTIENT: 65536 SHUFFLE_ROUND_COUNT: 10 # [customized] MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 64 +# Jan 3, 2020 +MIN_GENESIS_TIME: 1578009600 + # Deposit contract @@ -40,7 +43,6 @@ EFFECTIVE_BALANCE_INCREMENT: 1000000000 # Initial values # --------------------------------------------------------------- -GENESIS_FORK_VERSION: 0x00000000 # 0, GENESIS_EPOCH is derived from this constant GENESIS_SLOT: 0 BLS_WITHDRAWAL_PREFIX: 0x00 @@ -50,8 +52,8 @@ BLS_WITHDRAWAL_PREFIX: 0x00 # --------------------------------------------------------------- # 6 seconds 6 seconds SECONDS_PER_SLOT: 6 -# [customized] 2 slots -MIN_ATTESTATION_INCLUSION_DELAY: 2 +# 2**0 (= 1) slots 6 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 # [customized] fast epochs SLOTS_PER_EPOCH: 8 # 2**0 (= 1) epochs @@ -88,8 +90,8 @@ VALIDATOR_REGISTRY_LIMIT: 1099511627776 # Reward and penalty quotients # --------------------------------------------------------------- -# 2**5 (= 32) -BASE_REWARD_FACTOR: 32 +# 2**6 (= 64) +BASE_REWARD_FACTOR: 64 # 2**9 (= 512) WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8)