Merge branch 'master' into server-api

.github/workflows/fluffy.yml

@@ -80,6 +80,8 @@ jobs:
 
       - name: run test app with simulator
         run: |
+          : find / -name docker-compose -printf "%h\n%f\n%m\n\n" 2>/dev/null
+          PATH=$PATH$(find /usr/libexec/docker -name docker-compose -printf ":%h")
           SCENARIO="drop-rate --delay=15ms --bandwidth=10Mbps --queue=25 --rate_to_client=10 --rate_to_server=10" docker-compose -f fluffy/tools/utp_testing/docker/docker-compose.yml up -d
 
       - name: wait 5 seconds for containers to start
@@ -98,7 +100,9 @@ jobs:
 
       - name: Stop containers
         if: always()
-        run: docker-compose -f fluffy/tools/utp_testing/docker/docker-compose.yml down
+        run: |
+          PATH=$PATH$(find /usr/libexec/docker -name docker-compose -printf ":%h")
+          docker-compose -f fluffy/tools/utp_testing/docker/docker-compose.yml down
 
   build:
     strategy:

fluffy/docs/the_fluffy_book/docs/fluffy-with-portal-hive.md

@@ -17,11 +17,11 @@ go build .
 Example commands for running test suites:
 
 ```sh
-# Run the history tests with the 3 different clients
-./hive --sim history --client fluffy,trin,ultralight
+# Run the portal tests with only the fluffy client
+./hive --sim portal --client fluffy
 
-# Run the state tests with only the fluffy client
-./hive --sim state --client fluffy
+# Run the portal tests with the 3 different clients
+./hive --sim portal --client fluffy,trin,ultralight
 
 # Access results through web-ui:
 ```sh

fluffy/network/state/content/content_keys.nim

@@ -23,7 +23,7 @@ export ssz_serialization, common_types, hash, results
 type
   NodeHash* = KeccakHash
   CodeHash* = KeccakHash
-  Address* = EthAddress
+  AddressHash* = KeccakHash
 
   ContentType* = enum
     # Note: Need to add this unused value as a case object with an enum without
@@ -43,12 +43,12 @@ type
     nodeHash*: NodeHash
 
   ContractTrieNodeKey* = object
-    address*: Address
+    addressHash*: AddressHash
     path*: Nibbles
     nodeHash*: NodeHash
 
   ContractCodeKey* = object
-    address*: Address
+    addressHash*: AddressHash
     codeHash*: CodeHash
 
   ContentKey* = object
@@ -68,12 +68,15 @@ func init*(T: type AccountTrieNodeKey, path: Nibbles, nodeHash: NodeHash): T =
   AccountTrieNodeKey(path: path, nodeHash: nodeHash)
 
 func init*(
-    T: type ContractTrieNodeKey, address: Address, path: Nibbles, nodeHash: NodeHash
+    T: type ContractTrieNodeKey,
+    addressHash: AddressHash,
+    path: Nibbles,
+    nodeHash: NodeHash,
 ): T =
-  ContractTrieNodeKey(address: address, path: path, nodeHash: nodeHash)
+  ContractTrieNodeKey(addressHash: addressHash, path: path, nodeHash: nodeHash)
 
-func init*(T: type ContractCodeKey, address: Address, codeHash: CodeHash): T =
-  ContractCodeKey(address: address, codeHash: codeHash)
+func init*(T: type ContractCodeKey, addressHash: AddressHash, codeHash: CodeHash): T =
+  ContractCodeKey(addressHash: addressHash, codeHash: codeHash)
 
 func toContentKey*(key: AccountTrieNodeKey): ContentKey =
   ContentKey(contentType: accountTrieNode, accountTrieNodeKey: key)

fluffy/network/state/state_endpoints.nim

@@ -67,7 +67,7 @@ proc getNextNodeHash(
     raiseAssert(e.msg)
 
 proc getAccountProof(
-    n: StateNetwork, stateRoot: KeccakHash, address: Address
+    n: StateNetwork, stateRoot: KeccakHash, address: EthAddress
 ): Future[Opt[TrieProof]] {.async: (raises: [CancelledError]).} =
   let nibbles = address.toPath().unpackNibbles()
 
@@ -94,13 +94,14 @@ proc getAccountProof(
   Opt.some(proof)
 
 proc getStorageProof(
-    n: StateNetwork, storageRoot: KeccakHash, address: Address, storageKey: UInt256
+    n: StateNetwork, storageRoot: KeccakHash, address: EthAddress, storageKey: UInt256
 ): Future[Opt[TrieProof]] {.async: (raises: [CancelledError]).} =
   let nibbles = storageKey.toPath().unpackNibbles()
 
   var
+    addressHash = keccakHash(address)
     nibblesIdx = 0
-    key = ContractTrieNodeKey.init(address, Nibbles.empty(), storageRoot)
+    key = ContractTrieNodeKey.init(addressHash, Nibbles.empty(), storageRoot)
     proof = TrieProof.empty()
 
   while nibblesIdx < nibbles.len():
@@ -116,12 +117,12 @@ proc getStorageProof(
     let (nextPath, nextNodeHash) = trieNode.getNextNodeHash(nibbles, nibblesIdx).valueOr:
       break
 
-    key = ContractTrieNodeKey.init(address, nextPath, nextNodeHash)
+    key = ContractTrieNodeKey.init(addressHash, nextPath, nextNodeHash)
 
   Opt.some(proof)
 
 proc getAccount(
-    n: StateNetwork, blockHash: BlockHash, address: Address
+    n: StateNetwork, blockHash: BlockHash, address: EthAddress
 ): Future[Opt[Account]] {.async: (raises: [CancelledError]).} =
   let
     stateRoot = (await n.getStateRootByBlockHash(blockHash)).valueOr:
@@ -138,7 +139,7 @@ proc getAccount(
 
 # Used by: eth_getBalance,
 proc getBalance*(
-    n: StateNetwork, blockHash: BlockHash, address: Address
+    n: StateNetwork, blockHash: BlockHash, address: EthAddress
 ): Future[Opt[UInt256]] {.async: (raises: [CancelledError]).} =
   let account = (await n.getAccount(blockHash, address)).valueOr:
     return Opt.none(UInt256)
@@ -147,7 +148,7 @@ proc getBalance*(
 
 # Used by: eth_getTransactionCount
 proc getTransactionCount*(
-    n: StateNetwork, blockHash: BlockHash, address: Address
+    n: StateNetwork, blockHash: BlockHash, address: EthAddress
 ): Future[Opt[AccountNonce]] {.async: (raises: [CancelledError]).} =
   let account = (await n.getAccount(blockHash, address)).valueOr:
     return Opt.none(AccountNonce)
@@ -156,7 +157,7 @@ proc getTransactionCount*(
 
 # Used by: eth_getStorageAt
 proc getStorageAt*(
-    n: StateNetwork, blockHash: BlockHash, address: Address, slotKey: UInt256
+    n: StateNetwork, blockHash: BlockHash, address: EthAddress, slotKey: UInt256
 ): Future[Opt[UInt256]] {.async: (raises: [CancelledError]).} =
   let
     account = (await n.getAccount(blockHash, address)).valueOr:
@@ -172,12 +173,12 @@ proc getStorageAt*(
 
 # Used by: eth_getCode
 proc getCode*(
-    n: StateNetwork, blockHash: BlockHash, address: Address
+    n: StateNetwork, blockHash: BlockHash, address: EthAddress
 ): Future[Opt[Bytecode]] {.async: (raises: [CancelledError]).} =
   let
     account = (await n.getAccount(blockHash, address)).valueOr:
       return Opt.none(Bytecode)
-    contractCodeKey = ContractCodeKey.init(address, account.codeHash)
+    contractCodeKey = ContractCodeKey.init(keccakHash(address), account.codeHash)
 
   let contractCodeRetrieval = (await n.getContractCode(contractCodeKey)).valueOr:
     warn "Failed to get contract code"

fluffy/network/state/state_gossip.nim

@@ -60,7 +60,9 @@ func getParent(p: ProofWithPath): ProofWithPath =
     # leaf or extension node so we need to remove one or more nibbles
     let (_, _, prefixNibbles) = decodePrefix(parentEndNode.listElem(0))
 
-    parentProof.withPath(unpackedNibbles.dropN(prefixNibbles.len()).packNibbles())
+    parentProof.withPath(
+      unpackedNibbles.dropN(prefixNibbles.unpackNibbles().len()).packNibbles()
+    )
   except RlpError as e:
     raiseAssert(e.msg)
 
@@ -79,7 +81,7 @@ func getParent*(offerWithKey: ContractTrieOfferWithKey): ContractTrieOfferWithKe
     (key, offer) = offerWithKey
     parent = offer.storageProof.withPath(key.path).getParent()
     parentKey = ContractTrieNodeKey.init(
-      key.address, parent.path, keccakHash(parent.proof[^1].asSeq())
+      key.addressHash, parent.path, keccakHash(parent.proof[^1].asSeq())
     )
     parentOffer =
       ContractTrieNodeOffer.init(parent.proof, offer.accountProof, offer.blockHash)
@@ -97,7 +99,7 @@ proc gossipOffer*(
   let req1Peers = await p.neighborhoodGossip(
     srcNodeId, ContentKeysList.init(@[keyBytes]), @[offerBytes]
   )
-  info "Offered content gossipped successfully with peers", keyBytes, peers = req1Peers
+  debug "Offered content gossipped successfully with peers", keyBytes, peers = req1Peers
 
 proc gossipOffer*(
     p: PortalProtocol,
@@ -110,7 +112,7 @@ proc gossipOffer*(
   let req1Peers = await p.neighborhoodGossip(
     srcNodeId, ContentKeysList.init(@[keyBytes]), @[offerBytes]
   )
-  info "Offered content gossipped successfully with peers", keyBytes, peers = req1Peers
+  debug "Offered content gossipped successfully with peers", keyBytes, peers = req1Peers
 
 proc gossipOffer*(
     p: PortalProtocol,
@@ -123,7 +125,7 @@ proc gossipOffer*(
   let peers = await p.neighborhoodGossip(
     srcNodeId, ContentKeysList.init(@[keyBytes]), @[offerBytes]
   )
-  info "Offered content gossipped successfully with peers", keyBytes, peers
+  debug "Offered content gossipped successfully with peers", keyBytes, peers
 
 # Currently only used for testing to gossip an entire account trie proof
 # This may also be useful for the state network bridge
@@ -134,12 +136,12 @@ proc recursiveGossipOffer*(
     offerBytes: seq[byte],
     key: AccountTrieNodeKey,
     offer: AccountTrieNodeOffer,
-) {.async: (raises: [CancelledError]).} =
+): Future[ContentKeyByteList] {.async: (raises: [CancelledError]).} =
   await gossipOffer(p, srcNodeId, keyBytes, offerBytes, key, offer)
 
   # root node, recursive gossip is finished
   if key.path.unpackNibbles().len() == 0:
-    return
+    return keyBytes
 
   # continue the recursive gossip by sharing the parent offer with peers
   let
@@ -159,12 +161,12 @@ proc recursiveGossipOffer*(
     offerBytes: seq[byte],
     key: ContractTrieNodeKey,
     offer: ContractTrieNodeOffer,
-) {.async: (raises: [CancelledError]).} =
+): Future[ContentKeyByteList] {.async: (raises: [CancelledError]).} =
   await gossipOffer(p, srcNodeId, keyBytes, offerBytes, key, offer)
 
   # root node, recursive gossip is finished
   if key.path.unpackNibbles().len() == 0:
-    return
+    return keyBytes
 
   # continue the recursive gossip by sharing the parent offer with peers
   let

fluffy/network/state/state_network.nim

@@ -173,7 +173,7 @@ proc processOffer*(
   n.portalProtocol.storeContent(
     contentKeyBytes, contentId, contentValue.toRetrievalValue().encode()
   )
-  info "Offered content validated successfully", contentKeyBytes
+  debug "Offered content validated successfully", contentKeyBytes
 
   await gossipOffer(
     n.portalProtocol, maybeSrcNodeId, contentKeyBytes, contentValueBytes, contentKey,

fluffy/network/state/state_utils.nim

@@ -87,7 +87,7 @@ func removeLeafKeyEndNibbles*(
 func toPath*(hash: KeccakHash): Nibbles {.inline.} =
   Nibbles.init(hash.data, isEven = true)
 
-func toPath*(address: Address): Nibbles {.inline.} =
+func toPath*(address: EthAddress): Nibbles {.inline.} =
   keccakHash(address).toPath()
 
 func toPath*(slotKey: UInt256): Nibbles {.inline.} =

fluffy/network/state/state_validation.nim

@@ -156,7 +156,7 @@ proc validateOffer*(
 ): Result[void, string] =
   ?validateTrieProof(
     trustedStateRoot,
-    key.address.toPath(),
+    key.addressHash.toPath(),
     offer.accountProof,
     allowKeyEndInPathForLeafs = true,
   )
@@ -172,7 +172,7 @@ proc validateOffer*(
 ): Result[void, string] =
   ?validateTrieProof(
     trustedStateRoot,
-    key.address.toPath(),
+    key.addressHash.toPath(),
     offer.accountProof,
     allowKeyEndInPathForLeafs = true,
   )

fluffy/tests/state_network_tests/state_test_helpers.nim

@@ -26,17 +26,11 @@ export yaml_utils
 const testVectorDir* = "./vendor/portal-spec-tests/tests/mainnet/state/validation/"
 
 type
-  YamlTrieNodeRecursiveGossipKV* = ref object
-    content_key*: string
-    content_value_offer*: string
-    content_value_retrieval*: string
-
   YamlTrieNodeKV* = object
     state_root*: string
     content_key*: string
     content_value_offer*: string
     content_value_retrieval*: string
-    recursive_gossip*: YamlTrieNodeRecursiveGossipKV
 
   YamlTrieNodeKVs* = seq[YamlTrieNodeKV]
 
@@ -48,16 +42,6 @@ type
 
   YamlContractBytecodeKVs* = seq[YamlContractBytecodeKV]
 
-  YamlRecursiveGossipKV* = object
-    content_key*: string
-    content_value*: string
-
-  YamlRecursiveGossipData* = object
-    state_root*: string
-    recursive_gossip*: seq[YamlRecursiveGossipKV]
-
-  YamlRecursiveGossipKVs* = seq[YamlRecursiveGossipData]
-
 func asNibbles*(key: openArray[byte], isEven = true): Nibbles =
   Nibbles.init(key, isEven)
 

fluffy/tests/state_network_tests/test_state_content_keys_vectors.nim

@@ -8,6 +8,7 @@
 import
   unittest2,
   stew/byteutils,
+  eth/common,
   ../../network/state/state_content,
   ../../eth_data/yaml_utils
 
@@ -58,10 +59,10 @@ suite "State Content Keys":
         raiseAssert "Cannot read test vector: " & error
 
       packedNibbles = packNibbles(testCase.path)
-      address = Address.fromHex(testCase.address)
+      addressHash = EthAddress.fromHex(testCase.address).keccakHash()
       nodeHash = NodeHash.fromHex(testCase.node_hash)
       contentKey =
-        ContractTrieNodeKey.init(address, packedNibbles, nodeHash).toContentKey()
+        ContractTrieNodeKey.init(addressHash, packedNibbles, nodeHash).toContentKey()
       encoded = contentKey.encode()
 
     check:
@@ -73,7 +74,9 @@ suite "State Content Keys":
       decoded.isOk()
       decoded.value().contentType == contractTrieNode
       decoded.value().contractTrieNodeKey ==
-        ContractTrieNodeKey(address: address, path: packedNibbles, nodeHash: nodeHash)
+        ContractTrieNodeKey(
+          addressHash: addressHash, path: packedNibbles, nodeHash: nodeHash
+        )
 
   test "Encode/decode ContractCodeKey":
     const file = testVectorDir & "contract_bytecode_key.yaml"
@@ -88,9 +91,9 @@ suite "State Content Keys":
       testCase = YamlContractBytecodeKey.loadFromYaml(file).valueOr:
         raiseAssert "Cannot read test vector: " & error
 
-      address = Address.fromHex(testCase.address)
+      addressHash = EthAddress.fromHex(testCase.address).keccakHash()
       codeHash = CodeHash.fromHex(testCase.code_hash)
-      contentKey = ContractCodeKey.init(address, codeHash).toContentKey()
+      contentKey = ContractCodeKey.init(addressHash, codeHash).toContentKey()
       encoded = contentKey.encode()
 
     check:
@@ -101,7 +104,7 @@ suite "State Content Keys":
     check:
       decoded.isOk()
       decoded.value().contentType == contractCode
-      decoded.value().contractCodeKey.address == address
+      decoded.value().contractCodeKey.addressHash == addressHash
       decoded.value().contractCodeKey.codeHash == codeHash
 
   test "Invalid prefix - 0 value":

fluffy/tests/state_network_tests/test_state_endpoints_genesis.nim

@@ -43,8 +43,8 @@ suite "State Endpoints - Genesis JSON Files":
         let
           proof = accountState.generateAccountProof(address)
           leafNode = proof[^1]
-          addressHash = keccakHash(address).data
-          path = removeLeafKeyEndNibbles(Nibbles.init(addressHash, true), leafNode)
+          addressHash = keccakHash(address)
+          path = removeLeafKeyEndNibbles(Nibbles.init(addressHash.data, true), leafNode)
           key = AccountTrieNodeKey.init(path, keccakHash(leafNode.asSeq()))
           offer = AccountTrieNodeOffer(proof: proof)
 
@@ -94,8 +94,9 @@ suite "State Endpoints - Genesis JSON Files":
           block:
             # store the code
             let
-              key =
-                ContractCodeKey(address: address, codeHash: keccakHash(account.code))
+              key = ContractCodeKey(
+                addressHash: addressHash, codeHash: keccakHash(account.code)
+              )
               value = ContractCodeRetrieval(code: Bytecode.init(account.code))
 
             let contentKey = key.toContentKey().encode()
@@ -121,7 +122,9 @@ suite "State Endpoints - Genesis JSON Files":
                 Nibbles.init(keccakHash(toBytesBE(slotKey)).data, true), leafNode
               )
               key = ContractTrieNodeKey(
-                address: address, path: path, nodeHash: keccakHash(leafNode.asSeq())
+                addressHash: addressHash,
+                path: path,
+                nodeHash: keccakHash(leafNode.asSeq()),
               )
               offer =
                 ContractTrieNodeOffer(storageProof: storageProof, accountProof: proof)