Fluffy recursive gossip improvements, fixes and tests. (#2231)

* Refactor get parent gossip code and add Nibbles helper function.

* Add logging to state gossip.

* Unit test recursive gossip using state gossip getParent functions.

* Add recursive gossip genesis json test and fix bug in state gossip getParent.

* Add Nibbles len function.

fluffy/network/state/content/nibbles.nim

@@ -25,6 +25,7 @@ const
  MAX_UNPACKED_NIBBLES_LEN = 64
 
 type Nibbles* = List[byte, MAX_PACKED_NIBBLES_LEN]
+type UnpackedNibbles* = seq[byte]
 
 func init*(T: type Nibbles, packed: openArray[byte], isEven: bool): T =
  doAssert(packed.len() <= MAX_PACKED_NIBBLES_LEN)
@@ -79,7 +80,7 @@ func packNibbles*(unpacked: openArray[byte]): Nibbles =
 
  Nibbles(output)
 
-func unpackNibbles*(packed: Nibbles): seq[byte] =
+func unpackNibbles*(packed: Nibbles): UnpackedNibbles =
  doAssert(packed.len() <= MAX_PACKED_NIBBLES_LEN, "Packed nibbles length is too long")
 
  var output = newSeqOfCap[byte](packed.len() * 2)
@@ -98,4 +99,17 @@ func unpackNibbles*(packed: Nibbles): seq[byte] =
  output.add(first)
  output.add(second)
 
- output
+ move(output)
+
+func len(packed: Nibbles): int =
+ let lenExclPrefix = (packed.len() - 1) * 2
+
+ if packed[0] == 0x00: # is even length
+ lenExclPrefix
+ else:
+ lenExclPrefix + 1
+
+func dropN*(unpacked: UnpackedNibbles, num: int): UnpackedNibbles =
+ var nibbles = unpacked
+ nibbles.setLen(nibbles.len() - num)
+ move(nibbles)

fluffy/network/state/state_gossip.nim

@@ -19,57 +19,71 @@ export results, state_content
 logScope:
  topics = "portal_state"
 
-func getParent(nibbles: Nibbles, proof: TrieProof): (Nibbles, TrieProof) =
- doAssert(nibbles.len() > 0, "nibbles too short")
- doAssert(proof.len() > 1, "proof too short")
+type ProofWithPath = tuple[path: Nibbles, proof: TrieProof]
+
+type AccountTrieOfferWithKey* =
+ tuple[key: AccountTrieNodeKey, offer: AccountTrieNodeOffer]
+
+type ContractTrieOfferWithKey* =
+ tuple[key: ContractTrieNodeKey, offer: ContractTrieNodeOffer]
+
+func withPath(proof: TrieProof, path: Nibbles): ProofWithPath =
+ (path: path, proof: proof)
+
+func withKey*(
+ offer: AccountTrieNodeOffer, key: AccountTrieNodeKey
+): AccountTrieOfferWithKey =
+ (key: key, offer: offer)
+
+func withKey*(
+ offer: ContractTrieNodeOffer, key: ContractTrieNodeKey
+): ContractTrieOfferWithKey =
+ (key: key, offer: offer)
+
+func getParent(p: ProofWithPath): ProofWithPath =
+ doAssert(p.path.len() > 0, "nibbles too short")
+ doAssert(p.proof.len() > 1, "proof too short")
 
  let
- parentProof = TrieProof.init(proof[0 ..^ 2])
+ parentProof = TrieProof.init(p.proof[0 ..^ 2])
  parentEndNode = rlpFromBytes(parentProof[^1].asSeq())
 
  # the trie proof should have already been validated when receiving the offer content
  doAssert(parentEndNode.listLen() == 2 or parentEndNode.listLen() == 17)
 
- var unpackedNibbles = nibbles.unpackNibbles()
+ var unpackedNibbles = p.path.unpackNibbles()
 
  if parentEndNode.listLen() == 17:
  # branch node so only need to remove a single nibble
- unpackedNibbles.setLen(unpackedNibbles.len() - 1)
- return (unpackedNibbles.packNibbles(), parentProof)
+ return parentProof.withPath(unpackedNibbles.dropN(1).packNibbles())
 
  # leaf or extension node so we need to remove one or more nibbles
- let (_, isEven, prefixNibbles) = decodePrefix(parentEndNode.listElem(0))
-
- var removeCount = (prefixNibbles.len() - 1) * 2
- if not isEven:
- inc removeCount
+ let prefixNibbles = decodePrefix(parentEndNode.listElem(0))[2]
 
- unpackedNibbles.setLen(unpackedNibbles.len() - removeCount)
- (unpackedNibbles.packNibbles(), parentProof)
+ parentProof.withPath(unpackedNibbles.dropN(prefixNibbles.len()).packNibbles())
 
-func getParent*(
- key: AccountTrieNodeKey, offer: AccountTrieNodeOffer
-): (AccountTrieNodeKey, AccountTrieNodeOffer) =
+func getParent*(offerWithKey: AccountTrieOfferWithKey): AccountTrieOfferWithKey =
  let
- (parentNibbles, parentProof) = getParent(key.path, offer.proof)
- parentKey =
- AccountTrieNodeKey.init(parentNibbles, keccakHash(parentProof[^1].asSeq()))
+ (key, offer) = offerWithKey
+ (parentPath, parentProof) = offer.proof.withPath(key.path).getParent()
+
+ parentKey = AccountTrieNodeKey.init(parentPath, keccakHash(parentProof[^1].asSeq()))
  parentOffer = AccountTrieNodeOffer.init(parentProof, offer.blockHash)
 
- (parentKey, parentOffer)
+ parentOffer.withKey(parentKey)
 
-func getParent*(
- key: ContractTrieNodeKey, offer: ContractTrieNodeOffer
-): (ContractTrieNodeKey, ContractTrieNodeOffer) =
+func getParent*(offerWithKey: ContractTrieOfferWithKey): ContractTrieOfferWithKey =
  let
- (parentNibbles, parentProof) = getParent(key.path, offer.storageProof)
+ (key, offer) = offerWithKey
+ (parentPath, parentProof) = offer.storageProof.withPath(key.path).getParent()
+
  parentKey = ContractTrieNodeKey.init(
- key.address, parentNibbles, keccakHash(parentProof[^1].asSeq())
+ key.address, parentPath, keccakHash(parentProof[^1].asSeq())
  )
  parentOffer =
  ContractTrieNodeOffer.init(parentProof, offer.accountProof, offer.blockHash)
 
- (parentKey, parentOffer)
+ parentOffer.withKey(parentKey)
 
 proc gossipOffer*(
  p: PortalProtocol,
@@ -79,20 +93,25 @@ proc gossipOffer*(
  key: AccountTrieNodeKey,
  offer: AccountTrieNodeOffer,
 ) {.async.} =
- asyncSpawn p.neighborhoodGossipDiscardPeers(
+ let req1Peers = await p.neighborhoodGossip(
  srcNodeId, ContentKeysList.init(@[keyBytes]), @[offerBytes]
  )
+ info "Offered content gossipped successfully with peers", keyBytes, peers = req1Peers
 
  # root node, recursive gossip is finished
  if key.path.unpackNibbles().len() == 0:
  return
 
- let (parentKey, parentOffer) = getParent(key, offer)
- asyncSpawn p.neighborhoodGossipDiscardPeers(
- srcNodeId,
- ContentKeysList.init(@[parentKey.toContentKey().encode()]),
- @[parentOffer.encode()],
- )
+ # continue the recursive gossip by sharing the parent offer with peers
+ let
+ (parentKey, parentOffer) = offer.withKey(key).getParent()
+ parentKeyBytes = parentKey.toContentKey().encode()
+ req2Peers = await p.neighborhoodGossip(
+ srcNodeId, ContentKeysList.init(@[parentKeyBytes]), @[parentOffer.encode()]
+ )
+
+ info "Offered content parent gossipped successfully with peers",
+ parentKeyBytes, peers = req2Peers
 
 proc gossipOffer*(
  p: PortalProtocol,
@@ -102,20 +121,25 @@ proc gossipOffer*(
  key: ContractTrieNodeKey,
  offer: ContractTrieNodeOffer,
 ) {.async.} =
- asyncSpawn p.neighborhoodGossipDiscardPeers(
+ let req1Peers = await p.neighborhoodGossip(
  srcNodeId, ContentKeysList.init(@[keyBytes]), @[offerBytes]
  )
+ info "Offered content gossipped successfully with peers", keyBytes, peers = req1Peers
 
  # root node, recursive gossip is finished
  if key.path.unpackNibbles().len() == 0:
  return
 
- let (parentKey, parentOffer) = getParent(key, offer)
- asyncSpawn p.neighborhoodGossipDiscardPeers(
- srcNodeId,
- ContentKeysList.init(@[parentKey.toContentKey().encode()]),
- @[parentOffer.encode()],
- )
+ # continue the recursive gossip by sharing the parent offer with peers
+ let
+ (parentKey, parentOffer) = offer.withKey(key).getParent()
+ parentKeyBytes = parentKey.toContentKey().encode()
+ req2Peers = await p.neighborhoodGossip(
+ srcNodeId, ContentKeysList.init(@[parentKeyBytes]), @[parentOffer.encode()]
+ )
+
+ info "Offered content parent gossipped successfully with peers",
+ parentKeyBytes, peers = req2Peers
 
 proc gossipOffer*(
  p: PortalProtocol,
@@ -125,6 +149,7 @@ proc gossipOffer*(
  key: ContractCodeKey,
  offer: ContractCodeOffer,
 ) {.async.} =
- asyncSpawn p.neighborhoodGossipDiscardPeers(
+ let peers = await p.neighborhoodGossip(
  srcNodeId, ContentKeysList.init(@[keyBytes]), @[offerBytes]
  )
+ info "Offered content gossipped successfully with peers", keyBytes, peers

fluffy/network/state/state_network.nim

@@ -162,15 +162,15 @@ proc processOffer(
 
  let res = validateOffer(stateRoot, contentKey, contentValue)
  if res.isErr():
- return err("Received offered content failed validation: " & res.error())
+ return err("Offered content failed validation: " & res.error())
 
  let contentId = n.portalProtocol.toContentId(contentKeyBytes).valueOr:
  return err("Received offered content with invalid content key")
 
  n.portalProtocol.storeContent(
  contentKeyBytes, contentId, contentValue.toRetrievalValue().encode()
  )
- info "Received offered content validated successfully", contentKeyBytes
+ info "Offered content validated successfully", contentKeyBytes
 
  asyncSpawn gossipOffer(
  n.portalProtocol, maybeSrcNodeId, contentKeyBytes, contentValueBytes, contentKey,

fluffy/network/state/state_validation.nim

@@ -59,7 +59,7 @@ proc validateTrieProof*(
  if isLastNode:
  break
  else:
- return err("empty nibbles but proof has more nodes")
+ return err("proof has more nodes then expected for given path")
 
  case thisNodeRlp.listLen()
  of 2:

fluffy/tests/state_network_tests/all_state_network_tests.nim

@@ -9,10 +9,12 @@
 
 import
  ./test_state_content_keys,
- ./test_state_content_values,
  ./test_state_content_nibbles,
- ./test_state_network,
+ ./test_state_content_values,
  #./test_state_network_gossip,
- ./test_state_validation,
+ ./test_state_network,
+ ./test_state_recursivegossip_genesis,
+ ./test_state_recursivegossip_vectors,
  ./test_state_validation_genesis,
- ./test_state_validation_trieproof
+ ./test_state_validation_trieproof,
+ ./test_state_validation_vectors

fluffy/tests/state_network_tests/state_test_helpers.nim

@@ -11,7 +11,45 @@ import
  std/[sugar, sequtils],
  eth/[common, trie, trie/db],
  ../../nimbus/common/chain_config,
- ../../network/state/[state_content, state_utils]
+ ../../network/state/[state_content, state_utils],
+ ../../eth_data/yaml_utils
+
+export yaml_utils
+
+const testVectorDir* = "./vendor/portal-spec-tests/tests/mainnet/state/validation/"
+
+type
+ YamlTrieNodeRecursiveGossipKV* = ref object
+ content_key*: string
+ content_value_offer*: string
+ content_value_retrieval*: string
+
+ YamlTrieNodeKV* = object
+ state_root*: string
+ content_key*: string
+ content_value_offer*: string
+ content_value_retrieval*: string
+ recursive_gossip*: YamlTrieNodeRecursiveGossipKV
+
+ YamlTrieNodeKVs* = seq[YamlTrieNodeKV]
+
+ YamlContractBytecodeKV* = object
+ state_root*: string
+ content_key*: string
+ content_value_offer*: string
+ content_value_retrieval*: string
+
+ YamlContractBytecodeKVs* = seq[YamlContractBytecodeKV]
+
+ YamlRecursiveGossipKV* = object
+ content_key*: string
+ content_value*: string
+
+ YamlRecursiveGossipData* = object
+ state_root*: string
+ recursive_gossip*: seq[YamlRecursiveGossipKV]
+
+ YamlRecursiveGossipKVs* = seq[YamlRecursiveGossipData]
 
 func asNibbles*(key: openArray[byte], isEven = true): Nibbles =
  Nibbles.init(key, isEven)
@@ -28,8 +66,7 @@ func removeLeafKeyEndNibbles*(
  var unpackedNibbles = nibbles.unpackNibbles()
  doAssert(unpackedNibbles[^leafPrefix.len() .. ^1] == leafPrefix)
 
- unpackedNibbles.setLen(unpackedNibbles.len() - leafPrefix.len())
- unpackedNibbles.packNibbles()
+ unpackedNibbles.dropN(leafPrefix.len()).packNibbles()
 
 func asTrieProof*(branch: openArray[seq[byte]]): TrieProof =
  TrieProof.init(branch.map(node => TrieNode.init(node)))
@@ -38,6 +75,8 @@ proc getTrieProof*(
  state: HexaryTrie, key: openArray[byte]
 ): TrieProof {.raises: [RlpError].} =
  let branch = state.getBranch(key)
+ # for node in branch:
+ # debugEcho rlp.decode(node)
  branch.asTrieProof()
 
 proc generateAccountProof*(