Updates to Fluffy to support hive tests. (#2333)

* Updates to Fluffy book for hive tests.

* Add support to disable state root checks for state content from the command line.

* Update portal_stateStore endpoint to support decoding offer and storing retrieval value.

fluffy/conf.nim

@@ -299,6 +299,13 @@ type
  name: "disable-poke"
  .}: bool
 
+ disableStateRootValidation* {.
+ hidden,
+ desc: "Disables state root validation for content received by the state network.",
+ defaultValue: false,
+ name: "disable-state-root-validation"
+ .}: bool
+
  case cmd* {.command, defaultValue: noCommand.}: PortalCmd
  of noCommand:
  discard

fluffy/docs/the_fluffy_book/docs/fluffy-with-portal-hive.md

@@ -1,27 +1,27 @@
 # Fluffy with Portal-hive
 
-Fluffy is one of the Portal clients that is being tested with [portal-hive](https://github.com/ethereum/portal-hive).
+Fluffy is one of the Portal clients that is being tested with [hive](https://github.com/ethereum/hive).
 
 To see the status of the tests for the current version you can access [https://portal-hive.ethdevops.io/](https://portal-hive.ethdevops.io/).
 
 ## Run the hive tests locally
 
-Build portal-hive:
+Build hive:
 
 ```sh
-git clone https://github.com/ethereum/portal-hive.git
-cd ./portal-hive
+git clone https://github.com/ethereum/hive.git
+cd ./hive
 go build .
 ```
 
 Example commands for running test suites:
 
 ```sh
-# Run the rpc-compat tests with the 3 different clients
-./hive --sim rpc-compat --client fluffy,trin,ultralight
+# Run the history tests with the 3 different clients
+./hive --sim history --client fluffy,trin,ultralight
 
-# Run the portal-interop tests with only the fluffy client
-./hive --sim portal-interop --client fluffy
+# Run the state tests with only the fluffy client
+./hive --sim state --client fluffy
 
 # Access results through web-ui:
 ```sh
@@ -30,7 +30,7 @@ go build ./cmd/hiveview
 ```
 
 !!! note
- You can see all the implemented simulators in [https://github.com/ethereum/portal-hive/tree/main/simulators](https://github.com/ethereum/portal-hive/tree/main/simulators)
+ You can see all the implemented simulators in [https://github.com/ethereum/hive/tree/master/simulators](https://github.com/ethereum/hive/tree/master/simulators)
 
 ## Build a local development Docker image for portal-hive
 
@@ -55,4 +55,3 @@ docker build --tag fluffy-dev --file ./fluffy/tools/docker/Dockerfile.portalhive
  The `./vendors` dir is dockerignored and cached. If you have to make local
  changes to one of the dependencies in that directory you will have to remove
  `vendors/` from `./fluffy/tools/docker/Dockerfile.portalhive.dockerignore`.
-

fluffy/fluffy.nim

@@ -234,6 +234,7 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
  bootstrapRecords = bootstrapRecords,
  portalConfig = portalConfig,
  historyNetwork = historyNetwork,
+ not config.disableStateRootValidation,
  )
  )
  else:

fluffy/network/state/state_network.nim

@@ -32,6 +32,7 @@ type StateNetwork* = ref object
  contentQueue*: AsyncQueue[(Opt[NodeId], ContentKeysList, seq[seq[byte]])]
  processContentLoop: Future[void]
  historyNetwork: Opt[HistoryNetwork]
+ validateStateIsCanonical: bool
 
 func toContentIdHandler(contentKey: ByteList): results.Opt[ContentId] =
  ok(toContentId(contentKey))
@@ -44,6 +45,7 @@ proc new*(
  bootstrapRecords: openArray[Record] = [],
  portalConfig: PortalProtocolConfig = defaultPortalProtocolConfig,
  historyNetwork = Opt.none(HistoryNetwork),
+ validateStateIsCanonical = true,
 ): T =
  let cq = newAsyncQueue[(Opt[NodeId], ContentKeysList, seq[seq[byte]])](50)
 
@@ -67,6 +69,7 @@ proc new*(
  contentDB: contentDB,
  contentQueue: cq,
  historyNetwork: historyNetwork,
+ validateStateIsCanonical: validateStateIsCanonical,
  )
 
 proc getContent(
@@ -146,10 +149,15 @@ proc processOffer*(
  let contentValue = V.decode(contentValueBytes).valueOr:
  return err("Unable to decode offered content value")
 
- let stateRoot = (await n.getStateRootByBlockHash(contentValue.blockHash)).valueOr:
- return err("Failed to get state root by block hash")
+ let res =
+ if n.validateStateIsCanonical:
+ let stateRoot = (await n.getStateRootByBlockHash(contentValue.blockHash)).valueOr:
+ return err("Failed to get state root by block hash")
+ validateOffer(Opt.some(stateRoot), contentKey, contentValue)
+ else:
+ # Skip state root validation
+ validateOffer(Opt.none(KeccakHash), contentKey, contentValue)
 
- let res = validateOffer(stateRoot, contentKey, contentValue)
  if res.isErr():
  return err("Offered content failed validation: " & res.error())
 

fluffy/network/state/state_validation.nim

@@ -30,16 +30,18 @@ proc isValidNextNode(thisNodeRlp: Rlp, rlpIdx: int, nextNode: TrieNode): bool =
  nextNode.hashEquals(nextHash)
 
 proc validateTrieProof*(
- expectedRootHash: KeccakHash,
+ expectedRootHash: Opt[KeccakHash],
  path: Nibbles,
  proof: TrieProof,
  allowKeyEndInPathForLeafs = false,
 ): Result[void, string] =
  if proof.len() == 0:
  return err("proof is empty")
 
- if not proof[0].hashEquals(expectedRootHash):
- return err("hash of proof root node doesn't match the expected root hash")
+ # TODO: Remove this once the hive tests support passing in state roots from the history network
+ if expectedRootHash.isSome():
+ if not proof[0].hashEquals(expectedRootHash.get()):
+ return err("hash of proof root node doesn't match the expected root hash")
 
  let nibbles = path.unpackNibbles()
  if nibbles.len() == 0:
@@ -131,14 +133,18 @@ proc validateRetrieval*(
  err("hash of bytecode doesn't match the expected code hash")
 
 proc validateOffer*(
- trustedStateRoot: KeccakHash, key: AccountTrieNodeKey, offer: AccountTrieNodeOffer
+ trustedStateRoot: Opt[KeccakHash],
+ key: AccountTrieNodeKey,
+ offer: AccountTrieNodeOffer,
 ): Result[void, string] =
  ?validateTrieProof(trustedStateRoot, key.path, offer.proof)
 
  validateRetrieval(key, offer.toRetrievalValue())
 
 proc validateOffer*(
- trustedStateRoot: KeccakHash, key: ContractTrieNodeKey, offer: ContractTrieNodeOffer
+ trustedStateRoot: Opt[KeccakHash],
+ key: ContractTrieNodeKey,
+ offer: ContractTrieNodeOffer,
 ): Result[void, string] =
  ?validateTrieProof(
  trustedStateRoot,
@@ -149,12 +155,12 @@ proc validateOffer*(
 
  let account = ?offer.accountProof.toAccount()
 
- ?validateTrieProof(account.storageRoot, key.path, offer.storageProof)
+ ?validateTrieProof(Opt.some(account.storageRoot), key.path, offer.storageProof)
 
  validateRetrieval(key, offer.toRetrievalValue())
 
 proc validateOffer*(
- trustedStateRoot: KeccakHash, key: ContractCodeKey, offer: ContractCodeOffer
+ trustedStateRoot: Opt[KeccakHash], key: ContractCodeKey, offer: ContractCodeOffer
 ): Result[void, string] =
  ?validateTrieProof(
  trustedStateRoot,

fluffy/rpc/rpc_portal_api.nim

@@ -13,6 +13,7 @@ import
  json_serialization/std/tables,
  stew/byteutils,
  ../network/wire/portal_protocol,
+ ../network/state/state_content,
  ./rpc_types
 
 {.warning[UnusedImport]: off.}
@@ -42,6 +43,12 @@ TraceResponse.useDefaultSerializationIn JrpcConv
 proc installPortalApiHandlers*(
  rpcServer: RpcServer | RpcProxy, p: PortalProtocol, network: static string
 ) =
+ let
+ invalidKeyErr =
+ (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+ invalidValueErr =
+ (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content value")
+
  rpcServer.rpc("portal_" & network & "NodeInfo") do() -> NodeInfo:
  return p.routingTable.getNodeInfo()
 
@@ -212,14 +219,39 @@ proc installPortalApiHandlers*(
  rpcServer.rpc("portal_" & network & "Store") do(
  contentKey: string, contentValue: string
  ) -> bool:
- let key = ByteList.init(hexToSeqByte(contentKey))
- let contentId = p.toContentId(key)
+ let
+ key = ByteList.init(hexToSeqByte(contentKey))
+ contentValueBytes = hexToSeqByte(contentValue)
+
+ let valueToStore =
+ if network == "state":
+ let decodedKey = ContentKey.decode(key).valueOr:
+ raise invalidKeyErr
+
+ case decodedKey.contentType
+ of unused:
+ raise invalidKeyErr
+ of accountTrieNode:
+ let offerValue = AccountTrieNodeOffer.decode(contentValueBytes).valueOr:
+ raise invalidValueErr
+ offerValue.toRetrievalValue.encode()
+ of contractTrieNode:
+ let offerValue = ContractTrieNodeOffer.decode(contentValueBytes).valueOr:
+ raise invalidValueErr
+ offerValue.toRetrievalValue.encode()
+ of contractCode:
+ let offerValue = ContractCodeOffer.decode(contentValueBytes).valueOr:
+ raise invalidValueErr
+ offerValue.toRetrievalValue.encode()
+ else:
+ contentValueBytes
 
+ let contentId = p.toContentId(key)
  if contentId.isSome():
- p.storeContent(key, contentId.get(), hexToSeqByte(contentValue))
+ p.storeContent(key, contentId.get(), valueToStore)
  return true
  else:
- raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+ raise invalidKeyErr
 
  rpcServer.rpc("portal_" & network & "LocalContent") do(contentKey: string) -> string:
  let

fluffy/tests/state_network_tests/test_state_gossip_getparent_genesis.nim

@@ -42,12 +42,16 @@ suite "State Gossip getParent - Genesis JSON Files":
 
  # validate each parent offer until getting to the root node
  var parent = offer.withKey(key).getParent()
- check validateOffer(accountState.rootHash(), parent.key, parent.offer).isOk()
+ check validateOffer(Opt.some(accountState.rootHash()), parent.key, parent.offer)
+ .isOk()
  db.put(parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq())
 
  for i in proof.low ..< proof.high - 1:
  parent = parent.getParent()
- check validateOffer(accountState.rootHash(), parent.key, parent.offer).isOk()
+ check validateOffer(
+ Opt.some(accountState.rootHash()), parent.key, parent.offer
+ )
+ .isOk()
  db.put(parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq())
 
  # after putting all parent nodes into the trie, verify can lookup the leaf
@@ -89,14 +93,19 @@ suite "State Gossip getParent - Genesis JSON Files":
 
  # validate each parent offer until getting to the root node
  var parent = offer.withKey(key).getParent()
- check validateOffer(accountState.rootHash(), parent.key, parent.offer).isOk()
+ check validateOffer(
+ Opt.some(accountState.rootHash()), parent.key, parent.offer
+ )
+ .isOk()
  db.put(
  parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq()
  )
 
  for i in storageProof.low ..< storageProof.high - 1:
  parent = parent.getParent()
- check validateOffer(accountState.rootHash(), parent.key, parent.offer)
+ check validateOffer(
+ Opt.some(accountState.rootHash()), parent.key, parent.offer
+ )
  .isOk()
  db.put(
  parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq()

fluffy/tests/state_network_tests/test_state_validation_genesis.nim

@@ -34,15 +34,17 @@ template checkValidProofsForExistingLeafs(
  path: accountPath, nodeHash: keccakHash(accountProof[^1].asSeq())
  )
  accountTrieOffer = AccountTrieNodeOffer(proof: accountProof)
- proofResult =
- validateOffer(accountState.rootHash(), accountTrieNodeKey, accountTrieOffer)
+ proofResult = validateOffer(
+ Opt.some(accountState.rootHash()), accountTrieNodeKey, accountTrieOffer
+ )
  check proofResult.isOk()
 
  let
  contractCodeKey = ContractCodeKey(address: address, codeHash: acc.codeHash)
  contractCode =
  ContractCodeOffer(code: Bytecode.init(account.code), accountProof: accountProof)
- codeResult = validateOffer(accountState.rootHash(), contractCodeKey, contractCode)
+ codeResult =
+ validateOffer(Opt.some(accountState.rootHash()), contractCodeKey, contractCode)
  check codeResult.isOk()
 
  if account.code.len() > 0:
@@ -64,7 +66,7 @@ template checkValidProofsForExistingLeafs(
  storageProof: storageProof, accountProof: accountProof
  )
  proofResult = validateOffer(
- accountState.rootHash(), contractTrieNodeKey, contractTrieOffer
+ Opt.some(accountState.rootHash()), contractTrieNodeKey, contractTrieOffer
  )
  check proofResult.isOk()
 
@@ -88,8 +90,9 @@ template checkInvalidProofsWithBadValue(
  accountProof[^1][^1] += 1 # bad account leaf value
  let
  accountTrieOffer = AccountTrieNodeOffer(proof: accountProof)
- proofResult =
- validateOffer(accountState.rootHash(), accountTrieNodeKey, accountTrieOffer)
+ proofResult = validateOffer(
+ Opt.some(accountState.rootHash()), accountTrieNodeKey, accountTrieOffer
+ )
  check proofResult.isErr()
 
  let
@@ -98,7 +101,8 @@ template checkInvalidProofsWithBadValue(
  code: Bytecode.init(@[1u8, 2, 3]), # bad code value
  accountProof: accountProof,
  )
- codeResult = validateOffer(accountState.rootHash(), contractCodeKey, contractCode)
+ codeResult =
+ validateOffer(Opt.some(accountState.rootHash()), contractCodeKey, contractCode)
  check codeResult.isErr()
 
  if account.code.len() > 0:
@@ -122,7 +126,7 @@ template checkInvalidProofsWithBadValue(
  storageProof: storageProof, accountProof: accountProof
  )
  proofResult = validateOffer(
- accountState.rootHash(), contractTrieNodeKey, contractTrieOffer
+ Opt.some(accountState.rootHash()), contractTrieNodeKey, contractTrieOffer
  )
  check proofResult.isErr()