Fluffy state network refactor (#2200)

* Add additional getTrieProof test.

* Refactor state network: Removed variant objects for values and split state_content into multiple files.

fluffy/common/common_utils.nim

@@ -14,7 +14,7 @@ import
  stew/[io2, arrayops],
  eth/p2p/discoveryv5/enr
 
-func init*(T: type KeccakHash, hash: openArray[byte]): T =
+func fromBytes*(T: type KeccakHash, hash: openArray[byte]): T =
  doAssert(hash.len() == 32)
  KeccakHash(data: array[32, byte].initCopyFrom(hash))
 

fluffy/network/state/content/content_keys.nim

@@ -0,0 +1,112 @@
+# Fluffy
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# As per spec:
+# https://github.com/ethereum/portal-network-specs/blob/master/state-network.md#content-keys-and-content-ids
+
+{.push raises: [].}
+
+import
+ nimcrypto/[hash, sha2, keccak],
+ results,
+ stint,
+ eth/common/eth_types,
+ ssz_serialization,
+ ./nibbles
+
+export ssz_serialization, common_types, hash, results
+
+type
+ NodeHash* = KeccakHash
+ CodeHash* = KeccakHash
+ Address* = EthAddress
+
+ ContentType* = enum
+ # Note: Need to add this unused value as a case object with an enum without
+ # a 0 valueis not allowed: "low(contentType) must be 0 for discriminant".
+ # For prefix values that are in the enum gap, the deserialization will fail
+ # at runtime as is wanted.
+ # In the future it might be possible that this will fail at compile time for
+ # the SSZ Union type, but currently it is allowed in the implementation, and
+ # the SSZ spec is not explicit about disallowing this.
+ unused = 0x00
+ accountTrieNode = 0x20
+ contractTrieNode = 0x21
+ contractCode = 0x22
+
+ AccountTrieNodeKey* = object
+ path*: Nibbles
+ nodeHash*: NodeHash
+
+ ContractTrieNodeKey* = object
+ address*: Address
+ path*: Nibbles
+ nodeHash*: NodeHash
+
+ ContractCodeKey* = object
+ address*: Address
+ codeHash*: CodeHash
+
+ ContentKey* = object
+ case contentType*: ContentType
+ of unused:
+ discard
+ of accountTrieNode:
+ accountTrieNodeKey*: AccountTrieNodeKey
+ of contractTrieNode:
+ contractTrieNodeKey*: ContractTrieNodeKey
+ of contractCode:
+ contractCodeKey*: ContractCodeKey
+
+func init*(T: type AccountTrieNodeKey, path: Nibbles, nodeHash: NodeHash): T =
+ AccountTrieNodeKey(path: path, nodeHash: nodeHash)
+
+func init*(
+ T: type ContractTrieNodeKey, address: Address, path: Nibbles, nodeHash: NodeHash
+): T =
+ ContractTrieNodeKey(address: address, path: path, nodeHash: nodeHash)
+
+func init*(T: type ContractCodeKey, address: Address, codeHash: CodeHash): T =
+ ContractCodeKey(address: address, codeHash: codeHash)
+
+func initAccountTrieNodeKey*(path: Nibbles, nodeHash: NodeHash): ContentKey =
+ ContentKey(
+ contentType: accountTrieNode,
+ accountTrieNodeKey: AccountTrieNodeKey.init(path, nodeHash),
+ )
+
+func initContractTrieNodeKey*(
+ address: Address, path: Nibbles, nodeHash: NodeHash
+): ContentKey =
+ ContentKey(
+ contentType: contractTrieNode,
+ contractTrieNodeKey: ContractTrieNodeKey.init(address, path, nodeHash),
+ )
+
+func initContractCodeKey*(address: Address, codeHash: CodeHash): ContentKey =
+ ContentKey(
+ contentType: contractCode, contractCodeKey: ContractCodeKey.init(address, codeHash)
+ )
+
+proc readSszBytes*(data: openArray[byte], val: var ContentKey) {.raises: [SszError].} =
+ mixin readSszValue
+ if data.len() > 0 and data[0] == ord(unused):
+ raise newException(MalformedSszError, "SSZ selector is unused value")
+
+ readSszValue(data, val)
+
+func encode*(contentKey: ContentKey): ByteList =
+ doAssert(contentKey.contentType != unused)
+ ByteList.init(SSZ.encode(contentKey))
+
+func decode*(T: type ContentKey, contentKey: ByteList): Result[T, string] =
+ decodeSsz(contentKey.asSeq(), T)
+
+func toContentId*(contentKey: ByteList): ContentId =
+ # TODO: Should we try to parse the content key here for invalid ones?
+ let idHash = sha256.digest(contentKey.asSeq())
+ readUintBE[256](idHash.data)

fluffy/network/state/content/content_values.nim

@@ -0,0 +1,97 @@
+# Fluffy
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# As per spec:
+# https://github.com/ethereum/portal-network-specs/blob/master/state-network.md#content-keys-and-content-ids
+
+{.push raises: [].}
+
+import results, eth/common/eth_types, ssz_serialization, ../../../common/common_types
+
+export ssz_serialization, common_types, hash, results
+
+const
+ MAX_TRIE_NODE_LEN = 1024
+ MAX_TRIE_PROOF_LEN = 65
+ MAX_BYTECODE_LEN = 32768
+
+type
+ TrieNode* = List[byte, MAX_TRIE_NODE_LEN]
+ TrieProof* = List[TrieNode, MAX_TRIE_PROOF_LEN]
+ Bytecode* = List[byte, MAX_BYTECODE_LEN]
+
+ AccountTrieNodeOffer* = object
+ proof*: TrieProof
+ blockHash*: BlockHash
+
+ AccountTrieNodeRetrieval* = object
+ node*: TrieNode
+
+ ContractTrieNodeOffer* = object
+ storageProof*: TrieProof
+ accountProof*: TrieProof
+ blockHash*: BlockHash
+
+ ContractTrieNodeRetrieval* = object
+ node*: TrieNode
+
+ ContractCodeOffer* = object
+ code*: Bytecode
+ accountProof*: TrieProof
+ blockHash*: BlockHash
+
+ ContractCodeRetrieval* = object
+ code*: Bytecode
+
+ ContentValue* =
+ AccountTrieNodeOffer | ContractTrieNodeOffer | ContractCodeOffer |
+ AccountTrieNodeRetrieval | ContractTrieNodeRetrieval | ContractCodeRetrieval
+
+func init*(T: type AccountTrieNodeOffer, proof: TrieProof, blockHash: BlockHash): T =
+ AccountTrieNodeOffer(proof: proof, blockHash: blockHash)
+
+func init*(T: type AccountTrieNodeRetrieval, node: TrieNode): T =
+ AccountTrieNodeRetrieval(node: node)
+
+func init*(
+ T: type ContractTrieNodeOffer,
+ storageProof: TrieProof,
+ accountProof: TrieProof,
+ blockHash: BlockHash,
+): T =
+ ContractTrieNodeOffer(
+ storageProof: storageProof, accountProof: accountProof, blockHash: blockHash
+ )
+
+func init*(T: type ContractTrieNodeRetrieval, node: TrieNode): T =
+ ContractTrieNodeRetrieval(node: node)
+
+func init*(
+ T: type ContractCodeOffer,
+ code: Bytecode,
+ accountProof: TrieProof,
+ blockHash: BlockHash,
+): T =
+ ContractCodeOffer(code: code, accountProof: accountProof, blockHash: blockHash)
+
+func init*(T: type ContractCodeRetrieval, code: Bytecode): T =
+ ContractCodeRetrieval(code: code)
+
+func toRetrievalValue*(offer: AccountTrieNodeOffer): AccountTrieNodeRetrieval =
+ AccountTrieNodeRetrieval.init(offer.proof[^1])
+
+func toRetrievalValue*(offer: ContractTrieNodeOffer): ContractTrieNodeRetrieval =
+ ContractTrieNodeRetrieval.init(offer.storageProof[^1])
+
+func toRetrievalValue*(offer: ContractCodeOffer): ContractCodeRetrieval =
+ ContractCodeRetrieval.init(offer.code)
+
+func encode*(value: ContentValue): seq[byte] =
+ SSZ.encode(value)
+
+func decode*(T: type ContentValue, bytes: openArray[byte]): Result[T, string] =
+ decodeSsz(bytes, T)

fluffy/network/state/content/nibbles.nim

@@ -0,0 +1,101 @@
+# Fluffy
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# As per spec:
+# https://github.com/ethereum/portal-network-specs/blob/master/state-network.md#content-keys-and-content-ids
+
+{.push raises: [].}
+
+import
+ nimcrypto/hash,
+ results,
+ stint,
+ eth/common/eth_types,
+ ssz_serialization,
+ ../../../common/common_types
+
+export ssz_serialization, common_types, hash, results
+
+const
+ MAX_PACKED_NIBBLES_LEN = 33
+ MAX_UNPACKED_NIBBLES_LEN = 64
+
+type Nibbles* = List[byte, MAX_PACKED_NIBBLES_LEN]
+
+func init*(T: type Nibbles, packed: openArray[byte], isEven: bool): T =
+ doAssert(packed.len() <= MAX_PACKED_NIBBLES_LEN)
+
+ var output = newSeqOfCap[byte](packed.len() + 1)
+ if isEven:
+ output.add(0x00)
+ else:
+ doAssert(packed.len() > 0)
+ # set the first nibble to 1 and copy the second nibble from the input
+ output.add((packed[0] and 0x0F) or 0x10)
+
+ let startIdx = if isEven: 0 else: 1
+ for i in startIdx ..< packed.len():
+ output.add(packed[i])
+
+ Nibbles(output)
+
+func encode*(nibbles: Nibbles): seq[byte] =
+ SSZ.encode(nibbles)
+
+func decode*(T: type Nibbles, bytes: openArray[byte]): Result[T, string] =
+ decodeSsz(bytes, T)
+
+func packNibbles*(unpacked: openArray[byte]): Nibbles =
+ doAssert(
+ unpacked.len() <= MAX_UNPACKED_NIBBLES_LEN, "Can't pack more than 64 nibbles"
+ )
+
+ if unpacked.len() == 0:
+ return Nibbles(@[byte(0x00)])
+
+ let isEvenLength = unpacked.len() mod 2 == 0
+
+ var
+ output = newSeqOfCap[byte](unpacked.len() div 2 + 1)
+ highNibble = isEvenLength
+ currentByte: byte = 0
+
+ if isEvenLength:
+ output.add(0x00)
+ else:
+ currentByte = 0x10
+
+ for i, nibble in unpacked:
+ if highNibble:
+ currentByte = nibble shl 4
+ else:
+ output.add(currentByte or nibble)
+ currentByte = 0
+ highNibble = not highNibble
+
+ Nibbles(output)
+
+func unpackNibbles*(packed: Nibbles): seq[byte] =
+ doAssert(packed.len() <= MAX_PACKED_NIBBLES_LEN, "Packed nibbles length is too long")
+
+ var output = newSeqOfCap[byte](packed.len() * 2)
+
+ for i, pair in packed:
+ if i == 0 and pair == 0x00:
+ continue
+
+ let
+ first = (pair and 0xF0) shr 4
+ second = pair and 0x0F
+
+ if i == 0 and first == 0x01:
+ output.add(second)
+ else:
+ output.add(first)
+ output.add(second)
+
+ output