Replace status-im/portal-spec-tests with ethereum fork version

.gitmodules

@@ -212,11 +212,6 @@
  url = https://github.com/status-im/nimbus-security-resources.git
  ignore = untracked
  branch = master
-[submodule "vendor/portal-spec-tests"]
- path = vendor/portal-spec-tests
- url = https://github.com/status-im/portal-spec-tests.git
- ignore = untracked
- branch = master
 [submodule "vendor/libtommath"]
  path = vendor/libtommath
  url = https://github.com/libtom/libtommath
@@ -232,3 +227,18 @@
  url = https://github.com/arnetheduck/nim-results
  ignore = untracked
  branch = master
+[submodule "vendor/NimYAML"]
+ path = vendor/NimYAML
+ url = https://github.com/status-im/NimYAML.git
+ ignore = untracked
+ branch = master
+[submodule "vendor/portal-spec-tests"]
+ path = vendor/portal-spec-tests
+ url = https://github.com/ethereum/portal-spec-tests.git
+ ignore = untracked
+ branch = master
+[submodule "vendor/portal-mainnet"]
+ path = vendor/portal-mainnet
+ url = https://github.com/status-im/portal-mainnet.git
+ ignore = untracked
+ branch = master

Makefile

@@ -51,7 +51,8 @@ EXCLUDED_NIM_PACKAGES := \
  vendor/nimbus-eth2/vendor/nim-presto \
  vendor/nimbus-eth2/vendor/nim-zxcvbn \
  vendor/nimbus-eth2/vendor/nim-kzg4844 \
- vendor/nimbus-eth2/vendor/nimbus-security-resources
+ vendor/nimbus-eth2/vendor/nimbus-security-resources \
+ vendor/nimbus-eth2/vendor/NimYAML
 
 # we don't want an error here, so we can handle things later, in the ".DEFAULT" target
 -include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk

fluffy/network_metadata.nim

@@ -29,15 +29,9 @@ proc loadCompileTimeBootstrapNodes(path: string): seq[string] =
  macros.error "Failed to load bootstrap nodes metadata at '" & path & "': " & err.msg
 
 const
- # TODO: Change this from our local repo to an eth-client repo if/when this
- # gets created for the Portal networks.
- portalNetworksDir = currentSourcePath.parentDir.replace('\\', '/') / "network_data"
-
- # TODO: Using a repo for test vectors for now, as it is something to test
- # against, but at the same time could also go in a network metadata repo.
- portalTestDir =
+ portalConfigDir =
  currentSourcePath.parentDir.parentDir.replace('\\', '/') / "vendor" /
- "portal-spec-tests" / "tests"
+ "portal-mainnet" / "config"
  # Note:
  # For now it gets called testnet0 but this Portal network serves Eth1 mainnet
  # data. Giving the actual Portal (test)networks different names might not be
@@ -48,16 +42,12 @@ const
  # TODO: It would be nice to be able to use `loadBootstrapFile` here, but that
  # doesn't work at compile time. The main issue seems to be the usage of
  # rlp.rawData() in the enr code.
- testnet0BootstrapNodes* = loadCompileTimeBootstrapNodes(
- portalNetworksDir / "testnet0" / "bootstrap_nodes.txt"
- )
+ testnet0BootstrapNodes* =
+ loadCompileTimeBootstrapNodes(portalConfigDir / "bootstrap_nodes.txt")
 
- finishedAccumulatorSSZ* = slurp(
- portalTestDir / "mainnet" / "history" / "accumulator" / "finished_accumulator.ssz"
- )
+ finishedAccumulatorSSZ* = slurp(portalConfigDir / "finished_accumulator.ssz")
 
- historicalRootsSSZ* =
- slurp(portalTestDir / "mainnet" / "beacon_chain" / "historical_roots.ssz")
+ historicalRootsSSZ* = slurp(portalConfigDir / "historical_roots.ssz")
 
 func loadAccumulator*(): FinishedAccumulator =
  try:

fluffy/tests/portal_spec_tests/mainnet/test_history_content.nim

@@ -16,7 +16,8 @@ import
  ../../../network_metadata,
  ../../../eth_data/[history_data_json_store, history_data_ssz_e2s],
  ../../../network/history/[history_content, history_network, accumulator],
- ../../test_history_util
+ ../../test_history_util,
+ ../../test_yaml_utils
 
 suite "History Content Encodings":
  test "HeaderWithProof Building and Encoding":
@@ -79,83 +80,73 @@ suite "History Content Encodings":
 
  test "HeaderWithProof Encoding/Decoding and Verification":
  const dataFile =
- "./vendor/portal-spec-tests/tests/mainnet/history/headers_with_proof/14764013.json"
+ "./vendor/portal-spec-tests/tests/mainnet/history/headers_with_proof/14764013.yaml"
 
- let accumulator = loadAccumulator()
-
- let res = readJsonType(dataFile, JsonPortalContentTable)
- check res.isOk()
- let content = res.get()
-
- for k, v in content:
- # TODO: strange assignment failure when using try/except ValueError
- # for the hexToSeqByte() here.
- let
- contentKeyEncoded = v.content_key.hexToSeqByte()
- contentValueEncoded = v.content_value.hexToSeqByte()
+ let
+ content = YamlPortalContent.loadFromYaml(dataFile).valueOr:
+ raiseAssert "Invalid data file: " & error
+ accumulator = loadAccumulator()
+ contentKeyEncoded = content.content_key.hexToSeqByte()
+ contentValueEncoded = content.content_value.hexToSeqByte()
 
-  # Decode content
-  let
-  contentKey = decodeSsz(contentKeyEncoded, ContentKey)
-  contentValue = decodeSsz(contentValueEncoded, BlockHeaderWithProof)
+ # Decode content
+ let
+ contentKey = decodeSsz(contentKeyEncoded, ContentKey)
+ contentValue = decodeSsz(contentValueEncoded, BlockHeaderWithProof)
 
-  check:
-  contentKey.isOk()
-  contentValue.isOk()
+ check:
+ contentKey.isOk()
+ contentValue.isOk()
 
-  let blockHeaderWithProof = contentValue.get()
+ let blockHeaderWithProof = contentValue.get()
 
-  let res = decodeRlp(blockHeaderWithProof.header.asSeq(), BlockHeader)
-  check res.isOk()
-  let header = res.get()
+ let res = decodeRlp(blockHeaderWithProof.header.asSeq(), BlockHeader)
+ check res.isOk()
+ let header = res.get()
 
-  check accumulator.verifyHeader(header, blockHeaderWithProof.proof).isOk()
+ check accumulator.verifyHeader(header, blockHeaderWithProof.proof).isOk()
 
-  # Encode content
-  check:
-  SSZ.encode(blockHeaderWithProof) == contentValueEncoded
-  encode(contentKey.get()).asSeq() == contentKeyEncoded
+ # Encode content
+ check:
+ SSZ.encode(blockHeaderWithProof) == contentValueEncoded
+ encode(contentKey.get()).asSeq() == contentKeyEncoded
 
  test "PortalBlockBody (Legacy) Encoding/Decoding and Verification":
  const
- dataFile = "./vendor/portal-spec-tests/tests/mainnet/history/bodies/14764013.json"
- headersWithProofFile =
- "./vendor/portal-spec-tests/tests/mainnet/history/headers_with_proof/14764013.json"
+ dataFile = "./vendor/portal-spec-tests/tests/mainnet/history/bodies/14764013.yaml"
+ headersDataFile =
+ "./vendor/portal-spec-tests/tests/mainnet/history/headers_with_proof/14764013.yaml"
 
- let res = readJsonType(dataFile, JsonPortalContentTable)
- check res.isOk()
- let content = res.get()
-
- let headersRes = readJsonType(headersWithProofFile, JsonPortalContentTable)
- check headersRes.isOk()
- let headers = headersRes.get()
-
- for k, v in content:
- let
- contentKeyEncoded = v.content_key.hexToSeqByte()
- contentValueEncoded = v.content_value.hexToSeqByte()
-
- # Get the header for validation of body
- let
- headerEncoded = headers[k].content_value.hexToSeqByte()
- headerWithProofRes = decodeSsz(headerEncoded, BlockHeaderWithProof)
- check headerWithProofRes.isOk()
- let headerRes = decodeRlp(headerWithProofRes.get().header.asSeq(), BlockHeader)
- check headerRes.isOk()
- let header = headerRes.get()
-
- # Decode content key
- let contentKey = decodeSsz(contentKeyEncoded, ContentKey)
- check contentKey.isOk()
+ let
+ content = YamlPortalContent.loadFromYaml(dataFile).valueOr:
+ raiseAssert "Invalid data file: " & error
+ headerContent = YamlPortalContent.loadFromYaml(headersDataFile).valueOr:
+ raiseAssert "Invalid data file: " & error
 
- # Decode (SSZ + RLP decode step) and validate block body
- let contentValue = validateBlockBodyBytes(contentValueEncoded, header)
- check contentValue.isOk()
+ contentKeyEncoded = content.content_key.hexToSeqByte()
+ contentValueEncoded = content.content_value.hexToSeqByte()
 
- # Encode content and content key
- check:
- encode(contentValue.get()) == contentValueEncoded
- encode(contentKey.get()).asSeq() == contentKeyEncoded
+ # Get the header for validation of body
+ let
+ headerEncoded = headerContent.content_value.hexToSeqByte()
+ headerWithProofRes = decodeSsz(headerEncoded, BlockHeaderWithProof)
+ check headerWithProofRes.isOk()
+ let headerRes = decodeRlp(headerWithProofRes.get().header.asSeq(), BlockHeader)
+ check headerRes.isOk()
+ let header = headerRes.get()
+
+ # Decode content key
+ let contentKey = decodeSsz(contentKeyEncoded, ContentKey)
+ check contentKey.isOk()
+
+ # Decode (SSZ + RLP decode step) and validate block body
+ let contentValue = validateBlockBodyBytes(contentValueEncoded, header)
+ check contentValue.isOk()
+
+ # Encode content and content key
+ check:
+ encode(contentValue.get()) == contentValueEncoded
+ encode(contentKey.get()).asSeq() == contentKeyEncoded
 
  test "PortalBlockBody (Shanghai) Encoding/Decoding":
  # TODO: We don't have the header (without proof) ready here so cannot do
@@ -188,41 +179,37 @@ suite "History Content Encodings":
  test "Receipts Encoding/Decoding and Verification":
  const
  dataFile =
- "./vendor/portal-spec-tests/tests/mainnet/history/receipts/14764013.json"
- headersWithProofFile =
- "./vendor/portal-spec-tests/tests/mainnet/history/headers_with_proof/14764013.json"
-
- let res = readJsonType(dataFile, JsonPortalContentTable)
- check res.isOk()
- let content = res.get()
+ "./vendor/portal-spec-tests/tests/mainnet/history/receipts/14764013.yaml"
+ headersDataFile =
+ "./vendor/portal-spec-tests/tests/mainnet/history/headers_with_proof/14764013.yaml"
 
- let headersRes = readJsonType(headersWithProofFile, JsonPortalContentTable)
- check headersRes.isOk()
- let headers = headersRes.get()
-
- for k, v in content:
- let
- contentKeyEncoded = v.content_key.hexToSeqByte()
- contentValueEncoded = v.content_value.hexToSeqByte()
-
- # Get the header for validation of receipts
- let
- headerEncoded = headers[k].content_value.hexToSeqByte()
- headerWithProofRes = decodeSsz(headerEncoded, BlockHeaderWithProof)
- check headerWithProofRes.isOk()
- let headerRes = decodeRlp(headerWithProofRes.get().header.asSeq(), BlockHeader)
- check headerRes.isOk()
- let header = headerRes.get()
-
- # Decode content key
- let contentKey = decodeSsz(contentKeyEncoded, ContentKey)
- check contentKey.isOk()
+ let
+ content = YamlPortalContent.loadFromYaml(dataFile).valueOr:
+ raiseAssert "Invalid data file: " & error
+ headerContent = YamlPortalContent.loadFromYaml(headersDataFile).valueOr:
+ raiseAssert "Invalid data file: " & error
 
- # Decode (SSZ + RLP decode step) and validate receipts
- let contentValue = validateReceiptsBytes(contentValueEncoded, header.receiptRoot)
- check contentValue.isOk()
+ contentKeyEncoded = content.content_key.hexToSeqByte()
+ contentValueEncoded = content.content_value.hexToSeqByte()
 
- # Encode content
- check:
- encode(contentValue.get()) == contentValueEncoded
- encode(contentKey.get()).asSeq() == contentKeyEncoded
+ # Get the header for validation of receipts
+ let
+ headerEncoded = headerContent.content_value.hexToSeqByte()
+ headerWithProofRes = decodeSsz(headerEncoded, BlockHeaderWithProof)
+ check headerWithProofRes.isOk()
+ let headerRes = decodeRlp(headerWithProofRes.get().header.asSeq(), BlockHeader)
+ check headerRes.isOk()
+ let header = headerRes.get()
+
+ # Decode content key
+ let contentKey = decodeSsz(contentKeyEncoded, ContentKey)
+ check contentKey.isOk()
+
+ # Decode (SSZ + RLP decode step) and validate receipts
+ let contentValue = validateReceiptsBytes(contentValueEncoded, header.receiptRoot)
+ check contentValue.isOk()
+
+ # Encode content
+ check:
+ encode(contentValue.get()) == contentValueEncoded
+ encode(contentKey.get()).asSeq() == contentKeyEncoded