Premix persist tool improvements (#2000)

* Add premix persist tool to Makefile to build as separate binary.

* Fix rootBytes.len > 0 assertion defect in HexaryTrie by adding call to com.db.compensateLegacySetup().

* Run hardForkTransition before executing transactions in getBlockWitness.

* Improve logging and add help message.

* Retry requests for block data.

Makefile

@@ -58,9 +58,11 @@ EXCLUDED_NIM_PACKAGES := \
 
 # debugging tools + testing tools
 TOOLS := \
- test_tools_build
+ test_tools_build \
+ persist
 TOOLS_DIRS := \
- tests
+ tests \
+ premix
 # comma-separated values for the "clean" target
 TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 

nimbus/rpc/experimental.nim

@@ -45,7 +45,9 @@ proc getBlockWitness*(
  # before trying to initialize the VM as we do here.
  vmState = BaseVMState.new(blockHeader, com)
  flags = if vmState.fork >= FKSpurious: {wfEIP170} else: {}
+
  vmState.generateWitness = true # Enable saving witness data
+ vmState.com.hardForkTransition(blockHeader)
 
  var dbTx = vmState.com.db.beginTransaction()
  defer: dbTx.dispose()

premix/configuration.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2020-2023 Status Research & Development GmbH
+# Copyright (c) 2020-2024 Status Research & Development GmbH
 # Licensed under either of
 # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 # http:https://www.apache.org/licenses/LICENSE-2.0)
@@ -10,7 +10,10 @@
 
 import
  std/[os, parseopt, strutils],
- eth/common, stint, ../nimbus/config
+ eth/common,
+ stint,
+ chronicles,
+ ../nimbus/config
 
 from ../nimbus/common/chain_config import
  MainNet,
@@ -101,10 +104,12 @@ proc processArguments*(msg: var string): ConfigStatus =
  for kind, key, value in opt.getopt():
  case kind
  of cmdArgument:
- discard
+ return EmptyOption
  of cmdLongOption, cmdShortOption:
  inc(length)
  case key.toLowerAscii()
+ of "help":
+ return EmptyOption
  of "datadir": config.dataDir = value
  of "maxblocks":
  checkArgument(processInteger, config.maxBlocks, value)
@@ -118,9 +123,14 @@ proc processArguments*(msg: var string): ConfigStatus =
  else:
  msg = "Unknown option " & key
  if value.len > 0: msg = msg & " : " & value
- result = ErrorUnknownOption
- break
+ return ErrorUnknownOption
  of cmdEnd:
  msg = "Error processing option [" & key & "]"
- result = ErrorParseOption
- break
+ return ErrorParseOption
+
+ info "Using configuration parameters: ",
+ datadir = config.dataDir,
+ maxblocks = config.maxBlocks,
+ head = config.head,
+ numcommits = config.numCommits,
+ netid = config.netId

premix/persist.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2020-2023 Status Research & Development GmbH
+# Copyright (c) 2020-2024 Status Research & Development GmbH
 # Licensed under either of
 # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 # http:https://www.apache.org/licenses/LICENSE-2.0)
@@ -11,6 +11,7 @@
 # use this module to quickly populate db with data from geth/parity
 
 import
+ std/os,
  chronicles,
  ../nimbus/errors,
  ../nimbus/core/chain,
@@ -37,7 +38,7 @@ else:
 
 template persistToDb(db: CoreDbRef, body: untyped) =
  block: body
- 
+
 proc main() {.used.} =
  # 97 block with uncles
  # 46147 block with first transaction
@@ -58,12 +59,13 @@ proc main() {.used.} =
 
  # move head to block number ...
  if conf.head != 0.u256:
- var parentBlock = requestBlock(conf.head)
+ var parentBlock = requestBlock(conf.head, { DownloadAndValidate })
  discard com.db.setHead(parentBlock.header)
 
  if canonicalHeadHashKey().toOpenArray notin com.db.kvt:
  persistToDb(com.db):
  com.initializeEmptyDb()
+ com.db.compensateLegacySetup()
  doAssert(canonicalHeadHashKey().toOpenArray in com.db.kvt)
 
  var head = com.db.getCanonicalHead()
@@ -78,9 +80,21 @@ proc main() {.used.} =
 
  var numBlocks = 0
  var counter = 0
+ var retryCount = 0
 
  while true:
- var thisBlock = requestBlock(blockNumber)
+
+ var thisBlock: Block
+ try:
+ thisBlock = requestBlock(blockNumber, { DownloadAndValidate })
+ except CatchableError as e:
+ if retryCount < 3:
+ warn "Unable to get block data via JSON-RPC API", error = e.msg
+ inc retryCount
+ sleep(1000)
+ continue
+ else:
+ raise e
 
  headers.add thisBlock.header
  bodies.add thisBlock.body
@@ -111,7 +125,9 @@ when isMainModule:
 
  ## Processing command line arguments
  if configuration.processArguments(message) != Success:
- echo message
+ if len(message) > 0:
+ echo message
+ echo "Usage: persist --datadir=<DATA_DIR> --maxblocks=<MAX_BLOCKS> --head=<HEAD> --numcommits=<NUM_COMMITS> --netid=<NETWORK_ID>"
  quit(QuitFailure)
  else:
  if len(message) > 0: