Cleanup stateless and block witness code. (#2295)

* Cleanup unneeded stateless and block witness code. Keeping MultiKeys which is used in the eth_getProofsByBlockNumber RPC endpoint which is needed for the Fluffy state network bridge. * Rename generateWitness flag to collectWitnessData to better describe what the flag does. We only collect the keys of the touched accounts and storage slots but no block witness generation is supported for now. * Move remaining stateless code into nimbus directory. * Add vmstate parameter to ChainRef to fix test. * Exclude *.in from check copyright year --------- Co-authored-by: jangko <[email protected]>
status-im · Jun 8, 2024 · db8c5b9 · db8c5b9
1 parent 0524fe8
commit db8c5b9
Show file tree

Hide file tree

Showing 68 changed files with 182 additions and 6,321 deletions.
diff --git a/nimbus/config.nim b/nimbus/config.nim
@@ -197,12 +197,6 @@ type
  defaultValueDesc: ""
  name: "verify-from" }: Option[uint64]
 
- generateWitness* {.
- hidden
- desc: "Enable experimental generation and storage of block witnesses"
- defaultValue: false
- name: "generate-witness" }: bool
-
  evm* {.
  desc: "Load alternative EVM from EVMC-compatible shared library" & sharedLibText
  defaultValue: ""

diff --git a/nimbus/core/chain/chain_desc.nim b/nimbus/core/chain/chain_desc.nim
@@ -33,27 +33,29 @@ type
  ## Trigger extra validation, currently within `persistBlocks()`
  ## function only.
 
- generateWitness: bool ##\
- ## Enable generation of block witness, currently within `persistBlocks()`
- ## function only.
-
  verifyFrom: BlockNumber ##\
  ## First block to when `extraValidation` will be applied (only
  ## effective if `extraValidation` is true.)
 
+ vmState: BaseVMState
+ ## If it's not nil, block validation will use this
+ ## If it's nil, a new vmState state will be created.
+
 # ------------------------------------------------------------------------------
 # Public constructors
 # ------------------------------------------------------------------------------
 
 proc newChain*(com: CommonRef,
- extraValidation: bool): ChainRef =
+ extraValidation: bool,
+ vmState = BaseVMState(nil)): ChainRef =
  ## Constructor for the `Chain` descriptor object.
  ## The argument `extraValidation` enables extra block
  ## chain validation if set `true`.
  ChainRef(
  com: com,
  validateBlock: true,
  extraValidation: extraValidation,
+ vmState: vmState
  )
 
 func newChain*(com: CommonRef): ChainRef =
@@ -93,10 +95,6 @@ proc extraValidation*(c: ChainRef): bool =
  ## Getter
  c.extraValidation
 
-proc generateWitness*(c: ChainRef): bool =
- ## Getter
- c.generateWitness
-
 proc verifyFrom*(c: ChainRef): BlockNumber =
  ## Getter
  c.verifyFrom
@@ -121,11 +119,6 @@ proc `extraValidation=`*(c: ChainRef; extraValidation: bool) =
  ## extra block chain validation.
  c.extraValidation = extraValidation
 
-proc `generateWitness=`*(c: ChainRef; generateWitness: bool) =
- ## Setter. If set `true`, the assignment value `generateWitness` enables
- ## block witness generation.
- c.generateWitness = generateWitness
-
 proc `verifyFrom=`*(c: ChainRef; verifyFrom: BlockNumber) =
  ## Setter. The assignment value `verifyFrom` defines the first block where
  ## validation should start if the `Clique` field `extraValidation` was set

diff --git a/nimbus/core/chain/persist_blocks.nim b/nimbus/core/chain/persist_blocks.nim
@@ -111,11 +111,8 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
  body,
  checkSealOK = false) # TODO: how to checkseal from here
 
- if c.generateWitness:
- vmState.generateWitness = true
-
  let
- validationResult = if c.validateBlock or c.generateWitness:
+ validationResult = if c.validateBlock:
  vmState.processBlock(header, body)
  else:
  ValidationResult.OK
@@ -129,21 +126,6 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
  if validationResult != ValidationResult.OK:
  return err("Failed to validate block")
 
- if c.generateWitness:
- let dbTx = c.db.newTransaction()
- defer: dbTx.dispose()
-
- let
- mkeys = vmState.stateDB.makeMultiKeys()
- # Reset state to what it was before executing the block of transactions
- initialState = BaseVMState.new(header, c.com).valueOr:
- return err("Failed to create vm state")
- witness = initialState.buildWitness(mkeys)
-
- dbTx.rollback()
-
- c.db.setBlockWitness(header.blockHash(), witness)
-
  if NoPersistHeader notin flags:
  discard c.db.persistHeaderToDb(
  header, c.com.consensus == ConsensusType.POS, c.com.startOfHistory)

diff --git a/nimbus/core/executor/process_block.nim b/nimbus/core/executor/process_block.nim
@@ -114,7 +114,7 @@ proc procBlkEpilogue(vmState: BaseVMState;
  {.gcsafe, raises: [].} =
  # Reward beneficiary
  vmState.mutateStateDB:
- if vmState.generateWitness:
+ if vmState.collectWitnessData:
  db.collectWitnessData()
 
  db.persist(clearEmptyAccount = vmState.determineFork >= FkSpurious)

diff --git a/nimbus/core/executor/process_transaction.nim b/nimbus/core/executor/process_transaction.nim
@@ -115,7 +115,7 @@ proc processTransactionImpl(
  else:
  res = err(txRes.error)
 
- if vmState.generateWitness:
+ if vmState.collectWitnessData:
  vmState.stateDB.collectWitnessData()
  vmState.stateDB.persist(clearEmptyAccount = fork >= FkSpurious)
 

diff --git a/nimbus/core/tx_pool/tx_tasks/tx_packer.nim b/nimbus/core/tx_pool/tx_tasks/tx_packer.nim
@@ -108,7 +108,7 @@ proc runTxCommit(pst: TxPackerStateRef; item: TxItemRef; gasBurned: GasInt)
  vmState.stateDB.addBalance(xp.chain.feeRecipient, reward)
  xp.blockValue += reward
 
- if vmState.generateWitness:
+ if vmState.collectWitnessData:
  vmState.stateDB.collectWitnessData()
 
  # Save accounts via persist() is not needed unless the fork is smaller
@@ -245,7 +245,7 @@ proc vmExecCommit(pst: TxPackerStateRef)
 
  # Reward beneficiary
  vmState.mutateStateDB:
- if vmState.generateWitness:
+ if vmState.collectWitnessData:
  db.collectWitnessData()
  # Finish up, then vmState.stateDB.rootHash may be accessed
  db.persist(clearEmptyAccount = xp.chain.nextFork >= FkSpurious)

diff --git a/nimbus/db/core_db/core_apps_newapi.nim b/nimbus/db/core_db/core_apps_newapi.nim
@@ -984,20 +984,6 @@ proc haveBlockAndState*(db: CoreDbRef, headerHash: Hash256): bool =
  # see if stateRoot exists
  db.exists(header.stateRoot)
 
-proc getBlockWitness*(
- db: CoreDbRef, blockHash: Hash256): Result[seq[byte], string] {.gcsafe.} =
- let res = db.newKvt().get(blockHashToBlockWitnessKey(blockHash).toOpenArray)
- if res.isErr():
- err("Failed to get block witness from database: " & $res.error.error)
- else:
- ok(res.value())
-
-proc setBlockWitness*(db: CoreDbRef, blockHash: Hash256, witness: seq[byte]) =
- let witnessKey = blockHashToBlockWitnessKey(blockHash)
- db.newKvt.put(witnessKey.toOpenArray, witness).isOkOr:
- warn logTxt "setBlockWitness()", witnessKey, action="put()", error=($$error)
- return
-
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------
diff --git a/nimbus/db/ledger/accounts_ledger.nim b/nimbus/db/ledger/accounts_ledger.nim
@@ -29,7 +29,7 @@ import
  chronicles,
  eth/[common, rlp],
  results,
- ../../../stateless/multi_keys,
+ ../../stateless/multi_keys,
  "../.."/[constants, utils/utils],
  ../access_list as ac_access_list,
  ".."/[core_db, storage_types, transient_storage],

diff --git a/nimbus/db/ledger/base.nim b/nimbus/db/ledger/base.nim
@@ -14,7 +14,7 @@
 
 import
  eth/common,
- ../../../stateless/multi_keys,
+ ../../stateless/multi_keys,
  ../core_db,
  ./base/[api_tracking, base_desc]
 

diff --git a/nimbus/evm/state.nim b/nimbus/evm/state.nim
@@ -13,7 +13,6 @@
 import
  std/[options, sets, strformat],
  eth/keys,
- ../../stateless/[witness_from_tree, witness_types, multi_keys],
  ../db/ledger,
  ../common/[common, evmforks],
  ./interpreter/[op_codes, gas_costs],
@@ -259,31 +258,12 @@ proc `status=`*(vmState: BaseVMState, status: bool) =
  if status: vmState.flags.incl ExecutionOK
  else: vmState.flags.excl ExecutionOK
 
-proc generateWitness*(vmState: BaseVMState): bool =
- GenerateWitness in vmState.flags
+proc collectWitnessData*(vmState: BaseVMState): bool =
+ CollectWitnessData in vmState.flags
 
-proc `generateWitness=`*(vmState: BaseVMState, status: bool) =
- if status: vmState.flags.incl GenerateWitness
- else: vmState.flags.excl GenerateWitness
-
-proc buildWitness*(
- vmState: BaseVMState,
- mkeys: MultiKeysRef): seq[byte] {.raises: [CatchableError].} =
- let rootHash = vmState.stateDB.rootHash
- let flags = if vmState.fork >= FkSpurious: {wfEIP170} else: {}
-
- # A valid block having no transactions should return an empty witness
- if mkeys.keys.len() == 0:
- return @[]
-
- # build witness from tree
- var wb = initWitnessBuilder(vmState.com.db, rootHash, flags)
- wb.buildWitness(mkeys)
-
-proc buildWitness*(
- vmState: BaseVMState): seq[byte] {.raises: [CatchableError].} =
- let mkeys = vmState.stateDB.makeMultiKeys()
- buildWitness(vmState, mkeys)
+proc `collectWitnessData=`*(vmState: BaseVMState, status: bool) =
+ if status: vmState.flags.incl CollectWitnessData
+ else: vmState.flags.excl CollectWitnessData
 
 func forkDeterminationInfoForVMState*(vmState: BaseVMState): ForkDeterminationInfo =
  # FIXME-Adam: Is this timestamp right? Note that up above in blockNumber we add 1;

diff --git a/nimbus/evm/types.nim b/nimbus/evm/types.nim
@@ -36,7 +36,7 @@ const vm_use_recursion* = defined(evmc_enabled)
 type
  VMFlag* = enum
  ExecutionOK
- GenerateWitness
+ CollectWitnessData
 
  BlockContext* = object
  timestamp* : EthTime

diff --git a/nimbus/nimbus.nim b/nimbus/nimbus.nim
@@ -54,7 +54,6 @@ proc basicServices(nimbus: NimbusNode,
  nimbus.chainRef.extraValidation = 0 < verifyFrom
  nimbus.chainRef.verifyFrom = verifyFrom
 
- nimbus.chainRef.generateWitness = conf.generateWitness
  nimbus.beaconEngine = BeaconEngineRef.new(nimbus.txPool, nimbus.chainRef)
 
 proc manageAccounts(nimbus: NimbusNode, conf: NimbusConf) =

diff --git a/nimbus/rpc/experimental.nim b/nimbus/rpc/experimental.nim
@@ -22,17 +22,17 @@ import
  ./filters,
  ../core/executor/process_block,
  ../db/ledger,
- ../../stateless/[witness_verification, multi_keys],
+ ../stateless/multi_keys,
  ./p2p
 
 type
  BlockHeader = eth_types.BlockHeader
  ReadOnlyStateDB = state_db.ReadOnlyStateDB
 
-proc getBlockWitness*(
+proc getMultiKeys*(
  com: CommonRef,
  blockHeader: BlockHeader,
- statePostExecution: bool): (MultiKeysRef, BlockWitness)
+ statePostExecution: bool): MultiKeysRef
  {.raises: [RlpError, BlockNotFound, ValueError, CatchableError].} =
 
  let
@@ -45,7 +45,7 @@ proc getBlockWitness*(
  vmState = BaseVMState.new(blockHeader, com).valueOr:
  raise newException(ValueError, "Cannot create vm state")
 
- vmState.generateWitness = true # Enable saving witness data
+ vmState.collectWitnessData = true # Enable saving witness data
  vmState.com.hardForkTransition(blockHeader)
 
  let dbTx = vmState.com.db.newTransaction()
@@ -57,16 +57,10 @@ proc getBlockWitness*(
 
  let mkeys = vmState.stateDB.makeMultiKeys()
 
- if statePostExecution:
- result = (mkeys, vmState.buildWitness(mkeys))
- else:
- # Use the initial state from prior to executing the block of transactions
- let initialState = BaseVMState.new(blockHeader, com).valueOr:
- raise newException(ValueError, "Cannot create vm state")
- result = (mkeys, initialState.buildWitness(mkeys))
-
  dbTx.rollback()
 
+ mkeys
+
 proc getBlockProofs*(
  accDB: ReadOnlyStateDB,
  mkeys: MultiKeysRef): seq[ProofResponse] {.raises: [RlpError].} =
@@ -95,19 +89,6 @@ proc setupExpRpc*(com: CommonRef, server: RpcServer) =
  let ac = newAccountStateDB(chainDB, header.stateRoot)
  result = ReadOnlyStateDB(ac)
 
- server.rpc("exp_getWitnessByBlockNumber") do(quantityTag: BlockTag, statePostExecution: bool) -> seq[byte]:
- ## Returns the block witness for a block by block number or tag.
- ##
- ## quantityTag: integer of a block number, or the string "earliest", "latest" or "pending", as in the default block parameter.
- ## statePostExecution: bool which indicates whether to return the witness based on the state before or after executing the block.
- ## Returns seq[byte]
-
- let
- blockHeader = chainDB.headerFromTag(quantityTag)
- (_, witness) = getBlockWitness(com, blockHeader, statePostExecution)
-
- return witness
-
  server.rpc("exp_getProofsByBlockNumber") do(quantityTag: BlockTag, statePostExecution: bool) -> seq[ProofResponse]:
  ## Returns the block proofs for a block by block number or tag.
  ##
@@ -117,7 +98,7 @@ proc setupExpRpc*(com: CommonRef, server: RpcServer) =
 
  let
  blockHeader = chainDB.headerFromTag(quantityTag)
- (mkeys, _) = getBlockWitness(com, blockHeader, statePostExecution)
+ mkeys = getMultiKeys(com, blockHeader, statePostExecution)
 
  let accDB = if statePostExecution:
  getStateDB(blockHeader)

diff --git a/stateless/multi_keys.nim → nimbus/stateless/multi_keys.nim b/stateless/multi_keys.nim → nimbus/stateless/multi_keys.nim
@@ -9,11 +9,11 @@
 # according to those terms.
 
 import
- eth/common, eth/trie/nibbles, algorithm,
- ./witness_types
+ eth/common, eth/trie/nibbles, algorithm
 
 type
  KeyHash* = array[32, byte]
+ StorageSlot* = array[32, byte]
 
  KeyData* = object
  visited*: bool
@@ -47,6 +47,10 @@ type
  match*: bool
  group*: Group
 
+proc setBranchMaskBit(x: var uint, i: int) =
+ assert(i >= 0 and i < 17)
+ x = x or (1 shl i).uint
+
 func cmpHash(a, b: KeyHash): int =
  var i = 0
  var m = min(a.len, b.len)

diff --git a/nimbus/vm_state.nim b/nimbus/vm_state.nim
@@ -17,14 +17,13 @@ export
 export
  vms.`$`,
  vms.blockNumber,
- vms.buildWitness,
  vms.coinbase,
  vms.determineFork,
  vms.difficultyOrPrevRandao,
  vms.baseFee,
  vms.forkDeterminationInfoForVMState,
- vms.generateWitness,
- vms.`generateWitness=`,
+ vms.collectWitnessData,
+ vms.`collectWitnessData=`,
  vms.getAncestorHash,
  vms.getAndClearLogEntries,
  vms.init,

diff --git a/scripts/check_copyright_year.sh b/scripts/check_copyright_year.sh
@@ -8,7 +8,7 @@
 # according to those terms.
 
 excluded_files="config.yaml|.gitmodules|.gitignore"
-excluded_extensions="json|md|png|txt|toml|gz|key|rlp|era1|cfg|py|sh"
+excluded_extensions="json|md|png|txt|toml|gz|key|rlp|era1|cfg|py|sh|in"
 
 current_year=$(date +"%Y")
 outdated_files=()