Storage of block witnesses (#1986)

* Added procs to get and store block witness in db and add generate-witness cli flag.

* Completed initial implementation of block witness storage.

* Added test to verify witness is persisted to db after call to persistBlock.

* Update getBlockWitness to return witness using Result type.

* Make generate witness parameter hidden.

nimbus/config.nim

@@ -200,6 +200,12 @@ type
  defaultValueDesc: ""
  name: "verify-from" }: Option[uint64]
 
+ generateWitness* {.
+ hidden
+ desc: "Enable experimental generation and storage of block witnesses"
+ defaultValue: false
+ name: "generate-witness" }: bool
+
  evm* {.
  desc: "Load alternative EVM from EVMC-compatible shared library" & sharedLibText
  defaultValue: ""

nimbus/core/chain/chain_desc.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 # http:https://www.apache.org/licenses/LICENSE-2.0)
@@ -34,6 +34,10 @@ type
  ## Trigger extra validation, currently within `persistBlocks()`
  ## function only.
 
+ generateWitness: bool ##\
+ ## Enable generation of block witness, currently within `persistBlocks()`
+ ## function only.
+
  verifyFrom: BlockNumber ##\
  ## First block to when `extraValidation` will be applied (only
  ## effective if `extraValidation` is true.)
@@ -101,6 +105,10 @@ proc extraValidation*(c: ChainRef): bool =
  ## Getter
  c.extraValidation
 
+proc generateWitness*(c: ChainRef): bool =
+ ## Getter
+ c.generateWitness
+
 proc verifyFrom*(c: ChainRef): BlockNumber =
  ## Getter
  c.verifyFrom
@@ -125,6 +133,11 @@ proc `extraValidation=`*(c: ChainRef; extraValidation: bool) =
  ## extra block chain validation.
  c.extraValidation = extraValidation
 
+proc `generateWitness=`*(c: ChainRef; generateWitness: bool) =
+ ## Setter. If set `true`, the assignment value `generateWitness` enables
+ ## block witness generation.
+ c.generateWitness = generateWitness
+
 proc `verifyFrom=`*(c: ChainRef; verifyFrom: BlockNumber) =
  ## Setter. The assignment value `verifyFrom` defines the first block where
  ## validation should start if the `Clique` field `extraValidation` was set

nimbus/core/chain/persist_blocks.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 # http:https://www.apache.org/licenses/LICENSE-2.0)
@@ -100,8 +100,11 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
  msg = res.error
  return ValidationResult.Error
 
+ if c.generateWitness:
+ vmState.generateWitness = true
+
  let
- validationResult = if c.validateBlock:
+ validationResult = if c.validateBlock or c.generateWitness:
  vmState.processBlock(header, body)
  else:
  ValidationResult.OK
@@ -132,6 +135,21 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
  msg = $rc.error
  return ValidationResult.Error
 
+ if c.generateWitness:
+ let dbTx = c.db.beginTransaction()
+ defer: dbTx.dispose()
+
+ let
+ mkeys = vmState.stateDB.makeMultiKeys()
+ # Reset state to what it was before executing the block of transactions
+ initialState = BaseVMState.new(header, c.com)
+ witness = initialState.buildWitness(mkeys)
+
+ dbTx.rollback()
+
+ c.db.setBlockWitness(header.blockHash(), witness)
+
+
  if NoPersistHeader notin flags:
  discard c.db.persistHeaderToDb(
  header, c.com.consensus == ConsensusType.POS, c.com.startOfHistory)

nimbus/db/core_db/core_apps_legacy.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 # http:https://www.apache.org/licenses/LICENSE-2.0)
@@ -734,6 +734,12 @@ proc haveBlockAndState*(db: CoreDbRef, headerHash: Hash256): bool =
  # see if stateRoot exists
  db.exists(header.stateRoot)
 
+proc getBlockWitness*(db: CoreDbRef, blockHash: Hash256): seq[byte] {.gcsafe.} =
+ db.kvt.get(blockHashToBlockWitnessKey(blockHash).toOpenArray)
+
+proc setBlockWitness*(db: CoreDbRef, blockHash: Hash256, witness: seq[byte]) =
+ db.kvt.put(blockHashToBlockWitnessKey(blockHash).toOpenArray, witness)
+
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/db/core_db/core_apps_newapi.nim

@@ -924,6 +924,21 @@ proc haveBlockAndState*(db: CoreDbRef, headerHash: Hash256): bool =
  # see if stateRoot exists
  db.exists(header.stateRoot)
 
+proc getBlockWitness*(
+ db: CoreDbRef, blockHash: Hash256): Result[seq[byte], string] {.gcsafe.} =
+ let res = db.newKvt(Shared)
+ .get(blockHashToBlockWitnessKey(blockHash).toOpenArray)
+ if res.isErr():
+ err("Failed to get block witness from database: " & $res.error.error)
+ else:
+ ok(res.value())
+
+proc setBlockWitness*(db: CoreDbRef, blockHash: Hash256, witness: seq[byte]) =
+ let witnessKey = blockHashToBlockWitnessKey(blockHash)
+ db.newKvt.put(witnessKey.toOpenArray, witness).isOkOr:
+ warn logTxt "setBlockWitness()", witnessKey, action="put()", error=($$error)
+ return
+
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/db/storage_types.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2023 Status Research & Development GmbH
+# Copyright (c) 2023-2024 Status Research & Development GmbH
 # Licensed under either of
 # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 # http:https://www.apache.org/licenses/LICENSE-2.0)
@@ -31,6 +31,7 @@ type
  snapSyncAccount
  snapSyncStorageSlot
  snapSyncStateRoot
+ blockHashToBlockWitness
 
  DbKey* = object
  # The first byte stores the key type. The rest are key-specific values
@@ -129,6 +130,11 @@ proc snapSyncStateRootKey*(h: openArray[byte]): DbKey {.inline.} =
  result.data[1 .. 32] = h
  result.dataEndPos = uint8 sizeof(h)
 
+proc blockHashToBlockWitnessKey*(h: Hash256): DbKey {.inline.} =
+ result.data[0] = byte ord(blockHashToBlockWitness)
+ result.data[1 .. 32] = h.data
+ result.dataEndPos = uint8 32
+
 template toOpenArray*(k: DbKey): openArray[byte] =
  k.data.toOpenArray(0, int(k.dataEndPos))
 

nimbus/nimbus.nim

@@ -67,6 +67,7 @@ proc basicServices(nimbus: NimbusNode,
  nimbus.chainRef.extraValidation = 0 < verifyFrom
  nimbus.chainRef.verifyFrom = verifyFrom
 
+ nimbus.chainRef.generateWitness = conf.generateWitness
  nimbus.beaconEngine = BeaconEngineRef.new(nimbus.txPool, nimbus.chainRef)
 
 proc manageAccounts(nimbus: NimbusNode, conf: NimbusConf) =

nimbus/rpc/experimental.nim

@@ -47,7 +47,7 @@ proc getBlockWitness*(
  vmState.generateWitness = true # Enable saving witness data
  vmState.com.hardForkTransition(blockHeader)
 
- var dbTx = vmState.com.db.beginTransaction()
+ let dbTx = vmState.com.db.beginTransaction()
  defer: dbTx.dispose()
 
  # Execute the block of transactions and collect the keys of the touched account state

tests/all_tests.nim

@@ -54,4 +54,5 @@ cliBuilder:
  ./test_beacon/test_skeleton,
  ./test_overflow,
  ./test_getproof_json,
- ./test_rpc_experimental_json
+ ./test_rpc_experimental_json,
+ ./test_persistblock_witness_json

tests/test_configuration.nim

@@ -335,5 +335,21 @@ proc configurationMain*() =
  check conf.dataDir.string == defaultDataDir()
  check conf.keyStore.string == "banana"
 
+ test "generate-witness default":
+ let conf = makeTestConfig()
+ check conf.generateWitness == false
+
+ test "generate-witness enabled":
+ let conf = makeConfig(@["--generate-witness"])
+ check conf.generateWitness == true
+
+ test "generate-witness equals true":
+ let conf = makeConfig(@["--generate-witness=true"])
+ check conf.generateWitness == true
+
+ test "generate-witness equals false":
+ let conf = makeConfig(@["--generate-witness=false"])
+ check conf.generateWitness == false
+
 when isMainModule:
  configurationMain()

tests/test_persistblock_witness_json.nim

@@ -0,0 +1,69 @@
+# Nimbus
+# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Licensed under either of
+# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http:https://www.apache.org/licenses/LICENSE-2.0)
+# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http:https://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+ std/[json, os, tables, strutils],
+ unittest2,
+ stew/byteutils,
+ ./test_helpers,
+ ../nimbus/core/chain,
+ ../nimbus/common/common,
+ ../stateless/[witness_verification, witness_types]
+
+# use tracerTestGen.nim to generate additional test data
+proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus) =
+ var
+ blockNumber = UInt256.fromHex(node["blockNumber"].getStr())
+ memoryDB = newCoreDbRef LegacyDbMemory
+ config = chainConfigForNetwork(MainNet)
+ com = CommonRef.new(memoryDB, config, pruneTrie = false)
+ state = node["state"]
+
+ for k, v in state:
+ let key = hexToSeqByte(k)
+ let value = hexToSeqByte(v.getStr())
+ memoryDB.kvt.put(key, value)
+
+ let
+ parentNumber = blockNumber - 1
+ parent = com.db.getBlockHeader(parentNumber)
+ header = com.db.getBlockHeader(blockNumber)
+ headerHash = header.blockHash
+ blockBody = com.db.getBlockBody(headerHash)
+ chain = newChain(com)
+ headers = @[header]
+ bodies = @[blockBody]
+
+ chain.generateWitness = true # Enable code to generate and store witness in the db
+
+ # it's ok if setHead fails here because of missing ancestors
+ discard com.db.setHead(parent, true)
+ let validationResult = chain.persistBlocks(headers, bodies)
+ check validationResult == ValidationResult.OK
+
+ let
+ blockHash = memoryDB.getBlockHash(blockNumber)
+ witness = memoryDB.getBlockWitness(blockHash).value()
+ verifyWitnessResult = verifyWitness(parent.stateRoot, witness, {wfNoFlag})
+
+ check verifyWitnessResult.isOk()
+ let witnessData = verifyWitnessResult.value()
+
+ if blockBody.transactions.len() > 0:
+ check:
+ witness.len() > 0
+ witnessData.len() > 0
+
+proc persistBlockWitnessJsonMain*() =
+ suite "persist block json tests":
+ jsonTest("PersistBlockTests", testFixture)
+ #var testStatusIMPL: TestStatus
+ #let n = json.parseFile("tests" / "fixtures" / "PersistBlockTests" / "block420301.json")
+ #testFixture(n, testStatusIMPL)
+
+when isMainModule:
+ persistBlockWitnessJsonMain()