Culling legacy DB and accounts cache (#2197)

details:
+ Compiles nimbus all_tests
+ Failing tests have been commented out

hive_integration/nodocker/consensus/consensus_sim.nim

@@ -22,7 +22,6 @@ proc processChainData(cd: ChainData): TestStatus =
  let
  networkId = NetworkId(cd.params.config.chainId)
  com = CommonRef.new(newCoreDbRef DefaultDbMemory,
- pruneTrie = false,
  networkId,
  cd.params
  )

hive_integration/nodocker/engine/engine_env.nim

@@ -61,8 +61,7 @@ const
 
 proc makeCom*(conf: NimbusConf): CommonRef =
  CommonRef.new(
- newCoreDbRef LegacyDbMemory,
- conf.chainDbMode == ChainDbMode.Prune,
+ newCoreDbRef DefaultDbMemory,
  conf.networkId,
  conf.networkParams
  )

hive_integration/nodocker/graphql/graphql_sim.nim

@@ -78,8 +78,7 @@ proc main() =
  conf = makeConfig(@["--custom-network:" & genesisFile])
  ethCtx = newEthContext()
  ethNode = setupEthNode(conf, ethCtx, eth)
- com = CommonRef.new(newCoreDbRef LegacyDbMemory,
- pruneTrie = false,
+ com = CommonRef.new(newCoreDbRef DefaultDbMemory,
  conf.networkId,
  conf.networkParams
  )

hive_integration/nodocker/pyspec/test_env.nim

@@ -47,17 +47,16 @@ proc genesisHeader(node: JsonNode): BlockHeader =
  rlp.decode(genesisRLP, EthBlock).header
 
 proc setupELClient*(t: TestEnv, conf: ChainConfig, node: JsonNode) =
- let memDB = newCoreDbRef LegacyDbMemory
+ let memDB = newCoreDbRef DefaultDbMemory
  t.ctx = newEthContext()
  t.ethNode = setupEthNode(t.conf, t.ctx, eth)
  t.com = CommonRef.new(
  memDB,
- conf,
- t.conf.chainDbMode == ChainDbMode.Prune
+ conf
  )
  t.chainRef = newChain(t.com, extraValidation = true)
  let
- stateDB = AccountsCache.init(memDB, emptyRlpHash, t.conf.chainDbMode == ChainDbMode.Prune)
+ stateDB = LedgerCache.init(memDB, emptyRlpHash)
  genesisHeader = node.genesisHeader
 
  setupStateDB(node["pre"], stateDB)

hive_integration/nodocker/rpc/test_env.nim

@@ -76,8 +76,7 @@ proc setupEnv*(): TestEnv =
  let
  ethCtx = newEthContext()
  ethNode = setupEthNode(conf, ethCtx, eth)
- com = CommonRef.new(newCoreDbRef LegacyDbMemory,
- conf.chainDbMode == ChainDbMode.Prune,
+ com = CommonRef.new(newCoreDbRef DefaultDbMemory,
  conf.networkId,
  conf.networkParams
  )

nimbus/common/common.nim

@@ -46,9 +46,6 @@ type
  # all purpose storage
  db: CoreDbRef
 
- # prune underlying state db?
- pruneTrie: bool
-
  # block chain config
  config: ChainConfig
 
@@ -103,10 +100,8 @@ type
  ldgType: LedgerType
  ## Optional suggestion for the ledger cache to be used as state DB
 
-const
- CommonLegacyDbLedgerTypeDefault = LegacyAccountsCache
- ## Default ledger type to use, see `ldgType` above. This default will be
- ## superseded by `LedgerCache` as default for `Aristo` type deb backend.
+ pruneHistory: bool
+ ## Must not not set for a full node, might go away some time
 
 # ------------------------------------------------------------------------------
 # Forward declarations
@@ -147,31 +142,24 @@ proc daoCheck(conf: ChainConfig) =
  if conf.daoForkSupport and conf.daoForkBlock.isNone:
  conf.daoForkBlock = conf.homesteadBlock
 
-proc init(com : CommonRef,
- db : CoreDbRef,
- pruneTrie: bool,
- networkId: NetworkId,
- config : ChainConfig,
- genesis : Genesis,
- ldgType : LedgerType,
+proc init(com  : CommonRef,
+ db  : CoreDbRef,
+ networkId : NetworkId,
+ config : ChainConfig,
+ genesis  : Genesis,
+ ldgType : LedgerType,
+ pruneHistory: bool,
  ) {.gcsafe, raises: [CatchableError].} =
 
  config.daoCheck()
 
  com.db = db
- com.pruneTrie = pruneTrie
  com.config = config
  com.forkTransitionTable = config.toForkTransitionTable()
  com.networkId = networkId
  com.syncProgress= SyncProgress()
- com.ldgType = block:
- if ldgType != LedgerType(0):
- ldgType
- elif db.dbType in {AristoDbMemory,AristoDbRocks,AristoDbVoid}:
- # The `Aristo` backend does not work well with the `LegacyAccountsCache`
- LedgerCache
- else:
- CommonLegacyDbLedgerTypeDefault
+ com.ldgType = LedgerCache
+ com.pruneHistory= pruneHistory
 
  # Initalise the PoA state regardless of whether it is needed on the current
  # network. For non-PoA networks this descriptor is ignored.
@@ -235,10 +223,10 @@ proc getTdIfNecessary(com: CommonRef, blockHash: Hash256): Option[DifficultyInt]
 proc new*(
  _: type CommonRef;
  db: CoreDbRef;
- pruneTrie: bool = true;
  networkId: NetworkId = MainNet;
  params = networkParams(MainNet);
  ldgType = LedgerType(0);
+ pruneHistory = false;
  ): CommonRef
  {.gcsafe, raises: [CatchableError].} =
 
@@ -247,19 +235,19 @@ proc new*(
  new(result)
  result.init(
  db,
- pruneTrie,
  networkId,
  params.config,
  params.genesis,
- ldgType)
+ ldgType,
+ pruneHistory)
 
 proc new*(
  _: type CommonRef;
  db: CoreDbRef;
  config: ChainConfig;
- pruneTrie: bool = true;
  networkId: NetworkId = MainNet;
  ldgType = LedgerType(0);
+ pruneHistory = false;
  ): CommonRef
  {.gcsafe, raises: [CatchableError].} =
 
@@ -268,18 +256,17 @@ proc new*(
  new(result)
  result.init(
  db,
- pruneTrie,
  networkId,
  config,
  nil,
- ldgType)
+ ldgType,
+ pruneHistory)
 
 proc clone*(com: CommonRef, db: CoreDbRef): CommonRef =
  ## clone but replace the db
  ## used in EVM tracer whose db is CaptureDB
  CommonRef(
  db : db,
- pruneTrie : com.pruneTrie,
  config : com.config,
  forkTransitionTable: com.forkTransitionTable,
  forkIdCalculator: com.forkIdCalculator,
@@ -292,8 +279,8 @@ proc clone*(com: CommonRef, db: CoreDbRef): CommonRef =
  pow : com.pow,
  poa : com.poa,
  pos : com.pos,
- ldgType : com.ldgType
- )
+ ldgType : com.ldgType,
+  pruneHistory : com.pruneHistory)
 
 proc clone*(com: CommonRef): CommonRef =
  com.clone(com.db)
@@ -492,8 +479,8 @@ func cliqueEpoch*(com: CommonRef): int =
  if com.config.clique.epoch.isSome:
  return com.config.clique.epoch.get()
 
-func pruneTrie*(com: CommonRef): bool =
- com.pruneTrie
+func pruneHistory*(com: CommonRef): bool =
+ com.pruneHistory
 
 # always remember ChainId and NetworkId
 # are two distinct things that often got mixed

nimbus/common/genesis.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 # http:https://www.apache.org/licenses/LICENSE-2.0)
@@ -13,8 +13,7 @@
 import
  std/tables,
  eth/[common, eip1559],
- eth/trie/trie_defs,
- ../db/[ledger, core_db, state_db/read_write],
+ ../db/[ledger, core_db],
  ../constants,
  ./chain_config
 
@@ -28,8 +27,6 @@ type
  address: EthAddress; nonce: AccountNonce; balance: UInt256;
  code: openArray[byte]) {.catchRaise.}
 
- GenesisCompensateLegacySetupFn = proc() {.noRaise.}
-
  GenesisSetStorageFn = proc(
  address: EthAddress; slot: UInt256; val: UInt256) {.rlpRaise.}
 
@@ -40,70 +37,23 @@ type
  GenesisGetTrieFn = proc: CoreDbMptRef {.noRaise.}
 
  GenesisLedgerRef* = ref object
- ## Exportable ledger DB just for initialising Genesis. This is needed
- ## when using the `Aristo` backend which is not fully supported by the
- ## `AccountStateDB` object.
- ##
- ## Currently, using other than the `AccountStateDB` ledgers are
- ## experimental and test only. Eventually, the `GenesisLedgerRef` wrapper
- ## should disappear so that the `Ledger` object (which encapsulates
- ## `AccountsCache` and `AccountsLedger`) will prevail.
+ ## Exportable ledger DB just for initialising Genesis.
  ##
  addAccount: GenesisAddAccountFn
- compensateLegacySetup: GenesisCompensateLegacySetupFn
  setStorage: GenesisSetStorageFn
  commit: GenesisCommitFn
  rootHash: GenesisRootHashFn
  getTrie: GenesisGetTrieFn
 
-const
- GenesisLedgerTypeDefault* = LedgerType(0)
- ## Default ledger type to use, `LedgerType(0)` uses `AccountStateDB`
- ## rather than a `Ledger` variant.
-
 # ------------------------------------------------------------------------------
 # Private functions
 # ------------------------------------------------------------------------------
 
-proc initStateDbledgerRef(db: CoreDbRef; pruneTrie: bool): GenesisLedgerRef =
- let sdb = newAccountStateDB(db, emptyRlpHash, pruneTrie)
-
- GenesisLedgerRef(
- addAccount: proc(
- address: EthAddress;
- nonce: AccountNonce;
- balance: UInt256;
- code: openArray[byte];
- ) {.catchRaise.} =
- sdb.setAccount(address, newAccount(nonce, balance))
- sdb.setCode(address, code),
-
- compensateLegacySetup: proc() =
- if pruneTrie: db.compensateLegacySetup(),
-
- setStorage: proc(
- address: EthAddress;
- slot: UInt256;
- val: UInt256;
- ) {.rlpRaise.} =
- sdb.setStorage(address, slot, val),
-
- commit: proc() =
- discard,
-
- rootHash: proc(): Hash256 =
- sdb.rootHash(),
-
- getTrie: proc(): CoreDbMptRef =
- sdb.getTrie())
-
-
 proc initAccountsLedgerRef(
  db: CoreDbRef;
- pruneTrie: bool;
- ledgerType: LedgerType;
  ): GenesisLedgerRef =
- let ac = ledgerType.init(db, emptyRlpHash, pruneTrie)
+ ## Methods jump table
+ let ac = LedgerCache.init(db, EMPTY_ROOT_HASH)
 
  GenesisLedgerRef(
  addAccount: proc(
@@ -116,9 +66,6 @@ proc initAccountsLedgerRef(
  ac.setBalance(address, balance)
  ac.setCode(address, @code),
 
- compensateLegacySetup: proc() =
- if pruneTrie: db.compensateLegacySetup(),
-
  setStorage: proc(
  address: EthAddress;
  slot: UInt256;
@@ -141,15 +88,11 @@ proc initAccountsLedgerRef(
 
 proc newStateDB*(
  db: CoreDbRef;
- pruneTrie: bool;
- ledgerType = LedgerType(0);
+ ledgerType: LedgerType;
  ): GenesisLedgerRef =
- ## The flag `ledgerType` is set to zero for compatibility with legacy apps
- ## (see `test_state_network`).
- if ledgerType != LedgerType(0):
- db.initAccountsLedgerRef(pruneTrie, ledgerType)
- else:
- db.initStateDbledgerRef pruneTrie
+ ## Currently only `LedgerCache` supported for `ledgerType`.
+ doAssert ledgerType == LedgerCache
+ db.initAccountsLedgerRef()
 
 proc getTrie*(sdb: GenesisLedgerRef): CoreDbMptRef =
  ## Getter, used in `test_state_network`
@@ -167,22 +110,9 @@ proc toGenesisHeader*(
  ## The function returns the `Genesis` block header.
  ##
 
- # The following kludge is needed for the `LegacyDbPersistent` type database
- # when `pruneTrie` is enabled. For other cases, this code is irrelevant.
- sdb.compensateLegacySetup()
-
  for address, account in g.alloc:
  sdb.addAccount(address, account.nonce, account.balance, account.code)
 
- # Kludge:
- #
- # See https://github.com/status-im/nim-eth/issues/9 where other,
- # probably related debilities are discussed.
- #
- # This kludge also fixes the initial crash described in
- # https://github.com/status-im/nimbus-eth1/issues/932.
- sdb.compensateLegacySetup() # <-- kludge
-
  for k, v in account.storage:
  sdb.setStorage(address, k, v)
 
@@ -226,20 +156,20 @@ proc toGenesisHeader*(
  genesis: Genesis;
  fork: HardFork;
  db = CoreDbRef(nil);
- ledgerType = GenesisLedgerTypeDefault;
+ ledgerType = LedgerCache;
  ): BlockHeader
  {.gcsafe, raises: [CatchableError].} =
  ## Generate the genesis block header from the `genesis` and `config`
  ## argument value.
  let
- db = if db.isNil: newCoreDbRef LegacyDbMemory else: db
- sdb = newStateDB(db, pruneTrie = true, ledgerType)
+ db = if db.isNil: AristoDbMemory.newCoreDbRef() else: db
+ sdb = db.newStateDB(ledgerType)
  toGenesisHeader(genesis, sdb, fork)
 
 proc toGenesisHeader*(
  params: NetworkParams;
  db = CoreDbRef(nil);
- ledgerType = GenesisLedgerTypeDefault;
+ ledgerType = LedgerCache;
  ): BlockHeader
  {.raises: [CatchableError].} =
  ## Generate the genesis block header from the `genesis` and `config`