Unified database frontend (#1661)

* Remove 32bit os support from `custom_network` unit test

also:
* Fix compilation annoyance #1648
* Fix unit test on Kiln (changed `merge` logic?)

* Hide unused sources do not compile

why:
* Get them out of the way before major update
* Import and function prototype mismatch -- maybe some changes got out
  of scope.

* Re-implemented `db_chain` as `core_db`

why:
  Hiding `TrieDatabaseRef` and `HexaryTrie` by default allows to replace
  the current db wrapper by some other one, e.g. Aristo

* Support compiler exception warnings for CoreDbRef base methods.

* Allow `pairs()` iterator on all memory based key-value tables

why:
  Previously only available for capture recorder.

* Backport `chain_db.nim` changes into its re-implementation `core_apps.nim`

* Fix exception annotation

nimbus/db/core_db.nim

@@ -0,0 +1,162 @@
+# Nimbus
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed under either of
+# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+# http:https://www.apache.org/licenses/LICENSE-2.0)
+# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+# http:https://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+## Core database replacement wrapper object
+## ========================================
+##
+## See `core_db/README.md`
+##
+{.push raises: [].}
+
+import
+ chronicles,
+ eth/[common, trie/db],
+ ./core_db/[base, core_apps, legacy]
+
+export
+ common,
+ core_apps,
+
+ # Not all symbols from the object sources will be exported by default
+ CoreDbCaptFlags,
+ CoreDbCaptRef,
+ CoreDbKvtRef,
+ CoreDbMptRef,
+ CoreDbPhkRef,
+ CoreDbRef,
+ CoreDbTxID,
+ CoreDbTxRef,
+ CoreDbType,
+ LegacyCoreDbRef, # for shortTimeReadOnly()
+ beginTransaction,
+ commit,
+ compensateLegacySetup,
+ contains,
+ dbType,
+ del,
+ dispose,
+ get,
+ getTransactionID,
+ isPruning,
+ kvt,
+ maybeGet,
+ mpt,
+ mptPrune,
+ newCoreDbCaptRef,
+ parent,
+ phk,
+ phkPrune,
+ put,
+ recorder,
+ rollback,
+ rootHash,
+ safeDispose,
+ setTransactionID
+
+logScope:
+ topics = "core_db"
+
+# ------------------------------------------------------------------------------
+# Private functions: helpers
+# ------------------------------------------------------------------------------
+
+template logTxt(info: static[string]): static[string] =
+ "ChainDB " & info
+
+proc itNotImplemented(db: CoreDbRef|CoreDbKvtRef, name: string) {.used.} =
+ debug logTxt "iterator not implemented", dbType=db.dbType, meth=name
+
+proc tmplNotImplemented*(db: CoreDbRef, name: string) {.used.} =
+ debug logTxt "template not implemented", dbType=db.dbType, meth=name
+
+# ------------------------------------------------------------------------------
+# Public constructor
+# ------------------------------------------------------------------------------
+
+proc newCoreDbRef*(
+ db: TrieDatabaseRef;
+ ): CoreDbRef
+ {.gcsafe, deprecated: "use newCoreDbRef(LegacyDbPersistent,<path>)".} =
+ ## Legacy constructor.
+ ##
+ ## Note: Using legacy notation `newCoreDbRef()` rather than
+ ## `CoreDbRef.init()` because of compiler coughing.
+ db.newLegacyCoreDbRef()
+
+proc newCoreDbRef*(
+ dbType: static[CoreDbType];
+ ): CoreDbRef =
+ ## Constructor for volatile/memory type DB
+ ##
+ ## Note: Using legacy notation `newCoreDbRef()` rather than
+ ## `CoreDbRef.init()` because of compiler coughing.
+ when dbType == LegacyDbMemory:
+ newLegacyMemoryCoreDbRef()
+ else:
+ {.error: "Unsupported dbType for CoreDbRef.init()".}
+
+proc newCoreDbRef*(
+ dbType: static[CoreDbType];
+ path: string;
+ ): CoreDbRef =
+ ## General constructor (the `path` argument is ignored for volatile/memory
+ ## type DB)
+ ##
+ ## Note: Using legacy notation `newCoreDbRef()` rather than
+ ## `CoreDbRef.init()` because of compiler coughing.
+ when dbType == LegacyDbMemory:
+ newLegacyMemoryCoreDbRef()
+ elif dbType == LegacyDbPersistent:
+ newLegacyPersistentCoreDbRef path
+ else:
+ {.error: "Unsupported dbType for CoreDbRef.init()".}
+
+# ------------------------------------------------------------------------------
+# Public template wrappers
+# ------------------------------------------------------------------------------
+
+template shortTimeReadOnly*(db: CoreDbRef; id: CoreDbTxID; body: untyped) =
+ proc action() {.gcsafe, raises: [CatchableError].} =
+ body
+ case db.dbType:
+ of LegacyDbMemory, LegacyDbPersistent:
+ db.LegacyCoreDbRef.shortTimeReadOnly(id, action)
+ else:
+ db.tmplNotImplemented "shortTimeReadOnly"
+
+# ------------------------------------------------------------------------------
+# Public iterators
+# ------------------------------------------------------------------------------
+
+iterator pairs*(
+ db: CoreDbKvtRef;
+ ): (Blob, Blob)
+ {.gcsafe.} =
+ case db.dbType:
+ of LegacyDbMemory:
+ for k,v in db.LegacyCoreDbKvtRef:
+ yield (k,v)
+ else:
+ db.itNotImplemented "pairs/kvt"
+
+iterator pairs*(
+ db: CoreDbMptRef;
+ ): (Blob, Blob)
+ {.gcsafe, raises: [RlpError].} =
+ case db.parent.dbType:
+ of LegacyDbMemory, LegacyDbPersistent:
+ for k,v in db.LegacyCoreDbMptRef:
+ yield (k,v)
+ else:
+ db.parent.itNotImplemented "pairs/mpt"
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------

nimbus/db/core_db/.gitignore

@@ -0,0 +1 @@
+*.html

nimbus/db/core_db/README.md

@@ -0,0 +1,79 @@
+Core database replacement wrapper object
+========================================
+This wrapper replaces the *TrieDatabaseRef* and its derivatives by the new
+object *CoreDbRef*.
+
+Relations to current *TrieDatabaseRef* implementation
+-----------------------------------------------------
+Here are some incomplete translations for objects and constructors.
+
+### Object types:
+
+| **Legacy notation** | **CoreDbRef based replacement** |
+|:----------------------------|:--------------------------------------|
+| | |
+| ChainDB | (don't use/avoid) |
+| ChainDbRef | CoreDbRef |
+| TrieDatabaseRef | CoreDbKvtRef |
+| HexaryTrie | CoreDbMptRef |
+| SecureHexaryTrie | CoreDbPhkRef |
+| DbTransaction | CoreDbTxRef |
+| TransactionID | CoreDbTxID |
+
+
+### Constructors:
+
+| **Legacy notation** | **CoreDbRef based replacement** |
+|:----------------------------|:--------------------------------------|
+| | |
+| trieDB newChainDB("..") | newCoreDbRef(LegacyDbPersistent,"..") |
+| newMemoryDB() | newCoreDbRef(LegacyDbMemory) |
+| -- | |
+| initHexaryTrie(db,..) | db.mpt(..) (no pruning) |
+| | db.mptPrune(..) (w/pruning true/false)|
+| -- | |
+| initSecureHexaryTrie(db,..) | db.phk(..) (no pruning) |
+| | db.phkPrune(..) (w/pruning true/false)|
+| -- | |
+| newCaptureDB(db,memDB) | newCoreDbCaptRef(db) (see below) |
+
+
+Usage of the replacement wrapper
+--------------------------------
+
+### Objects pedigree:
+
+  CoreDbRef -- base descriptor
+  | | | |
+  | | | +-- CoreDbMptRef -- hexary trie instance
+  | | | | : :
+  | | | +-- CoreDbMptRef -- hexary trie instance
+  | | |
+  | | |
+  | | +---- CoreDbPhkRef -- pre-hashed key hexary trie instance
+  | | | : :
+  | | +---- CoreDbPhkRef -- pre-hashed key hexary trie instance
+  | |
+  | |
+  | +------ CoreDbKvtRef -- single static key-value table
+  |
+  |
+  +-------- CoreDbCaptRef -- tracer support descriptor
+
+### Instantiating standard database object descriptors works as follows:
+
+  let
+  db = newCoreDbRef(..) # new base descriptor
+  mpt = db.mpt(..) # hexary trie/Merkle Patricia Tree
+  phk = db.phk(..) # pre-hashed key hexary trie/MPT
+  kvt = db.kvt # key-value table
+
+### Tracer support setup by hiding the current *CoreDbRef* behind a replacement:
+
+  let
+  capture = newCoreDbCaptRef(db)
+  db = capture.recorder # use the recorder in place of db
+  ...
+
+  for key,value in capture.recorder.kvt:
+  ... # process recorded data