Initial implementation of the merge spec

Includes a simple test harness for the merge interop M1 milestone

This aims to enable connecting nimbus-eth2 to nimbus-eth1 within
the testing protocol described here:

https://github.com/status-im/nimbus-eth2/blob/amphora-merge-interop/docs/interop_merge.md

To execute the work-in-progress test, please run:

In terminal 1:
tests/amphora/launch-nimbus.sh

In terminal 2:
tests/amphora/check-merge-test-vectors.sh

.gitignore

@@ -27,3 +27,5 @@ nimcache
 /debug*.json
 /block*.json
 /.update.timestamp
+
+*.generated.nim

.gitmodules

@@ -114,8 +114,8 @@
  ignore = dirty
  branch = master
 [submodule "vendor/nim-web3"]
- path = vendor/nim-web3
- url = https://github.com/status-im/nim-web3.git
+ path = vendor/nim-web3
+ url = https://github.com/status-im/nim-web3.git
  ignore = dirty
  branch = master
 [submodule "vendor/nim-byteutils"]

nimbus/accounts/manager.nim

@@ -8,7 +8,7 @@
 # those terms.
 
 import
- std/[os, json, tables],
+ std/[os, json, tables, strutils],
  stew/[byteutils, results],
  eth/[keyfile, common, keys],
  chronicles
@@ -87,7 +87,7 @@ iterator addresses*(am: AccountsManager): EthAddress =
 proc importPrivateKey*(am: var AccountsManager, fileName: string): Result[void, string] =
  try:
  let pkhex = readFile(fileName)
- let res = PrivateKey.fromHex(pkhex)
+ let res = PrivateKey.fromHex(pkhex.strip)
  if res.isErr:
  return err("not a valid private key, expect 32 bytes hex")
 

nimbus/chain_config.nim

@@ -40,6 +40,7 @@ type
  londonBlock : Option[BlockNumber]
  arrowGlacierBlock : Option[BlockNumber]
  clique : CliqueOptions
+ terminalTotalDifficulty*: Option[UInt256]
 
  ChainConfig* = object
  chainId* : ChainId
@@ -67,6 +68,8 @@ type
  cliquePeriod* : int
  cliqueEpoch* : int
 
+ terminalTotalDifficulty*: Option[UInt256]
+
  Genesis* = object
  nonce* : BlockNonce
  timestamp* : EthTime
@@ -76,6 +79,9 @@ type
  mixHash* : Hash256
  coinbase* : EthAddress
  alloc* : GenesisAlloc
+ number* : BlockNumber
+ gasUser* : GasInt
+ parentHash* : Hash256
  baseFeePerGas*: Option[UInt256]
 
  GenesisAlloc* = Table[EthAddress, GenesisAccount]
@@ -97,6 +103,21 @@ type
  config : ChainOptions
  genesis: Genesis
 
+ GenesisFile* = object
+ config : ChainOptions
+ nonce* : BlockNonce
+ timestamp* : EthTime
+ extraData* : seq[byte]
+ gasLimit* : GasInt
+ difficulty* : DifficultyInt
+ mixHash* : Hash256
+ coinbase* : EthAddress
+ alloc* : GenesisAlloc
+ number* : BlockNumber
+ gasUser* : GasInt
+ parentHash* : Hash256
+ baseFeePerGas*: Option[UInt256]
+
 const
  CustomNet* = 0.NetworkId
  # these are public network id
@@ -194,6 +215,8 @@ proc loadNetworkParams*(fileName: string, cg: var NetworkParams):
  if cc.config.clique.epoch.isSome:
  cg.config.cliqueEpoch = cc.config.clique.epoch.get()
 
+ cg.config.terminalTotalDifficulty = cc.config.terminalTotalDifficulty
+
  template validateFork(forkName: untyped, nextBlock: BlockNumber) =
  let fork = astToStr(forkName)
  if cc.config.forkName.isSome:

nimbus/config.nim

@@ -102,6 +102,7 @@ const
  defaultEthRpcPort = 8545
  defaultEthWsPort = 8546
  defaultEthGraphqlPort = 8547
+ defaultEngineApiPort = 8550
  defaultListenAddress = (static ValidIpAddress.init("0.0.0.0"))
  defaultAdminListenAddress = (static ValidIpAddress.init("127.0.0.1"))
  defaultListenAddressDesc = $defaultListenAddress & ", meaning all network interfaces"
@@ -306,6 +307,27 @@ type
  defaultValueDesc: $DiscoveryType.V4
  name: "discovery" .}: DiscoveryType
 
+ terminalTotalDifficulty* {.
+ desc: "The terminal total difficulty of the eth2 merge transition block"
+ name: "terminal-total-difficulty" .}: Option[UInt256]
+
+ engineApiEnabled* {.
+ desc: "Enable the Engine API"
+ defaultValue: false
+ name: "engine-api" .}: bool
+
+ engineApiPort* {.
+ desc: "Listening port for the Engine API"
+ defaultValue: defaultEngineApiPort
+ defaultValueDesc: $defaultEngineApiPort
+ name: "engine-api-port" .}: Port
+
+ engineApiAddress* {.
+ desc: "Listening address for the Engine API"
+ defaultValue: defaultAdminListenAddress
+ defaultValueDesc: defaultAdminListenAddressDesc
+ name: "engine-api-address" .}: ValidIpAddress
+
  nodeKeyHex* {.
  desc: "P2P node private key (as 32 bytes hex string)"
  defaultValue: ""
@@ -418,13 +440,18 @@ type
  defaultValue: ""
  name: "blocks-file" }: InputFile
 
-
 proc parseCmdArg(T: type NetworkId, p: TaintedString): T =
  parseInt(p.string).T
 
 proc completeCmdArg(T: type NetworkId, val: TaintedString): seq[string] =
  return @[]
 
+proc parseCmdArg(T: type UInt256, p: TaintedString): T =
+ parse(string p, T)
+
+proc completeCmdArg(T: type UInt256, val: TaintedString): seq[string] =
+ return @[]
+
 proc parseCmdArg(T: type EthAddress, p: TaintedString): T =
  try:
  result = hexToByteArray(p.string, 20)

nimbus/db/db_chain.nim

@@ -55,8 +55,13 @@ proc exists*(self: BaseChainDB, hash: Hash256): bool =
 proc getBlockHeader*(self: BaseChainDB; blockHash: Hash256, output: var BlockHeader): bool =
  let data = self.db.get(genericHashKey(blockHash).toOpenArray)
  if data.len != 0:
- output = rlp.decode(data, BlockHeader)
- result = true
+ try:
+ output = rlp.decode(data, BlockHeader)
+ true
+ except RlpError:
+ false
+ else:
+ false
 
 proc getBlockHeader*(self: BaseChainDB, blockHash: Hash256): BlockHeader =
  ## Returns the requested block header as specified by block hash.
@@ -273,17 +278,24 @@ proc setAsCanonicalChainHead(self: BaseChainDB; headerHash: Hash256): seq[BlockH
 
  return newCanonicalHeaders
 
+proc headerExists*(self: BaseChainDB; blockHash: Hash256): bool =
+ ## Returns True if the header with the given block hash is in our DB.
+ self.db.contains(genericHashKey(blockHash).toOpenArray)
+
+proc setHead*(self: BaseChainDB, blockHash: Hash256): bool =
+ if self.headerExists(blockHash):
+ self.db.put(canonicalHeadHashKey().toOpenArray, rlp.encode(blockHash))
+ return true
+ else:
+ return false
+
 proc setHead*(self: BaseChainDB, header: BlockHeader, writeHeader = false) =
  var headerHash = rlpHash(header)
  if writeHeader:
  self.db.put(genericHashKey(headerHash).toOpenArray, rlp.encode(header))
  self.addBlockNumberToHashLookup(header)
  self.db.put(canonicalHeadHashKey().toOpenArray, rlp.encode(headerHash))
 
-proc headerExists*(self: BaseChainDB; blockHash: Hash256): bool =
- ## Returns True if the header with the given block hash is in our DB.
- self.db.contains(genericHashKey(blockHash).toOpenArray)
-
 proc persistReceipts*(self: BaseChainDB, receipts: openArray[Receipt]): Hash256 =
  var trie = initHexaryTrie(self.db)
  for idx, rec in receipts:

nimbus/nimbus.nim

@@ -18,11 +18,12 @@ import
  chronos, json_rpc/rpcserver, chronicles,
  eth/p2p/rlpx_protocols/les_protocol,
  ./p2p/blockchain_sync, eth/net/nat, eth/p2p/peer_pool,
+ ./p2p/clique/[clique_desc, clique_sealer],
  ./sync/protocol_eth65,
- config, genesis, rpc/[common, p2p, debug], p2p/chain,
+ config, genesis, rpc/[common, p2p, debug, engine_api], p2p/chain,
  eth/trie/db, metrics, metrics/[chronos_httpserver, chronicles_support],
  graphql/ethapi, context,
- "."/[conf_utils, sealer, constants]
+ "."/[conf_utils, sealer, constants, utils]
 
 when defined(evmc_enabled):
  import transaction/evmc_dynamic_loader
@@ -38,6 +39,7 @@ type
 
  NimbusNode = ref object
  rpcServer: RpcHttpServer
+ engineApiServer: RpcHttpServer
  ethNode: EthereumNode
  state: NimbusState
  graphqlServer: GraphqlHttpServerRef
@@ -183,11 +185,38 @@ proc localServices(nimbus: NimbusNode, conf: NimbusConf,
  if rs.isErr:
  echo rs.error
  quit(QuitFailure)
+
+ proc signFunc(signer: EthAddress, message: openArray[byte]): Result[RawSignature, cstring] {.gcsafe.} =
+ let
+ hashData = keccakHash(message)
+ acc = nimbus.ctx.am.getAccount(signer).tryGet()
+ rawSign = sign(acc.privateKey, SkMessage(hashData.data)).toRaw
+
+ ok(rawSign)
+
+ # TODO: There should be a better place to initialize this
+ nimbus.chainRef.clique.authorize(conf.engineSigner, signFunc)
+
+ let initialSealingEngineState =
+ if conf.networkParams.config.terminalTotalDifficulty.isSome and
+ conf.networkParams.config.terminalTotalDifficulty.get.isZero:
+ nimbus.chainRef.ttdReachedAt = some(BlockNumber.zero)
+ EnginePostMerge
+ else:
+ EngineStopped
  nimbus.sealingEngine = SealingEngineRef.new(
- nimbus.chainRef, nimbus.ctx, conf.engineSigner
- )
+ # TODO: Implement the initial state correctly
+  nimbus.chainRef, nimbus.ctx, conf.engineSigner, initialSealingEngineState)
  nimbus.sealingEngine.start()
 
+ if conf.engineApiEnabled:
+ nimbus.engineApiServer = newRpcHttpServer([
+ initTAddress(conf.engineApiAddress, conf.engineApiPort)
+ ])
+ setupEngineAPI(nimbus.sealingEngine, nimbus.engineApiServer)
+ nimbus.engineAPiServer.start()
+ info "Starting engine API server", port = conf.engineApiPort
+
  # metrics server
  if conf.metricsEnabled:
  info "Starting metrics HTTP server", address = conf.metricsAddress, port = conf.metricsPort
@@ -241,6 +270,8 @@ proc stop*(nimbus: NimbusNode, conf: NimbusConf) {.async, gcsafe.} =
  trace "Graceful shutdown"
  if conf.rpcEnabled:
  await nimbus.rpcServer.stop()
+ if conf.engineApiEnabled:
+ await nimbus.engineAPiServer.stop()
  if conf.wsEnabled:
  nimbus.wsRpcServer.stop()
  if conf.graphqlEnabled:

nimbus/p2p/chain/chain_desc.nim

@@ -63,6 +63,11 @@ type
  ## For non-PoA networks (when `db.config.poaEngine` is `false`),
  ## this descriptor is ignored.
 
+ ttdReachedAt*: Option[BlockNumber]
+ ## The first block which difficulty was above the terminal
+ ## total difficulty. In networks with TTD=0, this would be
+ ## the very first block.
+
 {.push raises: [Defect].}
 
 # ------------------------------------------------------------------------------
@@ -75,6 +80,9 @@ func toNextFork(n: BlockNumber): uint64 =
  else:
  result = n.truncate(uint64)
 
+func isBlockAfterTtd*(c: Chain, blockNum: BlockNumber): bool =
+ c.ttdReachedAt.isSome and blockNum > c.ttdReachedAt.get
+
 func getNextFork(c: ChainConfig, fork: ChainFork): uint64 =
  let next: array[ChainFork, uint64] = [
  0'u64,
@@ -165,7 +173,6 @@ proc newChain*(db: BaseChainDB; poa: Clique; extraValidation: bool): Chain
  new result
  result.initChain(db, poa, extraValidation)
 
-
 proc newChain*(db: BaseChainDB, extraValidation: bool): Chain
  {.gcsafe, raises: [Defect,CatchableError].} =
  ## Constructor for the `Chain` descriptor object with default initialisation

nimbus/p2p/chain/persist_blocks.nim

@@ -90,7 +90,8 @@ proc persistBlocksImpl(c: Chain; headers: openarray[BlockHeader];
  header,
  body,
  checkSealOK = false, # TODO: how to checkseal from here
- c.pow)
+ ttdReached = c.isBlockAfterTtd(header.blockNumber),
+ pow = c.pow)
  if res.isErr:
  debug "block validation error",
  msg = res.error

nimbus/p2p/clique/clique_sealer.nim

@@ -167,18 +167,14 @@ proc verifyUncles*(c: Clique; ethBlock: EthBlock): CliqueOkResult =
 
 # clique/clique.go(506): func (c *Clique) Prepare(chain [..]
 proc prepare*(c: Clique; parent: BlockHeader, header: var BlockHeader): CliqueOkResult
- {.gcsafe, raises: [Defect,CatchableError].} =
+ {.gcsafe, raises: [Defect, CatchableError].} =
  ## For the Consensus Engine, `prepare()` initializes the consensus fields
  ## of a block header according to the rules of a particular engine. The
  ## changes are executed inline.
  ##
  ## This implementation prepares all the consensus fields of the header for
  ## running the transactions on top.
 
- # If the block isn't a checkpoint, cast a random vote (good enough for now)
- header.coinbase.reset
- header.nonce.reset
-
  # Assemble the voting snapshot to check which votes make sense
  let rc = c.cliqueSnapshot(header.parentHash, @[])
  if rc.isErr:
@@ -225,7 +221,6 @@ proc authorize*(c: Clique; signer: EthAddress; signFn: CliqueSignerFn) =
  c.signer = signer
  c.signFn = signFn
 
-
 # clique/clique.go(724): func CliqueRLP(header [..]
 proc cliqueRlp*(header: BlockHeader): seq[byte] =
  ## Returns the rlp bytes which needs to be signed for the proof-of-authority
@@ -238,7 +233,6 @@ proc cliqueRlp*(header: BlockHeader): seq[byte] =
  ##hashes for the same header.
  header.encodeSealHeader
 
-
 # clique/clique.go(688): func SealHash(header *types.Header) common.Hash {
 proc sealHash*(header: BlockHeader): Hash256 =
  ## For the Consensus Engine, `sealHash()` returns the hash of a block prior