diff --git a/hive_integration/nodocker/graphql/graphql_sim.nim b/hive_integration/nodocker/graphql/graphql_sim.nim index 997180f24..b17ff2ec2 100644 --- a/hive_integration/nodocker/graphql/graphql_sim.nim +++ b/hive_integration/nodocker/graphql/graphql_sim.nim @@ -91,7 +91,11 @@ proc main() = var stat: SimStat let start = getTime() - for fileName {.inject.} in walkDirRec( + + #let fileName = caseFolder & "/37_eth_sendRawTransaction_nonceTooLow.json" + #block: + + for fileName in walkDirRec( caseFolder, yieldFilter = {pcFile,pcLinkToFile}): if not fileName.endsWith(".json"): continue @@ -101,6 +105,9 @@ proc main() = let status = ctx.processNode(node, fileName) stat.inc(name, status) + # simulate the real simulator + txPool.disposeAll() + let elpd = getTime() - start print(stat, elpd, "graphql") diff --git a/hive_integration/nodocker/graphql/init/blocks.rlp b/hive_integration/nodocker/graphql/init/blocks.rlp index d29453d3e..480efc87d 100644 Binary files a/hive_integration/nodocker/graphql/init/blocks.rlp and b/hive_integration/nodocker/graphql/init/blocks.rlp differ diff --git a/hive_integration/nodocker/graphql/init/genesis.json b/hive_integration/nodocker/graphql/init/genesis.json index d26036f5a..50c586e53 100644 --- a/hive_integration/nodocker/graphql/init/genesis.json +++ b/hive_integration/nodocker/graphql/init/genesis.json @@ -1,4 +1,20 @@ { + "config": { + "chainId": 1, + "homesteadBlock": 33, + "eip150Block": 33, + "eip155Block": 33, + "eip158Block": 33, + "byzantiumBlock": 33, + "constantinopleBlock": 33, + "petersburgBlock": 33, + "istanbulBlock": 33, + "muirGlacierBlock": 33, + "berlinBlock": 33, + "londonBlock": 33, + "terminalTotalDifficulty": 4357120, + "shanghaiTime": 1444660030 + }, "genesis": { "coinbase" : "0x8888f1f195afa192cfee860698584c030f4c9db1", "difficulty" : "0x020000", diff --git a/nimbus/common/chain_config.nim b/nimbus/common/chain_config.nim index 4f55339bb..f808d3c54 100644 --- a/nimbus/common/chain_config.nim +++ b/nimbus/common/chain_config.nim @@ -35,8 +35,8 @@ type gasUser* : GasInt parentHash* : Hash256 baseFeePerGas*: Option[UInt256] - dataGasUsed* : Option[uint64] # EIP-4844 - excessDataGas*: Option[uint64] # EIP-4844 + blobGasUsed* : Option[uint64] # EIP-4844 + excessBlobGas*: Option[uint64] # EIP-4844 GenesisAlloc* = Table[EthAddress, GenesisAccount] GenesisAccount* = object @@ -67,8 +67,8 @@ type gasUser* : GasInt parentHash* : Hash256 baseFeePerGas*: Option[UInt256] - dataGasUsed* : Option[uint64] # EIP-4844 - excessDataGas*: Option[uint64] # EIP-4844 + blobGasUsed* : Option[uint64] # EIP-4844 + excessBlobGas*: Option[uint64] # EIP-4844 const CustomNet* = 0.NetworkId @@ -166,6 +166,7 @@ proc readValue(reader: var JsonReader, value: var BlockNonce) except ValueError as ex: reader.raiseUnexpectedValue(ex.msg) +# genesis timestamp is in hex proc readValue(reader: var JsonReader, value: var EthTime) {.gcsafe, raises: [SerializationError, IOError].} = try: @@ -173,6 +174,21 @@ proc readValue(reader: var JsonReader, value: var EthTime) except ValueError as ex: reader.raiseUnexpectedValue(ex.msg) +# but shanghaiTime and cancunTime in config is in int literal +proc readValue(reader: var JsonReader, value: var Option[EthTime]) + {.gcsafe, raises: [SerializationError, IOError].} = + let tok = reader.lexer.lazyTok + if tok == tkNull: + reset value + reader.lexer.next() + else: + # both readValue(GasInt/AccountNonce) will be called if + # we use readValue(int64/uint64) + let tok {.used.} = reader.lexer.tok # resove lazy token + let val = reader.lexer.absIntVal.int64 + value = some val.fromUnix + reader.lexer.next() + proc readValue(reader: var JsonReader, value: var seq[byte]) {.gcsafe, raises: [SerializationError, IOError].} = try: diff --git a/nimbus/common/genesis.nim b/nimbus/common/genesis.nim index a92afa748..42b2f836f 100644 --- a/nimbus/common/genesis.nim +++ b/nimbus/common/genesis.nim @@ -98,8 +98,8 @@ proc toGenesisHeader*( result.withdrawalsRoot = some(EMPTY_ROOT_HASH) if fork >= Cancun: - result.dataGasUsed = g.dataGasUsed - result.excessDataGas = g.excessDataGas + result.blobGasUsed = g.blobGasUsed + result.excessBlobGas = g.excessBlobGas proc toGenesisHeader*( genesis: Genesis; diff --git a/nimbus/constants.nim b/nimbus/constants.nim index ee3b6b466..9cb0f9217 100644 --- a/nimbus/constants.nim +++ b/nimbus/constants.nim @@ -78,11 +78,11 @@ const MAX_TX_WRAP_COMMITMENTS* = 1 shl 12 # 2^12 BLOB_COMMITMENT_VERSION_KZG* = 0x01.byte FIELD_ELEMENTS_PER_BLOB* = 4096 - DATA_GAS_PER_BLOB* = (1 shl 17).uint64 # 2^17 - TARGET_DATA_GAS_PER_BLOCK* = (1 shl 18).uint64 # 2^18 - MIN_DATA_GASPRICE* = 1'u64 - DATA_GASPRICE_UPDATE_FRACTION* = 2225652'u64 - MAX_DATA_GAS_PER_BLOCK* = (1 shl 19).uint64 # 2^19 - MaxAllowedBlob* = MAX_DATA_GAS_PER_BLOCK div DATA_GAS_PER_BLOB + GAS_PER_BLOB* = (1 shl 17).uint64 # 2^17 + TARGET_BLOB_GAS_PER_BLOCK* = (1 shl 18).uint64 # 2^18 + MIN_BLOB_GASPRICE* = 1'u64 + BLOB_GASPRICE_UPDATE_FRACTION* = 2225652'u64 + MAX_BLOB_GAS_PER_BLOCK* = (1 shl 19).uint64 # 2^19 + MaxAllowedBlob* = MAX_BLOB_GAS_PER_BLOCK div GAS_PER_BLOB # End diff --git a/nimbus/core/eip4844.nim b/nimbus/core/eip4844.nim index 742628b72..d89dd9fa4 100644 --- a/nimbus/core/eip4844.nim +++ b/nimbus/core/eip4844.nim @@ -77,16 +77,16 @@ proc pointEvaluation*(input: openArray[byte]): Result[void, string] = ok() -# calcExcessDataGas implements calc_excess_data_gas from EIP-4844 -proc calcExcessDataGas*(parent: BlockHeader): uint64 = +# calcExcessBlobGas implements calc_excess_data_gas from EIP-4844 +proc calcExcessBlobGas*(parent: BlockHeader): uint64 = let - excessDataGas = parent.excessDataGas.get(0'u64) - dataGasUsed = parent.dataGasUsed.get(0'u64) + excessBlobGas = parent.excessBlobGas.get(0'u64) + blobGasUsed = parent.blobGasUsed.get(0'u64) - if excessDataGas + dataGasUsed < TARGET_DATA_GAS_PER_BLOCK: + if excessBlobGas + blobGasUsed < TARGET_BLOB_GAS_PER_BLOCK: 0'u64 else: - excessDataGas + dataGasUsed - TARGET_DATA_GAS_PER_BLOCK + excessBlobGas + blobGasUsed - TARGET_BLOB_GAS_PER_BLOCK # fakeExponential approximates factor * e ** (num / denom) using a taylor expansion # as described in the EIP-4844 spec. @@ -103,33 +103,33 @@ func fakeExponential*(factor, numerator, denominator: uint64): uint64 = output div denominator -proc getTotalDataGas*(tx: Transaction): uint64 = - DATA_GAS_PER_BLOB * tx.versionedHashes.len.uint64 +proc getTotalBlobGas*(tx: Transaction): uint64 = + GAS_PER_BLOB * tx.versionedHashes.len.uint64 -proc getTotalDataGas*(versionedHashesLen: int): uint64 = - DATA_GAS_PER_BLOB * versionedHasheslen.uint64 +proc getTotalBlobGas*(versionedHashesLen: int): uint64 = + GAS_PER_BLOB * versionedHasheslen.uint64 -# getDataGasPrice implements get_data_gas_price from EIP-4844 -func getDataGasprice*(parentExcessDataGas: uint64): uint64 = +# getBlobGasPrice implements get_data_gas_price from EIP-4844 +func getBlobGasprice*(parentExcessBlobGas: uint64): uint64 = fakeExponential( - MIN_DATA_GASPRICE, - parentExcessDataGas, - DATA_GASPRICE_UPDATE_FRACTION + MIN_BLOB_GASPRICE, + parentExcessBlobGas, + BLOB_GASPRICE_UPDATE_FRACTION ) proc calcDataFee*(tx: Transaction, - parentExcessDataGas: Option[uint64]): uint64 = - tx.getTotalDataGas * - getDataGasprice(parentExcessDataGas.get(0'u64)) + parentExcessBlobGas: Option[uint64]): uint64 = + tx.getTotalBlobGas * + getBlobGasprice(parentExcessBlobGas.get(0'u64)) proc calcDataFee*(versionedHashesLen: int, - parentExcessDataGas: Option[uint64]): uint64 = - getTotalDataGas(versionedHashesLen) * - getDataGasprice(parentExcessDataGas.get(0'u64)) + parentExcessBlobGas: Option[uint64]): uint64 = + getTotalBlobGas(versionedHashesLen) * + getBlobGasprice(parentExcessBlobGas.get(0'u64)) -func dataGasUsed(txs: openArray[Transaction]): uint64 = +func blobGasUsed(txs: openArray[Transaction]): uint64 = for tx in txs: - result += tx.getTotalDataGas + result += tx.getTotalBlobGas # https://eips.ethereum.org/EIPS/eip-4844 func validateEip4844Header*( @@ -137,34 +137,34 @@ func validateEip4844Header*( txs: openArray[Transaction]): Result[void, string] {.raises: [].} = if not com.forkGTE(Cancun): - if header.dataGasUsed.isSome: - return err("unexpected EIP-4844 dataGasUsed in block header") + if header.blobGasUsed.isSome: + return err("unexpected EIP-4844 blobGasUsed in block header") - if header.excessDataGas.isSome: - return err("unexpected EIP-4844 excessDataGas in block header") + if header.excessBlobGas.isSome: + return err("unexpected EIP-4844 excessBlobGas in block header") return ok() - if header.dataGasUsed.isNone: - return err("expect EIP-4844 dataGasUsed in block header") + if header.blobGasUsed.isNone: + return err("expect EIP-4844 blobGasUsed in block header") - if header.excessDataGas.isNone: - return err("expect EIP-4844 excessDataGas in block header") + if header.excessBlobGas.isNone: + return err("expect EIP-4844 excessBlobGas in block header") let - headerDataGasUsed = header.dataGasUsed.get() - dataGasUsed = dataGasUsed(txs) - headerExcessDataGas = header.excessDataGas.get - excessDataGas = calcExcessDataGas(parentHeader) + headerBlobGasUsed = header.blobGasUsed.get() + blobGasUsed = blobGasUsed(txs) + headerExcessBlobGas = header.excessBlobGas.get + excessBlobGas = calcExcessBlobGas(parentHeader) - if dataGasUsed > MAX_DATA_GAS_PER_BLOCK: - return err("dataGasUsed " & $dataGasUsed & " exceeds maximum allowance " & $MAX_DATA_GAS_PER_BLOCK) + if blobGasUsed > MAX_BLOB_GAS_PER_BLOCK: + return err("blobGasUsed " & $blobGasUsed & " exceeds maximum allowance " & $MAX_BLOB_GAS_PER_BLOCK) - if headerDataGasUsed != dataGasUsed: - return err("calculated dataGas not equal header.dataGasUsed") + if headerBlobGasUsed != blobGasUsed: + return err("calculated blobGas not equal header.blobGasUsed") - if headerExcessDataGas != excessDataGas: - return err("calculated excessDataGas not equal header.excessDataGas") + if headerExcessBlobGas != excessBlobGas: + return err("calculated excessBlobGas not equal header.excessBlobGas") return ok() diff --git a/nimbus/core/executor/process_transaction.nim b/nimbus/core/executor/process_transaction.nim index 24b94b33e..5063012e9 100644 --- a/nimbus/core/executor/process_transaction.nim +++ b/nimbus/core/executor/process_transaction.nim @@ -78,7 +78,7 @@ proc asyncProcessTransactionImpl( baseFee = baseFee256.truncate(GasInt) tx = eip1559TxNormalization(tx, baseFee, fork) priorityFee = min(tx.maxPriorityFee, tx.maxFee - baseFee) - excessDataGas = vmState.parent.excessDataGas.get(0'u64) + excessBlobGas = vmState.parent.excessBlobGas.get(0'u64) # Return failure unless explicitely set `ok()` var res: Result[GasInt, string] = err("") @@ -100,7 +100,7 @@ proc asyncProcessTransactionImpl( # before leaving is crucial for some unit tests that us a direct/deep call # of the `processTransaction()` function. So there is no `return err()` # statement, here. - let txRes = roDB.validateTransaction(tx, sender, header.gasLimit, baseFee256, excessDataGas, fork) + let txRes = roDB.validateTransaction(tx, sender, header.gasLimit, baseFee256, excessBlobGas, fork) if txRes.isOk: # EIP-1153 diff --git a/nimbus/core/tx_pool.nim b/nimbus/core/tx_pool.nim index d11b0f379..bbf25f424 100644 --- a/nimbus/core/tx_pool.nim +++ b/nimbus/core/tx_pool.nim @@ -804,6 +804,14 @@ iterator okPairs*(xp: TxPoolRef): (Hash256, TxItemRef) = proc numTxs*(xp: TxPoolRef): int = xp.txDB.byItemID.len +proc disposeAll*(xp: TxpoolRef) {.gcsafe,raises: [CatchableError].} = + let numTx = xp.numTxs + var list = newSeqOfCap[TxItemRef](numTx) + for x in nextPairs(xp.txDB.byItemID): + list.add x.data + for x in list: + xp.disposeItems(x) + # ------------------------------------------------------------------------------ # Public functions, local/remote accounts # ------------------------------------------------------------------------------ diff --git a/nimbus/core/tx_pool/tx_chain.nim b/nimbus/core/tx_pool/tx_chain.nim index 566c37038..a464968ff 100644 --- a/nimbus/core/tx_pool/tx_chain.nim +++ b/nimbus/core/tx_pool/tx_chain.nim @@ -52,10 +52,10 @@ type profit: UInt256 ## Net reward (w/o PoW specific block rewards) txRoot: Hash256 ## `rootHash` after packing stateRoot: Hash256 ## `stateRoot` after packing - dataGasUsed: - Option[uint64] ## EIP-4844 block dataGasUsed - excessDataGas: - Option[uint64] ## EIP-4844 block excessDataGas + blobGasUsed: + Option[uint64] ## EIP-4844 block blobGasUsed + excessBlobGas: + Option[uint64] ## EIP-4844 block excessBlobGas TxChainRef* = ref object ##\ ## State cache of the transaction environment for creating a new\ @@ -144,8 +144,8 @@ proc resetTxEnv(dh: TxChainRef; parent: BlockHeader; fee: Option[UInt256]) dh.txEnv.txRoot = EMPTY_ROOT_HASH dh.txEnv.stateRoot = dh.txEnv.vmState.parent.stateRoot - dh.txEnv.dataGasUsed = none(uint64) - dh.txEnv.excessDataGas = none(uint64) + dh.txEnv.blobGasUsed = none(uint64) + dh.txEnv.excessBlobGas = none(uint64) proc update(dh: TxChainRef; parent: BlockHeader) {.gcsafe,raises: [CatchableError].} = @@ -224,8 +224,8 @@ proc getHeader*(dh: TxChainRef): BlockHeader # mixDigest: Hash256 # mining hash for given difficulty # nonce: BlockNonce # mining free vaiable fee: dh.txEnv.vmState.fee, - dataGasUsed: dh.txEnv.dataGasUsed, - excessDataGas: dh.txEnv.excessDataGas) + blobGasUsed: dh.txEnv.blobGasUsed, + excessBlobGas: dh.txEnv.excessBlobGas) if dh.com.forkGTE(Shanghai): result.withdrawalsRoot = some(calcWithdrawalsRoot(dh.withdrawals)) @@ -378,13 +378,13 @@ proc `txRoot=`*(dh: TxChainRef; val: Hash256) = proc `withdrawals=`*(dh: TxChainRef, val: sink seq[Withdrawal]) = dh.withdrawals = system.move(val) -proc `excessDataGas=`*(dh: TxChainRef; val: Option[uint64]) = +proc `excessBlobGas=`*(dh: TxChainRef; val: Option[uint64]) = ## Setter - dh.txEnv.excessDataGas = val + dh.txEnv.excessBlobGas = val -proc `dataGasUsed=`*(dh: TxChainRef; val: Option[uint64]) = +proc `blobGasUsed=`*(dh: TxChainRef; val: Option[uint64]) = ## Setter - dh.txEnv.dataGasUsed = val + dh.txEnv.blobGasUsed = val # ------------------------------------------------------------------------------ # End diff --git a/nimbus/core/tx_pool/tx_tasks/tx_classify.nim b/nimbus/core/tx_pool/tx_tasks/tx_classify.nim index bc4fc6c0a..c0a96cfbb 100644 --- a/nimbus/core/tx_pool/tx_tasks/tx_classify.nim +++ b/nimbus/core/tx_pool/tx_tasks/tx_classify.nim @@ -233,9 +233,9 @@ proc classifyValidatePacked*(xp: TxPoolRef; else: xp.chain.limits.trgLimit tx = item.tx.eip1559TxNormalization(xp.chain.baseFee.GasInt, fork) - excessDataGas = vmState.parent.excessDataGas.get(0'u64) + excessBlobGas = vmState.parent.excessBlobGas.get(0'u64) - roDB.validateTransaction(tx, item.sender, gasLimit, baseFee, excessDataGas, fork).isOk + roDB.validateTransaction(tx, item.sender, gasLimit, baseFee, excessBlobGas, fork).isOk proc classifyPacked*(xp: TxPoolRef; gasBurned, moreBurned: GasInt): bool = ## Classifier for *packing* (i.e. adding up `gasUsed` values after executing diff --git a/nimbus/core/tx_pool/tx_tasks/tx_packer.nim b/nimbus/core/tx_pool/tx_tasks/tx_packer.nim index d8ca5436d..42596db01 100644 --- a/nimbus/core/tx_pool/tx_tasks/tx_packer.nim +++ b/nimbus/core/tx_pool/tx_tasks/tx_packer.nim @@ -43,7 +43,7 @@ type tr: HexaryTrie cleanState: bool balance: UInt256 - dataGasUsed: uint64 + blobGasUsed: uint64 const receiptsExtensionSize = ##\ @@ -138,9 +138,9 @@ proc runTxCommit(pst: TxPackerStateRef; item: TxItemRef; gasBurned: GasInt) vmState.cumulativeGasUsed += gasBurned vmState.receipts[inx] = vmState.makeReceipt(item.tx.txType) - # EIP-4844, count dataGasUsed + # EIP-4844, count blobGasUsed if item.tx.txType >= TxEip4844: - pst.dataGasUsed += item.tx.getTotalDataGas + pst.blobGasUsed += item.tx.getTotalBlobGas # Update txRoot pst.tr.put(rlp.encode(inx), rlp.encode(item.tx.removeNetworkPayload)) @@ -251,9 +251,9 @@ proc vmExecCommit(pst: TxPackerStateRef) if vmState.com.forkGTE(Cancun): # EIP-4844 - let excessDataGas = calcExcessDataGas(vmState.parent) - xp.chain.excessDataGas = some(excessDataGas) - xp.chain.dataGasUsed = some(pst.dataGasUsed) + let excessBlobGas = calcExcessBlobGas(vmState.parent) + xp.chain.excessBlobGas = some(excessBlobGas) + xp.chain.blobGasUsed = some(pst.blobGasUsed) proc balanceDelta: UInt256 = let postBalance = vmState.readOnlyStateDB.getBalance(xp.chain.feeRecipient) diff --git a/nimbus/core/validate.nim b/nimbus/core/validate.nim index 69bdd87a2..c59c62d5f 100644 --- a/nimbus/core/validate.nim +++ b/nimbus/core/validate.nim @@ -234,7 +234,7 @@ proc validateUncles(com: CommonRef; header: BlockHeader; func gasCost(tx: Transaction): UInt256 = if tx.txType >= TxEip4844: - tx.gasLimit.u256 * tx.maxFee.u256 + tx.getTotalDataGas.u256 * tx.maxFeePerDataGas.u256 + tx.gasLimit.u256 * tx.maxFee.u256 + tx.getTotalBlobGas.u256 * tx.maxFeePerBlobGas.u256 elif tx.txType >= TxEip1559: tx.gasLimit.u256 * tx.maxFee.u256 else: @@ -246,7 +246,7 @@ proc validateTransaction*( sender: EthAddress; ## tx.getSender or tx.ecRecover maxLimit: GasInt; ## gasLimit from block header baseFee: UInt256; ## baseFee from block header - excessDataGas: uint64; ## excessDataGas from parent block header + excessBlobGas: uint64; ## excessBlobGas from parent block header fork: EVMFork): Result[void, string] = let @@ -359,10 +359,10 @@ proc validateTransaction*( "get=$1, expect=$2" % [$bv.data[0].int, $BLOB_COMMITMENT_VERSION_KZG.int]) # ensure that the user was willing to at least pay the current data gasprice - let dataGasPrice = getDataGasPrice(excessDataGas) - if tx.maxFeePerDataGas.uint64 < dataGasPrice: - return err("invalid tx: maxFeePerDataGas smaller than dataGasPrice. " & - "maxFeePerDataGas=$1, dataGasPrice=$2" % [$tx.maxFeePerDataGas, $dataGasPrice]) + let blobGasPrice = getBlobGasPrice(excessBlobGas) + if tx.maxFeePerBlobGas.uint64 < blobGasPrice: + return err("invalid tx: maxFeePerBlobGas smaller than blobGasPrice. " & + "maxFeePerBlobGas=$1, blobGasPrice=$2" % [$tx.maxFeePerBlobGas, $blobGasPrice]) except CatchableError as ex: return err(ex.msg) diff --git a/nimbus/db/core_db.nim b/nimbus/db/core_db.nim new file mode 100644 index 000000000..641eb409a --- /dev/null +++ b/nimbus/db/core_db.nim @@ -0,0 +1,124 @@ +# Nimbus +# Copyright (c) 2018 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +## Core database replacement wrapper object +## ======================================== +## +## See `core_db/README.md` +## +{.push raises: [].} + +import + chronicles, + eth/[common, trie/db], + ./core_db/[base, core_apps, legacy] + +export + common, + core_apps, + + # Not all symbols from the object sources will be exported by default + CoreDbCaptFlags, + CoreDbCaptRef, + CoreDbKvtObj, + CoreDbMptRef, + CoreDbPhkRef, + CoreDbRef, + CoreDbTxID, + CoreDbTxRef, + CoreDbType, + beginTransaction, + capture, + commit, + compensateLegacySetup, + contains, + dbType, + del, + dispose, + get, + getTransactionID, + isPruning, + kvt, + maybeGet, + mpt, + mptPrune, + pairs, + parent, + phk, + phkPrune, + put, + recorder, + replicate, + rollback, + rootHash, + safeDispose, + setTransactionID, + toMpt, + toPhk + +logScope: + topics = "core_db" + +# ------------------------------------------------------------------------------ +# Private functions: helpers +# ------------------------------------------------------------------------------ + +template logTxt(info: static[string]): static[string] = + "CoreDb " & info + +proc tmplNotImplemented*(db: CoreDbRef, name: string) {.used.} = + debug logTxt "template not implemented", dbType=db.dbType, meth=name + +# ------------------------------------------------------------------------------ +# Public constructor +# ------------------------------------------------------------------------------ + +proc newCoreDbRef*( + db: TrieDatabaseRef; + ): CoreDbRef + {.gcsafe, deprecated: "use newCoreDbRef(LegacyDbPersistent,)".} = + ## Legacy constructor. + ## + ## Note: Using legacy notation `newCoreDbRef()` rather than + ## `CoreDbRef.init()` because of compiler coughing. + db.newLegacyPersistentCoreDbRef() + +proc newCoreDbRef*(dbType: static[CoreDbType]): CoreDbRef = + ## Constructor for volatile/memory type DB + ## + ## Note: Using legacy notation `newCoreDbRef()` rather than + ## `CoreDbRef.init()` because of compiler coughing. + when dbType == LegacyDbMemory: + newLegacyMemoryCoreDbRef() + else: + {.error: "Unsupported dbType for memory newCoreDbRef()".} + +proc newCoreDbRef*(dbType: static[CoreDbType]; path: string): CoreDbRef = + ## Constructor for persistent type DB + ## + ## Note: Using legacy notation `newCoreDbRef()` rather than + ## `CoreDbRef.init()` because of compiler coughing. + when dbType == LegacyDbPersistent: + newLegacyPersistentCoreDbRef path + else: + {.error: "Unsupported dbType for persistent newCoreDbRef()".} + +# ------------------------------------------------------------------------------ +# Public template wrappers +# ------------------------------------------------------------------------------ + +template shortTimeReadOnly*(id: CoreDbTxID; body: untyped) = + proc action() {.gcsafe, raises: [CatchableError].} = + body + id.shortTimeReadOnly action + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/.gitignore b/nimbus/db/core_db/.gitignore new file mode 100644 index 000000000..0b84df0f0 --- /dev/null +++ b/nimbus/db/core_db/.gitignore @@ -0,0 +1 @@ +*.html \ No newline at end of file diff --git a/nimbus/db/core_db/README.md b/nimbus/db/core_db/README.md new file mode 100644 index 000000000..a6c4758c1 --- /dev/null +++ b/nimbus/db/core_db/README.md @@ -0,0 +1,79 @@ +Core database replacement wrapper object +======================================== +This wrapper replaces the *TrieDatabaseRef* and its derivatives by the new +object *CoreDbRef*. + +Relations to current *TrieDatabaseRef* implementation +----------------------------------------------------- +Here are some incomplete translations for objects and constructors. + +### Object types: + +| **Legacy notation** | **CoreDbRef based replacement** | +|:----------------------------|:--------------------------------------| +| | | +| ChainDB | (don't use/avoid) | +| ChainDbRef | CoreDbRef | +| TrieDatabaseRef | CoreDbKvtRef | +| HexaryTrie | CoreDbMptRef | +| SecureHexaryTrie | CoreDbPhkRef | +| DbTransaction | CoreDbTxRef | +| TransactionID | CoreDbTxID | + + +### Constructors: + +| **Legacy notation** | **CoreDbRef based replacement** | +|:----------------------------|:--------------------------------------| +| | | +| trieDB newChainDB("..") | newCoreDbRef(LegacyDbPersistent,"..") | +| newMemoryDB() | newCoreDbRef(LegacyDbMemory) | +| -- | | +| initHexaryTrie(db,..) | db.mpt(..) (no pruning) | +| | db.mptPrune(..) (w/pruning true/false)| +| -- | | +| initSecureHexaryTrie(db,..) | db.phk(..) (no pruning) | +| | db.phkPrune(..) (w/pruning true/false)| +| -- | | +| newCaptureDB(db,memDB) | db.capture() (see below) | + + +Usage of the replacement wrapper +-------------------------------- + +### Objects pedigree: + + CoreDbRef -- base descriptor + | | | | + | | | +-- CoreDbMptRef -- hexary trie instance + | | | | : : + | | | +-- CoreDbMptRef -- hexary trie instance + | | | + | | | + | | +---- CoreDbPhkRef -- pre-hashed key hexary trie instance + | | | : : + | | +---- CoreDbPhkRef -- pre-hashed key hexary trie instance + | | + | | + | +------ CoreDbKvtRef -- single static key-value table + | + | + +-------- CoreDbCaptRef -- tracer support descriptor + +### Instantiating standard database object descriptors works as follows: + + let + db = newCoreDbRef(..) # new base descriptor + mpt = db.mpt(..) # hexary trie/Merkle Patricia Tree + phk = db.phk(..) # pre-hashed key hexary trie/MPT + kvt = db.kvt # key-value table + +### Tracer support setup by hiding the current *CoreDbRef* behind a replacement: + + let + capture = db.capure() + db = capture.recorder # use the recorder in place of db + ... + + for key,value in capture.recorder.kvt: + ... # process recorded data diff --git a/nimbus/db/core_db/base.nim b/nimbus/db/core_db/base.nim new file mode 100644 index 000000000..5d0c0afce --- /dev/null +++ b/nimbus/db/core_db/base.nim @@ -0,0 +1,630 @@ +# Nimbus +# Copyright (c) 2018 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +{.push raises: [].} + +import + std/options, + chronicles, + eth/common, + ../../constants + +logScope: + topics = "core_db-base" + +when defined(release): + const AutoValidateDescriptors = false +else: + const AutoValidateDescriptors = true + +type + CoreDbCaptFlags* {.pure.} = enum + PersistPut + PersistDel + + CoreDbType* = enum + Ooops + LegacyDbMemory + LegacyDbPersistent + # AristoDbMemory + # AristoDbPersistent + + # -------------------------------------------------- + # Constructors + # -------------------------------------------------- + CoreDbNewMptFn* = proc(root: Hash256): CoreDbMptRef {.gcsafe, raises: [].} + CoreDbNewLegacyMptFn* = proc(root: Hash256; prune: bool): CoreDbMptRef + {.gcsafe, raises: [].} + CoreDbNewTxGetIdFn* = proc(): CoreDbTxID {.gcsafe, raises: [].} + CoreDbNewTxBeginFn* = proc(): CoreDbTxRef {.gcsafe, raises: [].} + CoreDbNewCaptFn = proc(flags: set[CoreDbCaptFlags] = {}): CoreDbCaptRef + {.gcsafe, raises: [].} + + CoreDbConstructors* = object + ## Constructors + + # Hexary trie + mptFn*: CoreDbNewMptFn + legacyMptFn*: CoreDbNewLegacyMptFn # Legacy handler, should go away + + # Transactions + getIdFn*: CoreDbNewTxGetIdFn + beginFn*: CoreDbNewTxBeginFn + + # capture/tracer + captureFn*: CoreDbNewCaptFn + + + # -------------------------------------------------- + # Sub-descriptor: Misc methods for main descriptor + # -------------------------------------------------- + CoreDbInitLegaSetupFn* = proc() {.gcsafe, raises: [].} + + CoreDbMiscFns* = object + legacySetupFn*: CoreDbInitLegaSetupFn + + # -------------------------------------------------- + # Sub-descriptor: KVT methods + # -------------------------------------------------- + CoreDbKvtGetFn* = proc(k: openArray[byte]): Blob {.gcsafe, raises: [].} + CoreDbKvtMaybeGetFn* = proc(key: openArray[byte]): Option[Blob] + {.gcsafe, raises: [].} + CoreDbKvtDelFn* = proc(k: openArray[byte]) {.gcsafe, raises: [].} + CoreDbKvtPutFn* = proc(k: openArray[byte]; v: openArray[byte]) + {.gcsafe, raises: [].} + CoreDbKvtContainsFn* = proc(k: openArray[byte]): bool {.gcsafe, raises: [].} + CoreDbKvtPairsIt* = iterator(): (Blob,Blob) {.gcsafe, raises: [].} + + CoreDbKvtFns* = object + ## Methods for key-value table + getFn*: CoreDbKvtGetFn + maybeGetFn*: CoreDbKvtMaybeGetFn + delFn*: CoreDbKvtDelFn + putFn*: CoreDbKvtPutFn + containsFn*: CoreDbKvtContainsFn + pairsIt*: CoreDbKvtPairsIt + + + # -------------------------------------------------- + # Sub-descriptor: Mpt/hexary trie methods + # -------------------------------------------------- + CoreDbMptGetFn* = proc(k: openArray[byte]): Blob + {.gcsafe, raises: [RlpError].} + CoreDbMptMaybeGetFn* = proc(k: openArray[byte]): Option[Blob] + {.gcsafe, raises: [RlpError].} + CoreDbMptDelFn* = proc(k: openArray[byte]) {.gcsafe, raises: [RlpError].} + CoreDbMptPutFn* = proc(k: openArray[byte]; v: openArray[byte]) + {.gcsafe, raises: [RlpError].} + CoreDbMptContainsFn* = proc(k: openArray[byte]): bool + {.gcsafe, raises: [RlpError].} + CoreDbMptRootHashFn* = proc(): Hash256 {.gcsafe, raises: [].} + CoreDbMptIsPruningFn* = proc(): bool {.gcsafe, raises: [].} + CoreDbMptPairsIt* = iterator(): (Blob,Blob) {.gcsafe, raises: [RlpError].} + CoreDbMptReplicateIt* = iterator(): (Blob,Blob) {.gcsafe, raises: [RlpError].} + + CoreDbMptFns* = object + ## Methods for trie objects `CoreDbMptRef` + getFn*: CoreDbMptGetFn + maybeGetFn*: CoreDbMptMaybeGetFn + delFn*: CoreDbMptDelFn + putFn*: CoreDbMptPutFn + containsFn*: CoreDbMptContainsFn + rootHashFn*: CoreDbMptRootHashFn + pairsIt*: CoreDbMptPairsIt + replicateIt*: CoreDbMptReplicateIt + isPruningFn*: CoreDbMptIsPruningFn # Legacy handler, should go away + + + # -------------------------------------------------- + # Sub-descriptor: Transaction frame management + # -------------------------------------------------- + CoreDbTxCommitFn* = proc(applyDeletes: bool) {.gcsafe, raises: [].} + CoreDbTxRollbackFn* = proc() {.gcsafe, raises: [].} + CoreDbTxDisposeFn* = proc() {.gcsafe, raises: [].} + CoreDbTxSafeDisposeFn* = proc() {.gcsafe, raises: [].} + + CoreDbTxFns* = object + commitFn*: CoreDbTxCommitFn + rollbackFn*: CoreDbTxRollbackFn + disposeFn*: CoreDbTxDisposeFn + safeDisposeFn*: CoreDbTxSafeDisposeFn + + # -------------------------------------------------- + # Sub-descriptor: Transaction ID management + # -------------------------------------------------- + CoreDbTxIdSetIdFn* = proc() {.gcsafe, raises: [].} + CoreDbTxIdActionFn* = proc() {.gcsafe, raises: [CatchableError].} + CoreDbTxIdRoWrapperFn* = proc(action: CoreDbTxIdActionFn) + {.gcsafe, raises: [CatchableError].} + CoreDbTxIdFns* = object + setIdFn*: CoreDbTxIdSetIdFn + roWrapperFn*: CoreDbTxIdRoWrapperFn + + + # -------------------------------------------------- + # Sub-descriptor: capture recorder methods + # -------------------------------------------------- + CoreDbCaptRecorderFn* = proc(): CoreDbRef {.gcsafe, raises: [].} + CoreDbCaptFlagsFn* = proc(): set[CoreDbCaptFlags] {.gcsafe, raises: [].} + + CoreDbCaptFns* = object + recorderFn*: CoreDbCaptRecorderFn + getFlagsFn*: CoreDbCaptFlagsFn + + # -------------------------------------------------- + # Production descriptors + # -------------------------------------------------- + + CoreDbRef* = ref object of RootRef + ## Database descriptor + kvt: CoreDbKvtObj + new: CoreDbConstructors + methods: CoreDbMiscFns + + CoreDbKvtObj* = object + ## Statically initialised Key-Value pair table living in `CoreDbRef` + dbType: CoreDbType + methods: CoreDbKvtFns + + CoreDbMptRef* = ref object + ## Hexary/Merkle-Patricia tree derived from `CoreDbRef`, will be + ## initialised on-the-fly. + parent: CoreDbRef + methods: CoreDbMptFns + + CoreDbPhkRef* = ref object + ## Similar to `CoreDbMptRef` but with pre-hashed keys. That is, any + ## argument key for `put()`, `get()` etc. will be hashed first before + ## being applied. + parent: CoreDbMptRef + methods: CoreDbMptFns + + CoreDbTxRef* = ref object + ## Transaction descriptor derived from `CoreDbRef` + parent: CoreDbRef + methods: CoreDbTxFns + + CoreDbTxID* = ref object + ## Transaction ID descriptor derived from `CoreDbRef` + parent: CoreDbRef + methods: CoreDbTxIdFns + + CoreDbCaptRef* = ref object + ## Db transaction tracer derived from `CoreDbRef` + parent: CoreDbRef + methods: CoreDbCaptFns + + MethodsDesc = + CoreDbKvtObj | + CoreDbMptRef | CoreDbPhkRef | + CoreDbTxRef | CoreDbTxID | + CoreDbCaptRef + +# ------------------------------------------------------------------------------ +# Private functions: helpers +# ------------------------------------------------------------------------------ + +template logTxt(info: static[string]): static[string] = + "CoreDb " & info + +template itNotImplemented(db: CoreDbRef, name: string) = + warn logTxt "iterator not implemented", dbType=db.kvt.dbType, meth=name + +# --------- + +proc validateMethodsDesc(db: CoreDbRef) = + doAssert not db.methods.legacySetupFn.isNil + +proc validateMethodsDesc(kvt: CoreDbKvtObj) = + doAssert kvt.dbType != CoreDbType(0) + doAssert not kvt.methods.getFn.isNil + doAssert not kvt.methods.maybeGetFn.isNil + doAssert not kvt.methods.delFn.isNil + doAssert not kvt.methods.putFn.isNil + doAssert not kvt.methods.containsFn.isNil + doAssert not kvt.methods.pairsIt.isNil + +proc validateMethodsDesc(trie: CoreDbMptRef|CoreDbPhkRef) = + doAssert not trie.parent.isNil + doAssert not trie.methods.getFn.isNil + doAssert not trie.methods.maybeGetFn.isNil + doAssert not trie.methods.delFn.isNil + doAssert not trie.methods.putFn.isNil + doAssert not trie.methods.containsFn.isNil + doAssert not trie.methods.rootHashFn.isNil + doAssert not trie.methods.isPruningFn.isNil + doAssert not trie.methods.pairsIt.isNil + doAssert not trie.methods.replicateIt.isNil + +proc validateMethodsDesc(cpt: CoreDbCaptRef) = + doAssert not cpt.parent.isNil + doAssert not cpt.methods.recorderFn.isNil + doAssert not cpt.methods.getFlagsFn.isNil + +proc validateMethodsDesc(tx: CoreDbTxRef) = + doAssert not tx.parent.isNil + doAssert not tx.methods.commitFn.isNil + doAssert not tx.methods.rollbackFn.isNil + doAssert not tx.methods.disposeFn.isNil + doAssert not tx.methods.safeDisposeFn.isNil + +proc validateMethodsDesc(id: CoreDbTxID) = + doAssert not id.parent.isNil + doAssert not id.methods.setIdFn.isNil + doAssert not id.methods.roWrapperFn.isNil + +proc validateConstructors(new: CoreDbConstructors) = + doAssert not new.mptFn.isNil + doAssert not new.legacyMptFn.isNil + doAssert not new.getIdFn.isNil + doAssert not new.beginFn.isNil + doAssert not new.captureFn.isNil + +# --------- + +proc toCoreDbPhkRef(mpt: CoreDbMptRef): CoreDbPhkRef = + ## MPT => pre-hashed MPT (aka PHK) + result = CoreDbPhkRef( + parent: mpt, + methods: CoreDbMptFns( + getFn: proc(k: openArray[byte]): Blob + {.gcsafe, raises: [RlpError].} = + return mpt.methods.getFn(k.keccakHash.data), + + maybeGetFn: proc(k: openArray[byte]): Option[Blob] + {.gcsafe, raises: [RlpError].} = + return mpt.methods.maybeGetFn(k.keccakHash.data), + + delFn: proc(k: openArray[byte]) + {.gcsafe, raises: [RlpError].} = + mpt.methods.delFn(k.keccakHash.data), + + putFn: proc(k:openArray[byte]; v:openArray[byte]) + {.gcsafe, raises: [RlpError].} = + mpt.methods.putFn(k.keccakHash.data, v), + + containsFn: proc(k: openArray[byte]): bool + {.gcsafe, raises: [RlpError].} = + return mpt.methods.containsFn(k.keccakHash.data), + + pairsIt: iterator(): (Blob, Blob) {.gcsafe.} = + mpt.parent.itNotImplemented("pairs/phk"), + + replicateIt: iterator(): (Blob, Blob) {.gcsafe.} = + mpt.parent.itNotImplemented("replicate/phk"), + + rootHashFn: mpt.methods.rootHashFn, + isPruningFn: mpt.methods.isPruningFn)) + + when AutoValidateDescriptors: + result.validateMethodsDesc + + +proc kvtUpdate(db: CoreDbRef) = + ## Disable interator for non-memory instances + case db.kvt.dbType + of LegacyDbMemory: + discard + else: + db.kvt.methods.pairsIt = iterator(): (Blob, Blob) = + db.itNotImplemented "pairs/kvt" + +# ------------------------------------------------------------------------------ +# Public debugging helpers +# ------------------------------------------------------------------------------ + +proc validate*(desc: MethodsDesc) = + desc.validateMethodsDesc + +proc validate*(db: CoreDbRef) = + db.validateMethodsDesc + db.kvt.validateMethodsDesc + db.new.validateConstructors + +# ------------------------------------------------------------------------------ +# Public constructor +# ------------------------------------------------------------------------------ + +proc init*( + db: CoreDbRef; + dbType: CoreDbType; + dbMethods: CoreDbMiscFns; + kvtMethods: CoreDbKvtFns; + new: CoreDbConstructors + ) = + ## Base descriptor initaliser + db.methods = dbMethods + db.new = new + + db.kvt.dbType = dbType + db.kvt.methods = kvtMethods + db.kvtUpdate() + + when AutoValidateDescriptors: + db.validate + + +proc newCoreDbMptRef*(db: CoreDbRef; methods: CoreDbMptFns): CoreDbMptRef = + ## Hexary trie constructor helper. Will be needed for the + ## sub-constructors defined in `CoreDbMptConstructor`. + result = CoreDbMptRef( + parent: db, + methods: methods) + + when AutoValidateDescriptors: + result.validate + + +proc newCoreDbTxRef*(db: CoreDbRef; methods: CoreDbTxFns): CoreDbTxRef = + ## Transaction frame constructor helper. Will be needed for the + ## sub-constructors defined in `CoreDbTxConstructor`. + result = CoreDbTxRef( + parent: db, + methods: methods) + + when AutoValidateDescriptors: + result.validate + + +proc newCoreDbTxID*(db: CoreDbRef; methods: CoreDbTxIdFns): CoreDbTxID = + ## Transaction ID constructor helper. + result = CoreDbTxID( + parent: db, + methods: methods) + + when AutoValidateDescriptors: + result.validate + + +proc newCoreDbCaptRef*(db: CoreDbRef; methods: CoreDbCaptFns): CoreDbCaptRef = + ## Capture constructor helper. + result = CoreDbCaptRef( + parent: db, + methods: methods) + + when AutoValidateDescriptors: + db.validate + +# ------------------------------------------------------------------------------ +# Public main descriptor methods +# ------------------------------------------------------------------------------ + +proc dbType*(db: CoreDbRef): CoreDbType = + ## Getter + db.kvt.dbType + +# On the persistent legacy hexary trie, this function is needed for +# bootstrapping and Genesis setup when the `purge` flag is activated. +proc compensateLegacySetup*(db: CoreDbRef) = + db.methods.legacySetupFn() + +# ------------------------------------------------------------------------------ +# Public key-value table methods +# ------------------------------------------------------------------------------ + +proc kvt*(db: CoreDbRef): CoreDbKvtObj = + ## Getter (pseudo constructor) + db.kvt + +proc dbType*(db: CoreDbKvtObj): CoreDbType = + ## Getter + db.dbType + +proc get*(db: CoreDbKvtObj; key: openArray[byte]): Blob = + db.methods.getFn key + +proc maybeGet*(db: CoreDbKvtObj; key: openArray[byte]): Option[Blob] = + db.methods.maybeGetFn key + +proc del*(db: CoreDbKvtObj; key: openArray[byte]) = + db.methods.delFn key + +proc put*(db: CoreDbKvtObj; key: openArray[byte]; value: openArray[byte]) = + db.methods.putFn(key, value) + +proc contains*(db: CoreDbKvtObj; key: openArray[byte]): bool = + db.methods.containsFn key + +iterator pairs*(db: CoreDbKvtObj): (Blob, Blob) = + ## Iterator supported on memory DB (otherwise implementation dependent) + for k,v in db.methods.pairsIt(): + yield (k,v) + +# ------------------------------------------------------------------------------ +# Public Merkle Patricia Tree, hexary trie constructors +# ------------------------------------------------------------------------------ + +proc mpt*(db: CoreDbRef; root=EMPTY_ROOT_HASH): CoreDbMptRef = + ## Constructor + db.new.mptFn root + +proc mptPrune*(db: CoreDbRef; root=EMPTY_ROOT_HASH): CoreDbMptRef = + ## Constructor + db.new.legacyMptFn(root, true) + +proc mptPrune*(db: CoreDbRef; root: Hash256; prune: bool): CoreDbMptRef = + ## Constructor + db.new.legacyMptFn(root, prune) + +proc mptPrune*(db: CoreDbRef; prune: bool): CoreDbMptRef = + ## Constructor + db.new.legacyMptFn(EMPTY_ROOT_HASH, prune) + +# ------------------------------------------------------------------------------ +# Public pre-hashed key hexary trie constructors +# ------------------------------------------------------------------------------ + +proc phk*(db: CoreDbRef; root=EMPTY_ROOT_HASH): CoreDbPhkRef = + ## Constructor + db.new.mptFn(root).toCoreDbPhkRef + +proc phkPrune*(db: CoreDbRef; root=EMPTY_ROOT_HASH): CoreDbPhkRef = + ## Constructor + db.new.legacyMptFn(root, true).toCoreDbPhkRef + +proc phkPrune*(db: CoreDbRef; root: Hash256; prune: bool): CoreDbPhkRef = + ## Constructor + db.new.legacyMptFn(root, prune).toCoreDbPhkRef + +proc phkPrune*(db: CoreDbRef; prune: bool): CoreDbPhkRef = + ## Constructor + db.new.legacyMptFn(EMPTY_ROOT_HASH, prune).toCoreDbPhkRef + +# ------------------------------------------------------------------------------ +# Public hexary trie switch methods +# ------------------------------------------------------------------------------ + +proc toPhk*(mpt: CoreDbMptRef): CoreDbPhkRef = + ## Getter + mpt.toCoreDbPhkRef + +proc toMpt*(trie: CoreDbPhkRef): CoreDbMptRef = + ## Getter + trie.parent + +# ------------------------------------------------------------------------------ +# Public hexary trie database methods (`mpt` or `phk`) +# ------------------------------------------------------------------------------ + +proc parent*(mpt: CoreDbMptRef): CoreDbRef = + ## Getter + mpt.parent + +proc parent*(trie: CoreDbPhkRef): CoreDbRef = + ## Getter + trie.parent.parent + +proc isPruning*(trie: CoreDbMptRef|CoreDbPhkRef): bool = + ## Getter + trie.methods.isPruningFn() + +proc get*( + trie: CoreDbMptRef|CoreDbPhkRef; + key: openArray[byte]; + ): Blob + {.gcsafe, raises: [RlpError].} = + trie.methods.getFn key + +proc maybeGet*( + trie: CoreDbMptRef|CoreDbPhkRef; + key: openArray[byte]; + ): Option[Blob] + {.gcsafe, raises: [RlpError].} = + trie.methods.maybeGetFn key + +proc del*( + trie: CoreDbMptRef|CoreDbPhkRef; + key: openArray[byte]; + ) {.gcsafe, raises: [RlpError].} = + trie.methods.delFn key + +proc put*( + trie: CoreDbMptRef|CoreDbPhkRef; + key: openArray[byte]; + value: openArray[byte]; + ) {.gcsafe, raises: [RlpError].} = + trie.methods.putFn(key, value) + +proc contains*( + trie: CoreDbMptRef|CoreDbPhkRef; + key: openArray[byte]; + ): bool + {.gcsafe, raises: [RlpError].} = + trie.methods.containsFn key + +proc rootHash*( + trie: CoreDbMptRef|CoreDbPhkRef; + ): Hash256 + {.gcsafe.} = + trie.methods.rootHashFn() + +iterator pairs*( + trie: CoreDbMptRef; + ): (Blob, Blob) + {.gcsafe, raises: [RlpError].} = + ## Trie traversal, only supported for `CoreDbMptRef` + for k,v in trie.methods.pairsIt(): + yield (k,v) + +iterator replicate*( + trie: CoreDbMptRef; + ): (Blob, Blob) + {.gcsafe, raises: [RlpError].} = + ## Low level trie dump, only supported for `CoreDbMptRef` + for k,v in trie.methods.replicateIt(): + yield (k,v) + +# ------------------------------------------------------------------------------ +# Public transaction related methods +# ------------------------------------------------------------------------------ + +proc getTransactionID*(db: CoreDbRef): CoreDbTxID = + ## Getter, current transaction state + db.new.getIdFn() + +proc parent*(id: CoreDbTxID): CoreDbRef = + ## Getter + id.parent + +proc setTransactionID*(id: CoreDbTxID) = + ## Setter, revert to some earlier transaction state + id.methods.setIdFn() + +proc shortTimeReadOnly*( + id: CoreDbTxID; + action: proc() {.gcsafe, raises: [CatchableError].}; + ) {.gcsafe, raises: [CatchableError].} = + ## Run `action()` in an earlier transaction environment. + id.methods.roWrapperFn action + + +proc beginTransaction*(db: CoreDbRef): CoreDbTxRef = + ## Constructor + db.new.beginFn() + +proc parent*(db: CoreDbTxRef): CoreDbRef = + ## Getter + db.parent + +proc commit*(tx: CoreDbTxRef, applyDeletes = true) = + tx.methods.commitFn applyDeletes + +proc rollback*(tx: CoreDbTxRef) = + tx.methods.rollbackFn() + +proc dispose*(tx: CoreDbTxRef) = + tx.methods.disposeFn() + +proc safeDispose*(tx: CoreDbTxRef) = + tx.methods.safeDisposeFn() + +# ------------------------------------------------------------------------------ +# Public tracer methods +# ------------------------------------------------------------------------------ + +proc capture*(db: CoreDbRef; flags: set[CoreDbCaptFlags] = {}): CoreDbCaptRef = + ## Constructor + db.new.captureFn flags + +proc parent*(db: CoreDbCaptRef): CoreDbRef = + ## Getter + db.parent + +proc recorder*(db: CoreDbCaptRef): CoreDbRef = + ## Getter + db.methods.recorderFn() + +proc flags*(db: CoreDbCaptRef): set[CoreDbCaptFlags] = + ## Getter + db.methods.getFlagsFn() + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/core_apps.nim b/nimbus/db/core_db/core_apps.nim new file mode 100644 index 000000000..9f987057b --- /dev/null +++ b/nimbus/db/core_db/core_apps.nim @@ -0,0 +1,737 @@ +# Nimbus +# Copyright (c) 2018 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +{.push raises: [].} + +import + std/[algorithm, options, sequtils], + chronicles, + eth/[common, rlp], + stew/byteutils, + "../.."/[errors, constants], + ../storage_types, + "."/base + +logScope: + topics = "core_db-apps" + +type + TransactionKey = tuple + blockNumber: BlockNumber + index: int + +# ------------------------------------------------------------------------------ +# Forward declarations +# ------------------------------------------------------------------------------ + +proc getBlockHeader*( + db: CoreDbRef; + n: BlockNumber; + output: var BlockHeader; + ): bool + {.gcsafe, raises: [RlpError].} + +proc getBlockHeader*( + db: CoreDbRef, + blockHash: Hash256; + ): BlockHeader + {.gcsafe, raises: [BlockNotFound].} + +proc getBlockHash*( + db: CoreDbRef; + n: BlockNumber; + output: var Hash256; + ): bool + {.gcsafe, raises: [RlpError].} + +proc addBlockNumberToHashLookup*( + db: CoreDbRef; + header: BlockHeader; + ) {.gcsafe.} + +proc getBlockHeader*( + db: CoreDbRef; + blockHash: Hash256; + output: var BlockHeader; + ): bool + {.gcsafe.} + +# Copied from `utils/utils` which cannot be imported here in order to +# avoid circular imports. +func hash(b: BlockHeader): Hash256 + +# ------------------------------------------------------------------------------ +# Private iterators +# ------------------------------------------------------------------------------ + +iterator findNewAncestors( + db: CoreDbRef; + header: BlockHeader; + ): BlockHeader + {.gcsafe, raises: [RlpError,BlockNotFound].} = + ## Returns the chain leading up from the given header until the first + ## ancestor it has in common with our canonical chain. + var h = header + var orig: BlockHeader + while true: + if db.getBlockHeader(h.blockNumber, orig) and orig.hash == h.hash: + break + + yield h + + if h.parentHash == GENESIS_PARENT_HASH: + break + else: + h = db.getBlockHeader(h.parentHash) + +# ------------------------------------------------------------------------------ +# Public iterators +# ------------------------------------------------------------------------------ + +iterator getBlockTransactionData*( + db: CoreDbRef; + transactionRoot: Hash256; + ): seq[byte] + {.gcsafe, raises: [RlpError].} = + var transactionDb = db.mptPrune transactionRoot + var transactionIdx = 0 + while true: + let transactionKey = rlp.encode(transactionIdx) + if transactionKey in transactionDb: + yield transactionDb.get(transactionKey) + else: + break + inc transactionIdx + +iterator getBlockTransactions*( + db: CoreDbRef; + header: BlockHeader; + ): Transaction + {.gcsafe, raises: [RlpError].} = + for encodedTx in db.getBlockTransactionData(header.txRoot): + yield rlp.decode(encodedTx, Transaction) + +iterator getBlockTransactionHashes*( + db: CoreDbRef; + blockHeader: BlockHeader; + ): Hash256 + {.gcsafe, raises: [RlpError].} = + ## Returns an iterable of the transaction hashes from th block specified + ## by the given block header. + for encodedTx in db.getBlockTransactionData(blockHeader.txRoot): + let tx = rlp.decode(encodedTx, Transaction) + yield rlpHash(tx) # beware EIP-4844 + +iterator getWithdrawalsData*( + db: CoreDbRef; + withdrawalsRoot: Hash256; + ): seq[byte] + {.gcsafe, raises: [RlpError].} = + var wddb = db.mptPrune withdrawalsRoot + var idx = 0 + while true: + let wdKey = rlp.encode(idx) + if wdKey in wddb: + yield wddb.get(wdKey) + else: + break + inc idx + +iterator getReceipts*( + db: CoreDbRef; + receiptRoot: Hash256; + ): Receipt + {.gcsafe, raises: [RlpError].} = + var receiptDb = db.mptPrune receiptRoot + var receiptIdx = 0 + while true: + let receiptKey = rlp.encode(receiptIdx) + if receiptKey in receiptDb: + let receiptData = receiptDb.get(receiptKey) + yield rlp.decode(receiptData, Receipt) + else: + break + inc receiptIdx + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +func hash(b: BlockHeader): Hash256 = + rlpHash(b) + +proc removeTransactionFromCanonicalChain( + db: CoreDbRef; + transactionHash: Hash256; + ) = + ## Removes the transaction specified by the given hash from the canonical + ## chain. + db.kvt.del(transactionHashToBlockKey(transactionHash).toOpenArray) + +proc setAsCanonicalChainHead( + db: CoreDbRef; + headerHash: Hash256; + ): seq[BlockHeader] + {.gcsafe, raises: [RlpError,BlockNotFound].} = + ## Sets the header as the canonical chain HEAD. + let header = db.getBlockHeader(headerHash) + + var newCanonicalHeaders = sequtils.toSeq(db.findNewAncestors(header)) + reverse(newCanonicalHeaders) + for h in newCanonicalHeaders: + var oldHash: Hash256 + if not db.getBlockHash(h.blockNumber, oldHash): + break + + let oldHeader = db.getBlockHeader(oldHash) + for txHash in db.getBlockTransactionHashes(oldHeader): + db.removeTransactionFromCanonicalChain(txHash) + # TODO re-add txn to internal pending pool (only if local sender) + + for h in newCanonicalHeaders: + db.addBlockNumberToHashLookup(h) + + db.kvt.put(canonicalHeadHashKey().toOpenArray, rlp.encode(headerHash)) + + return newCanonicalHeaders + +proc markCanonicalChain( + db: CoreDbRef; + header: BlockHeader; + headerHash: Hash256; + ): bool + {.gcsafe, raises: [RlpError].} = + ## mark this chain as canonical by adding block number to hash lookup + ## down to forking point + var + currHash = headerHash + currHeader = header + + # mark current header as canonical + let key = blockNumberToHashKey(currHeader.blockNumber) + db.kvt.put(key.toOpenArray, rlp.encode(currHash)) + + # it is a genesis block, done + if currHeader.parentHash == Hash256(): + return true + + # mark ancestor blocks as canonical too + currHash = currHeader.parentHash + if not db.getBlockHeader(currHeader.parentHash, currHeader): + return false + + while currHash != Hash256(): + let key = blockNumberToHashKey(currHeader.blockNumber) + let data = db.kvt.get(key.toOpenArray) + if data.len == 0: + # not marked, mark it + db.kvt.put(key.toOpenArray, rlp.encode(currHash)) + elif rlp.decode(data, Hash256) != currHash: + # replace prev chain + db.kvt.put(key.toOpenArray, rlp.encode(currHash)) + else: + # forking point, done + break + + if currHeader.parentHash == Hash256(): + break + + currHash = currHeader.parentHash + if not db.getBlockHeader(currHeader.parentHash, currHeader): + return false + + return true + + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc exists*(db: CoreDbRef, hash: Hash256): bool = + db.kvt.contains(hash.data) + +proc getBlockHeader*( + db: CoreDbRef; + blockHash: Hash256; + output: var BlockHeader; + ): bool = + let data = db.kvt.get(genericHashKey(blockHash).toOpenArray) + if data.len != 0: + try: + output = rlp.decode(data, BlockHeader) + true + except RlpError: + false + else: + false + +proc getBlockHeader*( + db: CoreDbRef, + blockHash: Hash256; + ): BlockHeader = + ## Returns the requested block header as specified by block hash. + ## + ## Raises BlockNotFound if it is not present in the db. + if not db.getBlockHeader(blockHash, result): + raise newException( + BlockNotFound, "No block with hash " & blockHash.data.toHex) + +proc getHash( + db: CoreDbRef; + key: DbKey; + output: var Hash256; + ): bool + {.gcsafe, raises: [RlpError].} = + let data = db.kvt.get(key.toOpenArray) + if data.len != 0: + output = rlp.decode(data, Hash256) + result = true + +proc getCanonicalHead*( + db: CoreDbRef; + ): BlockHeader + {.gcsafe, raises: [RlpError,EVMError].} = + var headHash: Hash256 + if not db.getHash(canonicalHeadHashKey(), headHash) or + not db.getBlockHeader(headHash, result): + raise newException( + CanonicalHeadNotFound, "No canonical head set for this chain") + +proc getCanonicalHeaderHash*( + db: CoreDbRef; + ): Hash256 + {.gcsafe, raises: [RlpError].}= + discard db.getHash(canonicalHeadHashKey(), result) + +proc getBlockHash*( + db: CoreDbRef; + n: BlockNumber; + output: var Hash256; + ): bool = + ## Return the block hash for the given block number. + db.getHash(blockNumberToHashKey(n), output) + +proc getBlockHash*( + db: CoreDbRef; + n: BlockNumber; + ): Hash256 + {.gcsafe, raises: [RlpError,BlockNotFound].} = + ## Return the block hash for the given block number. + if not db.getHash(blockNumberToHashKey(n), result): + raise newException(BlockNotFound, "No block hash for number " & $n) + +proc getHeadBlockHash*( + db: CoreDbRef; + ): Hash256 + {.gcsafe, raises: [RlpError].} = + if not db.getHash(canonicalHeadHashKey(), result): + result = Hash256() + +proc getBlockHeader*( + db: CoreDbRef; + n: BlockNumber; + output: var BlockHeader; + ): bool = + ## Returns the block header with the given number in the canonical chain. + var blockHash: Hash256 + if db.getBlockHash(n, blockHash): + result = db.getBlockHeader(blockHash, output) + +proc getBlockHeaderWithHash*( + db: CoreDbRef; + n: BlockNumber; + ): Option[(BlockHeader, Hash256)] + {.gcsafe, raises: [RlpError].} = + ## Returns the block header and its hash, with the given number in the canonical chain. + ## Hash is returned to avoid recomputing it + var hash: Hash256 + if db.getBlockHash(n, hash): + # Note: this will throw if header is not present. + var header: BlockHeader + if db.getBlockHeader(hash, header): + return some((header, hash)) + else: + # this should not happen, but if it happen lets fail laudly as this means + # something is super wrong + raiseAssert("Corrupted database. Mapping number->hash present, without header in database") + else: + return none[(BlockHeader, Hash256)]() + +proc getBlockHeader*( + db: CoreDbRef; + n: BlockNumber; + ): BlockHeader + {.gcsafe, raises: [RlpError,BlockNotFound].} = + ## Returns the block header with the given number in the canonical chain. + ## Raises BlockNotFound error if the block is not in the DB. + db.getBlockHeader(db.getBlockHash(n)) + +proc getScore*( + db: CoreDbRef; + blockHash: Hash256; + ): UInt256 + {.gcsafe, raises: [RlpError].} = + rlp.decode(db.kvt.get(blockHashToScoreKey(blockHash).toOpenArray), UInt256) + +proc setScore*(db: CoreDbRef; blockHash: Hash256, score: UInt256) = + ## for testing purpose + db.kvt.put(blockHashToScoreKey(blockHash).toOpenArray, rlp.encode(score)) + +proc getTd*(db: CoreDbRef; blockHash: Hash256, td: var UInt256): bool = + let bytes = db.kvt.get(blockHashToScoreKey(blockHash).toOpenArray) + if bytes.len == 0: return false + try: + td = rlp.decode(bytes, UInt256) + except RlpError: + return false + return true + +proc headTotalDifficulty*( + db: CoreDbRef; + ): UInt256 + {.gcsafe, raises: [RlpError].} = + # this is actually a combination of `getHash` and `getScore` + const key = canonicalHeadHashKey() + let data = db.kvt.get(key.toOpenArray) + if data.len == 0: + return 0.u256 + + let blockHash = rlp.decode(data, Hash256) + rlp.decode(db.kvt.get(blockHashToScoreKey(blockHash).toOpenArray), UInt256) + +proc getAncestorsHashes*( + db: CoreDbRef; + limit: UInt256; + header: BlockHeader; + ): seq[Hash256] + {.gcsafe, raises: [BlockNotFound].} = + var ancestorCount = min(header.blockNumber, limit).truncate(int) + var h = header + + result = newSeq[Hash256](ancestorCount) + while ancestorCount > 0: + h = db.getBlockHeader(h.parentHash) + result[ancestorCount - 1] = h.hash + dec ancestorCount + +proc addBlockNumberToHashLookup*(db: CoreDbRef; header: BlockHeader) = + db.kvt.put( + blockNumberToHashKey(header.blockNumber).toOpenArray, + rlp.encode(header.hash)) + +proc persistTransactions*( + db: CoreDbRef; + blockNumber: BlockNumber; + transactions: openArray[Transaction]; + ): Hash256 + {.gcsafe, raises: [RlpError].} = + var trie = db.mptPrune() + for idx, tx in transactions: + let + encodedTx = rlp.encode(tx.removeNetworkPayload) + txHash = rlpHash(tx) # beware EIP-4844 + txKey: TransactionKey = (blockNumber, idx) + trie.put(rlp.encode(idx), encodedTx) + db.kvt.put(transactionHashToBlockKey(txHash).toOpenArray, rlp.encode(txKey)) + trie.rootHash + +proc getTransaction*( + db: CoreDbRef; + txRoot: Hash256; + txIndex: int; + res: var Transaction; + ): bool + {.gcsafe, raises: [RlpError].} = + var db = db.mptPrune txRoot + let txData = db.get(rlp.encode(txIndex)) + if txData.len > 0: + res = rlp.decode(txData, Transaction) + result = true + +proc getTransactionCount*( + db: CoreDbRef; + txRoot: Hash256; + ): int + {.gcsafe, raises: [RlpError].} = + var trie = db.mptPrune txRoot + var txCount = 0 + while true: + let txKey = rlp.encode(txCount) + if txKey in trie: + inc txCount + else: + return txCount + + doAssert(false, "unreachable") + +proc getUnclesCount*( + db: CoreDbRef; + ommersHash: Hash256; + ): int + {.gcsafe, raises: [RlpError].} = + if ommersHash != EMPTY_UNCLE_HASH: + let encodedUncles = db.kvt.get(genericHashKey(ommersHash).toOpenArray) + if encodedUncles.len != 0: + let r = rlpFromBytes(encodedUncles) + result = r.listLen + +proc getUncles*( + db: CoreDbRef; + ommersHash: Hash256; + ): seq[BlockHeader] + {.gcsafe, raises: [RlpError].} = + if ommersHash != EMPTY_UNCLE_HASH: + let encodedUncles = db.kvt.get(genericHashKey(ommersHash).toOpenArray) + if encodedUncles.len != 0: + result = rlp.decode(encodedUncles, seq[BlockHeader]) + +proc persistWithdrawals*( + db: CoreDbRef; + withdrawals: openArray[Withdrawal]; + ): Hash256 + {.gcsafe, raises: [RlpError].} = + var trie = db.mptPrune() + for idx, wd in withdrawals: + let encodedWd = rlp.encode(wd) + trie.put(rlp.encode(idx), encodedWd) + trie.rootHash + +proc getWithdrawals*( + db: CoreDbRef; + withdrawalsRoot: Hash256; + ): seq[Withdrawal] + {.gcsafe, raises: [RlpError].} = + for encodedWd in db.getWithdrawalsData(withdrawalsRoot): + result.add(rlp.decode(encodedWd, Withdrawal)) + +proc getBlockBody*( + db: CoreDbRef; + header: BlockHeader; + output: var BlockBody; + ): bool + {.gcsafe, raises: [RlpError].} = + result = true + output.transactions = @[] + output.uncles = @[] + for encodedTx in db.getBlockTransactionData(header.txRoot): + output.transactions.add(rlp.decode(encodedTx, Transaction)) + + if header.ommersHash != EMPTY_UNCLE_HASH: + let encodedUncles = db.kvt.get(genericHashKey(header.ommersHash).toOpenArray) + if encodedUncles.len != 0: + output.uncles = rlp.decode(encodedUncles, seq[BlockHeader]) + else: + result = false + + if header.withdrawalsRoot.isSome: + output.withdrawals = some(db.getWithdrawals(header.withdrawalsRoot.get)) + +proc getBlockBody*( + db: CoreDbRef; + blockHash: Hash256; + output: var BlockBody; + ): bool + {.gcsafe, raises: [RlpError].} = + var header: BlockHeader + if db.getBlockHeader(blockHash, header): + return db.getBlockBody(header, output) + +proc getBlockBody*( + db: CoreDbRef; + hash: Hash256; + ): BlockBody + {.gcsafe, raises: [RlpError,ValueError].} = + if not db.getBlockBody(hash, result): + raise newException(ValueError, "Error when retrieving block body") + +proc getUncleHashes*( + db: CoreDbRef; + blockHashes: openArray[Hash256]; + ): seq[Hash256] + {.gcsafe, raises: [RlpError,ValueError].} = + for blockHash in blockHashes: + var blockBody = db.getBlockBody(blockHash) + for uncle in blockBody.uncles: + result.add uncle.hash + +proc getUncleHashes*( + db: CoreDbRef; + header: BlockHeader; + ): seq[Hash256] + {.gcsafe, raises: [RlpError].} = + if header.ommersHash != EMPTY_UNCLE_HASH: + let encodedUncles = db.kvt.get(genericHashKey(header.ommersHash).toOpenArray) + if encodedUncles.len != 0: + let uncles = rlp.decode(encodedUncles, seq[BlockHeader]) + for x in uncles: + result.add x.hash + +proc getTransactionKey*( + db: CoreDbRef; + transactionHash: Hash256; + ): tuple[blockNumber: BlockNumber, index: int] + {.gcsafe, raises: [RlpError].} = + let tx = db.kvt.get(transactionHashToBlockKey(transactionHash).toOpenArray) + + if tx.len > 0: + let key = rlp.decode(tx, TransactionKey) + result = (key.blockNumber, key.index) + else: + result = (0.toBlockNumber, -1) + +proc headerExists*(db: CoreDbRef; blockHash: Hash256): bool = + ## Returns True if the header with the given block hash is in our DB. + db.kvt.contains(genericHashKey(blockHash).toOpenArray) + +proc setHead*( + db: CoreDbRef; + blockHash: Hash256; + ): bool + {.gcsafe, raises: [RlpError].} = + var header: BlockHeader + if not db.getBlockHeader(blockHash, header): + return false + + if not db.markCanonicalChain(header, blockHash): + return false + + db.kvt.put(canonicalHeadHashKey().toOpenArray, rlp.encode(blockHash)) + return true + +proc setHead*( + db: CoreDbRef; + header: BlockHeader; + writeHeader = false; + ): bool + {.gcsafe, raises: [RlpError].} = + var headerHash = rlpHash(header) + if writeHeader: + db.kvt.put(genericHashKey(headerHash).toOpenArray, rlp.encode(header)) + if not db.markCanonicalChain(header, headerHash): + return false + db.kvt.put(canonicalHeadHashKey().toOpenArray, rlp.encode(headerHash)) + return true + +proc persistReceipts*( + db: CoreDbRef; + receipts: openArray[Receipt]; + ): Hash256 + {.gcsafe, raises: [RlpError].} = + var trie = db.mptPrune() + for idx, rec in receipts: + trie.put(rlp.encode(idx), rlp.encode(rec)) + trie.rootHash + +proc getReceipts*( + db: CoreDbRef; + receiptRoot: Hash256; + ): seq[Receipt] + {.gcsafe, raises: [RlpError].} = + var receipts = newSeq[Receipt]() + for r in db.getReceipts(receiptRoot): + receipts.add(r) + return receipts + +proc persistHeaderToDb*( + db: CoreDbRef; + header: BlockHeader; + forceCanonical: bool; + startOfHistory = GENESIS_PARENT_HASH; + ): seq[BlockHeader] + {.gcsafe, raises: [RlpError,EVMError].} = + let isStartOfHistory = header.parentHash == startOfHistory + let headerHash = header.blockHash + if not isStartOfHistory and not db.headerExists(header.parentHash): + raise newException(ParentNotFound, "Cannot persist block header " & + $headerHash & " with unknown parent " & $header.parentHash) + db.kvt.put(genericHashKey(headerHash).toOpenArray, rlp.encode(header)) + + let score = if isStartOfHistory: header.difficulty + else: db.getScore(header.parentHash) + header.difficulty + db.kvt.put(blockHashToScoreKey(headerHash).toOpenArray, rlp.encode(score)) + + db.addBlockNumberToHashLookup(header) + + var headScore: UInt256 + try: + headScore = db.getScore(db.getCanonicalHead().hash) + except CanonicalHeadNotFound: + return db.setAsCanonicalChainHead(headerHash) + + if score > headScore or forceCanonical: + return db.setAsCanonicalChainHead(headerHash) + +proc persistHeaderToDbWithoutSetHead*( + db: CoreDbRef; + header: BlockHeader; + startOfHistory = GENESIS_PARENT_HASH; + ) {.gcsafe, raises: [RlpError].} = + let isStartOfHistory = header.parentHash == startOfHistory + let headerHash = header.blockHash + let score = if isStartOfHistory: header.difficulty + else: db.getScore(header.parentHash) + header.difficulty + + db.kvt.put(blockHashToScoreKey(headerHash).toOpenArray, rlp.encode(score)) + db.kvt.put(genericHashKey(headerHash).toOpenArray, rlp.encode(header)) + +# FIXME-Adam: This seems like a bad idea. I don't see a way to get the score +# in stateless mode, but it seems dangerous to just shove the header into +# the DB *without* also storing the score. +proc persistHeaderToDbWithoutSetHeadOrScore*(db: CoreDbRef; header: BlockHeader) = + db.addBlockNumberToHashLookup(header) + db.kvt.put(genericHashKey(header.blockHash).toOpenArray, rlp.encode(header)) + +proc persistUncles*(db: CoreDbRef, uncles: openArray[BlockHeader]): Hash256 = + ## Persists the list of uncles to the database. + ## Returns the uncles hash. + let enc = rlp.encode(uncles) + result = keccakHash(enc) + db.kvt.put(genericHashKey(result).toOpenArray, enc) + +proc safeHeaderHash*( + db: CoreDbRef; + ): Hash256 + {.gcsafe, raises: [RlpError].} = + discard db.getHash(safeHashKey(), result) + +proc safeHeaderHash*(db: CoreDbRef, headerHash: Hash256) = + db.kvt.put(safeHashKey().toOpenArray, rlp.encode(headerHash)) + +proc finalizedHeaderHash*( + db: CoreDbRef; + ): Hash256 + {.gcsafe, raises: [RlpError].} = + discard db.getHash(finalizedHashKey(), result) + +proc finalizedHeaderHash*(db: CoreDbRef, headerHash: Hash256) = + db.kvt.put(finalizedHashKey().toOpenArray, rlp.encode(headerHash)) + +proc safeHeader*( + db: CoreDbRef; + ): BlockHeader + {.gcsafe, raises: [RlpError,BlockNotFound].} = + db.getBlockHeader(db.safeHeaderHash) + +proc finalizedHeader*( + db: CoreDbRef; + ): BlockHeader + {.gcsafe, raises: [RlpError,BlockNotFound].} = + db.getBlockHeader(db.finalizedHeaderHash) + +proc haveBlockAndState*(db: CoreDbRef, headerHash: Hash256): bool = + var header: BlockHeader + if not db.getBlockHeader(headerHash, header): + return false + # see if stateRoot exists + db.exists(header.stateRoot) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/legacy.nim b/nimbus/db/core_db/legacy.nim new file mode 100644 index 000000000..f1de43666 --- /dev/null +++ b/nimbus/db/core_db/legacy.nim @@ -0,0 +1,258 @@ +# Nimbus +# Copyright (c) 2018 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +{.push raises: [].} + +import + std/options, + eth/[common, rlp, trie/db, trie/hexary], + results, + ../../constants, + ../select_backend, + ./base + +type + LegacyDbRef = ref object of CoreDbRef + backend: ChainDB # for backend access (legacy mode) + tdb: TrieDatabaseRef # copy of descriptor reference captured with closures + + HexaryTrieRef = ref object + trie: HexaryTrie # nedded for decriptor capturing with closures + + RecorderRef = ref object of RootRef + flags: set[CoreDbCaptFlags] + parent: TrieDatabaseRef + recorder: TrieDatabaseRef + appDb: CoreDbRef + +proc newLegacyDbRef( + dbType: CoreDbType; + tdb: TrieDatabaseRef; + ): CoreDbRef {.gcsafe.} + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +template ifLegacyOk(db: CoreDbRef; body: untyped) = + case db.dbType: + of LegacyDbMemory, LegacyDbPersistent: + body + else: + discard + +# ------------------------------------------------------------------------------ +# Private mixin methods for `trieDB` (backport from capturedb/tracer sources) +# ------------------------------------------------------------------------------ + +proc get(db: RecorderRef, key: openArray[byte]): Blob = + ## Mixin for `trieDB()` + result = db.recorder.get(key) + if result.len == 0: + result = db.parent.get(key) + if result.len != 0: + db.recorder.put(key, result) + +proc put(db: RecorderRef, key, value: openArray[byte]) = + ## Mixin for `trieDB()` + db.recorder.put(key, value) + if PersistPut in db.flags: + db.parent.put(key, value) + +proc contains(db: RecorderRef, key: openArray[byte]): bool = + ## Mixin for `trieDB()` + result = db.parent.contains(key) + doAssert(db.recorder.contains(key) == result) + +proc del(db: RecorderRef, key: openArray[byte]) = + ## Mixin for `trieDB()` + db.recorder.del(key) + if PersistDel in db.flags: + db.parent.del(key) + +proc newRecorderRef( + tdb: TrieDatabaseRef; + flags: set[CoreDbCaptFlags] = {}; + ): RecorderRef = + ## Capture constuctor, uses `mixin` values from above + result = RecorderRef( + flags: flags, + parent: tdb, + recorder: newMemoryDB()) + result.appDb = newLegacyDbRef(LegacyDbPersistent, trieDB result) + +# ------------------------------------------------------------------------------ +# Private database method function tables +# ------------------------------------------------------------------------------ + +proc miscMethods(tdb: TrieDatabaseRef): CoreDbMiscFns = + CoreDbMiscFns( + legacySetupFn: proc() = + tdb.put(EMPTY_ROOT_HASH.data, @[0x80u8])) + +proc kvtMethods(tdb: TrieDatabaseRef): CoreDbKvtFns = + ## Key-value database table handlers + CoreDbKvtFns( + getFn: proc(k: openArray[byte]): Blob = return tdb.get(k), + maybeGetFn: proc(k: openArray[byte]): Option[Blob] = return tdb.maybeGet(k), + delFn: proc(k: openArray[byte]) = tdb.del(k), + putFn: proc(k: openArray[byte]; v: openArray[byte]) = tdb.put(k,v), + containsFn: proc(k: openArray[byte]): bool = return tdb.contains(k), + pairsIt: iterator(): (Blob, Blob) {.gcsafe.} = + for k,v in tdb.pairsInMemoryDB: + yield (k,v)) + +proc mptMethods(mpt: HexaryTrieRef): CoreDbMptFns = + ## Hexary trie database handlers + CoreDbMptFns( + getFn: proc(k: openArray[byte]): Blob {.gcsafe, raises: [RlpError].} = + return mpt.trie.get(k), + + maybeGetFn: proc(k: openArray[byte]): Option[Blob] + {.gcsafe, raises: [RlpError].} = + return mpt.trie.maybeGet(k), + + delFn: proc(k: openArray[byte]) {.gcsafe, raises: [RlpError].} = + mpt.trie.del(k), + + putFn: proc(k: openArray[byte]; v: openArray[byte]) + {.gcsafe, raises: [RlpError].} = + mpt.trie.put(k,v), + + containsFn: proc(k: openArray[byte]): bool {.gcsafe, raises: [RlpError].} = + return mpt.trie.contains(k), + + rootHashFn: proc(): Hash256 = + return mpt.trie.rootHash, + + isPruningFn: proc(): bool = + return mpt.trie.isPruning, + + pairsIt: iterator(): (Blob, Blob) {.gcsafe, raises: [RlpError].} = + for k,v in mpt.trie.pairs(): + yield (k,v), + + replicateIt: iterator(): (Blob, Blob) {.gcsafe, raises: [RlpError].} = + for k,v in mpt.trie.replicate(): + yield (k,v)) + +proc txMethods(tx: DbTransaction): CoreDbTxFns = + CoreDbTxFns( + commitFn: proc(applyDeletes: bool) = tx.commit(applyDeletes), + rollbackFn: proc() = tx.rollback(), + disposeFn: proc() = tx.dispose(), + safeDisposeFn: proc() = tx.safeDispose()) + +proc tidMethods(tid: TransactionID; tdb: TrieDatabaseRef): CoreDbTxIdFns = + CoreDbTxIdFns( + setIdFn: proc() = + tdb.setTransactionID(tid), + + roWrapperFn: proc(action: CoreDbTxIdActionFn) + {.gcsafe, raises: [CatchableError].} = + tdb.shortTimeReadOnly(tid, action())) + +proc cptMethods(cpt: RecorderRef): CoreDbCaptFns = + CoreDbCaptFns( + recorderFn: proc(): CoreDbRef = + return cpt.appDb, + + getFlagsFn: proc(): set[CoreDbCaptFlags] = + return cpt.flags) + +# ------------------------------------------------------------------------------ +# Private constructor functions table +# ------------------------------------------------------------------------------ + +proc constructors(tdb: TrieDatabaseRef, parent: CoreDbRef): CoreDbConstructors = + CoreDbConstructors( + mptFn: proc(root: Hash256): CoreDbMptRef = + let mpt = HexaryTrieRef(trie: initHexaryTrie(tdb, root, false)) + return newCoreDbMptRef(parent, mpt.mptMethods), + + legacyMptFn: proc(root: Hash256; prune: bool): CoreDbMptRef = + let mpt = HexaryTrieRef(trie: initHexaryTrie(tdb, root, prune)) + return newCoreDbMptRef(parent, mpt.mptMethods), + + getIdFn: proc(): CoreDbTxID = + return newCoreDbTxID(parent, tdb.getTransactionID.tidMethods tdb), + + beginFn: proc(): CoreDbTxRef = + return newCoreDbTxRef(parent, tdb.beginTransaction.txMethods), + + captureFn: proc(flags: set[CoreDbCaptFlags] = {}): CoreDbCaptRef = + return newCoreDbCaptRef(parent, newRecorderRef(tdb, flags).cptMethods)) + +# ------------------------------------------------------------------------------ +# Private constructor helpers +# ------------------------------------------------------------------------------ + +proc newLegacyDbRef( + dbType: CoreDbType; + tdb: TrieDatabaseRef; + ): CoreDbRef = + result = LegacyDbRef(tdb: tdb) + result.init( + dbType = dbType, + dbMethods = tdb.miscMethods, + kvtMethods = tdb.kvtMethods, + new = tdb.constructors result) + +proc newLegacyDbRef( + dbType: CoreDbType; + tdb: TrieDatabaseRef; + backend: ChainDB; + ): CoreDbRef = + result = newLegacyDbRef(dbType, tdb) + result.LegacyDbRef.backend = backend + +# ------------------------------------------------------------------------------ +# Public constructor and low level data retrieval, storage & transation frame +# ------------------------------------------------------------------------------ + +proc newLegacyPersistentCoreDbRef*(db: TrieDatabaseRef): CoreDbRef = + newLegacyDbRef(LegacyDbPersistent, db) + +proc newLegacyPersistentCoreDbRef*(path: string): CoreDbRef = + # Kludge: Compiler bails out on `results.tryGet()` with + # :: + # fatal.nim(54) sysFatal + # Error: unhandled exception: types.nim(1251, 10) \ + # `b.kind in {tyObject} + skipPtrs` [AssertionDefect] + # + # when running `select_backend.newChainDB(path)`. The culprit seems to be + # the `ResultError` exception (or any other `CatchableError`). + # + doAssert dbBackend == rocksdb + let rc = RocksStoreRef.init(path, "nimbus") + doAssert(rc.isOk, "Cannot start RocksDB: " & rc.error) + doAssert(not rc.value.isNil, "Starting RocksDB returned nil") + + let backend = ChainDB(kv: rc.value.kvStore, rdb: rc.value) + newLegacyDbRef(LegacyDbPersistent, backend.trieDB, backend) + +proc newLegacyMemoryCoreDbRef*(): CoreDbRef = + newLegacyDbRef(LegacyDbMemory, newMemoryDB()) + +# ------------------------------------------------------------------------------ +# Public legacy helpers +# ------------------------------------------------------------------------------ + +proc toLegacyTrieRef*(db: CoreDbRef): TrieDatabaseRef = + db.ifLegacyOk: + return db.LegacyDbRef.tdb + +proc toLegacyBackend*(db: CoreDbRef): ChainDB = + db.ifLegacyOk: + return db.LegacyDbRef.backend + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/db_chain.nim b/nimbus/db/db_chain.nim index fe67ee03e..e1794daba 100644 --- a/nimbus/db/db_chain.nim +++ b/nimbus/db/db_chain.nim @@ -227,10 +227,12 @@ proc getTransactionCount*(chain: ChainDBRef, txRoot: Hash256): int = var txCount = 0 while true: let txKey = rlp.encode(txCount) - if txKey notin trie: - break - inc txCount - txCount + if txKey in trie: + inc txCount + else: + return txCount + + doAssert(false, "unreachable") proc getUnclesCount*(db: ChainDBRef, ommersHash: Hash256): int = if ommersHash != EMPTY_UNCLE_HASH: @@ -263,6 +265,10 @@ iterator getWithdrawalsData*(db: ChainDBRef, withdrawalsRoot: Hash256): seq[byte break inc idx +proc getWithdrawals*(db: ChainDBRef, withdrawalsRoot: Hash256): seq[Withdrawal] = + for encodedWd in db.getWithdrawalsData(withdrawalsRoot): + result.add(rlp.decode(encodedWd, Withdrawal)) + proc getBlockBody*(db: ChainDBRef, header: BlockHeader, output: var BlockBody): bool = result = true output.transactions = @[] @@ -278,10 +284,7 @@ proc getBlockBody*(db: ChainDBRef, header: BlockHeader, output: var BlockBody): result = false if header.withdrawalsRoot.isSome: - var withdrawals: seq[Withdrawal] - for encodedWd in db.getWithdrawalsData(header.withdrawalsRoot.get): - withdrawals.add(rlp.decode(encodedWd, Withdrawal)) - output.withdrawals = some(withdrawals) + output.withdrawals = some(db.getWithdrawals(header.withdrawalsRoot.get)) proc getBlockBody*(db: ChainDBRef, blockHash: Hash256, output: var BlockBody): bool = var header: BlockHeader diff --git a/nimbus/evm/async/data_sources/json_rpc_data_source.nim b/nimbus/evm/async/data_sources/json_rpc_data_source.nim index b01f8413c..f9bacae8d 100644 --- a/nimbus/evm/async/data_sources/json_rpc_data_source.nim +++ b/nimbus/evm/async/data_sources/json_rpc_data_source.nim @@ -79,7 +79,7 @@ proc makeAnRpcClient*(web3Url: string): Future[RpcClient] {.async.} = uncles*: seq[Hash256] # list of uncle hashes. baseFeePerGas*: Option[UInt256] # EIP-1559 withdrawalsRoot*: Option[Hash256] # EIP-4895 - excessDataGas*: Option[UInt256] # EIP-4844 + excessBlobGas*: Option[UInt256] # EIP-4844 ]# func fromQty(x: Option[Quantity]): Option[uint64] = @@ -108,8 +108,8 @@ func blockHeaderFromBlockObject(o: BlockObject): BlockHeader = nonce: nonce, fee: o.baseFeePerGas, withdrawalsRoot: o.withdrawalsRoot.map(toHash), - dataGasUsed: fromQty(o.dataGasUsed), - excessDataGas: fromQty(o.excessDataGas) + blobGasUsed: fromQty(o.blobGasUsed), + excessBlobGas: fromQty(o.excessBlobGas) ) proc fetchBlockHeaderWithHash*(rpcClient: RpcClient, h: Hash256): Future[BlockHeader] {.async.} = diff --git a/nimbus/evm/computation.nim b/nimbus/evm/computation.nim index 46ffd7abe..5c9cfd7b2 100644 --- a/nimbus/evm/computation.nim +++ b/nimbus/evm/computation.nim @@ -11,7 +11,7 @@ import ".."/[db/accounts_cache, constants], "."/[code_stream, memory, message, stack, state], - "."/[transaction_tracer, types], + "."/[types], ./interpreter/[gas_meter, gas_costs, op_codes], ../common/[common, evmforks], ../utils/utils, @@ -386,22 +386,37 @@ proc refundSelfDestruct*(c: Computation) = c.gasMeter.refundGas(cost * num) proc tracingEnabled*(c: Computation): bool = - TracerFlags.EnableTracing in c.vmState.tracer.flags - -proc traceOpCodeStarted*(c: Computation, op: Op): int - {.gcsafe, raises: [CatchableError].} = - c.vmState.tracer.traceOpCodeStarted(c, op) - -proc traceOpCodeEnded*(c: Computation, op: Op, lastIndex: int) - {.gcsafe, raises: [CatchableError].} = - c.vmState.tracer.traceOpCodeEnded(c, op, lastIndex) - -proc traceError*(c: Computation) - {.gcsafe, raises: [CatchableError].} = - c.vmState.tracer.traceError(c) + c.vmState.tracingEnabled + +proc traceOpCodeStarted*(c: Computation, op: Op): int {.gcsafe, raises: [].} = + c.vmState.captureOpStart( + c.code.pc - 1, + op, + c.gasMeter.gasRemaining, + c.msg.depth + 1) + +proc traceOpCodeEnded*(c: Computation, op: Op, opIndex: int) {.gcsafe, raises: [].} = + c.vmState.captureOpEnd( + c.code.pc - 1, + op, + c.gasMeter.gasRemaining, + c.gasMeter.gasRefunded, + c.returnData, + c.msg.depth + 1, + opIndex) + +proc traceError*(c: Computation) {.gcsafe, raises: [].} = + c.vmState.captureFault( + c.code.pc - 1, + c.instr, + c.gasMeter.gasRemaining, + c.gasMeter.gasRefunded, + c.returnData, + c.msg.depth + 1, + some(c.error.info)) proc prepareTracer*(c: Computation) = - c.vmState.tracer.prepare(c.msg.depth) + c.vmState.capturePrepare(c.msg.depth) # ------------------------------------------------------------------------------ # End diff --git a/nimbus/evm/interpreter/op_handlers/oph_memory.nim b/nimbus/evm/interpreter/op_handlers/oph_memory.nim index 40c058bb5..f355e892c 100644 --- a/nimbus/evm/interpreter/op_handlers/oph_memory.nim +++ b/nimbus/evm/interpreter/op_handlers/oph_memory.nim @@ -27,8 +27,7 @@ import ./oph_defs, ./oph_helpers, eth/common, - stint, - strformat + stint {.push raises: [CatchableError].} # basically the annotation type of a `Vm2OpFn` @@ -50,8 +49,7 @@ when evmc_enabled: gasCost = c.gasCosts[Sstore].c_handler(newValue, gasParam)[0] c.gasMeter.consumeGas( - gasCost, &"SSTORE: {c.msg.contractAddress}[{slot}] " & - &"-> {newValue} ({currentValue})") + gasCost, "SSTORE") else: proc sstoreImpl(c: Computation, slot, newValue: UInt256) = @@ -65,8 +63,7 @@ else: c.gasCosts[Sstore].c_handler(newValue, gasParam) c.gasMeter.consumeGas( - gasCost, &"SSTORE: {c.msg.contractAddress}[{slot}] " & - &"-> {newValue} ({currentValue})") + gasCost, "SSTORE") if gasRefund > 0: c.gasMeter.refundGas(gasRefund) @@ -87,8 +84,7 @@ else: (gasCost, gasRefund) = c.gasCosts[Sstore].c_handler(newValue, gasParam) c.gasMeter.consumeGas( - gasCost, &"SSTORE EIP2200: {c.msg.contractAddress}[{slot}]" & - &" -> {newValue} ({currentValue})") + gasCost, "SSTORE EIP2200") if gasRefund != 0: c.gasMeter.refundGas(gasRefund) diff --git a/nimbus/evm/interpreter_dispatch.nim b/nimbus/evm/interpreter_dispatch.nim index 47cbc149f..af20b1e9a 100644 --- a/nimbus/evm/interpreter_dispatch.nim +++ b/nimbus/evm/interpreter_dispatch.nim @@ -171,8 +171,30 @@ proc afterExecCreate(c: Computation) else: c.rollback() + +const + MsgKindToOp: array[CallKind, Op] = [ + CALL, + DELEGATECALL, + CALLCODE, + CREATE, + CREATE2 + ] + +func msgToOp(msg: Message): Op = + if emvcStatic == msg.flags: + return STATICCALL + MsgKindToOp[msg.kind] + proc beforeExec(c: Computation): bool {.gcsafe, raises: [ValueError].} = + + if c.msg.depth > 0: + c.vmState.captureEnter(msgToOp(c.msg), + c.msg.sender, c.msg.contractAddress, + c.msg.data, c.msg.gas, + c.msg.value) + if not c.msg.isCreate: c.beforeExecCall() false @@ -181,11 +203,20 @@ proc beforeExec(c: Computation): bool proc afterExec(c: Computation) {.gcsafe, raises: [CatchableError].} = + if not c.msg.isCreate: c.afterExecCall() else: c.afterExecCreate() + if c.msg.depth > 0: + let gasUsed = c.msg.gas - c.gasMeter.gasRemaining + let error = if c.isError: + some(c.error.info) + else: + none(string) + c.vmState.captureExit(c.output, gasUsed, error) + # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ @@ -306,7 +337,7 @@ proc asyncExecCallOrCreate*(c: Computation): Future[void] {.async.} = defer: c.dispose() await ifNecessaryGetCode(c.vmState, c.msg.contractAddress) - + if c.beforeExec(): return c.executeOpcodes() diff --git a/nimbus/evm/memory.nim b/nimbus/evm/memory.nim index f1271d245..7f0097bd8 100644 --- a/nimbus/evm/memory.nim +++ b/nimbus/evm/memory.nim @@ -37,8 +37,9 @@ proc newMemory*(size: Natural): Memory = result.extend(0, size) proc read*(memory: var Memory, startPos: Natural, size: Natural): seq[byte] = - # TODO: use an openArray[byte] - result = memory.bytes[startPos ..< (startPos + size)] + result = newSeq[byte](size) + if size > 0: + copyMem(result[0].addr, memory.bytes[startPos].addr, size) when defined(evmc_enabled): proc readPtr*(memory: var Memory, startPos: Natural): ptr byte = diff --git a/nimbus/evm/modexp.nim b/nimbus/evm/modexp.nim index 7d4118812..c1f20c232 100644 --- a/nimbus/evm/modexp.nim +++ b/nimbus/evm/modexp.nim @@ -183,6 +183,9 @@ proc modExp*(b, e, m: openArray[byte]): seq[byte] = var base, exp, modulo, res: mp_int + if m.len == 0: + return @[0.byte] + if mp_init_multi(base, exp.addr, modulo.addr, nil) != MP_OKAY: return @@ -192,7 +195,7 @@ proc modExp*(b, e, m: openArray[byte]): seq[byte] = # EVM special case 1 # If m == 0: EVM returns 0. # If m == 1: we can shortcut that to 0 as well - mp_clear(modulo) + mp_clear_multi(base, exp.addr, modulo.addr, nil) return @[0.byte] if e.len > 0: @@ -201,8 +204,11 @@ proc modExp*(b, e, m: openArray[byte]): seq[byte] = # EVM special case 2 # If 0^0: EVM returns 1 # For all x != 0, x^0 == 1 as well - mp_clear_multi(exp, modulo.addr, nil) + mp_clear_multi(base, exp.addr, modulo.addr, nil) return @[1.byte] + else: + mp_clear_multi(base, exp.addr, modulo.addr, nil) + return @[1.byte] if b.len > 0: discard mp_from_ubin(base, b[0].getPtr, b.len.csize_t) diff --git a/nimbus/evm/state.nim b/nimbus/evm/state.nim index aab009539..baa4818d9 100644 --- a/nimbus/evm/state.nim +++ b/nimbus/evm/state.nim @@ -11,13 +11,13 @@ {.push raises: [].} import - std/[json, options, sets, strformat, tables], + std/[options, sets, strformat, tables], eth/[keys], ../../stateless/[witness_from_tree, witness_types], ../db/accounts_cache, ../common/[common, evmforks], ./async/data_sources, - ./transaction_tracer, + ./interpreter/op_codes, ./types proc init( @@ -31,7 +31,7 @@ proc init( difficulty: UInt256; miner: EthAddress; com: CommonRef; - tracer: TransactionTracer, + tracer: TracerRef, asyncFactory: AsyncOperationFactory = AsyncOperationFactory(maybeDataSource: none[AsyncDataSource]())) {.gcsafe.} = ## Initialisation helper @@ -49,33 +49,6 @@ proc init( self.minerAddress = miner self.asyncFactory = asyncFactory -proc init( - self: BaseVMState; - ac: AccountsCache; - parent: BlockHeader; - timestamp: EthTime; - gasLimit: GasInt; - fee: Option[UInt256]; - prevRandao: Hash256; - difficulty: UInt256; - miner: EthAddress; - com: CommonRef; - tracerFlags: set[TracerFlags]) - {.gcsafe.} = - var tracer: TransactionTracer - tracer.initTracer(tracerFlags) - self.init( - ac = ac, - parent = parent, - timestamp = timestamp, - gasLimit = gasLimit, - fee = fee, - prevRandao= prevRandao, - difficulty= difficulty, - miner = miner, - com = com, - tracer = tracer) - # -------------- proc `$`*(vmState: BaseVMState): string @@ -96,7 +69,7 @@ proc new*( difficulty: UInt256, ## tx env: difficulty miner: EthAddress; ## tx env: coinbase(PoW) or signer(PoA) com: CommonRef; ## block chain config - tracerFlags: set[TracerFlags] = {}): T + tracer: TracerRef = nil): T {.gcsafe.} = ## Create a new `BaseVMState` descriptor from a parent block header. This ## function internally constructs a new account state cache rooted at @@ -116,7 +89,7 @@ proc new*( difficulty = difficulty, miner = miner, com = com, - tracerFlags = tracerFlags) + tracer = tracer) proc reinit*(self: BaseVMState; ## Object descriptor parent: BlockHeader; ## parent header, account sync pos. @@ -192,11 +165,11 @@ proc reinit*(self: BaseVMState; ## Object descriptor header = header) proc init*( - self: BaseVMState; ## Object descriptor - parent: BlockHeader; ## parent header, account sync position - header: BlockHeader; ## header with tx environment data fields - com: CommonRef; ## block chain config - tracerFlags: set[TracerFlags] = {}) + self: BaseVMState; ## Object descriptor + parent: BlockHeader; ## parent header, account sync position + header: BlockHeader; ## header with tx environment data fields + com: CommonRef; ## block chain config + tracer: TracerRef = nil) {.gcsafe, raises: [CatchableError].} = ## Variant of `new()` constructor above for in-place initalisation. The ## `parent` argument is used to sync the accounts cache and the `header` @@ -215,14 +188,14 @@ proc init*( difficulty = header.difficulty, miner = com.minerAddress(header), com = com, - tracerFlags = tracerFlags) + tracer = tracer) proc new*( - T: type BaseVMState; - parent: BlockHeader; ## parent header, account sync position - header: BlockHeader; ## header with tx environment data fields - com: CommonRef; ## block chain config - tracerFlags: set[TracerFlags] = {}): T + T: type BaseVMState; + parent: BlockHeader; ## parent header, account sync position + header: BlockHeader; ## header with tx environment data fields + com: CommonRef; ## block chain config + tracer: TracerRef = nil): T {.gcsafe, raises: [CatchableError].} = ## This is a variant of the `new()` constructor above where the `parent` ## argument is used to sync the accounts cache and the `header` is used @@ -232,41 +205,41 @@ proc new*( ## networks, the miner address is retrievable via `ecRecover()`. new result result.init( - parent = parent, - header = header, - com = com, - tracerFlags = tracerFlags) + parent = parent, + header = header, + com = com, + tracer = tracer) proc new*( - T: type BaseVMState; - header: BlockHeader; ## header with tx environment data fields - com: CommonRef; ## block chain config - tracerFlags: set[TracerFlags] = {}): T + T: type BaseVMState; + header: BlockHeader; ## header with tx environment data fields + com: CommonRef; ## block chain config + tracer: TracerRef = nil): T {.gcsafe, raises: [CatchableError].} = ## This is a variant of the `new()` constructor above where the field ## `header.parentHash`, is used to fetch the `parent` BlockHeader to be ## used in the `new()` variant, above. BaseVMState.new( - parent = com.db.getBlockHeader(header.parentHash), - header = header, - com = com, - tracerFlags = tracerFlags) + parent = com.db.getBlockHeader(header.parentHash), + header = header, + com = com, + tracer = tracer) proc init*( - vmState: BaseVMState; - header: BlockHeader; ## header with tx environment data fields - com: CommonRef; ## block chain config - tracerFlags: set[TracerFlags] = {}): bool + vmState: BaseVMState; + header: BlockHeader; ## header with tx environment data fields + com: CommonRef; ## block chain config + tracer: TracerRef = nil): bool {.gcsafe, raises: [CatchableError].} = ## Variant of `new()` which does not throw an exception on a dangling ## `BlockHeader` parent hash reference. var parent: BlockHeader if com.db.getBlockHeader(header.parentHash, parent): vmState.init( - parent = parent, - header = header, - com = com, - tracerFlags = tracerFlags) + parent = parent, + header = header, + com = com, + tracer = tracer) return true proc statelessInit*( @@ -275,10 +248,8 @@ proc statelessInit*( header: BlockHeader; ## header with tx environment data fields com: CommonRef; ## block chain config asyncFactory: AsyncOperationFactory; - tracerFlags: set[TracerFlags] = {}): bool + tracer: TracerRef = nil): bool {.gcsafe, raises: [CatchableError].} = - var tracer: TransactionTracer - tracer.initTracer(tracerFlags) vmState.init( ac = AccountsCache.init(com.db.db, parent.stateRoot, com.pruneTrie), parent = parent, @@ -345,42 +316,9 @@ template mutateStateDB*(vmState: BaseVMState, body: untyped) = var db {.inject.} = vmState.stateDB body -proc getTracingResult*(vmState: BaseVMState): JsonNode {.inline.} = - doAssert(EnableTracing in vmState.tracer.flags) - vmState.tracer.trace - proc getAndClearLogEntries*(vmState: BaseVMState): seq[Log] = vmState.stateDB.getAndClearLogEntries() -proc enableTracing*(vmState: BaseVMState) = - vmState.tracer.flags.incl EnableTracing - -proc disableTracing*(vmState: BaseVMState) = - vmState.tracer.flags.excl EnableTracing - -func tracingEnabled*(vmState: BaseVMState): bool = - EnableTracing in vmState.tracer.flags - -iterator tracedAccounts*(vmState: BaseVMState): EthAddress = - for acc in vmState.tracer.accounts: - yield acc - -iterator tracedAccountsPairs*(vmState: BaseVMState): (int, EthAddress) = - var idx = 0 - for acc in vmState.tracer.accounts: - yield (idx, acc) - inc idx - -proc removeTracedAccounts*(vmState: BaseVMState, accounts: varargs[EthAddress]) = - for acc in accounts: - vmState.tracer.accounts.excl acc - -proc tracerGasUsed*(vmState: BaseVMState, gasUsed: GasInt) = - vmState.tracer.gasUsed = gasUsed - -proc tracerGasUsed*(vmState: BaseVMState): GasInt = - vmState.tracer.gasUsed - proc status*(vmState: BaseVMState): bool = ExecutionOK in vmState.flags @@ -413,3 +351,62 @@ func forkDeterminationInfoForVMState*(vmState: BaseVMState): ForkDeterminationIn func determineFork*(vmState: BaseVMState): EVMFork = vmState.com.toEVMFork(vmState.forkDeterminationInfoForVMState) + +func tracingEnabled*(vmState: BaseVMState): bool = + vmState.tracer.isNil.not + +proc captureTxStart*(vmState: BaseVMState, gasLimit: GasInt) = + if vmState.tracingEnabled: + vmState.tracer.captureTxStart(gasLimit) + +proc captureTxEnd*(vmState: BaseVMState, restGas: GasInt) = + if vmState.tracingEnabled: + vmState.tracer.captureTxEnd(restGas) + +proc captureStart*(vmState: BaseVMState, c: Computation, + sender: EthAddress, to: EthAddress, + create: bool, input: openArray[byte], + gas: GasInt, value: UInt256) = + if vmState.tracingEnabled: + vmState.tracer.captureStart(c, sender, to, create, input, gas, value) + +proc captureEnd*(vmState: BaseVMState, output: openArray[byte], + gasUsed: GasInt, error: Option[string]) = + if vmState.tracingEnabled: + vmState.tracer.captureEnd(output, gasUsed, error) + +proc captureEnter*(vmState: BaseVMState, op: Op, + sender: EthAddress, to: EthAddress, + input: openArray[byte], gas: GasInt, + value: UInt256) = + if vmState.tracingEnabled: + vmState.tracer.captureEnter(op, sender, to, input, gas, value) + +proc captureExit*(vmState: BaseVMState, output: openArray[byte], + gasUsed: GasInt, error: Option[string]) = + if vmState.tracingEnabled: + vmState.tracer.captureExit(output, gasUsed, error) + +proc captureOpStart*(vmState: BaseVMState, pc: int, + op: Op, gas: GasInt, + depth: int): int = + if vmState.tracingEnabled: + result = vmState.tracer.captureOpStart(pc, op, gas, depth) + +proc captureOpEnd*(vmState: BaseVMState, pc: int, + op: Op, gas: GasInt, refund: GasInt, + rData: openArray[byte], + depth: int, opIndex: int) = + if vmState.tracingEnabled: + vmState.tracer.captureOpEnd(pc, op, gas, refund, rData, depth, opIndex) + +proc captureFault*(vmState: BaseVMState, pc: int, + op: Op, gas: GasInt, refund: GasInt, + rData: openArray[byte], + depth: int, error: Option[string]) = + if vmState.tracingEnabled: + vmState.tracer.captureFault(pc, op, gas, refund, rData, depth, error) + +proc capturePrepare*(vmState: BaseVMState, depth: int) = + if vmState.tracingEnabled: + vmState.tracer.capturePrepare(depth) diff --git a/nimbus/evm/tracer/json_tracer.nim b/nimbus/evm/tracer/json_tracer.nim new file mode 100644 index 000000000..e7fd9bd5e --- /dev/null +++ b/nimbus/evm/tracer/json_tracer.nim @@ -0,0 +1,193 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +import + std/[json, sets, streams, strutils], + eth/common/eth_types, + eth/rlp, + stew/byteutils, + chronicles, + ".."/[types, memory, stack], + ../interpreter/op_codes, + ../../db/accounts_cache, + ../../errors + +type + JsonTracer* = ref object of TracerRef + stream: Stream + pretty: bool + comp: Computation + gas: GasInt + pc: int + stack: JsonNode + storageKeys: seq[HashSet[UInt256]] + index: int + +template stripLeadingZeros(value: string): string = + var cidx = 0 + # ignore the last character so we retain '0' on zero value + while cidx < value.len - 1 and value[cidx] == '0': + cidx.inc + value[cidx .. ^1] + +proc encodeHexInt(x: SomeInteger): JsonNode = + %("0x" & x.toHex.stripLeadingZeros.toLowerAscii) + +proc `%`(x: openArray[byte]): JsonNode = + %("0x" & x.toHex) + +proc writeJson(ctx: JsonTracer, res: JsonNode) = + try: + if ctx.pretty: + ctx.stream.writeLine(res.pretty) + else: + ctx.stream.writeLine($res) + except IOError as ex: + error "JsonTracer writeJson", msg=ex.msg + except OSError as ex: + error "JsonTracer writeJson", msg=ex.msg + +proc rememberStorageKey(ctx: JsonTracer, compDepth: int, key: UInt256) = + ctx.storageKeys[compDepth].incl key + +iterator storage(ctx: JsonTracer, compDepth: int): UInt256 = + doAssert compDepth >= 0 and compDepth < ctx.storageKeys.len + for key in ctx.storageKeys[compDepth]: + yield key + +proc captureOpImpl(ctx: JsonTracer, pc: int, + op: Op, gas: GasInt, refund: GasInt, + rData: openArray[byte], + depth: int, error: Option[string]) {.gcsafe.} = + let + gasCost = ctx.gas - gas + c = ctx.comp + + var res = %{ + "pc": %(ctx.pc), + "op": %(op.int), + "gas": encodeHexInt(ctx.gas), + "gasCost": encodeHexInt(gasCost), + "memSize": %(c.memory.len) + } + + if TracerFlags.DisableMemory notin ctx.flags: + let mem = newJArray() + const chunkLen = 32 + let numChunks = c.memory.len div chunkLen + for i in 0 ..< numChunks: + let memHex = c.memory.bytes.toOpenArray(i * chunkLen, (i + 1) * chunkLen - 1).toHex() + mem.add(%("0x" & memHex.toLowerAscii)) + res["memory"] = mem + + if TracerFlags.DisableStack notin ctx.flags: + res["stack"] = ctx.stack + + if TracerFlags.DisableReturnData notin ctx.flags: + res["returnData"] = %(rData) + + if TracerFlags.DisableStorage notin ctx.flags: + var storage = newJObject() + if c.msg.depth < ctx.storageKeys.len: + var stateDB = c.vmState.stateDB + for key in ctx.storage(c.msg.depth): + let value = stateDB.getStorage(c.msg.contractAddress, key) + storage["0x" & key.dumpHex.stripLeadingZeros] = + %("0x" & value.dumpHex.stripLeadingZeros) + res["storage"] = storage + + res["depth"] = %(depth) + res["refund"] = %(refund) + res["opName"] = %(($op).toUpperAscii) + + if error.isSome: + res["error"] = %(error.get) + + ctx.writeJson(res) + +proc newJsonTracer*(stream: Stream, flags: set[TracerFlags], pretty: bool): JsonTracer = + JsonTracer( + flags: flags, + stream: stream, + pretty: pretty + ) + +method capturePrepare*(ctx: JsonTracer, depth: int) {.gcsafe.} = + if depth >= ctx.storageKeys.len: + let prevLen = ctx.storageKeys.len + ctx.storageKeys.setLen(depth + 1) + for i in prevLen ..< ctx.storageKeys.len - 1: + ctx.storageKeys[i] = initHashSet[UInt256]() + + ctx.storageKeys[depth] = initHashSet[UInt256]() + +# Top call frame +method captureStart*(ctx: JsonTracer, c: Computation, + sender: EthAddress, to: EthAddress, + create: bool, input: openArray[byte], + gas: GasInt, value: UInt256) {.gcsafe.} = + ctx.comp = c + +method captureEnd*(ctx: JsonTracer, output: openArray[byte], + gasUsed: GasInt, error: Option[string]) {.gcsafe.} = + var res = %{ + "output": %(output), + "gasUsed": encodeHexInt(gasUsed) + } + if error.isSome: + res["error"] = %(error.get()) + ctx.writeJson(res) + +# Opcode level +method captureOpStart*(ctx: JsonTracer, pc: int, + op: Op, gas: GasInt, + depth: int): int {.gcsafe.} = + ctx.gas = gas + ctx.pc = pc + + if TracerFlags.DisableStack notin ctx.flags: + let c = ctx.comp + ctx.stack = newJArray() + for v in c.stack.values: + ctx.stack.add(%("0x" & v.dumpHex.stripLeadingZeros)) + + if TracerFlags.DisableStorage notin ctx.flags and op == SSTORE: + try: + let c = ctx.comp + if c.stack.values.len > 1: + ctx.rememberStorageKey(c.msg.depth, c.stack[^1, UInt256]) + except InsufficientStack as ex: + error "JsonTracer captureOpStart", msg=ex.msg + except ValueError as ex: + error "JsonTracer captureOpStart", msg=ex.msg + + result = ctx.index + inc ctx.index + +method captureOpEnd*(ctx: JsonTracer, pc: int, + op: Op, gas: GasInt, refund: GasInt, + rData: openArray[byte], + depth: int, opIndex: int) {.gcsafe.} = + try: + ctx.captureOpImpl(pc, op, gas, refund, rData, depth, none(string)) + except RlpError as ex: + error "JsonTracer captureOpEnd", msg=ex.msg + +method captureFault*(ctx: JsonTracer, pc: int, + op: Op, gas: GasInt, refund: GasInt, + rData: openArray[byte], + depth: int, error: Option[string]) {.gcsafe.} = + try: + ctx.captureOpImpl(pc, op, gas, refund, rData, depth, error) + except RlpError as ex: + error "JsonTracer captureOpEnd", msg=ex.msg + +proc close*(ctx: JsonTracer) = + ctx.stream.close() diff --git a/nimbus/evm/tracer/legacy_tracer.nim b/nimbus/evm/tracer/legacy_tracer.nim new file mode 100644 index 000000000..851258e16 --- /dev/null +++ b/nimbus/evm/tracer/legacy_tracer.nim @@ -0,0 +1,205 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +import + std/[json, sets, strutils, hashes], + eth/common/eth_types, + eth/rlp, + stew/byteutils, + chronicles, + ".."/[types, memory, stack], + ../interpreter/op_codes, + ../../db/accounts_cache, + ../../errors + +type + LegacyTracer* = ref object of TracerRef + trace: JsonNode + accounts: HashSet[EthAddress] + storageKeys: seq[HashSet[UInt256]] + comp: Computation + gas: GasInt + +proc hash*(x: UInt256): Hash = + result = hash(x.toByteArrayBE) + +proc rememberStorageKey(ctx: LegacyTracer, compDepth: int, key: UInt256) = + ctx.storageKeys[compDepth].incl key + +iterator storage(ctx: LegacyTracer, compDepth: int): UInt256 = + doAssert compDepth >= 0 and compDepth < ctx.storageKeys.len + for key in ctx.storageKeys[compDepth]: + yield key + +template stripLeadingZeros(value: string): string = + var cidx = 0 + # ignore the last character so we retain '0' on zero value + while cidx < value.len - 1 and value[cidx] == '0': + cidx.inc + value[cidx .. ^1] + +proc encodeHexInt(x: SomeInteger): JsonNode = + %("0x" & x.toHex.stripLeadingZeros.toLowerAscii) + +proc newLegacyTracer*(flags: set[TracerFlags]): LegacyTracer = + let trace = newJObject() + + # make appear at the top of json object + trace["gas"] = %0 + trace["failed"] = %false + trace["returnValue"] = %"" + trace["structLogs"] = newJArray() + + LegacyTracer( + flags: flags, + trace: trace + ) + +method capturePrepare*(ctx: LegacyTracer, depth: int) {.gcsafe.} = + if depth >= ctx.storageKeys.len: + let prevLen = ctx.storageKeys.len + ctx.storageKeys.setLen(depth + 1) + for i in prevLen ..< ctx.storageKeys.len - 1: + ctx.storageKeys[i] = initHashSet[UInt256]() + + ctx.storageKeys[depth] = initHashSet[UInt256]() + +# Top call frame +method captureStart*(ctx: LegacyTracer, c: Computation, + sender: EthAddress, to: EthAddress, + create: bool, input: openArray[byte], + gas: GasInt, value: UInt256) {.gcsafe.} = + ctx.comp = c + +method captureEnd*(ctx: LegacyTracer, output: openArray[byte], + gasUsed: GasInt, error: Option[string]) {.gcsafe.} = + discard + +# Opcode level +method captureOpStart*(ctx: LegacyTracer, pc: int, + op: Op, gas: GasInt, + depth: int): int {.gcsafe.} = + try: + let + j = newJObject() + c = ctx.comp + ctx.trace["structLogs"].add(j) + + j["op"] = %(($op).toUpperAscii) + j["pc"] = %(c.code.pc - 1) + j["depth"] = %(c.msg.depth + 1) + j["gas"] = %(gas) + ctx.gas = gas + + # log stack + if TracerFlags.DisableStack notin ctx.flags: + let stack = newJArray() + for v in c.stack.values: + stack.add(%v.dumpHex()) + j["stack"] = stack + + # log memory + if TracerFlags.DisableMemory notin ctx.flags: + let mem = newJArray() + const chunkLen = 32 + let numChunks = c.memory.len div chunkLen + for i in 0 ..< numChunks: + let memHex = c.memory.bytes.toOpenArray(i * chunkLen, (i + 1) * chunkLen - 1).toHex() + mem.add(%(memHex.toUpperAscii)) + j["memory"] = mem + + if TracerFlags.EnableAccount in ctx.flags: + case op + of Call, CallCode, DelegateCall, StaticCall: + if c.stack.values.len > 2: + ctx.accounts.incl c.stack[^2, EthAddress] + of ExtCodeCopy, ExtCodeSize, Balance, SelfDestruct: + if c.stack.values.len > 1: + ctx.accounts.incl c.stack[^1, EthAddress] + else: + discard + + if TracerFlags.DisableStorage notin ctx.flags: + if op == Sstore: + if c.stack.values.len > 1: + ctx.rememberStorageKey(c.msg.depth, c.stack[^1, UInt256]) + + result = ctx.trace["structLogs"].len - 1 + except KeyError as ex: + error "LegacyTracer captureOpStart", msg=ex.msg + except ValueError as ex: + error "LegacyTracer captureOpStart", msg=ex.msg + except InsufficientStack as ex: + error "LegacyTracer captureOpEnd", msg=ex.msg + +method captureOpEnd*(ctx: LegacyTracer, pc: int, + op: Op, gas: GasInt, refund: GasInt, + rData: openArray[byte], + depth: int, opIndex: int) {.gcsafe.} = + try: + let + j = ctx.trace["structLogs"].elems[opIndex] + c = ctx.comp + + # TODO: figure out how to get storage + # when contract execution interrupted by exception + if TracerFlags.DisableStorage notin ctx.flags: + var storage = newJObject() + if c.msg.depth < ctx.storageKeys.len: + var stateDB = c.vmState.stateDB + for key in ctx.storage(c.msg.depth): + let value = stateDB.getStorage(c.msg.contractAddress, key) + storage[key.dumpHex] = %(value.dumpHex) + j["storage"] = storage + + j["gasCost"] = %(ctx.gas - gas) + + if op in {Return, Revert} and TracerFlags.DisableReturnData notin ctx.flags: + let returnValue = %("0x" & toHex(c.output)) + j["returnValue"] = returnValue + ctx.trace["returnValue"] = returnValue + except KeyError as ex: + error "LegacyTracer captureOpEnd", msg=ex.msg + except RlpError as ex: + error "LegacyTracer captureOpEnd", msg=ex.msg + +method captureFault*(ctx: LegacyTracer, pc: int, + op: Op, gas: GasInt, refund: GasInt, + rData: openArray[byte], + depth: int, error: Option[string]) {.gcsafe.} = + try: + let c = ctx.comp + if ctx.trace["structLogs"].elems.len > 0: + let j = ctx.trace["structLogs"].elems[^1] + j["error"] = %(c.error.info) + j["gasCost"] = %(ctx.gas - gas) + + ctx.trace["failed"] = %true + except KeyError as ex: + error "LegacyTracer captureOpEnd", msg=ex.msg + except InsufficientStack as ex: + error "LegacyTracer captureOpEnd", msg=ex.msg + +proc getTracingResult*(ctx: LegacyTracer): JsonNode = + ctx.trace + +iterator tracedAccounts*(ctx: LegacyTracer): EthAddress = + for acc in ctx.accounts: + yield acc + +iterator tracedAccountsPairs*(ctx: LegacyTracer): (int, EthAddress) = + var idx = 0 + for acc in ctx.accounts: + yield (idx, acc) + inc idx + +proc removeTracedAccounts*(ctx: LegacyTracer, accounts: varargs[EthAddress]) = + for acc in accounts: + ctx.accounts.excl acc diff --git a/nimbus/evm/transaction_tracer.nim b/nimbus/evm/transaction_tracer.nim deleted file mode 100644 index ce573a1db..000000000 --- a/nimbus/evm/transaction_tracer.nim +++ /dev/null @@ -1,190 +0,0 @@ -# Nimbus -# Copyright (c) 2023 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or -# http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or -# http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -import - std/[json, strutils, sets, hashes], - chronicles, eth/common, stint, - nimcrypto/utils, - ./types, ./memory, ./stack, ../db/accounts_cache, - ./interpreter/op_codes - -logScope: - topics = "vm opcode" - -proc hash*(x: UInt256): Hash = - result = hash(x.toByteArrayBE) - -proc initTracer*(tracer: var TransactionTracer, flags: set[TracerFlags] = {}) = - tracer.trace = newJObject() - - # make appear at the top of json object - tracer.trace["gas"] = %0 - tracer.trace["failed"] = %false - - if TracerFlags.GethCompatibility in tracer.flags: - tracer.trace["returnData"] = %"" - else: - tracer.trace["returnValue"] = %"" - - tracer.trace["structLogs"] = newJArray() - tracer.flags = flags - tracer.accounts = initHashSet[EthAddress]() - tracer.storageKeys = @[] - -proc prepare*(tracer: var TransactionTracer, compDepth: int) = - # this uncommon arragement is intentional - # compDepth will be varying up and down: 1,2,3,4,3,3,2,2,1 - # see issue #245 and PR #247 discussion - if compDepth >= tracer.storageKeys.len: - let prevLen = tracer.storageKeys.len - tracer.storageKeys.setLen(compDepth + 1) - for i in prevLen ..< tracer.storageKeys.len - 1: - tracer.storageKeys[i] = initHashSet[UInt256]() - - tracer.storageKeys[compDepth] = initHashSet[UInt256]() - -proc rememberStorageKey(tracer: var TransactionTracer, compDepth: int, key: UInt256) = - tracer.storageKeys[compDepth].incl key - -iterator storage(tracer: TransactionTracer, compDepth: int): UInt256 = - doAssert compDepth >= 0 and compDepth < tracer.storageKeys.len - for key in tracer.storageKeys[compDepth]: - yield key - -template stripLeadingZeros(value: string): string = - var cidx = 0 - # ignore the last character so we retain '0' on zero value - while cidx < value.len - 1 and value[cidx] == '0': - cidx.inc - value[cidx .. ^1] - -proc encodeHexInt(x: SomeInteger): JsonNode = - %("0x" & x.toHex.stripLeadingZeros.toLowerAscii) - -proc traceOpCodeStarted*(tracer: var TransactionTracer, c: Computation, op: Op): int = - if unlikely tracer.trace.isNil: - tracer.initTracer() - - let j = newJObject() - tracer.trace["structLogs"].add(j) - - if TracerFlags.GethCompatibility in tracer.flags: - j["pc"] = %(c.code.pc - 1) - j["op"] = %(op.int) - j["gas"] = encodeHexInt(c.gasMeter.gasRemaining) - j["gasCost"] = %("") - j["memSize"] = %c.memory.len - j["opName"] = %(($op).toUpperAscii) - j["depth"] = %(c.msg.depth + 1) - - # log stack - if TracerFlags.DisableStack notin tracer.flags: - let st = newJArray() - for v in c.stack.values: - st.add(%("0x" & v.dumpHex.stripLeadingZeros)) - j["stack"] = st - - else: - j["op"] = %(($op).toUpperAscii) - j["pc"] = %(c.code.pc - 1) - j["depth"] = %(c.msg.depth + 1) - j["gas"] = %c.gasMeter.gasRemaining - - # log stack - if TracerFlags.DisableStack notin tracer.flags: - let st = newJArray() - for v in c.stack.values: - st.add(%v.dumpHex()) - j["stack"] = st - - # log memory - if TracerFlags.DisableMemory notin tracer.flags: - let mem = newJArray() - const chunkLen = 32 - let numChunks = c.memory.len div chunkLen - for i in 0 ..< numChunks: - let memHex = c.memory.bytes.toOpenArray(i * chunkLen, (i + 1) * chunkLen - 1).toHex() - if TracerFlags.GethCompatibility in tracer.flags: - mem.add(%("0x" & memHex.toLowerAscii)) - else: - mem.add(%memHex) - j["memory"] = mem - - if TracerFlags.EnableAccount in tracer.flags: - case op - of Call, CallCode, DelegateCall, StaticCall: - if c.stack.values.len > 2: - tracer.accounts.incl c.stack[^2, EthAddress] - of ExtCodeCopy, ExtCodeSize, Balance, SelfDestruct: - if c.stack.values.len > 1: - tracer.accounts.incl c.stack[^1, EthAddress] - else: - discard - - if TracerFlags.DisableStorage notin tracer.flags: - if op == Sstore: - if c.stack.values.len > 1: - tracer.rememberStorageKey(c.msg.depth, c.stack[^1, UInt256]) - - result = tracer.trace["structLogs"].len - 1 - -proc traceOpCodeEnded*(tracer: var TransactionTracer, c: Computation, op: Op, lastIndex: int) = - let j = tracer.trace["structLogs"].elems[lastIndex] - - # TODO: figure out how to get storage - # when contract execution interrupted by exception - if TracerFlags.DisableStorage notin tracer.flags: - var storage = newJObject() - if c.msg.depth < tracer.storageKeys.len: - var stateDB = c.vmState.stateDB - for key in tracer.storage(c.msg.depth): - let value = stateDB.getStorage(c.msg.contractAddress, key) - if TracerFlags.GethCompatibility in tracer.flags: - storage["0x" & key.dumpHex.stripLeadingZeros] = - %("0x" & value.dumpHex.stripLeadingZeros) - else: - storage[key.dumpHex] = %(value.dumpHex) - j["storage"] = storage - - if TracerFlags.GethCompatibility in tracer.flags: - let gas = fromHex[GasInt](j["gas"].getStr) - j["gasCost"] = encodeHexInt(gas - c.gasMeter.gasRemaining) - else: - let gas = j["gas"].getBiggestInt() - j["gasCost"] = %(gas - c.gasMeter.gasRemaining) - - if op in {Return, Revert} and TracerFlags.DisableReturnData notin tracer.flags: - let returnValue = %("0x" & toHex(c.output, true)) - if TracerFlags.GethCompatibility in tracer.flags: - j["returnData"] = returnValue - tracer.trace["returnData"] = returnValue - else: - j["returnValue"] = returnValue - tracer.trace["returnValue"] = returnValue - - trace "Op", json = j.pretty() - -proc traceError*(tracer: var TransactionTracer, c: Computation) = - if tracer.trace["structLogs"].elems.len > 0: - let j = tracer.trace["structLogs"].elems[^1] - j["error"] = %(c.error.info) - trace "Error", json = j.pretty() - - # even though the gasCost is incorrect, - # we have something to display, - # it is an error anyway - if TracerFlags.GethCompatibility in tracer.flags: - let gas = fromHex[GasInt](j["gas"].getStr) - j["gasCost"] = encodeHexInt(gas - c.gasMeter.gasRemaining) - else: - let gas = j["gas"].getBiggestInt() - j["gasCost"] = %(gas - c.gasMeter.gasRemaining) - - tracer.trace["failed"] = %true diff --git a/nimbus/evm/types.nim b/nimbus/evm/types.nim index 38927175b..041075d0e 100644 --- a/nimbus/evm/types.nim +++ b/nimbus/evm/types.nim @@ -9,7 +9,6 @@ # according to those terms. import - std/[json, sets], chronos, json_rpc/rpcclient, "."/[stack, memory, code_stream], @@ -46,7 +45,7 @@ type prevRandao* : Hash256 blockDifficulty*: UInt256 flags* : set[VMFlag] - tracer* : TransactionTracer + tracer* : TracerRef receipts* : seq[Receipt] stateDB* : AccountsCache cumulativeGasUsed*: GasInt @@ -58,24 +57,6 @@ type minerAddress* : EthAddress asyncFactory* : AsyncOperationFactory - TracerFlags* {.pure.} = enum - EnableTracing - DisableStorage - DisableMemory - DisableStack - DisableState - DisableStateDiff - EnableAccount - DisableReturnData - GethCompatibility - - TransactionTracer* = object - trace*: JsonNode - flags*: set[TracerFlags] - accounts*: HashSet[EthAddress] - storageKeys*: seq[HashSet[UInt256]] - gasUsed*: GasInt - Computation* = ref object # The execution computation vmState*: BaseVMState @@ -129,3 +110,82 @@ type value*: UInt256 data*: seq[byte] flags*: MsgFlags + + TracerFlags* {.pure.} = enum + DisableStorage + DisableMemory + DisableStack + DisableState + DisableStateDiff + EnableAccount + DisableReturnData + + StructLog* = object + pc* : int + op* : Op + gas* : GasInt + gasCost* : GasInt + memory* : seq[byte] + memSize* : int + stack* : seq[UInt256] + returnData* : seq[byte] + storage* : Table[UInt256, UInt256] + depth* : int + refund* : GasInt + opName* : string + error* : string + + TracerRef* = ref object of RootObj + flags*: set[TracerFlags] + +# Transaction level +method captureTxStart*(ctx: TracerRef, gasLimit: GasInt) {.base, gcsafe.} = + discard + +method captureTxEnd*(ctx: TracerRef, restGas: GasInt) {.base, gcsafe.} = + discard + +# Top call frame +method captureStart*(ctx: TracerRef, c: Computation, + sender: EthAddress, to: EthAddress, + create: bool, input: openArray[byte], + gas: GasInt, value: UInt256) {.base, gcsafe.} = + discard + +method captureEnd*(ctx: TracerRef, output: openArray[byte], + gasUsed: GasInt, error: Option[string]) {.base, gcsafe.} = + discard + +# Rest of call frames +method captureEnter*(ctx: TracerRef, op: Op, + sender: EthAddress, to: EthAddress, + input: openArray[byte], gas: GasInt, + value: UInt256) {.base, gcsafe.} = + discard + +method captureExit*(ctx: TracerRef, output: openArray[byte], + gasUsed: GasInt, error: Option[string]) {.base, gcsafe.} = + discard + +# Opcode level +method captureOpStart*(ctx: TracerRef, pc: int, + op: Op, gas: GasInt, + depth: int): int {.base, gcsafe.} = + discard + +method captureOpEnd*(ctx: TracerRef, pc: int, + op: Op, gas: GasInt, refund: GasInt, + rData: openArray[byte], + depth: int, opIndex: int) {.base, gcsafe.} = + discard + +method captureFault*(ctx: TracerRef, pc: int, + op: Op, gas: GasInt, refund: GasInt, + rData: openArray[byte], + depth: int, error: Option[string]) {.base, gcsafe.} = + discard + + +method capturePrepare*(ctx: TracerRef, depth: int) {.base, gcsafe.} = + discard + diff --git a/nimbus/graphql/ethapi.nim b/nimbus/graphql/ethapi.nim index b52a55654..3dbaac6d9 100644 --- a/nimbus/graphql/ethapi.nim +++ b/nimbus/graphql/ethapi.nim @@ -10,7 +10,7 @@ import std/[strutils, times], stew/[results, byteutils], stint, - eth/[rlp], chronos, + eth/common/eth_types_rlp, chronos, stew/shims/net, graphql, graphql/graphql as context, graphql/common/types, graphql/httpserver, @@ -19,7 +19,7 @@ import ".."/[transaction, vm_state, config, constants], ../common/common, ../transaction/call_evm, - ../core/tx_pool, + ../core/[tx_pool, tx_pool/tx_item], ../utils/utils from eth/p2p import EthereumNode @@ -37,6 +37,7 @@ type ethQuery = "Query" ethMutation = "Mutation" ethAccessTuple = "AccessTuple" + ethWithdrawal = "Withdrawal" HeaderNode = ref object of Node header: BlockHeader @@ -62,6 +63,9 @@ type AclNode = ref object of Node acl: AccessPair + WdNode = ref object of Node + wd: Withdrawal + GraphqlContextRef = ref GraphqlContextObj GraphqlContextObj = object of Graphql ids: array[EthTypes, Name] @@ -79,7 +83,12 @@ proc toHash(n: Node): Hash256 = result.data = hexToByteArray[32](n.stringVal) proc toBlockNumber(n: Node): BlockNumber = - result = parse(n.intVal, UInt256, radix = 10) + if n.kind == nkInt: + result = parse(n.intVal, UInt256, radix = 10) + elif n.kind == nkString: + result = parse(n.stringVal, UInt256, radix = 16) + else: + doAssert(false, "unknown node type: " & $n.kind) proc headerNode(ctx: GraphqlContextRef, header: BlockHeader): Node = HeaderNode( @@ -128,6 +137,14 @@ proc aclNode(ctx: GraphqlContextRef, accessPair: AccessPair): Node = acl: accessPair ) +proc wdNode(ctx: GraphqlContextRef, wd: Withdrawal): Node = + WdNode( + kind: nkMap, + typeName: ctx.ids[ethWithdrawal], + pos: Pos(), + wd: wd + ) + proc getStateDB(com: CommonRef, header: BlockHeader): ReadOnlyStateDB = ## Retrieves the account db from canonical head ## we don't use accounst_cache here because it's read only operations @@ -190,6 +207,10 @@ proc bigIntNode(x: uint64 | int64): RespResult = # stdlib toHex is not suitable for hive const HexChars = "0123456789abcdef" + + if x == 0: + return ok(Node(kind: nkString, stringVal: "0x0", pos: Pos())) + var n = cast[uint64](x) r: array[2*sizeof(uint64), char] @@ -278,6 +299,19 @@ proc getTxs(ctx: GraphqlContextRef, header: BlockHeader): RespResult = except CatchableError as e: err("can't get transactions: " & e.msg) +proc getWithdrawals(ctx: GraphqlContextRef, header: BlockHeader): RespResult = + try: + if header.withdrawalsRoot.isSome: + let wds = getWithdrawals(ctx.chainDB, header.withdrawalsRoot.get) + var list = respList() + for wd in wds: + list.add wdNode(ctx, wd) + ok(list) + else: + ok(respNull()) + except CatchableError as e: + err("can't get transactions: " & e.msg) + proc getTxAt(ctx: GraphqlContextRef, header: BlockHeader, index: int): RespResult = try: var tx: Transaction @@ -323,9 +357,25 @@ proc accountNode(ctx: GraphqlContextRef, header: BlockHeader, address: EthAddres except RlpError as ex: err(ex.msg) +func hexCharToInt(c: char): uint64 = + case c + of 'a'..'f': return c.uint64 - 'a'.uint64 + 10'u64 + of 'A'..'F': return c.uint64 - 'A'.uint64 + 10'u64 + of '0'..'9': return c.uint64 - '0'.uint64 + else: doAssert(false, "invalid hex digit: " & $c) + proc parseU64(node: Node): uint64 = - for c in node.intVal: - result = result * 10 + uint64(c.int - '0'.int) + if node.kind == nkString: + if node.stringVal.len > 2 and node.stringVal[1] == 'x': + for i in 2.. maxU64: return err("long value overflow") - ok(Node(kind: nkInt, pos: node.pos, intVal: $val)) + ok(node) else: let val = parse(node.stringVal, UInt256, radix = 10) if val > maxU64: return err("long value overflow") - ok(Node(kind: nkInt, pos: node.pos, intVal: node.stringVal)) + ok(Node(kind: nkString, pos: node.pos, stringVal: "0x" & val.toHex)) of nkInt: let val = parse(node.intVal, UInt256, radix = 10) if val > maxU64: return err("long value overflow") - ok(node) + ok(Node(kind: nkString, pos: node.pos, stringVal: "0x" & val.toHex)) else: err("expect int, but got '$1'" % [$node.kind]) except CatchableError as e: @@ -583,25 +633,35 @@ proc txIndex(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} ok(resp(tx.index)) proc txFrom(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = - # TODO: with block param let ctx = GraphqlContextRef(ud) let tx = TxNode(parent) + + let blockNumber = if params[0].val.kind != nkEmpty: + parseU64(params[0].val).toBlockNumber + else: + tx.blockNumber + var sender: EthAddress if not getSender(tx.tx, sender): return ok(respNull()) - let hres = ctx.getBlockByNumber(tx.blockNumber) + let hres = ctx.getBlockByNumber(blockNumber) if hres.isErr: return hres let h = HeaderNode(hres.get()) ctx.accountNode(h.header, sender) proc txTo(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = - # TODO: with block param let ctx = GraphqlContextRef(ud) let tx = TxNode(parent) + + let blockNumber = if params[0].val.kind != nkEmpty: + parseU64(params[0].val).toBlockNumber + else: + tx.blockNumber + if tx.tx.contractCreation: return ok(respNull()) - let hres = ctx.getBlockByNumber(tx.blockNumber) + let hres = ctx.getBlockByNumber(blockNumber) if hres.isErr: return hres let h = HeaderNode(hres.get()) @@ -741,6 +801,35 @@ proc txAccessList(ud: RootRef, params: Args, parent: Node): RespResult {.apiPrag list.add aclNode(ctx, x) ok(list) +proc txMaxFeePerBlobGas(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let ctx = GraphqlContextRef(ud) + let tx = TxNode(parent) + if tx.tx.txType < TxEIP4844: + ok(respNull()) + else: + longNode(tx.tx.maxFeePerBlobGas) + +proc txVersionedHashes(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let ctx = GraphqlContextRef(ud) + let tx = TxNode(parent) + if tx.tx.txType < TxEIP4844: + ok(respNull()) + else: + var list = respList() + for hs in tx.tx.versionedHashes: + list.add resp("0x" & hs.data.toHex) + ok(list) + +proc txRaw(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let tx = TxNode(parent) + let txBytes = rlp.encode(tx.tx) + resp(txBytes) + +proc txRawReceipt(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let tx = TxNode(parent) + let recBytes = rlp.encode(tx.receipt) + resp(recBytes) + const txProcs = { "from": txFrom, "hash": txHash, @@ -765,7 +854,11 @@ const txProcs = { "maxFeePerGas": txMaxFeePerGas, "maxPriorityFeePerGas": txMaxPriorityFeePerGas, "effectiveGasPrice": txEffectiveGasPrice, - "chainID": txChainId + "chainID": txChainId, + "maxFeePerBlobGas": txmaxFeePerBlobGas, + "versionedHashes": txVersionedHashes, + "raw": txRaw, + "rawReceipt": txRawReceipt } proc aclAddress(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = @@ -787,6 +880,29 @@ const aclProcs = { "storageKeys": aclStorageKeys } +proc wdIndex(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let w = WdNode(parent) + longNode(w.wd.index) + +proc wdValidator(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let w = WdNode(parent) + longNode(w.wd.validatorIndex) + +proc wdAddress(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let w = WdNode(parent) + resp(w.wd.address) + +proc wdAmount(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let w = WdNode(parent) + longNode(w.wd.amount) + +const wdProcs = { + "index": wdIndex, + "validator": wdValidator, + "address": wdAddress, + "amount": wdAmount +} + proc blockNumberImpl(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = let ctx = GraphqlContextRef(ud) let h = HeaderNode(parent) @@ -1012,6 +1128,32 @@ proc blockBaseFeePerGas(ud: RootRef, params: Args, parent: Node): RespResult {.a else: ok(respNull()) +proc blockWithdrawalsRoot(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let h = HeaderNode(parent) + if h.header.withdrawalsRoot.isSome: + resp(h.header.withdrawalsRoot.get) + else: + ok(respNull()) + +proc blockWithdrawals(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let ctx = GraphqlContextRef(ud) + let h = HeaderNode(parent) + getWithdrawals(ctx, h.header) + +proc blockBlobGasUsed(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let h = HeaderNode(parent) + if h.header.blobGasUsed.isSome: + longNode(h.header.blobGasUsed.get) + else: + ok(respNull()) + +proc blockexcessBlobGas(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = + let h = HeaderNode(parent) + if h.header.excessBlobGas.isSome: + longNode(h.header.excessBlobGas.get) + else: + ok(respNull()) + const blockProcs = { "parent": blockParent, "number": blockNumberImpl, @@ -1040,7 +1182,11 @@ const blockProcs = { "account": blockAccount, "call": blockCall, "estimateGas": blockEstimateGas, - "baseFeePerGas": blockBaseFeePerGas + "baseFeePerGas": blockBaseFeePerGas, + "withdrawalsRoot": blockWithdrawalsRoot, + "withdrawals": blockWithdrawals, + "blobGasUsed": blockBlobGasUsed, + "excessBlobGas": blockExcessBlobGas } proc callResultData(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = @@ -1151,6 +1297,12 @@ proc queryBlock(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma err("only one param allowed, number or hash, not both") elif number.kind == nkInt: getBlockByNumber(ctx, number) + elif number.kind == nkString: + try: + let blockNumber = toBlockNumber(number) + getBlockByNumber(ctx, blockNumber) + except ValueError as ex: + err(ex.msg) elif hash.kind == nkString: getBlockByHash(ctx, hash) else: @@ -1241,16 +1393,26 @@ const queryProcs = { "chainID": queryChainId } +proc inPoolAndOk(ctx: GraphqlContextRef, txHash: Hash256): bool = + let res = ctx.txPool.getItem(txHash) + if res.isErr: return false + res.get().reject == txInfoOk + proc sendRawTransaction(ud: RootRef, params: Args, parent: Node): RespResult {.apiPragma.} = - # TODO: add tx validation and tx processing - # probably goes to tx pool # if tx validation failed, the result will be null let ctx = GraphqlContextRef(ud) try: let data = hexToSeqByte(params[0].val.stringVal) let tx = decodeTx(data) # we want to know if it is a valid tx blob let txHash = rlpHash(tx) # beware EIP-4844 - resp(txHash) + + ctx.txPool.add(tx) + + if ctx.inPoolAndOk(txHash): + return resp(txHash) + else: + return err("transaction rejected by txpool") + except CatchableError as em: return err("failed to process raw transaction: " & em.msg) @@ -1315,6 +1477,7 @@ proc initEthApi(ctx: GraphqlContextRef) = ctx.addResolvers(ctx, ctx.ids[ethQuery ], queryProcs) ctx.addResolvers(ctx, ctx.ids[ethMutation ], mutationProcs) ctx.addResolvers(ctx, ctx.ids[ethAccessTuple], aclProcs) + ctx.addResolvers(ctx, ctx.ids[ethWithdrawal ], wdProcs) var qc = newQC(ctx) ctx.addInstrument(qc) diff --git a/nimbus/graphql/ethapi.ql b/nimbus/graphql/ethapi.ql index 0e68f910c..5329209c4 100644 --- a/nimbus/graphql/ethapi.ql +++ b/nimbus/graphql/ethapi.ql @@ -5,7 +5,7 @@ scalar Bytes32 scalar Address # Bytes is an arbitrary length binary string, represented as 0x-prefixed hexadecimal. -# An empty byte string is represented as '0x'. Byte strings must have an even number of hexadecimal nybbles. +# An empty byte string is represented as '0x'. Byte strings must have an even number of hexadecimal nibbles. scalar Bytes # BigInt is a large integer. Input is accepted as either a JSON number or as a string. @@ -13,7 +13,9 @@ scalar Bytes # 0x-prefixed hexadecimal. scalar BigInt -# Long is a 64 bit unsigned integer. +# Long is a 64 bit unsigned integer. Input is accepted as either a JSON number or as a string. +# Strings may be either decimal or 0x-prefixed hexadecimal. Output values are all +# 0x-prefixed hexadecimal. scalar Long schema { @@ -46,7 +48,7 @@ type Account { # Log is an Ethereum event log. type Log { # Index is the index of this log in the block. - index: Int! + index: Long! # Account is the account which generated this log - this will always # be a contract account. @@ -64,13 +66,28 @@ type Log { # EIP-2718 Access List type AccessTuple { - # access list address + # access list address. address: Address! - # access list storage keys, null if not present + # access list storage keys, null if not present. storageKeys: [Bytes32!] } +# EIP-4895 +type Withdrawal { + # Index is a monotonically increasing identifier issued by consensus layer. + index: Long! + + # Validator is index of the validator associated with withdrawal. + validator: Long! + + # Recipient address of the withdrawn amount. + address: Address! + + # Amount is the withdrawal value in Gwei. + amount: Long! +} + # Transaction is an Ethereum transaction. type Transaction { # Hash is the hash of this transaction. @@ -81,7 +98,7 @@ type Transaction { # Index is the index of this transaction in the parent block. This will # be null if the transaction has not yet been mined. - index: Int + index: Long # From is the account that sent this transaction - this will always be # an externally owned account. @@ -157,14 +174,31 @@ type Transaction { v: BigInt! # EIP 2718: envelope transaction support - type: Int + type: Long # EIP 2930: optional access list, null if not present accessList: [AccessTuple!] + # EIP-4844: blob gas a user willing to pay + maxFeePerBlobGas: Long + + # EIP-4844: represents a list of hash outputs from kzg_to_versioned_hash + versionedHashes: [Bytes32!] + + #--------------------------Extensions------------------------------- + # If type == 0, chainID returns null. # If type > 0, chainID returns replay protection chainID chainID: Long + + # Raw is the canonical encoding of the transaction. + # For legacy transactions, it returns the RLP encoding. + # For EIP-2718 typed transactions, it returns the type and payload. + raw: Bytes! + + # RawReceipt is the canonical encoding of the receipt. For post EIP-2718 typed transactions + # this is equivalent to TxType || ReceiptEncoding. + rawReceipt: Bytes! } # BlockFilterCriteria encapsulates log filter criteria for a filter applied @@ -207,7 +241,7 @@ type Block { # TransactionCount is the number of transactions in this block. if # transactions are not available for this block, this field will be null. - transactionCount: Int + transactionCount: Long # StateRoot is the keccak256 hash of the state trie after this block was processed. stateRoot: Bytes32! @@ -249,7 +283,7 @@ type Block { # OmmerCount is the number of ommers (AKA uncles) associated with this # block. If ommers are unavailable, this field will be null. - ommerCount: Int + ommerCount: Long # Ommers is a list of ommer (AKA uncle) blocks associated with this block. # If ommers are unavailable, this field will be null. Depending on your @@ -286,6 +320,32 @@ type Block { # EstimateGas estimates the amount of gas that will be required for # successful execution of a transaction at the current block's state. estimateGas(data: CallData!): Long! + + # WithdrawalsRoot is the withdrawals trie root in this block. + # If withdrawals are unavailable for this block, this field will be null. + withdrawalsRoot: Bytes32 + + # Withdrawals is a list of withdrawals associated with this block. If + # withdrawals are unavailable for this block, this field will be null. + withdrawals: [Withdrawal!] + + # EIP-4844: is the total amount of blob gas consumed by the transactions + # within the block. + blobGasUsed: Long + + # EIP-4844: is a running total of blob gas consumed in excess of the target, + # prior to the block. Blocks with above-target blob gas consumption increase + # this value, blocks with below-target blob gas consumption decrease it + # (bounded at 0). + excessBlobGas: Long + + #--------------------------Extensions------------------------------- + + # RawHeader is the RLP encoding of the block's header. + rawHeader: Bytes! + + # Raw is the RLP encoding of the block. + raw: Bytes! } # CallData represents the data associated with a local contract call. @@ -379,7 +439,7 @@ type SyncState{ # Pending represents the current pending state. type Pending { # TransactionCount is the number of transactions in the pending state. - transactionCount: Int! + transactionCount: Long! # Transactions is a list of transactions in the current pending state. transactions: [Transaction!] diff --git a/nimbus/stateless_runner.nim b/nimbus/stateless_runner.nim index 156839045..60c6819b0 100644 --- a/nimbus/stateless_runner.nim +++ b/nimbus/stateless_runner.nim @@ -26,7 +26,7 @@ proc createVmStateForStatelessMode*(com: CommonRef, header: BlockHeader, body: B parentHeader: BlockHeader, asyncFactory: AsyncOperationFactory): Result[BaseVMState, string] {.inline.} = let vmState = BaseVMState() - if not vmState.statelessInit(parentHeader, header, com, asyncFactory, {}): + if not vmState.statelessInit(parentHeader, header, com, asyncFactory): return err("Cannot initialise VmState for block number " & $(header.blockNumber)) waitFor(ifNecessaryGetAccounts(vmState, coinbasesOfThisBlockAndUncles(header, body))) ok(vmState) diff --git a/nimbus/sync/skeleton.nim b/nimbus/sync/does-not-compile/skeleton.nim similarity index 100% rename from nimbus/sync/skeleton.nim rename to nimbus/sync/does-not-compile/skeleton.nim diff --git a/nimbus/tracer.nim b/nimbus/tracer.nim index 3eec3433c..ee72e041a 100644 --- a/nimbus/tracer.nim +++ b/nimbus/tracer.nim @@ -3,6 +3,7 @@ import ./common/common, ./db/[accounts_cache, capturedb], ./utils/utils, + ./evm/tracer/legacy_tracer, "."/[constants, vm_state, vm_types, transaction, core/executor], nimcrypto/utils as ncrutils, ./rpc/hexstrings, ./launcher, @@ -90,9 +91,9 @@ proc traceTransaction*(com: CommonRef, header: BlockHeader, memoryDB = newMemoryDB() captureDB = newCaptureDB(com.db.db, memoryDB) captureTrieDB = trieDB captureDB - captureFlags = tracerFlags + {EnableAccount} + tracerInst = newLegacyTracer(tracerFlags) captureCom = com.clone(captureTrieDB) - vmState = BaseVMState.new(header, captureCom, captureFlags) + vmState = BaseVMState.new(header, captureCom) var stateDb = vmState.stateDB @@ -115,7 +116,7 @@ proc traceTransaction*(com: CommonRef, header: BlockHeader, let recipient = tx.getRecipient(sender) if idx == txIndex: - vmState.enableTracing() + vmState.tracer = tracerInst # only enable tracer on target tx before.captureAccount(stateDb, sender, senderName) before.captureAccount(stateDb, recipient, recipientName) before.captureAccount(stateDb, miner, minerName) @@ -130,20 +131,20 @@ proc traceTransaction*(com: CommonRef, header: BlockHeader, after.captureAccount(stateDb, sender, senderName) after.captureAccount(stateDb, recipient, recipientName) after.captureAccount(stateDb, miner, minerName) - vmState.removeTracedAccounts(sender, recipient, miner) + tracerInst.removeTracedAccounts(sender, recipient, miner) stateDb.persist() stateDiff["afterRoot"] = %($stateDb.rootHash) break # internal transactions: var stateBefore = AccountsCache.init(captureTrieDB, beforeRoot, com.pruneTrie) - for idx, acc in tracedAccountsPairs(vmState): + for idx, acc in tracedAccountsPairs(tracerInst): before.captureAccount(stateBefore, acc, internalTxName & $idx) - for idx, acc in tracedAccountsPairs(vmState): + for idx, acc in tracedAccountsPairs(tracerInst): after.captureAccount(stateDb, acc, internalTxName & $idx) - result = vmState.getTracingResult() + result = tracerInst.getTracingResult() result["gas"] = %gasUsed if TracerFlags.DisableStateDiff notin tracerFlags: @@ -161,8 +162,9 @@ proc dumpBlockState*(com: CommonRef, header: BlockHeader, body: BlockBody, dumpS captureTrieDB = trieDB captureDB captureCom = com.clone(captureTrieDB) # we only need stack dump if we want to scan for internal transaction address - captureFlags = {EnableTracing, DisableMemory, DisableStorage, EnableAccount} - vmState = BaseVMState.new(header, captureCom, captureFlags) + captureFlags = {DisableMemory, DisableStorage, EnableAccount} + tracerInst = newLegacyTracer(captureFlags) + vmState = BaseVMState.new(header, captureCom, tracerInst) miner = vmState.coinbase() var @@ -190,20 +192,20 @@ proc dumpBlockState*(com: CommonRef, header: BlockHeader, body: BlockBody, dumpS let recipient = tx.getRecipient(sender) after.captureAccount(stateAfter, sender, senderName & $idx) after.captureAccount(stateAfter, recipient, recipientName & $idx) - vmState.removeTracedAccounts(sender, recipient) + tracerInst.removeTracedAccounts(sender, recipient) after.captureAccount(stateAfter, miner, minerName) - vmState.removeTracedAccounts(miner) + tracerInst.removeTracedAccounts(miner) for idx, uncle in body.uncles: after.captureAccount(stateAfter, uncle.coinbase, uncleName & $idx) - vmState.removeTracedAccounts(uncle.coinbase) + tracerInst.removeTracedAccounts(uncle.coinbase) # internal transactions: - for idx, acc in tracedAccountsPairs(vmState): + for idx, acc in tracedAccountsPairs(tracerInst): before.captureAccount(stateBefore, acc, internalTxName & $idx) - for idx, acc in tracedAccountsPairs(vmState): + for idx, acc in tracedAccountsPairs(tracerInst): after.captureAccount(stateAfter, acc, internalTxName & $idx) result = %{"before": before, "after": after} @@ -218,8 +220,8 @@ proc traceBlock*(com: CommonRef, header: BlockHeader, body: BlockBody, tracerFla captureDB = newCaptureDB(com.db.db, memoryDB) captureTrieDB = trieDB captureDB captureCom = com.clone(captureTrieDB) - captureFlags = tracerFlags + {EnableTracing} - vmState = BaseVMState.new(header, captureCom, captureFlags) + tracerInst = newLegacyTracer(tracerFlags) + vmState = BaseVMState.new(header, captureCom, tracerInst) if header.txRoot == EMPTY_ROOT_HASH: return newJNull() doAssert(body.transactions.calcTxRoot == header.txRoot) @@ -234,7 +236,7 @@ proc traceBlock*(com: CommonRef, header: BlockHeader, body: BlockBody, tracerFla if rc.isOk: gasUsed = gasUsed + rc.value - result = vmState.getTracingResult() + result = tracerInst.getTracingResult() result["gas"] = %gasUsed if TracerFlags.DisableState notin tracerFlags: diff --git a/nimbus/transaction/call_common.nim b/nimbus/transaction/call_common.nim index ebd64ca78..b9378d74b 100644 --- a/nimbus/transaction/call_common.nim +++ b/nimbus/transaction/call_common.nim @@ -191,6 +191,10 @@ proc setupHost(call: CallParams): TransactionHost = let cMsg = hostToComputationMessage(host.msg) host.computation = newComputation(vmState, cMsg) + vmState.captureStart(host.computation, call.sender, call.to, + call.isCreate, call.input, + call.gasLimit, call.value) + return host when defined(evmc_enabled): @@ -242,7 +246,7 @@ proc prepareToRunComputation(host: TransactionHost, call: CallParams) = # EIP-4844 if fork >= FkCancun: let blobFee = calcDataFee(call.versionedHashes.len, - vmState.parent.excessDataGas).GasInt + vmState.parent.excessBlobGas).GasInt db.subBalance(call.sender, blobFee.u256) proc calculateAndPossiblyRefundGas(host: TransactionHost, call: CallParams): GasInt = @@ -273,7 +277,13 @@ proc finishRunningComputation(host: TransactionHost, call: CallParams): CallResu let gasRemaining = calculateAndPossiblyRefundGas(host, call) # evm gas used without intrinsic gas - host.vmState.tracerGasUsed(host.msg.gas - gasRemaining) + let gasUsed = host.msg.gas - gasRemaining + let error = if c.isError: + some(c.error.info) + else: + none(string) + + host.vmState.captureEnd(c.output, gasUsed, error) result.isError = c.isError result.gasUsed = call.gasLimit - gasRemaining diff --git a/nimbus/utils/debug.nim b/nimbus/utils/debug.nim index 443c1a287..141730af3 100644 --- a/nimbus/utils/debug.nim +++ b/nimbus/utils/debug.nim @@ -51,10 +51,10 @@ proc debug*(h: BlockHeader): string = result.add "fee : " & $h.fee.get() & "\n" if h.withdrawalsRoot.isSome: result.add "withdrawalsRoot: " & $h.withdrawalsRoot.get() & "\n" - if h.dataGasUsed.isSome: - result.add "dataGasUsed : " & $h.dataGasUsed.get() & "\n" - if h.excessDataGas.isSome: - result.add "excessDataGas : " & $h.excessDataGas.get() & "\n" + if h.blobGasUsed.isSome: + result.add "blobGasUsed : " & $h.blobGasUsed.get() & "\n" + if h.excessBlobGas.isSome: + result.add "excessBlobGas : " & $h.excessBlobGas.get() & "\n" result.add "blockHash : " & $blockHash(h) & "\n" proc dumpAccount(stateDB: AccountsCache, address: EthAddress): JsonNode = diff --git a/nimbus/vm_state.nim b/nimbus/vm_state.nim index c0f2e29c4..1bf9a5244 100644 --- a/nimbus/vm_state.nim +++ b/nimbus/vm_state.nim @@ -21,27 +21,30 @@ export vms.coinbase, vms.determineFork, vms.difficulty, - vms.disableTracing, - vms.enableTracing, - vms.tracingEnabled, vms.baseFee, vms.forkDeterminationInfoForVMState, vms.generateWitness, vms.`generateWitness=`, vms.getAncestorHash, vms.getAndClearLogEntries, - vms.getTracingResult, vms.init, vms.statelessInit, vms.mutateStateDB, vms.new, vms.reinit, vms.readOnlyStateDB, - vms.removeTracedAccounts, vms.status, vms.`status=`, - vms.tracedAccounts, - vms.tracedAccountsPairs, - vms.tracerGasUsed + vms.tracingEnabled, + vms.captureTxStart, + vms.captureTxEnd, + vms.captureStart, + vms.captureEnd, + vms.captureEnter, + vms.captureExit, + vms.captureOpStart, + vms.captureOpEnd, + vms.captureFault, + vms.capturePrepare # End diff --git a/nimbus/vm_types.nim b/nimbus/vm_types.nim index 4932ea4dd..ff63025ca 100644 --- a/nimbus/vm_types.nim +++ b/nimbus/vm_types.nim @@ -20,7 +20,7 @@ export vmt.Message, vmt.MsgFlags, vmt.TracerFlags, - vmt.TransactionTracer, + vmt.TracerRef, vmt.VMFlag when defined(evmc_enabled): diff --git a/stateless/json_from_tree.nim b/stateless/does-not-compile/json_from_tree.nim similarity index 100% rename from stateless/json_from_tree.nim rename to stateless/does-not-compile/json_from_tree.nim diff --git a/stateless/json_witness_gen.nim b/stateless/does-not-compile/json_witness_gen.nim similarity index 100% rename from stateless/json_witness_gen.nim rename to stateless/does-not-compile/json_witness_gen.nim diff --git a/tests/fixtures/TracerTests/block46402.json b/tests/fixtures/TracerTests/block46402.json index 90514db11..06ee8ce70 100644 --- a/tests/fixtures/TracerTests/block46402.json +++ b/tests/fixtures/TracerTests/block46402.json @@ -283,7 +283,7 @@ "0000000000000000000000000000000000000000000000000000000000000000", "0000000000000000000000000000000000000000000000000000000000000060" ], - "error": "Opcode Dispatch Error msg=Out of gas: Needed 20000 - Remaining 412 - Reason: SSTORE: 9a049f5d18c239efaa258af9f3e7002949a977a0[0] -> 924236965777326770894530693462975209021625492404 (0), depth=0", + "error": "Opcode Dispatch Error msg=Out of gas: Needed 20000 - Remaining 412 - Reason: SSTORE, depth=0", "gasCost": 0 } ], @@ -705,7 +705,7 @@ "0000000000000000000000000000000000000000000000000000000000000000", "0000000000000000000000000000000000000000000000000000000000000060" ], - "error": "Opcode Dispatch Error msg=Out of gas: Needed 20000 - Remaining 412 - Reason: SSTORE: 9a049f5d18c239efaa258af9f3e7002949a977a0[0] -> 924236965777326770894530693462975209021625492404 (0), depth=0", + "error": "Opcode Dispatch Error msg=Out of gas: Needed 20000 - Remaining 412 - Reason: SSTORE, depth=0", "gasCost": 0 } ] diff --git a/tests/fixtures/eth_tests b/tests/fixtures/eth_tests index 55a6618e8..06e276776 160000 --- a/tests/fixtures/eth_tests +++ b/tests/fixtures/eth_tests @@ -1 +1 @@ -Subproject commit 55a6618e8eca201456ff82ce08a37eafb090f0f5 +Subproject commit 06e276776bc87817c38f6efb492bf6f4527fa904 diff --git a/tests/graphql/README.md b/tests/graphql/README.md deleted file mode 100644 index 87a3f9d30..000000000 --- a/tests/graphql/README.md +++ /dev/null @@ -1,5 +0,0 @@ -per protocool folders apply, e.g. - -- eth65/queries.toml (obsoleted *eth65* test specs) -- eth66/queries.toml -- ... diff --git a/tests/graphql/eth66/queries.toml b/tests/graphql/queries.toml similarity index 91% rename from tests/graphql/eth66/queries.toml rename to tests/graphql/queries.toml index ce605ea1d..4299c6278 100644 --- a/tests/graphql/eth66/queries.toml +++ b/tests/graphql/queries.toml @@ -23,6 +23,7 @@ {"name":"Address"}, {"name":"Block"}, {"name":"CallResult"}, + {"name":"Withdrawal"}, {"name":"Query"}, {"name":"Boolean"}, {"name":"FilterCriteria"}, @@ -106,38 +107,38 @@ """ result = """ { - "chainID":1, + "chainID":"0x1", "block":{ "__typename":"Block", - "number":3, + "number":"0x3", "hash":"0x72d69cc3c74c2740c27f4becae6c9f9fc524c702a9a48d23c564b13dce5fe0a1", "parent":{ "__typename":"Block", - "number":2 + "number":"0x2" }, "nonce":"0x0000000000000000", "transactionsRoot":"0x8c8775e959d553f9f991a21a8502dbe6819c53c518711f2cbbbd7ed998f65647", - "transactionCount":1, + "transactionCount":"0x1", "stateRoot":"0x81b1df384897709e96a4b6319fda4e8f7682a1c9d753b1fb772ed4fb19f1e443", "receiptsRoot":"0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", "miner":{ "__typename":"Account", "address":"0x8888f1f195afa192cfee860698584c030f4c9db1", "balance":"0x542253a12a8f8dc0", - "transactionCount":0, + "transactionCount":"0x0", "code":"0x", "storage":"0x0000000000000000000000000000000000000000000000000000000000000000" }, "extraData":"0x42", - "gasLimit":3141592, + "gasLimit":"0x2fefd8", "baseFeePerGas":null, - "gasUsed":21000, + "gasUsed":"0x5208", "timestamp":"0x54c99839", "logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000", "difficulty":"0x20000", "totalDifficulty":"0x80000", - "ommerCount":1, + "ommerCount":"0x1", "ommers":[ { "__typename":"Block", @@ -154,7 +155,7 @@ "__typename":"Account", "address":"0x095e7baea6a6c7c4c2dfeb977efac326af552d87", "balance":"0x1e", - "transactionCount":0, + "transactionCount":"0x0", "code":"0x", "storage":"0x0000000000000000000000000000000000000000000000000000000000000000" }, @@ -226,8 +227,8 @@ { "__typename":"Transaction", "hash":"0xbb8e2ffb7276bc688c0305d5c35ae219cf036ac7a3b058ffa9d32275cc75f31b", - "nonce":0, - "index":0, + "nonce":"0x0", + "index":"0x0", "from":{ "__typename":"Account", "address":"0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" @@ -238,21 +239,21 @@ }, "value":"0xa", "gasPrice":"0x3e8", - "gas":314159, + "gas":"0x4cb2f", "inputData":"0x", "block":{ "__typename":"Block", - "number":1 + "number":"0x1" }, - "status":1, - "gasUsed":21000, - "cumulativeGasUsed":21000, + "status":"0x1", + "gasUsed":"0x5208", + "cumulativeGasUsed":"0x5208", "createdContract":null, "logs":[], "r":"0x25d49a54362b5ae38cf895fa9a1d3ded6f7d5577e572c9a93cdebff6e33ceaf7", "s":"0x773806df18e22db29acde1dd96c0418e28738af7f520e5e2c5c673494029e5", "v":"0x1b", - "type":0, + "type":"0x0", "accessList":null, "maxFeePerGas":null, "maxPriorityFeePerGas":null, @@ -283,7 +284,7 @@ "block":{ "__typename":"Block", "hash":"0x72d69cc3c74c2740c27f4becae6c9f9fc524c702a9a48d23c564b13dce5fe0a1", - "number":3 + "number":"0x3" } } """ @@ -304,7 +305,7 @@ "block":{ "__typename":"Block", "hash":"0x72d69cc3c74c2740c27f4becae6c9f9fc524c702a9a48d23c564b13dce5fe0a1", - "number":3 + "number":"0x3" } } """ @@ -361,9 +362,9 @@ { "syncing":{ "__typename":"SyncState", - "startingBlock":0, - "currentBlock":3, - "highestBlock":0, + "startingBlock":"0x0", + "currentBlock":"0x3", + "highestBlock":"0x0", "pulledStates":null, "knownStates":null } @@ -386,22 +387,22 @@ "blocks":[ { "__typename":"Block", - "number":0, + "number":"0x0", "hash":"0x2b253498ad5e63a16978753398bad1fde371a3e513438297b52d65dc98e1db29" }, { "__typename":"Block", - "number":1, + "number":"0x1", "hash":"0xa4b71270e83c38d941d61fc2d8f3842f98a83203b82c3dea3176f1feb5a67b17" }, { "__typename":"Block", - "number":2, + "number":"0x2", "hash":"0x579d2bdb721bde7a6bc56b53f4962a58493ee11482e8f738702ddb3d65888a74" }, { "__typename":"Block", - "number":3, + "number":"0x3", "hash":"0x72d69cc3c74c2740c27f4becae6c9f9fc524c702a9a48d23c564b13dce5fe0a1" } ] @@ -426,7 +427,7 @@ result = """ { "block":{ - "estimateGas":21000 + "estimateGas":"0x5208" } } """ @@ -457,8 +458,8 @@ "call":{ "__typename":"CallResult", "data":"0x", - "gasUsed":21000, - "status":1 + "gasUsed":"0x5208", + "status":"0x1" } } } @@ -471,24 +472,8 @@ mutation { sendRawTransaction(data: "0xf86080018304cb2f94095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba077c7cd36820c71821c1aed59de46e70e701c4a8dd89c9ba508ab722210f60da8a03f29825d40c7c3f7bff3ca69267e0f3fb74b2d18b8c2c4e3c135b5d3b06e288d") } """ - result = """ -{ - "sendRawTransaction":"0x4ffa559ae277813fb886d1fa8743a590ba6b699f9893de99f5d1bcea9620b278" -} -""" - -[[units]] - name = "query.protocolVersion" - code = """ -{ - protocolVersion -} -""" - result = """ -{ - "protocolVersion":66 -} -""" + errors = ["[2, 3]: Fatal: Field 'sendRawTransaction' cannot be resolved: \"transaction rejected by txpool\": @[\"sendRawTransaction\"]"] + result = """null""" [[units]] name = "query.block(number) logs" diff --git a/tests/replay/pp.nim b/tests/replay/pp.nim index 2d2718c11..7fe6486bf 100644 --- a/tests/replay/pp.nim +++ b/tests/replay/pp.nim @@ -56,8 +56,8 @@ proc pp*(h: BlockHeader; sep = " "): string = &"stateRoot={h.stateRoot.pp}{sep}" & &"baseFee={h.baseFee}{sep}" & &"withdrawalsRoot={h.withdrawalsRoot.get(EMPTY_ROOT_HASH)}{sep}" & - &"dataGasUsed={h.dataGasUsed.get(0'u64)}" & - &"excessDataGas={h.excessDataGas.get(0'u64)}" + &"blobGasUsed={h.blobGasUsed.get(0'u64)}" & + &"excessBlobGas={h.excessBlobGas.get(0'u64)}" proc pp*(g: Genesis; sep = " "): string = "" & diff --git a/tests/test_blockchain_json.nim b/tests/test_blockchain_json.nim index b58316ebc..b7a5864d6 100644 --- a/tests/test_blockchain_json.nim +++ b/tests/test_blockchain_json.nim @@ -18,6 +18,7 @@ import ../nimbus/[vm_state, vm_types, errors, constants], ../nimbus/db/accounts_cache, ../nimbus/utils/[utils, debug], + ../nimbus/evm/tracer/legacy_tracer, ../nimbus/core/[executor, validate, pow/header], ../stateless/[tree_from_witness, witness_types], ../tools/common/helpers as chp, @@ -190,12 +191,16 @@ proc importBlock(tester: var Tester, com: CommonRef, let parentHeader = com.db.getBlockHeader(tb.header.parentHash) let td = some(com.db.getScore(tb.header.parentHash)) com.hardForkTransition(tb.header.blockNumber, td, some(tb.header.timestamp)) + let tracerInst = if tester.trace: + newLegacyTracer({}) + else: + LegacyTracer(nil) tester.vmState = BaseVMState.new( parentHeader, tb.header, com, - (if tester.trace: {TracerFlags.EnableTracing} else: {}), + tracerInst, ) if validation: @@ -230,7 +235,8 @@ proc collectDebugData(tester: var Tester) = return let vmState = tester.vmState - let tracingResult = if tester.trace: vmState.getTracingResult() else: %[] + let tracerInst = LegacyTracer(vmState.tracer) + let tracingResult = if tester.trace: tracerInst.getTracingResult() else: %[] tester.debugData.add %{ "blockNumber": %($vmState.blockNumber), "structLogs": tracingResult, diff --git a/tests/test_custom_network.nim b/tests/test_custom_network.nim index ff4701417..56939c1a2 100644 --- a/tests/test_custom_network.nim +++ b/tests/test_custom_network.nim @@ -28,15 +28,15 @@ ## import - std/[distros, os], + std/os, + chronicles, + results, + unittest2, + ../nimbus/core/chain, # must be early (compilation annoyance) ../nimbus/config, ../nimbus/db/select_backend, - ../nimbus/core/chain, ../nimbus/common/common, - ./replay/[undump_blocks, pp], - chronicles, - stew/results, - unittest2 + ./replay/[undump_blocks, pp] type ReplaySession = object @@ -85,31 +85,7 @@ const termTotalDff: 20_000_000_000_000.u256, mergeFork: 1000, ttdReachedAt: 55127, - failBlockAt: 9999999) - -when not defined(linux): - const isUbuntu32bit = false -else: - # The `detectOs(Ubuntu)` directive is not Windows compatible, causes an - # error when running the system command `lsb_release -d` in the background. - let isUbuntu32bit = detectOs(Ubuntu) and int.sizeof == 4 - -let - # There is a problem with the Github/CI which results in spurious crashes - # when leaving the `runner()` if the persistent ChainDBRef initialisation - # was present. The Github/CI set up for Linux/i386 is - # - # Ubuntu 10.04.06 LTS - # with repo kernel 5.4.0-1065-azure (see 'uname -a') - # - # base OS architecture is amd64 - # with i386 foreign architecture - # - # nimbus binary is an - # ELF 32-bit LSB shared object, - # Intel 80386, version 1 (SYSV), dynamically linked, - # - disablePersistentDB = isUbuntu32bit + failBlockAt: 1000) # Kludge, some change at the `merge` logic? # Block chains shared between test suites var @@ -170,9 +146,8 @@ proc setErrorLevel = # ------------------------------------------------------------------------------ proc ddbCleanUp(dir: string) = - if not disablePersistentDB: - ddbDir = dir - dir.flushDbDir + ddbDir = dir + dir.flushDbDir proc ddbCleanUp = ddbDir.ddbCleanUp @@ -195,14 +170,15 @@ proc importBlocks(c: ChainRef; h: seq[BlockHeader]; b: seq[BlockBody]; bRng = if 1 < h.len: &"s [#{first}..#{last}]={h.len}" else: &" #{first}" blurb = &"persistBlocks([#{first}..#" - noisy.say "***", &"block{bRng} #txs={nTxs} #uncles={nUnc}" - catchException("persistBlocks()", trace = true): if c.persistBlocks(h, b).isOk: + noisy.say "***", &"block{bRng} #txs={nTxs} #uncles={nUnc}" if not tddOk and c.com.ttdReached: noisy.say "***", &"block{bRng} => tddReached" return true + noisy.say "***", &"block{bRng} #txs={nTxs} #uncles={nUnc} -- failed" + # ------------------------------------------------------------------------------ # Test Runner # ------------------------------------------------------------------------------ @@ -216,8 +192,7 @@ proc genesisLoadRunner(noisy = true; gFileInfo = sSpcs.genesisFile.splitFile.name.split(".")[0] gFilePath = sSpcs.genesisFile.findFilePath.value - tmpDir = if disablePersistentDB: "*notused*" - else: gFilePath.splitFile.dir / "tmp" + tmpDir = gFilePath.splitFile.dir / "tmp" persistPruneInfo = if persistPruneTrie: "pruning enabled" else: "no pruning" @@ -240,24 +215,21 @@ proc genesisLoadRunner(noisy = true; check mcom.toHardFork(sSpcs.mergeFork.toBlockNumber.blockNumberToForkDeterminationInfo) == MergeFork test &"Construct persistent ChainDBRef on {tmpDir}, {persistPruneInfo}": - if disablePersistentDB: - skip() - else: - # Before allocating the database, the data directory needs to be - # cleared. There might be left overs from a previous crash or - # because there were file locks under Windows which prevented a - # previous clean up. - tmpDir.ddbCleanUp - - # Constructor ... - dcom = CommonRef.new( - tmpDir.newChainDB.trieDB, - networkId = params.config.chainId.NetworkId, - pruneTrie = persistPruneTrie, - params = params) - - check dcom.ttd.get == sSpcs.termTotalDff - check dcom.toHardFork(sSpcs.mergeFork.toBlockNumber.blockNumberToForkDeterminationInfo) == MergeFork + # Before allocating the database, the data directory needs to be + # cleared. There might be left overs from a previous crash or + # because there were file locks under Windows which prevented a + # previous clean up. + tmpDir.ddbCleanUp + + # Constructor ... + dcom = CommonRef.new( + tmpDir.newChainDB.trieDB, + networkId = params.config.chainId.NetworkId, + pruneTrie = persistPruneTrie, + params = params) + + check dcom.ttd.get == sSpcs.termTotalDff + check dcom.toHardFork(sSpcs.mergeFork.toBlockNumber.blockNumberToForkDeterminationInfo) == MergeFork test "Initialise in-memory Genesis": mcom.initializeEmptyDb @@ -270,18 +242,15 @@ proc genesisLoadRunner(noisy = true; check storedhHeaderPP == onTheFlyHeaderPP test "Initialise persistent Genesis": - if disablePersistentDB: - skip() - else: - dcom.initializeEmptyDb + dcom.initializeEmptyDb - # Must be the same as the in-memory DB value - check dcom.db.getBlockHash(0.u256) == mcom.db.getBlockHash(0.u256) + # Must be the same as the in-memory DB value + check dcom.db.getBlockHash(0.u256) == mcom.db.getBlockHash(0.u256) - let - storedhHeaderPP = dcom.db.getBlockHeader(0.u256).pp - onTheFlyHeaderPP = dcom.genesisHeader.pp - check storedhHeaderPP == onTheFlyHeaderPP + let + storedhHeaderPP = dcom.db.getBlockHeader(0.u256).pp + onTheFlyHeaderPP = dcom.genesisHeader.pp + check storedhHeaderPP == onTheFlyHeaderPP proc testnetChainRunner(noisy = true; @@ -386,6 +355,7 @@ when isMainModule: # typically on the `nimbus-eth1-blobs` module. noisy.testnetChainRunner( stopAfterBlock = 999999999) + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/tests/test_eip4844.nim b/tests/test_eip4844.nim index 4b312f9da..40c16d346 100644 --- a/tests/test_eip4844.nim +++ b/tests/test_eip4844.nim @@ -119,7 +119,7 @@ proc tx7(i: int): Transaction = maxFee: 10.GasInt, accessList: accesses, versionedHashes: @[digest], - maxFeePerDataGas: 10000000.GasInt, + maxFeePerBlobGas: 10000000.GasInt, ) proc tx8(i: int): Transaction = @@ -136,7 +136,7 @@ proc tx8(i: int): Transaction = maxFee: 10.GasInt, accessList: accesses, versionedHashes: @[digest], - maxFeePerDataGas: 10000000.GasInt, + maxFeePerBlobGas: 10000000.GasInt, ) proc privKey(keyHex: string): PrivateKey = diff --git a/tests/test_generalstate_json.nim b/tests/test_generalstate_json.nim index 477ea9f9d..899231891 100644 --- a/tests/test_generalstate_json.nim +++ b/tests/test_generalstate_json.nim @@ -14,6 +14,7 @@ import ../nimbus/db/accounts_cache, ../nimbus/common/common, ../nimbus/utils/[utils, debug], + ../nimbus/evm/tracer/legacy_tracer, ../tools/common/helpers as chp, ../tools/evmstate/helpers, ../tools/common/state_clearing, @@ -50,7 +51,8 @@ method getAncestorHash*(vmState: BaseVMState; blockNumber: BlockNumber): Hash256 return keccakHash(toBytes($blockNumber)) proc dumpDebugData(tester: Tester, vmState: BaseVMState, gasUsed: GasInt, success: bool) = - let tracingResult = if tester.trace: vmState.getTracingResult() else: %[] + let tracerInst = LegacyTracer(vmState.tracer) + let tracingResult = if tester.trace: tracerInst.getTracingResult() else: %[] let debugData = %{ "gasUsed": %gasUsed, "structLogs": tracingResult, @@ -63,12 +65,16 @@ proc testFixtureIndexes(tester: Tester, testStatusIMPL: var TestStatus) = let com = CommonRef.new(newMemoryDB(), tester.chainConfig, getConfiguration().pruning) parent = BlockHeader(stateRoot: emptyRlpHash) + tracer = if tester.trace: + newLegacyTracer({}) + else: + LegacyTracer(nil) let vmState = BaseVMState.new( - parent = parent, - header = tester.header, - com = com, - tracerFlags = (if tester.trace: {TracerFlags.EnableTracing} else: {}), + parent = parent, + header = tester.header, + com = com, + tracer = tracer, ) var gasUsed: GasInt diff --git a/tests/test_graphql.nim b/tests/test_graphql.nim index d608386d3..6786f4aef 100644 --- a/tests/test_graphql.nim +++ b/tests/test_graphql.nim @@ -8,7 +8,7 @@ # those terms. import - std/[os, json], + std/[json], stew/byteutils, eth/[p2p, rlp], graphql, ../nimbus/graphql/ethapi, graphql/test_common, @@ -25,8 +25,8 @@ type uncles: seq[BlockHeader] const - caseFolder = "tests" / "graphql" / "eth" & $ethVersion - dataFolder = "tests" / "fixtures" / "eth_tests" / "BlockchainTests" / "ValidBlocks" / "bcUncleTest" + caseFolder = "tests/graphql" + dataFolder = "tests/fixtures/eth_tests/BlockchainTests/ValidBlocks/bcUncleTest" proc toBlock(n: JsonNode, key: string): EthBlock = let rlpBlob = hexToSeqByte(n[key].str) @@ -43,7 +43,7 @@ proc setupChain(): CommonRef = berlinBlock : some(10.toBlockNumber) ) - var jn = json.parseFile(dataFolder / "oneUncle.json") + var jn = json.parseFile(dataFolder & "/oneUncle.json") for k, v in jn: if v["network"].str == "Istanbul": jn = v diff --git a/tools/evmstate/config.nim b/tools/evmstate/config.nim index e6a9badb8..f542306ea 100644 --- a/tools/evmstate/config.nim +++ b/tools/evmstate/config.nim @@ -44,7 +44,7 @@ type disableStorage* {. desc: "disable storage output" - defaultValue: false + defaultValue: true name: "nostorage" }: bool disableReturnData* {. diff --git a/tools/evmstate/evmstate.nim b/tools/evmstate/evmstate.nim index 31dc7e284..ee69ceb9a 100644 --- a/tools/evmstate/evmstate.nim +++ b/tools/evmstate/evmstate.nim @@ -9,7 +9,7 @@ # according to those terms. import - std/[json, strutils, sets, tables, options], + std/[json, strutils, sets, tables, options, streams], chronicles, eth/keys, stew/[results, byteutils], @@ -20,6 +20,7 @@ import ../../nimbus/transaction, ../../nimbus/core/executor, ../../nimbus/common/common, + ../../nimbus/evm/tracer/json_tracer, ../common/helpers as chp, "."/[config, helpers], ../common/state_clearing @@ -166,49 +167,29 @@ proc dumpState(vmState: BaseVMState): StateDump = accounts: dumpAccounts(vmState.stateDB) ) -template stripLeadingZeros(value: string): string = - var cidx = 0 - # ignore the last character so we retain '0' on zero value - while cidx < value.len - 1 and value[cidx] == '0': - cidx.inc - value[cidx .. ^1] - -proc encodeHexInt(x: SomeInteger): JsonNode = - %("0x" & x.toHex.stripLeadingZeros.toLowerAscii) - -proc writeTraceToStderr(vmState: BaseVMState, pretty: bool) = - let trace = vmState.getTracingResult() - trace["gasUsed"] = encodeHexInt(vmState.tracerGasUsed) - trace.delete("gas") +proc writeRootHashToStderr(vmState: BaseVMState) = let stateRoot = %{ "stateRoot": %(vmState.readOnlyStateDB.rootHash) } - if pretty: - stderr.writeLine(trace.pretty) - else: - let logs = trace["structLogs"] - trace.delete("structLogs") - for x in logs: - if "error" in x: - trace["error"] = x["error"] - x.delete("error") - stderr.writeLine($x) - - stderr.writeLine($trace) - stderr.writeLine($stateRoot) + stderr.writeLine($stateRoot) proc runExecution(ctx: var StateContext, conf: StateConf, pre: JsonNode): StateResult = let com = CommonRef.new(newMemoryDB(), ctx.chainConfig, pruneTrie = false) parent = BlockHeader(stateRoot: emptyRlpHash) fork = com.toEVMFork(ctx.header.forkDeterminationInfoForHeader) + stream = newFileStream(stderr) + tracer = if conf.jsonEnabled: + newJSonTracer(stream, ctx.tracerFlags, conf.pretty) + else: + JsonTracer(nil) let vmState = TestVMState() vmState.init( - parent = parent, - header = ctx.header, - com = com, - tracerFlags = ctx.tracerFlags) + parent = parent, + header = ctx.header, + com = com, + tracer = tracer) var gasUsed: GasInt let sender = ctx.tx.getSender() @@ -229,7 +210,7 @@ proc runExecution(ctx: var StateContext, conf: StateConf, pre: JsonNode): StateR if conf.dumpEnabled: result.state = dumpState(vmState) if conf.jsonEnabled: - writeTraceToStderr(vmState, conf.pretty) + writeRootHashToStderr(vmState) let rc = vmState.processTransaction( ctx.tx, sender, ctx.header, fork) @@ -241,9 +222,7 @@ proc runExecution(ctx: var StateContext, conf: StateConf, pre: JsonNode): StateR proc toTracerFlags(conf: Stateconf): set[TracerFlags] = result = { - TracerFlags.DisableStateDiff, - TracerFlags.EnableTracing, - TracerFlags.GethCompatibility + TracerFlags.DisableStateDiff } if conf.disableMemory : result.incl TracerFlags.DisableMemory diff --git a/tools/evmstate/helpers.nim b/tools/evmstate/helpers.nim index d4c7e1333..d26052a08 100644 --- a/tools/evmstate/helpers.nim +++ b/tools/evmstate/helpers.nim @@ -113,7 +113,9 @@ proc parseHeader*(n: JsonNode): BlockHeader = timestamp : required(EthTime, "currentTimestamp"), stateRoot : emptyRlpHash, mixDigest : omitZero(Hash256, "currentRandom"), - fee : optional(UInt256, "currentBaseFee") + fee : optional(UInt256, "currentBaseFee"), + excessBlobGas: optional(uint64, "excessBlobGas"), + blobGasUsed: optional(uint64, "blobGasUsed") ) proc parseTx*(n: JsonNode, dataIndex, gasIndex, valueIndex: int): Transaction = @@ -128,7 +130,7 @@ proc parseTx*(n: JsonNode, dataIndex, gasIndex, valueIndex: int): Transaction = maxFee : omitZero(GasInt, "maxFeePerGas"), accessList: omitZero(AccessList, "accessLists", dataIndex), maxPriorityFee: omitZero(GasInt, "maxPriorityFeePerGas"), - maxFeePerDataGas: omitZero(GasInt, "maxFeePerDataGas"), + maxFeePerBlobGas: omitZero(GasInt, "maxFeePerBlobGas"), versionedHashes: omitZero(VersionedHashes, "blobVersionedHashes") ) diff --git a/tools/evmstate/readme.md b/tools/evmstate/readme.md index 6fabbb730..5912fd5fa 100644 --- a/tools/evmstate/readme.md +++ b/tools/evmstate/readme.md @@ -6,9 +6,10 @@ The `evmstate` tool to execute state test. There are few options to build `evmstate` tool like any other nimbus tools. -1. Use nimble to install dependencies and your system Nim compiler(version <= 1.6.0). +1. Use your system Nim compiler(v1.6.12) and git to install dependencies. ``` - $> nimble install -y --depsOnly + $> git submodule update --init --recursive + $> ./env.sh (run once to generate nimbus-build-system.paths) $> nim c -d:release tools/evmstate/evmstate $> nim c -r -d:release tools/evmstate/evmstate_test ``` diff --git a/tools/t8n/helpers.nim b/tools/t8n/helpers.nim index 0a8369c97..2af9ed9c0 100644 --- a/tools/t8n/helpers.nim +++ b/tools/t8n/helpers.nim @@ -179,8 +179,10 @@ proc parseEnv*(ctx: var TransContext, n: JsonNode) = optional(ctx.env, UInt256, parentBaseFee) optional(ctx.env, GasInt, parentGasUsed) optional(ctx.env, GasInt, parentGasLimit) - optional(ctx.env, uint64, parentDataGasUsed) - optional(ctx.env, uint64, parentExcessDataGas) + optional(ctx.env, uint64, currentBlobGasUsed) + optional(ctx.env, uint64, currentExcessBlobGas) + optional(ctx.env, uint64, parentBlobGasUsed) + optional(ctx.env, uint64, parentExcessBlobGas) if n.hasKey("blockHashes"): let w = n["blockHashes"] @@ -234,7 +236,7 @@ proc parseTx(n: JsonNode, chainId: ChainID): Transaction = required(tx, GasInt, maxPriorityFeePerGas) required(tx, GasInt, maxFeePerGas) omitZero(tx, AccessList, accessList) - required(tx, GasInt, maxFeePerDataGas) + required(tx, GasInt, maxFeePerBlobGas) required(tx, VersionedHashes, blobVersionedHashes) var eip155 = true @@ -440,7 +442,7 @@ proc `@@`*(x: ExecutionResult): JsonNode = result["currentBaseFee"] = @@(x.currentBaseFee) if x.withdrawalsRoot.isSome: result["withdrawalsRoot"] = @@(x.withdrawalsRoot) - if x.dataGasUsed.isSome: - result["dataGasUsed"] = @@(x.dataGasUsed) - if x.excessDataGas.isSome: - result["excessDataGas"] = @@(x.excessDataGas) + if x.blobGasUsed.isSome: + result["blobGasUsed"] = @@(x.blobGasUsed) + if x.excessBlobGas.isSome: + result["excessBlobGas"] = @@(x.excessBlobGas) diff --git a/tools/t8n/readme.md b/tools/t8n/readme.md index d24d959f0..10b947ede 100644 --- a/tools/t8n/readme.md +++ b/tools/t8n/readme.md @@ -6,9 +6,10 @@ The `t8n` tool is a stateless state transition utility. There are few options to build `t8n` tool like any other nimbus tools. -1. Use nimble to install dependencies and your system Nim compiler(version <= 1.6.0). +1. Use your system Nim compiler(v1.6.12) and git to install dependencies. ``` - $> nimble install -y --depsOnly + $> git submodule update --init --recursive + $> ./env.sh (run once to generate nimbus-build-system.paths) $> nim c -d:release -d:chronicles_default_output_device=stderr tools/t8n/t8n $> nim c -r -d:release tools/t8n/t8n_test ``` diff --git a/tools/t8n/testdata/00-517/env.json b/tools/t8n/testdata/00-517/env.json index b1f22a39b..b07d44f70 100644 --- a/tools/t8n/testdata/00-517/env.json +++ b/tools/t8n/testdata/00-517/env.json @@ -14,6 +14,6 @@ "amount": "0x2a" } ], - "parentDataGasUsed": "0x100", - "parentExcessDataGas": "0x100" + "parentBlobGasUsed": "0x100", + "parentExcessBlobGas": "0x100" } diff --git a/tools/t8n/testdata/00-517/exp.json b/tools/t8n/testdata/00-517/exp.json index 311609cb4..b27eabd64 100644 --- a/tools/t8n/testdata/00-517/exp.json +++ b/tools/t8n/testdata/00-517/exp.json @@ -16,7 +16,7 @@ "gasUsed": "0x0", "currentBaseFee": "0x500", "withdrawalsRoot": "0x4921c0162c359755b2ae714a0978a1dad2eb8edce7ff9b38b9b6fc4cbc547eb5", - "dataGasUsed": "0x0", - "excessDataGas": "0x0" + "blobGasUsed": "0x0", + "excessBlobGas": "0x0" } } diff --git a/tools/t8n/testdata/00-518/env.json b/tools/t8n/testdata/00-518/env.json index 8dd2e1b4d..acb30c5ad 100644 --- a/tools/t8n/testdata/00-518/env.json +++ b/tools/t8n/testdata/00-518/env.json @@ -6,8 +6,8 @@ "currentNumber" : "0x01", "currentRandom" : "0x0000000000000000000000000000000000000000000000000000000000020000", "currentTimestamp" : "0x03e8", - "parentDataGasUsed" : "0x2000", - "parentExcessDataGas" : "0x1000", + "parentBlobGasUsed" : "0x2000", + "parentExcessBlobGas" : "0x1000", "previousHash" : "0x5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6", "withdrawals": [] } diff --git a/tools/t8n/testdata/00-518/exp.json b/tools/t8n/testdata/00-518/exp.json index 5c794271b..636aeae9d 100644 --- a/tools/t8n/testdata/00-518/exp.json +++ b/tools/t8n/testdata/00-518/exp.json @@ -25,7 +25,7 @@ ], "currentBaseFee": "0x7", "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "dataGasUsed": "0x0", - "excessDataGas": "0x0" + "blobGasUsed": "0x0", + "excessBlobGas": "0x0" } } diff --git a/tools/t8n/transition.nim b/tools/t8n/transition.nim index e6caa3085..9f277b005 100644 --- a/tools/t8n/transition.nim +++ b/tools/t8n/transition.nim @@ -21,7 +21,8 @@ import ../../nimbus/core/pow/difficulty, ../../nimbus/core/dao, ../../nimbus/core/executor/[process_transaction, executor_helpers], - ../../nimbus/core/eip4844 + ../../nimbus/core/eip4844, + ../../nimbus/evm/tracer/json_tracer import stew/byteutils const @@ -98,7 +99,9 @@ proc envToHeader(env: EnvStruct): BlockHeader = timestamp : env.currentTimestamp, stateRoot : emptyRlpHash, fee : env.currentBaseFee, - withdrawalsRoot: env.withdrawals.calcWithdrawalsRoot() + withdrawalsRoot: env.withdrawals.calcWithdrawalsRoot(), + blobGasUsed: env.currentBlobGasUsed, + excessBlobGas: env.currentExcessBlobGas ) proc postState(db: AccountsCache, alloc: var GenesisAlloc) = @@ -157,29 +160,36 @@ proc encodeHexInt(x: SomeInteger): JsonNode = proc toHex(x: Hash256): string = "0x" & x.data.toHex -proc dumpTrace(txIndex: int, txHash: Hash256, vmState: BaseVMstate) = - let txHash = "0x" & toLowerAscii($txHash) - let fName = "trace-$1-$2.jsonl" % [$txIndex, txHash] - let trace = vmState.getTracingResult() - var s = newFileStream(fName, fmWrite) - - trace["gasUsed"] = encodeHexInt(vmState.tracerGasUsed) - trace.delete("gas") - let stateRoot = %{ - "stateRoot": %(vmState.readOnlyStateDB.rootHash.toHex) +proc setupTrace(conf: T8NConf, txIndex: int, txHash: Hash256, vmState: BaseVMstate) = + var tracerFlags = { + TracerFlags.DisableMemory, + TracerFlags.DisableStorage, + TracerFlags.DisableState, + TracerFlags.DisableStateDiff, + TracerFlags.DisableReturnData } - let logs = trace["structLogs"] - trace.delete("structLogs") - for x in logs: - if "error" in x: - trace["error"] = x["error"] - x.delete("error") - s.writeLine($x) + if conf.traceEnabled: + if conf.traceMemory: tracerFlags.excl TracerFlags.DisableMemory + if conf.traceNostack: tracerFlags.incl TracerFlags.DisableStack + if conf.traceReturnData: tracerFlags.excl TracerFlags.DisableReturnData + + let + txHash = "0x" & toLowerAscii($txHash) + baseDir = if conf.outputBaseDir.len > 0: + conf.outputBaseDir + else: + "." + fName = "$1/trace-$2-$3.jsonl" % [baseDir, $txIndex, txHash] + stream = newFileStream(fName, fmWrite) + tracerInst = newJsonTracer(stream, tracerFlags, false) + + vmState.tracer = tracerInst - s.writeLine($trace) - s.writeLine($stateRoot) - s.close() +proc closeTrace(vmState: BaseVMstate) = + let tracer = JsonTracer(vmState.tracer) + if tracer.isNil.not: + tracer.close() func gwei(n: uint64): UInt256 = n.u256 * (10 ^ 9).u256 @@ -187,7 +197,8 @@ func gwei(n: uint64): UInt256 = proc exec(ctx: var TransContext, vmState: BaseVMState, stateReward: Option[UInt256], - header: BlockHeader): ExecOutput = + header: BlockHeader, + conf: T8NConf): ExecOutput = let txList = ctx.parseTxs(vmState.com.chainId) @@ -204,7 +215,7 @@ proc exec(ctx: var TransContext, vmState.receipts = newSeqOfCap[Receipt](txList.len) vmState.cumulativeGasUsed = 0 - var dataGasUsed = 0'u64 + var blobGasUsed = 0'u64 for txIndex, txRes in txList: if txRes.isErr: rejected.add RejectedTx( @@ -222,10 +233,13 @@ proc exec(ctx: var TransContext, ) continue + if conf.traceEnabled: + setupTrace(conf, txIndex, rlpHash(tx), vmState) + let rc = vmState.processTransaction(tx, sender, header) - if vmState.tracingEnabled: - dumpTrace(txIndex, rlpHash(tx), vmState) + if conf.traceEnabled: + closeTrace(vmState) if rc.isErr: rejected.add RejectedTx( @@ -241,7 +255,7 @@ proc exec(ctx: var TransContext, rec, tx, sender, txIndex, gasUsed ) includedTx.add tx - dataGasUsed += tx.getTotalDataGas + blobGasUsed += tx.getTotalBlobGas # Add mining reward? (-1 means rewards are disabled) if stateReward.isSome and stateReward.get >= 0: @@ -290,8 +304,8 @@ proc exec(ctx: var TransContext, ) if fork >= FkCancun: - result.result.dataGasUsed = some dataGasUsed - result.result.excessDataGas = some calcExcessDataGas(vmState.parent) + result.result.blobGasUsed = some blobGasUsed + result.result.excessBlobGas = some calcExcessBlobGas(vmState.parent) template wrapException(body: untyped) = when wrapExceptionEnabled: @@ -353,21 +367,6 @@ proc calcBaseFee(env: EnvStruct): UInt256 = proc transitionAction*(ctx: var TransContext, conf: T8NConf) = wrapException: - var tracerFlags = { - TracerFlags.DisableMemory, - TracerFlags.DisableStorage, - TracerFlags.DisableState, - TracerFlags.DisableStateDiff, - TracerFlags.DisableReturnData, - TracerFlags.GethCompatibility - } - - if conf.traceEnabled: - tracerFlags.incl TracerFlags.EnableTracing - if conf.traceMemory: tracerFlags.excl TracerFlags.DisableMemory - if conf.traceNostack: tracerFlags.incl TracerFlags.DisableStack - if conf.traceReturnData: tracerFlags.excl TracerFlags.DisableReturnData - if conf.inputAlloc.len == 0 and conf.inputEnv.len == 0 and conf.inputTxs.len == 0: raise newError(ErrorConfig, "either one of input is needeed(alloc, txs, or env)") @@ -411,8 +410,8 @@ proc transitionAction*(ctx: var TransContext, conf: T8NConf) = difficulty: ctx.env.parentDifficulty.get(0.u256), ommersHash: uncleHash, blockNumber: ctx.env.currentNumber - 1.toBlockNumber, - dataGasUsed: ctx.env.parentDataGasUsed, - excessDataGas: ctx.env.parentExcessDataGas + blobGasUsed: ctx.env.parentBlobGasUsed, + excessBlobGas: ctx.env.parentExcessBlobGas ) # Sanity check, to not `panic` in state_transition @@ -429,11 +428,11 @@ proc transitionAction*(ctx: var TransContext, conf: T8NConf) = raise newError(ErrorConfig, "Shanghai config but missing 'withdrawals' in env section") if com.isCancunOrLater(ctx.env.currentTimestamp): - if ctx.env.parentDataGasUsed.isNone: - raise newError(ErrorConfig, "Cancun config but missing 'parentDataGasUsed' in env section") + if ctx.env.parentBlobGasUsed.isNone: + raise newError(ErrorConfig, "Cancun config but missing 'parentBlobGasUsed' in env section") - if ctx.env.parentExcessDataGas.isNone: - raise newError(ErrorConfig, "Cancun config but missing 'parentExcessDataGas' in env section") + if ctx.env.parentExcessBlobGas.isNone: + raise newError(ErrorConfig, "Cancun config but missing 'parentExcessBlobGas' in env section") let res = loadKzgTrustedSetup() if res.isErr: @@ -472,15 +471,14 @@ proc transitionAction*(ctx: var TransContext, conf: T8NConf) = vmState.init( parent = parent, header = header, - com = com, - tracerFlags = (if conf.traceEnabled: tracerFlags else: {}) + com = com ) vmState.mutateStateDB: db.setupAlloc(ctx.alloc) db.persist(clearEmptyAccount = false, clearCache = false) - let res = exec(ctx, vmState, conf.stateReward, header) + let res = exec(ctx, vmState, conf.stateReward, header, conf) if vmState.hashError.len > 0: raise newError(ErrorMissingBlockhash, vmState.hashError) diff --git a/tools/t8n/types.nim b/tools/t8n/types.nim index f8feb55c5..48a127024 100644 --- a/tools/t8n/types.nim +++ b/tools/t8n/types.nim @@ -44,8 +44,10 @@ type parentGasUsed*: Option[GasInt] parentGasLimit*: Option[GasInt] withdrawals*: Option[seq[Withdrawal]] - parentDataGasUsed*: Option[uint64] - parentExcessDataGas*: Option[uint64] + currentBlobGasUsed*: Option[uint64] + currentExcessBlobGas*: Option[uint64] + parentBlobGasUsed*: Option[uint64] + parentExcessBlobGas*: Option[uint64] TxsType* = enum TxsNone @@ -94,8 +96,8 @@ type gasUsed*: GasInt currentBaseFee*: Option[UInt256] withdrawalsRoot*: Option[Hash256] - dataGasUsed*: Option[uint64] - excessDataGas*: Option[uint64] + blobGasUsed*: Option[uint64] + excessBlobGas*: Option[uint64] const ErrorEVM* = 2.T8NExitCode diff --git a/tools/txparse/readme.md b/tools/txparse/readme.md index 7bb8069b0..88b16e2a2 100644 --- a/tools/txparse/readme.md +++ b/tools/txparse/readme.md @@ -11,9 +11,10 @@ Otherwise, it outputs `err: ` and a suitable error message. There are few options to build `txparse` tool like any other nimbus tools. -1. Use nimble to install dependencies and your system Nim compiler(version <= 1.6.0). +1. Use your system Nim compiler(v1.6.12) and git to install dependencies. ``` - $> nimble install -y --depsOnly + $> git submodule update --init --recursive + $> ./env.sh (run once to generate nimbus-build-system.paths) $> nim c -d:release tools/txparse/txparse ``` 2. Use nimbus shipped Nim compiler and dependencies. diff --git a/vendor/nim-eth b/vendor/nim-eth index 26ae53959..2ed8e991b 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit 26ae539598e31efbaa016f6694b9a60ea08fc4b6 +Subproject commit 2ed8e991b543156dbf3faf0d000e1cfc168498d2 diff --git a/vendor/nim-web3 b/vendor/nim-web3 index 1f9fa11d0..04f56c593 160000 --- a/vendor/nim-web3 +++ b/vendor/nim-web3 @@ -1 +1 @@ -Subproject commit 1f9fa11d0e63c16aa4ec30e1f7328ae61254d7d0 +Subproject commit 04f56c593a035af1b7bebdc726543a2a73826412 diff --git a/vendor/nimbus-eth2 b/vendor/nimbus-eth2 index 970f5dfc4..b8a32419b 160000 --- a/vendor/nimbus-eth2 +++ b/vendor/nimbus-eth2 @@ -1 +1 @@ -Subproject commit 970f5dfc4e2df53d12ed71fc591e23ecc2bafb60 +Subproject commit b8a32419b8ffd030a14df0fee8863749a1832020