diff --git a/api.js b/api.js index 6d588230..c05146a2 100644 --- a/api.js +++ b/api.js @@ -423,7 +423,7 @@ app.post('/verify/:network/:codeHash', upload.single('package'), async (req, res let network = req.params["network"] let codeHash = req.params["codeHash"]; let publishSource = req.query.publishSource ? parseInt(req.query.publishSource, 10) : 1; - console.log("publishSource", publishSource); + let result = await query.postChainWASMContractVerification(network, codeHash, packageFile, signature, publishSource); if (result) { res.write(JSON.stringify(result)); diff --git a/index.js b/index.js index ae7c4e5d..7b82213d 100644 --- a/index.js +++ b/index.js @@ -1439,7 +1439,10 @@ app.get('/block/:chainID_or_chainName/:blockNumber', async (req, res) => { let [decorate, decorateExtra] = decorateOptUI(req) var b = await query.getBlock(chainID, blockNumber, blockHash, decorate, decorateExtra); if (b) { - let view = (chain.isEVM == 1 && b.evmBlock) ? 'evmBlock' : 'block'; + let view = "block"; + if ( chain.isEVM == 1 ) { + view = "evmBlock" + } res.render(view, { b: b, blockNumber: blockNumber, diff --git a/substrate/assetManager.js b/substrate/assetManager.js index 144768f6..94dc56e0 100644 --- a/substrate/assetManager.js +++ b/substrate/assetManager.js @@ -76,10 +76,18 @@ module.exports = class AssetManager extends PolkaholicDB { convertChainID(chainID_or_chainName) { let chainID = false let id = false + let evm_chainID = { + 1: "ethereum" , + 1284: "moonbeam", + 1285: "moonriver", + 592: "astar" + }; try { - chainID = parseInt(chainID_or_chainName, 10); - if (isNaN(chainID)) { - [chainID, id] = this.getChainIDByName(chainID_or_chainName) + chainID = parseInt(chainID_or_chainName, 10); + if (isNaN(chainID)) { + [chainID, id] = this.getChainIDByName(chainID_or_chainName) + } else if ( evm_chainID[chainID] !== undefined ) { + return [chainID, evm_chainID[chainID]] } else { [chainID, id] = this.getNameByChainID(chainID_or_chainName) } @@ -3140,4 +3148,4 @@ module.exports = class AssetManager extends PolkaholicDB { } -} \ No newline at end of file +} diff --git a/substrate/ethTool.js b/substrate/ethTool.js index b9994b6c..2203d83d 100644 --- a/substrate/ethTool.js +++ b/substrate/ethTool.js @@ -744,8 +744,9 @@ function decorateTxn(dTxn, dReceipt, dInternal, blockTS = false, chainID = false console.log(`decorateTxn: missing receipts`, dReceipt) return; } - if (dTxn.hash != dReceipt.transactionHash) { - console.log(`decorateTxn: txnHash mismatch (tx:${dTxn.hash}) vs (receipt: ${dReceipt.transactionHash})`, dTxn) + let dTxnTxHash = (dTxn.hash)? dTxn.hash: dTxn.transactionHash + if (dTxnTxHash != dReceipt.transactionHash) { + console.log(`decorateTxn: txnHash mismatch (tx:${dTxnTxHash}) vs (receipt: ${dReceipt.transactionHash})`, dTxn) return } //todo: how to detect reverted but successful case? @@ -790,7 +791,7 @@ function decorateTxn(dTxn, dReceipt, dInternal, blockTS = false, chainID = false } let fTxn = { chainID: (chainID) ? chainID : paraTool.dechexToInt(dTxn.chainId), - transactionHash: dTxn.hash, + transactionHash: (dTxn.hash)? dTxn.hash: dTxn.transactionHash, substrate: null, status: dReceipt.status, blockHash: dTxn.blockHash, @@ -1718,7 +1719,7 @@ function parseAbiSignature(abiStrArr) { } */ -async function fuseBlockTransactionReceipt(evmBlk, dTxns, dReceipts, dTrace, chainID) { +async function fuseBlockTransactionReceipt(evmBlk, dTxns, dReceipts, flatTraces, chainID) { let fBlk = evmBlk let blockTS = fBlk.timestamp let fTxns = [] @@ -1731,13 +1732,16 @@ async function fuseBlockTransactionReceipt(evmBlk, dTxns, dReceipts, dTrace, cha // recurse through all the trace records in dTrace, and for any with value, accumulate in feedTrace let feedtrace = [] let feedcreates = [] + let feedtraceMap = {}; + let traceCreateMap = {}; + /* if (dTrace) { process_evm_trace(dTrace, feedtrace, 0, [0], dTxns); - //console.log(`*** start process_evm_trace_creates`) + console.log(`*** start process_evm_trace_creates`) process_evm_trace_creates(dTrace, feedcreates, 0, [], dTxns); - //console.log(`*** done process_evm_trace_creates`, feedcreates) + console.log(`*** done process_evm_trace_creates`, feedcreates) } - let feedtraceMap = {}; + if (feedtrace.length > 0) { for (let t = 0; t < feedtrace.length; t++) { let internalTx = feedtrace[t]; @@ -1747,8 +1751,6 @@ async function fuseBlockTransactionReceipt(evmBlk, dTxns, dReceipts, dTrace, cha feedtraceMap[internalTx.transactionHash].push(internalTx); } } - - let traceCreateMap = {}; if (feedcreates.length > 0) { for (let t = 0; t < feedcreates.length; t++) { let createTx = feedcreates[t]; @@ -1758,7 +1760,29 @@ async function fuseBlockTransactionReceipt(evmBlk, dTxns, dReceipts, dTrace, cha traceCreateMap[createTx.transactionHash].push(createTx.to); } } + */ + if (flatTraces) { + for (const t of flatTraces) { + //check for create/create2 + let createsOps = ["create", "create2"] + if (createsOps.includes(t.trace_type)) { + console.log(`CONTRACT Create!`, t) + if (traceCreateMap[t.transaction_hash] == undefined) { + traceCreateMap[t.transaction_hash] = []; + } + let contractAddress = (t.to_address != undefined) ? t.to_address : t.to + traceCreateMap[t.transaction_hash].push(contractAddress); + } + //check for internal + if (t.value > 0 && trace_address_to_str(t.trace_address) != '') { + if (feedtraceMap[t.transaction_hash] == undefined) { + feedtraceMap[t.transaction_hash] = []; + } + feedtraceMap[t.transaction_hash].push(t); + } + } + } for (i = 0; i < dTxns.length; i += 1) { let dTxn = dTxns[i] let dReceipt = dReceipts[i] @@ -1792,6 +1816,62 @@ async function fuseBlockTransactionReceipt(evmBlk, dTxns, dReceipts, dTrace, cha return fBlk } +async function decorateEvmBlock(chainID, r){ + console.log('decorate_evm_block', r) + let contractABIs = this.contractABIs; + let contractABISignatures = this.contractABISignatures; + //let evmRPCInternalApi = this.evmRPCInternal + + let blkNum = false + let blkHash = false + let blockTS = false + let blockAvailable = false + let traceAvailable = false + let receiptsAvailable = false + let rpcBlock = r.block + let rpcReceipts = r.receipts + //let chainID = r.chain_id + let evmReceipts = [] + let evmTrace = false + let blk = false + let stream_bq = false + let write_bt = false + + if (rpcBlock) { + blockAvailable = true + blk = standardizeRPCBlock(rpcBlock) + console.log(`evmBlk++`, blk) + blkNum = blk.number; + blkHash = blk.hash; + blockTS = blk.timestamp; + } else { + console.log(`r.block missing`, r) + } + if (r.traces) { + traceAvailable = true + evmTrace = r.trace + } + if (rpcReceipts) { + receiptsAvailable = true + evmReceipts = standardizeRPCReceiptLogs(rpcReceipts) + } else { + console.log(`r.rpcReceipts missing`, r) + } + //console.log(`blk.transactions`, blk.transactions) + console.log(`decorate_evm_block [${blkNum}] [${blkHash}] Trace=${traceAvailable}, Receipts=${receiptsAvailable} , blockTS=${blockTS}`) + var statusesPromise = Promise.all([ + processTransactions(blk.transactions, contractABIs, contractABISignatures), + processReceipts(evmReceipts, contractABIs, contractABISignatures) + ]) + let [dTxns, dReceipts] = await statusesPromise + //console.log(`[#${blkNum} ${blkHash}] dTxns`, dTxns) + //console.log(`[#${blkNum} ${blkHash}] dReceipts`, dReceipts) + let flatTraces = debugTraceToFlatTraces(evmTrace, dTxns) + //console.log(`flatTraces[${flatTraces.length}]`, flatTraces) + let evmFullBlock = await fuseBlockTransactionReceipt(blk, dTxns, dReceipts, flatTraces, chainID) + return evmFullBlock; +} + async function processTransactions(txns, contractABIs, contractABISignatures) { let decodeTxns = [] let txnsAsync = await txns.map(async (txn) => { @@ -2069,8 +2149,83 @@ function categorizeTokenSwaps(dLog) { return false } +function debugTraceToFlatTraces(rpcTraces, txns) { + let traces = []; + if (!rpcTraces) { + return traces + } + for (let tx_index = 0; tx_index < rpcTraces.length; tx_index++) { + let tx = txns[tx_index] + let tx_transaction_hash = (tx.hash) ? tx.hash : tx.transactionHash + let tx_trace = (rpcTraces[tx_index].result) ? rpcTraces[tx_index].result : rpcTraces[tx_index]; + traces.push(...iterate_transaction_trace( + tx_transaction_hash, + tx_index, + tx_trace + )); + } + return traces; +} + +function iterate_transaction_trace(tx_transaction_hash, tx_index, tx_trace, trace_address = []) { + //console.log(`tx_trace`, tx_trace) + let trace = {}; + trace.transaction_hash = tx_transaction_hash + trace.transaction_index = tx_index; + + trace.from_address = (tx_trace['from'] != undefined) ? tx_trace['from'].toLowerCase() : null; + trace.to_address = (tx_trace['to'] != undefined) ? tx_trace['to'].toLowerCase() : null; + + trace.input = tx_trace['input']; + trace.output = (tx_trace['output'] != undefined) ? tx_trace['output'] : null; + + trace.value = (tx_trace['value'] != undefined) ? paraTool.dechexToIntStr(tx_trace['value']) : 0; + trace.gas = paraTool.dechexToIntStr(tx_trace['gas']); + trace.gas_used = paraTool.dechexToIntStr(tx_trace['gasUsed']); + + trace.error = (tx_trace['error'] != undefined) ? tx_trace['error'] : null; + + // lowercase for compatibility with parity traces + trace.trace_type = (tx_trace['type'] != undefined) ? tx_trace['type'].toLowerCase() : null; + trace.call_type = null + + if (trace.trace_type === 'selfdestruct') { + // rename to suicide for compatibility with parity traces + trace.trace_type = 'suicide'; + } else if (['call', 'callcode', 'delegatecall', 'staticcall'].includes(trace.trace_type)) { + trace.call_type = trace.trace_type; + trace.trace_type = 'call'; + } + + let result = [trace]; + + let calls = tx_trace['calls'] || []; + + trace.subtraces = calls.length; + trace.trace_address = trace_address; + trace.trace_id = `${trace.trace_type}_${tx_transaction_hash}_${trace_address_to_str(trace.trace_address)}` + + for (let call_index = 0; call_index < calls.length; call_index++) { + let call_trace = calls[call_index]; + result.push(...iterate_transaction_trace( + tx_transaction_hash, + tx_index, + call_trace, + [...trace_address, call_index] + )); + } + return result; +} + +function trace_address_to_str(trace_address) { + if (trace_address === null || trace_address.length === 0) { + return ''; + } + return trace_address.map(address_point => address_point.toString()).join('_'); +} function categorizeTokenTransfers(dLog) { + //console.log(`categorizeTokenTransfers`, dLog) if (dLog.decodeStatus == "success") { let dSig = dLog.signature let dEvents = dLog.events @@ -2391,14 +2546,6 @@ function decode_log(log, contractABIs, contractABISignatures) { return unknown } -//curl https://rpc.moonriver.moonbeam.network -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"eth_getLogs","params":[{"fromBlock": "0x10cca1", "toBlock": "0x10cca1"}],"id":1}' -//curl https://rpc.moonriver.moonbeam.network -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"eth_getLogs","params":[{"blockHash": "0x5843b718bd8310095c078643fbb33020bfe8a98efb99906a1980c95a5d8da04e"}],"id":1}' - -//retrive both block and log -//https://rpc.astar.network:8545 -//https://rpc.moonriver.moonbeam.network -//https://rpc.api.moonbeam.network - async function crawl_evm_logs(web3Api, bn) { var queryOpt = { fromBlock: bn, @@ -2538,128 +2685,6 @@ function loadWallet(pk) { return false } -/* 0.1MOVR to 22023 -> 22007 -Function: transfer(address, uint256, (uint8,bytes[]), uint64) -# Name Type Data -1 currency_address address 0x0000000000000000000000000000000000000802 -2 amount uint256 100000000000000000 -2 destination.parents uint8 1 -2 destination.interior bytes 0x00000007d7,0x015c88c4cdb316381d45d7dfbe59623516cfe5805808313e76f7ce6af6333a443700 - -Byte Value Selector Data Type -0x00 Parachain bytes4 -0x01 AccountId32 bytes32 -0x02 AccountIndex64 u64 -0x03 AccountKey20 bytes20 -0x04 PalletInstance byte -0x05 GeneralIndex u128 -0x06 GeneralKey bytes[] - -Selector Data Value Represents -Parachain "0x00+000007E7" Parachain ID 2023 -AccountId32 "0x01+AccountId32+00" AccountId32, Network Any -AccountKey20 "0x03+AccountKey20+00" AccountKey20, Network Any -PalletInstance "0x04+03" Pallet Instance 3 - -see: https://docs.moonbeam.network/builders/xcm/xc20/xtokens/#xtokens-transfer-function -*/ - -function xTokenBuilder(web3Api, currency_address = '0x0000000000000000000000000000000000000802', amount = 1, decimals = 18, beneficiary = '0xd2473025c560e31b005151ebadbc3e1f14a2af8fa60ed87e2b35fa930523cd3c', chainIDDest = 22006) { - console.log(`xTokenBuilder currency_address=${currency_address}, amount=${amount}, decimals=${decimals}, beneficiary=${beneficiary}, chainIDDest=${chainIDDest}`) - var xTokensContractAbiStr = '[{"inputs":[{"internalType":"address","name":"currency_address","type":"address"},{"internalType":"uint256","name":"amount","type":"uint256"},{"components":[{"internalType":"uint8","name":"parents","type":"uint8"},{"internalType":"bytes[]","name":"interior","type":"bytes[]"}],"internalType":"struct IxTokens.Multilocation","name":"destination","type":"tuple"},{"internalType":"uint64","name":"weight","type":"uint64"}],"name":"transfer","outputs":[],"stateMutability":"nonpayable","type":"function"}]' - var xTokensContractAbi = JSON.parse(xTokensContractAbiStr) - var xTokensContractAddress = '0x0000000000000000000000000000000000000804' //this is the precompiled interface - var xTokensContract = new web3Api.eth.Contract(xTokensContractAbi, xTokensContractAddress); - let weight = 6000000000 - let relayChain = paraTool.getRelayChainByChainID(chainIDDest) - let paraIDDest = paraTool.getParaIDfromChainID(chainIDDest) - let junction = [] - let junctionInterior = [] - junction.push(1) // local asset would have 0. (but this is not xcm?) - if (paraIDDest != 0) { - let parachainHex = paraTool.bnToHex(paraIDDest).substr(2) - parachainHex = '0x' + parachainHex.padStart(10, '0') - junctionInterior.push(parachainHex) - } - //assume "any" - if (beneficiary.length == 66) { - let accountId32 = `0x01${beneficiary.substr(2)}00` - junctionInterior.push(accountId32) - } else if (beneficiary.length == 42) { - let accountKey20 = `0x03${beneficiary.substr(2)}00` - junctionInterior.push(accountKey20) - } - let rawAmount = paraTool.toBaseUnit(`${amount}`, decimals) - junction.push(junctionInterior) - console.log(`junction`, junction) - console.log(`junctionInterior`, junctionInterior) - //[1,["0x00000007d4","0x01108cb67dcbaab765b66fdb99e3c4997ead09f1f82186e425dc9fec271e97aa7e00"]] - console.log(`xTokenBuilder currency_address=${currency_address}, Human Readable Amount=${amount}(rawAmount=${rawAmount}, using decimals=${decimals}), junction=${junction}, weight=${weight}`) - var data = xTokensContract.methods.transfer(currency_address, rawAmount, junction, weight).encodeABI() - let txStruct = { - to: xTokensContractAddress, - value: '0', - gas: 2000000, - data: data - } - console.log(`xTokenBuilder txStruct=`, txStruct) - return txStruct -} - -function xc20AssetWithdrawBuilder(web3Api, currency_address = '0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF', amount = 1, decimals = 18, beneficiary = '0xd2473025c560e31b005151ebadbc3e1f14a2af8fa60ed87e2b35fa930523cd3c', chainIDDest = 0) { - let isBeneficiaryEVM = (beneficiary.length == 42) ? true : false - console.log(`xc20Builder currency_address=${currency_address}, amount=${amount}, decimals=${decimals}, beneficiary=${beneficiary}(isEVM=${isBeneficiaryEVM}), chainIDDest=${chainIDDest}`) - //https://github.com/AstarNetwork/astar-frame/blob/polkadot-v0.9.28/precompiles/xcm/XCM.sol - var xc20ContractAbiStr = '[{"inputs":[{"internalType":"address[]","name":"asset_id","type":"address[]"},{"internalType":"uint256[]","name":"asset_amount","type":"uint256[]"},{"internalType":"bytes32","name":"recipient_account_id","type":"bytes32"},{"internalType":"bool","name":"is_relay","type":"bool"},{"internalType":"uint256","name":"parachain_id","type":"uint256"},{"internalType":"uint256","name":"fee_index","type":"uint256"}],"name":"assets_withdraw","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address[]","name":"asset_id","type":"address[]"},{"internalType":"uint256[]","name":"asset_amount","type":"uint256[]"},{"internalType":"address","name":"recipient_account_id","type":"address"},{"internalType":"bool","name":"is_relay","type":"bool"},{"internalType":"uint256","name":"parachain_id","type":"uint256"},{"internalType":"uint256","name":"fee_index","type":"uint256"}],"name":"assets_withdraw","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"}]' - - var xc20ContractAbi = JSON.parse(xc20ContractAbiStr) - var xc20ContractAddress = '0x0000000000000000000000000000000000005004' //this is the precompiled interface - var xc20Contract = new web3Api.eth.Contract(xc20ContractAbi, xc20ContractAddress); - let weight = 6000000000 - let relayChain = paraTool.getRelayChainByChainID(chainIDDest) - let paraIDDest = paraTool.getParaIDfromChainID(chainIDDest) - /* - function assets_withdraw( - address[] calldata asset_id, - uint256[] calldata asset_amount, - bytes32/address recipient_account_id, - bool is_relay, - uint256 parachain_id, - uint256 fee_index - ) external returns (bool); - */ - let currency_address_list = [] - let amountList = [] - let isRelay = true - let feeIndex = 0 - let data = '0x' - if (paraIDDest != 0) { - isRelay = false - } - //only teleport one asset for now - currency_address_list.push(currency_address) - let rawAmount = paraTool.toBaseUnit(`${amount}`, decimals) - amountList.push(rawAmount) - if (isBeneficiaryEVM) { - //0xecf766ff /BeneficiaryEVM - console.log(`xc20Builder method=0xecf766ff, currency_address_list=${currency_address_list}, Human Readable Amount=${amount}(amountList=${amountList}, beneficiary=${beneficiary}, using decimals=${decimals})`) - data = xc20Contract.methods['0xecf766ff'](currency_address_list, amountList, beneficiary, isRelay, paraIDDest, feeIndex).encodeABI() - } else if (beneficiary.length == 66) { - //0x019054d0 /BeneficiarySubstrate - console.log(`xc20Builder method=0x019054d0, currency_address_list=${currency_address_list}, Human Readable Amount=${amount}(amountList=${amountList}, using decimals=${decimals})`) - data = xc20Contract.methods['0x019054d0'](currency_address_list, amountList, beneficiary, isRelay, paraIDDest, feeIndex).encodeABI() - } - let txStruct = { - to: xc20ContractAddress, - gasPrice: web3.utils.numberToHex('30052000000'), //30.052Gwei - value: '0', - gas: 2000000, - data: data - } - console.log(`xc20Builder txStruct=`, txStruct) - return txStruct -} - function isTxContractCreate(tx) { if (!tx) return (false); let isCreate = (tx.creates != null) @@ -3263,6 +3288,101 @@ function standardizeRPCReceiptLogs(receipts) { return receipts } +function xTokenBuilder(web3Api, currency_address = '0x0000000000000000000000000000000000000802', amount = 1, decimals = 18, beneficiary = '0xd2473025c560e31b005151ebadbc3e1f14a2af8fa60ed87e2b35fa930523cd3c', chainIDDest = 22006) { + console.log(`xTokenBuilder currency_address=${currency_address}, amount=${amount}, decimals=${decimals}, beneficiary=${beneficiary}, chainIDDest=${chainIDDest}`) + var xTokensContractAbiStr = '[{"inputs":[{"internalType":"address","name":"currency_address","type":"address"},{"internalType":"uint256","name":"amount","type":"uint256"},{"components":[{"internalType":"uint8","name":"parents","type":"uint8"},{"internalType":"bytes[]","name":"interior","type":"bytes[]"}],"internalType":"struct IxTokens.Multilocation","name":"destination","type":"tuple"},{"internalType":"uint64","name":"weight","type":"uint64"}],"name":"transfer","outputs":[],"stateMutability":"nonpayable","type":"function"}]' + var xTokensContractAbi = JSON.parse(xTokensContractAbiStr) + var xTokensContractAddress = '0x0000000000000000000000000000000000000804' //this is the precompiled interface + var xTokensContract = new web3Api.eth.Contract(xTokensContractAbi, xTokensContractAddress); + let weight = 6000000000 + let relayChain = paraTool.getRelayChainByChainID(chainIDDest) + let paraIDDest = paraTool.getParaIDfromChainID(chainIDDest) + let junction = [] + let junctionInterior = [] + junction.push(1) // local asset would have 0. (but this is not xcm?) + if (paraIDDest != 0) { + let parachainHex = paraTool.bnToHex(paraIDDest).substr(2) + parachainHex = '0x' + parachainHex.padStart(10, '0') + junctionInterior.push(parachainHex) + } + //assume "any" + if (beneficiary.length == 66) { + let accountId32 = `0x01${beneficiary.substr(2)}00` + junctionInterior.push(accountId32) + } else if (beneficiary.length == 42) { + let accountKey20 = `0x03${beneficiary.substr(2)}00` + junctionInterior.push(accountKey20) + } + let rawAmount = paraTool.toBaseUnit(`${amount}`, decimals) + junction.push(junctionInterior) + console.log(`junction`, junction) + console.log(`junctionInterior`, junctionInterior) + //[1,["0x00000007d4","0x01108cb67dcbaab765b66fdb99e3c4997ead09f1f82186e425dc9fec271e97aa7e00"]] + console.log(`xTokenBuilder currency_address=${currency_address}, Human Readable Amount=${amount}(rawAmount=${rawAmount}, using decimals=${decimals}), junction=${junction}, weight=${weight}`) + var data = xTokensContract.methods.transfer(currency_address, rawAmount, junction, weight).encodeABI() + let txStruct = { + to: xTokensContractAddress, + value: '0', + gas: 2000000, + data: data + } + console.log(`xTokenBuilder txStruct=`, txStruct) + return txStruct +} + +function xc20AssetWithdrawBuilder(web3Api, currency_address = '0xFFfFfFffFFfffFFfFFfFFFFFffFFFffffFfFFFfF', amount = 1, decimals = 18, beneficiary = '0xd2473025c560e31b005151ebadbc3e1f14a2af8fa60ed87e2b35fa930523cd3c', chainIDDest = 0) { + let isBeneficiaryEVM = (beneficiary.length == 42) ? true : false + console.log(`xc20Builder currency_address=${currency_address}, amount=${amount}, decimals=${decimals}, beneficiary=${beneficiary}(isEVM=${isBeneficiaryEVM}), chainIDDest=${chainIDDest}`) + //https://github.com/AstarNetwork/astar-frame/blob/polkadot-v0.9.28/precompiles/xcm/XCM.sol + var xc20ContractAbiStr = '[{"inputs":[{"internalType":"address[]","name":"asset_id","type":"address[]"},{"internalType":"uint256[]","name":"asset_amount","type":"uint256[]"},{"internalType":"bytes32","name":"recipient_account_id","type":"bytes32"},{"internalType":"bool","name":"is_relay","type":"bool"},{"internalType":"uint256","name":"parachain_id","type":"uint256"},{"internalType":"uint256","name":"fee_index","type":"uint256"}],"name":"assets_withdraw","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address[]","name":"asset_id","type":"address[]"},{"internalType":"uint256[]","name":"asset_amount","type":"uint256[]"},{"internalType":"address","name":"recipient_account_id","type":"address"},{"internalType":"bool","name":"is_relay","type":"bool"},{"internalType":"uint256","name":"parachain_id","type":"uint256"},{"internalType":"uint256","name":"fee_index","type":"uint256"}],"name":"assets_withdraw","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"}]' + + var xc20ContractAbi = JSON.parse(xc20ContractAbiStr) + var xc20ContractAddress = '0x0000000000000000000000000000000000005004' //this is the precompiled interface + var xc20Contract = new web3Api.eth.Contract(xc20ContractAbi, xc20ContractAddress); + let weight = 6000000000 + let relayChain = paraTool.getRelayChainByChainID(chainIDDest) + let paraIDDest = paraTool.getParaIDfromChainID(chainIDDest) + /* + function assets_withdraw( + address[] calldata asset_id, + uint256[] calldata asset_amount, + bytes32/address recipient_account_id, + bool is_relay, + uint256 parachain_id, + uint256 fee_index + ) external returns (bool); + */ + let currency_address_list = [] + let amountList = [] + let isRelay = true + let feeIndex = 0 + let data = '0x' + if (paraIDDest != 0) { + isRelay = false + } + //only teleport one asset for now + currency_address_list.push(currency_address) + let rawAmount = paraTool.toBaseUnit(`${amount}`, decimals) + amountList.push(rawAmount) + if (isBeneficiaryEVM) { + //0xecf766ff /BeneficiaryEVM + console.log(`xc20Builder method=0xecf766ff, currency_address_list=${currency_address_list}, Human Readable Amount=${amount}(amountList=${amountList}, beneficiary=${beneficiary}, using decimals=${decimals})`) + data = xc20Contract.methods['0xecf766ff'](currency_address_list, amountList, beneficiary, isRelay, paraIDDest, feeIndex).encodeABI() + } else if (beneficiary.length == 66) { + //0x019054d0 /BeneficiarySubstrate + console.log(`xc20Builder method=0x019054d0, currency_address_list=${currency_address_list}, Human Readable Amount=${amount}(amountList=${amountList}, using decimals=${decimals})`) + data = xc20Contract.methods['0x019054d0'](currency_address_list, amountList, beneficiary, isRelay, paraIDDest, feeIndex).encodeABI() + } + let txStruct = { + to: xc20ContractAddress, + gasPrice: web3.utils.numberToHex('30052000000'), //30.052Gwei + value: '0', + gas: 2000000, + data: data + } + console.log(`xc20Builder txStruct=`, txStruct) + return txStruct +} module.exports = { toHex: function(bytes) { @@ -3285,9 +3405,6 @@ module.exports = { crawlEvmReceipts: async function(web3Api, blk, isParallel = true) { return crawl_evm_receipts(web3Api, blk, isParallel) }, - detect_contract_labels: async function(web3Api, blk, isParallel = true) { - return crawl_evm_receipts(web3Api, blk, isParallel) - }, detectContractLabels: async function(web3Api, contractAddress, bn) { return detect_contract_labels(web3Api, contractAddress, bn) }, @@ -3349,12 +3466,6 @@ module.exports = { sendSignedRLPTx: async function(web3Api, rlpTX) { return sendSignedRLPTx(web3Api, rlpTX) }, - xTokenBuilder: function(web3Api, currencyAddress, amount, decimal, beneficiary, chainIDDest) { - return xTokenBuilder(web3Api, currencyAddress, amount, decimal, beneficiary, chainIDDest) - }, - xc20AssetWithdrawBuilder: function(web3Api, currencyAddress, amount, decimal, beneficiary, chainIDDest) { - return xc20AssetWithdrawBuilder(web3Api, currencyAddress, amount, decimal, beneficiary, chainIDDest) - }, parseAbiSignature: function(abiStrArr) { return parseAbiSignature(abiStrArr) }, @@ -3364,8 +3475,8 @@ module.exports = { processTransactions: async function(txns, contractABIs, contractABISignatures) { return processTransactions(txns, contractABIs, contractABISignatures) }, - fuseBlockTransactionReceipt: async function(evmBlk, dTxns, dReceipts, dTrace, chainID) { - return fuseBlockTransactionReceipt(evmBlk, dTxns, dReceipts, dTrace, chainID) + fuseBlockTransactionReceipt: async function(evmBlk, dTxns, flatTraces, dTrace, chainID) { + return fuseBlockTransactionReceipt(evmBlk, dTxns, flatTraces, dTrace, chainID) }, decorateTxn: function(dTxn, dReceipt, dInternal, blockTS = false, chainID = false) { return decorateTxn(dTxn, dReceipt, dInternal, blockTS, chainID); @@ -3417,9 +3528,13 @@ module.exports = { return null } }, + debugTraceToFlatTraces: function(rpcTraces, txns) { + return debugTraceToFlatTraces(rpcTraces, txns) + }, processEVMTrace: function(evmTrace, evmTxs) { let res = [] process_evm_trace(evmTrace, res, 0, [0], evmTxs); + //res = debug_trace_to_flat_traces(evmTrace, evmTxs) return res }, keccak256: function(hex) { @@ -3458,4 +3573,11 @@ module.exports = { computeModifiedFingerprintID: computeModifiedFingerprintID, getEVMFlds: getEVMFlds, getSchemaWithoutDesc: getSchemaWithoutDesc, -}; \ No newline at end of file + xTokenBuilder: function(web3Api, currencyAddress, amount, decimal, beneficiary, chainIDDest) { + return xTokenBuilder(web3Api, currencyAddress, amount, decimal, beneficiary, chainIDDest) + }, + xc20AssetWithdrawBuilder: function(web3Api, currencyAddress, amount, decimal, beneficiary, chainIDDest) { + return xc20AssetWithdrawBuilder(web3Api, currencyAddress, amount, decimal, beneficiary, chainIDDest) + }, + decorateEvmBlock: decorateEvmBlock, +}; diff --git a/substrate/polkaholicDB.js b/substrate/polkaholicDB.js index 82caf6c5..68c7646b 100644 --- a/substrate/polkaholicDB.js +++ b/substrate/polkaholicDB.js @@ -1920,22 +1920,20 @@ from chain where chainID = '${chainID}' limit 1`); } async fetch_evm_block_gs(chainID, blockNumber) { - // TODO: shift back to substrate model + // TODO: shift back to substrate model let sql = `select UNIX_TIMESTAMP(blockDT) blockTS from block${chainID} from blockNumber = '${blockNumber}' limit 1` let blocks = await this.poolREADONLY.query(sql); if (blocks.length == 1) { let b = blocks[0]; let [logDT0, hr] = paraTool.ts_to_logDT_hr(b.blockTS); const storage = new Storage(); - const bucket = storage.bucket(bucketName); const bucketName = 'crypto_evm'; - const fileName = this.gs_evm_file_name(chainID, logDT, blockNumber); + const bucket = storage.bucket(bucketName); + const fileName = this.gs_evm_file_name(chainID, logDT0, blockNumber); const file = bucket.file(fileName); const buffer = await file.download(); - const r = JSON.parse(buffer[0]); - return { - evmFullBlock: r.block - } + const r = JSON.parse(buffer[0]); // block, receipts, evm + return r } return null; } @@ -1966,7 +1964,7 @@ from chain where chainID = '${chainID}' limit 1`); async fetch_block_gs(chainID, blockNumber) { try { - if (chainID == 1 || chainID == 10 || chainID == 1284 || chainID == 592 || chainID == 43114 || chainID == 42161) { + if (chainID == 1 || chainID == 10 || chainID == 43114 || chainID == 42161) { return this.fetch_evm_block_gs(chainID, blockNumber); } else { return this.fetch_substrate_block_gs(chainID, blockNumber); @@ -1989,7 +1987,7 @@ from chain where chainID = '${chainID}' limit 1`); const bucket = storage.bucket(bucketName); let jmp = 100; - + for (let bn0 = bnStart; bn0 <= bnEnd; bn0 += jmp) { let bn1 = bn0 + jmp - 1; if (bn1 > bnEnd) bn1 = bnEnd; @@ -2008,8 +2006,8 @@ from chain where chainID = '${chainID}' limit 1`); console.log("... workload: ", bn0, bn1, "archived", c.archived, "cnt", cnt); } } - - + + let start = paraTool.blockNumberToHex(bn0); let end = paraTool.blockNumberToHex(bn1); let [rows] = await tableChain.getRows({ @@ -2059,8 +2057,7 @@ from chain where chainID = '${chainID}' limit 1`); } async fetch_block(chainID, blockNumber, families = ["feed", "finalized"], feedOnly = false, blockHash = false) { - // WIP - if ( (chainID <= 2) && blockNumber < 35000 ) { // fetch { blockraw, events, feed } from GS storage + if ( ( chainID == 2004 && blockNumber >= 3683465 && blockNumber <= 3690513 ) || ( (chainID <= 2) && blockNumber < 35000 ) ) { // fetch { blockraw, events, feed } from GS storage try { let r = await this.fetch_block_gs(chainID, blockNumber); return r; diff --git a/substrate/query.js b/substrate/query.js index 3330efba..5736c1ca 100644 --- a/substrate/query.js +++ b/substrate/query.js @@ -2599,6 +2599,64 @@ module.exports = class Query extends AssetManager { return trace } + async decorate_evm_block(chainID, r){ + console.log('decorate_evm_block', r) + let contractABIs = this.contractABIs; + let contractABISignatures = this.contractABISignatures; + //let evmRPCInternalApi = this.evmRPCInternal + + let blkNum = false + let blkHash = false + let blockTS = false + let blockAvailable = false + let traceAvailable = false + let receiptsAvailable = false + let rpcBlock = r.block + let rpcReceipts = r.receipts + //let chainID = r.chain_id + let evmReceipts = [] + let evmTrace = false + let blk = false + let stream_bq = false + let write_bt = false + + if (rpcBlock) { + blockAvailable = true + blk = ethTool.standardizeRPCBlock(rpcBlock) + console.log(`evmBlk++`, blk) + blkNum = blk.number; + blkHash = blk.hash; + blockTS = blk.timestamp; + } else { + console.log(`r.block missing`, r) + } + if (r.traces) { + traceAvailable = true + evmTrace = r.trace + } + if (rpcReceipts) { + receiptsAvailable = true + evmReceipts = ethTool.standardizeRPCReceiptLogs(rpcReceipts) + } else { + console.log(`r.rpcReceipts missing`, r) + } + //console.log(`blk.transactions`, blk.transactions) + console.log(`decorate_evm_block [${blkNum}] [${blkHash}] Trace=${traceAvailable}, Receipts=${receiptsAvailable} , currTS=${this.getCurrentTS()}, blockTS=${blockTS}`) + var statusesPromise = Promise.all([ + ethTool.processTransactions(blk.transactions, contractABIs, contractABISignatures), + ethTool.processReceipts(evmReceipts, contractABIs, contractABISignatures) + ]) + let [dTxns, dReceipts] = await statusesPromise + console.log(`++++ [#${blkNum} ${blkHash}] dTxns len=${dTxns.length}`, dTxns) + console.log(`[#${blkNum} ${blkHash}] dReceipts len=${dReceipts.length}`, dReceipts) + let flatTraces = ethTool.debugTraceToFlatTraces(evmTrace, dTxns) + //console.log(`flatTraces[${flatTraces.length}]`, flatTraces) + //fuseBlockTransactionReceipt(evmBlk, dTxns, dReceipts, flatTraces, chainID) + let evmFullBlock = await ethTool.fuseBlockTransactionReceipt(blk, blk.transactions, dReceipts, flatTraces, chainID) + //let evmFullBlock = await ethTool.fuseBlockTransactionReceipt(blk, dTxns, dReceipts, flatTraces, chainID) + return evmFullBlock; + } + async getBlock(chainID_or_chainName, blockNumber, blockHash = false, decorate = true, decorateExtra = ["data", "address", "usd", "related"]) { let [chainID, id] = this.convertChainID(chainID_or_chainName) if (chainID === false) throw new paraTool.NotFoundError(`Invalid chain: ${chainID_or_chainName}`) @@ -2609,10 +2667,19 @@ module.exports = class Query extends AssetManager { try { let families = ["feed", "finalized", "feedevm"]; let row = await this.fetch_block(chainID, blockNumber, families, true, blockHash); - // TODO: check response - let block = row.feed; - if (block) { - block = await this.decorateBlock(row.feed, chainID, row.evmFullBlock, decorate, decorateExtra); + let evmFullBlock = (row.evmFullBlock)? row.evmFullBlock: false + console.log(`[${chainID}]${blockNumber} row`, row) + if (row && row.receipts != undefined){ + try { + row.evmFullBlock = await this.decorate_evm_block(chainID, row) + return row + } catch (err) { + console.log(err); + } + } + if (row.feed) { + let block = row.feed; + block = await this.decorateBlock(row.feed, chainID, evmFullBlock, decorate, decorateExtra); let sql = `select numXCMTransfersIn, numXCMMessagesIn, numXCMTransfersOut, numXCMMessagesOut, valXCMTransferIncomingUSD, valXCMTransferOutgoingUSD from block${chainID} where blockNumber = '${blockNumber}' limit 1`; let blockStatsRecs = await this.poolREADONLY.query(sql); @@ -6537,7 +6604,7 @@ module.exports = class Query extends AssetManager { r.metadata = JSON.parse(r.metadata); r.srcURLs = JSON.parse(r.srcURLs); } catch (err) { - + } return r; } @@ -6581,7 +6648,7 @@ module.exports = class Query extends AssetManager { // throw new paraTool.InvalidError(`Already verified ${codeHash} for ${network}`) } - // 1. creates local codehash-specific directory + // 1. creates local codehash-specific directory let targetDirectory = path.join(outputDirectory, codeHash); if (!fs.existsSync(targetDirectory)) { fs.mkdirSync(targetDirectory); @@ -6592,12 +6659,12 @@ module.exports = class Query extends AssetManager { //console.log(`cp ${zipFilePath} ${zipFilePathOriginalName}`); fs.copyFileSync(zipFilePath, zipFilePathOriginalName); - // 3. unzips zipfile into codehash-specific directory + // 3. unzips zipfile into codehash-specific directory const zip = new AdmZip(zipFilePath); zip.extractAllTo(targetDirectory, true); const extractedEntries = zip.getEntries(); - // 4. gets metadata from unzipped file of 3, parsing metadata JSON + // 4. gets metadata from unzipped file of 3, parsing metadata JSON let metadata = null; let cdnURL = "https://" + path.join(bucketDir, codeHash, originalname); // zip file let srcURLs = (publishSource>0) ? [cdnURL] : []; diff --git a/substrate/substrate-etl b/substrate/substrate-etl index 8d695623..914a4c9b 100755 --- a/substrate/substrate-etl +++ b/substrate/substrate-etl @@ -31,14 +31,26 @@ async function main() { program.command('extract') .description("Stream data out to GS") .option('-c, --chainID ', 'Chain ID', "2") - .option('-s, --startBlock ', 'Start Block', "1") - .option('-e, --endBlock ', 'End Block', "1000") + .option('-s, --startBlock ', 'Start Block') + .option('-e, --endBlock ', 'End Block') .action(async (opt) => { let substrateetl = new SubstrateETL(); let chainID = opt.chainID ? opt.chainID : 2; - let startBN = opt.startBlock ? parseInt(opt.startBlock, 10) : 1; - let endBN = opt.endBlock ? parseInt(opt.endBlock, 10) : 10000; - await substrateetl.extract_blocks(startBN, endBN, chainID); + let startBN = 1; + let endBN = 1000000; + if ( opt.startBlock && opt.endBlock ) { + startBN = parseInt(opt.startBlock, 10); + endBN = parseInt(opt.endBlock, 10); + await substrateetl.extract_blocks(startBN, endBN, chainID); + } else { + while ( 1 ) { + let [chainID, sector] = await substrateetl.get_archiver_chain_sector(); + console.log(chainID, sector); + await substrateetl.extract_blocks(sector * 10000, (sector+1)*10000, chainID); + await substrateetl.update_archiver_chain_sector(chainID, sector); + await substrateetl.sleep(2000); + } + } }) program.command('dump') @@ -655,4 +667,4 @@ main() .catch((e) => { console.error('ERROR', e); process.exit(1); - }); \ No newline at end of file + }); diff --git a/substrate/substrateetl.js b/substrate/substrateetl.js index 4f7c2323..52124021 100644 --- a/substrate/substrateetl.js +++ b/substrate/substrateetl.js @@ -149,6 +149,27 @@ module.exports = class SubstrateETL extends AssetManager { process.exit(0) } + async update_archiver_chain_sector(chainID, sector) { + let startBN = sector * 10000; + let endBN = startBN + 10000; + let sql = `insert into archiver (chainID, sector, archived, cnt) (select ${chainID} as chainID, ${sector} as sector, sum(archived) archived, count(*) as cnt from block${chainID} where blockNumber >= ${startBN} and blockNumber < ${endBN} group by chainID, sector) on duplicate key update archived = values(archived), cnt = values(cnt)` + this.batchedSQL.push(sql); + await this.update_batchedSQL(); + } + + async get_archiver_chain_sector() { + let sql = `select chainID, sector from archiver where archived < cnt order by rand() limit 1`; + let recs = await this.poolREADONLY.query(sql); + if ( recs.length == 1 ) { + let r = recs[0]; + let chainID = parseInt(r.chainID, 10); + let sector = parseInt(r.sector, 10); + return [chainID, sector]; + } + console.log("FAIL", sql) + return [null, null]; + } + async cleanReapedExcess(chainID) { let projectID = `${this.project}` if (chainID == 2004 || chainID == 22023) { diff --git a/views/evmBlockOverview.ejs b/views/evmBlockOverview.ejs index f722310f..afac80b0 100644 --- a/views/evmBlockOverview.ejs +++ b/views/evmBlockOverview.ejs @@ -3,24 +3,17 @@ let prevBlockUrl = `/block/${id}/${blockNumber - 1}`; let nextBlockUrl = `/block/${id}/${blockNumber + 1}`; let block = b.evmBlock; + console.log("THIS IS IT", block); %> - - - - - - - - <% } %> - - - - - - - - - - - - <% if ( b.author != undefined && b.miner != undefined && b.author.toLowerCase() != block.miner.toLowerCase()) { %> @@ -136,17 +111,5 @@ if ( block && block.transactions ) { %> <% } %> - - - - - - - -
<%- include("tooltip", {k: "block"}) %>Block < - <%= b.number %> + <%= block.number %> >
<%- include("tooltip", {k: "finalized"}) %>Finalized<%- include("finalized", {finalized: b.finalized}); %>
<%- include("tooltip", {k: "blocktimestamp"}) %>Timestamp<%- `${uiTool.presentSecondsAgoTS(b.blockTS)}`%><%- ` (${uiTool.presentTS(b.blockTS)})`; %>
<%- include("tooltip", {k: "transactions"}) %>Transactions @@ -106,24 +99,6 @@ if ( block && block.transactions ) { %>
<%- include("tooltip", {k: "blockhash"}) %>Substrate Block Hash - <%- uiTool.getFullHash(b.hash); %> -
<%- include("tooltip", {k: "parenthash"}) %>Substrate Parent Hash - <%- uiTool.getFullHash(b.header.parentHash); %> -
<%- include("tooltip", {k: "stateroot"}) %>Substrate State Root - <%- uiTool.getFullHash(b.header.stateRoot); %> -
<%- include("tooltip", {k: "author"}) %>Author
<%- include("tooltip", {k: "specversion"}) %>Spec Version - <%- uiTool.presentSpecVersion(id, b.specVersion); %> -
<%- include("tooltip", {k: "trace"}) %>Trace - <%- uiTool.presentTrace(id, blockNumber, b.hash); %> -