diff --git a/GeneralStateTests.md b/GeneralStateTests.md index 8c603a702f..caa48fcf17 100644 --- a/GeneralStateTests.md +++ b/GeneralStateTests.md @@ -387,7 +387,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 + CreateMessageRevertedOOGInInit.json OK + RevertDepthCreate2OOG.json OK + RevertDepthCreateAddressCollision.json OK - RevertInCreateInInitCreate2.json Skip ++ RevertInCreateInInitCreate2.json OK + RevertOpcodeCreate.json OK + RevertOpcodeInCreateReturnsCreate2.json OK + call_outsize_then_create2_successful_then_returndatasize.json OK @@ -413,7 +413,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 + returndatacopy_following_successful_create.json OK + returndatasize_following_successful_create.json OK ``` -OK: 41/44 Fail: 0/44 Skip: 3/44 +OK: 42/44 Fail: 0/44 Skip: 2/44 ## stCreateTest ```diff + CREATE_AcreateB_BSuicide_BStore.json OK @@ -1686,7 +1686,7 @@ OK: 38/38 Fail: 0/38 Skip: 0/38 + RevertDepthCreateAddressCollision.json OK + RevertDepthCreateOOG.json OK + RevertInCallCode.json OK - RevertInCreateInInit.json Skip ++ RevertInCreateInInit.json OK + RevertInDelegateCall.json OK + RevertInStaticCall.json OK + RevertOnEmptyStack.json OK @@ -1722,10 +1722,10 @@ OK: 38/38 Fail: 0/38 Skip: 0/38 + TouchToEmptyAccountRevert2.json OK + TouchToEmptyAccountRevert3.json OK ``` -OK: 40/46 Fail: 0/46 Skip: 6/46 +OK: 41/46 Fail: 0/46 Skip: 5/46 ## stSStoreTest ```diff - InitCollision.json Skip ++ InitCollision.json OK + InitCollisionNonZeroNonce.json OK + SstoreCallToSelfSubRefundBelowZero.json OK + sstore_0to0.json OK @@ -1755,7 +1755,7 @@ OK: 40/46 Fail: 0/46 Skip: 6/46 + sstore_combinations_initial1.json OK + sstore_combinations_initial2.json OK ``` -OK: 28/29 Fail: 0/29 Skip: 1/29 +OK: 29/29 Fail: 0/29 Skip: 0/29 ## stShift ```diff + sar00.json OK @@ -2645,4 +2645,4 @@ OK: 133/133 Fail: 0/133 Skip: 0/133 OK: 130/130 Fail: 0/130 Skip: 0/130 ---TOTAL--- -OK: 2340/2447 Fail: 0/2447 Skip: 107/2447 +OK: 2343/2447 Fail: 0/2447 Skip: 104/2447 diff --git a/PrecompileTests.md b/PrecompileTests.md index c9f855177d..15b11bd8d5 100644 --- a/PrecompileTests.md +++ b/PrecompileTests.md @@ -6,12 +6,12 @@ PrecompileTests + bn256mul.json OK + ecrecover.json OK + identity.json OK - modexp.json Skip ++ modexp.json OK + pairing.json OK + ripemd160.json OK + sha256.json OK ``` -OK: 7/8 Fail: 0/8 Skip: 1/8 +OK: 8/8 Fail: 0/8 Skip: 0/8 ---TOTAL--- -OK: 7/8 Fail: 0/8 Skip: 1/8 +OK: 8/8 Fail: 0/8 Skip: 0/8 diff --git a/tests/test_allowed_to_fail.nim b/tests/test_allowed_to_fail.nim new file mode 100644 index 0000000000..735f4df9a3 --- /dev/null +++ b/tests/test_allowed_to_fail.nim @@ -0,0 +1,176 @@ +# Nimbus +# Copyright (c) 2018 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +# XXX: when all but a relative few dozen, say, GeneralStateTests run, remove this, +# but for now, this enables some CI use before that to prevent regressions. In the +# separate file here because it would otherwise just distract. Could use all sorts +# of O(1) or O(log n) lookup structures, or be more careful to only initialize the +# table once, but notion's that it should shrink reasonable quickly and disappear, +# being mostly used for short-term regression prevention. + +import strutils + +func slowGSTTests(folder: string, name: string): bool = + result = folder == "stQuadraticComplexityTest" or + name in @["randomStatetest352.json", "randomStatetest1.json", + "randomStatetest32.json", "randomStatetest347.json", + "randomStatetest393.json", "randomStatetest626.json", + "CALLCODE_Bounds.json", "DELEGATECALL_Bounds3.json", + "CALLCODE_Bounds4.json", "CALL_Bounds.json", + "DELEGATECALL_Bounds2.json", "CALL_Bounds3.json", + "CALLCODE_Bounds2.json", "CALLCODE_Bounds3.json", + "DELEGATECALL_Bounds.json", "CALL_Bounds2a.json", + "CALL_Bounds2.json", + "CallToNameRegistratorMemOOGAndInsufficientBalance.json", + "CallToNameRegistratorTooMuchMemory0.json", + + # all these tests below actually pass + # but they are very slow + + # constantinople slow tests + "Create2Recursive.json", + + # byzantium slow tests + "LoopCallsDepthThenRevert3.json", + "LoopCallsDepthThenRevert2.json", + "LoopCallsDepthThenRevert.json", + "static_Call50000.json", + "static_Call50000_ecrec.json", + "static_Call50000_identity.json", + "static_Call50000_identity2.json", + "static_Call50000_rip160.json", + "static_Call50000_sha256.json", + "LoopCallsThenRevert.json", + "LoopDelegateCallsDepthThenRevert.json", + "recursiveCreateReturnValue.json", + "static_Call1024PreCalls2.json", + "Callcode1024BalanceTooLow.json", + "static_Call1024BalanceTooLow.json", + "static_Call1024BalanceTooLow2.json", + "static_Call1024OOG.json", + "static_Call1024PreCalls3.json", + "static_Call1024PreCalls.json", + "static_Call1MB1024Calldepth.json", + + # Homestead recursives + "ContractCreationSpam.json", + "Call1024OOG.json", + "Call1024PreCalls.json", + "CallRecursiveBombPreCall.json", + "Delegatecall1024.json", + "Delegatecall1024OOG.json", + "JUMPDEST_Attack.json", + "JUMPDEST_AttackwithJump.json", + "ABAcalls1.json", + "ABAcalls2.json", + "CallRecursiveBomb0.json", + "CallRecursiveBomb0_OOG_atMaxCallDepth.json", + "CallRecursiveBomb1.json", + "CallRecursiveBomb2.json", + "CallRecursiveBombLog.json", + "CallRecursiveBombLog2.json", + "Call1024BalanceTooLow.json", + + # Frontier recursives + "Callcode1024OOG.json", + "callcallcodecall_ABCB_RECURSIVE.json", + "callcallcodecallcode_ABCB_RECURSIVE.json", + "callcodecallcall_ABCB_RECURSIVE.json", + "callcodecallcallcode_ABCB_RECURSIVE.json", + "callcodecallcodecall_ABCB_RECURSIVE.json", + "callcodecallcodecallcode_ABCB_RECURSIVE.json", + "callcallcallcode_ABCB_RECURSIVE.json" + ] + +func failIn32Bits(folder, name: string): bool = + return name in @[ + # crash with OOM + "static_Return50000_2.json", + "randomStatetest185.json", + "randomStatetest159.json", + "randomStatetest48.json", + + # OOM in AppVeyor, not on my machine + "randomStatetest36.json" + ] + +func allowedFailingGeneralStateTest(folder, name: string): bool = + let allowedFailingGeneralStateTests = @[ + # conflicts between native int and big int. + # gasFee calculation in modexp precompiled + # contracts + "modexp.json", + # perhaps a design flaw with create/create2 opcode. + # a conflict between balance checker and + # static call context checker + "create2noCash.json" + ] + result = name in allowedFailingGeneralStateTests + +func allowedFailInCurrentBuild(folder, name: string): bool = + when sizeof(int) == 4: + if failIn32Bits(folder, name): + return true + return allowedFailingGeneralStateTest(folder, name) + +func skipGSTTests*(folder: string, name: string): bool = + # we skip tests that are slow or expected to fail for now + if slowGSTTests(folder, name): + return true + result = allowedFailInCurrentBuild(folder, name) + +func skipNewGSTTests*(folder: string, name: string): bool = + # share the same slow and failing tests + if skipGSTTests(folder, name): + return true + + result = name in @[ + # skip slow tests + "CALLBlake2f_MaxRounds.json", + + # py-evm claims these tests are incorrect + # nimbus also agree + "RevertInCreateInInit.json", + "RevertInCreateInInitCreate2.json", + "InitCollision.json" + ] + +func skipVMTests*(folder: string, name: string): bool = + when sizeof(int) == 4: + if name == "sha3_bigSize.json": + return true + result = (folder == "vmPerformance" and "loop" in name) + +func skipBCTests*(folder: string, name: string): bool = + let allowedFailingBCTests = @[ + # BlockChain slow tests + "SuicideIssue.json", + + # Failed tests + "SuicidesMixingCoinbase.json", + ] + + result = name in allowedFailingBCTests + +func skipNewBCTests*(folder: string, name: string): bool = + let allowedFailingBCTests = @[ + # Istanbul bc tests + # py-evm claims these tests are incorrect + # nimbus also agree + "RevertInCreateInInit.json", + "RevertInCreateInInitCreate2.json", + "InitCollision.json" + ] + + result = name in allowedFailingBCTests + +func skipTxTests*(folder: string, name: string): bool = + # from test_transaction_json + when sizeof(int) == 4: + result = name == "RLPHeaderSizeOverflowInt32.json" + else: + false diff --git a/tests/test_blockchain_json.nim b/tests/test_blockchain_json.nim index cf65c5d240..6307045497 100644 --- a/tests/test_blockchain_json.nim +++ b/tests/test_blockchain_json.nim @@ -10,7 +10,8 @@ import options, eth/[common, rlp, bloom], eth/trie/[db, trie_defs], ethash, stew/endians2, nimcrypto, - ./test_helpers, ../premix/parser, test_config, + ./test_helpers, , ./test_allowed_to_fail, + ../premix/parser, test_config, ../nimbus/vm/interpreter/vm_forks, ../nimbus/[vm_state, utils, vm_types, errors, transaction, constants], ../nimbus/db/[db_chain, state_db], @@ -668,9 +669,9 @@ proc main() = if paramCount() == 0: # run all test fixtures suite "block chain json tests": - jsonTest("BlockchainTests", testFixture) + jsonTest("BlockchainTests", testFixture, skipBCTests) suite "new block chain json tests": - jsonTest("newBlockchainTests", testFixture) + jsonTest("newBlockchainTests", testFixture, skipNewBCTests) else: # execute single test in debug mode let config = getConfiguration() diff --git a/tests/test_generalstate_failing.nim b/tests/test_generalstate_failing.nim deleted file mode 100644 index e401afda9b..0000000000 --- a/tests/test_generalstate_failing.nim +++ /dev/null @@ -1,32 +0,0 @@ -# Nimbus -# Copyright (c) 2018 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -# XXX: when all but a relative few dozen, say, GeneralStateTests run, remove this, -# but for now, this enables some CI use before that to prevent regressions. In the -# separate file here because it would otherwise just distract. Could use all sorts -# of O(1) or O(log n) lookup structures, or be more careful to only initialize the -# table once, but notion's that it should shrink reasonable quickly and disappear, -# being mostly used for short-term regression prevention. -func allowedFailingGeneralStateTest*(folder, name: string): bool = - let allowedFailingGeneralStateTests = @[ - # conflicts between native int and big int. - # gasFee calculation in modexp precompiled - # contracts - "modexp.json", - # perhaps a design flaw with create/create2 opcode. - # a conflict between balance checker and - # static call context checker - "create2noCash.json", - - # Istanbul bc test - # py-evm claims these tests are incorrect - # nimbus also agree - "RevertInCreateInInit.json", - "RevertInCreateInInitCreate2.json", - "InitCollision.json", - ] - result = name in allowedFailingGeneralStateTests diff --git a/tests/test_generalstate_json.nim b/tests/test_generalstate_json.nim index 38973cdefc..03f7a72eb7 100644 --- a/tests/test_generalstate_json.nim +++ b/tests/test_generalstate_json.nim @@ -9,7 +9,8 @@ import unittest2, strformat, strutils, tables, json, times, os, stew/ranges/typedranges, nimcrypto, options, eth/[rlp, common], eth/trie/[db, trie_defs], chronicles, - ./test_helpers, ../nimbus/p2p/executor, test_config, + ./test_helpers, ./test_allowed_to_fail, + ../nimbus/p2p/executor, test_config, ../nimbus/transaction, ../nimbus/[vm_state, vm_types, vm_state_transactions, utils], ../nimbus/vm/interpreter, @@ -178,10 +179,10 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus, proc generalStateJsonMain*(debugMode = false) = if paramCount() == 0 or not debugMode: # run all test fixtures - suite "generalstate json tests": - jsonTest("GeneralStateTests", testFixture) + #suite "generalstate json tests": + #jsonTest("GeneralStateTests", testFixture, skipGSTTests) suite "new generalstate json tests": - jsonTest("newGeneralStateTests", testFixture) + jsonTest("newGeneralStateTests", testFixture, skipNewGSTTests) else: # execute single test in debug mode let config = getConfiguration() diff --git a/tests/test_helpers.nim b/tests/test_helpers.nim index 41f030d935..27873c3deb 100644 --- a/tests/test_helpers.nim +++ b/tests/test_helpers.nim @@ -10,8 +10,7 @@ import stew/byteutils, stew/ranges/typedranges, net, eth/[common, keys, rlp, p2p], unittest2, ../nimbus/[vm_state, config, transaction, utils, errors], ../nimbus/db/[db_chain, state_db], - ../nimbus/vm/interpreter/vm_forks, - ../tests/test_generalstate_failing + ../nimbus/vm/interpreter/vm_forks func revmap(x: Table[Fork, string]): Table[string, Fork] = result = initTable[string, Fork]() @@ -44,112 +43,7 @@ const type Status* {.pure.} = enum OK, Fail, Skip -func slowTest*(folder: string, name: string): bool = - result = - (folder == "vmPerformance" and "loop" in name) or - folder == "stQuadraticComplexityTest" or - name in @["randomStatetest352.json", "randomStatetest1.json", - "randomStatetest32.json", "randomStatetest347.json", - "randomStatetest393.json", "randomStatetest626.json", - "CALLCODE_Bounds.json", "DELEGATECALL_Bounds3.json", - "CALLCODE_Bounds4.json", "CALL_Bounds.json", - "DELEGATECALL_Bounds2.json", "CALL_Bounds3.json", - "CALLCODE_Bounds2.json", "CALLCODE_Bounds3.json", - "DELEGATECALL_Bounds.json", "CALL_Bounds2a.json", - "CALL_Bounds2.json", - "CallToNameRegistratorMemOOGAndInsufficientBalance.json", - "CallToNameRegistratorTooMuchMemory0.json", - - # all these tests below actually pass - # but they are very slow - - # constantinople slow tests - "Create2Recursive.json", - - # byzantium slow tests - "LoopCallsDepthThenRevert3.json", - "LoopCallsDepthThenRevert2.json", - "LoopCallsDepthThenRevert.json", - "static_Call50000.json", - "static_Call50000_ecrec.json", - "static_Call50000_identity.json", - "static_Call50000_identity2.json", - "static_Call50000_rip160.json", - "static_Call50000_sha256.json", - "LoopCallsThenRevert.json", - "LoopDelegateCallsDepthThenRevert.json", - "recursiveCreateReturnValue.json", - "static_Call1024PreCalls2.json", - "Callcode1024BalanceTooLow.json", - "static_Call1024BalanceTooLow.json", - "static_Call1024BalanceTooLow2.json", - "static_Call1024OOG.json", - "static_Call1024PreCalls3.json", - "static_Call1024PreCalls.json", - "static_Call1MB1024Calldepth.json", - - # Homestead recursives - "ContractCreationSpam.json", - "Call1024OOG.json", - "Call1024PreCalls.json", - "CallRecursiveBombPreCall.json", - "Delegatecall1024.json", - "Delegatecall1024OOG.json", - "JUMPDEST_Attack.json", - "JUMPDEST_AttackwithJump.json", - "ABAcalls1.json", - "ABAcalls2.json", - "CallRecursiveBomb0.json", - "CallRecursiveBomb0_OOG_atMaxCallDepth.json", - "CallRecursiveBomb1.json", - "CallRecursiveBomb2.json", - "CallRecursiveBombLog.json", - "CallRecursiveBombLog2.json", - "Call1024BalanceTooLow.json", - - # Frontier recursives - "Callcode1024OOG.json", - "callcallcodecall_ABCB_RECURSIVE.json", - "callcallcodecallcode_ABCB_RECURSIVE.json", - "callcodecallcall_ABCB_RECURSIVE.json", - "callcodecallcallcode_ABCB_RECURSIVE.json", - "callcodecallcodecall_ABCB_RECURSIVE.json", - "callcodecallcodecallcode_ABCB_RECURSIVE.json", - "callcallcallcode_ABCB_RECURSIVE.json", - - # BlockChain slow tests - "SuicideIssue.json", - "CALLBlake2f_MaxRounds.json" - ] - -func failIn32Bits(folder, name: string): bool = - return name in @[ - "sha3_bigSize.json", # from vm_test - - # crash with OOM - "static_Return50000_2.json", - "randomStatetest185.json", - "randomStatetest159.json", - "randomStatetest48.json", - - # OOM in AppVeyor, not on my machine - "randomStatetest36.json", - - # from test_transaction_json - "RLPHeaderSizeOverflowInt32.json" - ] - -func allowedFailInCurrentBuild(folder, name: string): bool = - when sizeof(int) == 4: - if failIn32Bits(folder, name): - return true - return allowedFailingGeneralStateTest(folder, name) - -func validTest*(folder: string, name: string): bool = - # we skip tests that are slow or expected to fail for now - result = - not slowTest(folder, name) and - not allowedFailInCurrentBuild(folder, name) +func skipNothing*(folder: string, name: string): bool = false proc lacksSupportedForks*(fixtures: JsonNode): bool = # XXX: Until Nimbus supports all forks, some of the GeneralStateTests won't work. @@ -171,7 +65,7 @@ proc lacksSupportedForks*(fixtures: JsonNode): bool = var status = initOrderedTable[string, OrderedTable[string, Status]]() -macro jsonTest*(s: static[string], handler: untyped): untyped = +macro jsonTest*(s: static[string], handler: untyped, skipTest: untyped): untyped = let testStatusIMPL = ident("testStatusIMPL") testName = ident("testName") @@ -191,8 +85,9 @@ macro jsonTest*(s: static[string], handler: untyped): untyped = if not status.hasKey(last): status[last] = initOrderedTable[string, Status]() status[last][name] = Status.Skip - if last.validTest(name): - filenames.add(filename) + if `skipTest`(last, name): + continue + filenames.add(filename) for fname in filenames: test fname: {.gcsafe.}: diff --git a/tests/test_persistblock_json.nim b/tests/test_persistblock_json.nim index db0671648f..9a867452f2 100644 --- a/tests/test_persistblock_json.nim +++ b/tests/test_persistblock_json.nim @@ -40,5 +40,4 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus) = proc persistBlockJsonMain*() = suite "persist block json tests": - jsonTest("PersistBlockTests", testFixture) - + jsonTest("PersistBlockTests", testFixture, skipNothing) diff --git a/tests/test_precompiles.nim b/tests/test_precompiles.nim index 5652c47641..3917e9f1b7 100644 --- a/tests/test_precompiles.nim +++ b/tests/test_precompiles.nim @@ -102,7 +102,7 @@ const blake2InputTests = [ proc precompilesMain*() = suite "Precompiles": - jsonTest("PrecompileTests", testFixture) + jsonTest("PrecompileTests", testFixture, skipNothing) suite "blake2bf": var output: array[64, byte] diff --git a/tests/test_tracer_json.nim b/tests/test_tracer_json.nim index f1de13982a..262bb657bc 100644 --- a/tests/test_tracer_json.nim +++ b/tests/test_tracer_json.nim @@ -14,7 +14,7 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus) proc tracerJsonMain*() = suite "tracer json tests": - jsonTest("TracerTests", testFixture) + jsonTest("TracerTests", testFixture, skipNothing) # use tracerTestGen.nim to generate additional test data proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus) = diff --git a/tests/test_transaction_json.nim b/tests/test_transaction_json.nim index 2a251b05ae..3a0c90888e 100644 --- a/tests/test_transaction_json.nim +++ b/tests/test_transaction_json.nim @@ -1,7 +1,8 @@ import unittest2, json, os, tables, strformat, strutils, eth/[common, rlp], - ./test_helpers, ../nimbus/[transaction, utils, errors] + ./test_helpers, ./test_allowed_to_fail, + ../nimbus/[transaction, utils, errors] const FIXTURE_FORK_SKIPS = ["_info", "rlp", "Constantinople"] @@ -10,7 +11,7 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus) proc transactionJsonMain*() = suite "Transactions tests": - jsonTest("TransactionTests", testFixture) + jsonTest("TransactionTests", testFixture, skipTxTests) when isMainModule: transactionJsonMain() diff --git a/tests/test_vm_json.nim b/tests/test_vm_json.nim index 08f3316ec0..b67c874f78 100644 --- a/tests/test_vm_json.nim +++ b/tests/test_vm_json.nim @@ -8,7 +8,7 @@ import unittest2, strformat, strutils, tables, json, os, times, stew/byteutils, stew/ranges/typedranges, eth/[rlp, common], eth/trie/db, - ./test_helpers, ../nimbus/vm/interpreter, + ./test_helpers, ./test_allowed_to_fail, ../nimbus/vm/interpreter, ../nimbus/[constants, vm_state, vm_types, utils], ../nimbus/db/[db_chain, state_db] @@ -16,7 +16,7 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus) proc vmJsonMain*() = suite "vm json tests": - jsonTest("VMTests", testFixture) + jsonTest("VMTests", testFixture, skipVMTests) proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus) = var fixture: JsonNode