Skip to content
This repository was archived by the owner on Jul 1, 2021. It is now read-only.

Commit c3778a5

Browse files
pipermerriampipermerriam
authored
Merge pull request #608 from veox/dont-skip-constantinople-tests
tests: update to match `py-evm`: don't skip Constantinople, add Petersburg, use `--fork`; break out long-running categories
2 parents 0a7e121 + c3fe964 commit c3778a5

File tree

4 files changed

+123
-59
lines changed

4 files changed

+123
-59
lines changed

‎.circleci/config.yml‎

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -131,15 +131,13 @@ jobs:
131131
- image: circleci/python:3.6
132132
environment:
133133
TOXENV: py36-lint
134-
135134
py37-lint:
136135
<<: *common
137136
docker:
138137
- image: circleci/python:3.7
139138
environment:
140139
TOXENV: py37-lint
141140

142-
143141
py36-docs:
144142
<<: *common
145143
docker:
@@ -171,6 +169,13 @@ jobs:
171169
- image: circleci/python:3.6
172170
environment:
173171
TOXENV: py36-rpc-state-homestead
172+
py36-rpc-state-petersburg:
173+
<<: *common
174+
docker:
175+
- image: circleci/python:3.6
176+
environment:
177+
TOXENV: py36-rpc-state-petersburg
178+
174179
py36-rpc-state-tangerine_whistle:
175180
<<: *common
176181
docker:
@@ -221,12 +226,6 @@ jobs:
221226
- image: circleci/python:3.6
222227
environment:
223228
TOXENV: py36-p2p
224-
py36-rpc-state-quadratic:
225-
<<: *common
226-
docker:
227-
- image: circleci/python:3.6
228-
environment:
229-
TOXENV: py36-rpc-state-quadratic
230229
py36-eth2-core:
231230
<<: *common
232231
docker:
@@ -264,6 +263,24 @@ jobs:
264263
environment:
265264
TOXENV: py36-plugins
266265

266+
py37-rpc-state-quadratic:
267+
<<: *common
268+
docker:
269+
- image: circleci/python:3.7
270+
environment:
271+
TOXENV: py37-rpc-state-quadratic
272+
py37-rpc-state-sstore:
273+
<<: *common
274+
docker:
275+
- image: circleci/python:3.7
276+
environment:
277+
TOXENV: py37-rpc-state-sstore
278+
py37-rpc-state-zero_knowledge:
279+
<<: *common
280+
docker:
281+
- image: circleci/python:3.7
282+
environment:
283+
TOXENV: py37-rpc-state-zero_knowledge
267284

268285
py37-core:
269286
<<: *common
@@ -320,7 +337,6 @@ jobs:
320337
environment:
321338
TOXENV: py37-plugins
322339

323-
324340
docker-image-build-test:
325341
machine: true
326342
steps:
@@ -343,13 +359,17 @@ workflows:
343359
# - py37-libp2p
344360
- py37-plugins
345361

362+
- py37-rpc-state-quadratic
363+
- py37-rpc-state-sstore
364+
- py37-rpc-state-zero_knowledge
365+
346366
- py36-rpc-state-byzantium
347367
- py36-rpc-state-constantinople
348368
- py36-rpc-state-frontier
349369
- py36-rpc-state-homestead
350-
- py36-rpc-state-tangerine_whistle
370+
- py36-rpc-state-petersburg
351371
- py36-rpc-state-spurious_dragon
352-
- py36-rpc-state-quadratic
372+
- py36-rpc-state-tangerine_whistle
353373
- py36-rpc-blockchain
354374

355375
- py36-core

‎fixtures‎

Submodule fixtures updated 23665 files

‎tests/json-fixtures-over-rpc/test_rpc_fixtures.py‎

Lines changed: 76 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
is_hex,
1717
is_integer,
1818
is_string,
19+
to_tuple,
1920
)
2021

2122
from eth.chains.mainnet import (
@@ -47,16 +48,12 @@
4748
BASE_FIXTURE_PATH = os.path.join(ROOT_PROJECT_DIR, 'fixtures', 'BlockchainTests')
4849

4950
SLOW_TESTS = (
50-
'Call1024PreCalls_d0g0v0_Byzantium',
51-
'Call1024PreCalls_d0g0v0_EIP150',
52-
'Call1024PreCalls_d0g0v0_EIP158',
51+
'Call1024PreCalls_d0g0v0',
5352
'ContractCreationSpam_d0g0v0_Homestead',
5453
'ContractCreationSpam_d0g0v0_Frontier',
55-
'ForkStressTest_EIP150',
56-
'ForkStressTest_EIP158',
57-
'ForkStressTest_Homestead',
58-
'ForkStressTest_Frontier',
59-
'ForkStressTest_Byzantium',
54+
'Create2Recursive_d0g0v0',
55+
'Create2Recursive_d0g1v0',
56+
'ForkStressTest',
6057
'stQuadraticComplexityTest/Call50000_d0g1v0.json',
6158
'stQuadraticComplexityTest/QuadraticComplexitySolidity_CallDataCopy_d0g1v0.json',
6259
'stQuadraticComplexityTest/Return50000_2_d0g1v0.json',
@@ -66,39 +63,37 @@
6663
'stQuadraticComplexityTest/Call50000_ecrec_d0g1v0.json',
6764
'walletReorganizeOwners',
6865
'bcExploitTest/SuicideIssue.json',
69-
'DelegateCallSpam_Homestead',
70-
'static_Call50000_sha256_d0g0v0_Byzantium',
71-
'static_Call50000_rip160_d0g0v0_Byzantium',
72-
'static_Call50000_rip160_d1g0v0_Byzantium',
73-
'static_Call50000_sha256_d1g0v0_Byzantium',
74-
'static_Call50000_ecrec_d1g0v0_Byzantium',
75-
'static_Call50000_d1g0v0_Byzantium',
76-
'static_Call50000_d0g0v0_Byzantium',
77-
'static_Call50000_ecrec_d0g0v0_Byzantium',
78-
'static_Call50000_identity2_d0g0v0_Byzantium',
79-
'static_Call50000_identity2_d1g0v0_Byzantium',
80-
'static_Call50000_identity_d1g0v0_Byzantium',
81-
'static_Call50000_identity_d0g0v0_Byzantium',
82-
'static_Call50000bytesContract50_1_d1g0v0_Byzantium',
83-
'static_Call50000bytesContract50_2_d1g0v0_Byzantium',
84-
'static_LoopCallsThenRevert_d0g0v0_Byzantium',
85-
'static_LoopCallsThenRevert_d0g1v0_Byzantium',
86-
'Call1024PreCalls_d0g0v0_Byzantium',
87-
'Call1024PreCalls_d0g0v0_EIP158',
88-
'Call1024PreCalls_d0g0v0_EIP150',
89-
'Call1024PreCalls_d0g0v0_Byzantium',
90-
'Call1024PreCalls_d0g0v0_EIP150',
91-
'Call1024PreCalls_d0g0v0_EIP158',
66+
'static_Call1024PreCalls_d1g0v0',
67+
'static_Call1024PreCalls2_d0g0v0',
68+
'static_Call1024PreCalls2_d1g0v0',
69+
'static_Call1024PreCalls3_d1g0v0',
70+
'static_Call50000bytesContract50_1_d0g0v0',
71+
'static_Call50000_ecrec_d0g0v0',
72+
'static_Call50000_ecrec_d1g0v0',
73+
'static_Call50000_rip160_d0g0v0',
74+
'static_Call50000_rip160_d1g0v0',
75+
'static_Call50000_sha256_d0g0v0',
76+
'static_Call50000_sha256_d1g0v0',
77+
'static_Call50000_d0g0v0',
78+
'static_Call50000_d1g0v0',
79+
'static_Call50000_identity2_d0g0v0',
80+
'static_Call50000_identity2_d1g0v0',
81+
'static_Call50000_identity_d0g0v0',
82+
'static_Call50000_identity_d1g0v0',
83+
'static_Call50000bytesContract50_1_d1g0v0',
84+
'static_Call50000bytesContract50_2_d1g0v0',
85+
'static_LoopCallsThenRevert_d0g0v0',
86+
'static_LoopCallsThenRevert_d0g1v0',
87+
'static_Return50000_2_d0g0v0',
9288
'stQuadraticComplexityTest/Call50000_identity2_d0g1v0.json',
9389
'stQuadraticComplexityTest/Call50000_identity_d0g1v0.json',
9490
'stQuadraticComplexityTest/Call50000_rip160_d0g1v0.json',
9591
'stQuadraticComplexityTest/Call50000bytesContract50_1_d0g1v0.json',
92+
'stQuadraticComplexityTest/Call50000bytesContract50_2_d0g1v0.json',
9693
'stQuadraticComplexityTest/Create1000_d0g1v0.json',
9794
'ShanghaiLove_Homestead',
9895
'ShanghaiLove_Frontier',
99-
'DelegateCallSpam_EIP158',
100-
'DelegateCallSpam_Byzantium',
101-
'DelegateCallSpam_EIP150',
96+
'DelegateCallSpam',
10297
)
10398

10499
# These are tests that are thought to be incorrect or buggy upstream,
@@ -112,6 +107,28 @@
112107
# The result is in conflict with the yellow-paper:
113108
# * https://github.com/ethereum/py-evm/pull/1224#issuecomment-418800369
114109
('GeneralStateTests/stRevertTest/RevertInCreateInInit_d0g0v0.json', 'RevertInCreateInInit_d0g0v0_Byzantium'), # noqa: E501
110+
('GeneralStateTests/stRevertTest/RevertInCreateInInit_d0g0v0.json', 'RevertInCreateInInit_d0g0v0_Constantinople'), # noqa: E501
111+
('GeneralStateTests/stRevertTest/RevertInCreateInInit_d0g0v0.json', 'RevertInCreateInInit_d0g0v0_ConstantinopleFix'), # noqa: E501
112+
113+
# The CREATE2 variant seems to have been derived from the one above - it, too,
114+
# has a "synthetic" state, on which py-evm flips.
115+
# * https://github.com/ethereum/py-evm/pull/1181#issuecomment-446330609
116+
('GeneralStateTests/stCreate2/RevertInCreateInInitCreate2_d0g0v0.json', 'RevertInCreateInInitCreate2_d0g0v0_Constantinople'), # noqa: E501
117+
('GeneralStateTests/stCreate2/RevertInCreateInInitCreate2_d0g0v0.json', 'RevertInCreateInInitCreate2_d0g0v0_ConstantinopleFix'), # noqa: E501
118+
119+
# Four variants have been specifically added to test a collision type
120+
# like the above; therefore, they fail in the same manner.
121+
# * https://github.com/ethereum/py-evm/pull/1579#issuecomment-446591118
122+
# Interestingly, d2 passes in Constantinople after a py-evm refactor of storage handling,
123+
# the same test is already passing in ConstantinopleFix. Since the situation is synthetic,
124+
# not much research went into why, yet.
125+
('GeneralStateTests/stSStoreTest/InitCollision_d0g0v0.json', 'InitCollision_d0g0v0_Constantinople'), # noqa: E501
126+
('GeneralStateTests/stSStoreTest/InitCollision_d1g0v0.json', 'InitCollision_d1g0v0_Constantinople'), # noqa: E501
127+
('GeneralStateTests/stSStoreTest/InitCollision_d2g0v0.json', 'InitCollision_d2g0v0_Constantinople'), # noqa: E501
128+
('GeneralStateTests/stSStoreTest/InitCollision_d3g0v0.json', 'InitCollision_d3g0v0_Constantinople'), # noqa: E501
129+
('GeneralStateTests/stSStoreTest/InitCollision_d0g0v0.json', 'InitCollision_d0g0v0_ConstantinopleFix'), # noqa: E501
130+
('GeneralStateTests/stSStoreTest/InitCollision_d1g0v0.json', 'InitCollision_d1g0v0_ConstantinopleFix'), # noqa: E501
131+
('GeneralStateTests/stSStoreTest/InitCollision_d3g0v0.json', 'InitCollision_d3g0v0_ConstantinopleFix'), # noqa: E501
115132
}
116133

117134
RPC_STATE_NORMALIZERS = {
@@ -178,7 +195,7 @@ def fixture_transaction_in_rpc_format(state):
178195
}
179196

180197

181-
def blockchain_fixture_mark_fn(fixture_path, fixture_name):
198+
def blockchain_fixture_mark_fn(fixture_path, fixture_name, fixture_fork):
182199
for slow_test in SLOW_TESTS:
183200
if slow_test in fixture_path or slow_test in fixture_name:
184201
if not should_run_slow_tests():
@@ -188,13 +205,33 @@ def blockchain_fixture_mark_fn(fixture_path, fixture_name):
188205
return pytest.mark.xfail(reason="Listed in INCORRECT_UPSTREAM_TESTS.")
189206

190207

208+
def generate_ignore_fn_for_fork(passed_fork):
209+
if passed_fork:
210+
normalized_fork = passed_fork.lower()
211+
212+
def ignore_fn(fixture_path, fixture_key, fixture_fork):
213+
return fixture_fork.lower() != normalized_fork
214+
215+
return ignore_fn
216+
217+
218+
@to_tuple
219+
def expand_fixtures_forks(all_fixtures):
220+
for fixture_path, fixture_key in all_fixtures:
221+
fixture = load_fixture(fixture_path, fixture_key)
222+
yield fixture_path, fixture_key, fixture['network']
223+
224+
191225
def pytest_generate_tests(metafunc):
226+
ignore_fn = generate_ignore_fn_for_fork(metafunc.config.getoption('fork'))
192227
generate_fixture_tests(
193228
metafunc=metafunc,
194229
base_fixture_path=BASE_FIXTURE_PATH,
230+
preprocess_fn=expand_fixtures_forks,
195231
filter_fn=filter_fixtures(
196232
fixtures_base_dir=BASE_FIXTURE_PATH,
197233
mark_fn=blockchain_fixture_mark_fn,
234+
ignore_fn=ignore_fn
198235
),
199236
)
200237

@@ -375,9 +412,10 @@ async def validate_uncles(rpc, block_fixture, at_block):
375412

376413
@pytest.fixture
377414
def chain_fixture(fixture_data):
378-
fixture = load_fixture(*fixture_data)
379-
if fixture['network'] == 'Constantinople':
380-
pytest.skip('Constantinople VM rules not yet supported')
415+
fixture_path, fixture_key, fixture_fork = fixture_data
416+
fixture = load_fixture(fixture_path, fixture_key)
417+
if fixture_fork == 'Istanbul':
418+
pytest.skip('Istanbul VM rules not yet supported')
381419
return fixture
382420

383421

‎tox.ini‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
[tox]
22
envlist=
33
py{36,37}-{core,p2p,integration,lightchain_integration,eth2-core,eth2-fixtures,eth2-integration,plugins}
4-
py{36}-long_run_integration
5-
py{36}-rpc-state-{frontier,homestead,tangerine_whistle,spurious_dragon,byzantium,constantinople,quadratic}
6-
py{36}-rpc-blockchain
4+
py36-long_run_integration
5+
py36-rpc-blockchain
6+
py36-rpc-state-{frontier,homestead,tangerine_whistle,spurious_dragon,byzantium,constantinople,petersburg}
7+
py37-rpc-state-{quadratic,sstore,zero_knowledge}
78
py{36,37}-libp2p
89
py{36,37}-lint
910
py{36,37}-wheel-cli
@@ -26,14 +27,19 @@ commands=
2627
p2p: pytest -n 4 {posargs:tests/p2p}
2728
plugins: pytest -n 4 {posargs:tests/plugins/}
2829
rpc-blockchain: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'not GeneralStateTests'}
29-
rpc-state-frontier: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and not stQuadraticComplexityTest and Frontier'}
30-
rpc-state-homestead: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and not stQuadraticComplexityTest and Homestead'}
31-
rpc-state-tangerine_whistle: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and not stQuadraticComplexityTest and EIP150'}
32-
rpc-state-spurious_dragon: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and not stQuadraticComplexityTest and EIP158'}
30+
# Fork/VM-specific state transition tests; long-running categories run separately!
31+
rpc-state-frontier: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Frontier -k 'GeneralStateTests and not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
32+
rpc-state-homestead: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Homestead -k 'GeneralStateTests and not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
33+
rpc-state-tangerine_whistle: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork EIP150 -k 'GeneralStateTests and not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
34+
rpc-state-spurious_dragon: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork EIP158 -k 'GeneralStateTests and not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
3335
# The following test seems to consume a lot of memory. Restricting to 3 processes reduces crashes
34-
rpc-state-byzantium: pytest -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and not stQuadraticComplexityTest and Byzantium'}
35-
rpc-state-constantinople: pytest -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and not stQuadraticComplexityTest and Constantinople'}
36+
rpc-state-byzantium: pytest -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Byzantium -k 'GeneralStateTests and not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
37+
rpc-state-constantinople: pytest -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork Constantinople -k 'GeneralStateTests and not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
38+
rpc-state-petersburg: pytest -n 3 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py --fork ConstantinopleFix -k 'GeneralStateTests and not stQuadraticComplexityTest and not stSStoreTest and not stZeroKnowledge'}
39+
# Long-running categories.
3640
rpc-state-quadratic: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and stQuadraticComplexityTest'}
41+
rpc-state-sstore: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and stSStoreTest'}
42+
rpc-state-zero_knowledge: pytest -n 4 {posargs:tests/json-fixtures-over-rpc/test_rpc_fixtures.py -k 'GeneralStateTests and stZeroKnowledge'}
3743
lightchain_integration: pytest --integration {posargs:tests/integration/test_lightchain_integration.py}
3844

3945
deps = .[p2p,trinity,eth2,test]

0 commit comments

Comments
 (0)