From 3f119cae04826f529b438137ca19f87e02156afd Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 15 Apr 2020 10:48:30 +0100 Subject: [PATCH] Add pytest-xdist support Fixes #201. --- HISTORY.rst | 7 +++++++ requirements/py35.txt | 20 ++++++++++++++++++-- requirements/py36.txt | 20 ++++++++++++++++++-- requirements/py37.txt | 20 ++++++++++++++++++-- requirements/py38.txt | 20 ++++++++++++++++++-- requirements/requirements.in | 1 + src/pytest_randomly.py | 35 ++++++++++++++++++++++++++--------- tests/test_pytest_randomly.py | 22 ++++++++++++++++++++++ 8 files changed, 128 insertions(+), 17 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bfb844b..0ce4478 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,13 @@ History ------- +* Add `pytest-xdist `__ support. + Previously it only worked reliably when setting ``--randomly-seed`` + explicitly. When not provided, the default seed generated in workers could + differ and collection would fail. Now when it is not provided, all xdist + worker processes shared the same default seed generated in the master + process. + 3.2.1 (2020-01-13) ------------------ diff --git a/requirements/py35.txt b/requirements/py35.txt index b4b3f82..0523fe4 100644 --- a/requirements/py35.txt +++ b/requirements/py35.txt @@ -4,6 +4,10 @@ # # requirements/compile.py # +apipkg==1.5 \ + --hash=sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6 \ + --hash=sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c \ + # via execnet attrs==19.3.0 \ --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \ --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72 \ @@ -79,6 +83,10 @@ entrypoints==0.3 \ --hash=sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19 \ --hash=sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451 \ # via flake8 +execnet==1.7.1 \ + --hash=sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50 \ + --hash=sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547 \ + # via pytest-xdist factory-boy==2.12.0 \ --hash=sha256:728df59b372c9588b83153facf26d3d28947fc750e8e3c95cefa9bed0e6394ee \ --hash=sha256:faf48d608a1735f0d0a3c9cbf536d64f9132b547dae7ba452c4d99a79e84a370 \ @@ -194,10 +202,18 @@ pyparsing==2.4.7 \ --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \ # via packaging +pytest-forked==1.1.3 \ + --hash=sha256:1805699ed9c9e60cb7a8179b8d4fa2b8898098e82d229b0825d8095f0f261100 \ + --hash=sha256:1ae25dba8ee2e56fb47311c9638f9e58552691da87e82d25b0ce0e4bf52b7d87 \ + # via pytest-xdist +pytest-xdist==1.31.0 \ + --hash=sha256:0f46020d3d9619e6d17a65b5b989c1ebbb58fc7b1da8fb126d70f4bac4dfeed1 \ + --hash=sha256:7dc0d027d258cd0defc618fb97055fbd1002735ca7a6d17037018cf870e24011 \ + # via -r requirements.in pytest==5.4.1 \ --hash=sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172 \ --hash=sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970 \ - # via -r requirements.in + # via -r requirements.in, pytest-forked, pytest-xdist python-dateutil==2.8.1 \ --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \ --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \ @@ -221,7 +237,7 @@ secretstorage==3.1.2 \ six==1.14.0 \ --hash=sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a \ --hash=sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c \ - # via bleach, cryptography, packaging, pathlib2, python-dateutil, readme-renderer + # via bleach, cryptography, packaging, pathlib2, pytest-xdist, python-dateutil, readme-renderer text-unidecode==1.3 \ --hash=sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8 \ --hash=sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93 \ diff --git a/requirements/py36.txt b/requirements/py36.txt index 6c01c62..699fa12 100644 --- a/requirements/py36.txt +++ b/requirements/py36.txt @@ -4,6 +4,10 @@ # # requirements/compile.py # +apipkg==1.5 \ + --hash=sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6 \ + --hash=sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c \ + # via execnet attrs==19.3.0 \ --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \ --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72 \ @@ -79,6 +83,10 @@ entrypoints==0.3 \ --hash=sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19 \ --hash=sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451 \ # via flake8 +execnet==1.7.1 \ + --hash=sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50 \ + --hash=sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547 \ + # via pytest-xdist factory-boy==2.12.0 \ --hash=sha256:728df59b372c9588b83153facf26d3d28947fc750e8e3c95cefa9bed0e6394ee \ --hash=sha256:faf48d608a1735f0d0a3c9cbf536d64f9132b547dae7ba452c4d99a79e84a370 \ @@ -194,10 +202,18 @@ pyparsing==2.4.7 \ --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \ # via packaging +pytest-forked==1.1.3 \ + --hash=sha256:1805699ed9c9e60cb7a8179b8d4fa2b8898098e82d229b0825d8095f0f261100 \ + --hash=sha256:1ae25dba8ee2e56fb47311c9638f9e58552691da87e82d25b0ce0e4bf52b7d87 \ + # via pytest-xdist +pytest-xdist==1.31.0 \ + --hash=sha256:0f46020d3d9619e6d17a65b5b989c1ebbb58fc7b1da8fb126d70f4bac4dfeed1 \ + --hash=sha256:7dc0d027d258cd0defc618fb97055fbd1002735ca7a6d17037018cf870e24011 \ + # via -r requirements.in pytest==5.4.1 \ --hash=sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172 \ --hash=sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970 \ - # via -r requirements.in + # via -r requirements.in, pytest-forked, pytest-xdist python-dateutil==2.8.1 \ --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \ --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \ @@ -221,7 +237,7 @@ secretstorage==3.1.2 \ six==1.14.0 \ --hash=sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a \ --hash=sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c \ - # via bleach, cryptography, packaging, python-dateutil, readme-renderer + # via bleach, cryptography, packaging, pytest-xdist, python-dateutil, readme-renderer text-unidecode==1.3 \ --hash=sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8 \ --hash=sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93 \ diff --git a/requirements/py37.txt b/requirements/py37.txt index 6c01c62..699fa12 100644 --- a/requirements/py37.txt +++ b/requirements/py37.txt @@ -4,6 +4,10 @@ # # requirements/compile.py # +apipkg==1.5 \ + --hash=sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6 \ + --hash=sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c \ + # via execnet attrs==19.3.0 \ --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \ --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72 \ @@ -79,6 +83,10 @@ entrypoints==0.3 \ --hash=sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19 \ --hash=sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451 \ # via flake8 +execnet==1.7.1 \ + --hash=sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50 \ + --hash=sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547 \ + # via pytest-xdist factory-boy==2.12.0 \ --hash=sha256:728df59b372c9588b83153facf26d3d28947fc750e8e3c95cefa9bed0e6394ee \ --hash=sha256:faf48d608a1735f0d0a3c9cbf536d64f9132b547dae7ba452c4d99a79e84a370 \ @@ -194,10 +202,18 @@ pyparsing==2.4.7 \ --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \ # via packaging +pytest-forked==1.1.3 \ + --hash=sha256:1805699ed9c9e60cb7a8179b8d4fa2b8898098e82d229b0825d8095f0f261100 \ + --hash=sha256:1ae25dba8ee2e56fb47311c9638f9e58552691da87e82d25b0ce0e4bf52b7d87 \ + # via pytest-xdist +pytest-xdist==1.31.0 \ + --hash=sha256:0f46020d3d9619e6d17a65b5b989c1ebbb58fc7b1da8fb126d70f4bac4dfeed1 \ + --hash=sha256:7dc0d027d258cd0defc618fb97055fbd1002735ca7a6d17037018cf870e24011 \ + # via -r requirements.in pytest==5.4.1 \ --hash=sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172 \ --hash=sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970 \ - # via -r requirements.in + # via -r requirements.in, pytest-forked, pytest-xdist python-dateutil==2.8.1 \ --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \ --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \ @@ -221,7 +237,7 @@ secretstorage==3.1.2 \ six==1.14.0 \ --hash=sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a \ --hash=sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c \ - # via bleach, cryptography, packaging, python-dateutil, readme-renderer + # via bleach, cryptography, packaging, pytest-xdist, python-dateutil, readme-renderer text-unidecode==1.3 \ --hash=sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8 \ --hash=sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93 \ diff --git a/requirements/py38.txt b/requirements/py38.txt index b2d4a03..1a6ffae 100644 --- a/requirements/py38.txt +++ b/requirements/py38.txt @@ -4,6 +4,10 @@ # # requirements/compile.py # +apipkg==1.5 \ + --hash=sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6 \ + --hash=sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c \ + # via execnet appdirs==1.4.3 \ --hash=sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92 \ --hash=sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e \ @@ -95,6 +99,10 @@ entrypoints==0.3 \ --hash=sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19 \ --hash=sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451 \ # via flake8 +execnet==1.7.1 \ + --hash=sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50 \ + --hash=sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547 \ + # via pytest-xdist factory-boy==2.12.0 \ --hash=sha256:728df59b372c9588b83153facf26d3d28947fc750e8e3c95cefa9bed0e6394ee \ --hash=sha256:faf48d608a1735f0d0a3c9cbf536d64f9132b547dae7ba452c4d99a79e84a370 \ @@ -214,10 +222,18 @@ pyparsing==2.4.7 \ --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \ # via packaging +pytest-forked==1.1.3 \ + --hash=sha256:1805699ed9c9e60cb7a8179b8d4fa2b8898098e82d229b0825d8095f0f261100 \ + --hash=sha256:1ae25dba8ee2e56fb47311c9638f9e58552691da87e82d25b0ce0e4bf52b7d87 \ + # via pytest-xdist +pytest-xdist==1.31.0 \ + --hash=sha256:0f46020d3d9619e6d17a65b5b989c1ebbb58fc7b1da8fb126d70f4bac4dfeed1 \ + --hash=sha256:7dc0d027d258cd0defc618fb97055fbd1002735ca7a6d17037018cf870e24011 \ + # via -r requirements.in pytest==5.4.1 \ --hash=sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172 \ --hash=sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970 \ - # via -r requirements.in + # via -r requirements.in, pytest-forked, pytest-xdist python-dateutil==2.8.1 \ --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \ --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \ @@ -264,7 +280,7 @@ secretstorage==3.1.2 \ six==1.14.0 \ --hash=sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a \ --hash=sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c \ - # via bleach, cryptography, packaging, python-dateutil, readme-renderer + # via bleach, cryptography, packaging, pytest-xdist, python-dateutil, readme-renderer text-unidecode==1.3 \ --hash=sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8 \ --hash=sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93 \ diff --git a/requirements/requirements.in b/requirements/requirements.in index 1f23eff..e7d11b2 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -13,5 +13,6 @@ multilint numpy pygments pytest +pytest-xdist secretstorage # required for twine on linux twine diff --git a/src/pytest_randomly.py b/src/pytest_randomly.py index fda8339..3dcb524 100644 --- a/src/pytest_randomly.py +++ b/src/pytest_randomly.py @@ -46,7 +46,7 @@ def seed_type(string): - if string == "last": + if string in ("default", "last"): return string try: return int(string) @@ -62,7 +62,7 @@ def pytest_addoption(parser): "--randomly-seed", action="store", dest="randomly_seed", - default=str(default_seed), + default="default", type=seed_type, help="""Set the seed that pytest-randomly uses (int), or pass the special value 'last' to reuse the seed from the previous run. @@ -87,6 +87,29 @@ def pytest_addoption(parser): ) +def pytest_configure(config): + seed_value = config.getoption("randomly_seed") + if seed_value == "last": + seed = config.cache.get("randomly_seed", default_seed) + elif seed_value == "default": + if hasattr(config, 'workerinput'): + # pytest-xdist: use seed generated on master. + seed = config.workerinput['randomly_seed'] + else: + seed = default_seed + else: + seed = seed_value + config.cache.set("randomly_seed", seed) + config.option.randomly_seed = seed + + +def pytest_configure_node(node): + """ + pytest-xdist hook. Send the selected seed through to the nodes. + """ + node.workerinput['randomly_seed'] = node.config.getoption("randomly_seed") + + random_states = {} np_random_states = {} @@ -125,13 +148,7 @@ def _reseed(config, offset=0): def pytest_report_header(config): - seed_value = config.getoption("randomly_seed") - if seed_value == "last": - seed = config.cache.get("randomly_seed", default_seed) - else: - seed = seed_value - config.cache.set("randomly_seed", seed) - config.option.randomly_seed = seed + seed = config.getoption("randomly_seed") _reseed(config) return "Using --randomly-seed={}".format(seed) diff --git a/tests/test_pytest_randomly.py b/tests/test_pytest_randomly.py index 198c9f4..531f92b 100644 --- a/tests/test_pytest_randomly.py +++ b/tests/test_pytest_randomly.py @@ -668,3 +668,25 @@ def fake_entry_points(): # Need to run in-process so that monkeypatching works testdir.runpytest("--randomly-seed=1") + + +@pytest.mark.parametrize("n", list(range(5))) +def test_xdist(n, ourtestdir): + """ + This test does not expose the original bug (non-shared default seeds) with + a very high probability, hence multiple runs. + """ + ourtestdir.makepyfile( + test_one="def test_a(): pass", + test_two="def test_a(): pass", + test_three="def test_a(): pass", + test_four="def test_a(): pass", + test_five="def test_a(): pass", + test_six="def test_a(): pass", + ) + + out = ourtestdir.runpytest("-n 6", "-v", "--dist=loadfile") + out.assert_outcomes(passed=6) + + # Can't make any assertion on the order, since output comes back from + # workers non-deterministically