From 6d83034bb886f1605131e68da463070041ef67c0 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sun, 28 Nov 2021 10:30:05 -0300 Subject: [PATCH 1/4] Fix test_warning_captured_deprecated_in_pytest_6 This test started to fail in the 'py38-pytestmain' environment, the cause being PytestRemovedIn7Warning being raised by the conftest file of the test itself. Ignoring it is fine, the purpose of the test is to ensure the hook is not called by pytest-xdist. --- testing/acceptance_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 82513a4b..31d634bf 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -790,7 +790,7 @@ def test(): warnings.warn("my custom worker warning") """ ) - result = pytester.runpytest("-n1") + result = pytester.runpytest("-n1", "-Wignore") result.stdout.fnmatch_lines(["*1 passed*"]) result.stdout.no_fnmatch_line("*this hook should not be called in this version") From 52a395888ffb3e65e015041a7a23b602cc92f145 Mon Sep 17 00:00:00 2001 From: baekdohyeop Date: Mon, 29 Nov 2021 00:03:49 +0900 Subject: [PATCH 2/4] Create new dist option 'loadgroup' --- src/xdist/dsession.py | 2 + src/xdist/plugin.py | 10 ++- src/xdist/remote.py | 15 +++++ src/xdist/scheduler/__init__.py | 1 + src/xdist/scheduler/loadgroup.py | 67 +++++++++++++++++++ testing/acceptance_test.py | 109 +++++++++++++++++++++++++++++++ 6 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 src/xdist/scheduler/loadgroup.py diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index 12539a00..eb5ae751 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -7,6 +7,7 @@ LoadScheduling, LoadScopeScheduling, LoadFileScheduling, + LoadGroupScheduling, ) @@ -100,6 +101,7 @@ def pytest_xdist_make_scheduler(self, config, log): "load": LoadScheduling, "loadscope": LoadScopeScheduling, "loadfile": LoadFileScheduling, + "loadgroup": LoadGroupScheduling, } return schedulers[dist](config, log) diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index 17be6d04..b406d0ba 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -86,7 +86,7 @@ def pytest_addoption(parser): "--dist", metavar="distmode", action="store", - choices=["each", "load", "loadscope", "loadfile", "no"], + choices=["each", "load", "loadscope", "loadfile", "loadgroup", "no"], dest="dist", default="no", help=( @@ -98,6 +98,8 @@ def pytest_addoption(parser): " the same scope to any available environment.\n\n" "loadfile: load balance by sending test grouped by file" " to any available environment.\n\n" + "loadgroup: load balance by sending any pending test or test group" + " to any available enviroment.\n\n" "(default) no: run tests inprocess, don't distribute." ), ) @@ -204,6 +206,12 @@ def pytest_configure(config): config.issue_config_time_warning(warning, 2) config.option.forked = True + config_line = ( + "xgroup: specify group for tests should run in same session." + "in relation to one another. " + "Provided by pytest-xdist." + ) + config.addinivalue_line("markers", config_line) + @pytest.hookimpl(tryfirst=True) def pytest_cmdline_main(config): diff --git a/src/xdist/remote.py b/src/xdist/remote.py index 914040d4..410d3ca9 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -116,6 +116,20 @@ def run_one_test(self, torun): "runtest_protocol_complete", item_index=self.item_index, duration=duration ) + def pytest_collection_modifyitems(self, session, config, items): + # add the group name to nodeid as suffix if --dist=loadgroup + if config.getvalue("loadgroup"): + for item in items: + try: + mark = item.get_closest_marker("xgroup") + except AttributeError: + mark = item.get_marker("xgroup") + + if mark: + gname = mark.kwargs.get("name") + if gname: + item._nodeid = "{}@{}".format(item.nodeid, gname) + @pytest.hookimpl def pytest_collection_finish(self, session): try: @@ -236,6 +250,7 @@ def remote_initconfig(option_dict, args): def setup_config(config, basetemp): + config.option.loadgroup = True if config.getvalue("dist") == "loadgroup" else False config.option.looponfail = False config.option.usepdb = False config.option.dist = "no" diff --git a/src/xdist/scheduler/__init__.py b/src/xdist/scheduler/__init__.py index 06ba6b7b..ab2e830f 100644 --- a/src/xdist/scheduler/__init__.py +++ b/src/xdist/scheduler/__init__.py @@ -2,3 +2,4 @@ from xdist.scheduler.load import LoadScheduling # noqa from xdist.scheduler.loadfile import LoadFileScheduling # noqa from xdist.scheduler.loadscope import LoadScopeScheduling # noqa +from xdist.scheduler.loadgroup import LoadGroupScheduling # noqa diff --git a/src/xdist/scheduler/loadgroup.py b/src/xdist/scheduler/loadgroup.py new file mode 100644 index 00000000..49951d89 --- /dev/null +++ b/src/xdist/scheduler/loadgroup.py @@ -0,0 +1,67 @@ +from .loadscope import LoadScopeScheduling +from py.log import Producer + + +class LoadGroupScheduling(LoadScopeScheduling): + """Implement load scheduling across nodes, but grouping test only has group mark. + + This distributes the tests collected across all nodes so each test is run + just once. All nodes collect and submit the list of tests and when all + collections are received it is verified they are identical collections. + Then the collection gets divided up in work units, grouped by group mark + (If there is no group mark, it is itself a group.), and those work units + et submitted to nodes. Whenever a node finishes an item, it calls + ``.mark_test_complete()`` which will trigger the scheduler to assign more + work units if the number of pending tests for the node falls below a low-watermark. + + When created, ``numnodes`` defines how many nodes are expected to submit a + collection. This is used to know when all nodes have finished collection. + + This class behaves very much like LoadScopeScheduling, + but with a itself or group(by marked) scope. + """ + + def __init__(self, config, log=None): + super().__init__(config, log) + if log is None: + self.log = Producer("loadgroupsched") + else: + self.log = log.loadgroupsched + + def _split_scope(self, nodeid): + """Determine the scope (grouping) of a nodeid. + + There are usually 3 cases for a nodeid:: + + example/loadsuite/test/test_beta.py::test_beta0 + example/loadsuite/test/test_delta.py::Delta1::test_delta0 + example/loadsuite/epsilon/__init__.py::epsilon.epsilon + + #. Function in a test module. + #. Method of a class in a test module. + #. Doctest in a function in a package. + + With loadgroup, two cases are added:: + + example/loadsuite/test/test_beta.py::test_beta0 + example/loadsuite/test/test_delta.py::Delta1::test_delta0 + example/loadsuite/epsilon/__init__.py::epsilon.epsilon + example/loadsuite/test/test_gamma.py::test_beta0@gname + example/loadsuite/test/test_delta.py::Gamma1::test_gamma0@gname + + This function will group tests with the scope determined by splitting + the first ``@`` from the right. That is, test will be grouped in a + single work unit when they have same group name. + In the above example, scopes will be:: + + example/loadsuite/test/test_beta.py::test_beta0 + example/loadsuite/test/test_delta.py::Delta1::test_delta0 + example/loadsuite/epsilon/__init__.py::epsilon.epsilon + gname + gname + """ + if nodeid.rfind("@") > nodeid.rfind("]"): + # check the index of ']' to avoid the case: parametrize mark value has '@' + return nodeid.split("@")[-1] + else: + return nodeid diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 31d634bf..c7f99567 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -1326,6 +1326,115 @@ def test_2(): assert c1 == c2 +class TestGroupScope: + def test_by_module(self, testdir): + test_file = """ + import pytest + class TestA: + @pytest.mark.xgroup(name="xgroup") + @pytest.mark.parametrize('i', range(5)) + def test(self, i): + pass + """ + testdir.makepyfile(test_a=test_file, test_b=test_file) + result = testdir.runpytest("-n2", "--dist=loadgroup", "-v") + test_a_workers_and_test_count = get_workers_and_test_count_by_prefix( + "test_a.py::TestA", result.outlines + ) + test_b_workers_and_test_count = get_workers_and_test_count_by_prefix( + "test_b.py::TestA", result.outlines + ) + + assert ( + test_a_workers_and_test_count + in ( + {"gw0": 5}, + {"gw1": 0}, + ) + or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 5}) + ) + assert ( + test_b_workers_and_test_count + in ( + {"gw0": 5}, + {"gw1": 0}, + ) + or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 5}) + ) + assert ( + test_a_workers_and_test_count.items() + == test_b_workers_and_test_count.items() + ) + + def test_by_class(self, testdir): + testdir.makepyfile( + test_a=""" + import pytest + class TestA: + @pytest.mark.xgroup(name="xgroup") + @pytest.mark.parametrize('i', range(10)) + def test(self, i): + pass + class TestB: + @pytest.mark.xgroup(name="xgroup") + @pytest.mark.parametrize('i', range(10)) + def test(self, i): + pass + """ + ) + result = testdir.runpytest("-n2", "--dist=loadgroup", "-v") + test_a_workers_and_test_count = get_workers_and_test_count_by_prefix( + "test_a.py::TestA", result.outlines + ) + test_b_workers_and_test_count = get_workers_and_test_count_by_prefix( + "test_a.py::TestB", result.outlines + ) + + assert ( + test_a_workers_and_test_count + in ( + {"gw0": 10}, + {"gw1": 0}, + ) + or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10}) + ) + assert ( + test_b_workers_and_test_count + in ( + {"gw0": 10}, + {"gw1": 0}, + ) + or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10}) + ) + assert ( + test_a_workers_and_test_count.items() + == test_b_workers_and_test_count.items() + ) + + def test_module_single_start(self, testdir): + test_file1 = """ + import pytest + @pytest.mark.xgroup(name="xgroup") + def test(): + pass + """ + test_file2 = """ + import pytest + def test_1(): + pass + @pytest.mark.xgroup(name="xgroup") + def test_2(): + pass + """ + testdir.makepyfile(test_a=test_file1, test_b=test_file1, test_c=test_file2) + result = testdir.runpytest("-n2", "--dist=loadgroup", "-v") + a = get_workers_and_test_count_by_prefix("test_a.py::test", result.outlines) + b = get_workers_and_test_count_by_prefix("test_b.py::test", result.outlines) + c = get_workers_and_test_count_by_prefix("test_c.py::test_2", result.outlines) + + assert a.keys() == b.keys() and b.keys() == c.keys() + + class TestLocking: _test_content = """ class TestClassName%s(object): From 1b5d6b6db76d58a0c9985f6aeb634630590b06eb Mon Sep 17 00:00:00 2001 From: baekdohyeop Date: Mon, 29 Nov 2021 00:11:26 +0900 Subject: [PATCH 3/4] Add changelog --- changelog/733.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/733.feature.rst diff --git a/changelog/733.feature.rst b/changelog/733.feature.rst new file mode 100644 index 00000000..3ce8de0c --- /dev/null +++ b/changelog/733.feature.rst @@ -0,0 +1 @@ +Create new dist option 'loadgroup' From 62e50d00977b41e175b5f119381f9db760459ddc Mon Sep 17 00:00:00 2001 From: baekdohyeop Date: Mon, 29 Nov 2021 02:49:50 +0900 Subject: [PATCH 4/4] Address review --- README.rst | 22 ++++++++++++++++++++++ changelog/733.feature.rst | 2 +- src/xdist/plugin.py | 5 ++--- src/xdist/remote.py | 20 ++++++++++---------- src/xdist/scheduler/loadgroup.py | 25 ++++++------------------- testing/acceptance_test.py | 29 ++++++++++++++++++++++++----- 6 files changed, 65 insertions(+), 38 deletions(-) diff --git a/README.rst b/README.rst index 5768f7ca..c40c9f3e 100644 --- a/README.rst +++ b/README.rst @@ -96,6 +96,10 @@ distribution algorithm this with the ``--dist`` option. It takes these values: distributed to available workers as whole units. This guarantees that all tests in a file run in the same worker. +* ``--dist loadgroup``: Tests are grouped by xdist_group mark. Groups are + distributed to available workers as whole units. This guarantees that all + tests with same xdist_group name run in the same worker. + Making session-scoped fixtures execute only once ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -414,3 +418,21 @@ where the configuration file was found. .. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist .. _`pytest-xdist repository`: https://github.com/pytest-dev/pytest-xdist .. _`pytest`: http://pytest.org + +Groups tests by xdist_group mark +--------------------------------- + +*New in version 2.4.* + +Two or more tests belonging to different classes or modules can be executed in same worker through the xdist_group marker: + +.. code-block:: python + + @pytest.mark.xdist_group(name="group1") + def test1(): + pass + + class TestA: + @pytest.mark.xdist_group("group1") + def test2(): + pass diff --git a/changelog/733.feature.rst b/changelog/733.feature.rst index 3ce8de0c..28163e79 100644 --- a/changelog/733.feature.rst +++ b/changelog/733.feature.rst @@ -1 +1 @@ -Create new dist option 'loadgroup' +New ``--dist=loadgroup`` option, which ensures all tests marked with ``@pytest.mark.xdist_group`` run in the same session/worker. Other tests run distributed as in ``--dist=load``. diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index b406d0ba..85f76e82 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -98,8 +98,7 @@ def pytest_addoption(parser): " the same scope to any available environment.\n\n" "loadfile: load balance by sending test grouped by file" " to any available environment.\n\n" - "loadgroup: load balance by sending any pending test or test group" - " to any available enviroment.\n\n" + "loadgroup: like load, but sends tests marked with 'xdist_group' to the same worker.\n\n" "(default) no: run tests inprocess, don't distribute." ), ) @@ -207,7 +206,7 @@ def pytest_configure(config): config.option.forked = True config_line = ( - "xgroup: specify group for tests should run in same session." + "xdist_group: specify group for tests should run in same session." "in relation to one another. " + "Provided by pytest-xdist." ) config.addinivalue_line("markers", config_line) diff --git a/src/xdist/remote.py b/src/xdist/remote.py index 410d3ca9..160b042a 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -120,15 +120,15 @@ def pytest_collection_modifyitems(self, session, config, items): # add the group name to nodeid as suffix if --dist=loadgroup if config.getvalue("loadgroup"): for item in items: - try: - mark = item.get_closest_marker("xgroup") - except AttributeError: - mark = item.get_marker("xgroup") - - if mark: - gname = mark.kwargs.get("name") - if gname: - item._nodeid = "{}@{}".format(item.nodeid, gname) + mark = item.get_closest_marker("xdist_group") + if not mark: + continue + gname = ( + mark.args[0] + if len(mark.args) > 0 + else mark.kwargs.get("name", "default") + ) + item._nodeid = "{}@{}".format(item.nodeid, gname) @pytest.hookimpl def pytest_collection_finish(self, session): @@ -250,7 +250,7 @@ def remote_initconfig(option_dict, args): def setup_config(config, basetemp): - config.option.loadgroup = True if config.getvalue("dist") == "loadgroup" else False + config.option.loadgroup = config.getvalue("dist") == "loadgroup" config.option.looponfail = False config.option.usepdb = False config.option.dist = "no" diff --git a/src/xdist/scheduler/loadgroup.py b/src/xdist/scheduler/loadgroup.py index 49951d89..072f64ab 100644 --- a/src/xdist/scheduler/loadgroup.py +++ b/src/xdist/scheduler/loadgroup.py @@ -3,22 +3,10 @@ class LoadGroupScheduling(LoadScopeScheduling): - """Implement load scheduling across nodes, but grouping test only has group mark. + """Implement load scheduling across nodes, but grouping test by xdist_group mark. - This distributes the tests collected across all nodes so each test is run - just once. All nodes collect and submit the list of tests and when all - collections are received it is verified they are identical collections. - Then the collection gets divided up in work units, grouped by group mark - (If there is no group mark, it is itself a group.), and those work units - et submitted to nodes. Whenever a node finishes an item, it calls - ``.mark_test_complete()`` which will trigger the scheduler to assign more - work units if the number of pending tests for the node falls below a low-watermark. - - When created, ``numnodes`` defines how many nodes are expected to submit a - collection. This is used to know when all nodes have finished collection. - - This class behaves very much like LoadScopeScheduling, - but with a itself or group(by marked) scope. + This class behaves very much like LoadScopeScheduling, but it groups tests by xdist_group mark + instead of the module or class to which they belong to. """ def __init__(self, config, log=None): @@ -49,10 +37,9 @@ def _split_scope(self, nodeid): example/loadsuite/test/test_gamma.py::test_beta0@gname example/loadsuite/test/test_delta.py::Gamma1::test_gamma0@gname - This function will group tests with the scope determined by splitting - the first ``@`` from the right. That is, test will be grouped in a - single work unit when they have same group name. - In the above example, scopes will be:: + This function will group tests with the scope determined by splitting the first ``@`` + from the right. That is, test will be grouped in a single work unit when they have + same group name. In the above example, scopes will be:: example/loadsuite/test/test_beta.py::test_beta0 example/loadsuite/test/test_delta.py::Delta1::test_delta0 diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index c7f99567..c1391974 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -1331,7 +1331,7 @@ def test_by_module(self, testdir): test_file = """ import pytest class TestA: - @pytest.mark.xgroup(name="xgroup") + @pytest.mark.xdist_group(name="xdist_group") @pytest.mark.parametrize('i', range(5)) def test(self, i): pass @@ -1371,12 +1371,12 @@ def test_by_class(self, testdir): test_a=""" import pytest class TestA: - @pytest.mark.xgroup(name="xgroup") + @pytest.mark.xdist_group(name="xdist_group") @pytest.mark.parametrize('i', range(10)) def test(self, i): pass class TestB: - @pytest.mark.xgroup(name="xgroup") + @pytest.mark.xdist_group(name="xdist_group") @pytest.mark.parametrize('i', range(10)) def test(self, i): pass @@ -1414,7 +1414,7 @@ def test(self, i): def test_module_single_start(self, testdir): test_file1 = """ import pytest - @pytest.mark.xgroup(name="xgroup") + @pytest.mark.xdist_group(name="xdist_group") def test(): pass """ @@ -1422,7 +1422,7 @@ def test(): import pytest def test_1(): pass - @pytest.mark.xgroup(name="xgroup") + @pytest.mark.xdist_group(name="xdist_group") def test_2(): pass """ @@ -1434,6 +1434,25 @@ def test_2(): assert a.keys() == b.keys() and b.keys() == c.keys() + def test_with_two_group_names(self, testdir): + test_file = """ + import pytest + @pytest.mark.xdist_group(name="group1") + def test_1(): + pass + @pytest.mark.xdist_group("group2") + def test_2(): + pass + """ + testdir.makepyfile(test_a=test_file, test_b=test_file) + result = testdir.runpytest("-n2", "--dist=loadgroup", "-v") + a_1 = get_workers_and_test_count_by_prefix("test_a.py::test_1", result.outlines) + a_2 = get_workers_and_test_count_by_prefix("test_a.py::test_2", result.outlines) + b_1 = get_workers_and_test_count_by_prefix("test_b.py::test_1", result.outlines) + b_2 = get_workers_and_test_count_by_prefix("test_b.py::test_2", result.outlines) + + assert a_1.keys() == b_1.keys() and a_2.keys() == b_2.keys() + class TestLocking: _test_content = """