From b8245200af486af69dc32a2328773ead9bb43efc Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 4 Aug 2022 10:04:30 +0300 Subject: [PATCH 1/4] Introduce a new option for specifying the test execution order --- reframe/frontend/cli.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index eb0e5458d0..2c0df8e2b9 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -7,6 +7,7 @@ import itertools import json import os +import random import shlex import socket import sys @@ -25,6 +26,7 @@ import reframe.frontend.dependencies as dependencies import reframe.frontend.filters as filters import reframe.frontend.runreport as runreport +import reframe.utility as util import reframe.utility.jsonext as jsonext import reframe.utility.osext as osext import reframe.utility.typecheck as typ @@ -386,6 +388,11 @@ def main(): help=('Distribute the selected single-node jobs on every node that' 'is in STATE (default: "idle"') ) + run_options.add_argument( + '--exec-order', metavar='ORDER', action='store', + choices=['name', 'random', 'rname', 'ruid', 'uid'], + help='Impose an execution order for tests' + ) run_options.add_argument( '--exec-policy', metavar='POLICY', action='store', choices=['async', 'serial'], default='async', @@ -1061,8 +1068,8 @@ def _case_failed(t): "a non-negative integer" ) from None - testcases = repeat_tests(testcases, num_repeats) - testcases_all = testcases + testcases_all = repeat_tests(testcases, num_repeats) + testcases = testcases_all if options.distribute: node_map = getallnodes(options.distribute, parsed_job_options) @@ -1077,15 +1084,31 @@ def _case_failed(t): x for x in parsed_job_options if (not x.startswith('-w') and not x.startswith('--nodelist')) ] - testcases = distribute_tests(testcases, node_map) - testcases_all = testcases + testcases_all = distribute_tests(testcases, node_map) + testcases = testcases_all + + @logging.time_function + def _sort_testcases(testcases): + if options.exec_order in ('name', 'rname'): + testcases.sort(key=lambda c: c.check.display_name, + reverse=(options.exec_order == 'rname')) + elif options.exec_order in ('uid', 'ruid'): + testcases.sort(key=lambda c: c.check.unique_name, + reverse=(options.exec_order == 'ruid')) + elif options.exec_order == 'random': + random.shuffle(testcases) + + _sort_testcases(testcases) + if testcases_all is not testcases: + _sort_testcases(testcases_all) # Prepare for running printer.debug('Building and validating the full test DAG') testgraph, skipped_cases = dependencies.build_deps(testcases_all) if skipped_cases: # Some cases were skipped, so adjust testcases - testcases = list(set(testcases) - set(skipped_cases)) + testcases = list(util.OrderedSet(testcases) - + util.OrderedSet(skipped_cases)) printer.verbose( f'Filtering test case(s) due to unresolved dependencies: ' f'{len(testcases)} remaining' From 5b456a68dab40a6c8d27c73fb7898c6ae496156e Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 4 Aug 2022 10:34:05 +0300 Subject: [PATCH 2/4] Document the `--exec-order` option --- docs/manpage.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/manpage.rst b/docs/manpage.rst index 7c0f884f5d..c1aaa11ad1 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -418,6 +418,23 @@ Options controlling ReFrame execution .. versionadded:: 3.11.0 + +.. option:: --exec-order=ORDER + + Impose an execution order for the independent tests. + The ``ORDER`` argument can take one of the following values: + + - ``name``: Order tests by their display name. + - ``rname``: Order tests by their display name in reverse order. + - ``uid``: Order tests by their unique name. + - ``ruid``: Order tests by their unique name in reverse order. + - ``random``: Randomize the order of execution. + + If this option is not specified the order of execution of independent tests is implementation defined. + This option can be combined with any of the listing options (:option:`-l` or :option:`-L`) to list the tests in the order. + + .. versionadded:: 4.0.0 + .. option:: --exec-policy=POLICY The execution policy to be used for running tests. From ca5b79bff0a93d5deb3ca768c30d70279e808a41 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 4 Aug 2022 11:45:37 +0300 Subject: [PATCH 3/4] Add unit tests --- reframe/frontend/cli.py | 2 +- unittests/test_cli.py | 43 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 2c0df8e2b9..6b39c3ec1f 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -391,7 +391,7 @@ def main(): run_options.add_argument( '--exec-order', metavar='ORDER', action='store', choices=['name', 'random', 'rname', 'ruid', 'uid'], - help='Impose an execution order for tests' + help='Impose an execution order for independent tests' ) run_options.add_argument( '--exec-policy', metavar='POLICY', action='store', diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 35676f3bd5..8b4f791d00 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -834,7 +834,7 @@ def test_repeat_invalid_option(run_reframe): def test_repeat_negative(run_reframe): returncode, stdout, stderr = run_reframe( - more_options=['--repeat', 'foo'], + more_options=['--repeat', '-1'], checkpath=['unittests/resources/checks/hellocheck.py'] ) errmsg = "argument to '--repeat' option must be a non-negative integer" @@ -844,6 +844,47 @@ def test_repeat_negative(run_reframe): assert returncode == 1 +@pytest.fixture(params=['name', 'rname', 'uid', 'ruid', 'random']) +def exec_order(request): + return request.param + + +def test_exec_order(run_reframe, exec_order, monkeypatch): + import reframe.utility.sanity as sn + + # FIXME: Remove this as soon as GH #2574 is merged + monkeypatch.setenv('RFM_COMPACT_TEST_NAMES', '1') + + returncode, stdout, stderr = run_reframe( + more_options=['--repeat', '11', '-n', 'HelloTest', + f'--exec-order={exec_order}'], + checkpath=['unittests/resources/checks/hellocheck.py'], + action='list_detailed', + ) + assert 'Traceback' not in stdout + assert 'Traceback' not in stderr + assert 'Found 11 check(s)' in stdout + assert returncode == 0 + + # Verify the order + if exec_order == 'name': + repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)', + stdout, 1, int, 2).evaluate() + assert repeat_no == 10 + elif exec_order == 'rname': + repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)', + stdout, 1, int, -3).evaluate() + assert repeat_no == 10 + elif exec_order == 'uid': + repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)', + stdout, 1, int, -1).evaluate() + assert repeat_no == 10 + elif exec_order == 'ruid': + repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)', + stdout, 1, int, 0).evaluate() + assert repeat_no == 10 + + def test_detect_host_topology(run_reframe): from reframe.utility.cpuinfo import cpuinfo From 33a1df854612059a6a5f7b1d78b5fe8e73c1c50d Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 8 Aug 2022 12:47:21 +0300 Subject: [PATCH 4/4] Remove temporary monkey patch --- unittests/test_cli.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 3809eee842..12ce3ad0e0 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -849,12 +849,9 @@ def exec_order(request): return request.param -def test_exec_order(run_reframe, exec_order, monkeypatch): +def test_exec_order(run_reframe, exec_order): import reframe.utility.sanity as sn - # FIXME: Remove this as soon as GH #2574 is merged - monkeypatch.setenv('RFM_COMPACT_TEST_NAMES', '1') - returncode, stdout, stderr = run_reframe( more_options=['--repeat', '11', '-n', 'HelloTest', f'--exec-order={exec_order}'],