Merge pull request reframe-hpc#2575 from vkarak/feat/randomize-exec-o…

…rder

[feat] Add command line option to set the execution order of independent tests

docs/manpage.rst

@@ -418,6 +418,23 @@ Options controlling ReFrame execution
 
    .. versionadded:: 3.11.0
 
+
+.. option:: --exec-order=ORDER
+
+   Impose an execution order for the independent tests.
+   The ``ORDER`` argument can take one of the following values:
+
+   - ``name``: Order tests by their display name.
+   - ``rname``: Order tests by their display name in reverse order.
+   - ``uid``: Order tests by their unique name.
+   - ``ruid``: Order tests by their unique name in reverse order.
+   - ``random``: Randomize the order of execution.
+
+   If this option is not specified the order of execution of independent tests is implementation defined.
+   This option can be combined with any of the listing options (:option:`-l` or :option:`-L`) to list the tests in the order.
+
+   .. versionadded:: 4.0.0
+
 .. option:: --exec-policy=POLICY
 
    The execution policy to be used for running tests.

reframe/frontend/cli.py

@@ -7,6 +7,7 @@
 import itertools
 import json
 import os
+import random
 import shlex
 import socket
 import sys
@@ -25,6 +26,7 @@
 import reframe.frontend.dependencies as dependencies
 import reframe.frontend.filters as filters
 import reframe.frontend.runreport as runreport
+import reframe.utility as util
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 import reframe.utility.typecheck as typ
@@ -386,6 +388,11 @@ def main():
         help=('Distribute the selected single-node jobs on every node that'
               'is in STATE (default: "idle"')
     )
+    run_options.add_argument(
+        '--exec-order', metavar='ORDER', action='store',
+        choices=['name', 'random', 'rname', 'ruid', 'uid'],
+        help='Impose an execution order for independent tests'
+    )
     run_options.add_argument(
         '--exec-policy', metavar='POLICY', action='store',
         choices=['async', 'serial'], default='async',
@@ -1054,8 +1061,8 @@ def _case_failed(t):
                     "a non-negative integer"
                 ) from None
 
-            testcases = repeat_tests(testcases, num_repeats)
-            testcases_all = testcases
+            testcases_all = repeat_tests(testcases, num_repeats)
+            testcases = testcases_all
 
         if options.distribute:
             node_map = getallnodes(options.distribute, parsed_job_options)
@@ -1070,15 +1077,31 @@ def _case_failed(t):
                 x for x in parsed_job_options
                 if (not x.startswith('-w') and not x.startswith('--nodelist'))
             ]
-            testcases = distribute_tests(testcases, node_map)
-            testcases_all = testcases
+            testcases_all = distribute_tests(testcases, node_map)
+            testcases = testcases_all
+
+        @logging.time_function
+        def _sort_testcases(testcases):
+            if options.exec_order in ('name', 'rname'):
+                testcases.sort(key=lambda c: c.check.display_name,
+                               reverse=(options.exec_order == 'rname'))
+            elif options.exec_order in ('uid', 'ruid'):
+                testcases.sort(key=lambda c: c.check.unique_name,
+                               reverse=(options.exec_order == 'ruid'))
+            elif options.exec_order == 'random':
+                random.shuffle(testcases)
+
+        _sort_testcases(testcases)
+        if testcases_all is not testcases:
+            _sort_testcases(testcases_all)
 
         # Prepare for running
         printer.debug('Building and validating the full test DAG')
         testgraph, skipped_cases = dependencies.build_deps(testcases_all)
         if skipped_cases:
             # Some cases were skipped, so adjust testcases
-            testcases = list(set(testcases) - set(skipped_cases))
+            testcases = list(util.OrderedSet(testcases) -
+                             util.OrderedSet(skipped_cases))
             printer.verbose(
                 f'Filtering test case(s) due to unresolved dependencies: '
                 f'{len(testcases)} remaining'

unittests/test_cli.py

@@ -834,7 +834,7 @@ def test_repeat_invalid_option(run_reframe):
 
 def test_repeat_negative(run_reframe):
     returncode, stdout, stderr = run_reframe(
-        more_options=['--repeat', 'foo'],
+        more_options=['--repeat', '-1'],
         checkpath=['unittests/resources/checks/hellocheck.py']
     )
     errmsg = "argument to '--repeat' option must be a non-negative integer"
@@ -844,6 +844,44 @@ def test_repeat_negative(run_reframe):
     assert returncode == 1
 
 
+@pytest.fixture(params=['name', 'rname', 'uid', 'ruid', 'random'])
+def exec_order(request):
+    return request.param
+
+
+def test_exec_order(run_reframe, exec_order):
+    import reframe.utility.sanity as sn
+
+    returncode, stdout, stderr = run_reframe(
+        more_options=['--repeat', '11', '-n', 'HelloTest',
+                      f'--exec-order={exec_order}'],
+        checkpath=['unittests/resources/checks/hellocheck.py'],
+        action='list_detailed',
+    )
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert 'Found 11 check(s)' in stdout
+    assert returncode == 0
+
+    # Verify the order
+    if exec_order == 'name':
+        repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)',
+                                       stdout, 1, int, 2).evaluate()
+        assert repeat_no == 10
+    elif exec_order == 'rname':
+        repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)',
+                                       stdout, 1, int, -3).evaluate()
+        assert repeat_no == 10
+    elif exec_order == 'uid':
+        repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)',
+                                       stdout, 1, int, -1).evaluate()
+        assert repeat_no == 10
+    elif exec_order == 'ruid':
+        repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)',
+                                       stdout, 1, int, 0).evaluate()
+        assert repeat_no == 10
+
+
 def test_detect_host_topology(run_reframe):
     from reframe.utility.cpuinfo import cpuinfo