Merge branch 'develop' into mg/convert-json

docs/manpage.rst

@@ -545,6 +545,24 @@ Options controlling ReFrame execution
    .. versionchanged:: 4.1
       Options that can be specified multiple times are now combined between execution modes and the command line.
 
+.. option:: -P, --parameterize=[TEST.]VAR=VAL0,VAL1,...
+
+   Parameterize a test on an existing variable.
+
+   This option will create a new test with a parameter named ``$VAR`` with the values given in the comma-separated list ``VAL0,VAL1,...``.
+   The values will be converted based on the type of the target variable ``VAR``.
+   The ``TEST.`` prefix will only parameterize the variable ``VAR`` of test ``TEST``.
+
+   The :option:`-P` can be specified multiple times in order to parameterize multiple variables.
+
+   .. note::
+
+      Conversely to the :option:`-S` option that can set a variable in an arbitrarily nested fixture,
+      the :option:`-P` option can only parameterize the leaf test:
+      it cannot be used to parameterize a fixture of the test.
+
+   .. versionadded:: 4.3
+
 .. option:: --repeat=N
 
    Repeat the selected tests ``N`` times.
@@ -1011,7 +1029,7 @@ The format of the display name is the following in BNF notation:
 
 .. code-block:: bnf
 
-   <display_name> ::= <test_class_name> (<params>)* (<scope>)?
+   <display_name> ::= <test_class_name> (<params>)* (<scope> ("'"<fixtvar>)+)?
    <params> ::= "%" <parametrization> "=" <pvalue>
    <parametrization> ::= (<fname> ".")* <pname>
    <scope> ::= "~" <scope_descr>
@@ -1023,6 +1041,7 @@ The format of the display name is the following in BNF notation:
    <pvalue> ::= (* string *)
    <first> ::= (* string *)
    <second> ::= (* string *)
+   <fixtvar> ::= (* string *)
 
 The following is an example of a fictitious complex test that is itself parameterized and depends on parameterized fixtures as well.
 

docs/requirements.txt

@@ -1,6 +1,6 @@
 archspec==0.2.1
 docutils==0.17.1 # https://github.com/sphinx-doc/sphinx/issues/9001
 jsonschema==3.2.0
-semver==3.0.0
+semver==3.0.1
 Sphinx==5.3.0
-sphinx-rtd-theme==1.2.1
+sphinx-rtd-theme==1.2.2

reframe/__init__.py

@@ -6,7 +6,7 @@
 import os
 import sys
 
-VERSION = '4.3.0-dev.2'
+VERSION = '4.4.0-dev.0'
 INSTALL_PREFIX = os.path.normpath(
     os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 )

reframe/core/meta.py

@@ -936,20 +936,18 @@ def validate(obj):
     namespace = RegressionTestMeta.__prepare__(name, bases, **kwargs)
     methods = methods or []
 
-    # Add methods to the body
+    # Update the namespace with the body elements
+    #
+    # NOTE: We do not use `update()` here so as to force the `__setitem__`
+    # logic
+    for k, v in body.items():
+        namespace[k] = v
+
+    # Add methods to the namespace
     for m in methods:
-        body[m.__name__] = m
-
-    # We update the namespace with the body of the class and we explicitly
-    # call reset on each namespace key to trigger the functionality of
-    # `__setitem__()` as if the body elements were actually being typed in the
-    # class definition
-    namespace.update(body)
-    for k in list(namespace.keys()):
-        namespace.reset(k)
+        namespace[m.__name__] = m
 
     cls = RegressionTestMeta(name, bases, namespace, **kwargs)
-
     if not module:
         # Set the test's module to be that of our callers
         caller = inspect.currentframe().f_back

reframe/core/runtime.py

@@ -9,7 +9,7 @@
 
 import os
 import functools
-from datetime import datetime
+import time
 
 import reframe.core.config as config
 import reframe.utility.osext as osext
@@ -31,7 +31,7 @@ def __init__(self, site_config):
         self._site_config = site_config
         self._system = System.create(site_config)
         self._current_run = 0
-        self._timestamp = datetime.now()
+        self._timestamp = time.localtime()
 
     def _makedir(self, *dirs, wipeout=False):
         ret = os.path.join(*dirs)
@@ -111,7 +111,7 @@ def perflogdir(self):
     @property
     def timestamp(self):
         timefmt = self.site_config.get('general/0/timestamp_dirs')
-        return self._timestamp.strftime(timefmt)
+        return time.strftime(timefmt, self._timestamp)
 
     @property
     def output_prefix(self):

reframe/core/schedulers/local.py

@@ -12,7 +12,7 @@
 import reframe.core.schedulers as sched
 import reframe.utility.osext as osext
 from reframe.core.backends import register_scheduler
-from reframe.core.exceptions import ReframeError
+from reframe.core.exceptions import JobError, ReframeError
 
 
 class _TimeoutExpired(ReframeError):
@@ -187,9 +187,12 @@ def _poll_job(self, job):
             # Job has not finished; check if we have reached a timeout
             t_elapsed = time.time() - job.submit_time
             if job.time_limit and t_elapsed > job.time_limit:
-                self.log(f'Job {job.jobid} timed out; kill it')
                 self._kill_all(job)
                 job._state = 'TIMEOUT'
+                job._exception = JobError(
+                    f'job timed out ({t_elapsed:.6f}s > {job.time_limit}s)',
+                    job.jobid
+                )
 
             return
 

reframe/frontend/cli.py

@@ -30,7 +30,8 @@
 import reframe.utility.typecheck as typ
 
 from reframe.frontend.testgenerators import (distribute_tests,
-                                             getallnodes, repeat_tests)
+                                             getallnodes, repeat_tests,
+                                             parameterize_tests)
 from reframe.frontend.executors.policies import (SerialExecutionPolicy,
                                                  AsynchronousExecutionPolicy)
 from reframe.frontend.executors import Runner, generate_testcases
@@ -285,7 +286,7 @@ def main():
         envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files'
     )
     output_options.add_argument(
-        '--timestamp', action='store', nargs='?', const='%FT%T',
+        '--timestamp', action='store', nargs='?', const='%y%m%dT%H%M%S%z',
         metavar='TIMEFMT',
         help=('Append a timestamp to the output and stage directory prefixes '
               '(default: "%%FT%%T")'),
@@ -436,6 +437,10 @@ def main():
     run_options.add_argument(
         '--mode', action='store', help='Execution mode to use'
     )
+    run_options.add_argument(
+        '-P', '--parameterize', action='append', metavar='VAR:VAL0,VAL1,...',
+        default=[], help='Parameterize a test on a set of variables'
+    )
     run_options.add_argument(
         '--repeat', action='store', metavar='N',
         help='Repeat selected tests N times'
@@ -1096,6 +1101,22 @@ def _case_failed(t):
                 f'{len(testcases)} remaining'
             )
 
+        if options.parameterize:
+            # Prepare parameters
+            params = {}
+            for param_spec in options.parameterize:
+                try:
+                    var, values_spec = param_spec.split('=')
+                except ValueError:
+                    raise errors.CommandLineError(
+                        f'invalid parameter spec: {param_spec}'
+                    ) from None
+                else:
+                    params[var] = values_spec.split(',')
+
+            testcases_all = parameterize_tests(testcases, params)
+            testcases = testcases_all
+
         if options.repeat is not None:
             try:
                 num_repeats = int(options.repeat)

reframe/frontend/testgenerators.py

@@ -9,6 +9,7 @@
 import reframe.utility as util
 
 from reframe.core.decorators import TestRegistry
+from reframe.core.fields import make_convertible
 from reframe.core.logging import getlogger, time_function
 from reframe.core.meta import make_test
 from reframe.core.schedulers import Job
@@ -48,36 +49,7 @@ def getallnodes(state='all', jobs_cli_options=None):
     return nodes
 
 
-def _rfm_pin_run_nodes(obj):
-    nodelist = getattr(obj, '$nid')
-    if not obj.local:
-        obj.job.pin_nodes = nodelist
-
-
-def _rfm_pin_build_nodes(obj):
-    pin_nodes = getattr(obj, '$nid')
-    if not obj.local and not obj.build_locally:
-        obj.build_job.pin_nodes = pin_nodes
-
-
-def make_valid_systems_hook(systems):
-    '''Returns a function to be used as a hook that sets the
-    valid systems.
-
-    Since valid_systems change for each generated test, we need to
-    generate different post-init hooks for each one of them.
-    '''
-    def _rfm_set_valid_systems(obj):
-        obj.valid_systems = systems
-
-    return _rfm_set_valid_systems
-
-
-@time_function
-def distribute_tests(testcases, node_map):
-    '''Returns new testcases that will be parameterized to run in node of
-    their partitions based on the nodemap
-    '''
+def _generate_tests(testcases, gen_fn):
     tmp_registry = TestRegistry()
 
     # We don't want to register the same check for every environment
@@ -94,7 +66,41 @@ def distribute_tests(testcases, node_map):
         variant_info = cls.get_variant_info(
             check.variant_num, recurse=True
         )
-        nc = make_test(
+        nc, params = gen_fn(tc)
+        nc._rfm_custom_prefix = check.prefix
+        for i in range(nc.num_variants):
+            # Check if this variant should be instantiated
+            vinfo = nc.get_variant_info(i, recurse=True)
+            for p in params:
+                vinfo['params'].pop(p)
+
+            if vinfo == variant_info:
+                tmp_registry.add(nc, variant_num=i)
+
+    new_checks = tmp_registry.instantiate_all()
+    return generate_testcases(new_checks)
+
+
+@time_function
+def distribute_tests(testcases, node_map):
+    def _rfm_pin_run_nodes(obj):
+        nodelist = getattr(obj, '$nid')
+        if not obj.local:
+            obj.job.pin_nodes = nodelist
+
+    def _rfm_pin_build_nodes(obj):
+        pin_nodes = getattr(obj, '$nid')
+        if not obj.local and not obj.build_locally:
+            obj.build_job.pin_nodes = pin_nodes
+
+    def _make_dist_test(testcase):
+        check, partition, _ = testcase
+        cls = type(check)
+
+        def _rfm_set_valid_systems(obj):
+            obj.valid_systems = [partition.fullname]
+
+        return make_test(
             f'{cls.__name__}_{partition.fullname.replace(":", "_")}',
             (cls,),
             {
@@ -108,55 +114,70 @@ def distribute_tests(testcases, node_map):
                 builtins.run_before('run')(_rfm_pin_run_nodes),
                 builtins.run_before('compile')(_rfm_pin_build_nodes),
                 # We re-set the valid system in a hook to make sure that it
-                # will not be overwriten by a parent post-init hook
-                builtins.run_after('init')(
-                    make_valid_systems_hook([partition.fullname])
-                ),
+                # will not be overwritten by a parent post-init hook
+                builtins.run_after('init')(_rfm_set_valid_systems),
             ]
-        )
-
-        # We have to set the prefix manually
-        nc._rfm_custom_prefix = check.prefix
-        for i in range(nc.num_variants):
-            # Check if this variant should be instantiated
-            vinfo = nc.get_variant_info(i, recurse=True)
-            vinfo['params'].pop('$nid')
-            if vinfo == variant_info:
-                tmp_registry.add(nc, variant_num=i)
+        ), ['$nid']
 
-    new_checks = tmp_registry.instantiate_all()
-    return generate_testcases(new_checks)
+    return _generate_tests(testcases, _make_dist_test)
 
 
 @time_function
 def repeat_tests(testcases, num_repeats):
     '''Returns new test cases parameterized over their repetition number'''
 
-    tmp_registry = TestRegistry()
-    unique_checks = set()
-    for tc in testcases:
-        check = tc.check
-        if check.is_fixture() or check in unique_checks:
-            continue
-
-        unique_checks.add(check)
-        cls = type(check)
-        variant_info = cls.get_variant_info(
-            check.variant_num, recurse=True
-        )
-        nc = make_test(
+    def _make_repeat_test(testcase):
+        cls = type(testcase.check)
+        return make_test(
             f'{cls.__name__}', (cls,),
             {
                 '$repeat_no': builtins.parameter(range(num_repeats))
             }
-        )
-        nc._rfm_custom_prefix = check.prefix
-        for i in range(nc.num_variants):
-            # Check if this variant should be instantiated
-            vinfo = nc.get_variant_info(i, recurse=True)
-            vinfo['params'].pop('$repeat_no')
-            if vinfo == variant_info:
-                tmp_registry.add(nc, variant_num=i)
+        ), ['$repeat_no']
 
-    new_checks = tmp_registry.instantiate_all()
-    return generate_testcases(new_checks)
+    return _generate_tests(testcases, _make_repeat_test)
+
+
+@time_function
+def parameterize_tests(testcases, paramvars):
+    '''Returns new test cases parameterized over specific variables.'''
+
+    def _make_parameterized_test(testcase):
+        check = testcase.check
+        cls = type(check)
+
+        # Check that all the requested variables exist
+        body = {}
+        for var, values in paramvars.items():
+            var_parts = var.split('.')
+            if len(var_parts) == 1:
+                var = var_parts[0]
+            elif len(var_parts) == 2:
+                var_check, var = var_parts
+                if var_check != cls.__name__:
+                    continue
+            else:
+                getlogger().warning(f'cannot set a variable in a fixture')
+                continue
+
+            if not var in cls.var_space:
+                getlogger().warning(
+                    f'variable {var!r} not defined for test '
+                    f'{check.display_name!r}; ignoring parameterization'
+                )
+                continue
+
+            body[f'${var}'] = builtins.parameter(values)
+
+        def _set_vars(self):
+            for var in body.keys():
+                setattr(self, var[1:],
+                        make_convertible(getattr(self, f'{var}')))
+
+        return make_test(
+            f'{cls.__name__}', (cls,),
+            body=body,
+            methods=[builtins.run_after('init')(_set_vars)]
+        ), body.keys()
+
+    return _generate_tests(testcases, _make_parameterized_test)