diff --git a/changelog/4826.feature.rst b/changelog/4826.feature.rst new file mode 100644 index 00000000000..2afcba1ad8e --- /dev/null +++ b/changelog/4826.feature.rst @@ -0,0 +1,2 @@ +A warning is now emitted when unknown marks are used as a decorator. +This is often due to a typo, which can lead to silently broken tests. diff --git a/doc/en/mark.rst b/doc/en/mark.rst index e841a6780d6..a4dc8c9abf3 100644 --- a/doc/en/mark.rst +++ b/doc/en/mark.rst @@ -26,14 +26,15 @@ which also serve as documentation. :ref:`fixtures `. -Raising errors on unknown marks: --strict ------------------------------------------ +.. _unknown-marks: -When the ``--strict`` command-line flag is passed, any unknown marks applied -with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an error. -Marks defined or added by pytest or by a plugin will not trigger an error. +Raising errors on unknown marks +------------------------------- -Marks can be registered in ``pytest.ini`` like this: +Unknown marks applied with the ``@pytest.mark.name_of_the_mark`` decorator +will always emit a warning, in order to avoid silently doing something +surprising due to mis-typed names. You can disable the warning for custom +marks by registering them in ``pytest.ini`` like this: .. code-block:: ini @@ -42,8 +43,13 @@ Marks can be registered in ``pytest.ini`` like this: slow serial -This can be used to prevent users mistyping mark names by accident. Test suites that want to enforce this -should add ``--strict`` to ``addopts``: +When the ``--strict`` command-line flag is passed, any unregistered marks +applied with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an +error, including built-in marks such as ``skip`` or ``xfail``. + +Marks added by pytest or by a plugin instead of the decorator will not trigger +the warning or this error. Test suites that want to enforce a limited set of +markers can add ``--strict`` to ``addopts``: .. code-block:: ini diff --git a/src/_pytest/mark/__init__.py b/src/_pytest/mark/__init__.py index bc4c467f918..2dd3f4ed5df 100644 --- a/src/_pytest/mark/__init__.py +++ b/src/_pytest/mark/__init__.py @@ -146,8 +146,7 @@ def pytest_collection_modifyitems(items, config): def pytest_configure(config): config._old_mark_config = MARK_GEN._config - if config.option.strict: - MARK_GEN._config = config + MARK_GEN._config = config empty_parameterset = config.getini(EMPTY_PARAMETERSET_OPTION) diff --git a/src/_pytest/mark/structures.py b/src/_pytest/mark/structures.py index d65d8b9d806..c8dbfdc2f15 100644 --- a/src/_pytest/mark/structures.py +++ b/src/_pytest/mark/structures.py @@ -11,6 +11,7 @@ from ..compat import MappingMixin from ..compat import NOTSET from _pytest.outcomes import fail +from _pytest.warning_types import UnknownMarkWarning EMPTY_PARAMETERSET_OPTION = "empty_parameter_set_mark" @@ -129,7 +130,7 @@ def _for_parametrize(cls, argnames, argvalues, func, config, function_definition ) else: # empty parameter set (likely computed at runtime): create a single - # parameter set with NOSET values, with the "empty parameter set" mark applied to it + # parameter set with NOTSET values, with the "empty parameter set" mark applied to it mark = get_empty_parameterset_mark(config, argnames, func) parameters.append( ParameterSet(values=(NOTSET,) * len(argnames), marks=[mark], id=None) @@ -152,7 +153,7 @@ def combined_with(self, other): :type other: Mark :rtype: Mark - combines by appending aargs and merging the mappings + combines by appending args and merging the mappings """ assert self.name == other.name return Mark( @@ -284,27 +285,48 @@ def test_function(): _config = None + # These marks are always treated as known for the warning, which is designed + # to mitigate typos. However they are ignored by the --strict option, which + # requires explicit registration, to match previous behaviour. + _builtin_marks = { + "filterwarnings", + "parametrize", + "skip", + "skipif", + "usefixtures", + "xfail", + } + _markers = set() + def __getattr__(self, name): if name[0] == "_": raise AttributeError("Marker name must NOT start with underscore") - if self._config is not None: - self._check(name) + self._update_markers(name) + if name not in self._markers: + if name not in self._builtin_marks: + warnings.warn( + "Unknown mark %r. You can register custom marks to avoid this " + "warning, without risking typos that silently break your tests. " + "See https://docs.pytest.org/en/latest/mark.html for more detail.", + UnknownMarkWarning, + ) + if self._config is not None and self._config.option.strict: + fail("{!r} not a registered marker".format(name), pytrace=False) + return MarkDecorator(Mark(name, (), {})) - def _check(self, name): - try: - if name in self._markers: - return - except AttributeError: - pass - self._markers = values = set() - for line in self._config.getini("markers"): - marker = line.split(":", 1)[0] - marker = marker.rstrip() - x = marker.split("(", 1)[0] - values.add(x) - if name not in self._markers: - fail("{!r} not a registered marker".format(name), pytrace=False) + def _update_markers(self, name=None): + # We store a set of registered markers as a performance optimisation, + # but more could be added to `self._config` by other plugins at runtime. + # If we see an unknown marker, we therefore update the set and try again! + if ( + self._config is not None + and name not in self._markers + and (self._config.option.strict or name not in self._builtin_marks) + ): + for line in self._config.getini("markers"): + marker = line.split(":", 1)[0].rstrip().split("(", 1)[0] + self._markers.add(marker) MARK_GEN = MarkGenerator() diff --git a/src/_pytest/warning_types.py b/src/_pytest/warning_types.py index 55e1f037ae5..cd14cad785a 100644 --- a/src/_pytest/warning_types.py +++ b/src/_pytest/warning_types.py @@ -9,6 +9,14 @@ class PytestWarning(UserWarning): """ +class UnknownMarkWarning(PytestWarning): + """ + Bases: :class:`pytest.PytestWarning`. + + Warning class for unknown attributes of the pytest.mark decorator factory. + """ + + class PytestDeprecationWarning(PytestWarning, DeprecationWarning): """ Bases: :class:`pytest.PytestWarning`, :class:`DeprecationWarning`. diff --git a/tox.ini b/tox.ini index 1fc20cb5fb4..2d9396b5956 100644 --- a/tox.ini +++ b/tox.ini @@ -187,6 +187,9 @@ filterwarnings = # Do not cause SyntaxError for invalid escape sequences in py37. default:invalid escape sequence:DeprecationWarning pytester_example_dir = testing/example_scripts +markers = + issue + pytester_example_path [flake8] max-line-length = 120