Merge pull request #5015 from Zac-HD/mark-warning-prep

Pre-PR for warnings on unknown markers

changelog/4935.doc.rst

@@ -0,0 +1 @@
+Expand docs on registering marks and the effect of ``--strict``.

doc/en/mark.rst

@@ -26,12 +26,10 @@ which also serve as documentation.
     :ref:`fixtures <fixtures>`.
 
 
-Raising errors on unknown marks: --strict
------------------------------------------
+.. _unknown-marks:
 
-When the ``--strict`` command-line flag is passed, any unknown marks applied
-with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an error.
-Marks defined or added by pytest or by a plugin will not trigger an error.
+Raising errors on unknown marks
+-------------------------------
 
 Marks can be registered in ``pytest.ini`` like this:
 
@@ -42,8 +40,10 @@ Marks can be registered in ``pytest.ini`` like this:
         slow
         serial
 
-This can be used to prevent users mistyping mark names by accident. Test suites that want to enforce this
-should add ``--strict`` to ``addopts``:
+When the ``--strict`` command-line flag is passed, any unknown marks applied
+with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an error.
+Marks added by pytest or by a plugin instead of the decorator will not trigger
+this error.  To enforce validation of markers, add ``--strict`` to ``addopts``:
 
 .. code-block:: ini
 
@@ -53,6 +53,9 @@ should add ``--strict`` to ``addopts``:
         slow
         serial
 
+Third-party plugins should always :ref:`register their markers <registering-markers>`
+so that they appear in pytest's help text and do not emit warnings.
+
 
 .. _marker-revamp:
 

doc/en/writing_plugins.rst

@@ -286,6 +286,26 @@ the plugin manager like this:
 If you want to look at the names of existing plugins, use
 the ``--trace-config`` option.
 
+
+.. _registering-markers:
+
+Registering custom markers
+--------------------------
+
+If your plugin uses any markers, you should register them so that they appear in
+pytest's help text and do not :ref:`cause spurious warnings <unknown-marks>`.
+For example, the following plugin would register ``cool_marker`` and
+``mark_with`` for all users:
+
+.. code-block:: python
+
+    def pytest_configure(config):
+        config.addinivalue_line("markers", "cool_marker: this one is for cool tests.")
+        config.addinivalue_line(
+            "markers", "mark_with(arg, arg2): this marker takes arguments."
+        )
+
+
 Testing plugins
 ---------------
 

src/_pytest/mark/structures.py

@@ -129,7 +129,7 @@ def _for_parametrize(cls, argnames, argvalues, func, config, function_definition
                     )
         else:
             # empty parameter set (likely computed at runtime): create a single
-            # parameter set with NOSET values, with the "empty parameter set" mark applied to it
+            # parameter set with NOTSET values, with the "empty parameter set" mark applied to it
             mark = get_empty_parameterset_mark(config, argnames, func)
             parameters.append(
                 ParameterSet(values=(NOTSET,) * len(argnames), marks=[mark], id=None)
@@ -152,7 +152,7 @@ def combined_with(self, other):
         :type other: Mark
         :rtype: Mark
 
-        combines by appending aargs and merging the mappings
+        combines by appending args and merging the mappings
         """
         assert self.name == other.name
         return Mark(

src/_pytest/pytester.py

@@ -68,6 +68,12 @@ def pytest_configure(config):
         if checker.matching_platform():
             config.pluginmanager.register(checker)
 
+    config.addinivalue_line(
+        "markers",
+        "pytester_example_path(*path_segments): join the given path "
+        "segments to `pytester_example_dir` for this test.",
+    )
+
 
 def raise_on_kwargs(kwargs):
     if kwargs:

testing/python/fixtures.py

@@ -1927,7 +1927,7 @@ def test_hello(arg1):
         reprec = testdir.inline_run()
         reprec.assertoutcome(passed=1)
 
-    @pytest.mark.issue226
+    @pytest.mark.issue(226)
     @pytest.mark.parametrize("param1", ["", "params=[1]"], ids=["p00", "p01"])
     @pytest.mark.parametrize("param2", ["", "params=[1]"], ids=["p10", "p11"])
     def test_ordering_dependencies_torndown_first(self, testdir, param1, param2):
@@ -2709,7 +2709,7 @@ def test_3():
         reprec = testdir.inline_run("-v")
         reprec.assertoutcome(passed=5)
 
-    @pytest.mark.issue246
+    @pytest.mark.issue(246)
     @pytest.mark.parametrize("scope", ["session", "function", "module"])
     def test_finalizer_order_on_parametrization(self, scope, testdir):
         testdir.makepyfile(
@@ -2746,7 +2746,7 @@ def test_other():
         reprec = testdir.inline_run("-lvs")
         reprec.assertoutcome(passed=3)
 
-    @pytest.mark.issue396
+    @pytest.mark.issue(396)
     def test_class_scope_parametrization_ordering(self, testdir):
         testdir.makepyfile(
             """
@@ -2867,7 +2867,7 @@ def test_foo(fix):
         res = testdir.runpytest("-v")
         res.stdout.fnmatch_lines(["*test_foo*alpha*", "*test_foo*beta*"])
 
-    @pytest.mark.issue920
+    @pytest.mark.issue(920)
     def test_deterministic_fixture_collection(self, testdir, monkeypatch):
         testdir.makepyfile(
             """

testing/python/integration.py

@@ -393,7 +393,7 @@ def test_blah(self):
         assert not call.items
 
 
-@pytest.mark.issue351
+@pytest.mark.issue(351)
 class TestParameterize(object):
     def test_idfn_marker(self, testdir):
         testdir.makepyfile(

testing/python/metafunc.py

@@ -159,7 +159,7 @@ def func(x, y):
                 ("x", "y"), [("abc", "def"), ("ghi", "jkl")], ids=["one"]
             )
 
-    @pytest.mark.issue510
+    @pytest.mark.issue(510)
     def test_parametrize_empty_list(self):
         def func(y):
             pass
@@ -262,7 +262,7 @@ def test_function():
         for val, expected in values:
             assert _idval(val, "a", 6, None, item=None, config=None) == expected
 
-    @pytest.mark.issue250
+    @pytest.mark.issue(250)
     def test_idmaker_autoname(self):
         from _pytest.python import idmaker
 
@@ -356,7 +356,7 @@ def test_idmaker_enum(self):
         result = idmaker(("a", "b"), [pytest.param(e.one, e.two)])
         assert result == ["Foo.one-Foo.two"]
 
-    @pytest.mark.issue351
+    @pytest.mark.issue(351)
     def test_idmaker_idfn(self):
         from _pytest.python import idmaker
 
@@ -375,7 +375,7 @@ def ids(val):
         )
         assert result == ["10.0-IndexError()", "20-KeyError()", "three-b2"]
 
-    @pytest.mark.issue351
+    @pytest.mark.issue(351)
     def test_idmaker_idfn_unique_names(self):
         from _pytest.python import idmaker
 
@@ -459,7 +459,7 @@ def test_idmaker_with_ids_unique_names(self):
         )
         assert result == ["a0", "a1", "b0", "c", "b1"]
 
-    @pytest.mark.issue714
+    @pytest.mark.issue(714)
     def test_parametrize_indirect(self):
         def func(x, y):
             pass
@@ -473,7 +473,7 @@ def func(x, y):
         assert metafunc._calls[0].params == dict(x=1, y=2)
         assert metafunc._calls[1].params == dict(x=1, y=3)
 
-    @pytest.mark.issue714
+    @pytest.mark.issue(714)
     def test_parametrize_indirect_list(self):
         def func(x, y):
             pass
@@ -483,7 +483,7 @@ def func(x, y):
         assert metafunc._calls[0].funcargs == dict(y="b")
         assert metafunc._calls[0].params == dict(x="a")
 
-    @pytest.mark.issue714
+    @pytest.mark.issue(714)
     def test_parametrize_indirect_list_all(self):
         def func(x, y):
             pass
@@ -493,7 +493,7 @@ def func(x, y):
         assert metafunc._calls[0].funcargs == {}
         assert metafunc._calls[0].params == dict(x="a", y="b")
 
-    @pytest.mark.issue714
+    @pytest.mark.issue(714)
     def test_parametrize_indirect_list_empty(self):
         def func(x, y):
             pass
@@ -503,7 +503,7 @@ def func(x, y):
         assert metafunc._calls[0].funcargs == dict(x="a", y="b")
         assert metafunc._calls[0].params == {}
 
-    @pytest.mark.issue714
+    @pytest.mark.issue(714)
     def test_parametrize_indirect_list_functional(self, testdir):
         """
         Test parametrization with 'indirect' parameter applied on
@@ -532,7 +532,7 @@ def test_simple(x,y):
         result = testdir.runpytest("-v")
         result.stdout.fnmatch_lines(["*test_simple*a-b*", "*1 passed*"])
 
-    @pytest.mark.issue714
+    @pytest.mark.issue(714)
     def test_parametrize_indirect_list_error(self, testdir):
         def func(x, y):
             pass
@@ -541,7 +541,7 @@ def func(x, y):
         with pytest.raises(pytest.fail.Exception):
             metafunc.parametrize("x, y", [("a", "b")], indirect=["x", "z"])
 
-    @pytest.mark.issue714
+    @pytest.mark.issue(714)
     def test_parametrize_uses_no_fixture_error_indirect_false(self, testdir):
         """The 'uses no fixture' error tells the user at collection time
         that the parametrize data they've set up doesn't correspond to the
@@ -560,7 +560,7 @@ def test_simple(x):
         result = testdir.runpytest("--collect-only")
         result.stdout.fnmatch_lines(["*uses no argument 'y'*"])
 
-    @pytest.mark.issue714
+    @pytest.mark.issue(714)
     def test_parametrize_uses_no_fixture_error_indirect_true(self, testdir):
         testdir.makepyfile(
             """
@@ -580,7 +580,7 @@ def test_simple(x):
         result = testdir.runpytest("--collect-only")
         result.stdout.fnmatch_lines(["*uses no fixture 'y'*"])
 
-    @pytest.mark.issue714
+    @pytest.mark.issue(714)
     def test_parametrize_indirect_uses_no_fixture_error_indirect_string(self, testdir):
         testdir.makepyfile(
             """
@@ -597,7 +597,7 @@ def test_simple(x):
         result = testdir.runpytest("--collect-only")
         result.stdout.fnmatch_lines(["*uses no fixture 'y'*"])
 
-    @pytest.mark.issue714
+    @pytest.mark.issue(714)
     def test_parametrize_indirect_uses_no_fixture_error_indirect_list(self, testdir):
         testdir.makepyfile(
             """
@@ -614,7 +614,7 @@ def test_simple(x):
         result = testdir.runpytest("--collect-only")
         result.stdout.fnmatch_lines(["*uses no fixture 'y'*"])
 
-    @pytest.mark.issue714
+    @pytest.mark.issue(714)
     def test_parametrize_argument_not_in_indirect_list(self, testdir):
         testdir.makepyfile(
             """
@@ -1201,7 +1201,7 @@ def test_foo(x):
         reprec = testdir.runpytest()
         reprec.assert_outcomes(passed=4)
 
-    @pytest.mark.issue463
+    @pytest.mark.issue(463)
     @pytest.mark.parametrize("attr", ["parametrise", "parameterize", "parameterise"])
     def test_parametrize_misspelling(self, testdir, attr):
         testdir.makepyfile(
@@ -1386,7 +1386,7 @@ def pytest_generate_tests(metafunc):
         assert output.count("preparing foo-3") == 1
 
 
-@pytest.mark.issue308
+@pytest.mark.issue(308)
 class TestMarkersWithParametrization(object):
     def test_simple_mark(self, testdir):
         s = """
@@ -1575,7 +1575,7 @@ def test_increment(n, expected):
         reprec = testdir.inline_run(SHOW_PYTEST_WARNINGS_ARG)
         reprec.assertoutcome(passed=2, skipped=2)
 
-    @pytest.mark.issue290
+    @pytest.mark.issue(290)
     def test_parametrize_ID_generation_string_int_works(self, testdir):
         testdir.makepyfile(
             """

testing/test_capture.py

@@ -605,7 +605,7 @@ def test_hello(capfd):
         result.stdout.fnmatch_lines(["*KeyboardInterrupt*"])
         assert result.ret == 2
 
-    @pytest.mark.issue14
+    @pytest.mark.issue(14)
     def test_capture_and_logging(self, testdir):
         p = testdir.makepyfile(
             """\

testing/test_conftest.py

@@ -490,7 +490,7 @@ def test_no_conftest(fxtr):
             ("snc", ".", 1),
         ],
     )
-    @pytest.mark.issue616
+    @pytest.mark.issue(616)
     def test_parsefactories_relative_node_ids(
         self, testdir, chdir, testarg, expect_ntests_passed
     ):

testing/test_mark.py

@@ -448,7 +448,7 @@ def test_bar(self): pass
         items, rec = testdir.inline_genitems(p)
         self.assert_markers(items, test_foo=("a", "b"), test_bar=("a",))
 
-    @pytest.mark.issue568
+    @pytest.mark.issue(568)
     def test_mark_should_not_pass_to_siebling_class(self, testdir):
         p = testdir.makepyfile(
             """
@@ -651,7 +651,7 @@ def assert_markers(self, items, **expected):
             markers = {m.name for m in items[name].iter_markers()}
             assert markers == set(expected_markers)
 
-    @pytest.mark.issue1540
+    @pytest.mark.issue(1540)
     @pytest.mark.filterwarnings("ignore")
     def test_mark_from_parameters(self, testdir):
         testdir.makepyfile(

tox.ini

@@ -166,6 +166,8 @@ filterwarnings =
     # Do not cause SyntaxError for invalid escape sequences in py37.
     default:invalid escape sequence:DeprecationWarning
 pytester_example_dir = testing/example_scripts
+markers =
+    issue
 
 [flake8]
 max-line-length = 120