Emit warning on unknown marks via decorator

changelog/4826.feature.rst

@@ -0,0 +1,2 @@
+A warning is now emitted when unknown marks are used as a decorator.
+This is often due to a typo, which can lead to silently broken tests.

doc/en/mark.rst

@@ -26,14 +26,15 @@ which also serve as documentation.
     :ref:`fixtures <fixtures>`.
 
 
-Raising errors on unknown marks: --strict
------------------------------------------
+.. _unknown-marks:
 
-When the ``--strict`` command-line flag is passed, any unknown marks applied
-with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an error.
-Marks defined or added by pytest or by a plugin will not trigger an error.
+Raising errors on unknown marks
+-------------------------------
 
-Marks can be registered in ``pytest.ini`` like this:
+Unknown marks applied with the ``@pytest.mark.name_of_the_mark`` decorator
+will always emit a warning, in order to avoid silently doing something
+surprising due to mis-typed names.  You can disable the warning for custom
+marks by registering them in ``pytest.ini`` like this:
 
 .. code-block:: ini
 
@@ -42,8 +43,13 @@ Marks can be registered in ``pytest.ini`` like this:
         slow
         serial
 
-This can be used to prevent users mistyping mark names by accident. Test suites that want to enforce this
-should add ``--strict`` to ``addopts``:
+When the ``--strict`` command-line flag is passed, any unregistered marks
+applied with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an
+error, including built-in marks such as ``skip`` or ``xfail``.
+
+Marks added by pytest or by a plugin instead of the decorator will not trigger
+the warning or this error.  Test suites that want to enforce a limited set of
+markers can add ``--strict`` to ``addopts``:
 
 .. code-block:: ini
 

src/_pytest/mark/__init__.py

@@ -146,8 +146,7 @@ def pytest_collection_modifyitems(items, config):
 
 def pytest_configure(config):
     config._old_mark_config = MARK_GEN._config
-    if config.option.strict:
-        MARK_GEN._config = config
+    MARK_GEN._config = config
 
     empty_parameterset = config.getini(EMPTY_PARAMETERSET_OPTION)
 

src/_pytest/mark/structures.py

@@ -11,6 +11,7 @@
 from ..compat import MappingMixin
 from ..compat import NOTSET
 from _pytest.outcomes import fail
+from _pytest.warning_types import UnknownMarkWarning
 
 EMPTY_PARAMETERSET_OPTION = "empty_parameter_set_mark"
 
@@ -129,7 +130,7 @@ def _for_parametrize(cls, argnames, argvalues, func, config, function_definition
                     )
         else:
             # empty parameter set (likely computed at runtime): create a single
-            # parameter set with NOSET values, with the "empty parameter set" mark applied to it
+            # parameter set with NOTSET values, with the "empty parameter set" mark applied to it
             mark = get_empty_parameterset_mark(config, argnames, func)
             parameters.append(
                 ParameterSet(values=(NOTSET,) * len(argnames), marks=[mark], id=None)
@@ -152,7 +153,7 @@ def combined_with(self, other):
         :type other: Mark
         :rtype: Mark
 
-        combines by appending aargs and merging the mappings
+        combines by appending args and merging the mappings
         """
         assert self.name == other.name
         return Mark(
@@ -284,27 +285,48 @@ def test_function():
 
     _config = None
 
+    # These marks are always treated as known for the warning, which is designed
+    # to mitigate typos.  However they are ignored by the --strict option, which
+    # requires explicit registration, to match previous behaviour.
+    _builtin_marks = {
+        "filterwarnings",
+        "parametrize",
+        "skip",
+        "skipif",
+        "usefixtures",
+        "xfail",
+    }
+    _markers = set()
+
     def __getattr__(self, name):
         if name[0] == "_":
             raise AttributeError("Marker name must NOT start with underscore")
-        if self._config is not None:
-            self._check(name)
+        self._update_markers(name)
+        if name not in self._markers:
+            if name not in self._builtin_marks:
+                warnings.warn(
+                    "Unknown mark %r.  You can register custom marks to avoid this "
+                    "warning, without risking typos that silently break your tests. "
+                    "See https://docs.pytest.org/en/latest/mark.html for more detail.",
+                    UnknownMarkWarning,
+                )
+            if self._config is not None and self._config.option.strict:
+                fail("{!r} not a registered marker".format(name), pytrace=False)
+
         return MarkDecorator(Mark(name, (), {}))
 
-    def _check(self, name):
-        try:
-            if name in self._markers:
-                return
-        except AttributeError:
-            pass
-        self._markers = values = set()
-        for line in self._config.getini("markers"):
-            marker = line.split(":", 1)[0]
-            marker = marker.rstrip()
-            x = marker.split("(", 1)[0]
-            values.add(x)
-        if name not in self._markers:
-            fail("{!r} not a registered marker".format(name), pytrace=False)
+    def _update_markers(self, name=None):
+        # We store a set of registered markers as a performance optimisation,
+        # but more could be added to `self._config` by other plugins at runtime.
+        # If we see an unknown marker, we therefore update the set and try again!
+        if (
+            self._config is not None
+            and name not in self._markers
+            and (self._config.option.strict or name not in self._builtin_marks)
+        ):
+            for line in self._config.getini("markers"):
+                marker = line.split(":", 1)[0].rstrip().split("(", 1)[0]
+                self._markers.add(marker)
 
 
 MARK_GEN = MarkGenerator()

src/_pytest/warning_types.py

@@ -9,6 +9,14 @@ class PytestWarning(UserWarning):
     """
 
 
+class UnknownMarkWarning(PytestWarning):
+    """
+    Bases: :class:`pytest.PytestWarning`.
+
+    Warning class for unknown attributes of the pytest.mark decorator factory.
+    """
+
+
 class PytestDeprecationWarning(PytestWarning, DeprecationWarning):
     """
     Bases: :class:`pytest.PytestWarning`, :class:`DeprecationWarning`.

tox.ini

@@ -187,6 +187,9 @@ filterwarnings =
     # Do not cause SyntaxError for invalid escape sequences in py37.
     default:invalid escape sequence:DeprecationWarning
 pytester_example_dir = testing/example_scripts
+markers =
+    issue
+    pytester_example_path
 
 [flake8]
 max-line-length = 120