Allow wildcards inside of configuration section names (#5120)

This is to support Django-style usecases with patterns like
`site.*.migrations.*`.

The implementation works by mixing in the old-style glob matching with
the new structured matching.

Fixes #5014.

Author: msullivan

docs/source/config_file.rst

@@ -31,15 +31,35 @@ characters.
 
 - Additional sections named ``[mypy-PATTERN1,PATTERN2,...]`` may be
   present, where ``PATTERN1``, ``PATTERN2``, etc., are comma-separated
-  patterns of the form ``dotted_module_name`` or ``dotted_module_name.*``.
+  patterns of fully-qualified module names, with some components optionally
+  replaced by `*`s (e.g. ``foo.bar``, ``foo.bar.*``, ``foo.*.baz``).
   These sections specify additional flags that only apply to *modules*
   whose name matches at least one of the patterns.
 
-  A pattern of the form ``dotted_module_name`` matches only the named module,
-  while ``dotted_module_name.*`` matches ``dotted_module_name`` and any
+  A pattern of the form ``qualified_module_name`` matches only the named module,
+  while ``qualified_module_name.*`` matches ``dotted_module_name`` and any
   submodules (so ``foo.bar.*`` would match all of ``foo.bar``,
   ``foo.bar.baz``, and ``foo.bar.baz.quux``).
 
+  Patterns may also be "unstructured" wildcards, in which stars may
+  appear in the middle of a name (e.g
+  ``site.*.migrations.*``). Stars match zero or more module
+  components (so ``site.*.migrations.*`` can match ``site.migrations``).
+
+  When options conflict, the precedence order for the configuration sections is:
+    1. Sections with concrete module names (``foo.bar``)
+    2. Sections with "unstructured" wildcard patterns (``foo.*.baz``),
+       with sections later in the configuration file overriding
+       sections earlier.
+    3. Sections with "well-structured" wildcard patterns
+       (``foo.bar.*``), with more specific overriding more general.
+    4. Command line options.
+    5. Top-level configuration file options.
+
+The difference in precedence order between "structured" patterns (by
+specificity) and "unstructured" patterns (by order in the file) is
+unfortunate, and is subject to change in future versions.
+
 .. note::
 
    The ``warn_unused_configs`` flag may be useful to debug misspelled

mypy/main.py

@@ -746,8 +746,9 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None:
                     glob = glob.replace(os.altsep, '.')
 
                 if (any(c in glob for c in '?[]!') or
-                        ('*' in glob and (not glob.endswith('.*') or '*' in glob[:-2]))):
-                    print("%s: Invalid pattern. Patterns must be 'module_name' or 'module_name.*'"
+                        any('*' in x and x != '*' for x in glob.split('.'))):
+                    print("%s: Patterns must be fully-qualified module names, optionally "
+                          "with '*' in some components (e.g spam.*.eggs.*)"
                           % prefix,
                           file=sys.stderr)
                 else:

mypy/options.py

@@ -1,8 +1,9 @@
 from collections import OrderedDict
+import re
 import pprint
 import sys
 
-from typing import Dict, List, Mapping, MutableMapping, Optional, Set, Tuple
+from typing import Dict, List, Mapping, MutableMapping, Optional, Pattern, Set, Tuple
 
 from mypy import defaults
 
@@ -167,8 +168,8 @@ def __init__(self) -> None:
         self.plugins = []  # type: List[str]
 
         # Per-module options (raw)
-        pm_opts = OrderedDict()  # type: OrderedDict[str, Dict[str, object]]
-        self.per_module_options = pm_opts
+        self.per_module_options = OrderedDict()  # type: OrderedDict[str, Dict[str, object]]
+        self.glob_options = []  # type: List[Tuple[str, Pattern[str]]]
         self.unused_configs = set()  # type: Set[str]
 
         # -- development options --
@@ -208,27 +209,56 @@ def __ne__(self, other: object) -> bool:
     def __repr__(self) -> str:
         return 'Options({})'.format(pprint.pformat(self.snapshot()))
 
+    def apply_changes(self, changes: Dict[str, object]) -> 'Options':
+        new_options = Options()
+        new_options.__dict__.update(self.__dict__)
+        new_options.__dict__.update(changes)
+        return new_options
+
     def build_per_module_cache(self) -> None:
         self.per_module_cache = {}
-        # Since configs inherit from glob configs above them in the hierarchy,
+
+        # Config precedence is as follows:
+        #  1. Concrete section names: foo.bar.baz
+        #  2. "Unstructured" glob patterns: foo.*.baz, in the order
+        #     they appear in the file (last wins)
+        #  3. "Well-structured" wildcard patterns: foo.bar.*, in specificity order.
+
+        # Since structured configs inherit from structured configs above them in the hierarchy,
         # we need to process per-module configs in a careful order.
-        # We have to process foo.* before foo.bar.* before foo.bar.
-        # To do this, process all glob configs before non-glob configs and
+        # We have to process foo.* before foo.bar.* before foo.bar,
+        # and we need to apply *.bar to foo.bar but not to foo.bar.*.
+        # To do this, process all well-structured glob configs before non-glob configs and
         # exploit the fact that foo.* sorts earlier ASCIIbetically (unicodebetically?)
         # than foo.bar.*.
-        keys = (sorted(k for k in self.per_module_options.keys() if k.endswith('.*')) +
-                [k for k in self.per_module_options.keys() if not k.endswith('.*')])
-        for key in keys:
+        # (A section being "processed last" results in its config "winning".)
+        # Unstructured glob configs are stored and are all checked for each module.
+        unstructured_glob_keys = [k for k in self.per_module_options.keys()
+                                  if '*' in k[:-1]]
+        structured_keys = [k for k in self.per_module_options.keys()
+                           if '*' not in k[:-1]]
+        wildcards = sorted(k for k in structured_keys if k.endswith('.*'))
+        concrete = [k for k in structured_keys if not k.endswith('.*')]
+
+        for glob in unstructured_glob_keys:
+            self.glob_options.append((glob, self.compile_glob(glob)))
+
+        # We (for ease of implementation) treat unstructured glob
+        # sections as used if any real modules use them or if any
+        # concrete config sections use them. This means we need to
+        # track which get used while constructing.
+        self.unused_configs = set(unstructured_glob_keys)
+
+        for key in wildcards + concrete:
             # Find what the options for this key would be, just based
             # on inheriting from parent configs.
             options = self.clone_for_module(key)
             # And then update it with its per-module options.
-            new_options = Options()
-            new_options.__dict__.update(options.__dict__)
-            new_options.__dict__.update(self.per_module_options[key])
-            self.per_module_cache[key] = new_options
+            self.per_module_cache[key] = options.apply_changes(self.per_module_options[key])
 
-        self.unused_configs = set(keys)
+        # Add the more structured sections into unused configs, since
+        # they only count as used if actually used by a real module.
+        self.unused_configs.update(structured_keys)
 
     def clone_for_module(self, module: str) -> 'Options':
         """Create an Options object that incorporates per-module options.
@@ -250,18 +280,38 @@ def clone_for_module(self, module: str) -> 'Options':
         # in that order, looking for an entry.
         # This is technically quadratic in the length of the path, but module paths
         # don't actually get all that long.
+        options = self
         path = module.split('.')
         for i in range(len(path), 0, -1):
             key = '.'.join(path[:i] + ['*'])
             if key in self.per_module_cache:
                 self.unused_configs.discard(key)
-                return self.per_module_cache[key]
+                options = self.per_module_cache[key]
+                break
+
+        # OK and *now* we need to look for unstructured glob matches.
+        # We only do this for concrete modules, not structured wildcards.
+        if not module.endswith('.*'):
+            for key, pattern in self.glob_options:
+                if pattern.match(module):
+                    self.unused_configs.discard(key)
+                    options = options.apply_changes(self.per_module_options[key])
 
         # We could update the cache to directly point to modules once
         # they have been looked up, but in testing this made things
         # slower and not faster, so we don't bother.
 
-        return self
+        return options
+
+    def compile_glob(self, s: str) -> Pattern[str]:
+        # Compile one of the glob patterns to a regex so that '.*' can
+        # match *zero or more* module sections. This means we compile
+        # '.*' into '(\..*)?'.
+        parts = s.split('.')
+        expr = re.escape(parts[0]) if parts[0] != '*' else '.*'
+        for part in parts[1:]:
+            expr += re.escape('.' + part) if part != '*' else '(\..*)?'
+        return re.compile(expr + '\\Z')
 
     def select_options_affecting_cache(self) -> Mapping[str, object]:
         return {opt: getattr(self, opt) for opt in self.OPTIONS_AFFECTING_CACHE}

test-data/unit/cmdline.test

@@ -203,8 +203,8 @@ def g(a: int) -> int: return f(a)
 def f(a): pass
 def g(a: int) -> int: return f(a)
 [out]
-mypy.ini: [mypy-*x*]: Invalid pattern. Patterns must be 'module_name' or 'module_name.*'
-mypy.ini: [mypy-*y*]: Invalid pattern. Patterns must be 'module_name' or 'module_name.*'
+mypy.ini: [mypy-*x*]: Patterns must be fully-qualified module names, optionally with '*' in some components (e.g spam.*.eggs.*)
+mypy.ini: [mypy-*y*]: Patterns must be fully-qualified module names, optionally with '*' in some components (e.g spam.*.eggs.*)
 == Return code: 0
 
 [case testMultipleGlobConfigSection]
@@ -268,7 +268,6 @@ mypy.ini: [mypy]: ignore_missing_imports: Not a boolean: nah
 python_version = 3.4
 [out]
 mypy.ini: [mypy-*]: Per-module sections should only specify per-module flags (python_version)
-mypy.ini: [mypy-*]: Invalid pattern. Patterns must be 'module_name' or 'module_name.*'
 == Return code: 0
 
 [case testConfigMypyPath]
@@ -1179,10 +1178,48 @@ warn_unused_configs = True
 [[mypy-spam.eggs]
 [[mypy-emarg.*]
 [[mypy-emarg.hatch]
+-- Currently we don't treat an unstructured pattern like a.*.b as unused
+-- if it matches another section (like a.x.b). This would be reasonable
+-- to change.
+[[mypy-a.*.b]
+[[mypy-a.*.c]
+[[mypy-a.x.b]
 [file foo.py]
 [file quux.py]
 [file spam/__init__.py]
 [file spam/eggs.py]
 [out]
-Warning: unused section(s) in mypy.ini: [mypy-bar], [mypy-baz.*], [mypy-emarg.*], [mypy-emarg.hatch]
+Warning: unused section(s) in mypy.ini: [mypy-bar], [mypy-baz.*], [mypy-emarg.*], [mypy-emarg.hatch], [mypy-a.*.c], [mypy-a.x.b]
 == Return code: 0
+
+[case testConfigUnstructuredGlob]
+# cmd: mypy emarg foo
+[file mypy.ini]
+[[mypy]
+ignore_errors = true
+[[mypy-*.lol]
+ignore_errors = false
+[[mypy-emarg.*]
+ignore_errors = false
+[[mypy-emarg.*.villip.*]
+ignore_errors = true
+[[mypy-emarg.hatch.villip.mankangulisk]
+ignore_errors = false
+[file emarg/__init__.py]
+[file emarg/foo.py]
+fail
+[file emarg/villip.py]
+fail
+[file emarg/hatch/__init__.py]
+[file emarg/hatch/villip/__init__.py]
+[file emarg/hatch/villip/nus.py]
+fail
+[file emarg/hatch/villip/mankangulisk.py]
+fail
+[file foo/__init__.py]
+[file foo/lol.py]
+fail
+[out]
+foo/lol.py:1: error: Name 'fail' is not defined
+emarg/foo.py:1: error: Name 'fail' is not defined
+emarg/hatch/villip/mankangulisk.py:1: error: Name 'fail' is not defined