From 72d9ab6304fbdcee284e844bf65175647320b4d9 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Fri, 18 May 2018 16:07:41 -0700 Subject: [PATCH 01/16] Overhaul overload semantics, remove erasure, add union math This pull request: 1. Modifies how mypy handles overloads to match the proposal I made in the typing repo. 2. Starts removing type erasure from overload checks. 3. Adds support for basic union math. 4. Makes overloads respect keyword-only args This pull request does NOT implement the following: 1. Special-casing descriptors 2. Improving how operator methods are handled: I've tweaked one or two methods to take advantage of some of the changes I made in this PR, but stayed away from making any drastic changes for now. 3. Detecting partially overlapping argument types -- e.g. code like: @overload def f(x: Union[A, B]) -> str: ... @overload def f(x: Union[B, C]) -> int: ... @overload def g(x: Tuple[int, ...]) -> str: ... @overload def g(x: Tuple[int, int]) -> int: ... 4. Detecting overlapping "argument counts". For example, we should flag the following as an error since the first alternative could potentially overlap with the second. @overload def f(*args: int) -> int: ... @overload def f(x: int, y: int, z: int) -> str: ... 5. The "is-more-precise" relation. It's working in most normal cases but still contains a few bugs, mostly relating to type vars. For example, this currently isn't being flagged as an error: class Wrapper(Generic[T]): @overload def f(self, x: int) -> int: ... # No error? @overload def f(self, x: T) -> str: ... (This PR does the right thing if 'T' isn't bound to a containing class though:) class Wrapper: @overload def f(self, x: int, y: int) -> int: ... # Error @overload def f(self, x: T, y: T) -> str: ... Currently, what I'm doing is using the existing `is_more_precise` method, which calls `is_proper_subtype`. I think i'll either need to rewrite `is_more_precise` to do what I want with typevars or find a way of forcing the two methods to unify their typevars before running the `is_proper_subtype` check. The plan is to address these 5 TODOs in future pull requests. Items 1 and 2 are basically orthogonal to the overloads overhaul; items 3, 4, and 5 basically boil down to finding ways to teach mypy to detect if one thing is *potentially* compatible with another. For example, mypy contains code to tell if one type is *definitely* a subtype of another; fixing items 3 and 5 involve writing code to check if a type is *potentially* a subtype of another. --- mypy/checker.py | 171 ++++++++---- mypy/checkexpr.py | 312 ++++++++++++++++----- mypy/constraints.py | 4 +- mypy/messages.py | 17 +- mypy/subtypes.py | 83 ++++-- mypy/types.py | 21 +- test-data/unit/check-classes.test | 28 +- test-data/unit/check-overloading.test | 380 ++++++++++++++++++++++++-- test-data/unit/check-protocols.test | 4 +- test-data/unit/check-typeddict.test | 6 +- test-data/unit/fine-grained.test | 2 + 11 files changed, 846 insertions(+), 182 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 315c75b13d1f..28976d6ff775 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -39,7 +39,7 @@ from mypy import messages from mypy.subtypes import ( is_subtype, is_equivalent, is_proper_subtype, is_more_precise, - restrict_subtype_away, is_subtype_ignoring_tvars, is_callable_subtype, + restrict_subtype_away, is_subtype_ignoring_tvars, is_callable_compatible, unify_generic_callable, find_member ) from mypy.maptype import map_instance_to_supertype @@ -407,22 +407,32 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: if defn.info: self.check_method_override(defn) self.check_inplace_operator_method(defn) - self.check_overlapping_overloads(defn) + if not defn.is_property: + self.check_overlapping_overloads(defn) return None def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: # At this point we should have set the impl already, and all remaining # items are decorators for i, item in enumerate(defn.items): + # TODO overloads involving decorators assert isinstance(item, Decorator) sig1 = self.function_type(item.func) + for j, item2 in enumerate(defn.items[i + 1:]): - # TODO overloads involving decorators assert isinstance(item2, Decorator) sig2 = self.function_type(item2.func) - if is_unsafe_overlapping_signatures(sig1, sig2): - self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, - item.func) + + assert isinstance(sig1, CallableType) + assert isinstance(sig2, CallableType) + + if not are_argument_counts_overlapping(sig1, sig2): + continue + + if if_overload_can_never_match(sig1, sig2): + self.msg.overloaded_signature_will_never_match(i + 1, i + j + 2, item2.func) + elif is_unsafe_overlapping_overload_signatures(sig1, sig2): + self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, item.func) if defn.impl: if isinstance(defn.impl, FuncDef): impl_type = defn.impl.type @@ -437,7 +447,8 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: assert isinstance(impl_type, CallableType) assert isinstance(sig1, CallableType) - if not is_callable_subtype(impl_type, sig1, ignore_return=True): + if not is_callable_compatible(impl_type, sig1, + is_compat=is_subtype, ignore_return=True): self.msg.overloaded_signatures_arg_specific(i + 1, defn.impl) impl_type_subst = impl_type if impl_type.variables: @@ -1038,8 +1049,8 @@ def check_overlapping_op_methods(self, fallback=self.named_type('builtins.function'), name=reverse_type.name) - if is_unsafe_overlapping_signatures(forward_tweaked, - reverse_tweaked): + if is_unsafe_overlapping_operator_signatures( + forward_tweaked, reverse_tweaked): self.msg.operator_method_signatures_overlap( reverse_class, reverse_name, forward_base, forward_name, context) @@ -1832,10 +1843,18 @@ def check_multi_assignment_from_union(self, lvalues: List[Expression], rvalue: E # Bind a union of types collected in 'assignments' to every expression. if isinstance(expr, StarExpr): expr = expr.expr - types, declared_types = zip(*items) + + # TODO: See todo in binder.py, ConditionalTypeBinder.assign_type + # It's unclear why the 'declared_type' param is sometimes 'None' + clean_items = [] # type: List[Tuple[Type, Type]] + for type, declared_type in items: + assert declared_type is not None + clean_items.append((type, declared_type)) + + types, declared_types = zip(*clean_items) self.binder.assign_type(expr, - UnionType.make_simplified_union(types), - UnionType.make_simplified_union(declared_types), + UnionType.make_simplified_union(list(types)), + UnionType.make_simplified_union(list(declared_types)), False) for union, lv in zip(union_types, self.flatten_lvalues(lvalues)): # Properly store the inferred types. @@ -3547,18 +3566,96 @@ def type(self, type: Type) -> Type: return expand_type(type, self.map) -def is_unsafe_overlapping_signatures(signature: Type, other: Type) -> bool: - """Check if two signatures may be unsafely overlapping. +def are_argument_counts_overlapping(t: CallableType, s: CallableType) -> bool: + """Can a single call match both t and s, based just on positional argument counts? + """ + min_args = max(t.min_args, s.min_args) + max_args = min(t.max_possible_positional_args(), s.max_possible_positional_args()) + return min_args <= max_args - Two signatures s and t are overlapping if both can be valid for the same + +def is_unsafe_overlapping_overload_signatures(signature: CallableType, + other: CallableType) -> bool: + """Check if two overloaded function signatures may be unsafely overlapping. + + We consider two functions 's' and 't' to be unsafely overlapping both + of the following are true: + + 1. s's parameters are all more precise or partially overlapping with t's + 1. s's return type is NOT a subtype of t's. + + both can be valid for the same statically typed values and the return types are incompatible. + Assumes that 'signature' appears earlier in the list of overload + alternatives then 'other' and that their argument counts are overlapping. + """ + # TODO: Handle partially overlapping parameter types and argument counts + # + # For example, the signatures "f(x: Union[A, B]) -> int" and "f(x: Union[B, C]) -> str" + # is unsafe: the parameter types are partially overlapping. + # + # To fix this, we need to either modify meet.is_overlapping_types or add a new + # function and use "is_more_precise(...) or is_partially_overlapping(...)" for the is_compat + # checks. + # + # Similarly, the signatures "f(x: A, y: A) -> str" and "f(*x: A) -> int" are also unsafe: + # the parameter *counts* or arity are partially overlapping. + # + # To fix this, we need to modify is_callable_compatible so it can optionally detect + # functions that are *potentially* compatible rather then *definitely* compatible. + + # The reason we repeat this check twice is so we can do a slightly better job of + # checking for potentially overlapping param counts. Both calls will actually check + # the param and return types in the same "direction" -- the only thing that differs + # is how is_callable_compatible checks non-positional arguments. + return (is_callable_compatible(signature, other, + is_compat=is_more_precise, + is_compat_return=lambda l, r: not is_subtype(l, r), + check_args_covariantly=True) or + is_callable_compatible(other, signature, + is_compat=is_more_precise, + is_compat_return=lambda l, r: not is_subtype(r, l))) + + +def if_overload_can_never_match(signature: CallableType, other: CallableType) -> bool: + """Check if the 'other' method can never be matched due to 'signature'. + + This can happen if signature's parameters are all strictly broader then + other's parameters. + + Assumes that both signatures have overlapping argument counts. + """ + return is_callable_compatible(signature, other, + is_compat=is_more_precise, + ignore_return=True) + + +def is_unsafe_overlapping_operator_signatures(signature: Type, other: Type) -> bool: + """Check if two operator method signatures may be unsafely overlapping. + + Two signatures s and t are overlapping if both can be valid for the same + statically typed values and the return types are incompatible. + Assume calls are first checked against 'signature', then against 'other'. Thus if 'signature' is more general than 'other', there is no unsafe overlapping. - TODO If argument types vary covariantly, the return type may vary - covariantly as well. + TODO: Clean up this function and make it not perform type erasure. + + Context: This function was previously used to make sure both overloaded + functions and operator methods were not unsafely overlapping. + + We changed the semantics for we should handle overloaded definitions, + but not operator functions. (We can't reuse the same semantics for both: + the overload semantics are too restrictive here). + + We should rewrite this method so that: + + 1. It uses many of the improvements made to overloads: in particular, + eliminating type erasure. + + 2. It contains just the logic necessary for operator methods. """ if isinstance(signature, CallableType): if isinstance(other, CallableType): @@ -3601,12 +3698,11 @@ def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: """Does t have wider arguments than s?""" # TODO should an overload with additional items be allowed to be more # general than one with fewer items (or just one item)? - # TODO check argument kinds and otherwise make more general if isinstance(t, CallableType): if isinstance(s, CallableType): - t, s = unify_generic_callables(t, s) - return all(is_proper_subtype(args, argt) - for argt, args in zip(t.arg_types, s.arg_types)) + return is_callable_compatible(t, s, + is_compat=is_proper_subtype, + ignore_return=True) elif isinstance(t, FunctionLike): if isinstance(s, FunctionLike): if len(t.items()) == len(s.items()): @@ -3615,29 +3711,6 @@ def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool: return False -def unify_generic_callables(t: CallableType, - s: CallableType) -> Tuple[CallableType, - CallableType]: - """Make type variables in generic callables the same if possible. - - Return updated callables. If we can't unify the type variables, - return the unmodified arguments. - """ - # TODO: Use this elsewhere when comparing generic callables. - if t.is_generic() and s.is_generic(): - t_substitutions = {} - s_substitutions = {} - for tv1, tv2 in zip(t.variables, s.variables): - # Are these something we can unify? - if tv1.id != tv2.id and is_equivalent_type_var_def(tv1, tv2): - newdef = TypeVarDef.new_unification_variable(tv2) - t_substitutions[tv1.id] = TypeVarType(newdef) - s_substitutions[tv2.id] = TypeVarType(newdef) - return (cast(CallableType, expand_type(t, t_substitutions)), - cast(CallableType, expand_type(s, s_substitutions))) - return t, s - - def is_equivalent_type_var_def(tv1: TypeVarDef, tv2: TypeVarDef) -> bool: """Are type variable definitions equivalent? @@ -3653,17 +3726,17 @@ def is_equivalent_type_var_def(tv1: TypeVarDef, tv2: TypeVarDef) -> bool: def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool: - # TODO check argument kinds - return all(is_same_type(argt, args) - for argt, args in zip(t.arg_types, s.arg_types)) + return is_callable_compatible(t, s, + is_compat=is_same_type, + ignore_return=True, + check_args_covariantly=True, + ignore_pos_arg_names=True) def is_more_precise_signature(t: CallableType, s: CallableType) -> bool: """Is t more precise than s? - A signature t is more precise than s if all argument types and the return type of t are more precise than the corresponding types in s. - Assume that the argument kinds and names are compatible, and that the argument counts are overlapping. """ diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 0e349a436dab..c6577d7ac048 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -611,13 +611,15 @@ def check_call(self, callee: Type, args: List[Expression], arg_types = self.infer_arg_types_in_context(None, args) self.msg.enable_errors() - target = self.overload_call_target(arg_types, arg_kinds, arg_names, - callee, context, - messages=arg_messages) - return self.check_call(target, args, arg_kinds, context, arg_names, - arg_messages=arg_messages, - callable_name=callable_name, - object_type=object_type) + return self.check_overload_call(callee=callee, + args=args, + arg_types=arg_types, + arg_kinds=arg_kinds, + arg_names=arg_names, + callable_name=callable_name, + object_type=object_type, + context=context, + arg_messages=arg_messages) elif isinstance(callee, AnyType) or not self.chk.in_checked_function(): self.infer_arg_types_in_context(None, args) if isinstance(callee, AnyType): @@ -1104,68 +1106,246 @@ def check_arg(self, caller_type: Type, original_caller_type: Type, if call: self.msg.note_call(original_caller_type, call, context) - def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: Optional[Sequence[Optional[str]]], - overload: Overloaded, context: Context, - messages: Optional[MessageBuilder] = None) -> Type: - """Infer the correct overload item to call with given argument types. + def check_overload_call(self, + callee: Overloaded, + args: List[Expression], + arg_types: List[Type], + arg_kinds: List[int], + arg_names: Optional[Sequence[Optional[str]]], + callable_name: Optional[str], + object_type: Optional[Type], + context: Context, + arg_messages: MessageBuilder) -> Tuple[Type, Type]: + """Checks a call to an overloaded function.""" + # Step 1: Filter call targets to remove ones where the argument counts don't match + plausible_targets = self.plausible_overload_call_targets(arg_types, arg_kinds, + arg_names, callee) + + # Step 2: Attempt to find a matching overload + inferred_result = self.infer_overload_return_type(plausible_targets, args, arg_types, + arg_kinds, arg_names, callable_name, + object_type, context, arg_messages) + if inferred_result is not None: + # Success! Stop early. + return inferred_result + + # Step 3: At this point, we know none of the overload alternatives exactly match. + # We fall back to using the erased types to help with union math/help us + # produce a better error message. + erased_targets = self.overload_erased_call_targets(plausible_targets, arg_types, + arg_kinds, arg_names, context) + + # Step 4: Try and infer a second-best alternative. + if len(erased_targets) == 0: + # Step 4a: There are no viable targets, even if we relax our constraints. Give up. + if not self.chk.should_suppress_optional_error(arg_types): + arg_messages.no_variant_matches_arguments(callee, arg_types, context) + target = AnyType(TypeOfAny.from_error) # type: Type + elif any(isinstance(arg, UnionType) for arg in arg_types): + # Step 4b: Try performing union math + unioned_callable = self.union_overload_matches(erased_targets, args, arg_kinds, + arg_names, context) + target = unioned_callable if unioned_callable is not None else erased_targets[0] + else: + # Step 4c: Use the first matching erased target: it won't match, but at + # least we can have a nicer error message. + # TODO: Adjust the error message here to make it clear there was no match. + target = erased_targets[0] + + '''target = self.overload_call_target(args, arg_types, arg_kinds, arg_names, + callee, context, + messages=arg_messages)''' + return self.check_call(target, args, arg_kinds, context, arg_names, + arg_messages=arg_messages, + callable_name=callable_name, + object_type=object_type) + + def plausible_overload_call_targets(self, + arg_types: List[Type], + arg_kinds: List[int], + arg_names: Optional[Sequence[Optional[str]]], + overload: Overloaded) -> List[CallableType]: + """Returns all overload call targets that having matching argument counts.""" + matches = [] # type: List[CallableType] + for typ in overload.items(): + formal_to_actual = map_actuals_to_formals(arg_kinds, arg_names, + typ.arg_kinds, typ.arg_names, + lambda i: arg_types[i]) - The return value may be CallableType or AnyType (if an unique item - could not be determined). + if self.check_argument_count(typ, arg_types, arg_kinds, arg_names, + formal_to_actual, None, None): + matches.append(typ) + + return matches + + def infer_overload_return_type(self, + plausible_targets: List[CallableType], + args: List[Expression], + arg_types: List[Type], + arg_kinds: List[int], + arg_names: Optional[Sequence[Optional[str]]], + callable_name: Optional[str], + object_type: Optional[Type], + context: Context, + arg_messages: Optional[MessageBuilder] = None, + ) -> Optional[Tuple[Type, Type]]: + """Attempts to find the first matching callable from the given list. + + If multiple targets match due to ambiguous Any parameters, returns (AnyType, AnyType). + If no targets match, returns None. + + Assumes all of the given targets have argument counts compatible with the caller. """ - messages = messages or self.msg - # TODO: For overlapping signatures we should try to get a more precise - # result than 'Any'. - match = [] # type: List[CallableType] - best_match = 0 - for typ in overload.items(): - similarity = self.erased_signature_similarity(arg_types, arg_kinds, arg_names, - typ, context=context) - if similarity > 0 and similarity >= best_match: - if (match and not is_same_type(match[-1].ret_type, - typ.ret_type) and - (not mypy.checker.is_more_precise_signature(match[-1], typ) - or (any(isinstance(arg, AnyType) for arg in arg_types) - and any_arg_causes_overload_ambiguity( - match + [typ], arg_types, arg_kinds, arg_names)))): - # Ambiguous return type. Either the function overload is - # overlapping (which we don't handle very well here) or the - # caller has provided some Any argument types; in either - # case we'll fall back to Any. It's okay to use Any types - # in calls. - # - # Overlapping overload items are generally fine if the - # overlapping is only possible when there is multiple - # inheritance, as this is rare. See docstring of - # mypy.meet.is_overlapping_types for more about this. - # - # Note that there is no ambiguity if the items are - # covariant in both argument types and return types with - # respect to type precision. We'll pick the best/closest - # match. - # - # TODO: Consider returning a union type instead if the - # overlapping is NOT due to Any types? - return AnyType(TypeOfAny.special_form) + + arg_messages = self.msg if arg_messages is None else arg_messages + matches = [] # type: List[CallableType] + inferred = [] # type: List[Tuple[Type, Type]] + args_contain_any = any(isinstance(arg, AnyType) for arg in arg_types) + + for typ in plausible_targets: + overload_messages = self.msg.clean_copy() + prev_messages = self.msg + self.msg = overload_messages + try: + # Passing `overload_messages` as the `arg_messages` parameter doesn't + # seem to reliably catch all possible errors. + # + # TODO: Figure out why + result = self.check_call( + callee=typ, + args=args, + arg_kinds=arg_kinds, + arg_names=arg_names, + context=context, + arg_messages=overload_messages, + callable_name=callable_name, + object_type=object_type) + finally: + self.msg = prev_messages + + is_match = not overload_messages.is_errors() + if is_match: + if not args_contain_any: + # There is no possibility of ambiguity due to 'Any', so we can + # just end right away: + return result + elif (args_contain_any and matches + and not is_same_type(matches[-1].ret_type, typ.ret_type) + and any_arg_causes_overload_ambiguity( + matches + [typ], arg_types, arg_kinds, arg_names)): + # Ambiguous return type. The caller has provided some + # Any argument types (which are okay to use in calls), + # so we fall back to returning 'Any'. + source = AnyType(TypeOfAny.special_form) + return self.check_call(callee=source, + args=args, + arg_kinds=arg_kinds, + arg_names=arg_names, + context=context, + arg_messages=arg_messages, + callable_name=callable_name, + object_type=object_type) else: - match.append(typ) - best_match = max(best_match, similarity) - if not match: - if not self.chk.should_suppress_optional_error(arg_types): - messages.no_variant_matches_arguments(overload, arg_types, context) - return AnyType(TypeOfAny.from_error) - else: - if len(match) == 1: - return match[0] - else: - # More than one signature matches. Pick the first *non-erased* - # matching signature, or default to the first one if none - # match. - for m in match: - if self.match_signature_types(arg_types, arg_kinds, arg_names, m, - context=context): - return m - return match[0] + matches.append(typ) + inferred.append(result) + + return inferred[0] if len(inferred) > 0 else None + + def overload_erased_call_targets(self, + plausible_targets: List[CallableType], + arg_types: List[Type], + arg_kinds: List[int], + arg_names: Optional[Sequence[Optional[str]]], + context: Context) -> List[CallableType]: + """Returns a list of all targets that match the caller after erasing types. + + Assumes all of the given targets have argument counts compatible with the caller. + """ + matches = [] # type: List[CallableType] + for typ in plausible_targets: + if self.erased_signature_similarity(arg_types, arg_kinds, arg_names, typ, context): + matches.append(typ) + return matches + + def union_overload_matches(self, + callables: List[CallableType], + args: List[Expression], + arg_kinds: List[int], + arg_names: Optional[Sequence[Optional[str]]], + context: Context) -> Optional[CallableType]: + """Accepts a list of overload signatures and attempts to combine them together into a + new CallableType consisting of the union of all of the given arguments and return types. + + Returns None if it is not possible to combine the different callables together in a + sound manner. + + Assumes all of the given callables have argument counts compatible with the caller. + """ + assert len(callables) > 0 + if len(callables) == 1: + return callables[0] + + new_args = [[] for _ in range(len(callables[0].arg_types))] # type: List[List[Type]] + new_returns = [] # type: List[Type] + + expected_names = callables[0].arg_names + expected_kinds = callables[0].arg_kinds + + for target in callables: + if target.arg_names != expected_names or target.arg_kinds != expected_kinds: + # We conservatively end if the overloads do not have the exact same signature. + # TODO: Enhance the union overload logic to handle a wider variety of signatures. + return None + + if target.is_generic(): + formal_to_actual = map_actuals_to_formals( + arg_kinds, arg_names, + target.arg_kinds, target.arg_names, + lambda i: self.accept(args[i])) + + target = freshen_function_type_vars(target) + target = self.infer_function_type_arguments_using_context(target, context) + target = self.infer_function_type_arguments( + target, args, arg_kinds, formal_to_actual, context) + + for i, arg in enumerate(target.arg_types): + new_args[i].append(arg) + new_returns.append(target.ret_type) + + union_count = 0 + final_args = [] + for args_list in new_args: + new_type = UnionType.make_simplified_union(args_list) + union_count += 1 if isinstance(new_type, UnionType) else 0 + final_args.append(new_type) + + # TODO: Modify this check to be less conservative. + # + # Currently, we permit only one union union in the arguments because if we allow + # multiple, we can't always guarantee the synthesized callable will be correct. + # + # For example, suppose we had the following two overloads: + # + # @overload + # def f(x: A, y: B) -> None: ... + # @overload + # def f(x: B, y: A) -> None: ... + # + # If we continued and synthesize "def f(x: Union[A,B], y: Union[A,B]) -> None: ...", + # then we'd incorrectly accept calls like "f(A(), A())" when they really ought to + # be rejected. + # + # However, that means we'll also give up if the original overloads contained + # any unions. This is likely unnecessary -- we only really need to give up if + # there are more then one *synthesized* union arguments. + if union_count >= 2: + return None + + return callables[0].copy_modified( + arg_types=final_args, + ret_type=UnionType.make_simplified_union(new_returns), + implicit=True, + from_overloads=True) def erased_signature_similarity(self, arg_types: List[Type], arg_kinds: List[int], arg_names: Optional[Sequence[Optional[str]]], diff --git a/mypy/constraints.py b/mypy/constraints.py index 92a1f35b999b..67d42cf8c923 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -525,7 +525,9 @@ def find_matching_overload_item(overloaded: Overloaded, template: CallableType) for item in items: # Return type may be indeterminate in the template, so ignore it when performing a # subtype check. - if mypy.subtypes.is_callable_subtype(item, template, ignore_return=True): + if mypy.subtypes.is_callable_compatible(item, template, + is_compat=mypy.subtypes.is_subtype, + ignore_return=True): return item # Fall back to the first item if we can't find a match. This is totally arbitrary -- # maybe we should just bail out at this point. diff --git a/mypy/messages.py b/mypy/messages.py index e14fe64f3528..2654d29447c1 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -148,6 +148,11 @@ def copy(self) -> 'MessageBuilder': new.disable_type_names = self.disable_type_names return new + def clean_copy(self) -> 'MessageBuilder': + errors = self.errors.copy() + errors.error_info_map = OrderedDict() + return MessageBuilder(errors, self.modules) + def add_errors(self, messages: 'MessageBuilder') -> None: """Add errors in messages to this builder.""" if self.disable_count <= 0: @@ -937,11 +942,19 @@ def incompatible_typevar_value(self, self.format(typ)), context) - def overloaded_signatures_overlap(self, index1: int, index2: int, - context: Context) -> None: + def overloaded_signatures_overlap(self, index1: int, index2: int, context: Context) -> None: self.fail('Overloaded function signatures {} and {} overlap with ' 'incompatible return types'.format(index1, index2), context) + def overloaded_signature_will_never_match(self, index1: int, index2: int, + context: Context) -> None: + self.fail( + 'Overloaded function signature {index2} will never be matched: ' + 'function {index1}\'s parameter type(s) are the same or broader'.format( + index1=index1, + index2=index2), + context) + def overloaded_signatures_arg_specific(self, index1: int, context: Context) -> None: self.fail('Overloaded function implementation does not accept all possible arguments ' 'of signature {}'.format(index1), context) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index ecbcff4f292b..eb8b7fe92c08 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -203,8 +203,9 @@ def visit_type_var(self, left: TypeVarType) -> bool: def visit_callable_type(self, left: CallableType) -> bool: right = self.right if isinstance(right, CallableType): - return is_callable_subtype( + return is_callable_compatible( left, right, + is_compat=is_subtype, ignore_pos_arg_names=self.ignore_pos_arg_names) elif isinstance(right, Overloaded): return all(is_subtype(left, item, self.check_type_parameter, @@ -310,10 +311,12 @@ def visit_overloaded(self, left: Overloaded) -> bool: else: # If this one overlaps with the supertype in any way, but it wasn't # an exact match, then it's a potential error. - if (is_callable_subtype(left_item, right_item, ignore_return=True, - ignore_pos_arg_names=self.ignore_pos_arg_names) or - is_callable_subtype(right_item, left_item, ignore_return=True, - ignore_pos_arg_names=self.ignore_pos_arg_names)): + if (is_callable_compatible(left_item, right_item, + is_compat=is_subtype, ignore_return=True, + ignore_pos_arg_names=self.ignore_pos_arg_names) or + is_callable_compatible(right_item, left_item, + is_compat=is_subtype, ignore_return=True, + ignore_pos_arg_names=self.ignore_pos_arg_names)): # If this is an overload that's already been matched, there's no # problem. if left_item not in matched_overloads: @@ -568,16 +571,33 @@ def non_method_protocol_members(tp: TypeInfo) -> List[str]: return result -def is_callable_subtype(left: CallableType, right: CallableType, - ignore_return: bool = False, - ignore_pos_arg_names: bool = False, - use_proper_subtype: bool = False) -> bool: - """Is left a subtype of right?""" +def is_callable_compatible(left: CallableType, right: CallableType, + *, + is_compat: Callable[[Type, Type], bool], + is_compat_return: Optional[Callable[[Type, Type], bool]] = None, + ignore_return: bool = False, + ignore_pos_arg_names: bool = False, + check_args_covariantly: bool = False) -> bool: + """Is the left compatible with the right, using the provided compatibility check? - if use_proper_subtype: - is_compat = is_proper_subtype - else: - is_compat = is_subtype + is_compat: + The check we want to run against the parameters. + + is_compat_return: + The check we want to run against the return type. + If None, use the 'is_compat' check. + + check_args_covariantly: + If true, check if the left's args is compatible with the right's + instead of the other way around (contravariantly). + + This function is mostly used to check if the left is a subtype of the right which + is why the default is to check the args contravariantly. However, it's occasionally + useful to check the args using some other check, so we leave the variance + configurable. + """ + if is_compat_return is None: + is_compat_return = is_compat # If either function is implicitly typed, ignore positional arg names too if left.implicit or right.implicit: @@ -607,9 +627,12 @@ def is_callable_subtype(left: CallableType, right: CallableType, left = unified # Check return types. - if not ignore_return and not is_compat(left.ret_type, right.ret_type): + if not ignore_return and not is_compat_return(left.ret_type, right.ret_type): return False + if check_args_covariantly: + is_compat = flip_compat_check(is_compat) + if right.is_ellipsis_args: return True @@ -652,7 +675,7 @@ def is_callable_subtype(left: CallableType, right: CallableType, # Right has an infinite series of optional positional arguments # here. Get all further positional arguments of left, and make sure # they're more general than their corresponding member in this - # series. Also make sure left has its own inifite series of + # series. Also make sure left has its own infinite series of # optional positional arguments. if not left.is_var_arg: return False @@ -664,7 +687,7 @@ def is_callable_subtype(left: CallableType, right: CallableType, right_by_position = right.argument_by_position(j) assert right_by_position is not None if not are_args_compatible(left_by_position, right_by_position, - ignore_pos_arg_names, use_proper_subtype): + ignore_pos_arg_names, is_compat): return False j += 1 continue @@ -687,7 +710,7 @@ def is_callable_subtype(left: CallableType, right: CallableType, right_by_name = right.argument_by_name(name) assert right_by_name is not None if not are_args_compatible(left_by_name, right_by_name, - ignore_pos_arg_names, use_proper_subtype): + ignore_pos_arg_names, is_compat): return False continue @@ -696,7 +719,8 @@ def is_callable_subtype(left: CallableType, right: CallableType, if left_arg is None: return False - if not are_args_compatible(left_arg, right_arg, ignore_pos_arg_names, use_proper_subtype): + if not are_args_compatible(left_arg, right_arg, + ignore_pos_arg_names, is_compat): return False done_with_positional = False @@ -748,7 +772,7 @@ def are_args_compatible( left: FormalArgument, right: FormalArgument, ignore_pos_arg_names: bool, - use_proper_subtype: bool) -> bool: + is_compat: Callable[[Type, Type], bool]) -> bool: # If right has a specific name it wants this argument to be, left must # have the same. if right.name is not None and left.name != right.name: @@ -759,18 +783,20 @@ def are_args_compatible( if right.pos is not None and left.pos != right.pos: return False # Left must have a more general type - if use_proper_subtype: - if not is_proper_subtype(right.typ, left.typ): - return False - else: - if not is_subtype(right.typ, left.typ): - return False + if not is_compat(right.typ, left.typ): + return False # If right's argument is optional, left's must also be. if not right.required and left.required: return False return True +def flip_compat_check(is_compat: Callable[[Type, Type], bool]) -> Callable[[Type, Type], bool]: + def new_is_compat(left: Type, right: Type) -> bool: + return is_compat(right, left) + return new_is_compat + + def unify_generic_callable(type: CallableType, target: CallableType, ignore_return: bool) -> Optional[CallableType]: """Try to unify a generic callable type with another callable type. @@ -913,10 +939,7 @@ def visit_type_var(self, left: TypeVarType) -> bool: def visit_callable_type(self, left: CallableType) -> bool: right = self.right if isinstance(right, CallableType): - return is_callable_subtype( - left, right, - ignore_pos_arg_names=False, - use_proper_subtype=True) + return is_callable_compatible(left, right, is_compat=is_proper_subtype) elif isinstance(right, Overloaded): return all(is_proper_subtype(left, item) for item in right.items()) diff --git a/mypy/types.py b/mypy/types.py index 43015fbfe11a..89965eb99583 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1,5 +1,6 @@ """Classes for representing mypy types.""" +import sys import copy from abc import abstractmethod from collections import OrderedDict @@ -674,6 +675,8 @@ class CallableType(FunctionLike): # 'dict') 'from_type_type', # Was this callable generated by analyzing Type[...] # instantiation? + 'from_overloads', # Was this callable generated by synthesizing + # multiple overloads? 'bound_args', # Bound type args, mostly unused but may be useful for # tools that consume mypy ASTs ) @@ -694,6 +697,7 @@ def __init__(self, is_classmethod_class: bool = False, special_sig: Optional[str] = None, from_type_type: bool = False, + from_overloads: bool = False, bound_args: Sequence[Optional[Type]] = (), ) -> None: super().__init__(line, column) @@ -718,6 +722,7 @@ def __init__(self, self.is_classmethod_class = is_classmethod_class self.special_sig = special_sig self.from_type_type = from_type_type + self.from_overloads = from_overloads if not bound_args: bound_args = () self.bound_args = bound_args @@ -734,8 +739,10 @@ def copy_modified(self, line: int = _dummy, column: int = _dummy, is_ellipsis_args: bool = _dummy, + implicit: bool = _dummy, special_sig: Optional[str] = _dummy, from_type_type: bool = _dummy, + from_overloads: bool = _dummy, bound_args: List[Optional[Type]] = _dummy) -> 'CallableType': return CallableType( arg_types=arg_types if arg_types is not _dummy else self.arg_types, @@ -750,10 +757,11 @@ def copy_modified(self, column=column if column is not _dummy else self.column, is_ellipsis_args=( is_ellipsis_args if is_ellipsis_args is not _dummy else self.is_ellipsis_args), - implicit=self.implicit, + implicit=implicit if implicit is not _dummy else self.implicit, is_classmethod_class=self.is_classmethod_class, special_sig=special_sig if special_sig is not _dummy else self.special_sig, from_type_type=from_type_type if from_type_type is not _dummy else self.from_type_type, + from_overloads=from_overloads if from_overloads is not _dummy else self.from_overloads, bound_args=bound_args if bound_args is not _dummy else self.bound_args, ) @@ -789,6 +797,15 @@ def max_fixed_args(self) -> int: n -= 1 return n + def max_possible_positional_args(self) -> int: + """Returns maximum number of positional arguments this method could possibly accept. + + This takes into acount *arg and **kwargs but excludes keyword-only args.""" + if self.is_var_arg or self.is_kw_arg: + return sys.maxsize + blacklist = (ARG_NAMED, ARG_NAMED_OPT) + return len([kind not in blacklist for kind in self.arg_kinds]) + def corresponding_argument(self, model: FormalArgument) -> Optional[FormalArgument]: """Return the argument in this function that corresponds to `model`""" @@ -905,6 +922,7 @@ def serialize(self) -> JsonDict: 'is_ellipsis_args': self.is_ellipsis_args, 'implicit': self.implicit, 'is_classmethod_class': self.is_classmethod_class, + 'from_overloads': self.from_overloads, 'bound_args': [(None if t is None else t.serialize()) for t in self.bound_args], } @@ -923,6 +941,7 @@ def deserialize(cls, data: JsonDict) -> 'CallableType': is_ellipsis_args=data['is_ellipsis_args'], implicit=data['implicit'], is_classmethod_class=data['is_classmethod_class'], + from_overloads=data['from_overloads'], bound_args=[(None if t is None else deserialize_type(t)) for t in data['bound_args']], ) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index a56e246cd2c8..8e33e50d234e 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -1649,8 +1649,14 @@ class B(A): def __add__(self, x: 'A') -> 'A': pass @overload def __add__(self, x: 'B') -> 'B': pass +class C(A): + @overload + def __add__(self, x: 'B') -> 'B': pass + @overload + def __add__(self, x: 'A') -> 'A': pass [out] tmp/foo.pyi:8: error: Signature of "__add__" incompatible with supertype "A" +tmp/foo.pyi:11: error: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader [case testReverseOperatorMethodArgumentType] from typing import Any @@ -1717,9 +1723,9 @@ from foo import * from typing import overload, Any class A: @overload - def __radd__(self, x: 'A') -> str: pass # Error + def __radd__(self, x: 'A') -> str: pass @overload - def __radd__(self, x: 'A') -> Any: pass + def __radd__(self, x: 'A') -> Any: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader [out] [case testReverseOperatorMethodArgumentTypeAndOverloadedMethod] @@ -2626,15 +2632,15 @@ class Mbad(type): [case testTypeMatchesOverloadedFunctions] from foo import * [file foo.pyi] -from typing import Type, overload, Union +from typing import Type, overload, Any class User: pass UserType = User # type: Type[User] @overload -def f(a: object) -> int: pass +def f(a: int) -> Any: pass @overload -def f(a: int) -> str: pass +def f(a: object) -> int: pass reveal_type(f(User)) # E: Revealed type is 'builtins.int' reveal_type(f(UserType)) # E: Revealed type is 'builtins.int' @@ -2714,10 +2720,10 @@ reveal_type(f("hi")) # E: Revealed type is 'foo.User' [builtins fixtures/classmethod.pyi] [out] -[case testGeneralTypeDoesNotMatchSpecificTypeInOverloadedFunctions] +[case testGeneralTypeMatchesSpecificTypeInOverloadedFunctions] from foo import * [file foo.pyi] -from typing import Type, overload +from typing import Type, Any, overload class User: pass @@ -2726,10 +2732,12 @@ def f(a: Type[User]) -> None: pass @overload def f(a: int) -> None: pass -def mock() -> type: return User +def mock_1() -> type: return User +def mock_2() -> Type[Any]: return User f(User) -f(mock()) # E: No overload variant of "f" matches argument type "type" +f(mock_1()) +f(mock_2()) [builtins fixtures/classmethod.pyi] [out] @@ -2915,7 +2923,7 @@ class Sub(Super): @overload # E: Signature of "foo" incompatible with supertype "Super" def foo(self, a: A) -> A: pass @overload - def foo(self, a: B) -> C: pass + def foo(self, a: B) -> C: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader @overload def foo(self, a: C) -> C: pass [builtins fixtures/classmethod.pyi] diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 3fa332c9193e..5b8f85f9dba7 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -655,8 +655,7 @@ class B(A): pass @overload def f(x: A) -> A: pass @overload -def f(x: B) -> B: pass # This is more specific than the first item, and thus - # will never be called. +def f(x: B) -> B: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader [case testPartiallyCovariantOverlappingOverloadSignatures] from foo import * @@ -676,9 +675,9 @@ from typing import overload class A: pass class B(A): pass @overload -def g(x: A) -> int: pass # Fine, since A us supertype of B. +def g(x: A) -> int: pass @overload -def g(x: B) -> str: pass +def g(x: B) -> str: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader [case testCovariantOverlappingOverloadSignatures] from foo import * @@ -716,9 +715,9 @@ from foo import * [file foo.pyi] from typing import Any, overload @overload -def g(x: Any) -> Any: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def g(x: Any) -> Any: pass @overload -def g(x: int) -> int: pass +def g(x: int) -> int: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader [case testOverloadedLtAndGtMethods] from foo import * @@ -924,7 +923,7 @@ def f(x: int, y: List[str] = None) -> int: pass f(y=[1], x=0)() # E: "int" not callable f(y=[''], x=0)() # E: "int" not callable a = f(y=[['']], x=0) # E: List item 0 has incompatible type "List[str]"; expected "int" -a() # E: "int" not callable +reveal_type(a) # E: Revealed type is 'builtins.int' [builtins fixtures/list.pyi] [case testOverloadWithDerivedFromAny] @@ -987,15 +986,29 @@ def g(x: U, y: V) -> None: [case testOverlapWithTypeVars] from foo import * [file foo.pyi] -from typing import overload, TypeVar, Sequence +from typing import overload, TypeVar, Sequence, List T = TypeVar('T', bound=str) @overload def f(x: Sequence[T]) -> None: pass @overload def f(x: Sequence[int]) -> int: pass -# These are considered overlapping despite the bound on T due to runtime type erasure. -[out] -tmp/foo.pyi:4: error: Overloaded function signatures 1 and 2 overlap with incompatible return types + +@overload +def g(x: Sequence[T]) -> None: pass +@overload +def g(x: Sequence[str]) -> int: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader + +@overload +def h(x: Sequence[str]) -> int: pass +@overload +def h(x: Sequence[T]) -> None: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader + +@overload +def i(x: List[str]) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def i(x: List[T]) -> None: pass +[builtins fixtures/list.pyi] + [case testOverlapWithTypeVarsWithValues] from foo import * @@ -1026,16 +1039,21 @@ g(1, 'foo') g(1, 'foo', b'bar') # E: Value of type variable "AnyStr" of "g" cannot be "object" [builtins fixtures/primitives.pyi] -[case testBadOverlapWithTypeVarsWithValues] +[case testOverlapWithTypeVarsWithValuesOrdering] from foo import * [file foo.pyi] from typing import overload, TypeVar AnyStr = TypeVar('AnyStr', bytes, str) @overload -def f(x: AnyStr) -> None: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: AnyStr) -> AnyStr: pass +@overload +def f(x: str) -> str: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader + +@overload +def g(x: str) -> str: pass @overload -def f(x: str) -> bool: pass +def g(x: AnyStr) -> AnyStr: pass [builtins fixtures/primitives.pyi] [case testOverlappingOverloadCounting] @@ -1362,11 +1380,11 @@ def r1(x: Any) -> Any: ... @overload def r2(x: Tuple[A, ...]) -> A: ... @overload -def r2(x: Tuple[A, A]) -> B: ... +def r2(x: Tuple[A, A]) -> B: ... # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader @overload -def r2(x: Tuple[A]) -> B: ... +def r2(x: Tuple[A]) -> B: ... # E: Overloaded function signature 3 will never be matched: function 1's parameter type(s) are the same or broader @overload -def r2(x: Tuple[()]) -> B: ... +def r2(x: Tuple[()]) -> B: ... # E: Overloaded function signature 4 will never be matched: function 1's parameter type(s) are the same or broader def r2(x: Any) -> Any: ... [builtins fixtures/tuple.pyi] @@ -1391,7 +1409,7 @@ def r(x: Any) -> Any:... @overload def g(x: A) -> A: ... @overload -def g(x: Tuple[A1, ...]) -> B: ... # E: Overloaded function signatures 2 and 3 overlap with incompatible return types +def g(x: Tuple[A1, ...]) -> B: ... # TODO: This should be an error @overload def g(x: Tuple[A, A]) -> C: ... @overload @@ -1504,3 +1522,329 @@ def bar(x: None) -> object: ... # E: Overloaded function signatures 1 and 2 ove @overload def bar(x: Optional[str]) -> str: ... def bar(x): pass + +[case testOverloadWithNonPositionalArgs] +from typing import overload + +class A: ... +class B: ... +class C: ... + +@overload +def foo(*, p1: A, p2: B = B()) -> A: ... +@overload +def foo(*, p2: B = B()) -> B: ... +def foo(p1, p2=None): ... + +reveal_type(foo()) # E: Revealed type is '__main__.B' +reveal_type(foo(p2=B())) # E: Revealed type is '__main__.B' +reveal_type(foo(p1=A())) # E: Revealed type is '__main__.A' + +[case testOverloadWithNonPositionalArgsIgnoresOrder] +from typing import overload + +class A: ... +class B(A): ... +class X: ... +class Y: ... + +@overload +def f(*, p1: X, p2: A) -> X: ... +@overload +def f(*, p2: B, p1: X) -> Y: ... # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +def f(*, p1, p2): ... + +@overload +def g(*, p1: X, p2: B) -> X: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def g(*, p2: A, p1: X) -> Y: ... +def g(*, p1, p2): ... + +[case testOverloadWithVariableArgsAreOverlapping-skip] +# TODO: Re-enable this after adding support for partially overlapping arg counts +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(*x: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def foo1(x: int, y: int, z: int) -> str: ... + +@overload +def foo2(*x: int) -> int: ... +@overload +def foo2(x: int, y: str, z: int) -> str: ... + +@overload +def bar1(x: int, y: int, z: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def bar1(*x: int) -> int: ... + +@overload +def bar2(x: int, y: str, z: int) -> str: ... +@overload +def bar2(*x: int) -> int: ... + +[case testOverloadDetectsPossibleMatchesWithGenerics] +from typing import overload, TypeVar, Generic + +T = TypeVar('T') + +@overload +def foo(x: None, y: None) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def foo(x: T, y: T) -> int: ... +def foo(x): ... + +# TODO: We should allow this; T can't be bound to two distinct types +@overload +def bar(x: None, y: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def bar(x: T, y: T) -> int: ... +def bar(x, y): ... + +class Wrapper(Generic[T]): + # TODO: This should be an error + @overload + def foo(self, x: None, y: None) -> str: ... + @overload + def foo(self, x: T, y: None) -> str: ... + def foo(self, x): ... + + @overload + def bar(self, x: None, y: int) -> str: ... + @overload + def bar(self, x: T, y: T) -> str: ... + def bar(self, x, y): ... + +[case testOverloadFlagsPossibleMatches] +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(x: str) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def foo1(x: str, y: str = ...) -> int: ... + +@overload +def foo2(x: str, y: str = ...) -> int: ... +@overload +def foo2(x: str) -> str: ... # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader + +@overload +def foo3(x: str) -> str: ... +@overload +def foo3(x: str, y: str) -> int: ... + +[case testOverloadPossibleOverlapWithArgsAndKwargs-skip] +# TODO: Re-enable this after adding support for partially overlapping arg counts +from wrapper import * +[file wrapper.pyi] +from typing import overload + +@overload +def foo1(*args: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def foo1(**kwargs: int) -> str: ... + +@overload +def foo2(**kwargs: int) -> str: ... +@overload +def foo2(*args: int) -> int: ... # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +[builtins fixtures/dict.pyi] + +[case testOverloadWithPartiallyOverlappingUnions] +from typing import overload, Union + +class A: ... +class B: ... +class C: ... + +@overload +def f(x: Union[A, B]) -> int: ... # TODO: This should be an error +@overload +def f(x: Union[B, C]) -> str: ... +def f(x): ... + +[case testOverloadNotConfusedForProperty] +from typing import overload + +class PropertyClass: + @property + def foo(self) -> str: return "..." + @foo.setter + def foo(self, value: str) -> None: pass + @foo.deleter + def foo(self) -> None: pass + +class OverloadClass: + @overload + def foo(self) -> str: pass + @overload + def foo(self, value: str) -> None: pass + @overload + def foo(self) -> None: pass # E: Overloaded function signature 3 will never be matched: function 1's parameter type(s) are the same or broader + def foo(self, *args): pass + +[builtins fixtures/property.pyi] + +[case testOverloadInferUnionReturnBasic] +from typing import overload, Union + +class A: ... +class B: ... +class C: ... +class D: ... + +@overload +def f1(x: A) -> B: ... +@overload +def f1(x: C) -> D: ... +def f1(x): ... + +arg1: Union[A, C] +reveal_type(f1(arg1)) # E: Revealed type is 'Union[__main__.B, __main__.D]' + +arg2: Union[A, B] +f1(arg2) # E: Argument 1 to "f1" has incompatible type "Union[A, B]"; expected "A" + +@overload +def f2(x: A) -> B: ... +@overload +def f2(x: C) -> B: ... +def f2(x): ... + +reveal_type(f2(arg1)) # E: Revealed type is '__main__.B' + +[case testOverloadInferUnionReturnMultipleArguments] +from typing import overload, Union + +class A: ... +class B: ... +class C: ... +class D: ... + +@overload +def f1(x: A, y: C) -> B: ... +@overload +def f1(x: C, y: A) -> D: ... +def f1(x, y): ... + +arg1: Union[A, C] +reveal_type(f1(arg1, arg1)) + +@overload +def f2(x: A, y: C) -> B: ... +@overload +def f2(x: C, y: C) -> D: ... +def f2(x, y): ... + +reveal_type(f2(arg1, arg1)) +reveal_type(f2(arg1, C())) + +[out] +main:15: error: Revealed type is '__main__.B' +main:15: error: Argument 1 to "f1" has incompatible type "Union[A, C]"; expected "A" +main:15: error: Argument 2 to "f1" has incompatible type "Union[A, C]"; expected "C" +main:23: error: Revealed type is 'Union[__main__.B, __main__.D]' +main:23: error: Argument 2 to "f2" has incompatible type "Union[A, C]"; expected "C" +main:24: error: Revealed type is 'Union[__main__.B, __main__.D]' + +[case testOverloadInferUnionSkipIfParameterNamesAreDifferent] +from typing import overload, Union + +class A: ... +class B: ... +class C: ... + +@overload +def f(x: A) -> B: ... +@overload +def f(y: B) -> C: ... +def f(x): ... + +x: Union[A, B] +reveal_type(f(A())) # E: Revealed type is '__main__.B' +reveal_type(f(B())) # E: Revealed type is '__main__.C' +f(x) # E: Argument 1 to "f" has incompatible type "Union[A, B]"; expected "A" + +[case testOverloadInferUnionReturnFunctionsWithKwargs] +from typing import overload, Union, Optional + +class A: ... +class B: ... +class C: ... +class D(B, C): ... + +@overload +def f(x: A) -> D: ... +@overload +def f(x: A, y: Optional[B] = None) -> C: ... +@overload +def f(x: A, z: Optional[C] = None) -> B: ... +def f(x, y=None, z=None): ... + +reveal_type(f(A(), B())) +reveal_type(f(A(), C())) + +arg: Union[B, C] +reveal_type(f(A(), arg)) +reveal_type(f(A())) + +[builtins fixtures/tuple.pyi] +[out] +main:16: error: Revealed type is '__main__.C' +main:17: error: Revealed type is '__main__.B' +main:20: error: Revealed type is '__main__.C' +main:20: error: Argument 2 to "f" has incompatible type "Union[B, C]"; expected "Optional[B]" +main:21: error: Revealed type is '__main__.D' + +[case testOverloadingInferUnionReturnWithTypevarWithValueRestriction] +from typing import overload, Union, TypeVar, Generic + +class A: pass +class B: pass +class C: pass + +T = TypeVar('T', B, C) + +class Wrapper(Generic[T]): + @overload + def f(self, x: T) -> B: ... + + @overload + def f(self, x: A) -> C: ... + + def f(self, x): ... + +obj: Wrapper[B] = Wrapper() +x: Union[A, B] + +reveal_type(obj.f(A())) # E: Revealed type is '__main__.C' +reveal_type(obj.f(B())) # E: Revealed type is '__main__.B' +reveal_type(obj.f(x)) # E: Revealed type is 'Union[__main__.B, __main__.C]' + +[case testOverloadingInferUnionReturnWithTypevarReturn] +from typing import overload, Union, TypeVar, Generic + +T = TypeVar('T') + +class Wrapper1(Generic[T]): pass +class Wrapper2(Generic[T]): pass +class A: pass +class B: pass + +@overload +def f(x: Wrapper1[T]) -> T: ... +@overload +def f(x: Wrapper2[T]) -> T: ... +def f(x): ... + +obj1: Union[Wrapper1[A], Wrapper2[A]] +reveal_type(f(obj1)) # E: Revealed type is '__main__.A' + +obj2: Union[Wrapper1[A], Wrapper2[B]] +reveal_type(f(obj2)) # E: Revealed type is 'Union[__main__.A, __main__.B]' + diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index d731a406da44..14e1129cc78a 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -1301,8 +1301,8 @@ def f(x): reveal_type(f(C1())) # E: Revealed type is 'builtins.int' reveal_type(f(C2())) # E: Revealed type is 'builtins.str' class D(C1, C2): pass # Compatible with both P1 and P2 -# FIXME: the below is not right, see #1322 -reveal_type(f(D())) # E: Revealed type is 'Any' +# TODO: Should this return a union instead? +reveal_type(f(D())) # E: Revealed type is 'builtins.int' f(C()) # E: No overload variant of "f" matches argument type "C" [builtins fixtures/isinstance.pyi] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index ae769e9a933d..2619e9c8f8ab 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1261,15 +1261,15 @@ A = TypedDict('A', {'x': int}) B = TypedDict('B', {'y': str}) @overload -def f(x: A) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: A) -> int: ... @overload def f(x: B) -> str: ... def f(x): pass a: A b: B -reveal_type(f(a)) # E: Revealed type is 'Any' -reveal_type(f(b)) # E: Revealed type is 'Any' +reveal_type(f(a)) # E: Revealed type is 'builtins.int' +reveal_type(f(b)) # E: Revealed type is 'builtins.str' [builtins fixtures/dict.pyi] [typing fixtures/typing-full.pyi] diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 954c1fd62ec3..58196ee63c8d 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -2055,6 +2055,7 @@ def g() -> None: pass == main:2: error: Cannot find module named 'm' main:2: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) +main:7: error: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader main:9: error: Cannot find module named 'n' [case testOverloadSpecialCase] @@ -2083,6 +2084,7 @@ def g() -> None: pass == main:2: error: Cannot find module named 'm' main:2: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) +main:12: error: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader main:14: error: Cannot find module named 'n' [case testRefreshGenericClass] From 96486b66a053d4896542e8ec9c1be8a04f70310f Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Sun, 20 May 2018 12:15:24 -0700 Subject: [PATCH 02/16] Make overload definitions always strict optional --- mypy/checker.py | 89 ++++++++++++------------ mypy/checkexpr.py | 4 -- test-data/unit/check-overloading.test | 99 +++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 47 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 28976d6ff775..97b3cd97d32a 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -414,53 +414,56 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: # At this point we should have set the impl already, and all remaining # items are decorators - for i, item in enumerate(defn.items): - # TODO overloads involving decorators - assert isinstance(item, Decorator) - sig1 = self.function_type(item.func) + with experiments.strict_optional_set(True): + for i, item in enumerate(defn.items): + # TODO overloads involving decorators + assert isinstance(item, Decorator) + sig1 = self.function_type(item.func) - for j, item2 in enumerate(defn.items[i + 1:]): - assert isinstance(item2, Decorator) - sig2 = self.function_type(item2.func) + for j, item2 in enumerate(defn.items[i + 1:]): + assert isinstance(item2, Decorator) + sig2 = self.function_type(item2.func) - assert isinstance(sig1, CallableType) - assert isinstance(sig2, CallableType) + assert isinstance(sig1, CallableType) + assert isinstance(sig2, CallableType) - if not are_argument_counts_overlapping(sig1, sig2): - continue - - if if_overload_can_never_match(sig1, sig2): - self.msg.overloaded_signature_will_never_match(i + 1, i + j + 2, item2.func) - elif is_unsafe_overlapping_overload_signatures(sig1, sig2): - self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, item.func) - if defn.impl: - if isinstance(defn.impl, FuncDef): - impl_type = defn.impl.type - elif isinstance(defn.impl, Decorator): - impl_type = defn.impl.var.type - else: - assert False, "Impl isn't the right type" - # This can happen if we've got an overload with a different - # decorator too -- we gave up on the types. - if impl_type is None or isinstance(impl_type, AnyType) or sig1 is None: - return + if not are_argument_counts_overlapping(sig1, sig2): + continue - assert isinstance(impl_type, CallableType) - assert isinstance(sig1, CallableType) - if not is_callable_compatible(impl_type, sig1, - is_compat=is_subtype, ignore_return=True): - self.msg.overloaded_signatures_arg_specific(i + 1, defn.impl) - impl_type_subst = impl_type - if impl_type.variables: - unified = unify_generic_callable(impl_type, sig1, ignore_return=False) - if unified is None: - self.fail("Type variable mismatch between " + - "overload signature {} and implementation".format(i + 1), - defn.impl) + if overload_can_never_match(sig1, sig2): + self.msg.overloaded_signature_will_never_match( + i + 1, i + j + 2, item2.func) + elif is_unsafe_overlapping_overload_signatures(sig1, sig2): + self.msg.overloaded_signatures_overlap( + i + 1, i + j + 2, item.func) + if defn.impl: + if isinstance(defn.impl, FuncDef): + impl_type = defn.impl.type + elif isinstance(defn.impl, Decorator): + impl_type = defn.impl.var.type + else: + assert False, "Impl isn't the right type" + # This can happen if we've got an overload with a different + # decorator too -- we gave up on the types. + if impl_type is None or isinstance(impl_type, AnyType) or sig1 is None: return - impl_type_subst = unified - if not is_subtype(sig1.ret_type, impl_type_subst.ret_type): - self.msg.overloaded_signatures_ret_specific(i + 1, defn.impl) + + assert isinstance(impl_type, CallableType) + assert isinstance(sig1, CallableType) + if not is_callable_compatible(impl_type, sig1, + is_compat=is_subtype, ignore_return=True): + self.msg.overloaded_signatures_arg_specific(i + 1, defn.impl) + impl_type_subst = impl_type + if impl_type.variables: + unified = unify_generic_callable(impl_type, sig1, ignore_return=False) + if unified is None: + self.fail("Type variable mismatch between " + + "overload signature {} and implementation".format(i + 1), + defn.impl) + return + impl_type_subst = unified + if not is_subtype(sig1.ret_type, impl_type_subst.ret_type): + self.msg.overloaded_signatures_ret_specific(i + 1, defn.impl) # Here's the scoop about generators and coroutines. # @@ -3618,7 +3621,7 @@ def is_unsafe_overlapping_overload_signatures(signature: CallableType, is_compat_return=lambda l, r: not is_subtype(r, l))) -def if_overload_can_never_match(signature: CallableType, other: CallableType) -> bool: +def overload_can_never_match(signature: CallableType, other: CallableType) -> bool: """Check if the 'other' method can never be matched due to 'signature'. This can happen if signature's parameters are all strictly broader then diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index c6577d7ac048..18e8497d94ef 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1152,9 +1152,6 @@ def check_overload_call(self, # TODO: Adjust the error message here to make it clear there was no match. target = erased_targets[0] - '''target = self.overload_call_target(args, arg_types, arg_kinds, arg_names, - callee, context, - messages=arg_messages)''' return self.check_call(target, args, arg_kinds, context, arg_names, arg_messages=arg_messages, callable_name=callable_name, @@ -1209,7 +1206,6 @@ def infer_overload_return_type(self, try: # Passing `overload_messages` as the `arg_messages` parameter doesn't # seem to reliably catch all possible errors. - # # TODO: Figure out why result = self.check_call( callee=typ, diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 5b8f85f9dba7..3946dd2e19c2 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -1848,3 +1848,102 @@ reveal_type(f(obj1)) # E: Revealed type is '__main__.A' obj2: Union[Wrapper1[A], Wrapper2[B]] reveal_type(f(obj2)) # E: Revealed type is 'Union[__main__.A, __main__.B]' +[case testOverloadsAndNoneWithoutStrictOptional] +# flags: --no-strict-optional +from typing import overload, Optional + +@overload +def f(x: None) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: object) -> str: ... +def f(x): ... + +# We pretend strict-optional is enabled for overload definitions, +# even in non-strict optional mode +@overload +def g(x: None) -> int: ... +@overload +def g(x: int) -> str: ... +def g(x): ... + +# Calls are still checked normally though +a: None +b: int +c: Optional[int] +reveal_type(g(a)) # E: Revealed type is 'builtins.int' +reveal_type(g(b)) # E: Revealed type is 'builtins.str' +reveal_type(g(c)) # E: Revealed type is 'builtins.str' + +[case testOverloadsAndNoneWithStrictOptional] +# flags: --strict-optional +from typing import overload, Optional + +@overload +def f(x: None) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: object) -> str: ... +def f(x): ... + +@overload +def g(x: None) -> int: ... +@overload +def g(x: int) -> str: ... +def g(x): ... + +a: None +b: int +c: Optional[int] +reveal_type(g(a)) # E: Revealed type is 'builtins.int' +reveal_type(g(b)) # E: Revealed type is 'builtins.str' +reveal_type(g(c)) # E: Revealed type is 'Union[builtins.int, builtins.str]' + +[case testOverloadsNoneAndTypeVarsWithNoStrictOptional] +# flags: --no-strict-optional +from typing import Callable, Iterable, TypeVar, overload, Optional + +T = TypeVar('T') +S = TypeVar('S') + +@overload +def mymap(func: None, seq: Iterable[T]) -> Iterable[T]: ... +@overload +def mymap(func: Callable[[T], S], seq: Iterable[T]) -> Iterable[S]: ... +def mymap(*args): ... + +seq = [1, 2, 3] +f1: Callable[[int], str] +f2: None +f3: Optional[Callable[[int], str]] + +reveal_type(mymap(f1, seq)) # E: Revealed type is 'typing.Iterable[builtins.str*]' +reveal_type(mymap(f2, seq)) # E: Revealed type is 'typing.Iterable[builtins.int*]' +reveal_type(mymap(f3, seq)) # E: Revealed type is 'typing.Iterable[builtins.str*]' + +[builtins fixtures/list.pyi] +[typing fixtures/typing-full.pyi] + +[case testOverloadsNoneAndTypeVarsWithStrictOptional] +# flags: --strict-optional +from typing import Callable, Iterable, TypeVar, overload, Optional + +T = TypeVar('T') +S = TypeVar('S') + +@overload +def mymap(func: None, seq: Iterable[T]) -> Iterable[T]: ... +@overload +def mymap(func: Callable[[T], S], seq: Iterable[T]) -> Iterable[S]: ... +def mymap(*args): ... + +seq = [1, 2, 3] +f1: Callable[[int], str] +f2: None +f3: Optional[Callable[[int], str]] + +reveal_type(mymap(f1, seq)) # E: Revealed type is 'typing.Iterable[builtins.str*]' +reveal_type(mymap(f2, seq)) # E: Revealed type is 'typing.Iterable[builtins.int*]' +reveal_type(mymap(f3, seq)) # E: Revealed type is 'Union[typing.Iterable[builtins.int], typing.Iterable[builtins.str]]' + +[builtins fixtures/list.pyi] +[typing fixtures/typing-full.pyi] + From f37efdd55a63daa88a0d44069ec03cde9506abc5 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Sun, 20 May 2018 15:07:00 -0700 Subject: [PATCH 03/16] Rearrange Python 2 test Apparently, in Python 2, 'bytecode' promotes to 'unicode'. --- test-data/unit/python2eval.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-data/unit/python2eval.test b/test-data/unit/python2eval.test index 665ccf04e893..6007cad7780c 100644 --- a/test-data/unit/python2eval.test +++ b/test-data/unit/python2eval.test @@ -313,10 +313,10 @@ f(b'') [file m.pyi] from typing import overload @overload -def f(x): # type: (unicode) -> int +def f(x): # type: (bytearray) -> int pass @overload -def f(x): # type: (bytearray) -> int +def f(x): # type: (unicode) -> int pass [out] _program.py:2: error: No overload variant of "f" matches argument type "int" From c906385b6228d8777a996ca90fa819bb9809159c Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 24 May 2018 16:26:38 -0700 Subject: [PATCH 04/16] Adjust overlap error message to use "signature" instead of "function" This commit is a superficial one: it modifies the error message for overlapping alternatives to use the word "signature" instead of "function", as suggested. The majority of the changes are updates in tests. --- mypy/messages.py | 2 +- test-data/unit/check-classes.test | 6 +++--- test-data/unit/check-overloading.test | 28 +++++++++++++-------------- test-data/unit/fine-grained.test | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index 2654d29447c1..c744b135b572 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -950,7 +950,7 @@ def overloaded_signature_will_never_match(self, index1: int, index2: int, context: Context) -> None: self.fail( 'Overloaded function signature {index2} will never be matched: ' - 'function {index1}\'s parameter type(s) are the same or broader'.format( + 'signature {index1}\'s parameter type(s) are the same or broader'.format( index1=index1, index2=index2), context) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 8e33e50d234e..b32f93668502 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -1656,7 +1656,7 @@ class C(A): def __add__(self, x: 'A') -> 'A': pass [out] tmp/foo.pyi:8: error: Signature of "__add__" incompatible with supertype "A" -tmp/foo.pyi:11: error: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +tmp/foo.pyi:11: error: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader [case testReverseOperatorMethodArgumentType] from typing import Any @@ -1725,7 +1725,7 @@ class A: @overload def __radd__(self, x: 'A') -> str: pass @overload - def __radd__(self, x: 'A') -> Any: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader + def __radd__(self, x: 'A') -> Any: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader [out] [case testReverseOperatorMethodArgumentTypeAndOverloadedMethod] @@ -2923,7 +2923,7 @@ class Sub(Super): @overload # E: Signature of "foo" incompatible with supertype "Super" def foo(self, a: A) -> A: pass @overload - def foo(self, a: B) -> C: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader + def foo(self, a: B) -> C: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader @overload def foo(self, a: C) -> C: pass [builtins fixtures/classmethod.pyi] diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 3946dd2e19c2..afe9e8bbdcc4 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -655,7 +655,7 @@ class B(A): pass @overload def f(x: A) -> A: pass @overload -def f(x: B) -> B: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +def f(x: B) -> B: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader [case testPartiallyCovariantOverlappingOverloadSignatures] from foo import * @@ -677,7 +677,7 @@ class B(A): pass @overload def g(x: A) -> int: pass @overload -def g(x: B) -> str: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +def g(x: B) -> str: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader [case testCovariantOverlappingOverloadSignatures] from foo import * @@ -717,7 +717,7 @@ from typing import Any, overload @overload def g(x: Any) -> Any: pass @overload -def g(x: int) -> int: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +def g(x: int) -> int: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader [case testOverloadedLtAndGtMethods] from foo import * @@ -983,7 +983,7 @@ def g(x: U, y: V) -> None: [builtins fixtures/list.pyi] [out] -[case testOverlapWithTypeVars] +[case testOverloadOverlapWithTypeVars] from foo import * [file foo.pyi] from typing import overload, TypeVar, Sequence, List @@ -996,12 +996,12 @@ def f(x: Sequence[int]) -> int: pass @overload def g(x: Sequence[T]) -> None: pass @overload -def g(x: Sequence[str]) -> int: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +def g(x: Sequence[str]) -> int: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader @overload def h(x: Sequence[str]) -> int: pass @overload -def h(x: Sequence[T]) -> None: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +def h(x: Sequence[T]) -> None: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader @overload def i(x: List[str]) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types @@ -1039,7 +1039,7 @@ g(1, 'foo') g(1, 'foo', b'bar') # E: Value of type variable "AnyStr" of "g" cannot be "object" [builtins fixtures/primitives.pyi] -[case testOverlapWithTypeVarsWithValuesOrdering] +[case testOverloadOverlapWithTypeVarsWithValuesOrdering] from foo import * [file foo.pyi] from typing import overload, TypeVar @@ -1048,7 +1048,7 @@ AnyStr = TypeVar('AnyStr', bytes, str) @overload def f(x: AnyStr) -> AnyStr: pass @overload -def f(x: str) -> str: pass # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +def f(x: str) -> str: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader @overload def g(x: str) -> str: pass @@ -1380,11 +1380,11 @@ def r1(x: Any) -> Any: ... @overload def r2(x: Tuple[A, ...]) -> A: ... @overload -def r2(x: Tuple[A, A]) -> B: ... # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +def r2(x: Tuple[A, A]) -> B: ... # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader @overload -def r2(x: Tuple[A]) -> B: ... # E: Overloaded function signature 3 will never be matched: function 1's parameter type(s) are the same or broader +def r2(x: Tuple[A]) -> B: ... # E: Overloaded function signature 3 will never be matched: signature 1's parameter type(s) are the same or broader @overload -def r2(x: Tuple[()]) -> B: ... # E: Overloaded function signature 4 will never be matched: function 1's parameter type(s) are the same or broader +def r2(x: Tuple[()]) -> B: ... # E: Overloaded function signature 4 will never be matched: signature 1's parameter type(s) are the same or broader def r2(x: Any) -> Any: ... [builtins fixtures/tuple.pyi] @@ -1551,7 +1551,7 @@ class Y: ... @overload def f(*, p1: X, p2: A) -> X: ... @overload -def f(*, p2: B, p1: X) -> Y: ... # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +def f(*, p2: B, p1: X) -> Y: ... # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader def f(*, p1, p2): ... @overload @@ -1631,7 +1631,7 @@ def foo1(x: str, y: str = ...) -> int: ... @overload def foo2(x: str, y: str = ...) -> int: ... @overload -def foo2(x: str) -> str: ... # E: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +def foo2(x: str) -> str: ... # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader @overload def foo3(x: str) -> str: ... @@ -1685,7 +1685,7 @@ class OverloadClass: @overload def foo(self, value: str) -> None: pass @overload - def foo(self) -> None: pass # E: Overloaded function signature 3 will never be matched: function 1's parameter type(s) are the same or broader + def foo(self) -> None: pass # E: Overloaded function signature 3 will never be matched: signature 1's parameter type(s) are the same or broader def foo(self, *args): pass [builtins fixtures/property.pyi] diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 58196ee63c8d..d2fe0281453a 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -2055,7 +2055,7 @@ def g() -> None: pass == main:2: error: Cannot find module named 'm' main:2: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) -main:7: error: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +main:7: error: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader main:9: error: Cannot find module named 'n' [case testOverloadSpecialCase] @@ -2084,7 +2084,7 @@ def g() -> None: pass == main:2: error: Cannot find module named 'm' main:2: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) -main:12: error: Overloaded function signature 2 will never be matched: function 1's parameter type(s) are the same or broader +main:12: error: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader main:14: error: Cannot find module named 'n' [case testRefreshGenericClass] From e5008612eec6a7919661a5484be6ad3a0ce24876 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 24 May 2018 16:31:58 -0700 Subject: [PATCH 05/16] Add more comments and clean up some existing ones This commit adds extra comments/cleans up some existing ones as prompted by the first code review. --- mypy/checker.py | 21 ++++++++++++++++----- mypy/checkexpr.py | 2 ++ mypy/subtypes.py | 21 +++++++++++++++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 97b3cd97d32a..9a5f5fabac55 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -414,6 +414,20 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: # At this point we should have set the impl already, and all remaining # items are decorators + # + # Note: we force mypy to check overload signatures in strict-optional mode + # so we don't incorrectly report errors when a user tries typing an overload + # that happens to have a 'if the argument is None' fallback. + # + # For example, the following is fine in strict-optional mode but would throw + # the unsafe overlap error when strict-optional is disabled: + # + # @overload + # def foo(x: None) -> int: ... + # @overload + # def foo(x: str) -> str: ... + # + # See Python 2's map function for a concrete example of this kind of overload. with experiments.strict_optional_set(True): for i, item in enumerate(defn.items): # TODO overloads involving decorators @@ -3581,14 +3595,11 @@ def is_unsafe_overlapping_overload_signatures(signature: CallableType, other: CallableType) -> bool: """Check if two overloaded function signatures may be unsafely overlapping. - We consider two functions 's' and 't' to be unsafely overlapping both + We consider two functions 's' and 't' to be unsafely overlapping both if of the following are true: 1. s's parameters are all more precise or partially overlapping with t's - 1. s's return type is NOT a subtype of t's. - - both can be valid for the same - statically typed values and the return types are incompatible. + 2. s's return type is NOT a subtype of t's. Assumes that 'signature' appears earlier in the list of overload alternatives then 'other' and that their argument counts are overlapping. diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 18e8497d94ef..9b741a1b0800 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1188,6 +1188,8 @@ def infer_overload_return_type(self, ) -> Optional[Tuple[Type, Type]]: """Attempts to find the first matching callable from the given list. + If a match is found, returns a tuple containing the result type and the inferred + callee type. (This tuple is meant to be eventually returned by check_call.) If multiple targets match due to ambiguous Any parameters, returns (AnyType, AnyType). If no targets match, returns None. diff --git a/mypy/subtypes.py b/mypy/subtypes.py index eb8b7fe92c08..c45a0fd94071 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -595,6 +595,27 @@ def is_callable_compatible(left: CallableType, right: CallableType, is why the default is to check the args contravariantly. However, it's occasionally useful to check the args using some other check, so we leave the variance configurable. + + For example, when checking the validity of overloads, it's useful to see if + the first overload alternative has more precise arguments then the second. + We would want to check the arguments covariantly in that case. + + Note! The following two function calls are NOT equivalent: + + is_callable_compatible(f, g, is_compat=is_subtype, check_args_covariantly=False) + is_callable_compatible(g, f, is_compat=is_subtype, check_args_covariantly=True) + + The two calls are similar in that they both check the function arguments in + the same direction: they both run `is_subtype(argument_from_g, argument_from_f)`. + + However, the two calls differ in which direction they check things likee + keyword arguments. For example, suppose f and g are defined like so: + + def f(x: int, *y: int) -> int: ... + def g(x: int) -> int: ... + + In this case, the first call will succeed and the second will fail: f is a + valid stand-in for g but not vice-versa. """ if is_compat_return is None: is_compat_return = is_compat From 708e98bb71e65710ee05bea27afa02732afc18b0 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 24 May 2018 16:33:53 -0700 Subject: [PATCH 06/16] Remove vestigial 'from_overloads' field from UnionType I originally added the 'from_overloads' field because I thought it would help us produce better error messages when both union math and the "select the first match" algorithms fail. However, I found a slightly better approach for handling errors + decided to defer more adding more sophicated error messages for a future PR and forgot to remove this field. --- mypy/checkexpr.py | 4 ++-- mypy/types.py | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 9b741a1b0800..6dcee094607e 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1342,8 +1342,8 @@ def union_overload_matches(self, return callables[0].copy_modified( arg_types=final_args, ret_type=UnionType.make_simplified_union(new_returns), - implicit=True, - from_overloads=True) + variables=variables, + implicit=True) def erased_signature_similarity(self, arg_types: List[Type], arg_kinds: List[int], arg_names: Optional[Sequence[Optional[str]]], diff --git a/mypy/types.py b/mypy/types.py index 89965eb99583..a845ab10828f 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -675,8 +675,6 @@ class CallableType(FunctionLike): # 'dict') 'from_type_type', # Was this callable generated by analyzing Type[...] # instantiation? - 'from_overloads', # Was this callable generated by synthesizing - # multiple overloads? 'bound_args', # Bound type args, mostly unused but may be useful for # tools that consume mypy ASTs ) @@ -697,7 +695,6 @@ def __init__(self, is_classmethod_class: bool = False, special_sig: Optional[str] = None, from_type_type: bool = False, - from_overloads: bool = False, bound_args: Sequence[Optional[Type]] = (), ) -> None: super().__init__(line, column) @@ -722,7 +719,6 @@ def __init__(self, self.is_classmethod_class = is_classmethod_class self.special_sig = special_sig self.from_type_type = from_type_type - self.from_overloads = from_overloads if not bound_args: bound_args = () self.bound_args = bound_args @@ -761,7 +757,6 @@ def copy_modified(self, is_classmethod_class=self.is_classmethod_class, special_sig=special_sig if special_sig is not _dummy else self.special_sig, from_type_type=from_type_type if from_type_type is not _dummy else self.from_type_type, - from_overloads=from_overloads if from_overloads is not _dummy else self.from_overloads, bound_args=bound_args if bound_args is not _dummy else self.bound_args, ) @@ -922,7 +917,6 @@ def serialize(self) -> JsonDict: 'is_ellipsis_args': self.is_ellipsis_args, 'implicit': self.implicit, 'is_classmethod_class': self.is_classmethod_class, - 'from_overloads': self.from_overloads, 'bound_args': [(None if t is None else t.serialize()) for t in self.bound_args], } @@ -941,7 +935,6 @@ def deserialize(cls, data: JsonDict) -> 'CallableType': is_ellipsis_args=data['is_ellipsis_args'], implicit=data['implicit'], is_classmethod_class=data['is_classmethod_class'], - from_overloads=data['from_overloads'], bound_args=[(None if t is None else deserialize_type(t)) for t in data['bound_args']], ) From 1f312523fc6998e8c4c0a8fded61008542f8ae78 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 24 May 2018 16:39:00 -0700 Subject: [PATCH 07/16] Add rudimentary support for partial overlap checks The previous plan was that I would defer checking for partial overlaps for a future PR. However, that led to some regressions -- for example, the tuple ellipsis variance test case. So, I decided to add in very rudimentary partial overlap checks as a stop-gap measure to help catch more obviously unsafe signatures. The checks only attempt to check for overlaps at the top-level, and handle only unions, tuples, and unrestricted TypeVars. --- mypy/checker.py | 12 ++++++--- mypy/meet.py | 35 +++++++++++++++++++++++++++ test-data/unit/check-overloading.test | 28 +++++++++++++++++++-- 3 files changed, 70 insertions(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 9a5f5fabac55..963713cde638 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -51,7 +51,7 @@ from mypy.join import join_types from mypy.treetransform import TransformVisitor from mypy.binder import ConditionalTypeBinder, get_declaration -from mypy.meet import is_overlapping_types +from mypy.meet import is_overlapping_types, is_partially_overlapping_types from mypy.options import Options from mypy.plugin import Plugin, CheckerPluginInterface from mypy.sharedparse import BINARY_MAGIC_METHODS @@ -3613,22 +3613,28 @@ def is_unsafe_overlapping_overload_signatures(signature: CallableType, # function and use "is_more_precise(...) or is_partially_overlapping(...)" for the is_compat # checks. # + # (We already have a rudimentary implementation of 'is_partially_overlapping', but it only + # attempts to handle the obvious cases -- see its docstring for more info.) + # # Similarly, the signatures "f(x: A, y: A) -> str" and "f(*x: A) -> int" are also unsafe: # the parameter *counts* or arity are partially overlapping. # # To fix this, we need to modify is_callable_compatible so it can optionally detect # functions that are *potentially* compatible rather then *definitely* compatible. + def is_more_precise_or_partially_overlapping(t: Type, s: Type) -> bool: + return is_more_precise(t, s) or is_partially_overlapping_types(t, s) + # The reason we repeat this check twice is so we can do a slightly better job of # checking for potentially overlapping param counts. Both calls will actually check # the param and return types in the same "direction" -- the only thing that differs # is how is_callable_compatible checks non-positional arguments. return (is_callable_compatible(signature, other, - is_compat=is_more_precise, + is_compat=is_more_precise_or_partially_overlapping, is_compat_return=lambda l, r: not is_subtype(l, r), check_args_covariantly=True) or is_callable_compatible(other, signature, - is_compat=is_more_precise, + is_compat=is_more_precise_or_partially_overlapping, is_compat_return=lambda l, r: not is_subtype(r, l))) diff --git a/mypy/meet.py b/mypy/meet.py index 3fbd9c28b303..0d44d8082065 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -2,6 +2,7 @@ from typing import List, Optional, cast, Tuple from mypy.join import is_similar_callables, combine_similar_callables, join_type_list +from mypy.sametypes import is_same_type from mypy.types import ( Type, AnyType, TypeVisitor, UnboundType, NoneTyp, TypeVarType, Instance, CallableType, TupleType, TypedDictType, ErasedType, TypeList, UnionType, PartialType, DeletedType, @@ -49,6 +50,40 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type: return narrowed +def is_partially_overlapping_types(t: Type, s: Type) -> bool: + """Returns 'true' if the two types are partially, but not completely, overlapping. + + NOTE: This function is only a partial implementation. + + It exists mostly so that overloads correctly handle partial + overlaps for the more obvious cases. + """ + # Are unions partially overlapping? + if isinstance(t, UnionType) and isinstance(s, UnionType): + t_set = set(t.items) + s_set = set(s.items) + num_same = len(t_set.intersection(s_set)) + num_diff = len(t_set.symmetric_difference(s_set)) + return num_same > 0 and num_diff > 0 + + # Are tuples partially overlapping? + tup_overlap = is_overlapping_tuples(t, s, use_promotions=True) + if tup_overlap is not None and tup_overlap: + return tup_overlap + + def is_object(t: Type) -> bool: + return isinstance(t, Instance) and t.type.fullname() == 'builtins.object' + + # Is either 't' or 's' an unrestricted TypeVar? + if isinstance(t, TypeVarType) and is_object(t.upper_bound) and len(t.values) == 0: + return True + + if isinstance(s, TypeVarType) and is_object(s.upper_bound) and len(s.values) == 0: + return True + + return False + + def is_overlapping_types(t: Type, s: Type, use_promotions: bool = False) -> bool: """Can a value of type t be a value of type s, or vice versa? diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index afe9e8bbdcc4..2d3830a7424c 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -1409,7 +1409,7 @@ def r(x: Any) -> Any:... @overload def g(x: A) -> A: ... @overload -def g(x: Tuple[A1, ...]) -> B: ... # TODO: This should be an error +def g(x: Tuple[A1, ...]) -> B: ... # E: Overloaded function signatures 2 and 3 overlap with incompatible return types @overload def g(x: Tuple[A, A]) -> C: ... @overload @@ -1661,13 +1661,37 @@ from typing import overload, Union class A: ... class B: ... class C: ... +class D: ... @overload -def f(x: Union[A, B]) -> int: ... # TODO: This should be an error +def f(x: Union[A, B]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types @overload def f(x: Union[B, C]) -> str: ... def f(x): ... +@overload +def g(x: Union[A, B]) -> int: ... +@overload +def g(x: Union[C, D]) -> str: ... +def g(x): ... + +@overload +def h(x: Union[A, B]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def h(x: Union[A, B, C]) -> str: ... +def h(x): ... + +[case testOverloadPartialOverlapWithUnrestrictedTypeVar] +from typing import TypeVar, overload + +T = TypeVar('T') + +@overload +def f(x: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f(x: T) -> T: ... +def f(x): ... + [case testOverloadNotConfusedForProperty] from typing import overload From ff1d56f4668ee262c9e1f229fb3bc21bfae04855 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 24 May 2018 16:49:24 -0700 Subject: [PATCH 08/16] Redo how 'union_overload_matches' handles generics The previous implementation of union_overload_matches blindly copied code for handling generics from somewhere (I forget where). In a nutshell, what the code did was pre-emptively infer what the types of each callable ought to be based on the argument types before unioning. I think this was the wrong approach, after doing some more testing. This new approach does not attempt to do any inference and instead restricts itself to "combining" TypeVars by name before unioning. My understanding is that in Mypy, we normally ignore the name of the TypeVar and instead rely exclusively on inference to figure out how each TypeVar ought to be handled. However, here, I think it makes sense to go by name: if a user used the TypeVar 'T' in two different overload alternatives, they probably meant for that TypeVar to mean the same thing. (And if they didn't, well, I don't feel too bad about that: their code is probably confusing and ought to be re-written.) --- mypy/checkexpr.py | 82 ++++++++++++++++++++------- test-data/unit/check-overloading.test | 72 ++++++++++++++++++++--- 2 files changed, 123 insertions(+), 31 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 6dcee094607e..31b1f82c8667 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -42,7 +42,7 @@ from mypy.checkmember import analyze_member_access, type_object_type, bind_self from mypy.constraints import get_actual_type from mypy.checkstrformat import StringFormatterChecker -from mypy.expandtype import expand_type_by_instance, freshen_function_type_vars +from mypy.expandtype import expand_type, expand_type_by_instance, freshen_function_type_vars from mypy.util import split_module_names from mypy.typevars import fill_typevars from mypy.visitor import ExpressionVisitor @@ -1143,8 +1143,7 @@ def check_overload_call(self, target = AnyType(TypeOfAny.from_error) # type: Type elif any(isinstance(arg, UnionType) for arg in arg_types): # Step 4b: Try performing union math - unioned_callable = self.union_overload_matches(erased_targets, args, arg_kinds, - arg_names, context) + unioned_callable = self.union_overload_matches(erased_targets) target = unioned_callable if unioned_callable is not None else erased_targets[0] else: # Step 4c: Use the first matching erased target: it won't match, but at @@ -1265,12 +1264,7 @@ def overload_erased_call_targets(self, matches.append(typ) return matches - def union_overload_matches(self, - callables: List[CallableType], - args: List[Expression], - arg_kinds: List[int], - arg_names: Optional[Sequence[Optional[str]]], - context: Context) -> Optional[CallableType]: + def union_overload_matches(self, callables: List[CallableType]) -> Optional[CallableType]: """Accepts a list of overload signatures and attempts to combine them together into a new CallableType consisting of the union of all of the given arguments and return types. @@ -1279,10 +1273,24 @@ def union_overload_matches(self, Assumes all of the given callables have argument counts compatible with the caller. """ - assert len(callables) > 0 - if len(callables) == 1: + if len(callables) == 0: + return None + elif len(callables) == 1: return callables[0] + # Note: we are assuming here that if a user uses some TypeVar 'T' in + # two different overloads, they meant for that TypeVar to mean the + # same thing. + # + # This function will make sure that all instances of that TypeVar 'T' + # refer to the same underlying TypeVarType and TypeVarDef objects to + # simplify the union-ing logic below. + # + # (If the user did *not* mean for 'T' to be consistently bound to the + # same type in their overloads, well, their code is probably too + # confusing and ought to be re-written anyways.) + callables, variables = merge_typevars_in_callables_by_name(callables) + new_args = [[] for _ in range(len(callables[0].arg_types))] # type: List[List[Type]] new_returns = [] # type: List[Type] @@ -1295,17 +1303,6 @@ def union_overload_matches(self, # TODO: Enhance the union overload logic to handle a wider variety of signatures. return None - if target.is_generic(): - formal_to_actual = map_actuals_to_formals( - arg_kinds, arg_names, - target.arg_kinds, target.arg_names, - lambda i: self.accept(args[i])) - - target = freshen_function_type_vars(target) - target = self.infer_function_type_arguments_using_context(target, context) - target = self.infer_function_type_arguments( - target, args, arg_kinds, formal_to_actual, context) - for i, arg in enumerate(target.arg_types): new_args[i].append(arg) new_returns.append(target.ret_type) @@ -3171,3 +3168,44 @@ def map_formals_to_actuals(caller_kinds: List[int], for actual in actuals: actual_to_formal[actual].append(formal) return actual_to_formal + + +def merge_typevars_in_callables_by_name( + callables: List[CallableType]) -> Tuple[List[CallableType], List[TypeVarDef]]: + """Takes all the typevars present in the callables and 'combines' the ones with the same name. + + For example, suppose we have two callables with signatures "f(x: T, y: S) -> T" and + "f(x: List[Tuple[T, S]]) -> Tuple[T, S]". Both callables use typevars named "T" and + "S", but we treat them as distinct, unrelated typevars. (E.g. they could both have + distinct ids.) + + If we pass in both callables into this function, it returns a a list containing two + new callables that are identical in signature, but use the same underlying TypeVarDef + and TypeVarType objects for T and S. + + This is useful if we want to take the output lists and "merge" them into one callable + in some way -- for example, when unioning together overloads. + + Returns both the new list of callables and a list of all distinct TypeVarDef objects used. + """ + + output = [] # type: List[CallableType] + unique_typevars = {} # type: Dict[str, TypeVarType] + variables = [] # type: List[TypeVarDef] + + for target in callables: + if target.is_generic(): + target = freshen_function_type_vars(target) + + rename = {} # Dict[TypeVarId, TypeVar] + for tvdef in target.variables: + name = tvdef.fullname + if name not in unique_typevars: + unique_typevars[name] = TypeVarType(tvdef) + variables.append(tvdef) + rename[tvdef.id] = unique_typevars[name] + + target = cast(CallableType, expand_type(target, rename)) + output.append(target) + + return output, variables diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 2d3830a7424c..dda46ccc29d8 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -1855,22 +1855,76 @@ from typing import overload, Union, TypeVar, Generic T = TypeVar('T') -class Wrapper1(Generic[T]): pass -class Wrapper2(Generic[T]): pass +class W1(Generic[T]): pass +class W2(Generic[T]): pass class A: pass class B: pass @overload -def f(x: Wrapper1[T]) -> T: ... +def foo(x: W1[T]) -> T: ... @overload -def f(x: Wrapper2[T]) -> T: ... -def f(x): ... +def foo(x: W2[T]) -> T: ... +def foo(x): ... + +def bar(x: Union[W1[T], W2[T]]) -> T: ... + +class SomeType(Generic[T]): + @overload + def foo(self, x: W1[T]) -> T: ... + @overload + def foo(self, x: W2[T]) -> T: ... + def foo(self, x): ... + + def bar(self, x: Union[W1[T], W2[T]]) -> T: ... + +def wrapper(mysterious: T) -> T: + obj1: Union[W1[A], W2[A]] + + # Phase 1: Things mypy should be able to infer + + a1: A = foo(obj1) + a2 = foo(obj1) + a3 = SomeType[A]().foo(obj1) + + reveal_type(a1) # E: Revealed type is '__main__.A' + reveal_type(a2) # E: Revealed type is '__main__.A*' + reveal_type(a3) # E: Revealed type is '__main__.A*' + + # Phase 2: Things mypy should be able to infer, but does not + # due to bugs unrelated to overloads + # + # (We confirm this by checking to see that the error + # message is identical to what we would have gotten if + # we manually union the overload) + + a4_overload: A = SomeType().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[], W2[]]" + a4_union: A = SomeType().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[], W2[]]" + + SomeType().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[], W2[]]" + SomeType().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[], W2[]]" + + + # Phase 3: Things where type inference is impossible and + # mypy should report an error + + obj2: Union[W1[A], W2[B]] + + foo(obj2) # E: Cannot infer type argument 1 of "foo" + bar(obj2) # E: Cannot infer type argument 1 of "bar" + + b2_overload: A = foo(obj2) # E: Cannot infer type argument 1 of "foo" + b2_union: A = bar(obj2) # E: Cannot infer type argument 1 of "bar" + + SomeType().foo(obj2) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[], W2[]]" + SomeType().bar(obj2) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[], W2[]]" + + SomeType[A]().foo(obj2) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[A], W2[A]]" + SomeType[A]().bar(obj2) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[A], W2[A]]" -obj1: Union[Wrapper1[A], Wrapper2[A]] -reveal_type(f(obj1)) # E: Revealed type is '__main__.A' + SomeType[T]().foo(obj2) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[T], W2[T]]" + SomeType[T]().bar(obj2) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[T], W2[T]]" -obj2: Union[Wrapper1[A], Wrapper2[B]] -reveal_type(f(obj2)) # E: Revealed type is 'Union[__main__.A, __main__.B]' + return mysterious [case testOverloadsAndNoneWithoutStrictOptional] # flags: --no-strict-optional From 718b9de07a1941ea4dd643bad845f1b4762f9db9 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 24 May 2018 16:54:40 -0700 Subject: [PATCH 09/16] Make code for handling ambiguities due to 'Any' more robust The previous version of the code only checked for the presence of 'Any' at the top level. This new version does a more robust check on the types so we can handle inputs like 'List[Any]'. It also streamlines 'infer_overload_return_type': rather then checking to see if there's ambiguity on each iteration, we can just do it once at the end. This is theoretically a performance win: 'any_causes_overload_ambiguity' loops over all of the alternatives so unnesting this logic removes accidentally-quadratic behavior. In practice, this is probably a wash: overloads are unlikely to be super-long in practice. We also call 'check_call' more frequently: that method is probably the more expensive of the two. (But either way, I think this change does make the code look tidier.) --- mypy/checkexpr.py | 73 ++++++++++++++------------- test-data/unit/check-overloading.test | 37 ++++++++++++++ 2 files changed, 76 insertions(+), 34 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 31b1f82c8667..5e9b67c0dfdd 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1196,9 +1196,10 @@ def infer_overload_return_type(self, """ arg_messages = self.msg if arg_messages is None else arg_messages - matches = [] # type: List[CallableType] - inferred = [] # type: List[Tuple[Type, Type]] - args_contain_any = any(isinstance(arg, AnyType) for arg in arg_types) + matches = [] # type: List[CallableType] + return_types = [] # type: List[Type] + inferred_types = [] # type: List[Type] + args_contain_any = any(map(has_any_type, arg_types)) for typ in plausible_targets: overload_messages = self.msg.clean_copy() @@ -1208,7 +1209,7 @@ def infer_overload_return_type(self, # Passing `overload_messages` as the `arg_messages` parameter doesn't # seem to reliably catch all possible errors. # TODO: Figure out why - result = self.check_call( + ret_type, infer_type = self.check_call( callee=typ, args=args, arg_kinds=arg_kinds, @@ -1222,31 +1223,31 @@ def infer_overload_return_type(self, is_match = not overload_messages.is_errors() if is_match: + # Return early if possible; otherwise record info so we can + # check for ambiguity due to 'Any' below. if not args_contain_any: - # There is no possibility of ambiguity due to 'Any', so we can - # just end right away: - return result - elif (args_contain_any and matches - and not is_same_type(matches[-1].ret_type, typ.ret_type) - and any_arg_causes_overload_ambiguity( - matches + [typ], arg_types, arg_kinds, arg_names)): - # Ambiguous return type. The caller has provided some - # Any argument types (which are okay to use in calls), - # so we fall back to returning 'Any'. - source = AnyType(TypeOfAny.special_form) - return self.check_call(callee=source, - args=args, - arg_kinds=arg_kinds, - arg_names=arg_names, - context=context, - arg_messages=arg_messages, - callable_name=callable_name, - object_type=object_type) - else: - matches.append(typ) - inferred.append(result) - - return inferred[0] if len(inferred) > 0 else None + return ret_type, infer_type + matches.append(typ) + return_types.append(ret_type) + inferred_types.append(infer_type) + + if len(matches) == 0: + # No match was found + return None + elif any_causes_overload_ambiguity(matches, return_types, arg_types, arg_kinds, arg_names): + # An argument of type or containing the type 'Any' caused ambiguity. + # We infer a type of 'Any' + return self.check_call(callee=AnyType(TypeOfAny.special_form), + args=args, + arg_kinds=arg_kinds, + arg_names=arg_names, + context=context, + arg_messages=arg_messages, + callable_name=callable_name, + object_type=object_type) + else: + # Success! No ambiguity; return the first match. + return return_types[0], inferred_types[0] def overload_erased_call_targets(self, plausible_targets: List[CallableType], @@ -3101,11 +3102,12 @@ def overload_arg_similarity(actual: Type, formal: Type) -> int: return 2 if is_same_type(erasetype.erase_type(actual), erasetype.erase_type(formal)) else 0 -def any_arg_causes_overload_ambiguity(items: List[CallableType], - arg_types: List[Type], - arg_kinds: List[int], - arg_names: Optional[Sequence[Optional[str]]]) -> bool: - """May an Any actual argument cause ambiguous result type on call to overloaded function? +def any_causes_overload_ambiguity(items: List[CallableType], + return_types: List[Type], + arg_types: List[Type], + arg_kinds: List[int], + arg_names: Optional[Sequence[Optional[str]]]) -> bool: + """May an argument containing 'Any' cause ambiguous result type on call to overloaded function? Note that this sometimes returns True even if there is no ambiguity, since a correct implementation would be complex (and the call would be imprecisely typed due to Any @@ -3117,6 +3119,9 @@ def any_arg_causes_overload_ambiguity(items: List[CallableType], arg_kinds: Actual argument kinds arg_names: Actual argument names """ + if all_same_types(return_types): + return False + actual_to_formal = [ map_formals_to_actuals( arg_kinds, arg_names, item.arg_kinds, item.arg_names, lambda i: arg_types[i]) @@ -3124,7 +3129,7 @@ def any_arg_causes_overload_ambiguity(items: List[CallableType], ] for arg_idx, arg_type in enumerate(arg_types): - if isinstance(arg_type, AnyType): + if has_any_type(arg_type): matching_formals_unfiltered = [(item_idx, lookup[arg_idx]) for item_idx, lookup in enumerate(actual_to_formal) if lookup[arg_idx]] diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index dda46ccc29d8..c52205c55cf9 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -1056,6 +1056,43 @@ def g(x: str) -> str: pass def g(x: AnyStr) -> AnyStr: pass [builtins fixtures/primitives.pyi] +[case testOverloadsUsingAny] +from typing import overload, List, Any, Union + +@overload +def foo(x: List[int]) -> int: ... +@overload +def foo(x: List[str]) -> str: ... +def foo(x): pass + +a: List[int] +b: List[str] +c: List[Any] +d: Union[List[int], List[str]] +e: List[bool] +f: List[object] +g: List[Union[int, str]] + +reveal_type(foo(a)) +reveal_type(foo(b)) +reveal_type(foo(c)) +reveal_type(foo(d)) +foo(e) +foo(f) +foo(g) + +[builtins fixtures/list.pyi] +[out] +main:17: error: Revealed type is 'builtins.int' +main:18: error: Revealed type is 'builtins.str' +main:19: error: Revealed type is 'Any' +main:20: error: Revealed type is 'Union[builtins.int, builtins.str]' +main:21: error: Argument 1 to "foo" has incompatible type "List[bool]"; expected "List[int]" +main:21: note: "List" is invariant -- see http://mypy.readthedocs.io/en/latest/common_issues.html#variance +main:21: note: Consider using "Sequence" instead, which is covariant +main:22: error: Argument 1 to "foo" has incompatible type "List[object]"; expected "List[int]" +main:23: error: Argument 1 to "foo" has incompatible type "List[Union[int, str]]"; expected "List[int]" + [case testOverlappingOverloadCounting] from foo import * [file foo.pyi] From 123bc40b61ccbab0dcc40f9afe42c78de6cca357 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 24 May 2018 16:56:40 -0700 Subject: [PATCH 10/16] Make 'check_overload_call' try union math first, not second This commit rearranges the logic so we try performing union math first, not second. See the discussion in https://github.com/python/mypy/issues/4063 for details/justification about this change. --- mypy/checkexpr.py | 72 ++++++++++++------ test-data/unit/check-overloading.test | 103 +++++++++++++++++++++++++- 2 files changed, 153 insertions(+), 22 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 5e9b67c0dfdd..b1e22914ff1c 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1121,7 +1121,28 @@ def check_overload_call(self, plausible_targets = self.plausible_overload_call_targets(arg_types, arg_kinds, arg_names, callee) - # Step 2: Attempt to find a matching overload + # Step 2: If the arguments contain a union, we try performing union math first. + erased_targets = None # type: Optional[List[CallableType]] + unioned_result = None # type: Optional[Tuple[Type, Type]] + unioned_errors = None # type: Optional[MessageBuilder] + if any(isinstance(arg, UnionType) for arg in arg_types): + erased_targets = self.overload_erased_call_targets(plausible_targets, arg_types, + arg_kinds, arg_names, context) + unioned_callable = self.union_overload_matches(erased_targets) + + if unioned_callable is not None: + unioned_errors = arg_messages.clean_copy() + unioned_result = self.check_call(unioned_callable, args, arg_kinds, + context, arg_names, + arg_messages=unioned_errors, + callable_name=callable_name, + object_type=object_type) + if not unioned_errors.is_errors(): + # Success! Stop early. + return unioned_result + + # Step 3: If the union math fails, or if there was no union in the argument types, + # we fall back to checking each branch one-by-one. inferred_result = self.infer_overload_return_type(plausible_targets, args, arg_types, arg_kinds, arg_names, callable_name, object_type, context, arg_messages) @@ -1129,27 +1150,36 @@ def check_overload_call(self, # Success! Stop early. return inferred_result - # Step 3: At this point, we know none of the overload alternatives exactly match. - # We fall back to using the erased types to help with union math/help us - # produce a better error message. - erased_targets = self.overload_erased_call_targets(plausible_targets, arg_types, - arg_kinds, arg_names, context) - - # Step 4: Try and infer a second-best alternative. - if len(erased_targets) == 0: - # Step 4a: There are no viable targets, even if we relax our constraints. Give up. + # Step 4: Failure. At this point, we know there is no match. We fall back to trying + # to find a somewhat plausible overload target using the erased types + # so we can produce a nice error message. + # + # For example, suppose the user passes a value of type 'List[str]' into an + # overload with signatures f(x: int) -> int and f(x: List[int]) -> List[int]. + # + # Neither alternative matches, but we can guess the user probably wants the + # second one. + if erased_targets is None: + erased_targets = self.overload_erased_call_targets(plausible_targets, arg_types, + arg_kinds, arg_names, context) + + # Step 5: We try and infer a second-best alternative if possible. If not, fall back + # to using 'Any'. + if unioned_result is not None: + # When possible, return the error messages generated from the union-math attempt: + # they tend to be a little nicer. + assert unioned_errors is not None + arg_messages.add_errors(unioned_errors) + return unioned_result + elif len(erased_targets) > 0: + # Pick the first plausible erased target as the fallback + # TODO: Adjust the error message here to make it clear there was no match. + target = erased_targets[0] # type: Type + else: + # There was no plausible match: give up if not self.chk.should_suppress_optional_error(arg_types): arg_messages.no_variant_matches_arguments(callee, arg_types, context) - target = AnyType(TypeOfAny.from_error) # type: Type - elif any(isinstance(arg, UnionType) for arg in arg_types): - # Step 4b: Try performing union math - unioned_callable = self.union_overload_matches(erased_targets) - target = unioned_callable if unioned_callable is not None else erased_targets[0] - else: - # Step 4c: Use the first matching erased target: it won't match, but at - # least we can have a nicer error message. - # TODO: Adjust the error message here to make it clear there was no match. - target = erased_targets[0] + target = AnyType(TypeOfAny.from_error) return self.check_call(target, args, arg_kinds, context, arg_names, arg_messages=arg_messages, @@ -1230,7 +1260,7 @@ def infer_overload_return_type(self, matches.append(typ) return_types.append(ret_type) inferred_types.append(infer_type) - + if len(matches) == 0: # No match was found return None diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index c52205c55cf9..f0d8e5e67f3c 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -2057,8 +2057,109 @@ f3: Optional[Callable[[int], str]] reveal_type(mymap(f1, seq)) # E: Revealed type is 'typing.Iterable[builtins.str*]' reveal_type(mymap(f2, seq)) # E: Revealed type is 'typing.Iterable[builtins.int*]' -reveal_type(mymap(f3, seq)) # E: Revealed type is 'Union[typing.Iterable[builtins.int], typing.Iterable[builtins.str]]' +reveal_type(mymap(f3, seq)) # E: Revealed type is 'Union[typing.Iterable[builtins.int], typing.Iterable[builtins.str*]]' [builtins fixtures/list.pyi] [typing fixtures/typing-full.pyi] +[case testOverloadsAndNoReturnNarrowTypeNoStrictOptional] +# flags: --no-strict-optional +from typing import overload, Union, TypeVar, NoReturn, Optional + +@overload +def narrow_int(x: str) -> NoReturn: ... +@overload +def narrow_int(x: int) -> int: ... +def narrow_int(x): + assert isinstance(x, int) + return x + +T = TypeVar('T') +@overload +def narrow_none(x: None) -> NoReturn: ... +@overload +def narrow_none(x: T) -> T: ... +def narrow_none(x): + assert x is not None + return x + +def test_narrow_int() -> None: + a: Union[int, str] + a = narrow_int(a) + reveal_type(a) # E: Revealed type is 'builtins.int' + + b: int + b = narrow_int(b) + reveal_type(b) # E: Revealed type is 'builtins.int' + + c: str + c = narrow_int(c) + reveal_type(c) # Note: branch is now dead, so no type is revealed + # TODO: maybe we should make mypy report a warning instead? + +def test_narrow_none() -> None: + a: Optional[int] + a = narrow_none(a) + reveal_type(a) # E: Revealed type is 'Union[builtins.int, None]' + + b: int + b = narrow_none(b) + reveal_type(b) # E: Revealed type is 'builtins.int' + + c: None + c = narrow_none(c) + reveal_type(c) # Note: branch is now dead, so no type is revealed + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + +[case testOverloadsAndNoReturnNarrowTypeWithStrictOptional] +# flags: --strict-optional +from typing import overload, Union, TypeVar, NoReturn, Optional + +@overload +def narrow_int(x: str) -> NoReturn: ... +@overload +def narrow_int(x: int) -> int: ... +def narrow_int(x): + assert isinstance(x, int) + return x + +T = TypeVar('T') +@overload +def narrow_none(x: None) -> NoReturn: ... +@overload +def narrow_none(x: T) -> T: ... +def narrow_none(x): + assert x is not None + return x + +def test_narrow_int() -> None: + a: Union[int, str] + a = narrow_int(a) + reveal_type(a) # E: Revealed type is 'builtins.int' + + b: int + b = narrow_int(b) + reveal_type(b) # E: Revealed type is 'builtins.int' + + c: str + c = narrow_int(c) + reveal_type(c) # Note: branch is now dead, so no type is revealed + # TODO: maybe we should make mypy report a warning instead? + +def test_narrow_none() -> None: + a: Optional[int] + a = narrow_none(a) + reveal_type(a) # E: Revealed type is 'builtins.int' + + b: int + b = narrow_none(b) + reveal_type(b) # E: Revealed type is 'builtins.int' + + c: None + c = narrow_none(c) + reveal_type(c) # Branch is now dead + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] From 3e778c0cb906c7bc1ef9afe5574f14cda6d0adfe Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 24 May 2018 16:58:23 -0700 Subject: [PATCH 11/16] Add support for special-casing overloaded descriptors Since the code for special-casing overloaded descriptors looked so simple, I went ahead and just added it. This also gave me an excuse to add more complex tests. --- mypy/checker.py | 4 +- test-data/unit/check-overloading.test | 75 +++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 963713cde638..c7aae41e3c47 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -429,6 +429,7 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: # # See Python 2's map function for a concrete example of this kind of overload. with experiments.strict_optional_set(True): + is_descriptor_get = defn.info is not None and defn.name() == "__get__" for i, item in enumerate(defn.items): # TODO overloads involving decorators assert isinstance(item, Decorator) @@ -447,7 +448,8 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: if overload_can_never_match(sig1, sig2): self.msg.overloaded_signature_will_never_match( i + 1, i + j + 2, item2.func) - elif is_unsafe_overlapping_overload_signatures(sig1, sig2): + elif (not is_descriptor_get + and is_unsafe_overlapping_overload_signatures(sig1, sig2)): self.msg.overloaded_signatures_overlap( i + 1, i + j + 2, item.func) if defn.impl: diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index f0d8e5e67f3c..df8d511a11f5 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -2163,3 +2163,78 @@ def test_narrow_none() -> None: [builtins fixtures/isinstance.pyi] [typing fixtures/typing-full.pyi] + +[case testOverloadWithNonGenericDescriptor] +from typing import overload, Any, Optional, Union + +class FakeAttribute: + @overload + def dummy(self, instance: None, owner: Any) -> 'FakeAttribute': ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + @overload + def dummy(self, instance: object, owner: Any) -> int: ... + def dummy(self, instance: Optional[object], owner: Any) -> Union['FakeAttribute', int]: ... + +class NumberAttribute: + @overload + def __get__(self, instance: None, owner: Any) -> 'NumberAttribute': ... + @overload + def __get__(self, instance: object, owner: Any) -> int: ... + def __get__(self, instance: Optional[object], owner: Any) -> Union['NumberAttribute', int]: + if instance is None: + return self + else: + return 3 + + def foo(self) -> str: ... + +class MyModel: + my_number = NumberAttribute() + +reveal_type(MyModel().my_number) # E: Revealed type is 'builtins.int' +MyModel().my_number.foo() # E: "int" has no attribute "foo" + +reveal_type(MyModel.my_number) # E: Revealed type is '__main__.NumberAttribute' +reveal_type(MyModel.my_number.foo()) # E: Revealed type is 'builtins.str' + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + +[case testOverloadWithGenericDescriptor] +from typing import overload, Any, Optional, TypeVar, Type, Union, Generic + +T = TypeVar('T') + +class FakeAttribute(Generic[T]): + @overload + def dummy(self, instance: None, owner: Type[T]) -> 'FakeAttribute[T]': ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + @overload + def dummy(self, instance: T, owner: Type[T]) -> int: ... + def dummy(self, instance: Optional[T], owner: Type[T]) -> Union['FakeAttribute[T]', int]: ... + +class NumberAttribute(Generic[T]): + @overload + def __get__(self, instance: None, owner: Type[T]) -> 'NumberAttribute[T]': ... + @overload + def __get__(self, instance: T, owner: Type[T]) -> int: ... + def __get__(self, instance: Optional[T], owner: Type[T]) -> Union['NumberAttribute[T]', int]: + if instance is None: + return self + else: + return 3 + + def foo(self) -> str: ... + +class MyModel: + my_number = NumberAttribute[MyModel]() + +reveal_type(MyModel().my_number) # E: Revealed type is 'builtins.int' +MyModel().my_number.foo() # E: "int" has no attribute "foo" + +reveal_type(MyModel.my_number) # E: Revealed type is '__main__.NumberAttribute[__main__.MyModel*]' +reveal_type(MyModel.my_number.foo()) # E: Revealed type is 'builtins.str' + +reveal_type(NumberAttribute[MyModel]().__get__(None, MyModel)) # E: Revealed type is '__main__.NumberAttribute[__main__.MyModel*]' +reveal_type(NumberAttribute[str]().__get__(None, str)) # E: Revealed type is '__main__.NumberAttribute[builtins.str*]' + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] From 0133339f5ebe5925abc01bf5807b01069b970b9a Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Fri, 25 May 2018 09:51:18 -0700 Subject: [PATCH 12/16] Adjust code for checking overload implementation signature This commit cleans up and documents the code for checking overload implementations. In particular: 1. I think 'sig1' can actually never be None (at least going by the types) so there's no need to check it. 2. I think the second check was actually broken -- it seemed to break in some edge cases where the overload implementation returns a union. --- mypy/checker.py | 40 +++++++---- test-data/unit/check-overloading.test | 99 +++++++++++++++++++++++++-- 2 files changed, 118 insertions(+), 21 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index c7aae41e3c47..e172bbbe1167 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -434,12 +434,11 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: # TODO overloads involving decorators assert isinstance(item, Decorator) sig1 = self.function_type(item.func) + assert isinstance(sig1, CallableType) for j, item2 in enumerate(defn.items[i + 1:]): assert isinstance(item2, Decorator) sig2 = self.function_type(item2.func) - - assert isinstance(sig1, CallableType) assert isinstance(sig2, CallableType) if not are_argument_counts_overlapping(sig1, sig2): @@ -452,6 +451,7 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: and is_unsafe_overlapping_overload_signatures(sig1, sig2)): self.msg.overloaded_signatures_overlap( i + 1, i + j + 2, item.func) + if defn.impl: if isinstance(defn.impl, FuncDef): impl_type = defn.impl.type @@ -459,26 +459,36 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: impl_type = defn.impl.var.type else: assert False, "Impl isn't the right type" + # This can happen if we've got an overload with a different # decorator too -- we gave up on the types. - if impl_type is None or isinstance(impl_type, AnyType) or sig1 is None: + if impl_type is None or isinstance(impl_type, AnyType): return - assert isinstance(impl_type, CallableType) - assert isinstance(sig1, CallableType) + + # Is the overload alternative's arguments subtypes of the implementation's? if not is_callable_compatible(impl_type, sig1, - is_compat=is_subtype, ignore_return=True): + is_compat=is_subtype, + ignore_return=True): self.msg.overloaded_signatures_arg_specific(i + 1, defn.impl) - impl_type_subst = impl_type + + # Repeat the same unification process 'is_callable_compatible' + # internally performs so we can examine the return type separately. if impl_type.variables: - unified = unify_generic_callable(impl_type, sig1, ignore_return=False) - if unified is None: - self.fail("Type variable mismatch between " + - "overload signature {} and implementation".format(i + 1), - defn.impl) - return - impl_type_subst = unified - if not is_subtype(sig1.ret_type, impl_type_subst.ret_type): + # Note: we set 'ignore_return=True' because 'unify_generic_callable' + # normally checks the arguments and return types with differing variance. + # + # This is normally what we want, but for checking the validity of overload + # implementations, we actually want to use the same variance for both. + # + # TODO: Patch 'is_callable_compatible' and 'unify_generic_callable'? + # somehow so we can customize the variance in all different sorts + # of ways? This would let us infer more constraints, letting us + # infer more precise types. + impl_type = unify_generic_callable(impl_type, sig1, ignore_return=True) + + # Is the overload alternative's return type a subtype of the implementation's? + if impl_type is not None and not is_subtype(sig1.ret_type, impl_type.ret_type): self.msg.overloaded_signatures_ret_specific(i + 1, defn.impl) # Here's the scoop about generators and coroutines. diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index df8d511a11f5..b9561b6407f8 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -270,7 +270,7 @@ reveal_type(f(B())) # E: Revealed type is '__main__.B' [builtins fixtures/isinstance.pyi] [case testTypeCheckOverloadWithImplTypeVarProblems] -from typing import overload, Any, TypeVar +from typing import overload, Any, TypeVar, Union T = TypeVar('T', bound='A') @@ -284,7 +284,7 @@ def f(x: 'A') -> 'A': ... @overload def f(x: 'B') -> 'B': ... -def f(x: Any) -> T: # E: Type variable mismatch between overload signature 2 and implementation +def f(x: Union[T, B]) -> T: # E: Overloaded function implementation cannot produce return type of signature 2 ... reveal_type(f(A())) # E: Revealed type is '__main__.A' @@ -2070,7 +2070,7 @@ from typing import overload, Union, TypeVar, NoReturn, Optional def narrow_int(x: str) -> NoReturn: ... @overload def narrow_int(x: int) -> int: ... -def narrow_int(x): +def narrow_int(x: Union[int, str]) -> Union[int, NoReturn]: assert isinstance(x, int) return x @@ -2079,7 +2079,15 @@ T = TypeVar('T') def narrow_none(x: None) -> NoReturn: ... @overload def narrow_none(x: T) -> T: ... -def narrow_none(x): +def narrow_none(x: Optional[T]) -> Union[NoReturn, T]: + assert x is not None + return x + +@overload +def narrow_none_v2(x: None) -> NoReturn: ... +@overload +def narrow_none_v2(x: T) -> T: ... +def narrow_none_v2(x: Optional[T]) -> T: assert x is not None return x @@ -2110,6 +2118,19 @@ def test_narrow_none() -> None: c = narrow_none(c) reveal_type(c) # Note: branch is now dead, so no type is revealed +def test_narrow_none_v2() -> None: + a: Optional[int] + a = narrow_none_v2(a) + reveal_type(a) # E: Revealed type is 'Union[builtins.int, None]' + + b: int + b = narrow_none_v2(b) + reveal_type(b) # E: Revealed type is 'builtins.int' + + c: None + c = narrow_none_v2(c) + reveal_type(c) # Note: branch is now dead, so no type is revealed + [builtins fixtures/isinstance.pyi] [typing fixtures/typing-full.pyi] @@ -2121,7 +2142,7 @@ from typing import overload, Union, TypeVar, NoReturn, Optional def narrow_int(x: str) -> NoReturn: ... @overload def narrow_int(x: int) -> int: ... -def narrow_int(x): +def narrow_int(x: Union[int, str]) -> Union[int, NoReturn]: assert isinstance(x, int) return x @@ -2130,7 +2151,15 @@ T = TypeVar('T') def narrow_none(x: None) -> NoReturn: ... @overload def narrow_none(x: T) -> T: ... -def narrow_none(x): +def narrow_none(x: Optional[T]) -> Union[NoReturn, T]: + assert x is not None + return x + +@overload +def narrow_none_v2(x: None) -> NoReturn: ... +@overload +def narrow_none_v2(x: T) -> T: ... +def narrow_none_v2(x: Optional[T]) -> T: assert x is not None return x @@ -2161,6 +2190,64 @@ def test_narrow_none() -> None: c = narrow_none(c) reveal_type(c) # Branch is now dead +def test_narrow_none_v2() -> None: + a: Optional[int] + a = narrow_none_v2(a) + reveal_type(a) # E: Revealed type is 'builtins.int' + + b: int + b = narrow_none_v2(b) + reveal_type(b) # E: Revealed type is 'builtins.int' + + c: None + c = narrow_none_v2(c) + reveal_type(c) # Note: branch is now dead, so no type is revealed + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + +[case testOverloadsAndNoReturnNarrowWhenBlacklistingSubtype] +from typing import TypeVar, NoReturn, Union, overload + +class Parent: ... +class A(Parent): ... +class B(Parent): ... +T = TypeVar('T', bound=Parent) + +@overload +def narrow_to_not_a(x: A) -> NoReturn: ... +@overload +def narrow_to_not_a(x: T) -> T: ... +def narrow_to_not_a(x: T) -> Union[NoReturn, T]: + assert not isinstance(x, A) + return x + +@overload +def narrow_to_not_a_v2(x: A) -> NoReturn: ... +@overload +def narrow_to_not_a_v2(x: T) -> T: ... +def narrow_to_not_a_v2(x: T) -> T: + assert not isinstance(x, A) + return x + +def test() -> None: + val: Union[A, B] + val = narrow_to_not_a(val) + reveal_type(val) # E: Revealed type is '__main__.B' + + val2: A + val2 = narrow_to_not_a(val2) + reveal_type(val2) # Branch now dead + +def test_v2() -> None: + val: Union[A, B] + val = narrow_to_not_a_v2(val) + reveal_type(val) # E: Revealed type is '__main__.B' + + val2: A + val2 = narrow_to_not_a_v2(val2) + reveal_type(val2) # Branch now dead + [builtins fixtures/isinstance.pyi] [typing fixtures/typing-full.pyi] From 26be59068ade6dd079f5713bbb468124b0974845 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Fri, 25 May 2018 09:54:33 -0700 Subject: [PATCH 13/16] Add a few more tests This commit adds a few more miscellaneous tests. --- test-data/unit/check-overloading.test | 150 ++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index b9561b6407f8..4a67d977cf1e 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -1813,6 +1813,31 @@ main:23: error: Revealed type is 'Union[__main__.B, __main__.D]' main:23: error: Argument 2 to "f2" has incompatible type "Union[A, C]"; expected "C" main:24: error: Revealed type is 'Union[__main__.B, __main__.D]' +[case testOverloadInferUnionRespectsVariance] +from typing import overload, TypeVar, Union, Generic + +class A: pass +class B(A): pass +class C(B): pass + +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) + +class WrapperCo(Generic[T_co]): pass +class WrapperContra(Generic[T_contra]): pass + +@overload +def foo(x: WrapperCo[B]) -> int: ... +@overload +def foo(x: WrapperContra[B]) -> str: ... +def foo(x): pass + +compat: Union[WrapperCo[C], WrapperContra[A]] +reveal_type(foo(compat)) # E: Revealed type is 'Union[builtins.int, builtins.str]' + +not_compat: Union[WrapperCo[A], WrapperContra[C]] +foo(not_compat) # E: Argument 1 to "foo" has incompatible type "Union[WrapperCo[A], WrapperContra[C]]"; expected "Union[WrapperCo[B], WrapperContra[B]]" + [case testOverloadInferUnionSkipIfParameterNamesAreDifferent] from typing import overload, Union @@ -1963,6 +1988,131 @@ def wrapper(mysterious: T) -> T: return mysterious +[case testOverloadingInferUnionReturnWithMixedTypevars] +from typing import overload, Generic, TypeVar, List, Tuple, Union + +class A: pass +class B(A): pass +class C(A): pass + +T = TypeVar('T', bound=A) +S = TypeVar('S') + +class Dummy(Generic[T]): + @overload + def foo(self, x: List[Tuple[T, S]], y: S) -> T: ... + @overload + def foo(self, x: List[S], y: S) -> S: ... + def foo(self, x: Union[List[Tuple[T, S]], List[S]], y: S) -> Union[T, S]: ... + +T1 = TypeVar('T1', bound=A) + +def t_is_same_bound(arg1: T1, arg2: S) -> Tuple[T1, S]: + x1: Union[List[S], List[Tuple[T1, S]]] + y1: S + reveal_type(Dummy[T1]().foo(x1, y1)) # E: Revealed type is 'Union[T1`-1, S`-2]' + + x2: Union[List[T1], List[Tuple[T1, T1]]] + y2: T1 + reveal_type(Dummy[T1]().foo(x2, y2)) # E: Revealed type is 'T1`-1' + + # The arguments in the tuple are swapped + x3: Union[List[S], List[Tuple[S, T1]]] + y3: S + Dummy[T1]().foo(x3, y3) # E: Argument 1 to "foo" of "Dummy" has incompatible type "Union[List[S], List[Tuple[S, T1]]]"; expected "Union[List[Tuple[T1, S]], List[S]]" + + x4: Union[List[int], List[Tuple[C, int]]] + y4: int + reveal_type(Dummy[C]().foo(x4, y4)) # E: Revealed type is 'Union[__main__.C, builtins.int*]' + Dummy[A]().foo(x4, y4) # E: Cannot infer type argument 1 of "foo" of "Dummy" + + return arg1, arg2 + + +T2 = TypeVar('T2', bound=B) + +def t_is_tighter_bound(arg1: T2, arg2: S) -> Tuple[T2, S]: + x1: Union[List[S], List[Tuple[T2, S]]] + y1: S + reveal_type(Dummy[T2]().foo(x1, y1)) # E: Revealed type is 'Union[T2`-1, S`-2]' + + x2: Union[List[T2], List[Tuple[T2, T2]]] + y2: T2 + reveal_type(Dummy[T2]().foo(x2, y2)) # E: Revealed type is 'T2`-1' + + return arg1, arg2 + +[builtins fixtures/list.pyi] + +[case testOverloadingInferUnionReturnWithTypevarsAndValueRestrictions] +from typing import overload, Generic, TypeVar, List, Tuple, Union + +class A: pass +class B(A): pass +class C(A): pass + +T = TypeVar('T', bound=A) +S = TypeVar('S') + +class Dummy(Generic[T]): + @overload + def foo(self, x: List[Tuple[T, S]], y: S) -> T: ... + @overload + def foo(self, x: List[S], y: S) -> S: ... + def foo(self, x: Union[List[Tuple[T, S]], List[S]], y: S) -> Union[T, S]: ... + +T3 = TypeVar('T3', B, C) + +def t_is_compatible_bound(arg1: T3, arg2: S) -> Tuple[T3, S]: + x1: Union[List[S], List[Tuple[T3, S]]] + y1: S + reveal_type(Dummy[T3]().foo(x1, y1)) + + x2: Union[List[T3], List[Tuple[T3, T3]]] + y2: T3 + reveal_type(Dummy[T3]().foo(x2, y2)) + + return arg1, arg2 + +[builtins fixtures/list.pyi] +[out] +main:22: error: Revealed type is 'Union[__main__.B, S`-2]' +main:22: error: Revealed type is 'Union[__main__.C, S`-2]' +main:26: error: Revealed type is '__main__.B' +main:26: error: Revealed type is '__main__.C' + +[case testOverloadInferUnionReturnWithInconsistentTypevarNames] +from typing import overload, TypeVar, Union + +T = TypeVar('T') +S = TypeVar('S') + +@overload +def consistent(x: T, y: str) -> T: ... +@overload +def consistent(x: T, y: int) -> T: ... +def consistent(x: T, y: Union[str, int]) -> T: + return x + +@overload +def inconsistent(x: T, y: str) -> T: ... +@overload +def inconsistent(x: S, y: int) -> S: ... +def inconsistent(x: T, y: Union[str, int]) -> T: + return x + +def test(x: T) -> T: + y: Union[str, int] + + reveal_type(consistent(x, y)) # E: Revealed type is 'T`-1' + + # TODO: Should we try and handle this differently? + # On one hand, this overload is defined in a weird way so it's arguably + # the user's fault; on the other, there's nothing overtly wrong with it. + inconsistent(x, y) # E: Argument 2 to "inconsistent" has incompatible type "Union[str, int]"; expected "str" + + return x + [case testOverloadsAndNoneWithoutStrictOptional] # flags: --no-strict-optional from typing import overload, Optional From 2387ae8a9c2edf481bff8c980332dba079b7a530 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 28 May 2018 23:41:23 -0700 Subject: [PATCH 14/16] Clean up and expand comments --- mypy/checker.py | 2 +- mypy/checkexpr.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index e172bbbe1167..9e19bf9d96da 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3667,7 +3667,7 @@ def is_unsafe_overlapping_operator_signatures(signature: Type, other: Type) -> b """Check if two operator method signatures may be unsafely overlapping. Two signatures s and t are overlapping if both can be valid for the same - statically typed values and the return types are incompatible. + statically typed values and the return types are incompatible. Assume calls are first checked against 'signature', then against 'other'. Thus if 'signature' is more general than 'other', there is no unsafe diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 8a7207b57cfc..69c72198e32d 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1121,7 +1121,11 @@ def check_overload_call(self, plausible_targets = self.plausible_overload_call_targets(arg_types, arg_kinds, arg_names, callee) - # Step 2: If the arguments contain a union, we try performing union math first. + # Step 2: If the arguments contain a union, we try performing union math first, + # instead of picking the first matching overload. + # This is because picking the first overload often ends up being too greedy: + # for example, when we have a fallback alternative that accepts an unrestricted + # typevar. See https://github.com/python/mypy/issues/4063 for related discussion. erased_targets = None # type: Optional[List[CallableType]] unioned_result = None # type: Optional[Tuple[Type, Type]] unioned_errors = None # type: Optional[MessageBuilder] From 7f26467547aeedf967e4ff4f3c29e53e649a20dc Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 28 May 2018 23:42:04 -0700 Subject: [PATCH 15/16] Split up tests into shorter pieces --- test-data/unit/check-overloading.test | 347 +++++++++++++++++--------- 1 file changed, 230 insertions(+), 117 deletions(-) diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index adafeeab06d2..c103ea3b3ddd 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -1009,7 +1009,6 @@ def i(x: List[str]) -> int: pass # E: Overloaded function signatures 1 and 2 ov def i(x: List[T]) -> None: pass [builtins fixtures/list.pyi] - [case testOverlapWithTypeVarsWithValues] from foo import * [file foo.pyi] @@ -1912,7 +1911,7 @@ reveal_type(obj.f(A())) # E: Revealed type is '__main__.C' reveal_type(obj.f(B())) # E: Revealed type is '__main__.B' reveal_type(obj.f(x)) # E: Revealed type is 'Union[__main__.B, __main__.C]' -[case testOverloadingInferUnionReturnWithTypevarReturn] +[case testOverloadingInferUnionReturnWithFunctionTypevarReturn] from typing import overload, Union, TypeVar, Generic T = TypeVar('T') @@ -1930,6 +1929,32 @@ def foo(x): ... def bar(x: Union[W1[T], W2[T]]) -> T: ... +def wrapper() -> None: + obj1: Union[W1[A], W2[A]] + + a1: A = foo(obj1) + a2 = foo(obj1) + reveal_type(a1) # E: Revealed type is '__main__.A' + reveal_type(a2) # E: Revealed type is '__main__.A*' + + obj2: Union[W1[A], W2[B]] + + foo(obj2) # E: Cannot infer type argument 1 of "foo" + bar(obj2) # E: Cannot infer type argument 1 of "bar" + + b1_overload: A = foo(obj2) # E: Cannot infer type argument 1 of "foo" + b1_union: A = bar(obj2) # E: Cannot infer type argument 1 of "bar" + +[case testOverloadingInferUnionReturnWithObjectTypevarReturn] +from typing import overload, Union, TypeVar, Generic + +T = TypeVar('T') + +class W1(Generic[T]): pass +class W2(Generic[T]): pass +class A: pass +class B: pass + class SomeType(Generic[T]): @overload def foo(self, x: W1[T]) -> T: ... @@ -1939,52 +1964,50 @@ class SomeType(Generic[T]): def bar(self, x: Union[W1[T], W2[T]]) -> T: ... -def wrapper(mysterious: T) -> T: +def wrapper() -> None: obj1: Union[W1[A], W2[A]] - # Phase 1: Things mypy should be able to infer + a1 = SomeType[A]().foo(obj1) + reveal_type(a1) # E: Revealed type is '__main__.A*' - a1: A = foo(obj1) - a2 = foo(obj1) - a3 = SomeType[A]().foo(obj1) - - reveal_type(a1) # E: Revealed type is '__main__.A' - reveal_type(a2) # E: Revealed type is '__main__.A*' - reveal_type(a3) # E: Revealed type is '__main__.A*' - - # Phase 2: Things mypy should be able to infer, but does not - # due to bugs unrelated to overloads - # - # (We confirm this by checking to see that the error - # message is identical to what we would have gotten if - # we manually union the overload) - - a4_overload: A = SomeType().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[], W2[]]" - a4_union: A = SomeType().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[], W2[]]" + # Note: These should be fine, but mypy has an unrelated bug + # that makes them error out? + a2_overload: A = SomeType().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[], W2[]]" + a2_union: A = SomeType().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[], W2[]]" SomeType().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[], W2[]]" SomeType().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[A]]"; expected "Union[W1[], W2[]]" +[case testOverloadingInferUnionReturnWithBadObjectTypevarReturn] +from typing import overload, Union, TypeVar, Generic - # Phase 3: Things where type inference is impossible and - # mypy should report an error +T = TypeVar('T') - obj2: Union[W1[A], W2[B]] +class W1(Generic[T]): pass +class W2(Generic[T]): pass +class A: pass +class B: pass - foo(obj2) # E: Cannot infer type argument 1 of "foo" - bar(obj2) # E: Cannot infer type argument 1 of "bar" +class SomeType(Generic[T]): + @overload + def foo(self, x: W1[T]) -> T: ... + @overload + def foo(self, x: W2[T]) -> T: ... + def foo(self, x): ... - b2_overload: A = foo(obj2) # E: Cannot infer type argument 1 of "foo" - b2_union: A = bar(obj2) # E: Cannot infer type argument 1 of "bar" + def bar(self, x: Union[W1[T], W2[T]]) -> T: ... + +def wrapper(mysterious: T) -> T: + obj1: Union[W1[A], W2[B]] - SomeType().foo(obj2) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[], W2[]]" - SomeType().bar(obj2) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[], W2[]]" + SomeType().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[], W2[]]" + SomeType().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[], W2[]]" - SomeType[A]().foo(obj2) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[A], W2[A]]" - SomeType[A]().bar(obj2) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[A], W2[A]]" + SomeType[A]().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[A], W2[A]]" + SomeType[A]().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[A], W2[A]]" - SomeType[T]().foo(obj2) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[T], W2[T]]" - SomeType[T]().bar(obj2) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[T], W2[T]]" + SomeType[T]().foo(obj1) # E: Argument 1 to "foo" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[T], W2[T]]" + SomeType[T]().bar(obj1) # E: Argument 1 to "bar" of "SomeType" has incompatible type "Union[W1[A], W2[B]]"; expected "Union[W1[T], W2[T]]" return mysterious @@ -2016,6 +2039,30 @@ def t_is_same_bound(arg1: T1, arg2: S) -> Tuple[T1, S]: y2: T1 reveal_type(Dummy[T1]().foo(x2, y2)) # E: Revealed type is 'T1`-1' + return arg1, arg2 + +[builtins fixtures/list.pyi] + +[case testOverloadingInferUnionReturnWithMixedTypevarsInnerMismatch] +from typing import overload, Generic, TypeVar, List, Tuple, Union + +class A: pass +class B(A): pass +class C(A): pass + +T = TypeVar('T', bound=A) +S = TypeVar('S') + +class Dummy(Generic[T]): + @overload + def foo(self, x: List[Tuple[T, S]], y: S) -> T: ... + @overload + def foo(self, x: List[S], y: S) -> S: ... + def foo(self, x: Union[List[Tuple[T, S]], List[S]], y: S) -> Union[T, S]: ... + +T1 = TypeVar('T1', bound=A) + +def t_is_same_bound(arg1: T1, arg2: S) -> Tuple[T1, S]: # The arguments in the tuple are swapped x3: Union[List[S], List[Tuple[S, T1]]] y3: S @@ -2028,17 +2075,35 @@ def t_is_same_bound(arg1: T1, arg2: S) -> Tuple[T1, S]: return arg1, arg2 +[builtins fixtures/list.pyi] + +[case testOverloadingInferUnionReturnWithMixedTypevarsTighterBound] +from typing import overload, Generic, TypeVar, List, Tuple, Union + +class A: pass +class B(A): pass +class C(A): pass + +T = TypeVar('T', bound=A) +S = TypeVar('S') -T2 = TypeVar('T2', bound=B) +class Dummy(Generic[T]): + @overload + def foo(self, x: List[Tuple[T, S]], y: S) -> T: ... + @overload + def foo(self, x: List[S], y: S) -> S: ... + def foo(self, x: Union[List[Tuple[T, S]], List[S]], y: S) -> Union[T, S]: ... -def t_is_tighter_bound(arg1: T2, arg2: S) -> Tuple[T2, S]: - x1: Union[List[S], List[Tuple[T2, S]]] +T1 = TypeVar('T1', bound=B) + +def t_is_tighter_bound(arg1: T1, arg2: S) -> Tuple[T1, S]: + x1: Union[List[S], List[Tuple[T1, S]]] y1: S - reveal_type(Dummy[T2]().foo(x1, y1)) # E: Revealed type is 'Union[T2`-1, S`-2]' + reveal_type(Dummy[T1]().foo(x1, y1)) # E: Revealed type is 'Union[T1`-1, S`-2]' - x2: Union[List[T2], List[Tuple[T2, T2]]] - y2: T2 - reveal_type(Dummy[T2]().foo(x2, y2)) # E: Revealed type is 'T2`-1' + x2: Union[List[T1], List[Tuple[T1, T1]]] + y2: T1 + reveal_type(Dummy[T1]().foo(x2, y2)) # E: Revealed type is 'T1`-1' return arg1, arg2 @@ -2212,9 +2277,9 @@ reveal_type(mymap(f3, seq)) # E: Revealed type is 'Union[typing.Iterable[builti [builtins fixtures/list.pyi] [typing fixtures/typing-full.pyi] -[case testOverloadsAndNoReturnNarrowTypeNoStrictOptional] +[case testOverloadsAndNoReturnNarrowTypeNoStrictOptional1] # flags: --no-strict-optional -from typing import overload, Union, TypeVar, NoReturn, Optional +from typing import overload, Union, NoReturn @overload def narrow_int(x: str) -> NoReturn: ... @@ -2224,21 +2289,33 @@ def narrow_int(x: Union[int, str]) -> Union[int, NoReturn]: assert isinstance(x, int) return x -T = TypeVar('T') -@overload -def narrow_none(x: None) -> NoReturn: ... -@overload -def narrow_none(x: T) -> T: ... -def narrow_none(x: Optional[T]) -> Union[NoReturn, T]: - assert x is not None - return x +def test_narrow_int() -> None: + a: Union[int, str] + a = narrow_int(a) + reveal_type(a) # E: Revealed type is 'builtins.int' + + b: int + b = narrow_int(b) + reveal_type(b) # E: Revealed type is 'builtins.int' + + c: str + c = narrow_int(c) + reveal_type(c) # Note: branch is now dead, so no type is revealed + # TODO: maybe we should make mypy report a warning instead? + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + +[case testOverloadsAndNoReturnNarrowTypeWithStrictOptional1] +# flags: --strict-optional +from typing import overload, Union, NoReturn @overload -def narrow_none_v2(x: None) -> NoReturn: ... +def narrow_int(x: str) -> NoReturn: ... @overload -def narrow_none_v2(x: T) -> T: ... -def narrow_none_v2(x: Optional[T]) -> T: - assert x is not None +def narrow_int(x: int) -> int: ... +def narrow_int(x: Union[int, str]) -> Union[int, NoReturn]: + assert isinstance(x, int) return x def test_narrow_int() -> None: @@ -2255,6 +2332,22 @@ def test_narrow_int() -> None: reveal_type(c) # Note: branch is now dead, so no type is revealed # TODO: maybe we should make mypy report a warning instead? +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + +[case testOverloadsAndNoReturnNarrowTypeNoStrictOptional2] +# flags: --no-strict-optional +from typing import overload, Union, TypeVar, NoReturn, Optional + +T = TypeVar('T') +@overload +def narrow_none(x: None) -> NoReturn: ... +@overload +def narrow_none(x: T) -> T: ... +def narrow_none(x: Optional[T]) -> Union[NoReturn, T]: + assert x is not None + return x + def test_narrow_none() -> None: a: Optional[int] a = narrow_none(a) @@ -2268,34 +2361,13 @@ def test_narrow_none() -> None: c = narrow_none(c) reveal_type(c) # Note: branch is now dead, so no type is revealed -def test_narrow_none_v2() -> None: - a: Optional[int] - a = narrow_none_v2(a) - reveal_type(a) # E: Revealed type is 'Union[builtins.int, None]' - - b: int - b = narrow_none_v2(b) - reveal_type(b) # E: Revealed type is 'builtins.int' - - c: None - c = narrow_none_v2(c) - reveal_type(c) # Note: branch is now dead, so no type is revealed - [builtins fixtures/isinstance.pyi] [typing fixtures/typing-full.pyi] -[case testOverloadsAndNoReturnNarrowTypeWithStrictOptional] +[case testOverloadsAndNoReturnNarrowTypeWithStrictOptional2] # flags: --strict-optional from typing import overload, Union, TypeVar, NoReturn, Optional -@overload -def narrow_int(x: str) -> NoReturn: ... -@overload -def narrow_int(x: int) -> int: ... -def narrow_int(x: Union[int, str]) -> Union[int, NoReturn]: - assert isinstance(x, int) - return x - T = TypeVar('T') @overload def narrow_none(x: None) -> NoReturn: ... @@ -2305,6 +2377,27 @@ def narrow_none(x: Optional[T]) -> Union[NoReturn, T]: assert x is not None return x +def test_narrow_none() -> None: + a: Optional[int] + a = narrow_none(a) + reveal_type(a) # E: Revealed type is 'builtins.int' + + b: int + b = narrow_none(b) + reveal_type(b) # E: Revealed type is 'builtins.int' + + c: None + c = narrow_none(c) + reveal_type(c) # Branch is now dead + +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + + +[case testOverloadsAndNoReturnNarrowTypeNoStrictOptional3] +# flags: --no-strict-optional +from typing import overload, TypeVar, NoReturn, Optional + @overload def narrow_none_v2(x: None) -> NoReturn: ... @overload @@ -2313,32 +2406,33 @@ def narrow_none_v2(x: Optional[T]) -> T: assert x is not None return x -def test_narrow_int() -> None: - a: Union[int, str] - a = narrow_int(a) - reveal_type(a) # E: Revealed type is 'builtins.int' +def test_narrow_none_v2() -> None: + a: Optional[int] + a = narrow_none_v2(a) + reveal_type(a) # E: Revealed type is 'Union[builtins.int, None]' b: int - b = narrow_int(b) + b = narrow_none_v2(b) reveal_type(b) # E: Revealed type is 'builtins.int' - c: str - c = narrow_int(c) + c: None + c = narrow_none_v2(c) reveal_type(c) # Note: branch is now dead, so no type is revealed - # TODO: maybe we should make mypy report a warning instead? -def test_narrow_none() -> None: - a: Optional[int] - a = narrow_none(a) - reveal_type(a) # E: Revealed type is 'builtins.int' +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] - b: int - b = narrow_none(b) - reveal_type(b) # E: Revealed type is 'builtins.int' +[case testOverloadsAndNoReturnNarrowTypeWithStrictOptional3] +# flags: --strict-optional +from typing import overload, TypeVar, NoReturn, Optional - c: None - c = narrow_none(c) - reveal_type(c) # Branch is now dead +@overload +def narrow_none_v2(x: None) -> NoReturn: ... +@overload +def narrow_none_v2(x: T) -> T: ... +def narrow_none_v2(x: Optional[T]) -> T: + assert x is not None + return x def test_narrow_none_v2() -> None: a: Optional[int] @@ -2372,14 +2466,6 @@ def narrow_to_not_a(x: T) -> Union[NoReturn, T]: assert not isinstance(x, A) return x -@overload -def narrow_to_not_a_v2(x: A) -> NoReturn: ... -@overload -def narrow_to_not_a_v2(x: T) -> T: ... -def narrow_to_not_a_v2(x: T) -> T: - assert not isinstance(x, A) - return x - def test() -> None: val: Union[A, B] val = narrow_to_not_a(val) @@ -2389,6 +2475,25 @@ def test() -> None: val2 = narrow_to_not_a(val2) reveal_type(val2) # Branch now dead +[builtins fixtures/isinstance.pyi] +[typing fixtures/typing-full.pyi] + +[case testOverloadsAndNoReturnNarrowWhenBlacklistingSubtype2] +from typing import TypeVar, NoReturn, Union, overload + +class Parent: ... +class A(Parent): ... +class B(Parent): ... +T = TypeVar('T', bound=Parent) + +@overload +def narrow_to_not_a_v2(x: A) -> NoReturn: ... +@overload +def narrow_to_not_a_v2(x: T) -> T: ... +def narrow_to_not_a_v2(x: T) -> T: + assert not isinstance(x, A) + return x + def test_v2() -> None: val: Union[A, B] val = narrow_to_not_a_v2(val) @@ -2404,13 +2509,6 @@ def test_v2() -> None: [case testOverloadWithNonGenericDescriptor] from typing import overload, Any, Optional, Union -class FakeAttribute: - @overload - def dummy(self, instance: None, owner: Any) -> 'FakeAttribute': ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types - @overload - def dummy(self, instance: object, owner: Any) -> int: ... - def dummy(self, instance: Optional[object], owner: Any) -> Union['FakeAttribute', int]: ... - class NumberAttribute: @overload def __get__(self, instance: None, owner: Any) -> 'NumberAttribute': ... @@ -2436,18 +2534,21 @@ reveal_type(MyModel.my_number.foo()) # E: Revealed type is 'builtins.str' [builtins fixtures/isinstance.pyi] [typing fixtures/typing-full.pyi] +[case testOverloadWithNonGenericDescriptorLookalike] +from typing import overload, Any, Optional, Union + +class FakeAttribute: + @overload + def dummy(self, instance: None, owner: Any) -> 'FakeAttribute': ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + @overload + def dummy(self, instance: object, owner: Any) -> int: ... + def dummy(self, instance: Optional[object], owner: Any) -> Union['FakeAttribute', int]: ... + [case testOverloadWithGenericDescriptor] from typing import overload, Any, Optional, TypeVar, Type, Union, Generic T = TypeVar('T') -class FakeAttribute(Generic[T]): - @overload - def dummy(self, instance: None, owner: Type[T]) -> 'FakeAttribute[T]': ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types - @overload - def dummy(self, instance: T, owner: Type[T]) -> int: ... - def dummy(self, instance: Optional[T], owner: Type[T]) -> Union['FakeAttribute[T]', int]: ... - class NumberAttribute(Generic[T]): @overload def __get__(self, instance: None, owner: Type[T]) -> 'NumberAttribute[T]': ... @@ -2475,3 +2576,15 @@ reveal_type(NumberAttribute[str]().__get__(None, str)) # E: Revealed type i [builtins fixtures/isinstance.pyi] [typing fixtures/typing-full.pyi] + +[case testOverloadWithGenericDescriptorLookalike] +from typing import overload, Any, Optional, TypeVar, Type, Union, Generic + +T = TypeVar('T') + +class FakeAttribute(Generic[T]): + @overload + def dummy(self, instance: None, owner: Type[T]) -> 'FakeAttribute[T]': ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + @overload + def dummy(self, instance: T, owner: Type[T]) -> int: ... + def dummy(self, instance: Optional[T], owner: Type[T]) -> Union['FakeAttribute[T]', int]: ... From 3027d5783d720c41ce9cc7e0ad9465da948f4dd6 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 28 May 2018 23:42:18 -0700 Subject: [PATCH 16/16] Add test checking empty collections --- test-data/unit/check-overloading.test | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index c103ea3b3ddd..3208036afe36 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -1092,6 +1092,38 @@ main:21: note: Consider using "Sequence" instead, which is covariant main:22: error: Argument 1 to "foo" has incompatible type "List[object]"; expected "List[int]" main:23: error: Argument 1 to "foo" has incompatible type "List[Union[int, str]]"; expected "List[int]" +[case testOverloadAgainstEmptyCollections] +from typing import overload, List + +@overload +def f(x: List[int]) -> int: ... +@overload +def f(x: List[str]) -> str: ... +def f(x): pass + +reveal_type(f([])) # E: Revealed type is 'builtins.int' +[builtins fixtures/list.pyi] + +[case testOverloadAgainstEmptyCovariantCollections] +from typing import overload, TypeVar, Generic + +T = TypeVar('T', covariant=True) +class Wrapper(Generic[T]): pass + +class A: pass +class B(A): pass +class C: pass + +@overload +def f(x: Wrapper[A]) -> int: ... +@overload +def f(x: Wrapper[C]) -> str: ... +def f(x): pass + +reveal_type(f(Wrapper())) # E: Revealed type is 'builtins.int' +reveal_type(f(Wrapper[C]())) # E: Revealed type is 'builtins.str' +reveal_type(f(Wrapper[B]())) # E: Revealed type is 'builtins.int' + [case testOverlappingOverloadCounting] from foo import * [file foo.pyi]