Merge branch 'master' into fix/mypyc-defaults-setup-cache-load

Author: georgesittas

CHANGELOG.md

@@ -38,6 +38,7 @@ Contributed by Jukka Lehtosalo (PR [21433](https://github.com/python/mypy/pull/2
 
 ### Mypyc Improvements
 
+- Enable incremental self-compilation (Vaggelis Danias, PR [21369](https://github.com/python/mypy/pull/21369))
 - Make compilation order with multiple files consistent (Piotr Sawicki, PR [21419](https://github.com/python/mypy/pull/21419))
 - Fix crash on accessing `StopAsyncIteration` (Piotr Sawicki, PR [21406](https://github.com/python/mypy/pull/21406))
 - Fix incremental compilation with `separate` flag (Vaggelis Danias, PR [21299](https://github.com/python/mypy/pull/21299))
@@ -54,15 +55,13 @@ Contributed by Jukka Lehtosalo (PR [21433](https://github.com/python/mypy/pull/2
 ### Other Notable Fixes and Improvements
 
 - Rely on typeshed stubs for `slice` typing (Ivan Levkivskyi, PR [21401](https://github.com/python/mypy/pull/21401))
-- Improve negative narrowing for membership checks on tuples (Shantanu, PR [21456](https://github.com/python/mypy/pull/21456))
 - Narrow match captures based on previous cases (Shantanu, PR [21405](https://github.com/python/mypy/pull/21405))
 - Fix nondeterminism in overload resolution (Shantanu, PR [21455](https://github.com/python/mypy/pull/21455))
 - Respect file config comments for stale modules (Adam Turner, PR [21444](https://github.com/python/mypy/pull/21444))
 - Fix JSON output mode for syntax errors in parallel mode (Adam Turner, PR [21434](https://github.com/python/mypy/pull/21434))
 - Fix type variable with values as a supertype (Ivan Levkivskyi, PR [21431](https://github.com/python/mypy/pull/21431))
 - Add support for configuring `--num-workers` with an environment variable (Kevin Kannammalil, PR [21407](https://github.com/python/mypy/pull/21407))
 - Respect JSON output mode for syntax errors (Adam Turner, PR [21386](https://github.com/python/mypy/pull/21386))
-- Analyze `TypedDict` decorators (Pranav Manglik, PR [21267](https://github.com/python/mypy/pull/21267))
 
 ### Typeshed Updates
 

mypy/checkexpr.py

@@ -1948,12 +1948,7 @@ def analyze_type_type_callee(self, item: ProperType, context: Context) -> Type:
             # but better than AnyType...), but replace the return type
             # with typevar.
             callee = self.analyze_type_type_callee(get_proper_type(item.upper_bound), context)
-            callee = get_proper_type(callee)
-            if isinstance(callee, CallableType):
-                callee = callee.copy_modified(ret_type=item)
-            elif isinstance(callee, Overloaded):
-                callee = Overloaded([c.copy_modified(ret_type=item) for c in callee.items])
-            return callee
+            return self.replace_type_type_callee_ret_type(callee, item)
         # We support Type of namedtuples but not of tuples in general
         if isinstance(item, TupleType) and tuple_fallback(item).type.fullname != "builtins.tuple":
             return self.analyze_type_type_callee(tuple_fallback(item), context)
@@ -1963,6 +1958,23 @@ def analyze_type_type_callee(self, item: ProperType, context: Context) -> Type:
         self.msg.unsupported_type_type(item, context)
         return AnyType(TypeOfAny.from_error)
 
+    def replace_type_type_callee_ret_type(self, callee: Type, ret_type: Type) -> Type:
+        callee = get_proper_type(callee)
+        if isinstance(callee, CallableType):
+            return callee.copy_modified(ret_type=ret_type)
+        if isinstance(callee, Overloaded):
+            return Overloaded([c.copy_modified(ret_type=ret_type) for c in callee.items])
+        if isinstance(callee, UnionType):
+            return UnionType(
+                [
+                    self.replace_type_type_callee_ret_type(item, ret_type)
+                    for item in callee.relevant_items()
+                ],
+                line=callee.line,
+                column=callee.column,
+            )
+        return callee
+
     def infer_arg_types_in_empty_context(self, args: list[Expression]) -> list[Type]:
         """Infer argument expression types in an empty context.
 

mypy/meet.py

@@ -1090,7 +1090,28 @@ def visit_tuple_type(self, t: TupleType) -> ProperType:
         elif isinstance(self.s, Instance):
             # meet(Tuple[t1, t2, <...>], Tuple[s, ...]) == Tuple[meet(t1, s), meet(t2, s), <...>].
             if self.s.type.fullname in TUPLE_LIKE_INSTANCE_NAMES and self.s.args:
-                return t.copy_modified(items=[meet_types(it, self.s.args[0]) for it in t.items])
+                arg = self.s.args[0]
+                new_items: list[Type] = []
+                for it in t.items:
+                    # Unpack items need to be handled by the caller.
+                    if isinstance(it, UnpackType):
+                        unpacked = get_proper_type(it.type)
+                        if isinstance(unpacked, TypeVarTupleType):
+                            # We can't infer anything in this case.
+                            new_arg = UninhabitedType()
+                            instance = unpacked.tuple_fallback
+                        else:
+                            assert (
+                                isinstance(unpacked, Instance)
+                                and unpacked.type.fullname == "builtins.tuple"
+                            )
+                            new_arg = meet_types(unpacked.args[0], arg)
+                            instance = unpacked
+                        new_items.append(UnpackType(instance.copy_modified(args=[new_arg])))
+                    else:
+                        # All other items can be processed in a regular way.
+                        new_items.append(meet_types(it, arg))
+                return t.copy_modified(items=new_items)
             elif is_proper_subtype(t, self.s):
                 # A named tuple that inherits from a normal class
                 return t

mypy/message_registry.py

@@ -181,6 +181,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage:
     'Implicit generic "Any". Use "{}" and specify generic parameters'
 )
 NO_CYCLIC_DEFAULT: Final = "Cyclic type variable defaults are not supported"
+NO_DEFAULT_AFTER_TYPEVAR_TUPLE: Final = "A type variable with default cannot follow TypeVarTuple"
 INVALID_UNPACK: Final = "{} cannot be unpacked (must be tuple or TypeVarTuple)"
 INVALID_UNPACK_POSITION: Final = "Unpack is only valid in a variadic position"
 INVALID_PARAM_SPEC_LOCATION: Final = "Invalid location for ParamSpec {}"

mypy/mixedtraverser.py

@@ -16,6 +16,7 @@
     TypeApplication,
     TypedDictExpr,
     TypeFormExpr,
+    TypeInfo,
     TypeVarExpr,
     Var,
     WithStmt,
@@ -41,15 +42,29 @@ def visit_func(self, o: FuncItem, /) -> None:
         self.visit_optional_type(o.type)
 
     def visit_class_def(self, o: ClassDef, /) -> None:
-        # TODO: Should we visit generated methods/variables as well, either here or in
-        #       TraverserVisitor?
         super().visit_class_def(o)
-        info = o.info
-        if info:
-            for base in info.bases:
-                base.accept(self)
-            if info.special_alias:
-                info.special_alias.accept(self)
+        if o.info:
+            self.process_type_info(o.info)
+
+    def process_type_info(self, info: TypeInfo) -> None:
+        # TODO: Should we visit generated methods/variables as well?
+        # We should for methods generated by us (see below). But it is less clear for
+        # 3rd party plugin generated methods (since we don't want to emit errors there).
+        for base in info.bases:
+            base.accept(self)
+        if info.special_alias:
+            # We need to accept all types that are conceptually identical like special
+            # alias target and corresponding tuple_type or typeddict_type, since those
+            # may be copies, and not the same object.
+            info.special_alias.accept(self)
+        if info.tuple_type:
+            info.tuple_type.accept(self)
+        if info.typeddict_type:
+            info.typeddict_type.accept(self)
+        if info.is_named_tuple or info.is_newtype:
+            for sym in info.names.values():
+                if sym.plugin_generated and sym.node:
+                    sym.node.accept(self)
 
     def visit_type_alias_expr(self, o: TypeAliasExpr, /) -> None:
         super().visit_type_alias_expr(o)
@@ -64,19 +79,20 @@ def visit_type_var_expr(self, o: TypeVarExpr, /) -> None:
 
     def visit_typeddict_expr(self, o: TypedDictExpr, /) -> None:
         super().visit_typeddict_expr(o)
-        self.visit_optional_type(o.info.typeddict_type)
+        self.process_type_info(o.info)
 
     def visit_namedtuple_expr(self, o: NamedTupleExpr, /) -> None:
         super().visit_namedtuple_expr(o)
-        assert o.info.tuple_type
-        o.info.tuple_type.accept(self)
+        self.process_type_info(o.info)
 
     def visit__promote_expr(self, o: PromoteExpr, /) -> None:
         super().visit__promote_expr(o)
         o.type.accept(self)
 
     def visit_newtype_expr(self, o: NewTypeExpr, /) -> None:
         super().visit_newtype_expr(o)
+        if o.info:
+            self.process_type_info(o.info)
         self.visit_optional_type(o.old_type)
 
     # Statements

mypy/semanal.py

@@ -546,8 +546,13 @@ def __init__(
         # to create the set lazily.
         self.types_fixed: set[TypeInfo | TypeAlias] | None = None
 
+        # Stack of type variables that have been removed from current class because they
+        # cannot be bound unambiguously. This can happen if a (regular) type variable
+        # with a default follows a type variable tuple.
+        self.removed_type_vars: list[list[TypeVarType]] = [[]]
+
     # mypyc doesn't properly handle implementing an abstractproperty
-    # with a regular attribute so we make them properties
+    # with a regular attribute, so we make them properties
     @property
     def type(self) -> TypeInfo | None:
         return self._type
@@ -1151,7 +1156,8 @@ def prepare_method_signature(self, func: FuncDef, info: TypeInfo, has_self_type:
                         leading_type = fill_typevars(info)
                     if func.is_class or func.name == "__new__":
                         leading_type = self.class_type(leading_type)
-                    func.type = replace_implicit_first_type(functype, leading_type)
+                    if not has_placeholder(leading_type):
+                        func.type = replace_implicit_first_type(functype, leading_type)
                 elif has_self_type and isinstance(func.unanalyzed_type, CallableType):
                     if not isinstance(get_proper_type(func.unanalyzed_type.arg_types[0]), AnyType):
                         if self.is_expected_self_type(
@@ -1836,8 +1842,9 @@ def visit_class_def(self, defn: ClassDef) -> None:
             if self.push_type_args(defn.type_args, defn) is None:
                 self.mark_incomplete(defn.name, defn)
                 return
-
+            self.removed_type_vars.append([])
             self.analyze_class(defn)
+            self.removed_type_vars.pop()
             self.pop_type_args(defn.type_args)
         self.incomplete_type_stack.pop()
 
@@ -2084,7 +2091,21 @@ def check_type_alias_bases(self, bases: list[Expression]) -> None:
                 )
 
     def setup_type_vars(self, defn: ClassDef, tvar_defs: list[TypeVarLikeType]) -> None:
-        defn.type_vars = tvar_defs
+        seen_tvt = False
+        valid_tvar_defs = []
+        for tv in tvar_defs:
+            if seen_tvt and isinstance(tv, TypeVarType) and tv.has_default():
+                self.fail(
+                    message_registry.NO_DEFAULT_AFTER_TYPEVAR_TUPLE, defn, code=codes.TYPE_VAR
+                )
+                # Remove the ambiguous type variable, and record it, so that we can replace
+                # all its uses with Any.
+                self.removed_type_vars[-1].append(tv)
+                continue
+            if isinstance(tv, TypeVarTupleType):
+                seen_tvt = True
+            valid_tvar_defs.append(tv)
+        defn.type_vars = valid_tvar_defs
         defn.info.type_vars = []
         # we want to make sure any additional logic in add_type_vars gets run
         defn.info.add_type_vars()
@@ -4017,6 +4038,28 @@ def analyze_alias(
                 with self.allow_unbound_tvars_set():
                     rvalue.accept(self)
 
+            new_tvar_defs = []
+            erase_tvar_defs = []
+            variadic = False
+            for td in tvar_defs:
+                if variadic and isinstance(td, TypeVarType) and td.has_default():
+                    self.fail(
+                        message_registry.NO_DEFAULT_AFTER_TYPEVAR_TUPLE,
+                        rvalue,
+                        code=codes.TYPE_VAR,
+                    )
+                    # Remove the ambiguous type variable, and record it, so that we can
+                    # replace all its uses with Any.
+                    erase_tvar_defs.append(td)
+                    continue
+                if isinstance(td, TypeVarTupleType):
+                    # There can be only one variadic variable at most,
+                    # the error is reported elsewhere.
+                    if variadic:
+                        continue
+                    variadic = True
+                new_tvar_defs.append(td)
+
             analyzed, depends_on = analyze_type_alias(
                 typ,
                 self,
@@ -4029,20 +4072,11 @@ def analyze_alias(
                 in_dynamic_func=dynamic,
                 global_scope=global_scope,
                 allowed_alias_tvars=tvar_defs,
+                erase_tvar_defs=erase_tvar_defs,
                 alias_type_params_names=all_declared_type_params_names,
                 python_3_12_type_alias=python_3_12_type_alias,
             )
 
-        # There can be only one variadic variable at most, the error is reported elsewhere.
-        new_tvar_defs = []
-        variadic = False
-        for td in tvar_defs:
-            if isinstance(td, TypeVarTupleType):
-                if variadic:
-                    continue
-                variadic = True
-            new_tvar_defs.append(td)
-
         indexed = bool(isinstance(typ, UnboundType) and (typ.args or typ.empty_tuple_index))
         default_depends = {}
         for _, tv in alias_type_vars:
@@ -5771,7 +5805,10 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None:
                 res = make_any_non_unimported(res)
             eager = self.is_func_scope()
             if isinstance(res, ProperType) and isinstance(res, Instance):
-                fix_instance(res, self.fail, self.note, disallow_any=False, options=self.options)
+                if not validate_instance(res, self.fail, indexed):
+                    fix_instance(
+                        res, self.fail, self.note, disallow_any=False, options=self.options
+                    )
             alias_node = TypeAlias(
                 res,
                 self.qualified_name(s.name.name),
@@ -6396,7 +6433,7 @@ def analyze_comp_for(self, expr: GeneratorExpr | DictionaryComprehension) -> Non
             if i > 0:
                 sequence.accept(self)
             # Bind index variables.
-            self.analyze_lvalue(index)
+            self.analyze_lvalue(index, is_index_var=True)
             for cond in conditions:
                 cond.accept(self)
 
@@ -7789,6 +7826,7 @@ def type_analyzer(
             prohibit_special_class_field_types=prohibit_special_class_field_types,
             allow_type_any=allow_type_any,
             analyzing_tvar_def=analyzing_tvar_def,
+            erase_tvar_defs=self.removed_type_vars[-1],
         )
         tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic())
         tpan.global_scope = not self.type and not self.function_stack