variance modifiers

Author: KotlinIsland

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## [Unreleased]
 ### Added
 - `collections.User*` should have `__repr__`
+- explicit and use-site variance modifiers `In`/`Out`/`InOut`
 
 ## [2.8.1]
 ### Fixes

docs/source/based_features.rst

@@ -25,6 +25,12 @@ Using the ``&`` operator or ``basedtyping.Intersection`` you can denote intersec
         x.reset()
         x.add("first")
 
+Explicit and Use-Site variance
+------------------------------
+
+it is frequently desirable to explicitly declare the variance of type paramters on types and classes
+
+
 Type Joins
 ----------
 

mypy/checkmember.py

@@ -17,6 +17,8 @@
     ARG_POS,
     ARG_STAR,
     ARG_STAR2,
+    CONTRAVARIANT,
+    COVARIANT,
     EXCLUDED_ENUM_ATTRIBUTES,
     SYMBOL_FUNCBASE_TYPES,
     Context,
@@ -811,7 +813,9 @@ def analyze_var(
             mx.msg.cant_assign_to_classvar(name, mx.context)
         t = freshen_all_functions_type_vars(typ)
         t = expand_self_type_if_needed(t, mx, var, original_itype)
-        t = expand_type_by_instance(t, itype)
+        t = expand_type_by_instance(
+            t, itype, use_variance=CONTRAVARIANT if mx.is_lvalue else COVARIANT
+        )
         freeze_all_type_vars(t)
         result = t
         typ = get_proper_type(typ)

mypy/expandtype.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
-from typing import Final, Iterable, Mapping, Sequence, TypeVar, cast, overload
+from contextlib import contextmanager
+from typing import Final, Generator, Iterable, Mapping, Sequence, TypeVar, cast, overload
 
-from mypy.nodes import ARG_STAR, FakeInfo, Var
+from mypy.nodes import ARG_STAR, CONTRAVARIANT, COVARIANT, FakeInfo, Var
 from mypy.state import state
 from mypy.types import (
     ANY_STRATEGY,
@@ -38,6 +39,7 @@
     UninhabitedType,
     UnionType,
     UnpackType,
+    VarianceModifier,
     flatten_nested_unions,
     get_proper_type,
     split_with_prefix_and_suffix,
@@ -53,37 +55,49 @@
 
 
 @overload
-def expand_type(typ: CallableType, env: Mapping[TypeVarId, Type]) -> CallableType: ...
+def expand_type(
+    typ: CallableType, env: Mapping[TypeVarId, Type], *, variance: int | None = ...
+) -> CallableType: ...
 
 
 @overload
-def expand_type(typ: ProperType, env: Mapping[TypeVarId, Type]) -> ProperType: ...
+def expand_type(
+    typ: ProperType, env: Mapping[TypeVarId, Type], *, variance: int | None = ...
+) -> ProperType: ...
 
 
 @overload
-def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type: ...
+def expand_type(
+    typ: Type, env: Mapping[TypeVarId, Type], *, variance: int | None = ...
+) -> Type: ...
 
 
-def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type:
+def expand_type(typ: Type, env: Mapping[TypeVarId, Type], *, variance=None) -> Type:
     """Substitute any type variable references in a type given by a type
     environment.
     """
-    return typ.accept(ExpandTypeVisitor(env))
+    return typ.accept(ExpandTypeVisitor(env, variance=variance))
 
 
 @overload
-def expand_type_by_instance(typ: CallableType, instance: Instance) -> CallableType: ...
+def expand_type_by_instance(
+    typ: CallableType, instance: Instance, *, use_variance: int | None = ...
+) -> CallableType: ...
 
 
 @overload
-def expand_type_by_instance(typ: ProperType, instance: Instance) -> ProperType: ...
+def expand_type_by_instance(
+    typ: ProperType, instance: Instance, *, use_variance: int | None = ...
+) -> ProperType: ...
 
 
 @overload
-def expand_type_by_instance(typ: Type, instance: Instance) -> Type: ...
+def expand_type_by_instance(
+    typ: Type, instance: Instance, *, use_variance: int | None = ...
+) -> Type: ...
 
 
-def expand_type_by_instance(typ: Type, instance: Instance) -> Type:
+def expand_type_by_instance(typ: Type, instance: Instance, use_variance=None) -> Type:
     """Substitute type variables in type using values from an Instance.
     Type variables are considered to be bound by the class declaration."""
     if not instance.args and not instance.type.has_type_var_tuple_type:
@@ -108,12 +122,11 @@ def expand_type_by_instance(typ: Type, instance: Instance) -> Type:
         else:
             tvars = tuple(instance.type.defn.type_vars)
             instance_args = instance.args
-
         for binder, arg in zip(tvars, instance_args):
             assert isinstance(binder, TypeVarLikeType)
             variables[binder.id] = arg
 
-        return expand_type(typ, variables)
+        return expand_type(typ, variables, variance=use_variance)
 
 
 F = TypeVar("F", bound=FunctionLike)
@@ -181,10 +194,28 @@ class ExpandTypeVisitor(TrivialSyntheticTypeTranslator):
 
     variables: Mapping[TypeVarId, Type]  # TypeVar id -> TypeVar value
 
-    def __init__(self, variables: Mapping[TypeVarId, Type]) -> None:
+    def __init__(
+        self, variables: Mapping[TypeVarId, Type], *, variance: int | None = None
+    ) -> None:
         super().__init__()
         self.variables = variables
         self.recursive_tvar_guard: dict[TypeVarId, Type | None] = {}
+        self.variance = variance
+        self.using_variance: int | None = None
+
+    @contextmanager
+    def in_variance(self) -> Generator[None]:
+        using_variance = self.using_variance
+        self.using_variance = CONTRAVARIANT
+        yield
+        self.using_variance = using_variance
+
+    @contextmanager
+    def out_variance(self) -> Generator[None]:
+        using_variance = self.using_variance
+        self.using_variance = COVARIANT
+        yield
+        self.using_variance = using_variance
 
     def visit_unbound_type(self, t: UnboundType) -> Type:
         return t
@@ -238,6 +269,18 @@ def visit_type_var(self, t: TypeVarType) -> Type:
         if t.id.is_self():
             t = t.copy_modified(upper_bound=t.upper_bound.accept(self))
         repl = self.variables.get(t.id, t)
+        use_site_variance = repl.variance if isinstance(repl, VarianceModifier) else None
+        positional_variance = self.using_variance or self.variance
+        if (
+            positional_variance is not None
+            and use_site_variance is not None
+            and positional_variance != use_site_variance
+        ):
+            repl = (
+                t.upper_bound.accept(self)
+                if positional_variance == COVARIANT
+                else UninhabitedType()
+            )
         if isinstance(repl, ProperType) and isinstance(repl, Instance):
             # TODO: do we really need to do this?
             # If I try to remove this special-casing ~40 tests fail on reveal_type().
@@ -414,10 +457,15 @@ def visit_callable_type(self, t: CallableType) -> CallableType:
             needs_normalization = True
             arg_types = self.interpolate_args_for_unpack(t, var_arg.typ)
         else:
-            arg_types = self.expand_types(t.arg_types)
+            with self.in_variance():
+                arg_types = self.expand_types(t.arg_types)
+        with self.out_variance():
+            ret_type = t.ret_type.accept(self)
+            if isinstance(ret_type, VarianceModifier):
+                ret_type = ret_type.value
         expanded = t.copy_modified(
             arg_types=arg_types,
-            ret_type=t.ret_type.accept(self),
+            ret_type=ret_type,
             type_guard=t.type_guard and cast(TypeGuardType, t.type_guard.accept(self)),
             type_is=(t.type_is.accept(self) if t.type_is is not None else None),
         )
@@ -538,7 +586,10 @@ def visit_typeguard_type(self, t: TypeGuardType) -> Type:
     def expand_types(self, types: Iterable[Type]) -> list[Type]:
         a: list[Type] = []
         for t in types:
-            a.append(t.accept(self))
+            typ = t.accept(self)
+            if isinstance(typ, VarianceModifier):
+                typ = typ.value
+            a.append(typ)
         return a
 
 

mypy/message_registry.py

@@ -113,11 +113,11 @@ def with_additional_msg(self, info: str) -> ErrorMessage:
 )
 FORMAT_REQUIRES_MAPPING: Final = "Format requires a mapping"
 RETURN_TYPE_CANNOT_BE_CONTRAVARIANT: Final = ErrorMessage(
-    "This usage of this contravariant type variable is unsafe as a return type.",
+    "This usage of this contravariant type variable is unsafe as a return type",
     codes.UNSAFE_VARIANCE,
 )
 FUNCTION_PARAMETER_CANNOT_BE_COVARIANT: Final = ErrorMessage(
-    "This usage of this covariant type variable is unsafe as an input parameter.",
+    "This usage of this covariant type variable is unsafe as an input parameter",
     codes.UNSAFE_VARIANCE,
 )
 UNSAFE_VARIANCE_NOTE = ErrorMessage(

mypy/messages.py

@@ -93,6 +93,7 @@
     UninhabitedType,
     UnionType,
     UnpackType,
+    VarianceModifier,
     flatten_nested_unions,
     get_proper_type,
     get_proper_types,
@@ -2676,6 +2677,9 @@ def format_literal_value(typ: LiteralType) -> str:
                 type_str += f"[{format_list(typ.args)}]"
         return type_str
 
+    if isinstance(typ, VarianceModifier):
+        return typ.render(format)
+
     # TODO: always mention type alias names in errors.
     typ = get_proper_type(typ)
 

mypy/plugins/proper_plugin.py

@@ -107,6 +107,7 @@ def is_special_target(right: ProperType) -> bool:
             "mypy.types.DeletedType",
             "mypy.types.RequiredType",
             "mypy.types.ReadOnlyType",
+            "mypy.types.VarianceModifier",
         ):
             # Special case: these are not valid targets for a type alias and thus safe.
             # TODO: introduce a SyntheticType base to simplify this?

mypy/semanal.py

@@ -298,6 +298,7 @@
     UnionType,
     UnpackType,
     UntypedType,
+    VarianceModifier,
     get_proper_type,
     get_proper_types,
     has_type_vars,
@@ -1962,14 +1963,25 @@ def analyze_type_param(
         self, type_param: TypeParam, context: Context
     ) -> TypeVarLikeExpr | None:
         fullname = self.qualified_name(type_param.name)
+        variance = VARIANCE_NOT_READY
+        upper_bound = None
         if type_param.upper_bound:
-            upper_bound = self.anal_type(type_param.upper_bound, allow_placeholder=True)
-            # TODO: we should validate the upper bound is valid for a given kind.
-            if upper_bound is None:
-                # This and below copies special-casing for old-style type variables, that
-                # is equally necessary for new-style classes to break a vicious circle.
-                upper_bound = PlaceholderType(None, [], context.line)
-        else:
+            variance_or_bound = self.anal_type(
+                type_param.upper_bound,
+                allow_placeholder=True,
+                is_type_var_bound=isinstance(context, (ClassDef, TypeAliasStmt)),
+            )
+            if isinstance(variance_or_bound, VarianceModifier):
+                variance = variance_or_bound.variance
+                upper_bound = variance_or_bound._value
+            else:
+                upper_bound = variance_or_bound
+                # TODO: we should validate the upper bound is valid for a given kind.
+                if upper_bound is None:
+                    # This and below copies special-casing for old-style type variables, that
+                    # is equally necessary for new-style classes to break a vicious circle.
+                    upper_bound = PlaceholderType(None, [], context.line)
+        if upper_bound is None:
             if type_param.kind == TYPE_VAR_TUPLE_KIND:
                 upper_bound = self.named_type("builtins.tuple", [self.object_type()])
             else:
@@ -2012,7 +2024,7 @@ def analyze_type_param(
                 values=values,
                 upper_bound=upper_bound,
                 default=default,
-                variance=VARIANCE_NOT_READY,
+                variance=variance,
                 is_new_style=True,
                 line=context.line,
             )
@@ -6261,6 +6273,7 @@ def analyze_type_application_args(self, expr: IndexExpr) -> list[Type] | None:
                 allow_param_spec_literals=has_param_spec,
                 allow_unpack=allow_unpack,
                 runtime=True,
+                nested=True,
             )
             if analyzed is None:
                 return None
@@ -7545,6 +7558,7 @@ def type_analyzer(
         report_invalid_types: bool = True,
         prohibit_self_type: str | None = None,
         allow_type_any: bool = False,
+        is_type_var_bound=False,
     ) -> TypeAnalyser:
         if tvar_scope is None:
             tvar_scope = self.tvar_scope
@@ -7564,6 +7578,7 @@ def type_analyzer(
             allow_unpack=allow_unpack,
             prohibit_self_type=prohibit_self_type,
             allow_type_any=allow_type_any,
+            is_type_var_bound=is_type_var_bound,
         )
         tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic())
         tpan.global_scope = not self.type and not self.function_stack
@@ -7589,6 +7604,8 @@ def anal_type(
         prohibit_self_type: str | None = None,
         allow_type_any: bool = False,
         runtime: bool | None = None,
+        is_type_var_bound=False,
+        nested=False,
     ) -> Type | None:
         """Semantically analyze a type.
 
@@ -7624,13 +7641,16 @@ def anal_type(
             report_invalid_types=report_invalid_types,
             prohibit_self_type=prohibit_self_type,
             allow_type_any=allow_type_any,
+            is_type_var_bound=is_type_var_bound,
         )
         if not a.api.is_stub_file and runtime:
             a.always_allow_new_syntax = False
         if runtime is False:
             a.always_allow_new_syntax = True
         if self.is_stub_file:
             a.always_allow_new_syntax = True
+        if nested:
+            a.nesting_level += 1
         tag = self.track_incomplete_refs()
         typ = typ.accept(a)
         if self.found_incomplete_ref(tag):

mypy/subtypes.py

@@ -67,6 +67,7 @@
     UnionType,
     UnpackType,
     UntypedType,
+    VarianceModifier,
     find_unpack_in_list,
     get_proper_type,
     is_named_instance,
@@ -378,6 +379,8 @@ def check_type_parameter(
         p_left = get_proper_type(left)
         if isinstance(p_left, UninhabitedType) and p_left.ambiguous:
             variance = COVARIANT
+    if isinstance(right, VarianceModifier):
+        variance = right.variance
     # If variance hasn't been inferred yet, we are lenient and default to
     # covariance. This shouldn't happen often, but it's very difficult to
     # avoid these cases altogether.

mypy/type_visitor.py

@@ -49,6 +49,7 @@
     UninhabitedType,
     UnionType,
     UnpackType,
+    VarianceModifier,
     get_proper_type,
 )
 
@@ -87,6 +88,10 @@ def visit_erased_type(self, t: ErasedType) -> T:
     def visit_deleted_type(self, t: DeletedType) -> T:
         pass
 
+    def visit_variance_modifier(self, t: VarianceModifier) -> T:
+        assert t.value
+        return t.value.accept(self)
+
     @abstractmethod
     def visit_type_var(self, t: TypeVarType) -> T:
         pass
@@ -245,6 +250,9 @@ def visit_instance(self, t: Instance) -> Type:
         result.metadata = t.metadata
         return result
 
+    def visit_variance_modifier(self, t: VarianceModifier) -> Type:
+        return VarianceModifier(t.variance, t.value.accept(self))
+
     def visit_type_var(self, t: TypeVarType) -> Type:
         return t