From c8120f4f3da48910a2d78a537821e8f20c99bd4e Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Mon, 1 Aug 2022 18:44:11 -0400 Subject: [PATCH 1/7] Type the return value of lazy translation functions as Promise. The return value of the lazy translation functions is a proxied `Promise` object. https://github.com/django/django/blob/3.2.6/django/utils/translation/__init__.py#L135-L221. Signed-off-by: Zixuan James Li --- django-stubs/utils/translation/__init__.pyi | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/django-stubs/utils/translation/__init__.pyi b/django-stubs/utils/translation/__init__.pyi index a867ca686..abef8dd60 100644 --- a/django-stubs/utils/translation/__init__.pyi +++ b/django-stubs/utils/translation/__init__.pyi @@ -4,6 +4,7 @@ from contextlib import ContextDecorator from typing import Any, Callable, Optional, Type, Union from django.http.request import HttpRequest +from django.utils.functional import Promise LANGUAGE_SESSION_KEY: str @@ -34,13 +35,15 @@ def ungettext(singular: str, plural: str, number: float) -> str: ... def pgettext(context: str, message: str) -> str: ... def npgettext(context: str, singular: str, plural: str, number: int) -> str: ... -gettext_lazy = gettext -pgettext_lazy = pgettext +# lazy evaluated translation functions +def gettext_lazy(message: str) -> Promise: ... +def pgettext_lazy(context: str, message: str) -> Promise: ... +def ngettext_lazy(singular: str, plural: str, number: Union[int, str, None] = ...) -> Promise: ... +def npgettext_lazy(context: str, singular: str, plural: str, number: Union[int, str, None] = ...) -> Promise: ... + +ugettext_lazy = gettext_lazy +ungettext_lazy = ngettext_lazy -def ugettext_lazy(message: str) -> str: ... -def ngettext_lazy(singular: str, plural: str, number: Union[int, str, None] = ...) -> str: ... -def ungettext_lazy(singular: str, plural: str, number: Union[int, str, None] = ...) -> str: ... -def npgettext_lazy(context: str, singular: str, plural: str, number: Union[int, str, None] = ...) -> str: ... def activate(language: str) -> None: ... def deactivate() -> None: ... From 3c492941ed946d1a746631e6c29d2de63d748c91 Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Mon, 1 Aug 2022 19:00:47 -0400 Subject: [PATCH 2/7] Mark unicode translation functions for deprecation. https://docs.djangoproject.com/en/4.0/releases/4.0/#features-removed-in-4-0. Signed-off-by: Zixuan James Li --- django-stubs/utils/translation/__init__.pyi | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/django-stubs/utils/translation/__init__.pyi b/django-stubs/utils/translation/__init__.pyi index abef8dd60..cd9d50d01 100644 --- a/django-stubs/utils/translation/__init__.pyi +++ b/django-stubs/utils/translation/__init__.pyi @@ -27,11 +27,8 @@ class Trans: def __getattr__(self, real_name: Any): ... def gettext_noop(message: str) -> str: ... -def ugettext_noop(message: str) -> str: ... def gettext(message: str) -> str: ... -def ugettext(message: str) -> str: ... def ngettext(singular: str, plural: str, number: float) -> str: ... -def ungettext(singular: str, plural: str, number: float) -> str: ... def pgettext(context: str, message: str) -> str: ... def npgettext(context: str, singular: str, plural: str, number: int) -> str: ... @@ -41,6 +38,12 @@ def pgettext_lazy(context: str, message: str) -> Promise: ... def ngettext_lazy(singular: str, plural: str, number: Union[int, str, None] = ...) -> Promise: ... def npgettext_lazy(context: str, singular: str, plural: str, number: Union[int, str, None] = ...) -> Promise: ... +# NOTE: These translation functions are deprecated and removed in Django 4.0. We should remove them when we drop +# support for 3.2 +def ugettext_noop(message: str) -> str: ... +def ugettext(message: str) -> str: ... +def ungettext(singular: str, plural: str, number: float) -> str: ... + ugettext_lazy = gettext_lazy ungettext_lazy = ngettext_lazy From 48c3dba2dd5bcea1e43f90c916584c232e039190 Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Tue, 28 Jun 2022 17:54:34 -0400 Subject: [PATCH 3/7] Add proxied functions for Promise. Although there is nothing defined in `Promise` itself, the only instances of `Promise` are created by the `lazy` function, with magic methods defined on it. https://github.com/django/django/blob/3.2.6/django/utils/functional.py#L84-L191. Signed-off-by: Zixuan James Li --- django-stubs/utils/functional.pyi | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/django-stubs/utils/functional.pyi b/django-stubs/utils/functional.pyi index 0f1e3495a..8fb674466 100644 --- a/django-stubs/utils/functional.pyi +++ b/django-stubs/utils/functional.pyi @@ -15,7 +15,16 @@ class cached_property(Generic[_T]): @overload def __get__(self, instance: object, cls: Type[Any] = ...) -> _T: ... -class Promise: ... +# Promise is only subclassed by a proxy class defined in the lazy function +# so it makes sense for it to have all the methods available in that proxy class +class Promise: + def __init__(self, args: Any, kw: Any) -> None: ... + def __reduce__(self) -> Tuple[Any, Tuple[Any]]: ... + def __lt__(self, other: Any) -> bool: ... + def __mod__(self, rhs: Any) -> Any: ... + def __add__(self, other: Any) -> Any: ... + def __radd__(self, other: Any) -> Any: ... + def __deepcopy__(self, memo: Any): ... _C = TypeVar("_C", bound=Callable) From 3f39f0212e7e5c911e3dd5182b70a8f43cf6c2e1 Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Mon, 8 Aug 2022 11:04:54 -0400 Subject: [PATCH 4/7] Add _StrPromise as a special type for Promise objects for str. This allows the user to access methods defined on lazy strings while still letting mypy be aware of that they are not instances of `str`. The definitions for some of the magic methods are pulled from typeshed. We need those definitions in the stubs so that `_StrPromise` objects will work properly with operators, as refining operator types is tricky with the mypy plugins API. The rest of the methods will be covered by an attribute hook. Signed-off-by: Zixuan James Li --- django-stubs/utils/functional.pyi | 21 ++++++++++++++++++--- django-stubs/utils/translation/__init__.pyi | 10 +++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/django-stubs/utils/functional.pyi b/django-stubs/utils/functional.pyi index 8fb674466..2f3ce00de 100644 --- a/django-stubs/utils/functional.pyi +++ b/django-stubs/utils/functional.pyi @@ -1,8 +1,8 @@ from functools import wraps as wraps # noqa: F401 -from typing import Any, Callable, Generic, List, Optional, Tuple, Type, TypeVar, Union, overload +from typing import Any, Callable, Generic, List, Optional, Sequence, Tuple, Type, TypeVar, Union, overload from django.db.models.base import Model -from typing_extensions import Protocol +from typing_extensions import Protocol, SupportsIndex _T = TypeVar("_T") @@ -26,10 +26,25 @@ class Promise: def __radd__(self, other: Any) -> Any: ... def __deepcopy__(self, memo: Any): ... +class _StrPromise(Promise, Sequence[str]): + def __add__(self, __s: str) -> str: ... + # Incompatible with Sequence.__contains__ + def __contains__(self, __o: str) -> bool: ... # type: ignore[override] + def __ge__(self, __x: str) -> bool: ... + def __getitem__(self, __i: SupportsIndex | slice) -> str: ... + def __gt__(self, __x: str) -> bool: ... + def __le__(self, __x: str) -> bool: ... + # __len__ needed here because it defined abstract in Sequence[str] + def __len__(self) -> int: ... + def __lt__(self, __x: str) -> bool: ... + def __mod__(self, __x: Any) -> str: ... + def __mul__(self, __n: SupportsIndex) -> str: ... + def __rmul__(self, __n: SupportsIndex) -> str: ... + _C = TypeVar("_C", bound=Callable) def lazy(func: _C, *resultclasses: Any) -> _C: ... -def lazystr(text: Any) -> str: ... +def lazystr(text: Any) -> _StrPromise: ... def keep_lazy(*resultclasses: Any) -> Callable: ... def keep_lazy_text(func: Callable) -> Callable: ... diff --git a/django-stubs/utils/translation/__init__.pyi b/django-stubs/utils/translation/__init__.pyi index cd9d50d01..d82de72c9 100644 --- a/django-stubs/utils/translation/__init__.pyi +++ b/django-stubs/utils/translation/__init__.pyi @@ -4,7 +4,7 @@ from contextlib import ContextDecorator from typing import Any, Callable, Optional, Type, Union from django.http.request import HttpRequest -from django.utils.functional import Promise +from django.utils.functional import _StrPromise LANGUAGE_SESSION_KEY: str @@ -33,10 +33,10 @@ def pgettext(context: str, message: str) -> str: ... def npgettext(context: str, singular: str, plural: str, number: int) -> str: ... # lazy evaluated translation functions -def gettext_lazy(message: str) -> Promise: ... -def pgettext_lazy(context: str, message: str) -> Promise: ... -def ngettext_lazy(singular: str, plural: str, number: Union[int, str, None] = ...) -> Promise: ... -def npgettext_lazy(context: str, singular: str, plural: str, number: Union[int, str, None] = ...) -> Promise: ... +def gettext_lazy(message: str) -> _StrPromise: ... +def pgettext_lazy(context: str, message: str) -> _StrPromise: ... +def ngettext_lazy(singular: str, plural: str, number: Union[int, str, None] = ...) -> _StrPromise: ... +def npgettext_lazy(context: str, singular: str, plural: str, number: Union[int, str, None] = ...) -> _StrPromise: ... # NOTE: These translation functions are deprecated and removed in Django 4.0. We should remove them when we drop # support for 3.2 From d8f60bb89d2b95014a2e2fa0b6a102d12e68e937 Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Mon, 8 Aug 2022 12:18:48 -0400 Subject: [PATCH 5/7] Implement _StrPromise attribute hook. This implements an attribute hook that provides type information for methods that are available on `builtins.str` for `_StrPromise` except the supported operators. This allows us to avoid copying stubs from the builtins for all supported methods on `str`. Signed-off-by: Zixuan James Li --- django-stubs/utils/functional.pyi | 2 + mypy_django_plugin/lib/fullnames.py | 2 + mypy_django_plugin/main.py | 4 ++ mypy_django_plugin/transformers/functional.py | 40 +++++++++++++++++++ tests/typecheck/utils/test_functional.yml | 31 ++++++++++++++ 5 files changed, 79 insertions(+) create mode 100644 mypy_django_plugin/transformers/functional.py diff --git a/django-stubs/utils/functional.pyi b/django-stubs/utils/functional.pyi index 2f3ce00de..677582bc1 100644 --- a/django-stubs/utils/functional.pyi +++ b/django-stubs/utils/functional.pyi @@ -40,6 +40,8 @@ class _StrPromise(Promise, Sequence[str]): def __mod__(self, __x: Any) -> str: ... def __mul__(self, __n: SupportsIndex) -> str: ... def __rmul__(self, __n: SupportsIndex) -> str: ... + # Mypy requires this for the attribute hook to take effect + def __getattribute__(self, __name: str) -> Any: ... _C = TypeVar("_C", bound=Callable) diff --git a/mypy_django_plugin/lib/fullnames.py b/mypy_django_plugin/lib/fullnames.py index bb530d1df..17027aa22 100644 --- a/mypy_django_plugin/lib/fullnames.py +++ b/mypy_django_plugin/lib/fullnames.py @@ -41,3 +41,5 @@ F_EXPRESSION_FULLNAME = "django.db.models.expressions.F" ANY_ATTR_ALLOWED_CLASS_FULLNAME = "django_stubs_ext.AnyAttrAllowed" + +STR_PROMISE_FULLNAME = "django.utils.functional._StrPromise" diff --git a/mypy_django_plugin/main.py b/mypy_django_plugin/main.py index ca4b680f0..dde0b20d5 100644 --- a/mypy_django_plugin/main.py +++ b/mypy_django_plugin/main.py @@ -22,6 +22,7 @@ from mypy_django_plugin.django.context import DjangoContext from mypy_django_plugin.lib import fullnames, helpers from mypy_django_plugin.transformers import fields, forms, init_create, meta, querysets, request, settings +from mypy_django_plugin.transformers.functional import resolve_str_promise_attribute from mypy_django_plugin.transformers.managers import ( create_new_manager_class_from_from_queryset_method, resolve_manager_method, @@ -285,6 +286,9 @@ def get_attribute_hook(self, fullname: str) -> Optional[Callable[[AttributeConte ): return resolve_manager_method + if info and info.has_base(fullnames.STR_PROMISE_FULLNAME): + return resolve_str_promise_attribute + return None def get_type_analyze_hook(self, fullname: str) -> Optional[Callable[[AnalyzeTypeContext], MypyType]]: diff --git a/mypy_django_plugin/transformers/functional.py b/mypy_django_plugin/transformers/functional.py new file mode 100644 index 000000000..9e73845da --- /dev/null +++ b/mypy_django_plugin/transformers/functional.py @@ -0,0 +1,40 @@ +from mypy.errorcodes import ATTR_DEFINED +from mypy.nodes import CallExpr, MemberExpr +from mypy.plugin import AttributeContext +from mypy.types import AnyType, CallableType +from mypy.types import Type as MypyType +from mypy.types import TypeOfAny + +from mypy_django_plugin.lib import helpers + + +def resolve_str_promise_attribute(ctx: AttributeContext) -> MypyType: + if isinstance(ctx.context, MemberExpr): + method_name = ctx.context.name + elif isinstance(ctx.context, CallExpr) and isinstance(ctx.context.callee, MemberExpr): + method_name = ctx.context.callee.name + else: + ctx.api.fail(f'Cannot resolve the attribute of "{ctx.type}"', ctx.context, code=ATTR_DEFINED) + return AnyType(TypeOfAny.from_error) + + str_info = helpers.lookup_fully_qualified_typeinfo(helpers.get_typechecker_api(ctx), f"builtins.str") + assert str_info is not None + method = str_info.get(method_name) + + if method is None or method.type is None: + ctx.api.fail(f'"{ctx.type}" has no attribute "{method_name}"', ctx.context, code=ATTR_DEFINED) + return AnyType(TypeOfAny.from_error) + + if isinstance(method.type, CallableType): + # The proxied str methods are only meant to be used as instance methods. + # We need to drop the first `self` argument in them. + assert method.type.arg_names[0] == "self" + return method.type.copy_modified( + arg_kinds=method.type.arg_kinds[1:], + arg_names=method.type.arg_names[1:], + arg_types=method.type.arg_types[1:], + ) + else: + # Not possible with `builtins.str`, but we have error handling for this anyway. + ctx.api.fail(f'"{method_name}" on "{ctx.type}" is not a method', ctx.context) + return AnyType(TypeOfAny.from_error) diff --git a/tests/typecheck/utils/test_functional.yml b/tests/typecheck/utils/test_functional.yml index eb4417bfa..0fe0c8f92 100644 --- a/tests/typecheck/utils/test_functional.yml +++ b/tests/typecheck/utils/test_functional.yml @@ -16,3 +16,34 @@ f = Foo() reveal_type(f.attr) # N: Revealed type is "builtins.list[builtins.str]" f.attr.name # E: "List[str]" has no attribute "name" + +- case: str_promise_proxy + main: | + from typing import Union + + from django.utils.functional import Promise, lazystr, _StrPromise + + s = lazystr("asd") + + reveal_type(s) # N: Revealed type is "django.utils.functional._StrPromise" + + reveal_type(s.format("asd")) # N: Revealed type is "builtins.str" + reveal_type(s.capitalize()) # N: Revealed type is "builtins.str" + reveal_type(s.swapcase) # N: Revealed type is "def () -> builtins.str" + reveal_type(s.__getnewargs__) # N: Revealed type is "def () -> Tuple[builtins.str]" + s.nonsense # E: "django.utils.functional._StrPromise" has no attribute "nonsense" + f: Union[_StrPromise, str] + reveal_type(f.format("asd")) # N: Revealed type is "builtins.str" + + reveal_type(s + "bar") # N: Revealed type is "builtins.str" + reveal_type("foo" + s) # N: Revealed type is "Any" + reveal_type(s % "asd") # N: Revealed type is "builtins.str" + + def foo(content: str) -> None: + ... + + def bar(content: Promise) -> None: + ... + + foo(s) # E: Argument 1 to "foo" has incompatible type "_StrPromise"; expected "str" + bar(s) From 6e66d4e05f65b368b61bb976a5c1bd11f145ddc0 Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Fri, 19 Aug 2022 17:27:16 -0400 Subject: [PATCH 6/7] Allow message being a _StrPromise object for RegexValidator. One intended usage of lazystr is to postpone the translation of the error message of a validation error. It is possible that we pass a Promise (specifically _StrPromise) and only evaluate it when a ValidationError is raised. Signed-off-by: Zixuan James Li --- django-stubs/core/validators.pyi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/django-stubs/core/validators.pyi b/django-stubs/core/validators.pyi index 89653a556..d5b24d44b 100644 --- a/django-stubs/core/validators.pyi +++ b/django-stubs/core/validators.pyi @@ -3,6 +3,7 @@ from re import RegexFlag from typing import Any, Callable, Collection, Dict, List, Optional, Pattern, Sequence, Sized, Tuple, Union from django.core.files.base import File +from django.utils.functional import _StrPromise EMPTY_VALUES: Any @@ -11,14 +12,14 @@ _ValidatorCallable = Callable[[Any], None] class RegexValidator: regex: _Regex = ... # Pattern[str] on instance, but may be str on class definition - message: str = ... + message: Union[str, _StrPromise] = ... code: str = ... inverse_match: bool = ... flags: int = ... def __init__( self, regex: Optional[_Regex] = ..., - message: Optional[str] = ..., + message: Union[str, _StrPromise, None] = ..., code: Optional[str] = ..., inverse_match: Optional[bool] = ..., flags: Optional[RegexFlag] = ..., From bf68f6c0ed1d885a0d0760610377741e82634983 Mon Sep 17 00:00:00 2001 From: Zixuan James Li Date: Sat, 27 Aug 2022 13:57:15 -0400 Subject: [PATCH 7/7] Refactor _StrPromise attribtue hook with analyze_member_access. Signed-off-by: Zixuan James Li --- mypy_django_plugin/transformers/functional.py | 35 ++++++++----------- tests/typecheck/utils/test_functional.yml | 4 ++- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/mypy_django_plugin/transformers/functional.py b/mypy_django_plugin/transformers/functional.py index 9e73845da..ebea7cc40 100644 --- a/mypy_django_plugin/transformers/functional.py +++ b/mypy_django_plugin/transformers/functional.py @@ -1,7 +1,8 @@ +from mypy.checkmember import analyze_member_access from mypy.errorcodes import ATTR_DEFINED from mypy.nodes import CallExpr, MemberExpr from mypy.plugin import AttributeContext -from mypy.types import AnyType, CallableType +from mypy.types import AnyType, Instance from mypy.types import Type as MypyType from mypy.types import TypeOfAny @@ -19,22 +20,16 @@ def resolve_str_promise_attribute(ctx: AttributeContext) -> MypyType: str_info = helpers.lookup_fully_qualified_typeinfo(helpers.get_typechecker_api(ctx), f"builtins.str") assert str_info is not None - method = str_info.get(method_name) - - if method is None or method.type is None: - ctx.api.fail(f'"{ctx.type}" has no attribute "{method_name}"', ctx.context, code=ATTR_DEFINED) - return AnyType(TypeOfAny.from_error) - - if isinstance(method.type, CallableType): - # The proxied str methods are only meant to be used as instance methods. - # We need to drop the first `self` argument in them. - assert method.type.arg_names[0] == "self" - return method.type.copy_modified( - arg_kinds=method.type.arg_kinds[1:], - arg_names=method.type.arg_names[1:], - arg_types=method.type.arg_types[1:], - ) - else: - # Not possible with `builtins.str`, but we have error handling for this anyway. - ctx.api.fail(f'"{method_name}" on "{ctx.type}" is not a method', ctx.context) - return AnyType(TypeOfAny.from_error) + str_type = Instance(str_info, []) + return analyze_member_access( + method_name, + str_type, + ctx.context, + is_lvalue=False, + is_super=False, + # operators are already handled with magic methods defined in the stubs for _StrPromise + is_operator=False, + msg=ctx.api.msg, + original_type=ctx.type, + chk=helpers.get_typechecker_api(ctx), + ) diff --git a/tests/typecheck/utils/test_functional.yml b/tests/typecheck/utils/test_functional.yml index 0fe0c8f92..cb9ac299e 100644 --- a/tests/typecheck/utils/test_functional.yml +++ b/tests/typecheck/utils/test_functional.yml @@ -31,9 +31,11 @@ reveal_type(s.capitalize()) # N: Revealed type is "builtins.str" reveal_type(s.swapcase) # N: Revealed type is "def () -> builtins.str" reveal_type(s.__getnewargs__) # N: Revealed type is "def () -> Tuple[builtins.str]" - s.nonsense # E: "django.utils.functional._StrPromise" has no attribute "nonsense" + s.nonsense # E: "_StrPromise" has no attribute "nonsense" f: Union[_StrPromise, str] reveal_type(f.format("asd")) # N: Revealed type is "builtins.str" + reveal_type(f + "asd") # N: Revealed type is "builtins.str" + reveal_type("asd" + f) # N: Revealed type is "Union[Any, builtins.str]" reveal_type(s + "bar") # N: Revealed type is "builtins.str" reveal_type("foo" + s) # N: Revealed type is "Any"