diff --git a/spec-draft.rst b/spec-draft.rst index b14f9b7..006ddb7 100644 --- a/spec-draft.rst +++ b/spec-draft.rst @@ -191,6 +191,28 @@ We can put more in, but this is what typescript has. We can actually implement the case functions in terms of them and a bunch of conditionals, but shouldn't (especially if we want it to work for all unicode!). +--------- +Annotated +--------- + +Libraries like FastAPI use annotations heavily, and we would like to be able to use annotations to drive type-level computation decision making. + +We understand that this may be controversial, as currently Annotated may be fully ignored by typecheckers. The operations proposed are: + +* ``GetAnnotations[T]`` - Fetch the annotations of a potentially Annotated type, as Literals. Examples:: + + GetAnnotations[Annotated[int, 'xxx']] = Literal['xxx'] + GetAnnotations[Annotated[int, 'xxx', 5]] = Literal['xxx', 5] + GetAnnotations[int] = Never + + +* ``DropAnnotations[T]`` - Drop the annotations of a potentially Annotated type. Examples:: + + DropAnnotations[Annotated[int, 'xxx']] = int + DropAnnotations[Annotated[int, 'xxx', 5]] = int + DropAnnotations[int] = int + + ------------------- String manipulation ------------------- diff --git a/tests/test_fastapilike_1.py b/tests/test_fastapilike_1.py new file mode 100644 index 0000000..819130b --- /dev/null +++ b/tests/test_fastapilike_1.py @@ -0,0 +1,265 @@ +import dataclasses +import enum +import textwrap + +from typing import Annotated, Callable, Literal, Union, Self + +from typemap.type_eval import eval_typing +from typemap.typing import ( + NewProtocol, + Iter, + Attrs, + Is, + GetAnnotations, + DropAnnotations, + FromUnion, + GetType, + GetName, + GetQuals, + Member, + Members, + Param, +) + +from . import format_helper + + +class PropQuals(enum.StrEnum): + HIDDEN = "HIDDEN" + PRIMARY = "PRIMARY" + HAS_DEFAULT = "HAS_DEFAULT" + + +@dataclasses.dataclass(frozen=True) +class _Default: + val: object + + +type Hidden[T] = Annotated[T, Literal[PropQuals.HIDDEN]] +type Primary[T] = Annotated[T, Literal[PropQuals.PRIMARY]] +type HasDefault[T, default] = Annotated[ + T, _Default(default), Literal[PropQuals.HAS_DEFAULT] +] + + +#### + +type InitFnType[T] = Member[ + Literal["__init__"], + Callable[ + [ + Param[Literal["self"], Self], + *[ + Param[ + GetName[p], + DropAnnotations[GetType[p]], + Literal["keyword", "default"] + if Is[ + Literal[PropQuals.HAS_DEFAULT], + GetAnnotations[GetType[p]], + ] + else Literal["keyword"], + ] + for p in Iter[Attrs[T]] + ], + ], + None, + ], + Literal["ClassVar"], +] +type AddInit[T] = NewProtocol[ + InitFnType[T], + *[x for x in Iter[Members[T]]], +] + +"""TODO: + +We would really like to instead write: + +type AddInit[T] = NewProtocol[ + InitFnType[T], + *Members[T]], +] + +but we struggle here because typing wants to unpack the Members tuple +itself. I'm not sure if there is a nice way to resolve this. We +*could* make our consumers (NewProtocol etc) be more flexible about +these things but I don't think that is right. + +The frustrating thing is that it doesn't do much with the unpacked +version, just some checks! + +We could fix typing to allow it, and probably provide a hack around it +in the mean time. + +Lurr! Writing *this* gets past the typing checks (though we don't +support it yet): + +type AddInit[T] = NewProtocol[ + InitFnType[T], + *tuple[*Members[T]], +] +""" + +type AllOptional[T] = NewProtocol[ + *[ + Member[GetName[p], GetType[p] | None, GetQuals[p]] + for p in Iter[Attrs[T]] + ] +] + +# Strip `| None` from a type by iterating over its union components +# and filtering +type NotOptional[T] = Union[*[x for x in Iter[FromUnion[T]] if not Is[x, None]]] + +# Adjust an attribute type for use in Public below by dropping | None for +# primary keys and stripping all annotations. +type FixPublicType[T] = DropAnnotations[ + NotOptional[T] if Is[Literal[PropQuals.PRIMARY], GetAnnotations[T]] else T +] + +# Strip out everything that is Hidden and also make the primary key required +# Drop all the annotations, since this is for data getting returned to users +# from the DB, so we don't need default values. +type Public[T] = NewProtocol[ + *[ + Member[GetName[p], FixPublicType[GetType[p]], GetQuals[p]] + for p in Iter[Attrs[T]] + if not Is[Literal[PropQuals.HIDDEN], GetAnnotations[GetType[p]]] + ] +] + +# Create takes everything but the primary key and preserves defaults +type Create[T] = NewProtocol[ + *[ + Member[GetName[p], GetType[p], GetQuals[p]] + for p in Iter[Attrs[T]] + if not Is[Literal[PropQuals.PRIMARY], GetAnnotations[GetType[p]]] + ] +] + + +# Update takes everything but the primary key, but makes them all have +# None defaults +type Update[T] = NewProtocol[ + *[ + Member[ + GetName[p], + HasDefault[DropAnnotations[GetType[p]] | None, None], + GetQuals[p], + ] + for p in Iter[Attrs[T]] + if not Is[Literal[PropQuals.PRIMARY], GetAnnotations[GetType[p]]] + ] +] + + +#### + +# This is the FastAPI example code that we are trying to repair! +# Adapted from https://fastapi.tiangolo.com/tutorial/sql-databases/#heroupdate-the-data-model-to-update-a-hero +""" +class HeroBase(SQLModel): + name: str = Field(index=True) + age: int | None = Field(default=None, index=True) + + +class Hero(HeroBase, table=True): + id: int | None = Field(default=None, primary_key=True) + secret_name: str + + +class HeroPublic(HeroBase): + id: int + + +class HeroCreate(HeroBase): + secret_name: str + + +class HeroUpdate(HeroBase): + name: str | None = None + age: int | None = None + secret_name: str | None = None +""" + + +class Hero: + id: Primary[ + HasDefault[int | None, None] + ] # = Field(default=None, primary_key=True) + + name: str + age: HasDefault[int | None, None] # = Field(default=None, index=True) + + secret_name: Hidden[str] + + +####### + + +def test_eval_drop_optional_1(): + tgt = eval_typing(NotOptional[int | None]) + assert tgt is int + + +def test_fastapi_like_1(): + tgt = eval_typing(Public[Hero]) + fmt = format_helper.format_class(tgt) + + assert fmt == textwrap.dedent("""\ + class Public[tests.test_fastapilike_1.Hero]: + id: int + name: str + age: int | None + """) + + +def test_fastapi_like_2(): + tgt = eval_typing(Create[Hero]) + fmt = format_helper.format_class(tgt) + + assert fmt == textwrap.dedent("""\ + class Create[tests.test_fastapilike_1.Hero]: + name: str + age: typing.Annotated[int | None, _Default(val=None), typing.Literal[]] + secret_name: typing.Annotated[str, typing.Literal[]] + """) + + +def test_fastapi_like_3(): + tgt = eval_typing(Update[Hero]) + fmt = format_helper.format_class(tgt) + + assert fmt == textwrap.dedent("""\ + class Update[tests.test_fastapilike_1.Hero]: + name: typing.Annotated[str | None, _Default(val=None), typing.Literal[]] + age: typing.Annotated[int | None, _Default(val=None), typing.Literal[]] + secret_name: typing.Annotated[str | None, _Default(val=None), typing.Literal[]] + """) + + +def test_fastapi_like_4(): + tgt = eval_typing(AddInit[Public[Hero]]) + fmt = format_helper.format_class(tgt) + + assert fmt == textwrap.dedent("""\ + class AddInit[tests.test_fastapilike_1.Public[tests.test_fastapilike_1.Hero]]: + id: int + name: str + age: int | None + def __init__(self: Self, *, id: int, name: str, age: int | None) -> None: ... + """) + + +def test_fastapi_like_6(): + tgt = eval_typing(AddInit[Update[Hero]]) + fmt = format_helper.format_class(tgt) + + assert fmt == textwrap.dedent("""\ + class AddInit[tests.test_fastapilike_1.Update[tests.test_fastapilike_1.Hero]]: + name: typing.Annotated[str | None, _Default(val=None), typing.Literal[]] + age: typing.Annotated[int | None, _Default(val=None), typing.Literal[]] + secret_name: typing.Annotated[str | None, _Default(val=None), typing.Literal[]] + def __init__(self: Self, *, name: str | None = ..., age: int | None = ..., secret_name: str | None = ...) -> None: ... + """) diff --git a/tests/test_type_eval.py b/tests/test_type_eval.py index 32e84cc..2604263 100644 --- a/tests/test_type_eval.py +++ b/tests/test_type_eval.py @@ -2,6 +2,7 @@ import textwrap import unittest from typing import ( + Annotated, Any, Callable, Generic, @@ -24,6 +25,7 @@ GetAttr, GetName, GetType, + GetAnnotations, Is, Iter, Length, @@ -807,3 +809,38 @@ def test_callable_to_signature(): '(_arg0: int, /, b: int, c: int = ..., *args: int, ' 'd: int, e: int = ..., **kwargs: int) -> int' ) + + +############## + +type XTest[X] = Annotated[X, 'blah'] + + +class AnnoTest: + a: XTest[int] + b: XTest[Literal["test"]] + + +def test_type_eval_annotated_01(): + res = format_helper.format_class(eval_typing(AnnoTest)) + + assert res == textwrap.dedent("""\ + class AnnoTest: + a: typing.Annotated[int, 'blah'] + b: typing.Annotated[typing.Literal['test'], 'blah'] + """) + + +def test_type_eval_annotated_02(): + res = eval_typing(Is[GetAttr[AnnoTest, Literal["a"]], int]) + assert res is True + + +def test_type_eval_annotated_03(): + res = eval_typing(Uppercase[GetAttr[AnnoTest, Literal["b"]]]) + assert res == Literal["TEST"] + + +def test_type_eval_annotated_04(): + res = eval_typing(GetAnnotations[GetAttr[AnnoTest, Literal["b"]]]) + assert res == Literal["blah"] diff --git a/typemap/type_eval/_apply_generic.py b/typemap/type_eval/_apply_generic.py index 12bd207..73c3ba2 100644 --- a/typemap/type_eval/_apply_generic.py +++ b/typemap/type_eval/_apply_generic.py @@ -195,6 +195,9 @@ def _get_closure_types(af: types.FunctionType) -> dict[str, type]: } +EXCLUDED_ATTRIBUTES = typing.EXCLUDED_ATTRIBUTES - {'__init__'} # type: ignore[attr-defined] + + def get_local_defns(boxed: Boxed) -> tuple[dict[str, Any], dict[str, Any]]: annos: dict[str, Any] = {} dct: dict[str, Any] = {} @@ -238,7 +241,7 @@ def get_local_defns(boxed: Boxed) -> tuple[dict[str, Any], dict[str, Any]]: annos.update(af) for name, orig in boxed.cls.__dict__.items(): - if name in typing.EXCLUDED_ATTRIBUTES: # type: ignore[attr-defined] + if name in EXCLUDED_ATTRIBUTES: continue stuff = inspect.unwrap(orig) diff --git a/typemap/type_eval/_eval_operators.py b/typemap/type_eval/_eval_operators.py index 20cb4e0..8d09a02 100644 --- a/typemap/type_eval/_eval_operators.py +++ b/typemap/type_eval/_eval_operators.py @@ -7,6 +7,7 @@ import re import types import typing +from typing import _AnnotatedAlias as typing_AnnotatedAlias # type: ignore [attr-defined] # noqa: PLC2701 from typemap import type_eval from typemap.type_eval import _apply_generic, _typing_inspect @@ -19,6 +20,8 @@ GetArg, GetArgs, GetAttr, + GetAnnotations, + DropAnnotations, IsSubSimilar, IsSubtype, Iter, @@ -147,11 +150,31 @@ def _mk_literal_union(*parts): return typing.Literal[*parts] +def _unwrap_anno(tp): + if isinstance(tp, typing_AnnotatedAlias): + return tp.__origin__ + else: + return tp + + +def _lift_evaluated(func): + @functools.wraps(func) + def wrapper(*args, ctx): + return func( + *[_unwrap_anno(_eval_types(arg, ctx)) for arg in args], ctx=ctx + ) + + return wrapper + + def _lift_over_unions(func): @functools.wraps(func) def wrapper(*args, ctx): args2 = [_union_elems(x, ctx) for x in args] - parts = [func(*x, ctx=ctx) for x in itertools.product(*args2)] + parts = [ + func(*[_unwrap_anno(x) for x in xs], ctx=ctx) + for xs in itertools.product(*args2) + ] return _mk_union(*parts) return wrapper @@ -161,6 +184,7 @@ def wrapper(*args, ctx): @type_eval.register_evaluator(Iter) +@_lift_evaluated def _eval_Iter(tp, *, ctx): tp = _eval_types(tp, ctx) if ( @@ -180,19 +204,15 @@ def _eval_Iter(tp, *, ctx): @type_eval.register_evaluator(IsSubtype) +@_lift_evaluated def _eval_IsSubtype(lhs, rhs, *, ctx): - return type_eval.issubtype( - _eval_types(lhs, ctx), - _eval_types(rhs, ctx), - ) + return type_eval.issubtype(lhs, rhs) @type_eval.register_evaluator(IsSubSimilar) +@_lift_evaluated def _eval_IsSubSimilar(lhs, rhs, *, ctx): - return type_eval.issubsimilar( - _eval_types(lhs, ctx), - _eval_types(rhs, ctx), - ) + return type_eval.issubsimilar(lhs, rhs) ################################################################## @@ -317,6 +337,11 @@ def _callable_type_to_signature(callable_type: object) -> inspect.Signature: ) ) + # HACK: Makes output look nicer, but I'm not 100% where it is + # sneaking in... + if return_type is type(None): + return_type = None + return inspect.Signature( parameters=parameters, return_annotation=return_type, @@ -398,7 +423,7 @@ def _function_type(func, *, receiver_type): empty = inspect.Parameter.empty def _ann(x): - return typing.Any if x is empty else x + return typing.Any if x is empty else None if x is type(None) else x specified_receiver = receiver_type @@ -450,6 +475,7 @@ def _ann(x): @type_eval.register_evaluator(Attrs) +@_lift_over_unions def _eval_Attrs(tp, *, ctx): hints = get_annotated_type_hints(tp, include_extras=True) @@ -480,7 +506,6 @@ def _eval_Members(tp, *, ctx): ] for n, (t, qs, d) in hints.items() ] - return tuple[*attrs] @@ -488,6 +513,7 @@ def _eval_Members(tp, *, ctx): @type_eval.register_evaluator(FromUnion) +@_lift_evaluated def _eval_FromUnion(tp, *, ctx): if tp in ctx.known_recursive_types: return tuple[*_union_elems(ctx.known_recursive_types[tp], ctx)] @@ -695,6 +721,26 @@ def _eval_GetArgs(tp, base, *, ctx) -> typing.Any: return tuple[*args] # type: ignore[valid-type] +@type_eval.register_evaluator(GetAnnotations) +def _eval_GetAnnotations(tp, *, ctx) -> typing.Any: + tp = _eval_types(tp, ctx=ctx) + # XXX: Should *this* lift over unions?? + if isinstance(tp, typing_AnnotatedAlias): + return typing.Literal[*tp.__metadata__] + else: + return typing.Never + + +@type_eval.register_evaluator(DropAnnotations) +def _eval_DropAnnotations(tp, *, ctx) -> typing.Any: + tp = _eval_types(tp, ctx=ctx) + # XXX: Should *this* lift over unions?? + if isinstance(tp, typing_AnnotatedAlias): + return tp.__origin__ + else: + return tp + + @type_eval.register_evaluator(Length) @_lift_over_unions def _eval_Length(tp, *, ctx) -> typing.Any: @@ -710,6 +756,9 @@ def _eval_Length(tp, *, ctx) -> typing.Any: raise TypeError(f"Invalid type argument to Length: {tp} is not a tuple") +# String literals + + def _string_literal_op(typ, op): @_lift_over_unions def func(*args, ctx): @@ -749,6 +798,7 @@ def _is_method_like(typ): @type_eval.register_evaluator(NewProtocol) +@_lift_evaluated def _eval_NewProtocol(*etyps: Member, ctx): dct: dict[str, object] = {} dct["__annotations__"] = annos = {} @@ -781,5 +831,8 @@ def _eval_NewProtocol(*etyps: Member, ctx): mcls: type = type(typing.cast(type, typing.Protocol)) cls = mcls(name, (typing.Protocol,), dct) - cls = _eval_types(cls, ctx) + # Stick __init__ back in, since Protocol messes with it + if '__init__' in dct: + cls.__init__ = dct['__init__'] + return cls diff --git a/typemap/type_eval/_eval_typing.py b/typemap/type_eval/_eval_typing.py index 7f76660..b790dee 100644 --- a/typemap/type_eval/_eval_typing.py +++ b/typemap/type_eval/_eval_typing.py @@ -13,6 +13,7 @@ from typing import _GenericAlias as typing_GenericAlias # type: ignore [attr-defined] # noqa: PLC2701 from typing import _CallableGenericAlias as typing_CallableGenericAlias # type: ignore [attr-defined] # noqa: PLC2701 from typing import _LiteralGenericAlias as typing_LiteralGenericAlias # type: ignore [attr-defined] # noqa: PLC2701 +from typing import _AnnotatedAlias as typing_AnnotatedAlias # type: ignore [attr-defined] # noqa: PLC2701 if typing.TYPE_CHECKING: @@ -286,6 +287,15 @@ def _eval_literal(obj: typing_LiteralGenericAlias, ctx: EvalContext): return obj +@_eval_types_impl.register +def _eval_annotated(obj: typing_AnnotatedAlias, ctx: EvalContext): + # Don't evaluate the args + return typing.Annotated[ + _eval_types(obj.__origin__, ctx), + *obj.__metadata__, + ] + + @_eval_types_impl.register def _eval_type_alias(obj: typing.TypeAliasType, ctx: EvalContext): assert obj.__module__ # FIXME: or can this really happen? diff --git a/typemap/type_eval/_subsim.py b/typemap/type_eval/_subsim.py index 0c684c4..e87d5b7 100644 --- a/typemap/type_eval/_subsim.py +++ b/typemap/type_eval/_subsim.py @@ -8,6 +8,11 @@ def issubsimilar(lhs: typing.Any, rhs: typing.Any) -> bool: # TODO: Need to handle some cases + if lhs is None: + lhs = type(None) + if rhs is None: + rhs = type(None) + # N.B: All of the 'bool's in these are because black otherwise # formats the two-conditional chains in an unconscionably bad way. diff --git a/typemap/typing.py b/typemap/typing.py index ce2f5ec..41dfae9 100644 --- a/typemap/typing.py +++ b/typemap/typing.py @@ -30,6 +30,28 @@ class GenericCallable[ ### + +class GetAnnotations[T]: + """Fetch the annotations of a potentially Annotated type, as Literals. + + GetAnnotations[Annotated[int, 'xxx']] = Literal['xxx'] + GetAnnotations[Annotated[int, 'xxx', 5]] = Literal['xxx', 5] + GetAnnotations[int] = Never + """ + + +class DropAnnotations[T]: + """Drop the annotations of a potentially Annotated type + + DropAnnotations[Annotated[int, 'xxx']] = int + DropAnnotations[Annotated[int, 'xxx', 5]] = int + DropAnnotations[int] = int + """ + + +### + + MemberQuals = typing.Literal["ClassVar", "Final"]