From 8bbf12d18e37b7b634a6885bf40e96e89fac6b6a Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Tue, 13 Jan 2026 16:39:16 -0800 Subject: [PATCH] A new approach to fastapi/dataclass interfaces We want to be able to support transforming types based on dataclasses/attrs/pydantic style field descriptors. In order to do that, we need to be able to consume things like calls to `Field`. Our strategy for this is to introduce a new type `InitField[KwargDict]` that collects arguments defined by a `KwargDict: TypedDict`: ``` class InitField[KwargDict: BaseTypedDict]: def __init__(self, **kwargs: typing.Unpack[KwargDict]) -> None: ... def _get_kwargs(self) -> KwargDict: ... ``` When `InitField` or (more likely) a subtype of it is instantiated inside a class body, we infer a *more specific* type for it, based on `Literal` types for all the arguments passed. So if we write: class A: foo: int = InitField(default=0) then we would infer the type `InitField[TypedDict('...', {'default': Literal[0]})]` for the initializer, and that would be made available as the `Init` field of the `Member`. Honestly this is pretty subtle and will probably be controversial, but maybe *less* controversial than the ``Annotated`` stuff and it will produce a better result than that, so... Take a look at test_fastapilike_2.py for the full fastapi + `__init__` generation example using this system. --- spec-draft.rst | 29 +++- tests/format_helper.py | 6 +- tests/test_fastapilike_1.py | 9 +- tests/test_fastapilike_2.py | 238 +++++++++++++++++++++++++++ tests/test_type_dir.py | 43 ++++- typemap/type_eval/_eval_operators.py | 49 +++++- typemap/typing.py | 32 +++- 7 files changed, 384 insertions(+), 22 deletions(-) create mode 100644 tests/test_fastapilike_2.py diff --git a/spec-draft.rst b/spec-draft.rst index 4ea0f38..ed9a2da 100644 --- a/spec-draft.rst +++ b/spec-draft.rst @@ -146,7 +146,7 @@ Object inspection and creation * ``Members[T]`` produces a ``tuple`` of ``Member`` types. -* ``Member[N: Literal[str], T, Q: MemberQuals, D]`` - ``N`` is the name, ``T`` is the type, ``Q`` is a union of qualifiers, ``D`` is the defining class of the member +* ``Member[N: Literal[str], T, Q: MemberQuals, Init, D]`` - ``N`` is the name, ``T`` is the type, ``Q`` is a union of qualifiers, ``Init`` is the literal type of the class initializer (under what conditions!?) and ``D`` is the defining class of the member * ``MemberQuals = Literal['ClassVar', 'Final']`` - ``MemberQuals`` is the type of "qualifiers" that can apply to a member; currently ClassVar and Final Methods are returned as callables using the new ``Param`` based extended callables. staticmethod and classmethod will return ``staticmethod`` and ``classmethod`` types, which are subscriptable as of 3.14. @@ -159,6 +159,7 @@ We also have helpers for extracting those names; they are all definable in terms * ``GetName[T: Member | Param]`` * ``GetType[T: Member | Param]`` * ``GetQuals[T: Member | Param]`` +* ``GetInit[T: Member]`` * ``GetDefiner[T: Member]`` * ``NewProtocolWithBases[Bases, Ps: tuple[Member]]`` - A variant that allows specifying bases too. (UNIMPLEMENTED) @@ -178,6 +179,8 @@ Callable inspection and creation The names, type, and qualifiers share getter operations with ``Member``. +TODO: Should we make ``GetInit`` be literal types of default parameter values too? + ---- @@ -205,6 +208,30 @@ We understand that this may be controversial, as currently Annotated may be full DropAnnotations[int] = int +--------- +InitField +--------- + +We want to be able to support transforming types based on dataclasses/attrs/pydantic style field descriptors. In order to do that, we need to be able to consume things like calls to ``Field``. + +Our strategy for this is to introduce a new type ``InitField[KwargDict]`` that collects arguments defined by a ``KwargDict: TypedDict``:: + + class InitField[KwargDict: BaseTypedDict]: + def __init__(self, **kwargs: typing.Unpack[KwargDict]) -> None: + ... + + def _get_kwargs(self) -> KwargDict: + ... + +When ``InitField`` or (more likely) a subtype of it is instantiated inside a class body, we infer a *more specific* type for it, based on ``Literal`` types for all the arguments passed. + +So if we write:: + + class A: + foo: int = InitField(default=0) + +then we would infer the type ``InitField[TypedDict('...', {'default': Literal[0]})]`` for the initializer, and that would be made available as the ``Init`` field of the ``Member``. + ------------------- String manipulation ------------------- diff --git a/tests/format_helper.py b/tests/format_helper.py index a637e59..2eb9073 100644 --- a/tests/format_helper.py +++ b/tests/format_helper.py @@ -18,7 +18,11 @@ def format_meth(name, meth): code = f"class {cls.__name__}:\n" for attr_name, attr_type in cls.__annotations__.items(): attr_type_s = annotationlib.type_repr(attr_type) - code += f" {attr_name}: {attr_type_s}\n" + if attr_name in cls.__dict__: + eq = f' = {cls.__dict__[attr_name]!r}' + else: + eq = '' + code += f" {attr_name}: {attr_type_s}{eq}\n" for name, attr in cls.__dict__.items(): if attr is typing._no_init_or_replace_init: diff --git a/tests/test_fastapilike_1.py b/tests/test_fastapilike_1.py index 819130b..d14b2c8 100644 --- a/tests/test_fastapilike_1.py +++ b/tests/test_fastapilike_1.py @@ -24,7 +24,7 @@ from . import format_helper -class PropQuals(enum.StrEnum): +class PropQuals(enum.Enum): HIDDEN = "HIDDEN" PRIMARY = "PRIMARY" HAS_DEFAULT = "HAS_DEFAULT" @@ -101,13 +101,6 @@ class _Default: ] """ -type AllOptional[T] = NewProtocol[ - *[ - Member[GetName[p], GetType[p] | None, GetQuals[p]] - for p in Iter[Attrs[T]] - ] -] - # Strip `| None` from a type by iterating over its union components # and filtering type NotOptional[T] = Union[*[x for x in Iter[FromUnion[T]] if not Is[x, None]]] diff --git a/tests/test_fastapilike_2.py b/tests/test_fastapilike_2.py new file mode 100644 index 0000000..786279d --- /dev/null +++ b/tests/test_fastapilike_2.py @@ -0,0 +1,238 @@ +import textwrap + +from typing import ( + Callable, + Literal, + Union, + ReadOnly, + TypedDict, + Never, + Self, +) + +from typemap.type_eval import eval_typing +from typemap.typing import ( + NewProtocol, + Iter, + Attrs, + Is, + FromUnion, + GetArg, + GetAttr, + GetType, + GetName, + GetQuals, + GetInit, + InitField, + Member, + Members, + Param, +) + +from . import format_helper + + +class FieldArgs(TypedDict, total=False): + hidden: ReadOnly[bool] + primary_key: ReadOnly[bool] + index: ReadOnly[bool] + default: ReadOnly[object] + + +class Field[T: FieldArgs](InitField[T]): + pass + + +#### + +# TODO: Should this go into the stdlib? +type GetFieldItem[T: InitField, K] = GetAttr[GetArg[T, InitField, 0], K] + + +## + +# Strip `| None` from a type by iterating over its union components +# and filtering +type NotOptional[T] = Union[*[x for x in Iter[FromUnion[T]] if not Is[x, None]]] + +# Adjust an attribute type for use in Public below by dropping | None for +# primary keys and stripping all annotations. +type FixPublicType[T, Init] = ( + NotOptional[T] + if Is[Literal[True], GetFieldItem[Init, Literal["primary_key"]]] + else T +) + +# Extract the default type from an Init field. +# If it is a Field, then we try pulling out the "default" field, +# otherwise we return the type itself. +type GetDefault[Init] = ( + GetFieldItem[Init, Literal["default"]] if Is[Init, Field] else Init +) + +# Strip out everything that is Hidden and also make the primary key required +# Drop all the annotations, since this is for data getting returned to users +# from the DB, so we don't need default values. +type Public[T] = NewProtocol[ + *[ + Member[GetName[p], FixPublicType[GetType[p], GetInit[p]], GetQuals[p]] + for p in Iter[Attrs[T]] + if not Is[Literal[True], GetFieldItem[GetInit[p], Literal["hidden"]]] + ] +] + +# Create takes everything but the primary key and preserves defaults +type Create[T] = NewProtocol[ + *[ + Member[GetName[p], GetType[p], GetQuals[p], GetDefault[GetInit[p]]] + for p in Iter[Attrs[T]] + if not Is[ + Literal[True], GetFieldItem[GetInit[p], Literal["primary_key"]] + ] + ] +] + +# Update takes everything but the primary key, but makes them all have +# None defaults +type Update[T] = NewProtocol[ + *[ + Member[ + GetName[p], + GetType[p] | None, + GetQuals[p], + Literal[None], + ] + for p in Iter[Attrs[T]] + if not Is[ + Literal[True], GetFieldItem[GetInit[p], Literal["primary_key"]] + ] + ] +] + +## + +# Generate the Member field for __init__ for a class +type InitFnType[T] = Member[ + Literal["__init__"], + Callable[ + [ + Param[Literal["self"], Self], + *[ + Param[ + GetName[p], + GetType[p], + # All arguments are keyword-only + # It takes a default if a default is specified in the class + Literal["keyword"] + if Is[ + GetDefault[GetInit[p]], + Never, + ] + else Literal["keyword", "default"], + ] + for p in Iter[Attrs[T]] + ], + ], + None, + ], + Literal["ClassVar"], +] +type AddInit[T] = NewProtocol[ + InitFnType[T], + *[x for x in Iter[Members[T]]], +] + + +#### + +# This is the FastAPI example code that we are trying to repair! +# Adapted from https://fastapi.tiangolo.com/tutorial/sql-databases/#heroupdate-the-data-model-to-update-a-hero +""" +class HeroBase(SQLModel): + name: str = Field(index=True) + age: int | None = Field(default=None, index=True) + + +class Hero(HeroBase, table=True): + id: int | None = Field(default=None, primary_key=True) + secret_name: str + + +class HeroPublic(HeroBase): + id: int + + +class HeroCreate(HeroBase): + secret_name: str + + +class HeroUpdate(HeroBase): + name: str | None = None + age: int | None = None + secret_name: str | None = None +""" + + +class Hero: + id: int | None = Field(default=None, primary_key=True) + + name: str = Field(index=True) + age: int | None = Field(default=None, index=True) + + secret_name: str = Field(hidden=True) + + +####### + + +def test_fastapi_like_0(): + tgt = eval_typing(AddInit[Hero]) + fmt = format_helper.format_class(tgt) + + assert fmt == textwrap.dedent("""\ + class AddInit[tests.test_fastapilike_2.Hero]: + id: int | None = Field(default=None, primary_key=True) + name: str = Field(index=True) + age: int | None = Field(default=None, index=True) + secret_name: str = Field(hidden=True) + def __init__(self: Self, *, id: int | None = ..., name: str, age: int | None = ..., secret_name: str) -> None: ... + """) + + +def test_fastapi_like_1(): + tgt = eval_typing(AddInit[Public[Hero]]) + fmt = format_helper.format_class(tgt) + + assert fmt == textwrap.dedent("""\ + class AddInit[tests.test_fastapilike_2.Public[tests.test_fastapilike_2.Hero]]: + id: int + name: str + age: int | None + def __init__(self: Self, *, id: int, name: str, age: int | None) -> None: ... + """) + + +def test_fastapi_like_2(): + tgt = eval_typing(AddInit[Create[Hero]]) + fmt = format_helper.format_class(tgt) + + assert fmt == textwrap.dedent("""\ + class AddInit[tests.test_fastapilike_2.Create[tests.test_fastapilike_2.Hero]]: + name: str + age: int | None = None + secret_name: str + def __init__(self: Self, *, name: str, age: int | None = ..., secret_name: str) -> None: ... + """) + + +def test_fastapi_like_3(): + tgt = eval_typing(AddInit[Update[Hero]]) + fmt = format_helper.format_class(tgt) + + assert fmt == textwrap.dedent("""\ + class AddInit[tests.test_fastapilike_2.Update[tests.test_fastapilike_2.Hero]]: + name: str | None = None + age: int | None = None + secret_name: str | None = None + def __init__(self: Self, *, name: str | None = ..., age: int | None = ..., secret_name: str | None = ...) -> None: ... + """) diff --git a/tests/test_type_dir.py b/tests/test_type_dir.py index b6b9772..e2d6052 100644 --- a/tests/test_type_dir.py +++ b/tests/test_type_dir.py @@ -1,7 +1,6 @@ import textwrap import typing -from typing import Literal, Never, TypeVar, Union - +from typing import Literal, Never, TypeVar, TypedDict, Union, ReadOnly from typemap.type_eval import eval_typing from typemap.typing import ( @@ -11,6 +10,7 @@ GetName, GetQuals, GetType, + InitField, Is, Iter, Member, @@ -341,7 +341,7 @@ def test_type_members_attr_1(): d = eval_typing(Members[Final]) member = _get_member(d, "ordinary") assert typing.get_origin(member) is Member - _, _, _, origin = typing.get_args(member) + _, _, _, _, origin = typing.get_args(member) assert origin.__name__ == "Ordinary" @@ -349,7 +349,7 @@ def test_type_members_attr_2(): d = eval_typing(Members[Final]) member = _get_member(d, "last") assert typing.get_origin(member) is Member - _, typ, _, origin = typing.get_args(member) + _, typ, _, _, origin = typing.get_args(member) assert typ == int | Literal[True] assert str(origin) == "tests.test_type_dir.Last[int]" @@ -358,7 +358,7 @@ def test_type_members_attr_3(): d = eval_typing(Members[Last[int]]) member = _get_member(d, "last") assert typing.get_origin(member) is Member - _, typ, _, origin = typing.get_args(member) + _, typ, _, _, origin = typing.get_args(member) assert typ == int | Literal[True] assert str(origin) == "tests.test_type_dir.Last[int]" @@ -367,7 +367,7 @@ def test_type_members_func_1(): d = eval_typing(Members[Final]) member = _get_member(d, "foo") assert typing.get_origin(member) is Member - name, typ, quals, origin = typing.get_args(member) + name, typ, quals, _, origin = typing.get_args(member) assert name == typing.Literal["foo"] assert quals == typing.Literal["ClassVar"] @@ -389,7 +389,7 @@ def test_type_members_func_2(): d = eval_typing(Members[Final]) member = _get_member(d, "cbase") assert typing.get_origin(member) is Member - name, typ, quals, _origin = typing.get_args(member) + name, typ, quals, _origin, _ = typing.get_args(member) assert name == typing.Literal["cbase"] assert quals == typing.Literal["ClassVar"] @@ -404,7 +404,7 @@ def test_type_members_func_3(): d = eval_typing(Members[Final]) member = _get_member(d, "sbase") assert typing.get_origin(member) is Member - name, typ, quals, _origin = typing.get_args(member) + name, typ, quals, _origin, _ = typing.get_args(member) assert name == typing.Literal["sbase"] assert quals == typing.Literal["ClassVar"] @@ -415,3 +415,30 @@ def test_type_members_func_3(): == "\ typemap.typing.GenericCallable[tuple[Z], staticmethod[tuple[typemap.typing.Param[typing.Literal['a'], int | typing.Literal['gotcha!'] | Z | None, typing.Never], typemap.typing.Param[typing.Literal['b'], ~K, typing.Never]], dict[str, int | Z]]]" ) + + +# Test initializers + + +class FieldArgs(TypedDict, total=False): + foo: ReadOnly[bool] + bar: ReadOnly[int] + + +class Field[T: FieldArgs](InitField[T]): + pass + + +class Inited: + foo: int = 10 + bar: bool = Field(foo=False) + + +def test_type_dir_inits_1(): + d = eval_typing(Inited) + + assert format_helper.format_class(d) == textwrap.dedent("""\ + class Inited: + foo: int = 10 + bar: bool = Field(foo=False) + """) diff --git a/typemap/type_eval/_eval_operators.py b/typemap/type_eval/_eval_operators.py index 9e650c1..80b8a38 100644 --- a/typemap/type_eval/_eval_operators.py +++ b/typemap/type_eval/_eval_operators.py @@ -22,6 +22,7 @@ GetArg, GetArgs, GetAttr, + InitField, IsSubSimilar, IsSubtype, Iter, @@ -56,6 +57,21 @@ def _eval_literal(val, ctx): return _from_literal(_eval_types(val, ctx)) +def _make_init_type(v): + # Usually it's just a literal, but sometimes we need to handle + # InitField. + if isinstance(v, InitField): + return type(v)[ + typing.TypedDict( + type(v).__name__, + {k: _make_init_type(sv) for k, sv in v.get_kwargs().items()}, + ) + ] + else: + # Wrap in tuple when creating Literal in case it *is* a tuple + return typing.Literal[(v,)] + + def get_annotated_type_hints(cls, **kwargs): """Get the type hints/quals for a cls annotated with definition site. @@ -87,7 +103,13 @@ def get_annotated_type_hints(cls, **kwargs): else: break - hints[k] = ty, tuple(sorted(quals)), acls + if k in abox.cls.__dict__: + # Wrap in tuple when creating Literal in case it *is* a tuple + init = _make_init_type(abox.cls.__dict__[k]) + else: + init = typing.Never + + hints[k] = ty, tuple(sorted(quals)), init, acls return hints @@ -117,6 +139,7 @@ def get_annotated_method_hints(cls): hints[name] = ( _function_type(attr, receiver_type=acls), ("ClassVar",), + object, acls, ) @@ -482,9 +505,10 @@ def _hints_to_members(hints, ctx): typing.Literal[n], _eval_types(t, ctx), _mk_literal_union(*qs), + init, d, ] - for n, (t, qs, d) in hints.items() + for n, (t, qs, init, d) in hints.items() ] ] @@ -794,13 +818,31 @@ def _is_method_like(typ): return typing.get_origin(typ) in FUNC_LIKES +def _unpack_init(dct, name, init): + """Unpack an initializer type into a __dict__. + + If init is a literal with a single value, then dct[name] gets that + value. If it is an InitField subclass, we recursively unpack the + TypedDict it is parameterized over into a new InitField object, + and include that. + """ + origin = typing.get_origin(init) + if _typing_inspect.is_literal(init) and len(init.__args__) == 1: + dct[name] = init.__args__[0] + if isinstance(origin, type) and issubclass(origin, InitField): + args = {} + for k, v in typing.get_type_hints(init.__args__[0]).items(): + _unpack_init(args, k, v) + dct[name] = origin(**args) + + @type_eval.register_evaluator(NewProtocol) @_lift_evaluated def _eval_NewProtocol(*etyps: Member, ctx): dct: dict[str, object] = {} dct["__annotations__"] = annos = {} - for tname, typ, quals, _ in (typing.get_args(prop) for prop in etyps): + for tname, typ, quals, init, _ in (typing.get_args(prop) for prop in etyps): name = _eval_literal(tname, ctx) typ = _eval_types(typ, ctx) tquals = _eval_types(quals, ctx) @@ -811,6 +853,7 @@ def _eval_NewProtocol(*etyps: Member, ctx): dct[name] = _callable_type_to_method(name, typ) else: annos[name] = _add_quals(typ, tquals) + _unpack_init(dct, name, init) module_name = __name__ name = "NewProtocol" diff --git a/typemap/typing.py b/typemap/typing.py index 41dfae9..a05ac1e 100644 --- a/typemap/typing.py +++ b/typemap/typing.py @@ -31,6 +31,28 @@ class GenericCallable[ ### +class InitField[KwargDict: BaseTypedDict]: + """Base class to support dataclass.Field type initializers! + + Will require some magical treatment in typecheckers... + """ + + __kwargs: KwargDict + + def __init__(self, **kwargs: typing.Unpack[KwargDict]) -> None: # type: ignore[misc] + self.__kwargs = kwargs # type: ignore[assignment] + + def get_kwargs(self) -> KwargDict: + return self.__kwargs + + def __repr__(self) -> str: + args = ', '.join(f'{k}={v!r}' for k, v in self.__kwargs.items()) + return f'{type(self).__name__}({args})' + + +### + + class GetAnnotations[T]: """Fetch the annotations of a potentially Annotated type, as Literals. @@ -55,10 +77,17 @@ class DropAnnotations[T]: MemberQuals = typing.Literal["ClassVar", "Final"] -class Member[N: str, T, Q: MemberQuals = typing.Never, D = typing.Never]: +class Member[ + N: str, + T, + Q: MemberQuals = typing.Never, + I = typing.Never, + D = typing.Never, +]: name: N typ: T quals: Q + init: I definer: D @@ -74,6 +103,7 @@ class Param[N: str | None, T, Q: ParamQuals = typing.Never]: type GetName[T: Member | Param] = GetAttr[T, typing.Literal["name"]] type GetType[T: Member | Param] = GetAttr[T, typing.Literal["typ"]] type GetQuals[T: Member | Param] = GetAttr[T, typing.Literal["quals"]] +type GetInit[T: Member] = GetAttr[T, typing.Literal["init"]] type GetDefiner[T: Member] = GetAttr[T, typing.Literal["definer"]]