From 28a3c6217e4845f5a2572fb4e63d9dd0cba1b040 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Fri, 16 Jan 2026 10:20:37 -0800 Subject: [PATCH 1/6] Put Prisma-style first --- pre-pep.rst | 193 ++++++++++++++++++++++++++-------------------------- 1 file changed, 97 insertions(+), 96 deletions(-) diff --git a/pre-pep.rst b/pre-pep.rst index e19f586..e0e41fd 100644 --- a/pre-pep.rst +++ b/pre-pep.rst @@ -37,6 +37,103 @@ case of dataclass-like transformations (:pep:`PEP 681 <681>`). Examples: pydantic/fastapi, dataclasses, sqlalchemy +Prisma-style ORMs +----------------- + +`Prisma <#prisma_>`_, a popular ORM for TypeScript, allows writing +queries like (adapted from `this example <#prisma-example_>`_):: + + const user = await prisma.user.findMany({ + select: { + name: true, + email: true, + posts: true, + }, + }); + +for which the inferred type will be something like:: + + { + email: string; + name: string | null; + posts: { + id: number; + title: string; + content: string | null; + authorId: number | null; + }[]; + }[] + +Here, the output type is a combination of both existing information +about the type of ``prisma.user`` and the type of the argument to +``findMany``. It returns an array of objects containing the properties +of ``user`` that were requested; one of the requested elements, +``posts``, is a "relation" referencing another model; it has *all* of +its properties fetched but not its relations. + +We would like to be able to do something similar in Python, perhaps +with a schema defined like:: + + class Comment: + id: Property[int] + name: Property[str] + poster: Link[User] + + + class Post: + id: Property[int] + + title: Property[str] + content: Property[str] + + comments: MultiLink[Comment] + author: Link[Comment] + + + class User: + id: Property[int] + + name: Property[str] + email: Property[str] + posts: Link[Post] + +(In Prisma, a code generator generates type definitions based on a +prisma schema in its own custom format; you could imagine something +similar here, or that the definitions were hand written) + +and a call like:: + + db.select( + User, + name=True, + email=True, + posts=True, + ) + +which would have return type ``list[]`` where:: + + class : + name: str + email: str + posts: list[] + + class + id: int + title: str + content: str + + +Unlike the FastAPI-style example above, we probably don't have too +much need for runtime introspection of the types here, which is good: +inferring the type of a function is much less likely to be feasible. + + +Implementation +'''''''''''''' + +We have a more `worked example <#qb-test_>`_ in our test suite. + + Automatically deriving FastAPI CRUD models ------------------------------------------ @@ -166,102 +263,6 @@ from a ``default`` argument to a field or specified directly as an initializer). -Prisma-style ORMs ------------------ - -`Prisma <#prisma_>`_, a popular ORM for TypeScript, allows writing -queries like (adapted from `this example <#prisma-example_>`_):: - - const user = await prisma.user.findMany({ - select: { - name: true, - email: true, - posts: true, - }, - }); - -for which the inferred type will be something like:: - - { - email: string; - name: string | null; - posts: { - id: number; - title: string; - content: string | null; - authorId: number | null; - }[]; - }[] - -Here, the output type is a combination of both existing information -about the type of ``prisma.user`` and the type of the argument to -``findMany``. It returns an array of objects containing the properties -of ``user`` that were requested; one of the requested elements, -``posts``, is a "relation" referencing another model; it has *all* of -its properties fetched but not its relations. - -We would like to be able to do something similar in Python, perhaps -with a schema defined like:: - - class Comment: - id: Property[int] - name: Property[str] - poster: Link[User] - - - class Post: - id: Property[int] - - title: Property[str] - content: Property[str] - - comments: MultiLink[Comment] - author: Link[Comment] - - - class User: - id: Property[int] - - name: Property[str] - email: Property[str] - posts: Link[Post] - -(In Prisma, a code generator generates type definitions based on a -prisma schema in its own custom format; you could imagine something -similar here, or that the definitions were hand written) - -and a call like:: - - db.select( - User, - name=True, - email=True, - posts=True, - ) - -which would have return type ``list[]`` where:: - - class : - name: str - email: str - posts: list[] - - class - id: int - title: str - content: str - - -Unlike the FastAPI-style example above, we probably don't have too -much need for runtime introspection of the types here, which is good: -inferring the type of a function is much less likely to be feasible. - - -Implementation -'''''''''''''' - -We have a more `worked example <#qb-test_>`_ in our test suite. - dataclasses-style method generation ----------------------------------- From f7a96bc40cb4461f5f92e502fc6f734fb4f1d20c Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Fri, 16 Jan 2026 10:27:04 -0800 Subject: [PATCH 2/6] Make qblike test return a list --- tests/test_qblike_2.py | 20 ++++++++++++-------- typemap/type_eval/_apply_generic.py | 9 ++++++--- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/tests/test_qblike_2.py b/tests/test_qblike_2.py index ee71b84..ffc1980 100644 --- a/tests/test_qblike_2.py +++ b/tests/test_qblike_2.py @@ -57,18 +57,19 @@ class MultiLink[T](Link[T]): ) -# XXX: putting list here doesn't work! def select[K: BaseTypedDict]( rcv: type[User], /, **kwargs: Unpack[K], -) -> NewProtocol[ - *[ - Member[ - GetName[c], - ConvertField[GetAttr[User, GetName[c]]], +) -> list[ + NewProtocol[ + *[ + Member[ + GetName[c], + ConvertField[GetAttr[User, GetName[c]]], + ] + for c in Iter[Attrs[K]] ] - for c in Iter[Attrs[K]] ] ]: ... @@ -105,6 +106,8 @@ def test_qblike_1(): id=True, name=True, ) + assert ret.__origin__ is list + ret = ret.__args__[0] fmt = format_helper.format_class(ret) assert fmt == textwrap.dedent("""\ @@ -123,7 +126,8 @@ def test_qblike_2(): posts=True, ) - # ret = ret.__args__[0] + assert ret.__origin__ is list + ret = ret.__args__[0] fmt = format_helper.format_class(ret) assert fmt == textwrap.dedent("""\ diff --git a/typemap/type_eval/_apply_generic.py b/typemap/type_eval/_apply_generic.py index 73c3ba2..2f2eb4d 100644 --- a/typemap/type_eval/_apply_generic.py +++ b/typemap/type_eval/_apply_generic.py @@ -107,9 +107,12 @@ def _box(cls: type[Any], args: dict[Any, Any]) -> Boxed: return Boxed(cls, boxed_bases, args) if isinstance(cls, (typing._GenericAlias, types.GenericAlias)): # type: ignore[attr-defined] - args = dict( - zip(cls.__origin__.__parameters__, cls.__args__, strict=True) - ) + if params := getattr(cls.__origin__, "__parameters__", None): + args = dict( + zip(cls.__origin__.__parameters__, cls.__args__, strict=True) + ) + else: + args = {} cls = cls.__origin__ else: if params := getattr(cls, "__parameters__", None): From 32e3dacdab5ccdcae93c089ff7ce0013da5b8fe7 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Fri, 16 Jan 2026 10:46:11 -0800 Subject: [PATCH 3/6] Tweak qblike2 to take type[T] --- tests/test_qblike_2.py | 10 +++++----- typemap/type_eval/_eval_call.py | 17 +++++++++++++++++ typemap/type_eval/_eval_operators.py | 3 +++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/tests/test_qblike_2.py b/tests/test_qblike_2.py index ffc1980..6846abb 100644 --- a/tests/test_qblike_2.py +++ b/tests/test_qblike_2.py @@ -57,8 +57,8 @@ class MultiLink[T](Link[T]): ) -def select[K: BaseTypedDict]( - rcv: type[User], +def select[ModelT, K: BaseTypedDict]( + rcv: type[ModelT], /, **kwargs: Unpack[K], ) -> list[ @@ -66,7 +66,7 @@ def select[K: BaseTypedDict]( *[ Member[ GetName[c], - ConvertField[GetAttr[User, GetName[c]]], + ConvertField[GetAttr[ModelT, GetName[c]]], ] for c in Iter[Attrs[K]] ] @@ -99,7 +99,7 @@ class User: posts: Link[Post] -def test_qblike_1(): +def test_qblike2_1(): ret = eval_call( select, User, @@ -117,7 +117,7 @@ class select[...]: """) -def test_qblike_2(): +def test_qblike2_2(): ret = eval_call( select, User, diff --git a/typemap/type_eval/_eval_call.py b/typemap/type_eval/_eval_call.py index f874a25..d71007a 100644 --- a/typemap/type_eval/_eval_call.py +++ b/typemap/type_eval/_eval_call.py @@ -9,6 +9,7 @@ from . import _eval_typing +from . import _typing_inspect RtType = Any @@ -18,6 +19,8 @@ def _type(t): if t is None or isinstance(t, (int, str, bool, bytes, enum.Enum)): return typing.Literal[t] + elif isinstance(t, type): + return type[t] else: return type(t) @@ -39,6 +42,7 @@ def _get_bound_type_args( vars: dict[str, RtType] = {} # TODO: duplication, error cases for param in sig.parameters.values(): + # Unpack[TypeVarType] for *args if ( param.kind == inspect.Parameter.VAR_POSITIONAL # XXX: typing_extensions also @@ -51,6 +55,7 @@ def _get_bound_type_args( ): tps = bound.arguments.get(param.name, ()) vars[tv.__name__] = tuple[tps] # type: ignore[valid-type] + # Unpack[T] for **kwargs elif ( param.kind == inspect.Parameter.VAR_KEYWORD # XXX: typing_extensions also @@ -65,6 +70,18 @@ def _get_bound_type_args( ): tp = typing.TypedDict(f"**{param.name}", bound.kwargs) # type: ignore[misc, operator] vars[tv.__name__] = tp + # trivial type[T] bindings + elif ( + _typing_inspect.is_generic_alias(param.annotation) + and param.annotation.__origin__ is type + and (tv := param.annotation.__args__[0]) + and isinstance(tv, typing.TypeVar) + and (arg := bound.arguments.get(param.name)) + and _typing_inspect.is_generic_alias(arg) + and arg.__origin__ is type + ): + vars[tv.__name__] = arg.__args__[0] + # trivial T bindings elif ( isinstance(param.annotation, typing.TypeVar) and param.name in bound.arguments diff --git a/typemap/type_eval/_eval_operators.py b/typemap/type_eval/_eval_operators.py index c9c4349..0694821 100644 --- a/typemap/type_eval/_eval_operators.py +++ b/typemap/type_eval/_eval_operators.py @@ -218,6 +218,9 @@ def _eval_Iter(tp, *, ctx): return iter(tp.__args__) else: # XXX: Or should we return []? + # We *definitely* should return [] for Never + # Maybe we should lift over unions and return the union of + # each tuples position... raise TypeError( f"Invalid type argument to Iter: {tp} is not a fixed-length tuple" ) From 22c96e184b6ec51ce419925c6fa4600c2c9fd04e Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Fri, 16 Jan 2026 14:29:38 -0800 Subject: [PATCH 4/6] Write a kind of tutorial introduction to qblike_2 in docstrings --- pre-pep.rst | 7 +-- tests/test_qblike_2.py | 108 +++++++++++++++++++++++++++++++++-------- 2 files changed, 92 insertions(+), 23 deletions(-) diff --git a/pre-pep.rst b/pre-pep.rst index e0e41fd..acdd54f 100644 --- a/pre-pep.rst +++ b/pre-pep.rst @@ -131,7 +131,6 @@ inferring the type of a function is much less likely to be feasible. Implementation '''''''''''''' -We have a more `worked example <#qb-test_>`_ in our test suite. Automatically deriving FastAPI CRUD models @@ -237,7 +236,7 @@ suite, but here is a possible implementation of just ``Public``:: # If it is a Field, then we try pulling out the "default" field, # otherwise we return the type itself. type GetDefault[Init] = ( - GetFieldItem[Init, Literal["default"]] if Sub[Init, Field] else Init + GetFieldItem[Init, Literal["default"]] if IsSub[Init, Field] else Init ) # Create takes everything but the primary key and preserves defaults @@ -245,7 +244,7 @@ suite, but here is a possible implementation of just ``Public``:: *[ Member[GetName[p], GetType[p], GetQuals[p], GetDefault[GetInit[p]]] for p in Iter[Attrs[T]] - if not Sub[ + if not IsSub[ Literal[True], GetFieldItem[GetInit[p], Literal["primary_key"]] ] ] @@ -355,6 +354,8 @@ Reference Implementation Rejected Ideas ============== +* Don't attempt to support runtime evaluation, make + [Why certain ideas that were brought while discussing this PEP were not ultimately pursued.] diff --git a/tests/test_qblike_2.py b/tests/test_qblike_2.py index 6846abb..4f8ff7c 100644 --- a/tests/test_qblike_2.py +++ b/tests/test_qblike_2.py @@ -19,11 +19,29 @@ from . import format_helper -class Property[T]: +# Begin PEP section: Prisma-style ORMs + +""" +We will walk through the implementation. + +This will take something of a tutorial approach, and explain some of +the features being used. More details were appear in the specification +section. + +First, to support the annotations we saw above, we have a collection +of dummy classes with generic types. +""" + + +class Pointer[T]: + pass + + +class Property[T](Pointer[T]): pass -class Link[T]: +class Link[T](Pointer[T]): pass @@ -35,30 +53,25 @@ class MultiLink[T](Link[T]): pass -type DropProp[T] = GetArg[T, Property, 0] +""" +The ``select`` method is where we start seeing new things. -type PropsOnly[T] = list[ - NewProtocol[ - *[ - Member[GetName[p], DropProp[GetType[p]]] - for p in Iter[Attrs[T]] - if Sub[GetType[p], Property] - ] - ] -] +The ``**kwargs: Unpack[K]`` is part of this proposal, and allows +*inferring* a TypedDict from keyword args. -type AdjustLink[Tgt, LinkTy] = list[Tgt] if Sub[LinkTy, MultiLink] else Tgt +``Attrs[K]`` extracts ``Member`` types corresponding to every +type-annotated attribute of ``K``, while calling ``NewProtocol`` with +``Member`` arguments constructs a new structural type. -# Conditional type alias! -type ConvertField[T] = ( - AdjustLink[PropsOnly[GetArg[T, Link, 0]], T] - if Sub[T, Link] - else DropProp[T] -) +``GetName`` is a getter operator that fetches the name of a ``Member`` +as a literal type--all of these mechanisms lean very heavily on literal types. +``GetAttr`` gets the type of an attribute from a class. + +""" def select[ModelT, K: BaseTypedDict]( - rcv: type[ModelT], + typ: type[ModelT], /, **kwargs: Unpack[K], ) -> list[ @@ -74,6 +87,61 @@ def select[ModelT, K: BaseTypedDict]( ]: ... +"""ConvertField is our first type helper, and it is a conditional type +alias, which decides between two types based on a (limited) +subtype-ish check. + +In ``ConvertField``, we wish to drop the ``Property`` or ``Link`` +annotation and produce the underlying type, as well as, for links, +producing a new target type containing only properties and wrapping +``MultiLink`` in a list. +""" + +type ConvertField[T] = ( + AdjustLink[PropsOnly[PointerArg[T]], T] if Sub[T, Link] else PointerArg[T] +) + +"""``PointerArg`` gets the type argument to ``Pointer`` or a subclass. + +``GetArg[T, Base, I]`` is one of the core primitives; it fetches the +index ``I`` type argument to ``Base`` from a type ``T``, if ``T`` +inherits from ``Base``. + +(The subtleties of this will be discussed later; in this case, it just +grabs the argument to a ``Pointer``). + +""" +type PointerArg[T: Pointer] = GetArg[T, Pointer, 0] + +""" +```AdjustLink sticks a `list` around `MultiLink`, using features +we've discussed already. + +""" +type AdjustLink[Tgt, LinkTy] = list[Tgt] if Sub[LinkTy, MultiLink] else Tgt + +"""And the final helper, ``PropsOnly[T]``, generates a new type that +contains all the ``Property`` attributes of ``T``. + +""" +type PropsOnly[T] = list[ + NewProtocol[ + *[ + Member[GetName[p], PointerArg[GetType[p]]] + for p in Iter[Attrs[T]] + if Sub[GetType[p], Property] + ] + ] +] + +""" +The full test is `in our test suite <#qb-test_>`_. +""" + + +# End PEP section + + # Basic filtering class Comment: id: Property[int] From ac8103cc776664be37b0b4b44f5f2c767008172a Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Fri, 16 Jan 2026 14:39:44 -0800 Subject: [PATCH 5/6] Copy it over --- pre-pep.rst | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/pre-pep.rst b/pre-pep.rst index acdd54f..9fd6204 100644 --- a/pre-pep.rst +++ b/pre-pep.rst @@ -131,6 +131,108 @@ inferring the type of a function is much less likely to be feasible. Implementation '''''''''''''' +We will walk through the implementation. + +This will take something of a tutorial approach, and explain some of +the features being used. More details will appear in the specification +section. + +First, to support the annotations we saw above, we have a collection +of dummy classes with generic types:: + + class Pointer[T]: + pass + + + class Property[T](Pointer[T]): + pass + + + class Link[T](Pointer[T]): + pass + + + class SingleLink[T](Link[T]): + pass + + + class MultiLink[T](Link[T]): + pass + +The ``select`` method is where we start seeing new things. + +The ``**kwargs: Unpack[K]`` is part of this proposal, and allows +*inferring* a TypedDict from keyword args. + +``Attrs[K]`` extracts ``Member`` types corresponding to every +type-annotated attribute of ``K``, while calling ``NewProtocol`` with +``Member`` arguments constructs a new structural type. + +``GetName`` is a getter operator that fetches the name of a ``Member`` +as a literal type--all of these mechanisms lean very heavily on literal types. +``GetAttr`` gets the type of an attribute from a class. + +:: + + def select[ModelT, K: BaseTypedDict]( + typ: type[ModelT], + /, + **kwargs: Unpack[K], + ) -> list[ + NewProtocol[ + *[ + Member[ + GetName[c], + ConvertField[GetAttr[ModelT, GetName[c]]], + ] + for c in Iter[Attrs[K]] + ] + ] + ]: ... + +``ConvertField`` is our first type helper, and it is a conditional type +alias, which decides between two types based on a (limited) +subtype-ish check. + +In ``ConvertField``, we wish to drop the ``Property`` or ``Link`` +annotation and produce the underlying type, as well as, for links, +producing a new target type containing only properties and wrapping +``MultiLink`` in a list:: + + type ConvertField[T] = ( + AdjustLink[PropsOnly[PointerArg[T]], T] if Sub[T, Link] else PointerArg[T] + ) + +``PointerArg`` gets the type argument to ``Pointer`` or a subclass. + +``GetArg[T, Base, I]`` is one of the core primitives; it fetches the +index ``I`` type argument to ``Base`` from a type ``T``, if ``T`` +inherits from ``Base``. + +(The subtleties of this will be discussed later; in this case, it just +grabs the argument to a ``Pointer``):: + + type PointerArg[T: Pointer] = GetArg[T, Pointer, 0] + +``AdjustLink`` sticks a ``list`` around ``MultiLink``, using features +we've discussed already:: + + type AdjustLink[Tgt, LinkTy] = list[Tgt] if Sub[LinkTy, MultiLink] else Tgt + +And the final helper, ``PropsOnly[T]``, generates a new type that +contains all the ``Property`` attributes of ``T``:: + + type PropsOnly[T] = list[ + NewProtocol[ + *[ + Member[GetName[p], PointerArg[GetType[p]]] + for p in Iter[Attrs[T]] + if Sub[GetType[p], Property] + ] + ] + ] + +The full test is `in our test suite <#qb-test_>`_. Automatically deriving FastAPI CRUD models From 95fc8bf9f185b4439c32093608284f90440f09e0 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Fri, 16 Jan 2026 14:45:42 -0800 Subject: [PATCH 6/6] Vibe up scripts for automatically syncing examples to the pre-pep --- pre-pep.rst | 47 +++++---- scripts/py2rst.py | 178 +++++++++++++++++++++++++++++++++ scripts/rst_replace_section.py | 168 +++++++++++++++++++++++++++++++ scripts/update-examples.sh | 8 ++ tests/test_fastapilike_2.py | 6 ++ tests/test_qblike_2.py | 11 +- 6 files changed, 390 insertions(+), 28 deletions(-) create mode 100755 scripts/py2rst.py create mode 100755 scripts/rst_replace_section.py create mode 100755 scripts/update-examples.sh diff --git a/pre-pep.rst b/pre-pep.rst index 9fd6204..baa663f 100644 --- a/pre-pep.rst +++ b/pre-pep.rst @@ -128,34 +128,32 @@ much need for runtime introspection of the types here, which is good: inferring the type of a function is much less likely to be feasible. +.. _qb-impl: + Implementation '''''''''''''' -We will walk through the implementation. - -This will take something of a tutorial approach, and explain some of -the features being used. More details will appear in the specification -section. +This will take something of a tutorial approach in discussing the +implementation, and explain the features being used as we use +them. More details were appear in the specification section. First, to support the annotations we saw above, we have a collection -of dummy classes with generic types:: +of dummy classes with generic types. + +:: class Pointer[T]: pass - class Property[T](Pointer[T]): pass - class Link[T](Pointer[T]): pass - class SingleLink[T](Link[T]): pass - class MultiLink[T](Link[T]): pass @@ -190,14 +188,16 @@ as a literal type--all of these mechanisms lean very heavily on literal types. ] ]: ... -``ConvertField`` is our first type helper, and it is a conditional type +ConvertField is our first type helper, and it is a conditional type alias, which decides between two types based on a (limited) subtype-ish check. In ``ConvertField``, we wish to drop the ``Property`` or ``Link`` annotation and produce the underlying type, as well as, for links, producing a new target type containing only properties and wrapping -``MultiLink`` in a list:: +``MultiLink`` in a list. + +:: type ConvertField[T] = ( AdjustLink[PropsOnly[PointerArg[T]], T] if Sub[T, Link] else PointerArg[T] @@ -210,17 +210,23 @@ index ``I`` type argument to ``Base`` from a type ``T``, if ``T`` inherits from ``Base``. (The subtleties of this will be discussed later; in this case, it just -grabs the argument to a ``Pointer``):: +grabs the argument to a ``Pointer``). + +:: type PointerArg[T: Pointer] = GetArg[T, Pointer, 0] ``AdjustLink`` sticks a ``list`` around ``MultiLink``, using features -we've discussed already:: +we've discussed already. + +:: type AdjustLink[Tgt, LinkTy] = list[Tgt] if Sub[LinkTy, MultiLink] else Tgt And the final helper, ``PropsOnly[T]``, generates a new type that -contains all the ``Property`` attributes of ``T``:: +contains all the ``Property`` attributes of ``T``. + +:: type PropsOnly[T] = list[ NewProtocol[ @@ -377,6 +383,11 @@ This kind of pattern is widespread enough that :pep:`PEP 681 <681>` was created to represent a lowest-common denominator subset of what existing libraries do. +.. _init-impl: + +Implementation +'''''''''''''' + :: # Generate the Member field for __init__ for a class @@ -411,12 +422,6 @@ existing libraries do. ] -TODO: We still need a full story on *how* best to apply this kind of -type modifier to a type. With dataclasses, which is a decorator, we -could put it in the decorator type... But what about things that use -``__init_subclass__`` or even metaclasses? - - Rationale ========= diff --git a/scripts/py2rst.py b/scripts/py2rst.py new file mode 100755 index 0000000..9757b03 --- /dev/null +++ b/scripts/py2rst.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Convert Python source code to reStructuredText. + +Top-level docstrings become regular RST text, everything else becomes +code blocks. +""" + +import argparse +import re +import sys + + +def convert_py_to_rst(source: str) -> str: + """Convert Python source to RST. + + Args: + source: Python source code + + Returns: + reStructuredText content + """ + lines = source.split('\n') + result: list[str] = [] + code_buffer: list[str] = [] + i = 0 + + def flush_code(): + """Flush accumulated code as a code block.""" + nonlocal code_buffer + # Strip leading/trailing empty lines from code buffer + while code_buffer and not code_buffer[0].strip(): + code_buffer.pop(0) + while code_buffer and not code_buffer[-1].strip(): + code_buffer.pop() + + if code_buffer: + result.append('::') + result.append('') + for line in code_buffer: + result.append(' ' + line if line.strip() else '') + result.append('') + code_buffer = [] + + while i < len(lines): + line = lines[i] + stripped = line.strip() + + # Check for start of a top-level docstring (triple quotes at column 0) + if stripped.startswith('"""') or stripped.startswith("'''"): + quote = stripped[:3] + + # Flush any pending code + flush_code() + + # Check if it's a single-line docstring + if stripped.count(quote) >= 2 and stripped.endswith(quote) and len(stripped) > 6: + # Single line docstring: """text""" + docstring_content = stripped[3:-3] + result.append(docstring_content) + result.append('') + i += 1 + continue + + # Multi-line docstring + docstring_lines: list[str] = [] + + # Handle first line - might have content after opening quotes + first_line_content = stripped[3:] + if first_line_content: + docstring_lines.append(first_line_content) + + i += 1 + + # Collect lines until closing quotes + while i < len(lines): + docline = lines[i] + if quote in docline: + # Found closing quotes + end_idx = docline.find(quote) + final_content = docline[:end_idx] + if final_content.strip(): + docstring_lines.append(final_content.rstrip()) + i += 1 + break + else: + docstring_lines.append(docline.rstrip()) + i += 1 + + # Output docstring content as RST text + for dline in docstring_lines: + result.append(dline) + result.append('') + + else: + # Regular code line + code_buffer.append(line) + i += 1 + + # Flush any remaining code + flush_code() + + # Clean up multiple consecutive blank lines + output: list[str] = [] + prev_blank = False + for line in result: + is_blank = not line.strip() + if is_blank and prev_blank: + continue + output.append(line) + prev_blank = is_blank + + # Remove trailing blank lines + while output and not output[-1].strip(): + output.pop() + + return '\n'.join(output) + '\n' + + +def main(): + parser = argparse.ArgumentParser( + description='Convert Python source to reStructuredText' + ) + parser.add_argument( + 'input', + nargs='?', + type=argparse.FileType('r'), + default=sys.stdin, + help='Input Python file (default: stdin)' + ) + parser.add_argument( + '-o', '--output', + type=argparse.FileType('w'), + default=sys.stdout, + help='Output RST file (default: stdout)' + ) + parser.add_argument( + '--start', + type=str, + default=None, + help='Start marker comment (content after this line)' + ) + parser.add_argument( + '--end', + type=str, + default=None, + help='End marker comment (content before this line)' + ) + + args = parser.parse_args() + + source = args.input.read() + + # Extract section between markers if specified + if args.start or args.end: + lines = source.split('\n') + start_idx = 0 + end_idx = len(lines) + + if args.start: + for idx, line in enumerate(lines): + if args.start in line: + start_idx = idx + 1 + break + + if args.end: + for idx, line in enumerate(lines): + if args.end in line: + end_idx = idx + break + + source = '\n'.join(lines[start_idx:end_idx]) + + rst = convert_py_to_rst(source) + args.output.write(rst) + + +if __name__ == '__main__': + main() diff --git a/scripts/rst_replace_section.py b/scripts/rst_replace_section.py new file mode 100755 index 0000000..e35ebf7 --- /dev/null +++ b/scripts/rst_replace_section.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Replace the contents of an RST section with contents from another file. + +The section is identified by a label that appears right before it. +""" + +import argparse +import re +import sys + + +# RST section characters in order of precedence (most to least significant) +SECTION_CHARS = ['=', '-', '`', ':', "'", '"', '~', '^', '_', '*', '+', '#'] + + +def get_section_level(underline: str) -> int | None: + """Get the section level from an underline character. + + Returns None if not a valid underline. + """ + if not underline.strip(): + return None + char = underline.strip()[0] + if char in SECTION_CHARS and underline.strip() == char * len(underline.strip()): + return SECTION_CHARS.index(char) + return None + + +def is_section_underline(line: str, prev_line: str) -> bool: + """Check if a line is a section underline for the previous line.""" + if not line.strip() or not prev_line.strip(): + return False + char = line.strip()[0] + if char not in SECTION_CHARS: + return False + if line.strip() != char * len(line.strip()): + return False + # Underline must be at least as long as the title + return len(line.rstrip()) >= len(prev_line.rstrip()) + + +def replace_section(rst_content: str, label: str, new_content: str) -> str: + """Replace a section's content identified by a label. + + Args: + rst_content: The RST file content + label: The label before the section (e.g., 'qb-impl' for '.. _qb-impl:') + new_content: The new content to insert + + Returns: + The modified RST content + """ + lines = rst_content.split('\n') + + # Find the label + label_pattern = re.compile(rf'^\.\.\s+_({re.escape(label)}|#{re.escape(label)}):\s*$') + label_idx = None + for i, line in enumerate(lines): + if label_pattern.match(line): + label_idx = i + break + + if label_idx is None: + raise ValueError(f"Label '{label}' not found in RST content") + + # Find the section heading after the label + section_title_idx = None + section_underline_idx = None + section_level = None + + for i in range(label_idx + 1, len(lines) - 1): + if is_section_underline(lines[i + 1], lines[i]): + section_title_idx = i + section_underline_idx = i + 1 + section_level = get_section_level(lines[i + 1]) + break + + if section_title_idx is None: + raise ValueError(f"No section heading found after label '{label}'") + + # Find where the section ends (next section at same or higher level) + section_end_idx = len(lines) + + for i in range(section_underline_idx + 1, len(lines) - 1): + if is_section_underline(lines[i + 1], lines[i]): + next_level = get_section_level(lines[i + 1]) + if next_level is not None and next_level <= section_level: + section_end_idx = i + break + + # Build the result + result_lines = [] + + # Everything up to and including the section underline + result_lines.extend(lines[:section_underline_idx + 1]) + + # Add blank line after heading if new content doesn't start with one + new_content_lines = new_content.rstrip('\n').split('\n') + if new_content_lines and new_content_lines[0].strip(): + result_lines.append('') + + # Add the new content + result_lines.extend(new_content_lines) + + # Add blank line before next section if needed + if section_end_idx < len(lines): + if result_lines and result_lines[-1].strip(): + result_lines.append('') + result_lines.append('') + + # Everything from the next section onwards + result_lines.extend(lines[section_end_idx:]) + + return '\n'.join(result_lines) + + +def main(): + parser = argparse.ArgumentParser( + description='Replace an RST section with content from another file' + ) + parser.add_argument( + 'rst_file', + help='The RST file to modify' + ) + parser.add_argument( + 'label', + help='The label identifying the section (e.g., "qb-impl" for ".. _qb-impl:")' + ) + parser.add_argument( + 'content_file', + nargs='?', + type=argparse.FileType('r'), + default=sys.stdin, + help='File containing new section content (default: stdin)' + ) + parser.add_argument( + '-i', '--in-place', + action='store_true', + help='Modify the RST file in place' + ) + parser.add_argument( + '-o', '--output', + type=str, + default=None, + help='Output file (default: stdout, or same as input with -i)' + ) + + args = parser.parse_args() + + with open(args.rst_file, 'r') as f: + rst_content = f.read() + + new_content = args.content_file.read() + + result = replace_section(rst_content, args.label, new_content) + + if args.in_place: + with open(args.rst_file, 'w') as f: + f.write(result) + elif args.output: + with open(args.output, 'w') as f: + f.write(result) + else: + sys.stdout.write(result) + + +if __name__ == '__main__': + main() diff --git a/scripts/update-examples.sh b/scripts/update-examples.sh new file mode 100755 index 0000000..773b5b3 --- /dev/null +++ b/scripts/update-examples.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +scripts/py2rst.py tests/test_qblike_2.py --start "Begin PEP section" --end "End PEP section" \ + | scripts/rst_replace_section.py pre-pep.rst qb-impl -i + + +scripts/py2rst.py tests/test_fastapilike_2.py --start "Begin PEP section" --end "End PEP section" \ + | scripts/rst_replace_section.py pre-pep.rst init-impl -i diff --git a/tests/test_fastapilike_2.py b/tests/test_fastapilike_2.py index 30ba3f3..c21e46f 100644 --- a/tests/test_fastapilike_2.py +++ b/tests/test_fastapilike_2.py @@ -113,6 +113,9 @@ class Field[T: FieldArgs](InitField[T]): ## +# Begin PEP section: dataclass like __init__ + + # Generate the Member field for __init__ for a class type InitFnType[T] = Member[ Literal["__init__"], @@ -145,6 +148,9 @@ class Field[T: FieldArgs](InitField[T]): ] +# End PEP section + + #### # This is the FastAPI example code that we are trying to repair! diff --git a/tests/test_qblike_2.py b/tests/test_qblike_2.py index 4f8ff7c..de493f7 100644 --- a/tests/test_qblike_2.py +++ b/tests/test_qblike_2.py @@ -21,12 +21,9 @@ # Begin PEP section: Prisma-style ORMs -""" -We will walk through the implementation. - -This will take something of a tutorial approach, and explain some of -the features being used. More details were appear in the specification -section. +"""This will take something of a tutorial approach in discussing the +implementation, and explain the features being used as we use +them. More details were appear in the specification section. First, to support the annotations we saw above, we have a collection of dummy classes with generic types. @@ -114,7 +111,7 @@ def select[ModelT, K: BaseTypedDict]( type PointerArg[T: Pointer] = GetArg[T, Pointer, 0] """ -```AdjustLink sticks a `list` around `MultiLink`, using features +``AdjustLink`` sticks a ``list`` around ``MultiLink``, using features we've discussed already. """