diff --git a/.gitignore b/.gitignore index 46251f6..1c15be0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ dist/ src/pyawaitable/pyawaitable.h pcbuild/ *.o +site/ +_build/ # LSP compile_flags.txt diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/adding_awaits.md b/docs/adding_awaits.md deleted file mode 100644 index 2c361cd..0000000 --- a/docs/adding_awaits.md +++ /dev/null @@ -1,280 +0,0 @@ -# Executing Asynchronous Calls from C - -Let's say we wanted to replicate the following in C: - -```py -async def trampoline(coro: collections.abc.Coroutine) -> int: - return await coro -``` - -This is simply a function that we pass a coroutine to, and it will await it for us. It's not particularly useful, but it's just for learning purposes. - -We already know that `trampoline()` will evaluate to a magic coroutine object that supports `await`, via the `__await__` dunder, and needs to `yield from` its coroutines. So, if we wanted to break `trampoline` down into a synchronous Python function, it would look something like this: - -```py -class _trampoline_coroutine: - def __init__(self, coro: collections.abc.Coroutine) -> None: - self.coro = coro - - def __await__(self) -> collections.abc.Generator: - yield - yield from self.coro.__await__() - -def trampoline(coro: collections.abc.Coroutine) -> collections.abc.Coroutine: - return _trampoline_coroutine(coro) -``` - -But, this is using `yield from`; there's no `yield from` in C, so how do we actually await things, or more importantly, use their return value? This is where things get tricky. - -## Adding Awaits to a PyAwaitable Object - -There's one big function for "adding" coroutines to a PyAwaitable object: `PyAwaitable_AddAwait`. By "add," we mean that the asynchronous call won't happen right then and there. Instead, the PyAwaitable will store it, and then when something comes to call the `__await__` on the PyAwaitable object, it will mimick a `yield from` on that coroutine. - -`PyAwaitable_AddAwait` takes four arguments: - -- The PyAwaitable object. -- The _coroutine_ to store. (Not an `async def` function, but the result of calling one without `await`.) -- A callback. -- An error callback. - -Let's focus on the first two for now, and just pass `NULL` for the other two in the meantime. We can implement `trampoline` from our earlier example pretty easily: - -```c -static PyObject * -trampoline(PyObject *self, PyObject *coro) // METH_O -{ - PyObject *awaitable = PyAwaitable_New(); - if (awaitable == NULL) { - return NULL; - } - - if (PyAwaitable_AddAwait(awaitable, coro, NULL, NULL) < 0) { - Py_DECREF(awaitable); - return NULL; - } - - return awaitable; -} -``` - -To your eyes, the `yield from` and all of that mess is completely hidden; you give PyAwaitable your coroutine, and it handles the rest! `trampoline` now acts like our pure-Python function from earlier: - -```py ->>> from _yourmod import trampoline ->>> import asyncio ->>> await trampoline(asyncio.sleep(2)) # Sleeps for 2 seconds -``` - -Yay! We called an asynchronous function from C! - -## Getting the Return Value in a Callback - -In many cases, it's desirable to use the return value of a coroutine. For example, let's say we wanted to get the result of the following asynchronous function: - -```py -async def silly() -> int: - await asyncio.sleep(2) # Simulate doing some I/O work - return 42 -``` - -The details of how coroutines return values aren't relevant, but we do know that a coroutine isn't actually "awaited" until _after_ we've already returned our PyAwaitable object from C. That means we have to use a callback to get the return value of the coroutine. - -Specifically, we can pass a function pointer to the third parameter of `PyAwaitable_AddAwait`. A callback function takes two `PyObject *` parameters: - -- A _borrowed_ reference to the PyAwaitable object that called it. -- A _borrowed_ reference to the return value of the coroutine. - -A callback must return `0` to indicate success, or `-1` with an exception set to indicate failure. - -Now, we can use the result of `silly` in C: - -```c -static int -callback(PyObject *awaitable, PyObject *value) -{ - if (PyAwaitable_SetResult(awaitable, value) < 0) { - return -1; - } - - return 0; -} - -static PyObject * -call_silly(PyObject *self, PyObject *silly) -{ - PyObject *awaitable = PyAwaitable_New(); - if (awaitable == NULL) { - return NULL; - } - - // Get the coroutine by calling silly() - PyObject *coro = PyObject_CallNoArgs(silly); - if (coro == NULL) { - Py_DECREF(awaitable); - return NULL; - } - - if (PyAwaitable_AddAwait(awaitable, coro, callback, NULL) < 0) { - Py_DECREF(awaitable); - Py_DECREF(coro); - return NULL; - } - - Py_DECREF(coro); - return awaitable; -} -``` - -This can be used from Python as such: - -```py ->>> from _yourmod import call_silly ->>> await call_silly(silly) # Sleeps for 2 seconds -silly() returned: 42 -``` - -## Handling Errors with Callbacks - -Coroutines can raise exceptions, during execution. For example, imagine we wanted to use a function that makes a network request: - -```py -import asyncio - - -async def make_request() -> str: - async with asyncio.timeout(5): - await asyncio.sleep(10) # Simulate some I/O - return "..." -``` - -The above will raise `TimeoutError`, but not on simply calling `make_request()`; it will only raise once it's actually started executing in an `await`, and as we already know that coroutines don't execute at the `PyAwaitable_AddAwait` callsite, we can't simply check for errors there. So, similar to return value callbacks, PyAwaitable provides error callbacks, which--you guessed it--is the fourth argument to `PyAwaitable_AddAwait`. - -An error callback has the same signature as a return value callback, but instead of taking a reference to a return value, it takes a borrowed reference to an exception object that was caught and raised by either the coroutine or the coroutine's callback. - -!!! note - - Error callbacks are *not* called with an exception "set" (*i.e.*, `PyErr_Occurred()` returns `NULL`), so it's safe to call most of Python's C API without worrying about those kinds of failures. - -An error callback's return value can do a number of different things to the state of the PyAwaitable object's exception. Namely: - -- Returning `0` will consider the error successfully caught, so PyAwaitable object will clear the exception and continue executing the rest of its coroutine. -- Returning `-1` indicates that the error should be repropagated. The PyAwaitable object will officially "set" the Python exception (via `PyErr_SetRaisedException`), raise the error to the event loop and stop itself from executing any future coroutines. -- Returning `-2` indicates that a new error occurred while handling the other one; the original exception is _not_ restored, and an exception set by the error callback is used instead and propagated to the event loop. - -!!! note - - Return value callbacks are not called if an exception occurred while executing the coroutine. - -To try and give a real-world example of all three of these, let's implement the following function in C: - -```py -async def is_api_reachable(make_request: Callable[[], collections.abc.Coroutine]) -> bool: - try: - await make_request() - return True - except TimeoutError: - return False -``` - -!!! note - - `asyncio.TimeoutError` is an alias of the `TimeoutError` builtin. - -We have to do several things here: - -- Call `make_request` to get the coroutine object to `await`. -- Add an error callback for that coroutine. -- In a return value callback, set the return value to `True`, because that means the operation didn't time out. -- In an error callback, check if the exception is an instance of `TimeoutError`, and set the return value to `False` if it is. -- If its something other than `TimeoutError`, let it propagate. - -In C, all that would be implemented like this: - -```c -static int -return_true(PyObject *awaitable, PyObject *unused) -{ - return PyAwaitable_SetResult(awaitable, Py_True); -} - -static int -return_false(PyObject *awaitable, PyObject *exc) -{ - if (PyErr_GivenExceptionMatches(exc, PyExc_TimeoutError)) { - if (PyAwaitable_SetResult(exc, Py_False) < 0) { - // New exception occurred; give it to the event loop. - return -2; - } - - return 0; - } else { - // This isn't a TimeoutError! - return -1; - } -} - -static PyObject * -is_api_reachable(PyObject *self, PyObject *make_request) -{ - PyObject *awaitable = PyAwaitable_New(); - if (awaitable == NULL) { - return NULL; - } - - // Remember, this isn't the same as executing the coroutine, so - // the timeout doesn't show up here. But, we still need to handle - // an exception case, because something might have gone wrong - // in getting the coroutine object, e.g., the object isn't callable - // or we're out of memory. - PyObject *coro = PyObject_CallNoArgs(make_request); - if (coro == NULL) { - Py_DECREF(awaitable); - return NULL; - } - - if (PyAwaitable_AddAwait(awaitable, coro, return_true, return_false)) { - Py_DECREF(awaitable); - Py_DECREF(coro); - return NULL; - } - - Py_DECREF(coro); - return awaitable; -} -``` - -### Propagation of Errors in Return Value Callbacks - -By default, returning `-1` from a return value callback will implicitly call the error callback if one is set. But, this isn't always desirable; sometimes, you want to let errors in callbacks bubble up instead of getting handled by some default error handling mechanism you've installed. - -You can force the PyAwaitable object to propagate the exception by returning `-2` from a return value callback. If `-2` is returned, the exception set by the callback will always be raised back to whoever `await`ed the PyAwaitable object. - -For example, if we installed some global exception logger inside of the error callback, but don't want that to grab things like a `MemoryError` inside of the return callback, we would return `-2`: - -```c -static int -error_handler(PyObject *awaitable, PyObject *error) -{ - // Simply print the error and continue execution - PyErr_SetRaisedException(Py_NewRef(error)); - PyErr_Print(); - return 0; -} - -static int -handle_value(PyObject *awaitable, PyObject *something) -{ - PyObject *message = PyUnicode_FromString("LOG: Got value"); - if (message == NULL) { - // Skip the error callback - return -2; - } - - if (magically_log_value(message, something) < 0) { - // Skip the error callback - return -2; - } - - return 0; -} -``` diff --git a/docs/c_async.md b/docs/c_async.md deleted file mode 100644 index c37e2d1..0000000 --- a/docs/c_async.md +++ /dev/null @@ -1,109 +0,0 @@ -# Making a C Function Asynchronous - -Let's make a C function that replicates the following Python code: - -```py -async def hello() -> None: - print("Hello, PyAwaitable") -``` - -If you've tried to implement an asynchronous C function in the past, this is likely where you got stuck. How do we make a C function `async`? - -## Breaking Down Awaitable Functions - -In Python, you have to _call_ an `async def` function to use it with `await`. In our example above, the following would be invalid: - -```py ->>> await hello -``` - -Of course, you need to do `await hello()` instead. `hello()` is returning a _coroutine_, and coroutine objects are usable with the `await` keyword. So, `hello` as a synchronous function would really be like: - -```py -class _hello_coroutine: - def __await__(self) -> collections.abc.Generator: - print("Hello, PyAwaitable") - yield - -def hello() -> collections.abc.Coroutine: - return _hello_coroutine() -``` - -If there were to be `await` expressions inside `hello`, the returned coroutine object would handle those by yielding inside of the `__await__` dunder method. We can do the same kind of thing in C. - -## Creating PyAwaitable Objects - -You can create a new PyAwaitable object with `PyAwaitable_New`. This returns a _strong reference_ to a PyAwaitable object, and `NULL` with an exception set on failure. - -Think of a PyAwaitable object sort of like the `_hello_coroutine` example from above, but it's _generic_ instead of being special for `hello`. So, like our Python example, we need to return the coroutine to allow it to be used in `await` expressions: - -```c -static PyObject * -hello(PyObject *self, PyObject *nothing) // METH_NOARGS -{ - PyObject *awaitable = PyAwaitable_New(); - if (awaitable == NULL) { - return NULL; - } - - puts("Hello, PyAwaitable"); - return awaitable; -} -``` - -!!! note "There's a difference between native coroutines and implemented coroutines" - - "Coroutine" is a bit of an ambigious term in Python. There are two types of coroutines: native ones ([`types.CoroutineType`](https://docs.python.org/3/library/types.html#types.CoroutineType)), and objects that implement the *coroutine protocol* ([`collections.abc.Coroutine`](https://docs.python.org/3/library/types.html#types.CoroutineType)). Only the interpreter itself can create native coroutines, so a PyAwaitable object is an object that implements the coroutine protocol. - -Yay! We can now use `hello` in `await` expressions: - -```py ->>> from _yourmod import hello ->>> await hello() -Hello, PyAwaitable -``` - -## Changing the Return Value - -Note that in all code-paths, we should return the PyAwaitable object, or `NULL` with an exception set to indicate a failure. But that means you can't simply `return` your own value; how can the `await` expression evaluate to something useful? - -By default, the "return value" (_i.e._, what `await` will evaluate to) is `None`. That can be changed with `PyAwaitable_SetResult`, which takes a reference to the object you want to return. - -For example, if you wanted to return the Python integer `42` from `hello`, you would simply pass that to `PyAwaitable_SetResult`: - -```c -static PyObject * -hello(PyObject *self, PyObject *nothing) // METH_NOARGS -{ - PyObject *awaitable = PyAwaitable_New(); - if (awiatable == NULL) { - return NULL; - } - - PyObject *my_number = PyLong_FromLong(42); - if (my_number == NULL) { - Py_DECREF(awaitable); - return NULL; - } - - if (PyAwaitable_SetResult(awaitable, my_number) < 0) { - Py_DECREF(awaitable); - Py_DECREF(my_number); - return NULL; - } - - Py_DECREF(my_number); - - puts("Hello, PyAwaitable"); - return awaitable; -} -``` - -Now, the `await` expression evalutes to `42`: - -```py ->>> from _yourmod import hello ->>> await hello() -Hello, PyAwaitable -42 -``` diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..161e20e --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,29 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'PyAwaitable' +copyright = '2025, Peter Bierma' +author = 'Peter Bierma' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["sphinx.ext.intersphinx"] + +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} + + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'shibuya' +html_theme_options = { + "accent_color": "sky", +} diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index dc27039..0000000 --- a/docs/index.md +++ /dev/null @@ -1,68 +0,0 @@ -# PyAwaitable - -!!! note - - This project originates from a scrapped PEP. For the original text, see [here](https://gist.github.com/ZeroIntensity/8d32e94b243529c7e1c27349e972d926). - -## Introduction - -CPython currently has no existing C interface for writing asynchronous functions or doing any sort of `await` operations, other than defining extension types and manually implementing methods like `__await__` from scratch. This lack of an API can be seen in some Python-to-C transpilers (such as `mypyc`) having limited support for asynchronous code. - -In the C API, developers are forced to do one of three things when it comes to asynchronous code: - -- Manually implementing coroutines using extension types. -- Use an external tool to compile their asynchronous code to C. -- Defer their asynchronous logic to a synchronous Python function, and then call that natively. - -Since there are other event loop implementations, PyAwaitable aims to be a _generic_ interface for working with asynchronous operations from C (as in, we'll only be implementing features like `async def` and `await`, but not things like `asyncio.create_task`.) - -This documentation assumes that you're familiar with the C API already, and understand some essential concepts like reference counting (as well as borrowed and strong references). If you don't know what any of that means, it's highly advised that you read through the [Python docs](https://docs.python.org/3/extending/extending.html) before trying to use PyAwaitable. - -## Quickstart - -Add PyAwaitable as a build dependency: - -```toml -# pyproject.toml -[build-system] -requires = ["your_preferred_build_system", "pyawaitable>=2.0.0"] -build-backend = "your_preferred_build_system.build" -``` - -Use it in your extension: - -```c -/* - Equivalent to the following Python function: - - async def async_function(coro: collections.abc.Awaitable) -> None: - await coro - - */ -static PyObject * -async_function(PyObject *self, PyObject *coro) -{ - PyObject *awaitable = PyAwaitable_New(); - if (awaitable == NULL) { - return NULL; - } - - if (PyAwaitable_AddAwait(awaitable, coro, NULL, NULL) < 0) { - Py_DECREF(awaitable); - return NULL; - } - - return awaitable; -} -``` - -!!! note - - You need to call `PyAwaitable_Init` upon initializing your extension! This can be done in the `PyInit_` function, or a module-exec function if using [multi-phase initialization](https://docs.python.org/3/c-api/module.html#initializing-c-modules). - -## Acknowledgements - -Special thanks to: - -- [Petr Viktorin](https://github.com/encukou), for his feedback on the initial API design and PEP. -- [Sean Hunt](https://github.com/AraHaan), for beta testing. diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..3c67259 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,100 @@ +.. PyAwaitable documentation master file, created by + sphinx-quickstart on Sat Aug 16 11:38:02 2025. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +PyAwaitable documentation +========================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + installation + usage/c_async + usage/adding_awaits + usage/value_storage + reference + +.. note:: + + This project originates from a scrapped PEP. For the original text, + see `here `_. + +Introduction +------------ + +CPython currently has no existing C interface for writing asynchronous functions +or doing any sort of :keyword:`await` operations, other than defining +extension types and manually implementing methods like +:attr:`~object.__await__` from scratch. This lack of an API can be seen in +some Python-to-C transpilers (such as ``mypyc``) having limited support for +asynchronous code. + +In the C API, developers are forced to do one of three things when it comes +to asynchronous code: + +- Manually implementing coroutines using extension types. +- Use an external tool to compile their asynchronous code to C. +- Defer their asynchronous logic to a synchronous Python function, and then + call that natively. + +Since there are other event loop implementations, PyAwaitable aims to be a +generic interface for working with asynchronous operations from C (as in, +we'll only be implementing features like ``async def`` and :keyword:`await`, +but not things like :func:`asyncio.create_task`). + +This documentation assumes that you're familiar with the C API already, +and understand some essential concepts like reference counting (as well as +borrowed and strong references). If you don't know what any of that means, it's +highly advised that you read through the +:ref:`Python documentation ` before trying to use +PyAwaitable. + +Quickstart +---------- + +Add PyAwaitable as a build dependency: + +.. code-block:: toml + + # pyproject.toml + [build-system] + requires = ["your_preferred_build_system", "pyawaitable>=2.0.0"] + build-backend = "your_preferred_build_system.build" + +Use it in your extension: + +.. code-block:: c + + /* + Equivalent to the following Python function: + + async def async_function(coro: collections.abc.Awaitable) -> None: + await coro + */ + static PyObject * + async_function(PyObject *self, PyObject *coro) + { + PyObject *awaitable = PyAwaitable_New(); + if (awaitable == NULL) { + return NULL; + } + + if (PyAwaitable_AddAwait(awaitable, coro, NULL, NULL) < 0) { + Py_DECREF(awaitable); + return NULL; + } + + return awaitable; + } + + +Acknowledgements +---------------- + +Special thanks to: + +- `Petr Viktorin `_, for his feedback on the + initial API design and PEP. +- `Sean Hunt `_, for beta testing. diff --git a/docs/installation.md b/docs/installation.md deleted file mode 100644 index 89974c5..0000000 --- a/docs/installation.md +++ /dev/null @@ -1,171 +0,0 @@ -# Installation - -## Build Dependency - -PyAwaitable needs to be installed as a build dependency, not a runtime dependency. For the `project.dependencies` portion of your `pyproject.toml`, you can completely omit PyAwaitable as a dependency! - -For example, in a `setuptools` based project, your `build-system` section would look like: - -```toml -# pyproject.toml -[build-system] -requires = ["setuptools~=78.1", "pyawaitable>=2.0.0"] -build-backend = "setuptools.build_meta" -``` - -## Including the `pyawaitable.h` File - -PyAwaitable provides a number of APIs to access the include path for a number of different build systems. Namely: - -- `pyawaitable.include()` is available in Python code (typically for `setup.py`-based extensions). -- `PYAWAITABLE_INCLUDE` is accessible as an environment variable, but only if Python has been started without `-S` (this is useful for `scikit-build-core` projects). -- `pyawaitable --include` returns the path of the include directory (useful for everything else, such as `meson-python`). - -!!! note - - PyAwaitable uses a nifty trick for building itself into your project. Python's packaging ecosystem isn't exactly great at distributing C libraries, so the `pyawaitable.h` actually contains the entire PyAwaitable source code (but with mangled names to prevent collisions with your own project). - - This has some pros and cons: - - - PyAwaitable doesn't need to be installed at runtime, as it's embedded directly into your extension. This means it's *extremely* portable; completely different PyAwaitable versions can peacefully coexist in the same process. - - Enabling debug flags in your extension also means enabling debug flags in PyAwaitable, thus enabling assertions and whatnot. This is useful for debugging. - - However, PyAwaitable can't use the limited API, so it prevents your extension from using the limited API (see the note below). - -## Initializing PyAwaitable in Your Extension - -PyAwaitable has to do a one-time initialization to get its types and other state initialized in the Python process. This is done with `PyAwaitable_Init`, which can be called basically anywhere, as long as its called before any other PyAwaitable functions are used. - -Typically, you'll want to call this in your extension's `PyInit_` function, or in the module-exec function in multi-phase extensions. For example: - -```c -// Single-phase -PyMODINIT_FUNC -PyInit_mymodule() -{ - if (PyAwaitable_Init() < 0) { - return NULL; - } - - return PyModule_Create(/* ... */); -} -``` - -```c -// Multi-phase -static int -module_exec(PyObject *mod) -{ - return PyAwaitable_Init(); -} -``` - -!!! warning "No Limited API Support" - - Unfortunately, PyAwaitable cannot be used with the [limited C API](https://docs.python.org/3/c-api/stable.html#limited-c-api). This is due to PyAwaitable needing [am_send](https://docs.python.org/3/c-api/typeobj.html#c.PyAsyncMethods.am_send) to implement the coroutine protocol on 3.10+, but the corresponding heap-type slot `Py_am_send` was not added until 3.11. Therefore, PyAwaitable cannot support the limited API without dropping support for <3.11. - -## Examples - -### `setuptools` - -```py -# setup.py -from setuptools import setup, Extension -import pyawaitable - -if __name__ == "__main__": - setup( - ext_modules=[ - Extension("_module", ["src/module.c"], include_dirs=[pyawaitable.include()]) - ] - ) -``` - -### `scikit-build-core` - -```t -# CMakeLists.txt -cmake_minimum_required(VERSION 3.15...3.30) -project(${SKBUILD_PROJECT_NAME} LANGUAGES C) - -find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) - -Python_add_library(_module MODULE src/module.c WITH_SOABI) -target_include_directories(_module PRIVATE $ENV{PYAWAITABLE_INCLUDE}) -install(TARGETS _module DESTINATION .) -``` - -### `meson-python` - -```py -# meson.build -project('_module', 'c') - -py = import('python').find_installation(pure: false) -pyawaitable_include = run_command('pyawaitable --include', check: true).stdout().strip() - -py.extension_module( - '_module', - 'src/module.c', - install: true, - include_directories: [pyawaitable_include], -) -``` - -## Simple Extension Example - -```c -#include -#include - -static int -module_exec(PyObject *mod) -{ - return PyAwaitable_Init(); -} - -/* - Equivalent to the following Python function: - - async def async_function(coro: collections.abc.Awaitable) -> None: - await coro - - */ -static PyObject * -async_function(PyObject *self, PyObject *coro) -{ - PyObject *awaitable = PyAwaitable_New(); - if (awaitable == NULL) { - return NULL; - } - - if (PyAwaitable_AddAwait(awaitable, coro, NULL, NULL) < 0) { - Py_DECREF(awaitable); - return NULL; - } - - return awaitable; -} - -static PyModuleDef_Slot module_slots[] = { - {Py_mod_exec, module_exec}, - {0, NULL} -}; - -static PyMethodDef module_methods[] = { - {"async_function", async_function, METH_O, NULL}, - {NULL, NULL, 0, NULL}, -}; - -static PyModuleDef module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_size = 0, - .m_slots = module_slots, - .m_methods = module_methods -}; - -PyMODINIT_FUNC -PyInit__module() -{ - return PyModuleDef_Init(&module); -} -``` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..9da46c6 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,203 @@ +Installation +============ + +PyAwaitable needs to be installed as a build dependency, not a runtime +dependency. For the ``project.dependencies`` portion of your ``pyproject.toml`` +file, you can completely omit PyAwaitable as a dependency. + +For example, in a ``setuptools``-based project, your ``build-system`` section would look like: + +.. code-block:: toml + + # pyproject.toml + [build-system] + requires = ["setuptools~=78.1", "pyawaitable>=2.0.0"] + build-backend = "setuptools.build_meta" + + +Including the ``pyawaitable.h`` File +------------------------------------ + +PyAwaitable provides a number of APIs to access the include path for a +number of different build systems. Namely: + +- :func:`pyawaitable.include` is available in Python code (typically for + ``setup.py``-based extensions). +- :envvar:`PYAWAITABLE_INCLUDE` is accessible as an environment variable, + but only if Python has been started without :py:option:`-S` (this is + useful for ``scikit-build-core`` projects). +- ``pyawaitable --include`` on the command line returns the path of the + include directory (useful for ``meson-python`` projects). + +.. note:: + + PyAwaitable uses a nifty trick for building itself into your project. + Python's packaging ecosystem isn't exactly great at distributing C + libraries, so the ``pyawaitable.h`` actually contains the entire + PyAwaitable source code (but with mangled names to prevent collisions + with your own project). + + This has some pros and cons: + + - PyAwaitable doesn't need to be installed at runtime, as it's embedded + directly into your extension. This means it's extremely portable; + completely different PyAwaitable versions can peacefully coexist in the + same process. + - Enabling debug flags in your extension also means enabling debug flags + in PyAwaitable, thus enabling assertions and whatnot. + - However, PyAwaitable can't use the limited API, so it prevents your + extension from using the limited API (see the note below). + +Initializing PyAwaitable in Your Extension +------------------------------------------ + +PyAwaitable has to do a one-time initialization to get its types and other +state initialized in the Python process. This is done with +:c:func:`PyAwaitable_Init`, which can be called basically anywhere, +as long as it's called before any other PyAwaitable functions are used. + +Typically, you'll want to call this in your extension's `PyInit_` function, or +in the :c:data`Py_mod_exec` function in multi-phase extensions. For example: + +.. code-block:: c + + // Single-phase + PyMODINIT_FUNC + PyInit_mymodule() + { + if (PyAwaitable_Init() < 0) { + return NULL; + } + + return PyModule_Create(/* ... */); + } + +.. code-block:: c + + // Multi-phase + static int + module_exec(PyObject *mod) + { + return PyAwaitable_Init(); + } + +.. warning:: + + Unfortunately, PyAwaitable cannot be used with the + :ref:`limited C API `. This is due to PyAwaitable needing + :c:member:`~PyAsyncMethods.am_send` to implement the coroutine protocol + on 3.10+, but the corresponding :ref:`heap type ` slot + ``Py_am_send`` was not added until 3.11. Therefore, PyAwaitable + annot support the limited API without dropping support for <3.11. + +Examples +-------- + +``setuptools`` +************** + +.. code-block:: python + + # setup.py + from setuptools import setup, Extension + import pyawaitable + + if __name__ == "__main__": + setup( + ext_modules=[ + Extension("_module", ["src/module.c"], include_dirs=[pyawaitable.include()]) + ] + ) + +``scikit-build-core`` +********************* + +.. code-block:: cmake + + # CMakeLists.txt + cmake_minimum_required(VERSION 3.15...3.30) + project(${SKBUILD_PROJECT_NAME} LANGUAGES C) + + find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) + + Python_add_library(_module MODULE src/module.c WITH_SOABI) + target_include_directories(_module PRIVATE $ENV{PYAWAITABLE_INCLUDE}) + install(TARGETS _module DESTINATION .) + +``meson-python`` +**************** + +.. code-block:: meson + + # meson.build + project('_module', 'c') + + py = import('python').find_installation(pure: false) + pyawaitable_include = run_command('pyawaitable --include', check: true).stdout().strip() + + py.extension_module( + '_module', + 'src/module.c', + install: true, + include_directories: [pyawaitable_include], + ) + +Simple Extension Example +------------------------ + +.. code-block:: c + + #include + #include + + static int + module_exec(PyObject *mod) + { + return PyAwaitable_Init(); + } + + /* + Equivalent to the following Python function: + + async def async_function(coro: collections.abc.Awaitable) -> None: + await coro + + */ + static PyObject * + async_function(PyObject *self, PyObject *coro) + { + PyObject *awaitable = PyAwaitable_New(); + if (awaitable == NULL) { + return NULL; + } + + if (PyAwaitable_AddAwait(awaitable, coro, NULL, NULL) < 0) { + Py_DECREF(awaitable); + return NULL; + } + + return awaitable; + } + + static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, module_exec}, + {0, NULL} + }; + + static PyMethodDef module_methods[] = { + {"async_function", async_function, METH_O, NULL}, + {NULL, NULL, 0, NULL}, + }; + + static PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_size = 0, + .m_slots = module_slots, + .m_methods = module_methods + }; + + PyMODINIT_FUNC + PyInit__module() + { + return PyModuleDef_Init(&module); + } diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..32bb245 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/reference.rst b/docs/reference.rst new file mode 100644 index 0000000..190ee94 --- /dev/null +++ b/docs/reference.rst @@ -0,0 +1,223 @@ +Reference +========= + +General +------- + +.. function:: pyawaitable.include() + + Return the include path of ``pyawaitable.h``. + + +.. envvar:: PYAWAITABLE_INCLUDE + + Result of :func:`pyawaitable.include`, stored as an environment variable + through a ``.pth`` file (only available when Python isn't run through + :py:option:`-S`). + + +.. c:function:: int PyAwaitable_Init(void) + + Initialize PyAwaitable. This should typically be done in the :c:data:`Py_mod_exec` + slot of a module. + + This can safely be called multiple times. + + Return ``0`` on success, and ``-1`` with an exception set on failure. + + +.. c:function:: PyObject *PyAwaitable_New(void) + + Create a new empty PyAwaitable object. + + This returns a new :term:`strong reference` to a PyAwaitable object on + success, and returns ``NULL`` with an exception set on failure. + + +.. c:function:: int PyAwaitable_SetResult(PyObject *awaitable, PyObject *result) + + Set *result* to the :ref:`result ` of the PyAwaitable object. + The result will be returned to the :keyword:`await` expression on this + PyAwaitable object. + + *result* will not be stolen; this function will create its own reference to + *result* internally. + + Return ``0`` with the return value set on success, and ``-1`` with an + exception set on failure. + + +Coroutines +---------- + + +.. c:type:: int (*PyAwaitable_Callback)(PyObject *awaitable, PyObject *result) + + The type of a result callback, as submitted in :c:func:`PyAwaitable_AddAwait`. + + This takes two :c:type:`PyObject * ` references: an instance of + a PyAwaitable object, and the return value of the awaited coroutine. + + There are three possible return values: + + 1. ``0``: OK. + 2. ``-1``: Error, fall to the error callback registered alongside this one, + if it exists. There must be an exception set. + 3. ``-2``: Error, but skip the error callback if provided. There must be an + exception set. + + +.. c:type:: int (*PyAwaitable_Error)(PyObject *awaitable, PyObject *result) + + The type of an error callback. + + +.. c:type:: int (*PyAwaitable_Defer)(PyObject *awaitable) + + The type of a "defer" callback. + + +.. c:function:: int PyAwaitable_AddAwait(PyObject *awaitable, PyObject *coroutine, PyAwaitable_Callback result_callback, PyAwaitable_Error error_callback) + + Mark an :term:`awaitable` object, *coroutine*, for execution by the event + loop when the PyAwaitable object is awaited. + + *result_callback* is a :ref:`return value callback `. + It will be called with the result of *coroutine* when it has finished + execution. This may be ``NULL``, in which case the return value is simply + discarded. + + *error_callback* is an :ref:`error callback `. It is + called if *coroutine* raises an exception during execution, or when + *result_callback* returns ``-1``. This may be ``NULL``, which will cause + any exceptions to be propagated to the caller (the one who awaited the + PyAwaitable object). + + This function will return ``0`` on success, and ``-1`` with an exception + set on failure. + + +Value Storage +------------- + +.. c:function:: int PyAwaitable_SaveValues(PyObject *awaitable, Py_ssize_t nargs, ...) + + Store *nargs* amount of :ref:`object values ` in the + PyAwaitable object. + + The number of arguments passed to ``...`` must match *nargs*. The objects + passed will be stored in the PyAwaitable object internally to be unpacked + by :c:func:`PyAwaitable_UnpackValues` later. + + Return ``0`` with the values stored on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyAwaitable_UnpackValues(PyObject *awaitable, ...) + + Unpack :ref:`object values ` stored in the PyAwaitable + object. + + This function expects ``PyObject **`` pointers passed to the ``...``. + These will then be set to :term:`borrowed reference `. + The number of arguments passed to the ``...`` must match the sum of all + *nargs* to prior :c:func:`PyAwaitable_SaveValues` calls. For example, if + one call stored two values, and then another call stored three values, this + function would expect five pointers to be passed. + + Pointers passed to the ``...`` may be ``NULL``, in which case the object at + that position is skipped. + + Return ``0`` will all references set on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyAwaitable_SetValue(PyObject *awaitable, Py_ssize_t index, PyObject *value) + + Replace a single :ref:`object value ` at the position *index* + with *value*. The old reference to the object stored at the position *index* + is released, so *value* must not be ``NULL``. + + If *index* is below zero or out of bounds for the number of stored object + values, this function will fail. As such, this function cannot be used to + append new object values -- use :c:func:`PyAwaitable_SaveValues` for that. + + Return ``0`` with the object replaced on success, and ``-1`` with an exception + set on failure. + + +.. c:function:: PyObject *PyAwaitable_GetValue(PyObject *awaitable, Py_ssize_t index) + + Unpack a single :ref:`object value ` at the position *index*. + + If *index* is below zero or out of bounds for the number of stored object + values, this function will sanely fail. + + This is a low-level routine meant for complete-ness; always prefer using + :c:func:`PyAwaitable_UnpackValues` over this function. + + Return a :term:`borrowed reference` to the value on success, and ``NULL`` + with an exception set on failure. + + +.. c:function:: int PyAwaitable_SaveArbValues(PyObject *awaitable, Py_ssize_t nargs, ...) + + Similar to :c:func:`PyAwaitable_SaveValues`, but saves + :ref:`arbitrary values ` (``void *`` pointers) instead + of :c:type:`PyObject * ` references. + + Arbitrary values are separate from object values, so the number of Python + objects stored through :c:func:`PyAwaitable_SaveValues` has no effect + on this function. + + Return ``0`` with all pointers stored on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyAwaitable_UnpackArbValues(PyObject *awaitable, ...) + + Similar to :c:func:`PyAwaitable_UnpackValues`, but unpacks + :ref:`arbitrary values ` (``void *`` pointers) instead + of :c:type:`PyObject * ` references. + + Arbitrary values are separate from object values, so the number of Python + objects stored through :c:func:`PyAwaitable_SaveValues` has no effect + on this function. + + This function expects ``void **`` pointers passed to the ``...``. + The number of arguments passed to the ``...`` must match the sum of + all *nargs* to prior :c:func:`PyAwaitable_SaveArbValues` calls. For + example, if one call stored two values, and then another call stored + three values, this function would expect five pointers to be passed. + + Return ``0`` with all pointers set on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyAwaitable_SetArbValue(PyObject *awaitable, Py_ssize_t index, void *value) + + Similar to :c:func:`PyAwaitable_SetValue`, but replaces a single + :ref:`arbitrary value ` instead. + + If *index* is below zero or out of bounds for the number of stored object + values, this function will fail. As such, this function cannot be used to + append new object values -- use :c:func:`PyAwaitable_SaveArbValues` for that. + + Return ``0`` with the object replaced on success, and ``-1`` with an exception + set on failure. + + +.. c:function:: void *PyAwaitable_GetArbValue(PyObject *awaitable, Py_ssize_t index) + + Similar to :c:func:`PyAwaitable_GetValue`, but unpacks a single + :ref:`arbitrary value ` at the position *index*. + + If *index* is below zero or out of bounds for the number of stored object + values, this function will sanely fail. + + This is a low-level routine meant for complete-ness; always prefer using + :c:func:`PyAwaitable_UnpackArbValues` over this function. + + Return the ``void *`` pointer stored at *index* on success, and ``NULL`` + with an exception set on failure. If ``NULL`` is a valid value for the + arbitrary value, use :c:func:`PyErr_Occurred` to differentiate. diff --git a/docs/usage/adding_awaits.rst b/docs/usage/adding_awaits.rst new file mode 100644 index 0000000..3dda75f --- /dev/null +++ b/docs/usage/adding_awaits.rst @@ -0,0 +1,346 @@ +Executing Asynchronous Calls from C +=================================== + +Let's say we wanted to replicate the following in C: + +.. code-block:: py + + async def trampoline(coro: collections.abc.Coroutine) -> int: + return await coro + +This is simply a function that we pass a coroutine to, and it will await it for +us. It's not particularly useful, but it's just for learning purposes. + +We already know that ``trampoline()`` will evaluate to a magic coroutine object +that supports :py:keyword:`await`, via the :py:attr:`~object.__await__` dunder, +and needs to :py:ref:`yield from ` its coroutines. So, if we wanted +to break ``trampoline()`` down into a synchronous Python function, it would look +something like this: + +.. code-block:: py + + class _trampoline_coroutine: + def __init__(self, coro: collections.abc.Coroutine) -> None: + self.coro = coro + + def __await__(self) -> collections.abc.Generator: + yield + yield from self.coro.__await__() + + def trampoline(coro: collections.abc.Coroutine) -> collections.abc.Coroutine: + return _trampoline_coroutine(coro) + +But, this is using ``yield from``; there's no ``yield from`` in C, so how do we +actually await things, or more importantly, use their return value? This is +where things get tricky. + +Adding Awaits to a PyAwaitable Object +------------------------------------- + +There's one big function for "adding" coroutines to a PyAwaitable object: +:c:func:`PyAwaitable_AddAwait`. By "add", we mean that the asynchronous +call won't happen right then and there. Instead, the PyAwaitable will +store it, and then when something comes to call the :py:attr:`~object.__await__` +on the PyAwaitable object, it will mimick a :py:ref:`yield from ` +on that coroutine. + +:c:func:`PyAwaitable_AddAwait` takes four arguments: + +- The PyAwaitable object. +- The coroutine to store. (Not an `async def` function, but the result of + calling one without :py:keyword:`await`.) +- A callback. +- An error callback. + +Let's focus on the first two for now, and just pass ``NULL`` for the other +two in the meantime. We can implement ``trampoline()`` from our earlier +example pretty easily: + +.. code-block:: c + + static PyObject * + trampoline(PyObject *self, PyObject *coro) // METH_O + { + PyObject *awaitable = PyAwaitable_New(); + if (awaitable == NULL) { + return NULL; + } + + if (PyAwaitable_AddAwait(awaitable, coro, NULL, NULL) < 0) { + Py_DECREF(awaitable); + return NULL; + } + + return awaitable; + } + +To your eyes, the ``yield from`` and all of that mess is completely +hidden; you give PyAwaitable your coroutine, and it handles the rest! +``trampoline()`` now acts like our pure-Python function from earlier: + +.. code-block:: pycon + + >>> from _yourmod import trampoline + >>> import asyncio + >>> await trampoline(asyncio.sleep(2)) # Sleeps for 2 seconds + +Yay! We called an asynchronous function from C! + + +.. _return-value-callbacks: + +Getting the Return Value in a Callback +-------------------------------------- + +In many cases, it's desirable to use the return value of a coroutine. +For example, let's say we wanted to get the result of the following +asynchronous function: + +.. code-block:: py + + async def silly() -> int: + await asyncio.sleep(2) # Simulate doing some I/O work + return 42 + +The details of how coroutines return values aren't relevant, but we do know +that a coroutine isn't actually "awaited" until after we've already returned +our PyAwaitable object from C. That means we have to use a callback to get the +return value of the coroutine. + +Specifically, we can pass a function pointer to the third parameter of +:c:func:`PyAwaitable_AddAwait`. A callback function takes two +:c:type:`PyObject * ` parameters: + +- A reference to the PyAwaitable object that called it. +- A reference to the return value of the coroutine. + +A callback must return ``0`` to indicate success, or ``-1`` with an exception set to indicate failure. + +Now, we can use the result of ``silly()`` in C: + +.. code-block:: c + + static int + callback(PyObject *awaitable, PyObject *value) + { + if (PyAwaitable_SetResult(awaitable, value) < 0) { + return -1; + } + + return 0; + } + + static PyObject * + call_silly(PyObject *self, PyObject *silly) + { + PyObject *awaitable = PyAwaitable_New(); + if (awaitable == NULL) { + return NULL; + } + + // Get the coroutine by calling silly() + PyObject *coro = PyObject_CallNoArgs(silly); + if (coro == NULL) { + Py_DECREF(awaitable); + return NULL; + } + + if (PyAwaitable_AddAwait(awaitable, coro, callback, NULL) < 0) { + Py_DECREF(awaitable); + Py_DECREF(coro); + return NULL; + } + + Py_DECREF(coro); + return awaitable; + } + +This can be used from Python as such: + +.. code-block:: pycon + + >>> from _yourmod import call_silly + >>> await call_silly(silly) # Sleeps for 2 seconds + silly() returned: 42 + + +.. _error-callbacks: + +Handling Errors with Callbacks +------------------------------ + +Coroutines can raise exceptions, during execution. For example, imagine we +wanted to use a function that makes a network request: + +.. code-block:: py + + import asyncio + + + async def make_request() -> str: + async with asyncio.timeout(5): + await asyncio.sleep(10) # Simulate some I/O + return "..." + +The above will raise :py:exc:`TimeoutError`, but not on simply calling +``make_request()``; it will only raise once it's actually started executing +in an :py:keyword:`await`, and as we already know that coroutines don't execute +at the :c:func:`PyAwaitable_AddAwait` callsite, we can't simply check for +errors there. So, similar to return value callbacks, PyAwaitable provides error +callbacks, which--you guessed it--is the fourth argument to ``PyAwaitable_AddAwait``. + +An error callback has the same signature as a return value callback, but instead +of taking a reference to a return value, it takes a reference to an exception +object that was caught and raised by either the coroutine or the coroutine's +callback. + +.. note:: + + Error callbacks are *not* called with an exception "set" + (so :c:func:`PyErr_Occurred()` returns ``NULL``), so it's safe to call + most of Python's C API without worrying about those kinds of failures. + +An error callback's return value can do a number of different things to the +state of the PyAwaitable object's exception. Namely: + +- Returning ``0`` will consider the error successfully caught, so + the PyAwaitable object will clear the exception and continue + executing the rest of its coroutine. +- Returning ``-1`` indicates that the error should be repropagated. + The PyAwaitable object will officially "set" the Python exception + (via :c:func:`PyErr_SetRaisedException`), raise the error to the + event loop and stop itself from executing any future coroutines. +- Returning ``-2`` indicates that a new error occurred while handling + the other one; the original exception is _not_ restored, and an exception + set by the error callback is used instead and propagated to the event loop. + +.. note:: + + Return value callbacks are not called if an exception occurred while executing the coroutine. + +To try and give a real-world example of all three of these, let's implement +the following function in C: + +.. code-block:: py + + async def is_api_reachable(make_request: Callable[[], collections.abc.Coroutine]) -> bool: + try: + await make_request() + return True + except TimeoutError: + return False + +.. note:: + + :py:exc:`asyncio.TimeoutError` is an alias of the built-in + :py:exc:`TimeoutError` exception. + +We have to do several things here: + +- Call ``make_request()`` to get the coroutine object to :py:keyword:`await`. +- Add an error callback for that coroutine. +- In a return value callback, set the return value to ``True`` (or really, + :c:data:`Py_True`), because that means the operation didn't time out. +- In an error callback, check if the exception is an instance of + :py:exc:`TimeoutError`, and set the return value to ``False`` if it is. +- If it's something other than ``TimeoutError``, let it propagate. + +In C, all that would be implemented like this: + +.. code-block:: c + + static int + return_true(PyObject *awaitable, PyObject *unused) + { + return PyAwaitable_SetResult(awaitable, Py_True); + } + + static int + return_false(PyObject *awaitable, PyObject *exc) + { + if (PyErr_GivenExceptionMatches(exc, PyExc_TimeoutError)) { + if (PyAwaitable_SetResult(exc, Py_False) < 0) { + // New exception occurred; give it to the event loop. + return -2; + } + + return 0; + } else { + // This isn't a TimeoutError! + return -1; + } + } + + static PyObject * + is_api_reachable(PyObject *self, PyObject *make_request) + { + PyObject *awaitable = PyAwaitable_New(); + if (awaitable == NULL) { + return NULL; + } + + // Remember, this isn't the same as executing the coroutine, so + // the timeout doesn't show up here. But, we still need to handle + // an exception case, because something might have gone wrong + // in getting the coroutine object, e.g., the object isn't callable + // or we're out of memory. + PyObject *coro = PyObject_CallNoArgs(make_request); + if (coro == NULL) { + Py_DECREF(awaitable); + return NULL; + } + + if (PyAwaitable_AddAwait(awaitable, coro, return_true, return_false)) { + Py_DECREF(awaitable); + Py_DECREF(coro); + return NULL; + } + + Py_DECREF(coro); + return awaitable; + } + +Propagation of Errors in Return Value Callbacks +----------------------------------------------- + +By default, returning ``-1`` from a return value callback will implicitly +call the error callback if one is set. But, this isn't always desirable; +sometimes, you want to let errors in callbacks bubble up instead of +getting handled by some default error handling mechanism you've installed. + +You can force the PyAwaitable object to propagate the exception by returning +``-2`` from a return value callback. If ``-2`` is returned, the exception set +by the callback will always be raised back to whoever awaited the PyAwaitable +object. + +For example, if we installed some global exception logger inside of the error +callback, but don't want that to grab things like a :py:exc:`MemoryError` inside +of the return callback, we would return ``-2``: + +.. code-block:: c + + static int + error_handler(PyObject *awaitable, PyObject *error) + { + // Simply print the error and continue execution + PyErr_SetRaisedException(Py_NewRef(error)); + PyErr_Print(); + return 0; + } + + static int + handle_value(PyObject *awaitable, PyObject *something) + { + PyObject *message = PyUnicode_FromString("LOG: Got value"); + if (message == NULL) { + // Skip the error callback + return -2; + } + + if (magically_log_value(message, something) < 0) { + // Skip the error callback + return -2; + } + + return 0; + } diff --git a/docs/usage/c_async.rst b/docs/usage/c_async.rst new file mode 100644 index 0000000..21b5317 --- /dev/null +++ b/docs/usage/c_async.rst @@ -0,0 +1,143 @@ +Making a C Function Asynchronous +-------------------------------- + +Let's make a C function that replicates the following Python code: + +.. code-block:: + + async def hello() -> None: + print("Hello, PyAwaitable") + +If you've tried to implement an asynchronous C function in the past, this is +likely where you got stuck. How do we make a C function ``async def``? + +Breaking Down Awaitable Functions +--------------------------------- + +In Python, you have to call an ``async def`` function to use it with +:py:keyword:`await`. In our example above, the following would be invalid: + +.. code-block:: pycon + + >>> await hello + +Of course, you need to do ``await hello()`` instead. Why? + +``hello()`` is returning a :term:`coroutine`, and coroutine objects are +usable with the :py:keyword:`await` keyword (but not ``async def`` functions +themselves). So, ``hello()`` as a synchronous function would really be like: + +.. code-block:: python + + class _hello_coroutine: + def __await__(self) -> collections.abc.Generator: + print("Hello, PyAwaitable") + yield + + def hello() -> collections.abc.Coroutine: + return _hello_coroutine() + +If there were to be :keyword:`await` expressions inside ``hello()``, the +returned coroutine object would handle those by yielding inside of +the :py:attr:`~object.__await__` dunder method. We can do the same kind of +thing in C. + +Creating PyAwaitable Objects +---------------------------- + +You can create a new PyAwaitable object with :c:func:`PyAwaitable_New`. +This returns a :term:`strong reference` to a PyAwaitable object, and +``NULL`` with an exception set on failure. + +Think of a PyAwaitable object sort of like the ``_hello_coroutine`` example +from above, but it's generic instead of being specially built for the +``hello()`` function. So, like our Python example, we need to return the +coroutine to allow it to be used in :keyword:`await` expressions: + +.. code-block:: c + + static PyObject * + hello(PyObject *self, PyObject *unused) // METH_NOARGS + { + PyObject *awaitable = PyAwaitable_New(); + if (awaitable == NULL) { + return NULL; + } + + puts("Hello, PyAwaitable"); + return awaitable; + } + +.. note:: + + "Coroutine" is a bit of an ambigious term in Python. There are two types + of coroutines: native ones (instances of :py:class:`types.CoroutineType`), + and objects that implement the coroutine protocol + (:py:class:`collections.abc.Coroutine`). Only the interpreter itself can + create native coroutines, so a PyAwaitable object is an object that + implements the coroutine protocol. + +Yay! We can now use ``hello()`` in :keyword:`await` expressions: + +.. code-block:: pycon + + >>> from _yourmod import hello + >>> await hello() + Hello, PyAwaitable + +.. _return-values: + +Changing the Return Value +------------------------- + +Note that in all code-paths, we should return the PyAwaitable object, or +``NULL`` with an exception set to indicate a failure. But that means you can't +simply ``return`` your own :c:type:`PyObject * `; how can the +:keyword:`await` expression of our C coroutine evaluate to something useful? + +By default, the "return value" (_i.e._, what :keyword:`await` will evaluate to) +is :const:`None` (or really, :c:data:`Py_None` in C). That can be changed with +:c:func:`PyAwaitable_SetResult`, which takes a reference to the object you +want to return. PyAwaitable will store a :term:`strong reference` to this +object internally. + +For example, if you wanted to return the Python integer ``42`` from ``hello()``, +you would simply pass a :c:type:`PyObject * ` for ``42`` to +:c:func:`PyAwaitable_SetResult`: + +.. code-block:: c + + static PyObject * + hello(PyObject *self, PyObject *nothing) // METH_NOARGS + { + PyObject *awaitable = PyAwaitable_New(); + if (awiatable == NULL) { + return NULL; + } + + PyObject *my_number = PyLong_FromLong(42); + if (my_number == NULL) { + Py_DECREF(awaitable); + return NULL; + } + + if (PyAwaitable_SetResult(awaitable, my_number) < 0) { + Py_DECREF(awaitable); + Py_DECREF(my_number); + return NULL; + } + + Py_DECREF(my_number); + + puts("Hello, PyAwaitable"); + return awaitable; + } + +Now, the :keyword:`await` expression evalutes to ``42``: + +.. code-block:: pycon + + >>> from _yourmod import hello + >>> await hello() + Hello, PyAwaitable + 42 diff --git a/docs/usage/value_storage.rst b/docs/usage/value_storage.rst new file mode 100644 index 0000000..a77e343 --- /dev/null +++ b/docs/usage/value_storage.rst @@ -0,0 +1,166 @@ +Managing State Between Callbacks +================================ + +So far, all of our examples haven't needed any transfer of state between the +:c:func:`PyAwaitable_AddAwait` callsite and the time the coroutine is executed +(in a callback). You might have noticed that the callbacks don't take a +``void *arg`` parameter to make up for C's lack of closures, so how do we +manage state? + +First, let's get an example of what we might want state for. Our goal is to +implement a function that takes a paremeter and does something with it against +the result of a coroutine. For example: + +.. code-block:: python + + async def multiply_async( + number: int, + get_number_io: Callable[[], collections.abc.Awaitable[int]] + ) -> str: + value = await get_number_io() + return value * number + +Introducing Value Storage +------------------------- + +Instead of ``void *arg`` parameters, PyAwaitable provides APIs for storing +state directly on the PyAwaitable object. There are two types of value +storage: + +- Object value storage; :c:type:`PyObject * ` pointers that + PyAwaitable keeps references to. +- Arbitrary value storage; ``void *`` pointers that PyAwaitable never + dereferences -- it's your job to manage it. + +Value storage is generally a lot more convenient than something like a +``void *arg``, because you don't have to define any ``struct`` or make +extra allocations. It's especially convenient in the ``PyObject *`` case, +because you don't have to worry about dealing with their reference counts +or traversing reference cycles. And, even if a single state ``struct`` is +more convenient for your case, it's easy to implement it through the arbitrary +values API. + +There are four parts to the value APIs, each with a variant for object and +arbitrary values: + +- Saving values: :c:func:`PyAwaitable_SaveValues` and :c:func:`PyAwaitable_SaveArbValues`. +- Unpacking values: :c:func:`PyAwaitable_UnpackValues` and :c:func:`PyAwaitable_UnpackArbValues`. +- Getting values: :c:func:`PyAwaitable_GetValue` and :c:func:`PyAwaitable_GetArbValue`. +- Setting values: :c:func:`PyAwaitable_SetValue` and :c:func:`PyAwaitable_SetArbValue`. + +.. _object-values: + +Object Value Storage +-------------------- + +In most cases, you want to store Python objects on your PyAwaitable object. +This can be anything you want, such as arguments passed into your function. +The two main APIs you want when using value storage are +:c:func:`PyAwaitable_SaveValues` and :c:func:`PyAwaitable_UnpackValues`. + +These are variadic C functions; for ``Save``, pass the PyAwaitable object +and the number of objects you want to store, and then pass ``PyObject *`` +pointers matching that number. These references will not be stolen by +PyAwaitable. + +``Unpack``, on the other hand, does not require you to pass the number of +objects that you want -- it remembers how many you stored in ``Save``. +In ``Unpack``, you just pass the PyAwaitable object and pointers to local +``PyObject *`` variables, which will then be unpacked by the PyAwaitable +object (these may be ``NULL``, in which case the value is skipped). + +.. note:: + + Both :c:func:`PyAwaitable_SaveValues` and + :c:func:`PyAwaitable_UnpackValues` can fail. They return ``-1`` with an + exception set on failure, and ``0`` on success. + +For example, if you called ``PyAwaitable_SaveValues(awaitable, 3, /* ... */)``, +you must pass three non-``NULL`` ``PyObject *`` references, and then pass +three pointers-to-pointers to ``PyAwaitable_UnpackValues`` (but these may be +``NULL``). + +So, with all that in mind, we can implement ``multiply_async()`` from above +as such: + +.. code-block:: c + + static int + multiply_callback(PyObject *awaitable, PyObject *value) + { + PyObject *number; + if (PyAwaitable_UnpackValues(awaitable, &number) < 0) { + return -1; + } + + PyObject *result = PyNumber_Multiply(number, value); + if (result == NULL) { + return -1; + } + + if (PyAwaitable_SetResult(awaitable, result) < 0) { + Py_DECREF(result); + return -1; + } + + Py_DECREF(result); + return 0; + } + + static PyObject * + multiply_async(PyObject *self, PyObject *args) // METH_VARARGS + { + PyObject *number; + PyObject *get_number_io; + + if (!PyArg_ParseTuple(args, "OO", &number, &get_number_io)) { + return NULL; + } + + PyObject *awaitable = PyAwaitable_New(); + if (awaitable == NULL) { + return NULL; + } + + if (PyAwaitable_SaveValues(awaitable, 1, number) < 0) { + Py_DECREF(awaitable); + return NULL; + } + + PyObject *coro = PyObject_CallNoArgs(get_number_io); + if (coro == NULL) { + Py_DECREF(awaitable); + return NULL; + } + + if (PyAwaitable_AddAwait(awaitable, coro, multiply_callback, NULL) < 0) { + Py_DECREF(awaitable); + Py_DECREF(coro); + return NULL; + } + + Py_DECREF(coro); + return awaitable; + } + +Getting and Setting Values +-------------------------- + +In rare cases, it might be desirable to get or set a specific value at an +index. :c:func:`PyAwaitable_SetValue` (or :c:func:`PyAwaitable_SetArbValue`) +is useful if you intend to completely overwrite an object at a value index, +but :c:func:`PyAwaitable_GetValue` should basically never be preferred over +:c:func:`PyAwaitable_UnpackValues`; it's only there for complete-ness. + +.. _arbitrary-values: + +Arbitrary Value Storage +----------------------- + +Arbitrary value storage works exactly the same as +:ref:`object value storage `, with the exception of taking +``void *`` pointers instead of ``PyObject *`` pointers. PyAwaitable will +never attempt to read or write the pointers that you pass, so managing their +lifetime is up to you. In most cases, if your PyAwaitable object is supposed +to own the state of the arbitrary value, you deallocate it in the last +callback. diff --git a/docs/value_storage.md b/docs/value_storage.md deleted file mode 100644 index b5c86ce..0000000 --- a/docs/value_storage.md +++ /dev/null @@ -1,114 +0,0 @@ -# Managing State Between Callbacks - -So far, all of our examples haven't needed any transfer of state between the `PyAwaitable_AddAwait` callsite and the time the coroutine is executed (_i.e._, in a callback). You might have noticed that the callbacks don't take a `void *arg` parameter to make up for C's lack of closures, so how do we manage state? - -First, let's get an example of what we might want state for. Our goal is to implement a function that takes a paremeter and does something wiith it against the result of a coroutine. For example: - -```py -async def multiply_async( - number: int, - get_number_io: Callable[[], collections.abc.Awaitable[int]] -) -> str: - value = await get_number_io() - return value * number -``` - -## Introducing Value Storage - -Instead of `void *arg` parameters, PyAwaitable provides APIs for storing state directly on the PyAwaitable object. There are two types of value storage: - -- Object value storage; `PyObject *` pointers that PyAwaitable correctly stores references to. -- Arbitrary value storage; `void *` pointers that PyAwaitable never dereferences--it's your job to manage it. - -Value storage is generally a lot more convenient than something like a `void *arg`, because you don't have to define any `struct` or make an extra allocations. It's especially convenient in the `PyObject *` case, because you don't have to worry about dealing with their reference counts or traversing reference cycles. And, even if a single state `struct` is more convenient for your case, it's easy to implement it with the arbitrary values API. - -There are four parts to the value APIs, each with a variant for object and arbitrary values: - -- Saving values; `PyAwaitable_SaveValues`/`PyAwaitable_SaveArbValues`. -- Unpacking values; `PyAwaitable_UnpackValues`/`PyAwaitable_UnpackArbValues`. -- Getting values; `PyAwaitable_GetValue`/`PyAwaitable_GetArbValue`. -- Setting values; `PyAwaitable_SetValue`/`PyAwaitable_SetArbValue`. - -## Object Value Storage - -In most cases, you want to store Python objects on your PyAwaitable object. This can be anything you want, such as arguments passed into your function. The two main APIs you want when using value storage are `PyAwaitable_SaveValues` and `PyAwaitable_UnpackValues`. - -These are variadic C functions; for `Save`, pass the PyAwaitable object and the number of objects you want to store, and then pass `PyObject *` pointers matching that number. These references will _not_ be stolen by PyAwaitable. - -`Unpack`, on the other hand, does not require you to pass the number of objects that you want--it remembers how many you stored in `Save`. In `Unpack`, you just pass the PyAwaitable object and pointers to local `PyObject *` variables, which will then be unpacked by the PyAwaitable object (these may be `NULL`, in which case the value is skipped). - -!!! note - - Both `PyAwaitable_SaveValues` and `PyAwaitable_UnpackValues` can fail. They return `-1` with an exception set on failure, and `0` on success. - -For example, if you called `PyAwaitable_SaveValues(awaitable, 3, /* ... */)`, you must pass three non-`NULL` `PyObject *` references, and then pass three pointers-to-pointers to `PyAwaitable_UnpackValues` (but these may be `NULL`). - -So, with all that in mind, we can implement `multiply_async` above as such: - -```c -static int -multiply_callback(PyObject *awaitable, PyObject *value) -{ - PyObject *number; - if (PyAwaitable_UnpackValues(awaitable, &number) < 0) { - return -1; - } - - PyObject *result = PyNumber_Multiply(number, value); - if (result == NULL) { - return -1; - } - - if (PyAwaitable_SetResult(awaitable, result) < 0) { - Py_DECREF(result); - return -1; - } - - Py_DECREF(result); - return 0; -} - -static PyObject * -multiply_async(PyObject *self, PyObject *args) // METH_VARARGS -{ - PyObject *number; - PyObject *get_number_io; - - if (!PyArg_ParseTuple(args, "OO", &number, &get_number_io)) { - return NULL; - } - - PyObject *awaitable = PyAwaitable_New(); - if (awaitable == NULL) { - return NULL; - } - - if (PyAwaitable_SaveValues(awaitable, 1, number) < 0) { - Py_DECREF(awaitable); - return NULL; - } - - PyObject *coro = PyObject_CallNoArgs(get_number_io); - if (coro == NULL) { - Py_DECREF(awaitable); - return NULL; - } - - if (PyAwaitable_AddAwait(awaitable, coro, multiply_callback, NULL) < 0) { - Py_DECREF(awaitable); - Py_DECREF(coro); - return NULL; - } - - Py_DECREF(coro); - return awaitable; -} -``` - -## Getting and Setting Values - -In rare cases, it might be desirable to get or set a specific value at an index. `PyAwaitable_SetValue` is useful if you intend to completely overwrite an object at a value index, but `PyAwaitable_GetValue` should basically never be preferred over `PyAwaitable_UnpackValues`; it's, more or less, there for completion. - -## Arbitrary Value Storage - -Arbitrary value storage works exactly the same as object value storage, with the exception of taking `void *` pointers instead of `PyObject *` pointers. PyAwaitable will never attempt to read or write the pointers that you pass, so managing their lifetime is up to you. In most cases, if your PyAwaitable object is supposed to own the state of the arbitrary value, you deallocate it in the last callback. diff --git a/netlify.toml b/netlify.toml index 035f568..2c65f8f 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1,3 +1,3 @@ [build] -command = "mkdocs build" -publish = "site" \ No newline at end of file +command = "sphinx-build -M html ./docs ./site" +publish = "site/html" diff --git a/requirements.txt b/requirements.txt index 38d94eb..781e4d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ # Requirements for Netlify -mkdocs~=1.6 -pymdown-extensions~=10.15 +sphinx>=7.0 +shibuya~=2025.8 diff --git a/runtime.txt b/runtime.txt index bd28b9c..c8cfe39 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -3.9 +3.10