Skip to content

Commit f334da2

Browse files
committed
Add NIF-based logging and tracing for Python-Erlang integration
Implement fire-and-forget logging from Python's logging module to Erlang's logger, plus distributed tracing with span support. Features: - py:configure_logging/0,1 - Forward Python logs to Erlang logger - py:enable_tracing/0, py:disable_tracing/0 - Control span collection - py:get_traces/0, py:clear_traces/0 - Retrieve collected spans - Python erlang.Span context manager for tracing - Python @erlang.trace() decorator for function tracing - Level filtering at NIF level for performance Architecture: - Uses enif_send() for non-blocking message delivery - py_logger gen_server receives log messages - py_tracer gen_server collects trace spans - Both added to supervision tree Files: - c_src/py_logging.c - NIF implementations - src/py_logger.erl - Log message receiver - src/py_tracer.erl - Span collector - test/py_logging_SUITE.erl - 9 tests - docs/logging.md - Documentation - examples/logging_example.erl - Working example
1 parent 84ced63 commit f334da2

File tree

14 files changed

+1834
-2
lines changed

14 files changed

+1834
-2
lines changed

README.md

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ Key features:
2929
- **Streaming** - Iterate over Python generators chunk-by-chunk
3030
- **Virtual environments** - Activate venvs for dependency isolation
3131
- **AI/ML ready** - Examples for embeddings, semantic search, RAG, and LLMs
32+
- **Logging integration** - Python logging forwarded to Erlang logger
33+
- **Distributed tracing** - Span-based tracing from Python code
3234

3335
## Requirements
3436

@@ -320,6 +322,71 @@ ok = py:activate_venv(<<"/path/to/venv">>).
320322
ok = py:deactivate_venv().
321323
```
322324

325+
## Logging and Tracing
326+
327+
### Python Logging to Erlang Logger
328+
329+
Forward Python `logging` messages to Erlang's `logger`:
330+
331+
```erlang
332+
%% Configure Python logging
333+
ok = py:configure_logging(#{level => info}).
334+
335+
%% Python logs now appear in Erlang logger
336+
ok = py:exec(<<"
337+
import logging
338+
logging.info('Hello from Python!')
339+
logging.warning('Something needs attention')
340+
">>).
341+
```
342+
343+
From Python, you can also set up logging explicitly:
344+
345+
```python
346+
import erlang
347+
erlang.setup_logging(level=20) # 20 = INFO
348+
```
349+
350+
### Distributed Tracing
351+
352+
Collect trace spans from Python code:
353+
354+
```erlang
355+
%% Enable tracing
356+
ok = py:enable_tracing().
357+
358+
%% Run Python code with spans
359+
ok = py:exec(<<"
360+
import erlang
361+
362+
with erlang.Span('process-request', user_id=123):
363+
with erlang.Span('query-database'):
364+
pass # database work
365+
with erlang.Span('format-response'):
366+
pass # formatting work
367+
">>).
368+
369+
%% Retrieve collected spans
370+
{ok, Spans} = py:get_traces().
371+
%% Spans = [#{name => <<"query-database">>, status => ok, duration_us => 42, ...}, ...]
372+
373+
%% Clean up
374+
ok = py:clear_traces().
375+
ok = py:disable_tracing().
376+
```
377+
378+
Use the `@erlang.trace()` decorator for automatic function tracing:
379+
380+
```python
381+
import erlang
382+
383+
@erlang.trace()
384+
def my_function():
385+
return compute_something()
386+
```
387+
388+
See [docs/logging.md](docs/logging.md) for details.
389+
323390
## Examples
324391

325392
The `examples/` directory contains runnable demonstrations:
@@ -360,6 +427,11 @@ escript examples/erlang_concurrency.erl
360427
elixir --erl "-pa _build/default/lib/erlang_python/ebin" examples/elixir_example.exs
361428
```
362429

430+
### Logging and Tracing
431+
```bash
432+
escript examples/logging_example.erl
433+
```
434+
363435
## API Reference
364436

365437
### Function Calls
@@ -407,6 +479,22 @@ ok = py:tracemalloc_start().
407479
ok = py:tracemalloc_stop().
408480
```
409481

482+
### Logging
483+
484+
```erlang
485+
ok = py:configure_logging().
486+
ok = py:configure_logging(#{level => info, format => <<"%(name)s: %(message)s">>}).
487+
```
488+
489+
### Tracing
490+
491+
```erlang
492+
ok = py:enable_tracing().
493+
ok = py:disable_tracing().
494+
{ok, Spans} = py:get_traces().
495+
ok = py:clear_traces().
496+
```
497+
410498
## Type Mappings
411499

412500
### Erlang to Python
@@ -482,6 +570,8 @@ py:execution_mode(). %% => free_threaded | subinterp | multi_executor
482570
- [Scalability](docs/scalability.md)
483571
- [Streaming](docs/streaming.md)
484572
- [Threading](docs/threading.md)
573+
- [Logging and Tracing](docs/logging.md)
574+
- [Sandbox](docs/sandbox.md)
485575
- [Changelog](https://github.com/benoitc/erlang-python/releases)
486576

487577
## License

c_src/py_callback.c

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1300,6 +1300,19 @@ static PyMethodDef ErlangModuleMethods[] = {
13001300
{"_register_async_future", register_async_future, METH_VARARGS,
13011301
"Register a Future for an async callback.\n"
13021302
"Usage: erlang._register_async_future(callback_id, future)"},
1303+
/* Logging and tracing (from py_logging.c) */
1304+
{"_log", erlang_log_impl, METH_VARARGS,
1305+
"Log message to Erlang logger (fire-and-forget).\n"
1306+
"Usage: erlang._log(level, logger_name, message, metadata)"},
1307+
{"_trace_start", erlang_trace_start_impl, METH_VARARGS,
1308+
"Start a trace span.\n"
1309+
"Usage: erlang._trace_start(name, span_id, parent_id, attrs)"},
1310+
{"_trace_end", erlang_trace_end_impl, METH_VARARGS,
1311+
"End a trace span.\n"
1312+
"Usage: erlang._trace_end(span_id, status, attrs)"},
1313+
{"_trace_event", erlang_trace_event_impl, METH_VARARGS,
1314+
"Add event to a span.\n"
1315+
"Usage: erlang._trace_event(span_id, name, attrs)"},
13031316
{NULL, NULL, 0, NULL}
13041317
};
13051318

@@ -1452,6 +1465,126 @@ static int create_erlang_module(void) {
14521465
Py_DECREF(globals);
14531466
}
14541467

1468+
/* Inject logging and tracing Python code */
1469+
const char *erlang_logging_code =
1470+
"import logging\n"
1471+
"import threading\n"
1472+
"import random\n"
1473+
"\n"
1474+
"class ErlangHandler(logging.Handler):\n"
1475+
" '''Logging handler that forwards log records to Erlang logger.'''\n"
1476+
" def emit(self, record):\n"
1477+
" try:\n"
1478+
" msg = self.format(record)\n"
1479+
" meta = {'module': record.module, 'lineno': record.lineno,\n"
1480+
" 'funcName': record.funcName}\n"
1481+
" erlang._log(record.levelno, record.name, msg, meta)\n"
1482+
" except:\n"
1483+
" pass\n"
1484+
"\n"
1485+
"def setup_logging(level=10, format=None):\n"
1486+
" '''Set up Python logging to forward to Erlang.\n"
1487+
" \n"
1488+
" Args:\n"
1489+
" level: Minimum log level (10=DEBUG, 20=INFO, 30=WARNING, etc.)\n"
1490+
" format: Optional format string\n"
1491+
" \n"
1492+
" Returns:\n"
1493+
" The created ErlangHandler instance\n"
1494+
" '''\n"
1495+
" handler = ErlangHandler()\n"
1496+
" if format:\n"
1497+
" handler.setFormatter(logging.Formatter(format))\n"
1498+
" else:\n"
1499+
" handler.setFormatter(logging.Formatter('%(message)s'))\n"
1500+
" root = logging.getLogger()\n"
1501+
" root.addHandler(handler)\n"
1502+
" root.setLevel(level)\n"
1503+
" return handler\n"
1504+
"\n"
1505+
"# Thread-local span context for tracing\n"
1506+
"_span_ctx = threading.local()\n"
1507+
"\n"
1508+
"class Span:\n"
1509+
" '''Context manager for tracing spans.\n"
1510+
" \n"
1511+
" Usage:\n"
1512+
" with erlang.Span('operation-name', key='value') as span:\n"
1513+
" do_work()\n"
1514+
" span.event('checkpoint', items=10)\n"
1515+
" '''\n"
1516+
" def __init__(self, name, **attrs):\n"
1517+
" self.name = name\n"
1518+
" self.span_id = random.getrandbits(64)\n"
1519+
" self.attrs = attrs\n"
1520+
" self._prev = None\n"
1521+
"\n"
1522+
" def __enter__(self):\n"
1523+
" self._prev = getattr(_span_ctx, 'current', None)\n"
1524+
" _span_ctx.current = self.span_id\n"
1525+
" erlang._trace_start(self.name, self.span_id, self._prev, self.attrs)\n"
1526+
" return self\n"
1527+
"\n"
1528+
" def __exit__(self, et, ev, tb):\n"
1529+
" status = 'error' if et else 'ok'\n"
1530+
" attrs = {}\n"
1531+
" if et:\n"
1532+
" attrs['exception'] = str(ev)\n"
1533+
" erlang._trace_end(self.span_id, status, attrs)\n"
1534+
" _span_ctx.current = self._prev\n"
1535+
" return False\n"
1536+
"\n"
1537+
" def event(self, name, **attrs):\n"
1538+
" '''Add an event to this span.'''\n"
1539+
" erlang._trace_event(self.span_id, name, attrs)\n"
1540+
"\n"
1541+
"def trace(name=None):\n"
1542+
" '''Decorator to trace a function.\n"
1543+
" \n"
1544+
" Usage:\n"
1545+
" @erlang.trace()\n"
1546+
" def my_function():\n"
1547+
" pass\n"
1548+
" '''\n"
1549+
" def decorator(fn):\n"
1550+
" span_name = name or f'{fn.__module__}.{fn.__qualname__}'\n"
1551+
" def wrapper(*a, **kw):\n"
1552+
" with Span(span_name):\n"
1553+
" return fn(*a, **kw)\n"
1554+
" return wrapper\n"
1555+
" return decorator\n"
1556+
"\n"
1557+
"# Add to erlang module\n"
1558+
"erlang.ErlangHandler = ErlangHandler\n"
1559+
"erlang.setup_logging = setup_logging\n"
1560+
"erlang.Span = Span\n"
1561+
"erlang.trace = trace\n";
1562+
1563+
PyObject *log_globals = PyDict_New();
1564+
if (log_globals != NULL) {
1565+
PyObject *builtins = PyEval_GetBuiltins();
1566+
PyDict_SetItemString(log_globals, "__builtins__", builtins);
1567+
1568+
/* Import erlang module into globals so the code can reference it */
1569+
PyObject *sys_modules = PySys_GetObject("modules");
1570+
if (sys_modules != NULL) {
1571+
PyObject *erlang_mod = PyDict_GetItemString(sys_modules, "erlang");
1572+
if (erlang_mod != NULL) {
1573+
PyDict_SetItemString(log_globals, "erlang", erlang_mod);
1574+
}
1575+
}
1576+
1577+
PyObject *result = PyRun_String(erlang_logging_code, Py_file_input, log_globals, log_globals);
1578+
if (result == NULL) {
1579+
/* Non-fatal - logging features just won't be available */
1580+
PyErr_Print();
1581+
PyErr_Clear();
1582+
} else {
1583+
Py_DECREF(result);
1584+
}
1585+
Py_DECREF(log_globals);
1586+
}
1587+
14551588
return 0;
14561589
}
14571590

0 commit comments

Comments
 (0)