Merge pull request #334 from TTsangSC/cooperative-tracing

Restore trace callback when the profiler is disabled

Author: Erotemic

CHANGELOG.rst

@@ -16,6 +16,7 @@ Changes
 * ENH: Highlight final summary using rich if enabled
 * ENH: Made it possible to use multiple profiler instances simultaneously
 * ENH: various improvements related to auto-profiling:
+
   * ``kernprof -p`` target entities are now imported and profiled regardless of
     whether they are directly imported in the run script/module/code (old
     behavior restored by passing ``--no-preimports``)
@@ -25,7 +26,23 @@ Changes
     like class methods and properties
   * ``LineProfiler`` can now be used as a class decorator
 * FIX: Fixed line tracing for Cython code; superseded use of the legacy tracing system with ``sys.monitoring``
-* ENH: Fixed edge case where :py:meth:`LineProfiler.get_stats()` neglects data from duplicate code objects (#348)
+* FIX: Fixed edge cases where:
+
+  * ``LineProfiler.get_stats()`` neglected data from duplicate code objects
+    (#348)
+  * ``LineProfiler`` instances may stop receiving tracing events when multiple
+    instances were used (#350)
+  * Line events were not reported for ``raise`` statements and ``finally:``
+    bodies when using ``sys.monitoring`` (#355)
+* FIX: Tracing-system-related fixes (#333):
+
+  * ``LineProfiler`` now caches the existing ``sys`` or ``sys.monitoring`` trace
+    callbacks in ``.enable()`` and restores them in ``.disable()``, instead of
+    always discarding them on the way out
+  * Also added experimental support for calling (instead of suspending) said
+    callbacks during profiling
+  * Now allowing switching back to the "legacy" trace system on Python 3.12+,
+    controlled by an environment variable
 
 4.2.0
 ~~~~~

docs/source/auto/line_profiler._line_profiler.rst

@@ -2,6 +2,7 @@ line\_profiler.\_line\_profiler module
 ======================================
 
 .. automodule:: line_profiler._line_profiler
+   :private-members: _LineProfilerManager
    :members:
    :undoc-members:
    :show-inheritance:

docs/source/conf.py

@@ -194,6 +194,9 @@ def visit_Assign(self, node):
     'geowatch.tasks.cold.export_change_map',
 ]
 
+autodoc_default_options = {  # Document callable classes
+    'special-members': '__call__'}
+
 autodoc_member_order = 'bysource'
 autoclass_content = 'both'
 # autodoc_mock_imports = ['torch', 'torchvision', 'visdom']

line_profiler/CMakeLists.txt

@@ -10,6 +10,7 @@ add_cython_target(${module_name} "${cython_source}" C OUTPUT_VAR sources)
 # Add any other non-cython dependencies to the sources
 list(APPEND sources
   "${CMAKE_CURRENT_SOURCE_DIR}/timers.c"
+  "${CMAKE_CURRENT_SOURCE_DIR}/c_trace_callbacks.c"
 )
 message(STATUS "[OURS] sources = ${sources}")
 

line_profiler/Python_wrapper.h

@@ -0,0 +1,96 @@
+// Compatibility layer over `Python.h`.
+
+#ifndef LINE_PROFILER_PYTHON_WRAPPER_H
+#define LINE_PROFILER_PYTHON_WRAPPER_H
+
+#include "Python.h"
+
+// Ensure PyFrameObject availability as a concretely declared struct
+
+// _frame -> PyFrameObject
+#if PY_VERSION_HEX >= 0x030b00a6  // 3.11.0a6
+#   ifndef Py_BUILD_CORE
+#       define Py_BUILD_CORE 1
+#   endif
+#   include "internal/pycore_frame.h"
+#   include "cpython/code.h"
+#   include "pyframe.h"
+#else
+#   include "frameobject.h"
+#endif
+
+// Backport of Python 3.9 caller hooks
+
+#if PY_VERSION_HEX < 0x030900a4  // 3.9.0a4
+#   define PyObject_CallOneArg(func, arg) \
+        PyObject_CallFunctionObjArgs(func, arg, NULL)
+#   define PyObject_CallMethodOneArg(obj, name, arg) \
+        PyObject_CallMethodObjArgs(obj, name, arg, NULL)
+#   define PyObject_CallNoArgs(func) \
+        PyObject_CallFunctionObjArgs(func, NULL)
+#   define PyObject_CallMethodNoArgs(obj, name) \
+        PyObject_CallMethodObjArgs(obj, name, NULL)
+#endif
+
+#if PY_VERSION_HEX < 0x030900a5  // 3.9.0a5
+#   define PyThreadState_GetInterpreter(tstate) \
+        ((tstate)->interp)
+#endif
+
+#if PY_VERSION_HEX < 0x030900b1  // 3.9.0b1
+    /*
+     * Notes:
+     *     While 3.9.0a1 already has `PyFrame_GetCode()`, it doesn't
+     *     INCREF the code object until 0b1 (PR #19773), so override
+     *     that for consistency.
+     */
+#   define PyFrame_GetCode(x) PyFrame_GetCode_backport(x)
+    inline PyCodeObject *PyFrame_GetCode_backport(PyFrameObject *frame)
+    {
+        PyCodeObject *code;
+        assert(frame != NULL);
+        code = frame->f_code;
+        assert(code != NULL);
+        Py_INCREF(code);
+        return code;
+    }
+#endif
+
+#if PY_VERSION_HEX < 0x030b00b1  // 3.11.0b1
+    /*
+     * Notes:
+     *     Since 3.11.0a7 (PR #31888) `co_code` has been made a
+     *     descriptor, so:
+     *     - This already creates a NewRef, so don't INCREF in that
+     *       case; and
+     *     - `code->co_code` will not work.
+     */
+    inline PyObject *PyCode_GetCode(PyCodeObject *code)
+    {
+        PyObject *code_bytes;
+        if (code == NULL) return NULL;
+#       if PY_VERSION_HEX < 0x030b00a7  // 3.11.0a7
+            code_bytes = code->co_code;
+            Py_XINCREF(code_bytes);
+#       else
+            code_bytes = PyObject_GetAttrString(code, "co_code");
+#       endif
+        return code_bytes;
+    }
+#endif
+
+#if PY_VERSION_HEX < 0x030d00a1  // 3.13.0a1
+    inline PyObject *PyImport_AddModuleRef(const char *name)
+    {
+        PyObject *mod = NULL, *name_str = NULL;
+        name_str = PyUnicode_FromString(name);
+        if (name_str == NULL) goto cleanup;
+        mod = PyImport_AddModuleObject(name_str);
+        Py_XINCREF(mod);
+    cleanup:
+        Py_XDECREF(name_str);
+        return mod;
+    }
+#endif
+
+#endif // LINE_PROFILER_PYTHON_WRAPPER_H

line_profiler/_diagnostics.py

@@ -2,26 +2,102 @@
 Global state initialized at import time.
 Used for hidden arguments and developer features.
 """
-from line_profiler import _logger
 import os
+import sys
+from types import ModuleType
+from line_profiler import _logger
 
 
-def _boolean_environ(key):
-    """
+def _boolean_environ(
+        envvar,
+        truey=frozenset({'1', 'on', 'true', 'yes'}),
+        falsy=frozenset({'0', 'off', 'false', 'no'}),
+        default=False):
+    r"""
     Args:
-        key (str)
+        envvar (str)
+            Name for the environment variable to read from.
+        truey (Collection[str])
+            Values to be considered truey.
+        falsy (Collection[str])
+            Values to be considered falsy.
+        default (bool)
+            Default boolean value to resolve to.
 
     Returns:
-        bool
+        :py:data:`True`
+            If the (case-normalized) environment variable is equal to
+            any of ``truey``.
+        :py:data:`False`
+            If the (case-normalized) environment variable is equal to
+            any of ``falsy``.
+        ``default``
+            Otherwise.
+
+    Example:
+        >>> from os import environ
+        >>> from subprocess import run
+        >>> from sys import executable
+        >>> from textwrap import dedent
+        >>>
+        >>>
+        >>> def resolve_in_subproc(value, default,
+        ...                        envvar='MY_ENVVAR',
+        ...                        truey=('foo',), falsy=('bar',)):
+        ...     code = dedent('''
+        ...     from {0.__module__} import {0.__name__}
+        ...     print({0.__name__}({1!r}, {2!r}, {3!r}, {4!r}))
+        ...     ''').strip('\n').format(_boolean_environ, envvar,
+        ...                             tuple(truey), tuple(falsy),
+        ...                             bool(default))
+        ...     env = environ.copy()
+        ...     env[envvar] = value
+        ...     proc = run([executable, '-c', code],
+        ...                capture_output=True, env=env, text=True)
+        ...     proc.check_returncode()
+        ...     return {'True': True,
+        ...             'False': False}[proc.stdout.strip()]
+        ...
+        >>>
+        >>> # Truey value
+        >>> assert resolve_in_subproc('FOO', True) == True
+        >>> assert resolve_in_subproc('FOO', False) == True
+        >>> # Falsy value
+        >>> assert resolve_in_subproc('BaR', True) == False
+        >>> assert resolve_in_subproc('BaR', False) == False
+        >>> # Mismatch -> fall back to default
+        >>> assert resolve_in_subproc('baz', True) == True
+        >>> assert resolve_in_subproc('baz', False) == False
     """
-    value = os.environ.get(key, '').lower()
-    TRUTHY_ENVIRONS = {'true', 'on', 'yes', '1'}
-    return value in TRUTHY_ENVIRONS
+    # (TODO: migrate to `line_profiler.cli_utils.boolean()` after
+    # merging #335)
+    try:
+        value = os.environ.get(envvar).casefold()
+    except AttributeError:  # None
+        return default
+    non_default_values = falsy if default else truey
+    if value in {v.casefold() for v in non_default_values}:
+        return not default
+    return default
 
 
+# `kernprof` switches
 DEBUG = _boolean_environ('LINE_PROFILER_DEBUG')
 NO_EXEC = _boolean_environ('LINE_PROFILER_NO_EXEC')
 KEEP_TEMPDIRS = _boolean_environ('LINE_PROFILER_KEEP_TEMPDIRS')
 STATIC_ANALYSIS = _boolean_environ('LINE_PROFILER_STATIC_ANALYSIS')
 
+# `line_profiler._line_profiler` switches
+WRAP_TRACE = _boolean_environ('LINE_PROFILER_WRAP_TRACE')
+SET_FRAME_LOCAL_TRACE = _boolean_environ('LINE_PROFILER_SET_FRAME_LOCAL_TRACE')
+_MUST_USE_LEGACY_TRACE = not isinstance(
+    getattr(sys, 'monitoring', None), ModuleType)
+USE_LEGACY_TRACE = (
+    _MUST_USE_LEGACY_TRACE
+    or _boolean_environ('LINE_PROFILER_CORE',
+                        # Also provide `coverage-style` aliases
+                        truey={'old', 'legacy', 'ctrace'},
+                        falsy={'new', 'sys.monitoring', 'sysmon'},
+                        default=_MUST_USE_LEGACY_TRACE))
+
 log = _logger.Logger('line_profiler', backend='auto')