From 47544e0201d770d6918634d62387e6e3e8aaec25 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Wed, 4 Feb 2026 09:19:00 -0500 Subject: [PATCH 01/10] Add syntask information to SIGUSR2 output. --- changes/7a68cdaf97188251836d1f71709d33ce.yaml | 7 +++++++ synapse/glob.py | 5 +++++ 2 files changed, 12 insertions(+) create mode 100644 changes/7a68cdaf97188251836d1f71709d33ce.yaml diff --git a/changes/7a68cdaf97188251836d1f71709d33ce.yaml b/changes/7a68cdaf97188251836d1f71709d33ce.yaml new file mode 100644 index 00000000000..f4b8ccee6de --- /dev/null +++ b/changes/7a68cdaf97188251836d1f71709d33ce.yaml @@ -0,0 +1,7 @@ +--- +desc: Added information about ``SynTask`` data when printing asyncio tasks in response + to receving a ``SIGUSR2`` signal. +desc:literal: false +prs: [] +type: note +... diff --git a/synapse/glob.py b/synapse/glob.py index faab5cafe0f..4224bd42a24 100644 --- a/synapse/glob.py +++ b/synapse/glob.py @@ -1,4 +1,5 @@ import os +import pprint import signal import asyncio import logging @@ -20,6 +21,10 @@ def _asynciostacks(*args, **kwargs): # pragma: no cover tasks = asyncio.all_tasks(_glob_loop) for task in tasks: task.print_stack() + if hasattr(task, '_syn_task'): + st = task._syn_task + print(f'Task is a syn task with the following information: iden={st.iden} user={st.user.iden} username={st.user.name}') + pprint.pprint(st.info) print(80 * '*') print('Faulthandler stack frames per thread:') faulthandler.dump_traceback() From 4215ce65cb9b16a1d3198177f8ae045ccbcbb7d7 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Wed, 4 Feb 2026 10:33:16 -0500 Subject: [PATCH 02/10] Account for root --- synapse/glob.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/glob.py b/synapse/glob.py index 4224bd42a24..21c786f7d83 100644 --- a/synapse/glob.py +++ b/synapse/glob.py @@ -23,7 +23,10 @@ def _asynciostacks(*args, **kwargs): # pragma: no cover task.print_stack() if hasattr(task, '_syn_task'): st = task._syn_task - print(f'Task is a syn task with the following information: iden={st.iden} user={st.user.iden} username={st.user.name}') + root = None + if st.root is not None: + root = st.root.iden + print(f'Task is a syn task with the following information: iden={st.iden} name={st.name} root={root} user={st.user.iden} username={st.user.name}') pprint.pprint(st.info) print(80 * '*') print('Faulthandler stack frames per thread:') From 6e1aab3f3a39262b7538f52922407c7748883acf Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 5 Feb 2026 15:10:28 -0500 Subject: [PATCH 03/10] Add quick tests. --- synapse/glob.py | 6 +++--- synapse/tests/test_glob.py | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/synapse/glob.py b/synapse/glob.py index 21c786f7d83..a42bf06f692 100644 --- a/synapse/glob.py +++ b/synapse/glob.py @@ -17,7 +17,7 @@ def _asynciostacks(*args, **kwargs): # pragma: no cover A signal handler used to print asyncio task stacks and thread stacks. ''' print(80 * '*') - print('Asyncio tasks stacks:') + print('Asyncio task stacks:') tasks = asyncio.all_tasks(_glob_loop) for task in tasks: task.print_stack() @@ -26,8 +26,8 @@ def _asynciostacks(*args, **kwargs): # pragma: no cover root = None if st.root is not None: root = st.root.iden - print(f'Task is a syn task with the following information: iden={st.iden} name={st.name} root={root} user={st.user.iden} username={st.user.name}') - pprint.pprint(st.info) + print(f'Task is a syntask with the following information: iden={st.iden} name={st.name} root={root} user={st.user.iden} username={st.user.name}') + print(pprint.pformat(st.info)) print(80 * '*') print('Faulthandler stack frames per thread:') faulthandler.dump_traceback() diff --git a/synapse/tests/test_glob.py b/synapse/tests/test_glob.py index 536cf26bd5a..974b1fc00b1 100644 --- a/synapse/tests/test_glob.py +++ b/synapse/tests/test_glob.py @@ -1,3 +1,6 @@ +import asyncio +import unittest.mock as mock + import synapse.glob as s_glob import synapse.tests.utils as s_t_utils @@ -11,3 +14,42 @@ async def afoo(): retn = s_glob.sync(afoo()) self.eq(retn, 42) + + async def test_glob_stacks(self): + + lines = [] + def mock_print(*args, **kwargs): + self.isinstance(args[0], str) + lines.append(args[0]) + + with mock.patch('builtins.print', mock_print): + self.none(s_glob._threadstacks()) + + text = '\n'.join(lines) + self.isin('Faulthandler stack frames per thread', text) + + lines.clear() + + async with self.getTestCore() as core: + + q = 'while (true) { $lib.time.sleep(60) }' + event = asyncio.Event() + + async def coro(): + async for _ in core.storm(q): + event.set() + + fut = core.schedCoro(coro()) + + self.true(await asyncio.wait_for(event.wait(), timeout=12)) + + with mock.patch('builtins.print', mock_print): + self.none(s_glob._asynciostacks()) + + fut.cancel() + + text = ''.join(lines) + self.isin('Asyncio task stacks', text) + self.isin('Task is a syntask with the following information', text) + self.isin(q, text) + self.isin('Faulthandler stack frames per thread', text) From 92a04496cf6a5f88e84306e82c07798075e65d16 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 5 Feb 2026 15:15:09 -0500 Subject: [PATCH 04/10] Ensure that the root logic is handled --- synapse/tests/test_glob.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/tests/test_glob.py b/synapse/tests/test_glob.py index 974b1fc00b1..e4c8b305624 100644 --- a/synapse/tests/test_glob.py +++ b/synapse/tests/test_glob.py @@ -35,11 +35,14 @@ def mock_print(*args, **kwargs): q = 'while (true) { $lib.time.sleep(60) }' event = asyncio.Event() - async def coro(): - async for _ in core.storm(q): + async def coro(info): + async for mesg in core.storm(q): + if mesg[0] == 'init': + info |= mesg[1] event.set() - fut = core.schedCoro(coro()) + init_mesg = {} + fut = core.schedCoro(coro(init_mesg)) self.true(await asyncio.wait_for(event.wait(), timeout=12)) @@ -52,4 +55,6 @@ async def coro(): self.isin('Asyncio task stacks', text) self.isin('Task is a syntask with the following information', text) self.isin(q, text) + self.isin('root=None', text) + self.isin(f'root={init_mesg.get("task", "newp")}', text) self.isin('Faulthandler stack frames per thread', text) From 72a287395b193ae324000665ab733c5709732f54 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 5 Feb 2026 17:57:31 -0500 Subject: [PATCH 05/10] Embed time into sigusr1 / sigusr2 output --- synapse/glob.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/synapse/glob.py b/synapse/glob.py index a42bf06f692..4b2ac87f5ce 100644 --- a/synapse/glob.py +++ b/synapse/glob.py @@ -3,6 +3,7 @@ import signal import asyncio import logging +import datetime import threading import faulthandler @@ -12,12 +13,19 @@ _glob_thrd = None +def _get_ts(dt=None): + if dt is None: + datetime.datetime.now(datetime.timezone.utc) + millis = dt.microsecond / 1000 + return '%d%.2d%.2d%.2d%.2d%.2d%.3d' % (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, millis) + def _asynciostacks(*args, **kwargs): # pragma: no cover ''' A signal handler used to print asyncio task stacks and thread stacks. ''' + ts = _get_ts() print(80 * '*') - print('Asyncio task stacks:') + print(f'Asyncio task stacks @ {ts}:') tasks = asyncio.all_tasks(_glob_loop) for task in tasks: task.print_stack() @@ -29,7 +37,7 @@ def _asynciostacks(*args, **kwargs): # pragma: no cover print(f'Task is a syntask with the following information: iden={st.iden} name={st.name} root={root} user={st.user.iden} username={st.user.name}') print(pprint.pformat(st.info)) print(80 * '*') - print('Faulthandler stack frames per thread:') + print(f'Faulthandler stack frames per thread @ {ts}:') faulthandler.dump_traceback() print(80 * '*') @@ -37,8 +45,9 @@ def _threadstacks(*args, **kwargs): # pragma: no cover ''' A signal handler used to print thread stacks. ''' + ts = _get_ts() print(80 * '*') - print('Faulthandler stack frames per thread:') + print(f'Faulthandler stack frames per thread @ {ts}:') faulthandler.dump_traceback() print(80 * '*') From a7d55a0d4259a4f59b4d61d71781041f00db138d Mon Sep 17 00:00:00 2001 From: epiphyte Date: Fri, 6 Feb 2026 12:48:04 -0500 Subject: [PATCH 06/10] Fix dt --- synapse/glob.py | 2 +- synapse/tests/test_glob.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/glob.py b/synapse/glob.py index 4b2ac87f5ce..c0d50dd2ef0 100644 --- a/synapse/glob.py +++ b/synapse/glob.py @@ -15,7 +15,7 @@ def _get_ts(dt=None): if dt is None: - datetime.datetime.now(datetime.timezone.utc) + dt = datetime.datetime.now(datetime.timezone.utc) millis = dt.microsecond / 1000 return '%d%.2d%.2d%.2d%.2d%.2d%.3d' % (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, millis) diff --git a/synapse/tests/test_glob.py b/synapse/tests/test_glob.py index e4c8b305624..643031ffe7e 100644 --- a/synapse/tests/test_glob.py +++ b/synapse/tests/test_glob.py @@ -51,7 +51,7 @@ async def coro(info): fut.cancel() - text = ''.join(lines) + text = '\n'.join(lines) self.isin('Asyncio task stacks', text) self.isin('Task is a syntask with the following information', text) self.isin(q, text) From 248e7a85d2e7b074ecf0ca27c5bd199ed2e0d25e Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 19 Feb 2026 13:19:07 -0500 Subject: [PATCH 07/10] Use now() and repr() for time. --- synapse/glob.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/synapse/glob.py b/synapse/glob.py index c0d50dd2ef0..d6d62a9dc7e 100644 --- a/synapse/glob.py +++ b/synapse/glob.py @@ -3,27 +3,22 @@ import signal import asyncio import logging -import datetime import threading import faulthandler +import synapse.common as s_common +import synapse.lib.time as s_time + logger = logging.getLogger(__name__) _glob_loop = None _glob_thrd = None - -def _get_ts(dt=None): - if dt is None: - dt = datetime.datetime.now(datetime.timezone.utc) - millis = dt.microsecond / 1000 - return '%d%.2d%.2d%.2d%.2d%.2d%.3d' % (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, millis) - def _asynciostacks(*args, **kwargs): # pragma: no cover ''' A signal handler used to print asyncio task stacks and thread stacks. ''' - ts = _get_ts() + ts = s_time.repr(s_common.now()) print(80 * '*') print(f'Asyncio task stacks @ {ts}:') tasks = asyncio.all_tasks(_glob_loop) @@ -45,7 +40,7 @@ def _threadstacks(*args, **kwargs): # pragma: no cover ''' A signal handler used to print thread stacks. ''' - ts = _get_ts() + ts = s_time.repr(s_common.now()) print(80 * '*') print(f'Faulthandler stack frames per thread @ {ts}:') faulthandler.dump_traceback() From 51e4d990a266cbff189891e039daf78d74ccf2d2 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 19 Feb 2026 13:39:00 -0500 Subject: [PATCH 08/10] Print the task name directly --- synapse/glob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/glob.py b/synapse/glob.py index d6d62a9dc7e..ec5b996c855 100644 --- a/synapse/glob.py +++ b/synapse/glob.py @@ -29,7 +29,7 @@ def _asynciostacks(*args, **kwargs): # pragma: no cover root = None if st.root is not None: root = st.root.iden - print(f'Task is a syntask with the following information: iden={st.iden} name={st.name} root={root} user={st.user.iden} username={st.user.name}') + print(f'{task.get_name()} is a syntask with the following information: iden={st.iden} name={st.name} root={root} user={st.user.iden} username={st.user.name}') print(pprint.pformat(st.info)) print(80 * '*') print(f'Faulthandler stack frames per thread @ {ts}:') From 0eece08a2490267bdc7bc3440f9b60cf37a01d26 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Thu, 19 Feb 2026 14:00:43 -0500 Subject: [PATCH 09/10] Fix test --- synapse/tests/test_glob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/tests/test_glob.py b/synapse/tests/test_glob.py index 643031ffe7e..b11746c985a 100644 --- a/synapse/tests/test_glob.py +++ b/synapse/tests/test_glob.py @@ -53,7 +53,7 @@ async def coro(info): text = '\n'.join(lines) self.isin('Asyncio task stacks', text) - self.isin('Task is a syntask with the following information', text) + self.isin('is a syntask with the following information', text) self.isin(q, text) self.isin('root=None', text) self.isin(f'root={init_mesg.get("task", "newp")}', text) From 33651981dd113103e04a7802dd8569c8be3391a9 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Wed, 4 Mar 2026 10:41:42 -0500 Subject: [PATCH 10/10] updates from feedback --- synapse/glob.py | 2 +- synapse/tests/test_glob.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/glob.py b/synapse/glob.py index ec5b996c855..d6801dadae1 100644 --- a/synapse/glob.py +++ b/synapse/glob.py @@ -29,7 +29,7 @@ def _asynciostacks(*args, **kwargs): # pragma: no cover root = None if st.root is not None: root = st.root.iden - print(f'{task.get_name()} is a syntask with the following information: iden={st.iden} name={st.name} root={root} user={st.user.iden} username={st.user.name}') + print(f'syntask metadata: iden={st.iden} name={st.name} root={root} user={st.user.iden} username={st.user.name}') print(pprint.pformat(st.info)) print(80 * '*') print(f'Faulthandler stack frames per thread @ {ts}:') diff --git a/synapse/tests/test_glob.py b/synapse/tests/test_glob.py index b11746c985a..5d181213b1d 100644 --- a/synapse/tests/test_glob.py +++ b/synapse/tests/test_glob.py @@ -53,7 +53,7 @@ async def coro(info): text = '\n'.join(lines) self.isin('Asyncio task stacks', text) - self.isin('is a syntask with the following information', text) + self.isin('syntask metadata', text) self.isin(q, text) self.isin('root=None', text) self.isin(f'root={init_mesg.get("task", "newp")}', text)