From 93e163e848a80fd6347a1d8cdc524e8c343fc0dc Mon Sep 17 00:00:00 2001 From: Run Wang Date: Thu, 14 May 2026 17:04:09 +0200 Subject: [PATCH] feat(sim): add ring-buffer wrapper for gvsoc trace to bound disk usage When debugging gvsoc hangs on Siracusa (which has no working UART) the only progress signal is gvsoc's --trace output. A full --trace=insn run on a real-size network produces tens of GB before the simulation completes -- enough to fill the build host's disk on a single attempt. scripts/ring_tee.py is a ~170-line pure-stdlib wrapper that reads stdin, rotates writes across N files of fixed size each, and discards the oldest content once the keep * size quota is full. Three runtime features make it useful for hang debugging: * 5-second stderr heartbeat: distinguishes "gvsoc itself froze" (heartbeat stops because stdin is empty) from "simulated firmware deadlocked" (heartbeat continues but simulated PC stays still in the trace). * SIGUSR1 handler: writes .snapshot as a chronological concatenation of every file still on disk (oldest -> newest), so a single `kill -USR1 ` grabs the recent window at the moment of the hang. * Ring rotation: total disk use is bounded by keep * size (default 600 MB) regardless of how long the simulation runs. cmake/simulation.cmake gains three options: GVSOC_RING_TRACE OFF Enable the wrapper (default OFF; behaviour identical to before when not set). GVSOC_RING_TRACE_SIZE 500M Per-file size, K/M/G suffix accepted. GVSOC_RING_TRACE_KEEP 3 Number of rotating files. When GVSOC_RING_TRACE=ON, the gvsoc_ target pipes gvsoc's stderr through ring_tee.py via bash process substitution. Default OFF. Known limitation: this gvsoc build (PULP siracusa target) emits its --trace=insn output on stdout rather than stderr. To capture that output via this wrapper you currently need to invoke gvsoc through a hand-written bash script that pipes stdout into ring_tee, rather than using the CMake target -- the macro's bash -c quoting also loses the process substitution when expanded by make's /bin/sh. Both limitations left to a follow-up; this commit lands the wrapper script and CMake plumbing so the infrastructure is in place. Co-Authored-By: Claude Opus 4.7 (1M context) --- cmake/simulation.cmake | 46 ++++++++--- scripts/ring_tee.py | 173 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+), 9 deletions(-) create mode 100755 scripts/ring_tee.py diff --git a/cmake/simulation.cmake b/cmake/simulation.cmake index 55525fee..706d79c6 100644 --- a/cmake/simulation.cmake +++ b/cmake/simulation.cmake @@ -23,6 +23,13 @@ if(gvsoc_simulation) add_compile_definitions(GVSOC_SIMULATION) endif() +# When ON, gvsoc's stderr (which is where --trace=... output goes) is piped +# through scripts/ring_tee.py so the trace files rotate inside a fixed disk +# budget instead of growing to tens of GB. See add_gvsoc_emulation below. +OPTION(GVSOC_RING_TRACE "Pipe gvsoc stderr through a rotating ring buffer" OFF) +set(GVSOC_RING_TRACE_SIZE "500M" CACHE STRING "Per-file size for ring_tee rotation (e.g. 500M, 1G)") +set(GVSOC_RING_TRACE_KEEP "3" CACHE STRING "Number of rotating trace files to keep") + ######################### ## Utility Functions ## ######################### @@ -92,13 +99,34 @@ macro(add_gvsoc_emulation name target) make_directory(${GVSOC_WORKDIR}) set(GVSOC_EXECUTABLE "${GVSOC_INSTALL_DIR}/bin/gvsoc") set(GVSOC_BINARY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${name}") - add_custom_target(gvsoc_${name} - DEPENDS ${name} - WORKING_DIRECTORY ${GVSOC_WORKDIR} - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_BINARY_DIR}/*.bin ${GVSOC_WORKDIR}/ || true - COMMAND ${GVSOC_EXECUTABLE} --target=${target} --binary ${GVSOC_BINARY} --work-dir=${GVSOC_WORKDIR} ${GVSOC_EXTRA_FLAGS} image flash run - COMMENT "Simulating deeploytest ${name} with gvsoc for the target ${target}" - POST_BUILD - USES_TERMINAL - ) + if(GVSOC_RING_TRACE) + # Pipe gvsoc's stderr (where --trace events land) through ring_tee.py. + # We use bash process substitution so stdout still flows to the + # terminal untouched — only the trace stream gets capped. + set(_ring_tee "${CMAKE_SOURCE_DIR}/scripts/ring_tee.py") + string(REPLACE ";" " " _extra_flags_str "${GVSOC_EXTRA_FLAGS}") + set(_gvsoc_cmd + "'${GVSOC_EXECUTABLE}' --target=${target} --binary '${GVSOC_BINARY}' --work-dir='${GVSOC_WORKDIR}' ${_extra_flags_str} image flash run " + "2> >(python3 '${_ring_tee}' --prefix '${GVSOC_WORKDIR}/gv_trace' --size ${GVSOC_RING_TRACE_SIZE} --keep ${GVSOC_RING_TRACE_KEEP} >&2)") + string(CONCAT _gvsoc_cmd_str ${_gvsoc_cmd}) + add_custom_target(gvsoc_${name} + DEPENDS ${name} + WORKING_DIRECTORY ${GVSOC_WORKDIR} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_BINARY_DIR}/*.bin ${GVSOC_WORKDIR}/ || true + COMMAND bash -c "${_gvsoc_cmd_str}" + COMMENT "Simulating ${name} with gvsoc (ring trace: ${GVSOC_RING_TRACE_KEEP} x ${GVSOC_RING_TRACE_SIZE} at ${GVSOC_WORKDIR}/gv_trace.*)" + POST_BUILD + USES_TERMINAL + ) + else() + add_custom_target(gvsoc_${name} + DEPENDS ${name} + WORKING_DIRECTORY ${GVSOC_WORKDIR} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_BINARY_DIR}/*.bin ${GVSOC_WORKDIR}/ || true + COMMAND ${GVSOC_EXECUTABLE} --target=${target} --binary ${GVSOC_BINARY} --work-dir=${GVSOC_WORKDIR} ${GVSOC_EXTRA_FLAGS} image flash run + COMMENT "Simulating deeploytest ${name} with gvsoc for the target ${target}" + POST_BUILD + USES_TERMINAL + ) + endif() endmacro() diff --git a/scripts/ring_tee.py b/scripts/ring_tee.py new file mode 100755 index 00000000..1bf9d578 --- /dev/null +++ b/scripts/ring_tee.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# SPDX-License-Identifier: Apache-2.0 +""" +Ring-buffer tee. Reads stdin, writes to a fixed set of rotating files, +discarding the oldest data once total disk usage hits `--keep * --size`. + +Used by cmake/simulation.cmake to keep gvsoc's `--trace` output from +filling the disk: instead of one growing trace.log, you get + .0 .1 .2 ... +each capped at --size, overwritten round-robin. .current is a +symlink to the file being written right now. + +While running, a heartbeat is printed to stderr every --heartbeat +seconds so you can tell the sim is actually progressing. + +SIGUSR1 dumps a concatenated .snapshot (oldest -> newest) of +everything still on disk, for grabbing the recent window when gvsoc +hangs: `kill -USR1 `. +""" +import argparse +import os +import re +import signal +import sys +import time + + +def parse_size(s): + m = re.fullmatch(r'\s*(\d+)\s*([KMGkmg]?)B?\s*', s) + if not m: + raise ValueError("bad size %r (use e.g. 500M, 1G)" % s) + return int(m.group(1)) * {'': 1, 'k': 1 << 10, 'm': 1 << 20, 'g': 1 << 30}[m.group(2).lower()] + + +class RingTee: + def __init__(self, prefix, size, keep, passthrough, heartbeat): + self.prefix = prefix + self.size = size + self.keep = keep + self.passthrough = passthrough + self.heartbeat = heartbeat + self.idx = 0 + self.bytes_in_file = 0 + self.total_bytes = 0 + self.start = time.monotonic() + self.last_hb = self.start + self.fh = None + # Wipe any stale ring files from a previous run so old data + # can't masquerade as part of the current trace. + for i in range(keep): + try: + os.unlink("%s.%d" % (prefix, i)) + except OSError: + pass + self._open_current() + signal.signal(signal.SIGUSR1, self._snapshot) + + def _path(self, i): + return "%s.%d" % (self.prefix, i % self.keep) + + def _open_current(self): + if self.fh is not None: + self.fh.close() + path = self._path(self.idx) + self.fh = open(path, 'wb') + self.bytes_in_file = 0 + cur = "%s.current" % self.prefix + try: + if os.path.lexists(cur): + os.unlink(cur) + os.symlink(os.path.basename(path), cur) + except OSError: + pass + + def _snapshot(self, *_): + try: + self.fh.flush() + except Exception: + pass + snap = "%s.snapshot" % self.prefix + try: + with open(snap, 'wb') as out: + lo = max(0, self.idx - self.keep + 1) + for i in range(lo, self.idx + 1): + try: + with open(self._path(i), 'rb') as f: + while True: + buf = f.read(1 << 20) + if not buf: + break + out.write(buf) + except OSError: + pass + sys.stderr.write("[ring_tee] snapshot -> %s\n" % snap) + sys.stderr.flush() + except Exception as e: + sys.stderr.write("[ring_tee] snapshot failed: %s\n" % e) + + def _hb(self, force=False): + now = time.monotonic() + if not force and now - self.last_hb < self.heartbeat: + return + self.last_hb = now + sys.stderr.write( + "[ring_tee] +%4ds total=%dMB file=%d fill=%dMB/%dMB\n" % ( + int(now - self.start), + self.total_bytes >> 20, + self.idx % self.keep, + self.bytes_in_file >> 20, + self.size >> 20, + ) + ) + sys.stderr.flush() + + def run(self): + rd = sys.stdin.buffer + out = sys.stdout.buffer if self.passthrough else None + BUF = 1 << 16 + try: + while True: + # read1: return as soon as any bytes are available, so heartbeats + # and SIGUSR1 handlers fire promptly even on a slow trace stream. + chunk = rd.read1(BUF) + if not chunk: + break + # Rotate before writing if this chunk would push us over. + if self.bytes_in_file + len(chunk) > self.size: + self.idx += 1 + self._open_current() + self.fh.write(chunk) + self.bytes_in_file += len(chunk) + self.total_bytes += len(chunk) + if out is not None: + try: + out.write(chunk) + out.flush() + except BrokenPipeError: + out = None + self._hb() + finally: + try: + self.fh.flush() + self.fh.close() + except Exception: + pass + self._hb(force=True) + sys.stderr.write("[ring_tee] done, total=%dMB across %d file(s)\n" % ( + self.total_bytes >> 20, min(self.idx + 1, self.keep))) + sys.stderr.flush() + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--prefix', required=True, + help='Output prefix; writes .0 .. .{keep-1}') + ap.add_argument('--size', default='500M', + help='Per-file size (default 500M). Suffixes K/M/G.') + ap.add_argument('--keep', type=int, default=3, + help='Number of rotating files to keep (default 3)') + ap.add_argument('--passthrough', action='store_true', + help='Also echo stdin to stdout (default: silent on stdout)') + ap.add_argument('--heartbeat', type=float, default=5.0, + help='Heartbeat to stderr every N seconds (default 5)') + args = ap.parse_args() + os.makedirs(os.path.dirname(os.path.abspath(args.prefix)) or '.', exist_ok=True) + RingTee(args.prefix, parse_size(args.size), args.keep, + args.passthrough, args.heartbeat).run() + + +if __name__ == '__main__': + main()