From 6c9ce530588af49c06ad3f4ce4eb43b501a5b405 Mon Sep 17 00:00:00 2001 From: Larry White Date: Tue, 16 Jun 2026 11:35:27 -0400 Subject: [PATCH] Hard-exit ACQ on terminate to avoid device-finalizer SIGABRT (#710) run_acq handled TerminateServerRequest by tearing down devices via close_streams() and then returning, which let Python GC-finalize the device objects as the frame unwound. close_streams() has already freed their native handles, so the finalizers (LSL outlets, MetaWear/warble, camera SDKs) re-ran native teardown on freed handles and aborted the process with SIGABRT before it could exit. This is the dominant ACQ_0 silent death: 18 of 22 faulthandler dumps in the 2026-05-28..06-11 window, each with the main thread parked at server_acq.py:46 (run_acq just returned) and the last DB log line 'Microphone: Exiting LSL Thread' (device teardown complete). Flush logging and os._exit() directly from the handler instead of returning, so no device finalizer runs. The terminate path keeps its clean log trail (Stopping ACQ -> Closing app log db connection). Covers the TerminateServerRequest path. The exception path (run_acq raises -> frame unwind -> same finalizer abort) is a separate follow-up. Confirm the exact aborting module via WER LocalDumps before merge. --- neurobooth_os/server_acq.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/neurobooth_os/server_acq.py b/neurobooth_os/server_acq.py index f9b9cc20..8a178b0e 100644 --- a/neurobooth_os/server_acq.py +++ b/neurobooth_os/server_acq.py @@ -33,6 +33,20 @@ def countdown(period): t2 = local_clock() +def _flush_and_hard_exit(exit_code: int) -> None: + """Flush logging and terminate via ``os._exit`` without running finalizers. + + The acquisition process must not let Python GC-finalize its device objects + on the way out. ``DeviceManager.close_streams()`` already frees their native + handles; re-running native teardown on freed handles (LSL outlets, + MetaWear/warble, camera SDKs) aborts the process (#710). ``os._exit`` skips + finalizers and ``atexit`` entirely, so we flush logging explicitly first, + then exit, rather than returning and unwinding the frame. + """ + logging.shutdown() + os._exit(exit_code) + + def main(): acq_index = int(sys.argv[1]) if len(sys.argv) > 1 else 0 logger = None @@ -52,8 +66,7 @@ def main(): exc_info=sys.exc_info()) exit_code = 1 finally: - logging.shutdown() - os._exit(exit_code) + _flush_and_hard_exit(exit_code) def run_acq(logger, acq_index: int = 0): @@ -207,7 +220,14 @@ def run_acq(logger, acq_index: int = 0): if device_manager is not None: device_manager.close_streams() - break + + # #710: do not fall through to run_acq's return here. Unwinding + # the frame GC-finalizes `device_manager`, whose device objects + # re-run native teardown (LSL outlets, MetaWear/warble, camera + # SDKs) on handles close_streams() already freed -> SIGABRT before + # the process can exit. Hard-exit instead, so no finalizer runs. + logger.debug("Stopping ACQ") + _flush_and_hard_exit(0) else: logger.error(f'Unexpected message received: {message.model_dump_json()}') except Exception as argument: