From eea4104e4298f93e25e01a0c0f7c18514a747da8 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 24 Jul 2026 16:35:20 -0400 Subject: [PATCH 1/9] Test pytest-xdist fix [skip circle] [skip azp] --- .github/workflows/tests.yml | 22 +----------------- tools/github_actions_test.sh | 43 +++++------------------------------- 2 files changed, 6 insertions(+), 59 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 11c903dcc95..80457408174 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,30 +47,9 @@ jobs: fail-fast: false matrix: include: - - os: ubuntu-latest - python: '3.13' - kind: pip - - os: ubuntu-latest - python: '3.14' - kind: pip-pre - - os: ubuntu-latest - python: '3.13' - kind: conda - os: macos-latest # arm64 python: '3.14' kind: pip - - os: macos-15-intel # Intel - python: '3.13' - kind: pip - - os: ubuntu-latest - python: '3.12' - kind: minimal - - os: ubuntu-22.04 - python: '3.10' - kind: old - - os: ubuntu-latest - python: '3.14t' # free-threaded - kind: pip-ft steps: - uses: actions/checkout@v7.0.0 with: @@ -130,6 +109,7 @@ jobs: - run: bash ./tools/github_actions_verify_python.sh "${{ matrix.python }}" - run: bash ./tools/github_actions_dependencies.sh timeout-minutes: 10 + - run: pip install "pytest-xdist @ git+https://github.com/larsoner/pytest-xdist.git@master" - run: python ./tools/github_actions_check_old_env.py if: matrix.kind == 'old' # Minimal commands on Linux (macOS stalls) diff --git a/tools/github_actions_test.sh b/tools/github_actions_test.sh index 439aebab76f..7296f12f012 100755 --- a/tools/github_actions_test.sh +++ b/tools/github_actions_test.sh @@ -3,24 +3,16 @@ set -eo pipefail if [[ "${CI_OS_NAME}" == "ubuntu"* ]]; then - CONDITIONS=("not (ultraslowtest or pgtest)") + CONDITION="not (ultraslowtest or pgtest)" elif [[ "${CI_OS_NAME}" == "macos"* ]]; then # detect arch and run slowtest on arm64 only (pgtest is already ultraslow on macOS) if [[ "$(uname -m)" == "arm64" ]]; then - # Split the PyVista/VTK ("pvtest") tests into their own xdist invocation. - # Run all together, a worker that accumulates heavy state (loky pools, - # leftover ipykernel/jupyter_client asyncio loops, tqdm monitors) creates - # the scheduling jitter that trips an xdist loadscope dispatch deadlock at - # end-of-run: workers idle in TestQueue.get, controller loops forever in - # dsession.loop_once without dispatching the remaining scopes or SHUTDOWN - # (see pytest-xdist#1313). Two shorter, lighter invocations avoid it while - # keeping parallelism; pvtest alone under xdist does not hang. - CONDITIONS=("not (ultraslowtest or pgtest or pvtest)" "pvtest and not ultraslowtest") + CONDITION="not (ultraslowtest or pgtest)" else - CONDITIONS=("not (slowtest or pgtest)") + CONDITION="not (slowtest or pgtest)" fi elif [[ "${CI_OS_NAME}" == "windows"* ]]; then - CONDITIONS=("not (slowtest or pgtest)") + CONDITION="not (slowtest or pgtest)" else echo "✕ ERROR: Unrecognized CI_OS_NAME=${CI_OS_NAME}" exit 1 @@ -53,29 +45,4 @@ if [[ ! -z "$CONDA_ENV" ]] && [[ "${CI_OS_NAME}" != "windows"* ]] && [[ "${MNE_C fi # $COV_ARGS is set in github_actions_env_vars.sh (coverage only on Python >= 3.14) -# Run each marker expression as its own pytest invocation (usually just one; on -# macOS arm64 the pvtest tests are split out, see above). Later runs append -# coverage, each gets its own junit file, and we run them all before exiting -# with the first nonzero code so one failing split does not mask the other. -set +e # capture each pytest's exit code manually rather than aborting on it -CODE=0 -COV_APPEND="" -i=0 -for CONDITION in "${CONDITIONS[@]}"; do - if [ "${#CONDITIONS[@]}" -gt 1 ]; then - THIS_JUNIT="${JUNIT_PATH%.xml}-${i}.xml" - else - THIS_JUNIT="$JUNIT_PATH" - fi - set -x - pytest -m "${CONDITION}" -n "$PYTEST_XDIST_N" --dist loadscope --timeout=120 --timeout-method=thread -o faulthandler_timeout=110 ${COV_ARGS} ${COV_APPEND} --color=yes --continue-on-collection-errors --junit-xml="$THIS_JUNIT" -vv ${USE_DIRS} - THIS_CODE=$? - set +x - echo "Exited with code $THIS_CODE for: ${CONDITION}" - if [ "$THIS_CODE" -ne 0 ] && [ "$CODE" -eq 0 ]; then - CODE=$THIS_CODE - fi - COV_APPEND="--cov-append" - i=$((i + 1)) -done -exit $CODE +pytest -m "${CONDITION}" -n "$PYTEST_XDIST_N" --dist loadscope --timeout=120 --timeout-method=thread -o faulthandler_timeout=110 ${COV_ARGS} ${COV_APPEND} --color=yes --continue-on-collection-errors --junit-xml="$THIS_JUNIT" -vv ${USE_DIRS} From 124c594577e912518626a30c5b5ca88178230921 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 24 Jul 2026 17:05:17 -0400 Subject: [PATCH 2/9] TST: Try it [skip azp] [skip circle] --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 80457408174..a98a97166d2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,7 +109,7 @@ jobs: - run: bash ./tools/github_actions_verify_python.sh "${{ matrix.python }}" - run: bash ./tools/github_actions_dependencies.sh timeout-minutes: 10 - - run: pip install "pytest-xdist @ git+https://github.com/larsoner/pytest-xdist.git@master" + - run: pip install "pytest-xdist @ git+https://github.com/larsoner/pytest-xdist.git@lock" - run: python ./tools/github_actions_check_old_env.py if: matrix.kind == 'old' # Minimal commands on Linux (macOS stalls) From abd63271394835e070a34cb8bae83ed3e6966369 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 24 Jul 2026 19:19:46 -0400 Subject: [PATCH 3/9] WIP: Diagnostics [skip azp] [skip circle] --- .github/workflows/tests.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a98a97166d2..184e7b24ecd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -134,6 +134,26 @@ jobs: path: ~/mne_data - run: bash ./tools/github_actions_download.sh - run: bash ./tools/github_actions_test.sh # for some reason on macOS we need to run "bash X" in order for a failed test run to show up + # Diagnose silent pytest-xdist worker deaths on macOS: a JetsamEvent-*.ips + # report means the OS killed the worker under memory pressure (SIGKILL), + # while a python-*.ips crash report means a real native crash (with stack) + - name: Collect macOS crash diagnostics + if: (success() || failure()) && startsWith(matrix.os, 'macos') + run: | + set -x + mkdir -p crash_diagnostics/user_reports crash_diagnostics/system_reports + find ~/Library/Logs/DiagnosticReports -type f -mmin -90 -exec cp {} crash_diagnostics/user_reports/ \; 2>/dev/null || true + sudo find /Library/Logs/DiagnosticReports -type f -mmin -90 -exec cp {} crash_diagnostics/system_reports/ \; 2>/dev/null || true + sudo chown -R "$USER" crash_diagnostics || true + { sysctl hw.memsize vm.swapusage; vm_stat; } > crash_diagnostics/memory_snapshot.txt || true + log show --last 45m --style syslog --predicate 'eventMessage CONTAINS[c] "memorystatus" OR eventMessage CONTAINS[c] "jetsam"' > crash_diagnostics/memorystatus.log || true + ls -alR crash_diagnostics || true + - uses: actions/upload-artifact@v7 + if: (success() || failure()) && startsWith(matrix.os, 'macos') + with: + name: crash-diagnostics-${{ matrix.os }}-${{ matrix.kind }}-${{ matrix.python }} + path: crash_diagnostics + if-no-files-found: ignore - uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} From 766f8684138dd823f2f219c3226446d058776cf9 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 24 Jul 2026 19:53:58 -0400 Subject: [PATCH 4/9] Use native cursor [skip circle] [skip azp] --- mne/viz/backends/_qt.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mne/viz/backends/_qt.py b/mne/viz/backends/_qt.py index b86d946fdf6..ad7790904ca 100644 --- a/mne/viz/backends/_qt.py +++ b/mne/viz/backends/_qt.py @@ -131,6 +131,13 @@ os.environ.setdefault("QT_MAC_WANTS_LAYER", "1") +def _qcursor(name): + # map to a native name on macOS (instead of using Qt pixmap) + if sys.platform == "darwin" and name in ("WaitCursor", "BusyCursor"): + name = "ForbiddenCursor" + return QCursor(getattr(Qt, name)) + + # fix for qscroll needing two layouts, one parent, one child def _get_layout(layout): if hasattr(layout, "_parent_layout"): @@ -885,7 +892,7 @@ def func(button): button_id = widget.standardButton(button) for button_name in _QtDialog.supported_button_names: if button_id == getattr(QMessageBox, button_name): - widget.setCursor(QCursor(Qt.WaitCursor)) + widget.setCursor(_qcursor("WaitCursor")) try: callback(button_name) finally: @@ -1563,7 +1570,7 @@ def _window_set_cursor(self, cursor): self._window.setCursor(cursor) def _window_new_cursor(self, name): - return QCursor(getattr(Qt, name)) + return _qcursor(name) @contextmanager def _window_ensure_minimum_sizes(self): From b2febc7a9d0f482c8d639bc12f5f57e94075f827 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 24 Jul 2026 20:21:48 -0400 Subject: [PATCH 5/9] Push more diagnostics [skip azp] [skip circle] --- .github/workflows/tests.yml | 6 +++ mne/viz/backends/_qt.py | 5 +- tools/cg_canary.py | 105 ++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 tools/cg_canary.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 184e7b24ecd..38f52d46830 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -134,6 +134,11 @@ jobs: path: ~/mne_data - run: bash ./tools/github_actions_download.sh - run: bash ./tools/github_actions_test.sh # for some reason on macOS we need to run "bash X" in order for a failed test run to show up + env: + # CGColorSpace over-release canary (temporary diagnostic, see tools/cg_canary.py) + PYTHONPATH: ${{ github.workspace }}/tools + PYTEST_ADDOPTS: -p cg_canary + CG_CANARY_LOG: ${{ github.workspace }}/cg_canary.log # Diagnose silent pytest-xdist worker deaths on macOS: a JetsamEvent-*.ips # report means the OS killed the worker under memory pressure (SIGKILL), # while a python-*.ips crash report means a real native crash (with stack) @@ -146,6 +151,7 @@ jobs: sudo find /Library/Logs/DiagnosticReports -type f -mmin -90 -exec cp {} crash_diagnostics/system_reports/ \; 2>/dev/null || true sudo chown -R "$USER" crash_diagnostics || true { sysctl hw.memsize vm.swapusage; vm_stat; } > crash_diagnostics/memory_snapshot.txt || true + cp "$GITHUB_WORKSPACE/cg_canary.log" crash_diagnostics/ || true log show --last 45m --style syslog --predicate 'eventMessage CONTAINS[c] "memorystatus" OR eventMessage CONTAINS[c] "jetsam"' > crash_diagnostics/memorystatus.log || true ls -alR crash_diagnostics || true - uses: actions/upload-artifact@v7 diff --git a/mne/viz/backends/_qt.py b/mne/viz/backends/_qt.py index ad7790904ca..dea7ac94271 100644 --- a/mne/viz/backends/_qt.py +++ b/mne/viz/backends/_qt.py @@ -133,8 +133,9 @@ def _qcursor(name): # map to a native name on macOS (instead of using Qt pixmap) - if sys.platform == "darwin" and name in ("WaitCursor", "BusyCursor"): - name = "ForbiddenCursor" + # (temporarily disabled to diagnose the CGColorSpace crash on CI) + # if sys.platform == "darwin" and name in ("WaitCursor", "BusyCursor"): + # name = "ForbiddenCursor" return QCursor(getattr(Qt, name)) diff --git a/tools/cg_canary.py b/tools/cg_canary.py new file mode 100644 index 00000000000..a12da67aace --- /dev/null +++ b/tools/cg_canary.py @@ -0,0 +1,105 @@ +"""pytest plugin: detect CGColorSpace over-release, per test. + +Arms buffer-retains on the CG colorspace objects Qt's cursor conversion can +pass to CGImageCreate (named sRGB, deviceRGB, and the ICC cache entry for the +"VP2030 SERIES" profile embedded in Qt's macOS wait/busy cursor PNGs), then +probes CFGetRetainCount after every test. A persistent drop names the test +that stole references. The buffer retains also *prevent* the SIGTRAP crash. + +Use: PYTHONPATH= pytest -p cg_canary ... (macOS only, no-op elsewhere) + +Under pytest-xdist, worker stdout is redirected to /dev/null by execnet, so +set CG_CANARY_LOG= to also append findings to a file (one shared file +is fine; lines are tagged with the pid). +""" + +import os +import sys + +_armed = {} +_reported = set() + + +def _log(msg): + line = f"[cg-canary] [{os.getpid()}] {msg}" + print(line, flush=True) + path = os.environ.get("CG_CANARY_LOG") + if path: + with open(path, "a") as f: + f.write(line + "\n") + + +if sys.platform == "darwin": + import ctypes + + _cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") + _cf = ctypes.CDLL( + "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation" + ) + _cg.CGColorSpaceCreateWithName.restype = ctypes.c_void_p + _cg.CGColorSpaceCreateWithName.argtypes = [ctypes.c_void_p] + _cg.CGColorSpaceCreateDeviceRGB.restype = ctypes.c_void_p + _cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p + _cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] + _cf.CFDataCreate.restype = ctypes.c_void_p + _cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] + _cf.CFRetain.restype = ctypes.c_void_p + _cf.CFRetain.argtypes = [ctypes.c_void_p] + _cf.CFGetRetainCount.restype = ctypes.c_long + _cf.CFGetRetainCount.argtypes = [ctypes.c_void_p] + + _IMMORTAL = (1 << 32) - 2 + _BUFFER = 500 + + def _arm_one(label, ptr): + if not ptr or label in _armed: + return + rc = _cf.CFGetRetainCount(ptr) + if rc >= _IMMORTAL: + _armed[label] = (ptr, None) # immortal: track for info only + _log(f"{label}: immortal, not tracked") + return + for _ in range(_BUFFER): + _cf.CFRetain(ptr) + base = _cf.CFGetRetainCount(ptr) + _armed[label] = (ptr, base) + _log(f"{label}: armed ptr={ptr:#x} rc={base}") + + def _try_arm(): + if "srgb-named" not in _armed: + k = ctypes.c_void_p.in_dll(_cg, "kCGColorSpaceSRGB") + _arm_one("srgb-named", _cg.CGColorSpaceCreateWithName(k)) + if "device-rgb" not in _armed: + _arm_one("device-rgb", _cg.CGColorSpaceCreateDeviceRGB()) + if "vp2030-icc" not in _armed: + # needs the cocoa plugin resources -> only after QApplication + try: + from qtpy.QtGui import QImage + from qtpy.QtWidgets import QApplication + + if QApplication.instance() is None: + return + img = QImage(":/qt-project.org/mac/cursors/images/spincursor.png") + if img.isNull() or not img.colorSpace().isValid(): + return + icc = bytes(img.colorSpace().iccProfile().data()) + d = _cf.CFDataCreate(None, icc, len(icc)) + _arm_one("vp2030-icc", _cg.CGColorSpaceCreateWithICCData(d)) + except Exception: + pass + + def pytest_runtest_teardown(item): + _try_arm() + for label, (ptr, base) in _armed.items(): + if base is None: + continue + rc = _cf.CFGetRetainCount(ptr) + if rc != base and (label, rc) not in _reported: + _reported.add((label, rc)) + _log(f"*** {label} rc {base} -> {rc} after {item.nodeid} ***") + _armed[label] = (ptr, rc) # new baseline; report each change + + def pytest_terminal_summary(terminalreporter): + for label, (ptr, base) in _armed.items(): + state = "immortal" if base is None else f"final rc={base}" + terminalreporter.write_line(f"[cg-canary] {label}: {state}") From bb7d5ac74edec7f598983b377e76e029cb346fb2 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sat, 25 Jul 2026 09:08:52 -0400 Subject: [PATCH 6/9] More [skip azp] [skip circle] --- .github/workflows/tests.yml | 10 ++-- mne/viz/backends/_qt.py | 5 +- tools/cg_canary.py | 105 ------------------------------------ tools/cg_icc_probe.py | 92 +++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 114 deletions(-) delete mode 100644 tools/cg_canary.py create mode 100644 tools/cg_icc_probe.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 38f52d46830..0f601db1ea8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -121,6 +121,10 @@ jobs: - run: bash ./tools/get_minimal_commands.sh if: startswith(matrix.os, 'ubuntu') && matrix.kind != 'minimal' && matrix.kind != 'old' && matrix.kind != 'pip-ft' - run: bash ./tools/github_actions_infos.sh + # Probe CoreGraphics ICC colorspace cache lifetime (temporary diagnostic, + # expected: WEAK + child SIGTRAP on macOS 26.4, fixed on >= 26.5) + - run: python tools/cg_icc_probe.py + if: startsWith(matrix.os, 'macos') # Check Qt - run: bash ./tools/check_qt_import.sh $MNE_QT_BACKEND if: env.MNE_QT_BACKEND != '' @@ -134,11 +138,6 @@ jobs: path: ~/mne_data - run: bash ./tools/github_actions_download.sh - run: bash ./tools/github_actions_test.sh # for some reason on macOS we need to run "bash X" in order for a failed test run to show up - env: - # CGColorSpace over-release canary (temporary diagnostic, see tools/cg_canary.py) - PYTHONPATH: ${{ github.workspace }}/tools - PYTEST_ADDOPTS: -p cg_canary - CG_CANARY_LOG: ${{ github.workspace }}/cg_canary.log # Diagnose silent pytest-xdist worker deaths on macOS: a JetsamEvent-*.ips # report means the OS killed the worker under memory pressure (SIGKILL), # while a python-*.ips crash report means a real native crash (with stack) @@ -151,7 +150,6 @@ jobs: sudo find /Library/Logs/DiagnosticReports -type f -mmin -90 -exec cp {} crash_diagnostics/system_reports/ \; 2>/dev/null || true sudo chown -R "$USER" crash_diagnostics || true { sysctl hw.memsize vm.swapusage; vm_stat; } > crash_diagnostics/memory_snapshot.txt || true - cp "$GITHUB_WORKSPACE/cg_canary.log" crash_diagnostics/ || true log show --last 45m --style syslog --predicate 'eventMessage CONTAINS[c] "memorystatus" OR eventMessage CONTAINS[c] "jetsam"' > crash_diagnostics/memorystatus.log || true ls -alR crash_diagnostics || true - uses: actions/upload-artifact@v7 diff --git a/mne/viz/backends/_qt.py b/mne/viz/backends/_qt.py index dea7ac94271..ad7790904ca 100644 --- a/mne/viz/backends/_qt.py +++ b/mne/viz/backends/_qt.py @@ -133,9 +133,8 @@ def _qcursor(name): # map to a native name on macOS (instead of using Qt pixmap) - # (temporarily disabled to diagnose the CGColorSpace crash on CI) - # if sys.platform == "darwin" and name in ("WaitCursor", "BusyCursor"): - # name = "ForbiddenCursor" + if sys.platform == "darwin" and name in ("WaitCursor", "BusyCursor"): + name = "ForbiddenCursor" return QCursor(getattr(Qt, name)) diff --git a/tools/cg_canary.py b/tools/cg_canary.py deleted file mode 100644 index a12da67aace..00000000000 --- a/tools/cg_canary.py +++ /dev/null @@ -1,105 +0,0 @@ -"""pytest plugin: detect CGColorSpace over-release, per test. - -Arms buffer-retains on the CG colorspace objects Qt's cursor conversion can -pass to CGImageCreate (named sRGB, deviceRGB, and the ICC cache entry for the -"VP2030 SERIES" profile embedded in Qt's macOS wait/busy cursor PNGs), then -probes CFGetRetainCount after every test. A persistent drop names the test -that stole references. The buffer retains also *prevent* the SIGTRAP crash. - -Use: PYTHONPATH= pytest -p cg_canary ... (macOS only, no-op elsewhere) - -Under pytest-xdist, worker stdout is redirected to /dev/null by execnet, so -set CG_CANARY_LOG= to also append findings to a file (one shared file -is fine; lines are tagged with the pid). -""" - -import os -import sys - -_armed = {} -_reported = set() - - -def _log(msg): - line = f"[cg-canary] [{os.getpid()}] {msg}" - print(line, flush=True) - path = os.environ.get("CG_CANARY_LOG") - if path: - with open(path, "a") as f: - f.write(line + "\n") - - -if sys.platform == "darwin": - import ctypes - - _cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") - _cf = ctypes.CDLL( - "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation" - ) - _cg.CGColorSpaceCreateWithName.restype = ctypes.c_void_p - _cg.CGColorSpaceCreateWithName.argtypes = [ctypes.c_void_p] - _cg.CGColorSpaceCreateDeviceRGB.restype = ctypes.c_void_p - _cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p - _cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] - _cf.CFDataCreate.restype = ctypes.c_void_p - _cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] - _cf.CFRetain.restype = ctypes.c_void_p - _cf.CFRetain.argtypes = [ctypes.c_void_p] - _cf.CFGetRetainCount.restype = ctypes.c_long - _cf.CFGetRetainCount.argtypes = [ctypes.c_void_p] - - _IMMORTAL = (1 << 32) - 2 - _BUFFER = 500 - - def _arm_one(label, ptr): - if not ptr or label in _armed: - return - rc = _cf.CFGetRetainCount(ptr) - if rc >= _IMMORTAL: - _armed[label] = (ptr, None) # immortal: track for info only - _log(f"{label}: immortal, not tracked") - return - for _ in range(_BUFFER): - _cf.CFRetain(ptr) - base = _cf.CFGetRetainCount(ptr) - _armed[label] = (ptr, base) - _log(f"{label}: armed ptr={ptr:#x} rc={base}") - - def _try_arm(): - if "srgb-named" not in _armed: - k = ctypes.c_void_p.in_dll(_cg, "kCGColorSpaceSRGB") - _arm_one("srgb-named", _cg.CGColorSpaceCreateWithName(k)) - if "device-rgb" not in _armed: - _arm_one("device-rgb", _cg.CGColorSpaceCreateDeviceRGB()) - if "vp2030-icc" not in _armed: - # needs the cocoa plugin resources -> only after QApplication - try: - from qtpy.QtGui import QImage - from qtpy.QtWidgets import QApplication - - if QApplication.instance() is None: - return - img = QImage(":/qt-project.org/mac/cursors/images/spincursor.png") - if img.isNull() or not img.colorSpace().isValid(): - return - icc = bytes(img.colorSpace().iccProfile().data()) - d = _cf.CFDataCreate(None, icc, len(icc)) - _arm_one("vp2030-icc", _cg.CGColorSpaceCreateWithICCData(d)) - except Exception: - pass - - def pytest_runtest_teardown(item): - _try_arm() - for label, (ptr, base) in _armed.items(): - if base is None: - continue - rc = _cf.CFGetRetainCount(ptr) - if rc != base and (label, rc) not in _reported: - _reported.add((label, rc)) - _log(f"*** {label} rc {base} -> {rc} after {item.nodeid} ***") - _armed[label] = (ptr, rc) # new baseline; report each change - - def pytest_terminal_summary(terminalreporter): - for label, (ptr, base) in _armed.items(): - state = "immortal" if base is None else f"final rc={base}" - terminalreporter.write_line(f"[cg-canary] {label}: {state}") diff --git a/tools/cg_icc_probe.py b/tools/cg_icc_probe.py new file mode 100644 index 00000000000..59a0802e3dd --- /dev/null +++ b/tools/cg_icc_probe.py @@ -0,0 +1,92 @@ +"""Probe CoreGraphics' ICC-colorspace cache for the weak-reference bug. + +On macOS 26.4, CGColorSpaceCreateWithICCData appears to cache entries without +holding an owning reference: create -> release -> create returns a dangling +pointer and crashes (SIGTRAP) when used. This is the suspected root cause of +the Qt wait-cursor crash (QImage::toCGImage) seen on CI. + +Safe discriminator (parent process): retain count right after a single create +should be 2 (object + cache's strong ref, macOS >= 26.5) vs 1 (weak cache, +macOS 26.4). + +Crash probe (child process): create -> release -> create -> CGColorSpaceGetType. +Exit 0 = survived; killed by SIGTRAP/SIGSEGV = bug reproduced. +""" + +import ctypes +import pathlib +import platform +import subprocess +import sys + +CHILD = """ +import ctypes, pathlib +cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") +cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") +cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p +cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] +cg.CGColorSpaceGetType.restype = ctypes.c_int +cg.CGColorSpaceGetType.argtypes = [ctypes.c_void_p] +cf.CFDataCreate.restype = ctypes.c_void_p +cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] +cf.CFRelease.restype = None +cf.CFRelease.argtypes = [ctypes.c_void_p] +icc = bytearray( + pathlib.Path("/System/Library/ColorSync/Profiles/sRGB Profile.icc").read_bytes() +) +icc[-4:] = b"MNE!" # arbitrary profile -> generic cache path (like Qt's VP2030) +icc = bytes(icc) +d = cf.CFDataCreate(None, icc, len(icc)) +p1 = cg.CGColorSpaceCreateWithICCData(d) +cf.CFRelease(p1) # refcount -> 0; freed if the cache reference is weak +p2 = cg.CGColorSpaceCreateWithICCData(d) +print(f"child: p1={p1:#x} p2={p2:#x} same={p1 == p2}", flush=True) +print(f"child: type={cg.CGColorSpaceGetType(p2)}", flush=True) # may SIGTRAP +print("child: survived", flush=True) +""" + + +def main(): + """Run the probe.""" + print(f"macOS {platform.mac_ver()[0]}", flush=True) + cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") + cf = ctypes.CDLL( + "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation" + ) + cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p + cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] + cf.CFDataCreate.restype = ctypes.c_void_p + cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] + cf.CFGetRetainCount.restype = ctypes.c_long + cf.CFGetRetainCount.argtypes = [ctypes.c_void_p] + + system = pathlib.Path( + "/System/Library/ColorSync/Profiles/sRGB Profile.icc" + ).read_bytes() + arbitrary = bytearray(system) + arbitrary[-4:] = b"MNE!" + for label, icc in [("system-sRGB", system), ("arbitrary", bytes(arbitrary))]: + d = cf.CFDataCreate(None, icc, len(icc)) + p = cg.CGColorSpaceCreateWithICCData(d) + rc = cf.CFGetRetainCount(p) + cache = "strong/immortal (fixed)" if rc >= 2 else "WEAK (vulnerable)" + print( + f"parent: {label}: rc after single create = {rc} -> {cache}", + flush=True, + ) + + res = subprocess.run([sys.executable, "-c", CHILD], capture_output=True, text=True) + print(res.stdout, end="", flush=True) + if res.returncode == 0: + print("crash probe: SURVIVED (no bug on this OS)", flush=True) + else: + print( + f"crash probe: DIED rc={res.returncode} " + f"(negative = signal; -5 = SIGTRAP) -> BUG REPRODUCED", + flush=True, + ) + print(res.stderr[-500:], flush=True) + + +if __name__ == "__main__": + main() From b25251ebe7d34749fc728e3ac36f394ba8a08997 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sat, 25 Jul 2026 09:36:00 -0400 Subject: [PATCH 7/9] Try again [skip circle] [skip azp] --- tools/cg_icc_probe.py | 72 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/tools/cg_icc_probe.py b/tools/cg_icc_probe.py index 59a0802e3dd..9cc3d3c6f66 100644 --- a/tools/cg_icc_probe.py +++ b/tools/cg_icc_probe.py @@ -11,8 +11,19 @@ Crash probe (child process): create -> release -> create -> CGColorSpaceGetType. Exit 0 = survived; killed by SIGTRAP/SIGSEGV = bug reproduced. + +Stress probe (second child): several threads concurrently create/use/release +a colorspace from the same ICC data, so the cache entry's refcount repeatedly +bounces off zero while other threads look it up. If the cache lookup does not +atomically retain against the final release, a thread eventually gets a freed +entry -> SIGTRAP/SIGSEGV. This mimics Qt cursor conversion on the main thread +racing AppKit background-thread releases of the previous cursor's CGImage. """ +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + import ctypes import pathlib import platform @@ -45,6 +56,44 @@ print("child: survived", flush=True) """ +STRESS_CHILD = """ +import ctypes, pathlib, threading, time +cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") +cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") +cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p +cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] +cg.CGColorSpaceGetType.restype = ctypes.c_int +cg.CGColorSpaceGetType.argtypes = [ctypes.c_void_p] +cf.CFDataCreate.restype = ctypes.c_void_p +cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] +cf.CFRelease.restype = None +cf.CFRelease.argtypes = [ctypes.c_void_p] +icc = bytearray( + pathlib.Path("/System/Library/ColorSync/Profiles/sRGB Profile.icc").read_bytes() +) +icc[-4:] = b"MNE!" +icc = bytes(icc) +DURATION = 5.0 +stop = time.monotonic() + DURATION +count = [0] * 4 + +def churn(idx): + d = cf.CFDataCreate(None, icc, len(icc)) + while time.monotonic() < stop: + p = cg.CGColorSpaceCreateWithICCData(d) # cache hit + retain + cg.CGColorSpaceGetType(p) # may SIGTRAP if handed a freed entry + cf.CFRelease(p) # may drop entry rc to 0 while others look up + count[idx] += 1 + +threads = [threading.Thread(target=churn, args=(i,)) for i in range(1, 4)] +for t in threads: + t.start() +churn(0) +for t in threads: + t.join() +print(f"stress child: survived {sum(count)} cycles across 4 threads", flush=True) +""" + def main(): """Run the probe.""" @@ -75,17 +124,20 @@ def main(): flush=True, ) - res = subprocess.run([sys.executable, "-c", CHILD], capture_output=True, text=True) - print(res.stdout, end="", flush=True) - if res.returncode == 0: - print("crash probe: SURVIVED (no bug on this OS)", flush=True) - else: - print( - f"crash probe: DIED rc={res.returncode} " - f"(negative = signal; -5 = SIGTRAP) -> BUG REPRODUCED", - flush=True, + for label, code in [("crash probe", CHILD), ("stress probe", STRESS_CHILD)]: + res = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True ) - print(res.stderr[-500:], flush=True) + print(res.stdout, end="", flush=True) + if res.returncode == 0: + print(f"{label}: SURVIVED (no bug detected)", flush=True) + else: + print( + f"{label}: DIED rc={res.returncode} " + f"(negative = signal; -5 = SIGTRAP) -> BUG REPRODUCED", + flush=True, + ) + print(res.stderr[-500:], flush=True) if __name__ == "__main__": From d3a519451b1c81d2fb0ae362fac74ac4657c6f0c Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sat, 25 Jul 2026 09:58:18 -0400 Subject: [PATCH 8/9] Try again [skip circle] [skip azp] --- tools/cg_icc_probe.py | 108 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/tools/cg_icc_probe.py b/tools/cg_icc_probe.py index 9cc3d3c6f66..71242957d5d 100644 --- a/tools/cg_icc_probe.py +++ b/tools/cg_icc_probe.py @@ -94,6 +94,107 @@ def churn(idx): print(f"stress child: survived {sum(count)} cycles across 4 threads", flush=True) """ +EVICT_CHILD = """ +import ctypes, pathlib +cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") +cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") +cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p +cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] +cg.CGColorSpaceGetType.restype = ctypes.c_int +cg.CGColorSpaceGetType.argtypes = [ctypes.c_void_p] +cf.CFDataCreate.restype = ctypes.c_void_p +cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] +cf.CFRelease.restype = None +cf.CFRelease.argtypes = [ctypes.c_void_p] +icc = bytearray( + pathlib.Path("/System/Library/ColorSync/Profiles/sRGB Profile.icc").read_bytes() +) +icc[-4:] = b"MNE!" +d_target = cf.CFDataCreate(None, bytes(icc), len(icc)) +p1 = cg.CGColorSpaceCreateWithICCData(d_target) +cg.CGColorSpaceGetType(p1) +cf.CFRelease(p1) # cache-only (rc=1) -> eviction candidate +# pressure the cache with many distinct profiles +for i in range(20000): + v = bytearray(icc) + v[-8:-4] = i.to_bytes(4, "big") + dv = cf.CFDataCreate(None, bytes(v), len(v)) + q = cg.CGColorSpaceCreateWithICCData(dv) + if q: + cf.CFRelease(q) + cf.CFRelease(dv) +p2 = cg.CGColorSpaceCreateWithICCData(d_target) +print(f"evict child: p1={p1:#x} p2={p2:#x} same={p1 == p2}", flush=True) +print(f"evict child: type={cg.CGColorSpaceGetType(p2)}", flush=True) # crashable +print("evict child: survived", flush=True) +""" + +PRESSURE_CHILD = """ +import ctypes, pathlib, subprocess, threading, time +cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") +cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") +cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p +cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] +cg.CGColorSpaceGetType.restype = ctypes.c_int +cg.CGColorSpaceGetType.argtypes = [ctypes.c_void_p] +cf.CFDataCreate.restype = ctypes.c_void_p +cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] +cf.CFRelease.restype = None +cf.CFRelease.argtypes = [ctypes.c_void_p] +icc = bytearray( + pathlib.Path("/System/Library/ColorSync/Profiles/sRGB Profile.icc").read_bytes() +) +icc[-4:] = b"MNE!" +icc = bytes(icc) +DURATION = 12.0 +stop = time.monotonic() + DURATION +fired = [0] + +def pressure(): + # simulate memory-pressure notifications (CG caches flush on these) + while time.monotonic() < stop: + for level in ("warn", "critical"): + r = subprocess.run( + ["sudo", "-n", "memory_pressure", "-S", "-l", level], + capture_output=True, + ) + if r.returncode == 0: + fired[0] += 1 + time.sleep(1.0) + +def churn_distinct(): + i = 0 + while time.monotonic() < stop: + v = bytearray(icc) + v[-8:-4] = i.to_bytes(4, "big") + dv = cf.CFDataCreate(None, bytes(v), len(v)) + q = cg.CGColorSpaceCreateWithICCData(dv) + if q: + cf.CFRelease(q) + cf.CFRelease(dv) + i += 1 + +threads = [ + threading.Thread(target=pressure), + threading.Thread(target=churn_distinct), +] +for t in threads: + t.start() +d = cf.CFDataCreate(None, icc, len(icc)) +n = 0 +while time.monotonic() < stop: + p = cg.CGColorSpaceCreateWithICCData(d) # may return flushed/stale entry + cg.CGColorSpaceGetType(p) # crashable + cf.CFRelease(p) + n += 1 +for t in threads: + t.join() +print( + f"pressure child: survived {n} lookups, {fired[0]} pressure events fired", + flush=True, +) +""" + def main(): """Run the probe.""" @@ -124,7 +225,12 @@ def main(): flush=True, ) - for label, code in [("crash probe", CHILD), ("stress probe", STRESS_CHILD)]: + for label, code in [ + ("crash probe", CHILD), + ("stress probe", STRESS_CHILD), + ("evict probe", EVICT_CHILD), + ("pressure probe", PRESSURE_CHILD), + ]: res = subprocess.run( [sys.executable, "-c", code], capture_output=True, text=True ) From 6d2da5f9b9ece739a9f87cfab450a7dcef986273 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sat, 25 Jul 2026 11:01:44 -0400 Subject: [PATCH 9/9] FIX: Restore --- .github/workflows/tests.yml | 46 +++---- tools/cg_icc_probe.py | 250 ----------------------------------- tools/github_actions_test.sh | 2 +- 3 files changed, 22 insertions(+), 276 deletions(-) delete mode 100644 tools/cg_icc_probe.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0f601db1ea8..11c903dcc95 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,9 +47,30 @@ jobs: fail-fast: false matrix: include: + - os: ubuntu-latest + python: '3.13' + kind: pip + - os: ubuntu-latest + python: '3.14' + kind: pip-pre + - os: ubuntu-latest + python: '3.13' + kind: conda - os: macos-latest # arm64 python: '3.14' kind: pip + - os: macos-15-intel # Intel + python: '3.13' + kind: pip + - os: ubuntu-latest + python: '3.12' + kind: minimal + - os: ubuntu-22.04 + python: '3.10' + kind: old + - os: ubuntu-latest + python: '3.14t' # free-threaded + kind: pip-ft steps: - uses: actions/checkout@v7.0.0 with: @@ -109,7 +130,6 @@ jobs: - run: bash ./tools/github_actions_verify_python.sh "${{ matrix.python }}" - run: bash ./tools/github_actions_dependencies.sh timeout-minutes: 10 - - run: pip install "pytest-xdist @ git+https://github.com/larsoner/pytest-xdist.git@lock" - run: python ./tools/github_actions_check_old_env.py if: matrix.kind == 'old' # Minimal commands on Linux (macOS stalls) @@ -121,10 +141,6 @@ jobs: - run: bash ./tools/get_minimal_commands.sh if: startswith(matrix.os, 'ubuntu') && matrix.kind != 'minimal' && matrix.kind != 'old' && matrix.kind != 'pip-ft' - run: bash ./tools/github_actions_infos.sh - # Probe CoreGraphics ICC colorspace cache lifetime (temporary diagnostic, - # expected: WEAK + child SIGTRAP on macOS 26.4, fixed on >= 26.5) - - run: python tools/cg_icc_probe.py - if: startsWith(matrix.os, 'macos') # Check Qt - run: bash ./tools/check_qt_import.sh $MNE_QT_BACKEND if: env.MNE_QT_BACKEND != '' @@ -138,26 +154,6 @@ jobs: path: ~/mne_data - run: bash ./tools/github_actions_download.sh - run: bash ./tools/github_actions_test.sh # for some reason on macOS we need to run "bash X" in order for a failed test run to show up - # Diagnose silent pytest-xdist worker deaths on macOS: a JetsamEvent-*.ips - # report means the OS killed the worker under memory pressure (SIGKILL), - # while a python-*.ips crash report means a real native crash (with stack) - - name: Collect macOS crash diagnostics - if: (success() || failure()) && startsWith(matrix.os, 'macos') - run: | - set -x - mkdir -p crash_diagnostics/user_reports crash_diagnostics/system_reports - find ~/Library/Logs/DiagnosticReports -type f -mmin -90 -exec cp {} crash_diagnostics/user_reports/ \; 2>/dev/null || true - sudo find /Library/Logs/DiagnosticReports -type f -mmin -90 -exec cp {} crash_diagnostics/system_reports/ \; 2>/dev/null || true - sudo chown -R "$USER" crash_diagnostics || true - { sysctl hw.memsize vm.swapusage; vm_stat; } > crash_diagnostics/memory_snapshot.txt || true - log show --last 45m --style syslog --predicate 'eventMessage CONTAINS[c] "memorystatus" OR eventMessage CONTAINS[c] "jetsam"' > crash_diagnostics/memorystatus.log || true - ls -alR crash_diagnostics || true - - uses: actions/upload-artifact@v7 - if: (success() || failure()) && startsWith(matrix.os, 'macos') - with: - name: crash-diagnostics-${{ matrix.os }}-${{ matrix.kind }}-${{ matrix.python }} - path: crash_diagnostics - if-no-files-found: ignore - uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/tools/cg_icc_probe.py b/tools/cg_icc_probe.py deleted file mode 100644 index 71242957d5d..00000000000 --- a/tools/cg_icc_probe.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Probe CoreGraphics' ICC-colorspace cache for the weak-reference bug. - -On macOS 26.4, CGColorSpaceCreateWithICCData appears to cache entries without -holding an owning reference: create -> release -> create returns a dangling -pointer and crashes (SIGTRAP) when used. This is the suspected root cause of -the Qt wait-cursor crash (QImage::toCGImage) seen on CI. - -Safe discriminator (parent process): retain count right after a single create -should be 2 (object + cache's strong ref, macOS >= 26.5) vs 1 (weak cache, -macOS 26.4). - -Crash probe (child process): create -> release -> create -> CGColorSpaceGetType. -Exit 0 = survived; killed by SIGTRAP/SIGSEGV = bug reproduced. - -Stress probe (second child): several threads concurrently create/use/release -a colorspace from the same ICC data, so the cache entry's refcount repeatedly -bounces off zero while other threads look it up. If the cache lookup does not -atomically retain against the final release, a thread eventually gets a freed -entry -> SIGTRAP/SIGSEGV. This mimics Qt cursor conversion on the main thread -racing AppKit background-thread releases of the previous cursor's CGImage. -""" - -# Authors: The MNE-Python contributors. -# License: BSD-3-Clause -# Copyright the MNE-Python contributors. - -import ctypes -import pathlib -import platform -import subprocess -import sys - -CHILD = """ -import ctypes, pathlib -cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") -cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") -cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p -cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] -cg.CGColorSpaceGetType.restype = ctypes.c_int -cg.CGColorSpaceGetType.argtypes = [ctypes.c_void_p] -cf.CFDataCreate.restype = ctypes.c_void_p -cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] -cf.CFRelease.restype = None -cf.CFRelease.argtypes = [ctypes.c_void_p] -icc = bytearray( - pathlib.Path("/System/Library/ColorSync/Profiles/sRGB Profile.icc").read_bytes() -) -icc[-4:] = b"MNE!" # arbitrary profile -> generic cache path (like Qt's VP2030) -icc = bytes(icc) -d = cf.CFDataCreate(None, icc, len(icc)) -p1 = cg.CGColorSpaceCreateWithICCData(d) -cf.CFRelease(p1) # refcount -> 0; freed if the cache reference is weak -p2 = cg.CGColorSpaceCreateWithICCData(d) -print(f"child: p1={p1:#x} p2={p2:#x} same={p1 == p2}", flush=True) -print(f"child: type={cg.CGColorSpaceGetType(p2)}", flush=True) # may SIGTRAP -print("child: survived", flush=True) -""" - -STRESS_CHILD = """ -import ctypes, pathlib, threading, time -cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") -cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") -cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p -cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] -cg.CGColorSpaceGetType.restype = ctypes.c_int -cg.CGColorSpaceGetType.argtypes = [ctypes.c_void_p] -cf.CFDataCreate.restype = ctypes.c_void_p -cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] -cf.CFRelease.restype = None -cf.CFRelease.argtypes = [ctypes.c_void_p] -icc = bytearray( - pathlib.Path("/System/Library/ColorSync/Profiles/sRGB Profile.icc").read_bytes() -) -icc[-4:] = b"MNE!" -icc = bytes(icc) -DURATION = 5.0 -stop = time.monotonic() + DURATION -count = [0] * 4 - -def churn(idx): - d = cf.CFDataCreate(None, icc, len(icc)) - while time.monotonic() < stop: - p = cg.CGColorSpaceCreateWithICCData(d) # cache hit + retain - cg.CGColorSpaceGetType(p) # may SIGTRAP if handed a freed entry - cf.CFRelease(p) # may drop entry rc to 0 while others look up - count[idx] += 1 - -threads = [threading.Thread(target=churn, args=(i,)) for i in range(1, 4)] -for t in threads: - t.start() -churn(0) -for t in threads: - t.join() -print(f"stress child: survived {sum(count)} cycles across 4 threads", flush=True) -""" - -EVICT_CHILD = """ -import ctypes, pathlib -cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") -cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") -cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p -cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] -cg.CGColorSpaceGetType.restype = ctypes.c_int -cg.CGColorSpaceGetType.argtypes = [ctypes.c_void_p] -cf.CFDataCreate.restype = ctypes.c_void_p -cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] -cf.CFRelease.restype = None -cf.CFRelease.argtypes = [ctypes.c_void_p] -icc = bytearray( - pathlib.Path("/System/Library/ColorSync/Profiles/sRGB Profile.icc").read_bytes() -) -icc[-4:] = b"MNE!" -d_target = cf.CFDataCreate(None, bytes(icc), len(icc)) -p1 = cg.CGColorSpaceCreateWithICCData(d_target) -cg.CGColorSpaceGetType(p1) -cf.CFRelease(p1) # cache-only (rc=1) -> eviction candidate -# pressure the cache with many distinct profiles -for i in range(20000): - v = bytearray(icc) - v[-8:-4] = i.to_bytes(4, "big") - dv = cf.CFDataCreate(None, bytes(v), len(v)) - q = cg.CGColorSpaceCreateWithICCData(dv) - if q: - cf.CFRelease(q) - cf.CFRelease(dv) -p2 = cg.CGColorSpaceCreateWithICCData(d_target) -print(f"evict child: p1={p1:#x} p2={p2:#x} same={p1 == p2}", flush=True) -print(f"evict child: type={cg.CGColorSpaceGetType(p2)}", flush=True) # crashable -print("evict child: survived", flush=True) -""" - -PRESSURE_CHILD = """ -import ctypes, pathlib, subprocess, threading, time -cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") -cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") -cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p -cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] -cg.CGColorSpaceGetType.restype = ctypes.c_int -cg.CGColorSpaceGetType.argtypes = [ctypes.c_void_p] -cf.CFDataCreate.restype = ctypes.c_void_p -cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] -cf.CFRelease.restype = None -cf.CFRelease.argtypes = [ctypes.c_void_p] -icc = bytearray( - pathlib.Path("/System/Library/ColorSync/Profiles/sRGB Profile.icc").read_bytes() -) -icc[-4:] = b"MNE!" -icc = bytes(icc) -DURATION = 12.0 -stop = time.monotonic() + DURATION -fired = [0] - -def pressure(): - # simulate memory-pressure notifications (CG caches flush on these) - while time.monotonic() < stop: - for level in ("warn", "critical"): - r = subprocess.run( - ["sudo", "-n", "memory_pressure", "-S", "-l", level], - capture_output=True, - ) - if r.returncode == 0: - fired[0] += 1 - time.sleep(1.0) - -def churn_distinct(): - i = 0 - while time.monotonic() < stop: - v = bytearray(icc) - v[-8:-4] = i.to_bytes(4, "big") - dv = cf.CFDataCreate(None, bytes(v), len(v)) - q = cg.CGColorSpaceCreateWithICCData(dv) - if q: - cf.CFRelease(q) - cf.CFRelease(dv) - i += 1 - -threads = [ - threading.Thread(target=pressure), - threading.Thread(target=churn_distinct), -] -for t in threads: - t.start() -d = cf.CFDataCreate(None, icc, len(icc)) -n = 0 -while time.monotonic() < stop: - p = cg.CGColorSpaceCreateWithICCData(d) # may return flushed/stale entry - cg.CGColorSpaceGetType(p) # crashable - cf.CFRelease(p) - n += 1 -for t in threads: - t.join() -print( - f"pressure child: survived {n} lookups, {fired[0]} pressure events fired", - flush=True, -) -""" - - -def main(): - """Run the probe.""" - print(f"macOS {platform.mac_ver()[0]}", flush=True) - cg = ctypes.CDLL("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics") - cf = ctypes.CDLL( - "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation" - ) - cg.CGColorSpaceCreateWithICCData.restype = ctypes.c_void_p - cg.CGColorSpaceCreateWithICCData.argtypes = [ctypes.c_void_p] - cf.CFDataCreate.restype = ctypes.c_void_p - cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] - cf.CFGetRetainCount.restype = ctypes.c_long - cf.CFGetRetainCount.argtypes = [ctypes.c_void_p] - - system = pathlib.Path( - "/System/Library/ColorSync/Profiles/sRGB Profile.icc" - ).read_bytes() - arbitrary = bytearray(system) - arbitrary[-4:] = b"MNE!" - for label, icc in [("system-sRGB", system), ("arbitrary", bytes(arbitrary))]: - d = cf.CFDataCreate(None, icc, len(icc)) - p = cg.CGColorSpaceCreateWithICCData(d) - rc = cf.CFGetRetainCount(p) - cache = "strong/immortal (fixed)" if rc >= 2 else "WEAK (vulnerable)" - print( - f"parent: {label}: rc after single create = {rc} -> {cache}", - flush=True, - ) - - for label, code in [ - ("crash probe", CHILD), - ("stress probe", STRESS_CHILD), - ("evict probe", EVICT_CHILD), - ("pressure probe", PRESSURE_CHILD), - ]: - res = subprocess.run( - [sys.executable, "-c", code], capture_output=True, text=True - ) - print(res.stdout, end="", flush=True) - if res.returncode == 0: - print(f"{label}: SURVIVED (no bug detected)", flush=True) - else: - print( - f"{label}: DIED rc={res.returncode} " - f"(negative = signal; -5 = SIGTRAP) -> BUG REPRODUCED", - flush=True, - ) - print(res.stderr[-500:], flush=True) - - -if __name__ == "__main__": - main() diff --git a/tools/github_actions_test.sh b/tools/github_actions_test.sh index 7296f12f012..8a84e2b7ef8 100755 --- a/tools/github_actions_test.sh +++ b/tools/github_actions_test.sh @@ -45,4 +45,4 @@ if [[ ! -z "$CONDA_ENV" ]] && [[ "${CI_OS_NAME}" != "windows"* ]] && [[ "${MNE_C fi # $COV_ARGS is set in github_actions_env_vars.sh (coverage only on Python >= 3.14) -pytest -m "${CONDITION}" -n "$PYTEST_XDIST_N" --dist loadscope --timeout=120 --timeout-method=thread -o faulthandler_timeout=110 ${COV_ARGS} ${COV_APPEND} --color=yes --continue-on-collection-errors --junit-xml="$THIS_JUNIT" -vv ${USE_DIRS} +pytest -m "${CONDITION}" -n "$PYTEST_XDIST_N" --dist loadscope --timeout=120 --timeout-method=thread -o faulthandler_timeout=110 ${COV_ARGS} --color=yes --continue-on-collection-errors --junit-xml="$JUNIT_PATH" -vv ${USE_DIRS}