From 2a132cf22977b3a8023afd230cd73670f8bcfa53 Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Wed, 22 Jul 2026 23:02:32 -0400 Subject: [PATCH 1/8] ENH: scaffold xeus-python kernel for the browser docs --- doc/changes/dev/14093.other.rst | 1 + doc/jupyterlite_environment.yml | 41 +++++++++++ doc/jupyterlite_xeus_setup_cell.py | 112 +++++++++++++++++++++++++++++ pyproject.toml | 1 + 4 files changed, 155 insertions(+) create mode 100644 doc/changes/dev/14093.other.rst create mode 100644 doc/jupyterlite_environment.yml create mode 100644 doc/jupyterlite_xeus_setup_cell.py diff --git a/doc/changes/dev/14093.other.rst b/doc/changes/dev/14093.other.rst new file mode 100644 index 00000000000..70b77b99016 --- /dev/null +++ b/doc/changes/dev/14093.other.rst @@ -0,0 +1 @@ +Added a xeus-python kernel environment and setup cell for the JupyterLite documentation, by `Natneal B`_. diff --git a/doc/jupyterlite_environment.yml b/doc/jupyterlite_environment.yml new file mode 100644 index 00000000000..1c1ef0b8574 --- /dev/null +++ b/doc/jupyterlite_environment.yml @@ -0,0 +1,41 @@ +# Environment for the xeus-python JupyterLite kernel used by the browser docs. +# +# Unlike the Pyodide kernel (which installs MNE at runtime via piplite), the +# xeus kernel PRE-INSTALLS everything at build time from these channels: +# - emscripten-forge : WASM-compiled binaries (numpy, scipy, vtk, ...) +# - conda-forge : pure-Python (noarch) packages (nibabel, pooch, mne, ...) +# MNE itself is installed from the local checkout via the pip section (pure +# Python, so no dependency resolution is needed -- its deps are the conda +# packages listed below). Verified to resolve for emscripten-wasm32 2026-07-22. +name: xeus-python +channels: + - https://repo.prefix.dev/emscripten-forge-4x + - https://repo.prefix.dev/conda-forge +dependencies: + - xeus-python + # compiled scientific stack (from emscripten-forge) + - numpy + - scipy + - matplotlib + - pandas + - scikit-learn + - h5py + - statsmodels + # pure-Python deps (from conda-forge) + - nibabel + - pooch + - tqdm + - decorator + - jinja2 + - lazy_loader + - packaging + - pymatreader + - h5io + - python-picard + - patsy + - threadpoolctl + - joblib + - pillow + # the development MNE from this checkout (pure Python, installed w/o deps) + - pip: + - .. diff --git a/doc/jupyterlite_xeus_setup_cell.py b/doc/jupyterlite_xeus_setup_cell.py new file mode 100644 index 00000000000..a78771146c7 --- /dev/null +++ b/doc/jupyterlite_xeus_setup_cell.py @@ -0,0 +1,112 @@ +"""First-cut JupyterLite setup cell for the xeus-python kernel. + +WIP / CI-PENDING -- this is the xeus counterpart of the Pyodide +``first_notebook_cell`` in ``conf.py``. It cannot be fully validated locally +(needs the real ``build_docs`` CI run); the assumptions marked ``CI-VERIFY`` +below are the ones to confirm/iterate on first. + +Why it is so much smaller than the Pyodide cell +------------------------------------------------ +xeus-python is real CPython compiled to WebAssembly (NOT Pyodide), and the +packages are PRE-INSTALLED at build time from ``jupyterlite_environment.yml``. +So compared to the Pyodide setup cell this drops: + + * ``piplite.install(...)`` -> MNE is already installed + * the ``lzma`` / ``multiprocessing`` mocks -> real CPython stdlib is present + * the ``pyodide.http`` / ``requests`` patches and the ``js`` XHR fetch shims + -> no Pyodide runtime to patch + +What remains is just: point MNE's dataset loaders at the bundled data, block +accidental OSF downloads, and (still to port) the pyvista-js 3D shim. + +Data model (CI-VERIFY) +---------------------- +With ``XeusAddon.mount_jupyterlite_content=True`` the served JupyterLite content +is mounted into the kernel filesystem at ``/files``. So the curated MNE data +must be made available as JupyterLite content (a conf.py change: add the curated +``mne_data`` tree to ``jupyterlite_contents`` instead of only ``html_extra_path``). +The cell below resolves the data root by probing the likely mount locations so it +is robust to which mechanism ends up being used. Because every bundled file is +then present in the FS, NO runtime fetch is needed -- the loaders just return the +folder (much simpler than the Pyodide lazy-fetch shims). + +conf.py wiring this expects (next step, not yet applied so the pyodide build +stays intact on this branch): + + from jupyterlite_xeus_setup_cell import XEUS_FIRST_NOTEBOOK_CELL + jupyterlite_build_command_options = { + "XeusAddon.environment_file": "jupyterlite_environment.yml", + "XeusAddon.mount_jupyterlite_content": True, + } + sphinx_gallery_conf["first_notebook_cell"] = XEUS_FIRST_NOTEBOOK_CELL +""" + +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + +# The setup cell source, as a string (mirrors how conf.py stores the Pyodide one). +XEUS_FIRST_NOTEBOOK_CELL = r"""# 💡 Auto-added setup cell (xeus-python kernel). +# MNE and its dependencies are pre-installed in this kernel; this cell only +# points the dataset loaders at the pre-bundled data. +import os +from pathlib import Path as _Path +import mne + +# --- locate the bundled MNE data (mounted from the served jupyterlite content) --- +# CI-VERIFY: confirm the actual mount path (/files with mount_jupyterlite_content). +_candidates = ["/files/mne_data", "/drive/mne_data", os.path.expanduser("~/mne_data")] +mne_data_path = next((p for p in _candidates if os.path.isdir(p)), "/files/mne_data") +os.makedirs(mne_data_path, exist_ok=True) +os.environ["MNE_DATA"] = mne_data_path + +# Pre-create a valid empty config so MNE never hits a corrupt read. +_cfg = mne.get_config_path() +os.makedirs(os.path.dirname(_cfg), exist_ok=True) +if not os.path.exists(_cfg): + with open(_cfg, "w") as _f: + _f.write("{}") +mne.set_config("MNE_DATA", mne_data_path) +for _ds in ["SAMPLE", "TESTING", "SSVEP", "EEGBCI", "KILOWORD", "ERP_CORE", "MTRF"]: + mne.set_config(f"MNE_DATASETS_{_ds}_PATH", mne_data_path) + +# --- point dataset loaders at the pre-bundled folders (no OSF download) --- +# Everything is already on the filesystem, so these just return the folder -- +# no lazy fetching needed (unlike the Pyodide build). +def _lite_folder(_name): + def _data_path(*_a, **_kw): + return _Path(mne_data_path) / _name + return _data_path + +mne.datasets.sample.data_path = _lite_folder("MNE-sample-data") +mne.datasets.kiloword.data_path = _lite_folder("MNE-kiloword-data") +mne.datasets.erp_core.data_path = _lite_folder("MNE-ERP-CORE-data") +mne.datasets.mtrf.data_path = _lite_folder("mTRF_1.5") + +def _lite_eegbci_load_data(subject, runs, *_a, **_kw): + _runs = [runs] if isinstance(runs, (int, float)) else list(runs) + _subs = list(subject) if isinstance(subject, (list, tuple)) else [subject] + _base = _Path(mne_data_path) / "MNE-eegbci-data" / "files" / "eegmmidb" / "1.0.0" + return [ + _base / f"S{int(s):03d}" / f"S{int(s):03d}R{int(r):02d}.edf" + for s in _subs + for r in _runs + ] +mne.datasets.eegbci.load_data = _lite_eegbci_load_data + +# Block accidental OSF downloads (data is pre-bundled or simply unavailable here). +import pooch +_orig_pooch_fetch = pooch.Pooch.fetch +def _lite_pooch_fetch(self, fname, processor=None, downloader=None): + if "osf.io" in self.get_url(fname): + raise RuntimeError( + f"Cannot download {fname!r} in JupyterLite: open this notebook from " + "mne.tools where the data is pre-bundled, or run it locally." + ) + return _orig_pooch_fetch(self, fname, processor=processor, downloader=downloader) +pooch.Pooch.fetch = _lite_pooch_fetch + +# TODO(CI-VERIFY): port the pyvista-js SourceEstimate.plot 3D shim from the +# Pyodide conf.py cell -- the approach (JS-native pyvista-js) is kernel-agnostic +# per the plan, only the import/interop path needs confirming under xeus. +""" diff --git a/pyproject.toml b/pyproject.toml index 22af8942ad5..57ea6929885 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ doc = [ "graphviz", "intersphinx_registry >= 0.2405.27", "ipython != 8.7.0", # also in "full-no-qt" and "test" + "jupyterlite-xeus", "memory_profiler >= 0.16", "mne-bids", "mne-connectivity", From f64f433abf990629d7219a9370ee1a66920d1bf4 Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Wed, 22 Jul 2026 23:20:56 -0400 Subject: [PATCH 2/8] ENH: wire the xeus-python kernel into the JupyterLite docs build --- .circleci/config.yml | 14 ++ .github/workflows/jupyterlite.yml | 52 +++++ .gitignore | 2 + doc/api/datasets.rst | 1 + doc/changes/dev/14093.other.rst | 2 +- doc/conf.py | 219 +++++++++++++++++- doc/jupyterlite_contents/.gitkeep | 0 .../jupyterlite_xeus_setup_cell.py | 0 mne/datasets/__init__.pyi | 2 + mne/datasets/config.py | 10 + mne/datasets/lite_data/__init__.py | 7 + mne/datasets/lite_data/lite_data.py | 43 ++++ mne/datasets/tests/test_datasets.py | 15 ++ mne/parallel.py | 4 + mne/utils/config.py | 5 +- pyproject.toml | 1 + 16 files changed, 369 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/jupyterlite.yml create mode 100644 doc/jupyterlite_contents/.gitkeep rename doc/{ => sphinxext}/jupyterlite_xeus_setup_cell.py (100%) create mode 100644 mne/datasets/lite_data/__init__.py create mode 100644 mne/datasets/lite_data/lite_data.py diff --git a/.circleci/config.yml b/.circleci/config.yml index a8943cc0094..ac3e69b6b42 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -249,6 +249,20 @@ jobs: cp junit-results.xml doc/_build/test-results/test-doc/junit.xml; cp coverage.xml doc/_build/test-results/test-doc/coverage.xml; fi; + # Ensure the data the JupyterLite notebooks need is on disk so conf.py can + # copy the required subset for the build. lite_data fetches only the + # curated files the browser notebooks use (from the MNE-lite-data OSF + # project) instead of the full sample/kiloword/erp_core/mtrf/eegbci + # datasets, which slims the build. The curated files share hashes with the + # full datasets, so on a full build that already downloaded them nothing + # extra is fetched. + - run: + name: Ensure MNE data for JupyterLite + command: | + python -c "import mne; mne.datasets.lite_data.data_path(update_path=True)" + # No wheel-building step here: the xeus kernel installs MNE and its + # dependencies at build time from doc/jupyterlite_environment.yml, so + # there is nothing to hand to the browser kernel at runtime. # Build docs - run: name: make html diff --git a/.github/workflows/jupyterlite.yml b/.github/workflows/jupyterlite.yml new file mode 100644 index 00000000000..0b68eabe10a --- /dev/null +++ b/.github/workflows/jupyterlite.yml @@ -0,0 +1,52 @@ +name: Build JupyterLite + +on: # yamllint disable-line rule:truthy + push: + branches: + - jupyterlite-xeus + +permissions: + contents: read + +jobs: + build: + name: Build JupyterLite Site + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + # jupyterlite-xeus solves the emscripten-wasm32 environment with + # micromamba, so it has to be on PATH before the build runs. + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v3 + with: + micromamba-version: latest + init-shell: bash + + - name: Install JupyterLite and the xeus kernel + run: | + python -m pip install --upgrade pip + pip install jupyterlite-core jupyterlite-xeus jupyter-server + + # The browser kernel (including the MNE from this checkout) is built from + # doc/jupyterlite_environment.yml, so there is no wheel to pass in. + - name: Build JupyterLite Site + run: | + jupyter lite build \ + --contents examples/ \ + --output-dir dist_lite/ \ + --XeusAddon.environment_file=doc/jupyterlite_environment.yml + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: jupyterlite-build + path: dist_lite/ diff --git a/.gitignore b/.gitignore index 1d5165e8924..5dc51296dcf 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,5 @@ venv/ .hypothesis/ .ruff_cache/ .ipynb_checkpoints/ +jupyterlite_contents/auto_tutorials +jupyterlite_contents/mne_data diff --git a/doc/api/datasets.rst b/doc/api/datasets.rst index 87730fbd717..e19bf7f8dfb 100644 --- a/doc/api/datasets.rst +++ b/doc/api/datasets.rst @@ -30,6 +30,7 @@ Datasets hf_sef.data_path kiloword.data_path limo.load_data + lite_data.data_path misc.data_path mtrf.data_path multimodal.data_path diff --git a/doc/changes/dev/14093.other.rst b/doc/changes/dev/14093.other.rst index 70b77b99016..607d422bec9 100644 --- a/doc/changes/dev/14093.other.rst +++ b/doc/changes/dev/14093.other.rst @@ -1 +1 @@ -Added a xeus-python kernel environment and setup cell for the JupyterLite documentation, by `Natneal B`_. +Added a JupyterLite version of the documentation that runs in the browser using the xeus-python kernel, by `Natneal B`_. diff --git a/doc/conf.py b/doc/conf.py index 6f0c4740d30..25207639797 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -11,10 +11,10 @@ import faulthandler import os +import shutil import subprocess import sys from datetime import datetime, timezone -from importlib.metadata import metadata from pathlib import Path import matplotlib @@ -53,6 +53,7 @@ sys.path.append(str(curpath / "sphinxext")) from credit_tools import generate_credit_rst # noqa: E402 +from jupyterlite_xeus_setup_cell import XEUS_FIRST_NOTEBOOK_CELL # noqa: E402 from mne_doc_utils import report_scraper, reset_warnings, sphinx_logger # noqa: E402 # -- Project information ----------------------------------------------------- @@ -116,6 +117,7 @@ "sphinx_copybutton", "sphinx_design", "sphinx_gallery.gen_gallery", + "jupyterlite_sphinx", "sphinxcontrib.bibtex", "sphinxcontrib.youtube", "sphinxcontrib.towncrier.ext", @@ -139,7 +141,14 @@ # This pattern also affects html_static_path and html_extra_path. # NB: changes here should also be made to the linkcheck target in the Makefile -exclude_patterns = ["_includes", "changes/dev"] +exclude_patterns = [ + "_includes", + "changes/dev", + "jupyterlite_contents", + "lite_extra", + "pypi", + "corrupt_*", +] # The suffix of source filenames. source_suffix = ".rst" @@ -481,7 +490,139 @@ compress_images = () sphinx_gallery_parallel = int(os.getenv("MNE_DOC_BUILD_N_JOBS", "1")) +jupyterlite_contents = ["jupyterlite_contents"] +jupyterlite_bind_ipynb_suffix = False + +# Build the browser kernel from the xeus environment file: packages are installed +# at build time (see doc/jupyterlite_environment.yml) rather than at runtime, and +# the served JupyterLite content is mounted into the kernel filesystem so the +# notebooks can read the bundled MNE data without downloading anything. +jupyterlite_build_command_options = { + "XeusAddon.environment_file": "jupyterlite_environment.yml", + "XeusAddon.mount_jupyterlite_content": True, +} + +# Inject the required subset of MNE-sample-data for JupyterLite. The data is +# placed under doc/lite_extra/mne_data and served at the docs root via +# html_extra_path (added below). lite_data (mne.datasets.lite_data) extracts +# the curated subset here, with the files under their original dataset folders +# (MNE-sample-data/, ...). +# +# NOTE (xeus): the setup cell probes for this data inside the kernel filesystem +# (/files, /drive). Getting the served copy visible to the kernel is the next +# step and needs care: embedding the whole set would bloat the kernel, while +# the /drive service-worker bridge needs cross-origin-isolation (COOP/COEP) +# headers that static artifact servers (e.g. CircleCI) do not send. +lite_root = Path(os.path.expanduser("~/mne_data/MNE-lite-data")) +src_sample_data = lite_root / "MNE-sample-data" +lite_extra_base = ( + Path(os.path.abspath(os.path.dirname(__file__))) / "lite_extra" / "mne_data" +) +dst_sample_data = lite_extra_base / "MNE-sample-data" +dst_sample_data.mkdir(parents=True, exist_ok=True) +print(f"[JupyterLite] Sample data source exists: {src_sample_data.exists()}") +print(f"[JupyterLite] Source path: {src_sample_data}") +if src_sample_data.exists(): + required_files = [ + "version.txt", + "MEG/sample/sample_audvis_raw.fif", + "MEG/sample/sample_audvis_filt-0-40_raw.fif", + "MEG/sample/sample_audvis_raw-eve.fif", + "MEG/sample/sample_audvis_filt-0-40_raw-eve.fif", + "MEG/sample/sample_audvis_ecg-proj.fif", + "MEG/sample/sample_audvis-ave.fif", + "MEG/sample/sample_audvis-cov.fif", + "MEG/sample/sample_audvis-meg-eeg-oct-6-fwd.fif", + "MEG/sample/sample_audvis-meg-oct-6-meg-inv.fif", + "MEG/sample/sample_audvis-meg-oct-6-fwd.fif", + "MEG/sample/sample_audvis-meg-oct-6-meg-fixed-inv.fif", + "MEG/sample/ernoise_raw.fif", + "MEG/sample/sample_audvis-no-filter-ave.fif", + "MEG/sample/sample_audvis_raw-trans.fif", + "MEG/sample/sample_audvis-shrunk-cov.fif", + "MEG/sample/sample_audvis-meg-lh.stc", + "MEG/sample/sample_audvis-meg-rh.stc", + "subjects/sample/mri/T1.mgz", + "subjects/sample/bem/sample-oct-6-src.fif", + "subjects/sample/surf/rh.pial", + "subjects/sample/surf/lh.pial", + "subjects/sample/surf/rh.white", + "subjects/sample/surf/lh.white", + "subjects/sample/surf/rh.inflated", + "subjects/sample/surf/lh.inflated", + "subjects/sample/surf/rh.curv", + "subjects/sample/surf/lh.curv", + "subjects/sample/label/lh.aparc.annot", + "subjects/sample/label/rh.aparc.annot", + ] + for req in required_files: + s = src_sample_data / req + d = dst_sample_data / req + if s.exists(): + d.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(s, d) + print(f"[JupyterLite] Copied: {req}") + else: + print(f"[JupyterLite] MISSING: {req}") + + +# Also inject SSVEP and EEGLAB testing datasets for JupyterLite +mne_data_base = Path(os.path.expanduser("~/mne_data")) +lite_data_base = lite_extra_base +lite_data_base.mkdir(parents=True, exist_ok=True) + +src_ssvep = mne_data_base / "ssvep-example-data" +dst_ssvep = lite_data_base / "ssvep-example-data" +print(f"[JupyterLite] SSVEP data source exists: {src_ssvep.exists()}") +if src_ssvep.exists() and not dst_ssvep.exists(): + shutil.copytree(src_ssvep, dst_ssvep, dirs_exist_ok=True) + print("[JupyterLite] Copied ssvep-example-data") + +src_eeglab = mne_data_base / "MNE-testing-data" / "EEGLAB" +dst_eeglab = lite_data_base / "MNE-testing-data" / "EEGLAB" +print(f"[JupyterLite] EEGLAB data source exists: {src_eeglab.exists()}") +if src_eeglab.exists() and not dst_eeglab.exists(): + shutil.copytree(src_eeglab, dst_eeglab, dirs_exist_ok=True) + print("[JupyterLite] Copied MNE-testing-data/EEGLAB") + +# Inject the single needed file(s) from extra datasets used by the Epochs and +# decoding examples. Sizes are all within what we already serve +# (sample_audvis_raw.fif is 128.5 MB): kiloword 28.7 MB, erp_core 123.6 MB, +# mtrf speech_data.mat 17.2 MB, eegbci 3x2.6 MB. The CI "Ensure ... data" step +# downloads them so the sources exist here. +for _folder, _ds_files in ( + ("MNE-kiloword-data", ["kword_metadata-epo.fif"]), + ("MNE-ERP-CORE-data", ["ERP-CORE_Subject-001_Task-Flankers_eeg.fif"]), + ("mTRF_1.5", ["speech_data.mat"]), + ( + "MNE-eegbci-data", + [ + "files/eegmmidb/1.0.0/S001/S001R06.edf", + "files/eegmmidb/1.0.0/S001/S001R10.edf", + "files/eegmmidb/1.0.0/S001/S001R14.edf", + ], + ), +): + _src_ds = lite_root / _folder + _dst_ds = lite_data_base / _folder + print(f"[JupyterLite] {_folder} source exists: {_src_ds.exists()}") + for _ds_file in _ds_files: + s = _src_ds / _ds_file + d = _dst_ds / _ds_file + if s.exists(): + d.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(s, d) + print(f"[JupyterLite] Copied: {_folder}/{_ds_file}") + else: + print(f"[JupyterLite] MISSING: {_folder}/{_ds_file}") + + sphinx_gallery_conf = { + "jupyterlite": { + "use_jupyter_lab": True, + "jupyterlite_contents": "jupyterlite_contents", + }, + "first_notebook_cell": XEUS_FIRST_NOTEBOOK_CELL, "doc_module": ("mne",), "reference_url": dict(mne=None), "examples_dirs": examples_dirs, @@ -583,6 +724,55 @@ "parallel": sphinx_gallery_parallel, } assert is_serializable(sphinx_gallery_conf) + +# --------------------------------------------------------------------------- +# Drop the "Open in JupyterLite" launch badge from gallery pages whose +# notebooks cannot run in the browser kernel at all: they need the R runtime +# (rpy2), a compiled package the browser kernel does not ship (antio), or +# datasets that cannot be bundled/slimmed. sphinx-gallery adds the badge to +# every example unconditionally, so we wrap its badge generator and return an +# empty string for these files. This only removes the badge/link — the +# notebook source is untouched (no in-code guard). Files that merely need data +# bundled, a pure-Python package installed, or pyvista 3D are NOT listed here +# (they are fixable, not impossible). +JUPYTERLITE_EXCLUDE = ( + # Tier 1 — impossible: R runtime / compiled package / huge single dataset + "examples/stats/r_interop.py", # rpy2 -> needs the R runtime + "examples/io/read_impedances.py", # antio (compiled, unavailable in WASM) + "examples/decoding/decoding_rsa_sgskip.py", # visual_92_categories ~6 GB + "examples/decoding/decoding_spoc_CMC.py", # fieldtrip_cmc ~700 MB + "examples/decoding/ssd_spatial_filters.py", # fieldtrip_cmc ~700 MB + # Tier 2 — multi-GB datasets (brainstorm / spm_face / opm / hf_sef) + "examples/datasets/brainstorm_data.py", + "examples/datasets/hf_sef_data.py", + "examples/datasets/opm_data.py", + "examples/datasets/spm_faces_dataset.py", + "examples/preprocessing/movement_detection.py", + "examples/preprocessing/muscle_detection.py", + "examples/preprocessing/otp.py", + "examples/time_frequency/source_power_spectrum_opm.py", + "examples/visualization/evoked_arrowmap.py", + "examples/visualization/meg_sensors.py", + "tutorials/inverse/80_brainstorm_phantom_elekta.py", + "tutorials/inverse/85_brainstorm_phantom_ctf.py", + "tutorials/io/60_ctf_bst_auditory.py", + "tutorials/preprocessing/80_opm_processing.py", +) + +import sphinx_gallery.gen_rst as _sg_gen_rst # noqa: E402 + +_orig_gen_jupyterlite_rst = _sg_gen_rst.gen_jupyterlite_rst + + +def _lite_badge_filtered(fpath, gallery_conf): + """Return the JupyterLite badge reST, or "" for excluded notebooks.""" + _p = str(fpath).replace(os.sep, "/") + if any(_p.endswith(_ex) for _ex in JUPYTERLITE_EXCLUDE): + return "" + return _orig_gen_jupyterlite_rst(fpath, gallery_conf) + + +_sg_gen_rst.gen_jupyterlite_rst = _lite_badge_filtered # Files were renamed from plot_* with: # find . -type f -name 'plot_*.py' -exec sh -c 'x="{}"; xn=`basename "${x}"`; git mv "$x" `dirname "${x}"`/${xn:5}' \; # noqa @@ -884,6 +1074,9 @@ def fix_sklearn_inherited_docstrings(app, what, name, obj, options, lines): "documentation.html", "getting_started.html", "install_mne_python.html", + # Serve the pre-bundled JupyterLite sample data at the docs root + # (e.g. /mne_data/...). The lite setup cell fetches it over HTTP. + "lite_extra", ] # Custom sidebar templates, maps document names to template names. @@ -1106,11 +1299,10 @@ def fix_sklearn_inherited_docstrings(app, what, name, obj, options, lines): # -- Dependency info ---------------------------------------------------------- -min_py = metadata("mne")["Requires-Python"].lstrip(" =<>") +min_py = "3.10" +min_py_minor = "10" rst_prolog += f"\n.. |min_python_version| replace:: {min_py}\n" -# -- website redirects -------------------------------------------------------- - # Static list created 2021/04/13 based on what we needed to redirect, # since we don't need to add redirects for examples added after this date. needed_plot_redirects = { @@ -1513,8 +1705,25 @@ def rstjinja(app, docname, source): # -- Connect our handlers to the main Sphinx app --------------------------- +def _mark_jupyterlite_parallel_safe(app): + """Declare jupyterlite_sphinx safe for Sphinx's parallel (-j) read phase. + + The jupyterlite-sphinx version pinned by our JupyterLite stack + (0.9.3) predates the ``parallel_read_safe`` metadata that newer releases + declare, so the doc build's ``-j auto`` emits a "does not declare if it is + safe for parallel reading" warning that ``-W`` turns into a build error. + Newer jupyterlite-sphinx marks it read-safe; set the same flag here rather + than bumping the pin (which would drag the whole pinned JupyterLite stack + forward and risk the browser build). + """ + ext = app.extensions.get("jupyterlite_sphinx") + if ext is not None and ext.parallel_read_safe is None: + ext.parallel_read_safe = True + + def setup(app): """Set up the Sphinx app.""" + app.connect("builder-inited", _mark_jupyterlite_parallel_safe, priority=1) app.connect("autodoc-process-docstring", append_attr_meth_examples) app.connect("autodoc-process-docstring", fix_sklearn_inherited_docstrings) # High prio, will happen before SG diff --git a/doc/jupyterlite_contents/.gitkeep b/doc/jupyterlite_contents/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/doc/jupyterlite_xeus_setup_cell.py b/doc/sphinxext/jupyterlite_xeus_setup_cell.py similarity index 100% rename from doc/jupyterlite_xeus_setup_cell.py rename to doc/sphinxext/jupyterlite_xeus_setup_cell.py diff --git a/mne/datasets/__init__.pyi b/mne/datasets/__init__.pyi index 2f69a1027e5..f1a6db8b581 100644 --- a/mne/datasets/__init__.pyi +++ b/mne/datasets/__init__.pyi @@ -19,6 +19,7 @@ __all__ = [ "hf_sef", "kiloword", "limo", + "lite_data", "misc", "mtrf", "multimodal", @@ -48,6 +49,7 @@ from . import ( hf_sef, kiloword, limo, + lite_data, misc, mtrf, multimodal, diff --git a/mne/datasets/config.py b/mne/datasets/config.py index 5599c20e5d4..293c29bcbb3 100644 --- a/mne/datasets/config.py +++ b/mne/datasets/config.py @@ -209,6 +209,16 @@ config_key="MNE_DATASETS_SAMPLE_PATH", ) +# Curated subset of sample (plus a few files from kiloword/erp_core/mtrf/eegbci) +# used by the JupyterLite browser docs; see mne/datasets/lite_data/. +MNE_DATASETS["lite_data"] = dict( + archive_name="MNE-lite-data.tar.gz", + hash="md5:5f9c4fffed32e79bc2bc2061bf22ce99", + url="https://osf.io/download/a8qbx", + folder_name="MNE-lite-data", + config_key="MNE_DATASETS_LITE_DATA_PATH", +) + MNE_DATASETS["somato"] = dict( archive_name="MNE-somato-data.tar.gz", hash="md5:9a191907b326b9402341ee7a0d1240d8", diff --git a/mne/datasets/lite_data/__init__.py b/mne/datasets/lite_data/__init__.py new file mode 100644 index 00000000000..eff22a4ce84 --- /dev/null +++ b/mne/datasets/lite_data/__init__.py @@ -0,0 +1,7 @@ +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + +"""Curated data subset for the JupyterLite browser documentation.""" + +from .lite_data import data_path, get_version diff --git a/mne/datasets/lite_data/lite_data.py b/mne/datasets/lite_data/lite_data.py new file mode 100644 index 00000000000..e9e575f8403 --- /dev/null +++ b/mne/datasets/lite_data/lite_data.py @@ -0,0 +1,43 @@ +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + +"""Curated data subset used by the JupyterLite browser documentation. + +The full MNE datasets (``sample``, ``kiloword``, ``erp_core``, ``mtrf``, +``eegbci``) ship as separate multi-GB archives, so the docs build would download +several gigabytes just to serve a handful of files to the browser notebooks. +``lite_data`` is a small curated archive holding only those files -- same data, +same checksums -- so the build fetches just what the JupyterLite notebooks need. +It extracts to ``MNE-lite-data/`` with the files under their original dataset +folders (``MNE-sample-data/``, ``MNE-kiloword-data/``, ...). +""" + +from ...utils import verbose +from ..utils import _data_path_doc, _download_mne_dataset, _get_version, _version_doc + + +@verbose +def data_path( + path=None, force_update=False, update_path=True, download=True, *, verbose=None +): # noqa: D103 + return _download_mne_dataset( + name="lite_data", + processor="untar", + path=path, + force_update=force_update, + update_path=update_path, + download=download, + ) + + +data_path.__doc__ = _data_path_doc.format( + name="lite_data", conf="MNE_DATASETS_LITE_DATA_PATH" +) + + +def get_version(): # noqa: D103 + return _get_version("lite_data") + + +get_version.__doc__ = _version_doc.format(name="lite_data") diff --git a/mne/datasets/tests/test_datasets.py b/mne/datasets/tests/test_datasets.py index a7f985392e7..53bdb1b5b2a 100644 --- a/mne/datasets/tests/test_datasets.py +++ b/mne/datasets/tests/test_datasets.py @@ -322,3 +322,18 @@ def test_fetch_uncompressed_file(tmp_path): ) fetch_dataset(dataset_dict, path=None, force_update=True) assert (tmp_path / "foo" / "LICENSE.foo").is_file() + + +def test_lite_data(): + """Test the lite_data curated dataset is registered correctly.""" + from mne.datasets import lite_data + from mne.datasets.config import MNE_DATASETS + + assert "lite_data" in MNE_DATASETS + cfg = MNE_DATASETS["lite_data"] + assert cfg["archive_name"] == "MNE-lite-data.tar.gz" + assert cfg["hash"].startswith("md5:") + assert cfg["url"].startswith("https://osf.io/") + assert cfg["config_key"] == "MNE_DATASETS_LITE_DATA_PATH" + assert callable(lite_data.data_path) + assert callable(lite_data.get_version) diff --git a/mne/parallel.py b/mne/parallel.py index 22443dab762..df90c6d1c30 100644 --- a/mne/parallel.py +++ b/mne/parallel.py @@ -156,6 +156,10 @@ def parallel_progress(op_iter): def _running_in_joblib_context(): """Check if we are running in a joblib.parallel_config context manager.""" + import sys + + if sys.platform == "emscripten": + return False try: from joblib.parallel import get_active_backend except ImportError: diff --git a/mne/utils/config.py b/mne/utils/config.py index 34677ed4db4..ea2793074f9 100644 --- a/mne/utils/config.py +++ b/mne/utils/config.py @@ -170,6 +170,7 @@ def set_memmap_min_size(memmap_min_size): "MNE_DATASETS_TESTING_PATH": "str, path for testing data", "MNE_DATASETS_VISUAL_92_CATEGORIES_PATH": "str, path for visual_92_categories data", "MNE_DATASETS_KILOWORD_PATH": "str, path for kiloword data", + "MNE_DATASETS_LITE_DATA_PATH": "str, path for lite_data data", "MNE_DATASETS_FIELDTRIP_CMC_PATH": "str, path for fieldtrip_cmc data", "MNE_DATASETS_PHANTOM_KIT_PATH": "str, path for phantom_kit data", "MNE_DATASETS_PHANTOM_4DBTI_PATH": "str, path for phantom_4dbti data", @@ -268,8 +269,8 @@ def _load_config(config_path, raise_error=False): with _open_lock(config_path, "r+") as fid: try: config = json.load(fid) - except ValueError: - # No JSON object could be decoded --> corrupt file? + except Exception: + # Catch ANY exception (including SyntaxError from Pyodide json parser) msg = ( f"The MNE-Python config file ({config_path}) is not a valid JSON " "file and might be corrupted" diff --git a/pyproject.toml b/pyproject.toml index 57ea6929885..e51a6b9ed62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ doc = [ "graphviz", "intersphinx_registry >= 0.2405.27", "ipython != 8.7.0", # also in "full-no-qt" and "test" + "jupyterlite-sphinx", "jupyterlite-xeus", "memory_profiler >= 0.16", "mne-bids", From c77ee9ad0903a950ed3c6cc6c94729cf59bed95d Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Wed, 22 Jul 2026 23:34:37 -0400 Subject: [PATCH 3/8] FIX: install micromamba so the xeus kernel can build on CircleCI --- .circleci/config.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac3e69b6b42..970bfc3a6e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -262,7 +262,17 @@ jobs: python -c "import mne; mne.datasets.lite_data.data_path(update_path=True)" # No wheel-building step here: the xeus kernel installs MNE and its # dependencies at build time from doc/jupyterlite_environment.yml, so - # there is nothing to hand to the browser kernel at runtime. + # there is nothing to hand to the browser kernel at runtime. It does need + # micromamba though, which jupyterlite-xeus shells out to in order to + # solve the emscripten-wasm32 environment; it is not on the image, so + # fetch the static binary and put it on PATH for the build below. + - run: + name: Install micromamba for the JupyterLite xeus kernel + command: | + curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest \ + | tar -xj -C "$HOME" bin/micromamba + echo 'export PATH="$HOME/bin:$PATH"' >> "$BASH_ENV" + "$HOME/bin/micromamba" --version # Build docs - run: name: make html From e56e819ee67af40e32333d4efeff03677a797deb Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Thu, 23 Jul 2026 10:07:01 -0400 Subject: [PATCH 4/8] ENH: fetch sample data over HTTP and enable 3D in the xeus kernel The setup cell probed /files/mne_data but never fetched anything, so every data-dependent notebook failed on the first read and the rest of the cells cascaded into NameErrors. The curated data is already served at the docs root via html_extra_path, so fetch it the same way the pyodide build does: a synchronous XMLHttpRequest into /tmp/mne_data, small files eagerly and the heavy ones lazily via the reader shims, so each notebook only downloads what it touches. The sync request may set responseType='arraybuffer' because the kernel runs in a web worker (the spec only forbids that on a Window), which means this needs neither cross-origin isolation nor a service worker -- static hosts such as the CircleCI artifact server provide neither. Also port the pyvista-js SourceEstimate.plot shim, which turned out to have no kernel-specific code, and add pyvista-js to the environment since the built kernel had no pyvista or vtk at all. Drop the threadpoolctl patch (Pyodide-only API) but keep the generic WASM ones. --- doc/jupyterlite_environment.yml | 3 + doc/sphinxext/jupyterlite_xeus_setup_cell.py | 562 ++++++++++++++++--- 2 files changed, 485 insertions(+), 80 deletions(-) diff --git a/doc/jupyterlite_environment.yml b/doc/jupyterlite_environment.yml index 1c1ef0b8574..42f582e34f8 100644 --- a/doc/jupyterlite_environment.yml +++ b/doc/jupyterlite_environment.yml @@ -39,3 +39,6 @@ dependencies: # the development MNE from this checkout (pure Python, installed w/o deps) - pip: - .. + # pyvista-js renders 3D through vtk.js in the browser; MNE's normal + # VTK stack cannot load in WASM. Pure Python, so pip is fine here. + - pyvista-js diff --git a/doc/sphinxext/jupyterlite_xeus_setup_cell.py b/doc/sphinxext/jupyterlite_xeus_setup_cell.py index a78771146c7..0c082f56891 100644 --- a/doc/sphinxext/jupyterlite_xeus_setup_cell.py +++ b/doc/sphinxext/jupyterlite_xeus_setup_cell.py @@ -1,44 +1,52 @@ -"""First-cut JupyterLite setup cell for the xeus-python kernel. +"""JupyterLite setup cell for the xeus-python kernel. -WIP / CI-PENDING -- this is the xeus counterpart of the Pyodide -``first_notebook_cell`` in ``conf.py``. It cannot be fully validated locally -(needs the real ``build_docs`` CI run); the assumptions marked ``CI-VERIFY`` -below are the ones to confirm/iterate on first. +This is the xeus counterpart of the Pyodide ``first_notebook_cell`` in +``conf.py``. -Why it is so much smaller than the Pyodide cell ------------------------------------------------- +How it differs from the Pyodide cell +------------------------------------ xeus-python is real CPython compiled to WebAssembly (NOT Pyodide), and the packages are PRE-INSTALLED at build time from ``jupyterlite_environment.yml``. So compared to the Pyodide setup cell this drops: * ``piplite.install(...)`` -> MNE is already installed * the ``lzma`` / ``multiprocessing`` mocks -> real CPython stdlib is present - * the ``pyodide.http`` / ``requests`` patches and the ``js`` XHR fetch shims - -> no Pyodide runtime to patch - -What remains is just: point MNE's dataset loaders at the bundled data, block -accidental OSF downloads, and (still to port) the pyvista-js 3D shim. - -Data model (CI-VERIFY) ----------------------- -With ``XeusAddon.mount_jupyterlite_content=True`` the served JupyterLite content -is mounted into the kernel filesystem at ``/files``. So the curated MNE data -must be made available as JupyterLite content (a conf.py change: add the curated -``mne_data`` tree to ``jupyterlite_contents`` instead of only ``html_extra_path``). -The cell below resolves the data root by probing the likely mount locations so it -is robust to which mechanism ends up being used. Because every bundled file is -then present in the FS, NO runtime fetch is needed -- the loaders just return the -folder (much simpler than the Pyodide lazy-fetch shims). - -conf.py wiring this expects (next step, not yet applied so the pyodide build -stays intact on this branch): - - from jupyterlite_xeus_setup_cell import XEUS_FIRST_NOTEBOOK_CELL - jupyterlite_build_command_options = { - "XeusAddon.environment_file": "jupyterlite_environment.yml", - "XeusAddon.mount_jupyterlite_content": True, - } - sphinx_gallery_conf["first_notebook_cell"] = XEUS_FIRST_NOTEBOOK_CELL + * the ``pyodide.http`` / ``requests`` patches -> no Pyodide runtime to patch + * the ``threadpoolctl`` ``as_object_map()`` patch -> Pyodide-only API + +The WASM-level workarounds that are not Pyodide-specific (no OS threads for +MNE's ProgressBar and tqdm's monitor, inline matplotlib, single-render +displayhook) are kept, since they apply to any WASM kernel. + +Data model +---------- +Same as the Pyodide build: the curated data is served at the docs root under +``/mne_data/`` (via Sphinx ``html_extra_path``, see ``conf.py``) and fetched +over HTTP into the kernel's own filesystem at ``/tmp/mne_data``. + +The fetch is a *synchronous* ``XMLHttpRequest`` with +``responseType='arraybuffer'``. JupyterLite kernels run in a web worker, where +that combination is allowed (the spec only forbids setting ``responseType`` on +a synchronous request when the global is a ``Window``). This matters because it +needs neither cross-origin isolation (COOP/COEP) nor a service worker, and +static hosts such as the CircleCI artifact server provide neither. + +Being synchronous is what lets a plain ``data_path()`` / ``read_raw_fif()`` +call fetch on demand, so each notebook only downloads the files it actually +touches instead of the whole sample dataset up front. + +The one xeus-specific difference is how JavaScript is reached. Pyodide exposes +a ``js`` module whose proxies carry a ``.to_py()`` method; xeus exposes +``pyjs``, where the equivalents are the module-level ``pyjs.new()`` and +``pyjs.to_py()`` (a ``pyjs`` ``JsValue`` has no ``.to_py()`` method, and the +bundled ``pyjs.pyodide_polyfill`` deliberately provides only ``to_js``). + +3D rendering +------------ +The ``SourceEstimate.plot`` shim that routes through pyvista-js (vtk.js) is +carried over from the Pyodide cell unchanged -- it is plain numpy/nibabel/ +scipy/matplotlib plus ``pyvista_js``, with no kernel-specific code. It does +require ``pyvista-js`` to be installed, see ``jupyterlite_environment.yml``. """ # Authors: The MNE-Python contributors. @@ -48,65 +56,459 @@ # The setup cell source, as a string (mirrors how conf.py stores the Pyodide one). XEUS_FIRST_NOTEBOOK_CELL = r"""# 💡 Auto-added setup cell (xeus-python kernel). # MNE and its dependencies are pre-installed in this kernel; this cell only -# points the dataset loaders at the pre-bundled data. +# wires up the sample data, which is fetched on demand from the docs server. import os from pathlib import Path as _Path -import mne +import pyjs + +# --- locate the docs root --------------------------------------------------- +# The data is served at the docs root (/mne_data/...) via html_extra_path. +# `location` exists both in the main thread and in a web worker, so use it to +# find the docs root by splitting the page URL on '/lite/'. +_base = str(pyjs.js.location.href).split('/lite/')[0] + '/mne_data/' +mne_data_path = '/tmp/mne_data' +_sample_dir = mne_data_path + '/MNE-sample-data' + + +def _lite_bytes(_resp): + # JS ArrayBuffer -> Python bytes. pyjs converts an ArrayBuffer via + # Uint8Array into an object supporting the buffer protocol. Note this is + # the module-level pyjs.to_py(); unlike Pyodide, a pyjs JsValue has no + # .to_py() method. + return bytes(pyjs.to_py(_resp)) + + +def _lite_fetch_rel(_rel): + # Fetch one file, relative to the served mne_data root, and cache it. + _dst = mne_data_path + '/' + _rel + if not os.path.exists(_dst): + _xhr = pyjs.new(pyjs.js.XMLHttpRequest) + _xhr.open('GET', _base + _rel, False) # False -> synchronous + _xhr.responseType = 'arraybuffer' + _xhr.send() + if int(_xhr.status) != 200: + raise FileNotFoundError( + f'Could not fetch {_rel} (HTTP {int(_xhr.status)})' + ) + _data = _lite_bytes(_xhr.response) + # A static server answers a missing path with an HTML error page. + if _data[:4] == b'=0) + _fc = _cv[_tris].mean(1) + for _cm, _col in ( + (_fc < 0, (0.68, 0.68, 0.68)), + (_fc >= 0, (0.38, 0.38, 0.38))): + _s = _sub(_pts, _tris, _cm) + if _s is not None: + _plotter.add_mesh( + _pv.PolyData(points=_s[0], faces=_flat(_s[1])), + color=_col, smooth_shading=True) + # activation as a smooth hot gradient in N value bands, + # each lifted 2% off the surface to avoid z-fighting + _fv = _scal[_tris].mean(1) + _p90 = _np.percentile(_scal, 90.0) + _fmax = float(_scal.max()) + # keep the background gray: for sparse point sources the + # 90th pct is ~0 (most of the brain is zero), which would + # paint everything, so fall back to a fraction of the max. + _fmin = _p90 if _p90 > _fmax * 0.05 else _fmax * 0.4 + if _fmax > _fmin: + _edges = _np.linspace(_fmin, _fmax, _N + 1) + for _i in range(_N): + if _i < _N - 1: + _m = (_fv >= _edges[_i]) & (_fv < _edges[_i + 1]) + else: + _m = _fv >= _edges[_i] + if int(_m.sum()) == 0: + continue + _rgb = _hot(0.25 + 0.41 * (_i / (_N - 1))) + _col = (float(_rgb[0]), float(_rgb[1]), + float(_rgb[2])) + _s = _sub(_pts, _tris, _m, 0.02, _cen) + if _s is not None: + _plotter.add_mesh( + _pv.PolyData(points=_s[0], + faces=_flat(_s[1])), + color=_col, smooth_shading=True) + # Open on the lateral profile (camera along the medial-lateral + # X axis, superior up), like native MNE, instead of vtk.js's + # default anterior/face-on view. Guarded so a missing + # view_vector never costs us the render. + try: + _plotter.view_vector((-1.0, 0.0, 0.0), + viewup=(0.0, 0.0, 1.0)) + except Exception: + pass + _plotter.show() + except Exception as _e: + print('[JupyterLite] pyvista-js 3D render unavailable: ' + + repr(_e)) + return _LiteBrain() +mne.SourceEstimate.plot = _lite_stc_plot + +# WASM has no OS threads, so MNE's ProgressBar background +# updater thread (used by the ProgressBar context manager, e.g. in +# permutation cluster tests) crashes with 'can't start new thread'. +# That thread only animates a cosmetic bar — the computation runs on +# the main thread and __exit__ writes the final state — so no-op its +# start/join. Only affects notebooks that use it; results are unchanged. +try: + from mne.utils import progressbar as _mpb + _mpb._UpdateThread.start = lambda self: None + _mpb._UpdateThread.join = lambda self, *_a, **_kw: None +except Exception: + pass +# tqdm also spawns its own monitor thread, which likewise can't start in +# WASM and emits a TqdmMonitorWarning. Setting monitor_interval=0 before +# any bar is created skips that thread entirely (bars still display). +try: + import tqdm as _tqdm + _tqdm.tqdm.monitor_interval = 0 +except Exception: + pass + +# Switch matplotlib to inline so figures render in the notebook. +import IPython +IPython.get_ipython().run_line_magic('matplotlib', 'inline') +import matplotlib.pyplot as plt +# Silence the spurious 'FigureCanvasAgg is non-interactive' warning +# at its source. MNE's plt_show calls fig.show() (the inline backend +# isn't detected as 'agg'), and the inline Agg canvas warns. Patching +# viz.utils.plt_show is not enough: other modules did +# `from .utils import plt_show` and hold their own reference. Every +# path resolves fig.show on the class at call time, so a no-op here +# silences it everywhere. Figures still render via the inline backend. +import matplotlib.figure as _mfig +_mfig.Figure.show = lambda self, *a, **k: None +import importlib +viz_utils = importlib.import_module('mne.viz.utils') +# Also display+close via IPython for paths that call plt_show +# directly, so figures render exactly once. +def _lite_plt_show(show=True, fig=None, **kwargs): + if not show: + return + import IPython.display + _f = fig if fig is not None else plt.gcf() + IPython.display.display(_f) + plt.close(_f) +viz_utils.plt_show = _lite_plt_show + +# Each MNE plot is rendered once by _lite_plt_show above (display()). +# When a plot call is also a cell's last expression, the method returns +# the Figure, which Jupyter echoes a SECOND time as the Out[] result +# (the duplicate seen below inline plots). Drop that redundant echo for +# Figures (and pure lists of Figures, e.g. ica.plot_properties) so each +# plot appears exactly once. Non-figure results (numbers, DataFrames, +# reprs) are untouched, and raw matplotlib figures never shown still +# render via the inline backend's end-of-cell flush, so nothing hides. +# Wrapped in try/except (like the patches below): if anything about +# the displayhook is unexpected, silently keep the current behavior +# (harmless double render) rather than breaking the setup cell. +try: + _lite_dh = type(IPython.get_ipython().displayhook) + if not getattr(_lite_dh, '_lite_no_fig_echo', False): + _lite_dh_call = _lite_dh.__call__ + def _lite_displayhook(self, result=None): + if isinstance(result, _mfig.Figure): + result = None + elif (isinstance(result, (list, tuple)) and result + and all(isinstance(_x, _mfig.Figure) for _x in result)): + result = None + return _lite_dh_call(self, result) + _lite_dh.__call__ = _lite_displayhook + _lite_dh._lite_no_fig_echo = True +except Exception: + pass """ From 821f59f5e0aa739d2bf6f4afccf2297aec4f6986 Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Thu, 23 Jul 2026 10:52:58 -0400 Subject: [PATCH 5/8] MAINT: keep the browser 3D meshes in narrower dtypes The xeus kernel's WASM heap is capped at 2 GiB by the emscripten build (bin/xpython.js requests WebAssembly.Memory with maximum=32768 pages), and the 3D shim runs at the end of a notebook that is still holding the raw, epochs and source estimates. Build the meshes with int32 face indices and float32 points, which are the types vtk.js uses anyway, and drop the KDTree query results and the full-resolution surface arrays as soon as they are dead. This is a small win (~14 MB retained, ~28 MB transient for the sample surfaces) and is NOT on its own enough to keep 10_overview under the cap -- that notebook still runs out of memory in the 3D cell. --- doc/sphinxext/jupyterlite_xeus_setup_cell.py | 26 ++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/doc/sphinxext/jupyterlite_xeus_setup_cell.py b/doc/sphinxext/jupyterlite_xeus_setup_cell.py index 0c082f56891..aade794616f 100644 --- a/doc/sphinxext/jupyterlite_xeus_setup_cell.py +++ b/doc/sphinxext/jupyterlite_xeus_setup_cell.py @@ -334,10 +334,16 @@ def _lite_stc_plot(self, *_a, **_kw): _ti = int(_np.argmin(_np.abs(self.times - _init))) _hot = _cmaps['hot'] _N = 10 + # xeus-python's WASM heap is capped at 2 GiB (see bin/xpython.js: + # WebAssembly.Memory maximum=32768 pages), and this runs at the end + # of a notebook that is already holding raw/epochs/stc. Every band + # below is a separate mesh, so keep the geometry in the narrowest + # dtypes VTK accepts: int32 face indices and float32 points. That + # halves the two biggest arrays and does not change the render. def _flat(_t): return _np.hstack([ - _np.full((len(_t), 1), 3, dtype=_np.int64), - _t.astype(_np.int64)]).ravel() + _np.full((len(_t), 1), 3, dtype=_np.int32), + _t.astype(_np.int32)]).ravel() def _sub(_pts, _tris, _mask, _lift=0.0, _cen=None): _sel = _tris[_mask] if len(_sel) == 0: @@ -346,7 +352,7 @@ def _sub(_pts, _tris, _mask, _lift=0.0, _cen=None): _p = _pts[_u] if _lift and _cen is not None: _p = _cen + (_p - _cen) * (1.0 + _lift) - return _p, _iv.reshape(-1, 3) + return _p.astype(_np.float32, copy=False), _iv.reshape(-1, 3) _plotter = _pv.Plotter() _plotter.background_color = 'black' # even lighting so the surface isn't black when rotated @@ -373,20 +379,25 @@ def _sub(_pts, _tris, _mask, _lift=0.0, _cen=None): # color each surface vertex from the nearest ACTIVE source # within a small radius, so single-vertex (point) sources # show as visible blobs and dense sources fill in as usual - _sv = _hdata[:, _ti].astype(float) + _sv = _hdata[:, _ti].astype(_np.float32) _act = _sv != 0 - _scal = _np.zeros(len(_rr)) + _scal = _np.zeros(len(_rr), dtype=_np.float32) if _act.any(): _atree = _KDTree(_rr[_vno][_act]) _ad, _ai = _atree.query(_rr) _scal = _np.where(_ad <= 12.0, _sv[_act][_ai], 0.0) + # the tree and its query results are the largest temporaries + # here; drop them before building any meshes + del _atree, _ad, _ai # offset hemispheres along x so they do not overlap _off = -60.0 if _h == 'lh' else 60.0 - _pts = _np.round(_rr, 2) + _pts = _np.round(_rr, 2).astype(_np.float32, copy=False) _pts[:, 0] = _pts[:, 0] + _off _cen = _pts.mean(0) + del _rr # curvature base: light gyri (curv<0) + dark sulci (curv>=0) - _fc = _cv[_tris].mean(1) + _fc = _cv[_tris].astype(_np.float32, copy=False).mean(1) + del _cv for _cm, _col in ( (_fc < 0, (0.68, 0.68, 0.68)), (_fc >= 0, (0.38, 0.38, 0.38))): @@ -395,6 +406,7 @@ def _sub(_pts, _tris, _mask, _lift=0.0, _cen=None): _plotter.add_mesh( _pv.PolyData(points=_s[0], faces=_flat(_s[1])), color=_col, smooth_shading=True) + del _fc # activation as a smooth hot gradient in N value bands, # each lifted 2% off the surface to avoid z-fighting _fv = _scal[_tris].mean(1) From bbc00abb98e8dfac0b146c9b45c410658658755d Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Thu, 23 Jul 2026 11:03:41 -0400 Subject: [PATCH 6/8] FIX: honour the hemi argument in the browser 3D shim SourceEstimate.plot defaults to hemi='lh' and 10_overview asks for hemi='rh', but the shim always built both hemispheres, so it drew a hemisphere the tutorial never asked for. That is expensive as well as wrong. pyvista-js serialises every mesh to JSON through .tolist(), which turns ~6.8 MB of surface arrays into ~55 MB of Python floats plus ~17 MB of text, and the browser then holds a UTF-16 copy of that text. Skipping the unwanted hemisphere removes about half of the scene. Also centre the surface when only one hemisphere is drawn, instead of applying the +/-60 mm split offset. --- doc/sphinxext/jupyterlite_xeus_setup_cell.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/sphinxext/jupyterlite_xeus_setup_cell.py b/doc/sphinxext/jupyterlite_xeus_setup_cell.py index aade794616f..37dcee63c58 100644 --- a/doc/sphinxext/jupyterlite_xeus_setup_cell.py +++ b/doc/sphinxext/jupyterlite_xeus_setup_cell.py @@ -332,6 +332,12 @@ def _lite_stc_plot(self, *_a, **_kw): _ti = int(_np.argmax(_np.abs(self.data).mean(0))) else: _ti = int(_np.argmin(_np.abs(self.times - _init))) + # Honour hemi (MNE defaults to 'lh'). This matters for memory as + # much as for correctness: pyvista-js serialises every mesh to JSON + # via .tolist(), which turns ~7 MB of surface into ~70 MB of Python + # floats + text, so rendering a hemisphere nobody asked for is the + # single most expensive mistake we can make under the 2 GiB cap. + _hemi = _kw.get('hemi', 'lh') _hot = _cmaps['hot'] _N = 10 # xeus-python's WASM heap is capped at 2 GiB (see bin/xpython.js: @@ -365,6 +371,8 @@ def _sub(_pts, _tris, _mask, _lift=0.0, _cen=None): _nlh = len(self.vertices[0]) _hemis = (('lh', 0, self.vertices[0]), ('rh', 1, self.vertices[1])) + if _hemi in ('lh', 'rh'): + _hemis = tuple(_x for _x in _hemis if _x[0] == _hemi) for _h, _hi, _vno in _hemis: if len(_vno) == 0: continue @@ -389,8 +397,9 @@ def _sub(_pts, _tris, _mask, _lift=0.0, _cen=None): # the tree and its query results are the largest temporaries # here; drop them before building any meshes del _atree, _ad, _ai - # offset hemispheres along x so they do not overlap - _off = -60.0 if _h == 'lh' else 60.0 + # offset hemispheres along x so they do not overlap; a single + # hemisphere stays centred + _off = 0.0 if len(_hemis) == 1 else (-60.0 if _h == 'lh' else 60.0) _pts = _np.round(_rr, 2).astype(_np.float32, copy=False) _pts[:, 0] = _pts[:, 0] + _off _cen = _pts.mean(0) From 0070e086a2347ec649d4839931db69885c589ac3 Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Thu, 23 Jul 2026 11:39:52 -0400 Subject: [PATCH 7/8] TEMP: raise the xeus wasm heap cap to 4 GiB Experiment to check whether the 2 GiB cap is what kills 10_overview in ica.plot_properties. Revert once we know. --- .circleci/config.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 970bfc3a6e7..29f55fadd3f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -279,6 +279,27 @@ jobs: command: | # we have -o pipefail in #BASH_ENV so we should be okay set -x PATTERN=$(cat pattern.txt) make -C doc $(cat build.txt) 2>&1 | tee sphinx_log.txt + # TEMPORARY EXPERIMENT - revert before this PR leaves draft. + # emscripten-forge builds xeus-python with MAXIMUM_MEMORY=2GB while + # pyodide ships 4GB, and 10_overview dies with Aborted(OOM) inside + # ica.plot_properties. Raise both caps to find out whether that 2 GiB is + # the only thing stopping it. This is NOT a fix: the wasm itself was + # compiled for a 2 GiB heap, so it is only trustworthy far enough to + # answer the question. The real fix is an emscripten-forge rebuild. + - run: + name: EXPERIMENT raise the xeus wasm heap cap to 4 GiB + command: | + set -x + f=doc/_build/html/lite/xeus/xeus-python/bin/xpython.js + test -f "$f" + grep -c 'maximum:32768' "$f" + grep -c 'getHeapMax=()=>2147483648' "$f" + sed -i \ + -e 's/maximum:32768/maximum:65536/' \ + -e 's/getHeapMax=()=>2147483648/getHeapMax=()=>4294967296/' \ + "$f" + grep -c 'maximum:65536' "$f" + grep -c 'getHeapMax=()=>4294967296' "$f" - run: name: Check sphinx log for warnings (which are treated as errors) when: always From 4bced00087ff7d41a2ebc8e395e696589c646852 Mon Sep 17 00:00:00 2001 From: natinew77-creator Date: Thu, 23 Jul 2026 12:11:50 -0400 Subject: [PATCH 8/8] Revert "TEMP: raise the xeus wasm heap cap to 4 GiB" The wasm declares max=32768 pages on its env.memory import, so handing it a 4 GiB memory is a LinkError and the kernel never starts. Only an emscripten-forge rebuild can raise it. --- .circleci/config.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 29f55fadd3f..970bfc3a6e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -279,27 +279,6 @@ jobs: command: | # we have -o pipefail in #BASH_ENV so we should be okay set -x PATTERN=$(cat pattern.txt) make -C doc $(cat build.txt) 2>&1 | tee sphinx_log.txt - # TEMPORARY EXPERIMENT - revert before this PR leaves draft. - # emscripten-forge builds xeus-python with MAXIMUM_MEMORY=2GB while - # pyodide ships 4GB, and 10_overview dies with Aborted(OOM) inside - # ica.plot_properties. Raise both caps to find out whether that 2 GiB is - # the only thing stopping it. This is NOT a fix: the wasm itself was - # compiled for a 2 GiB heap, so it is only trustworthy far enough to - # answer the question. The real fix is an emscripten-forge rebuild. - - run: - name: EXPERIMENT raise the xeus wasm heap cap to 4 GiB - command: | - set -x - f=doc/_build/html/lite/xeus/xeus-python/bin/xpython.js - test -f "$f" - grep -c 'maximum:32768' "$f" - grep -c 'getHeapMax=()=>2147483648' "$f" - sed -i \ - -e 's/maximum:32768/maximum:65536/' \ - -e 's/getHeapMax=()=>2147483648/getHeapMax=()=>4294967296/' \ - "$f" - grep -c 'maximum:65536' "$f" - grep -c 'getHeapMax=()=>4294967296' "$f" - run: name: Check sphinx log for warnings (which are treated as errors) when: always