From 1bac704500c3f262cddc580b5d45ef95e22623d5 Mon Sep 17 00:00:00 2001 From: "anujkumar.singh" Date: Sat, 6 Jun 2026 01:43:46 +0530 Subject: [PATCH 01/22] Stream notebook loading progressively --- docs/source/configuration.md | 7 ++ .../src/__tests__/yprovider.spec.ts | 9 +++ packages/docprovider/src/yprovider.ts | 19 +++-- .../jupyter_server_ydoc/app.py | 26 ++++++ .../jupyter_server_ydoc/handlers.py | 8 ++ .../jupyter_server_ydoc/rooms.py | 76 ++++++++++++++++-- tests/test_app.py | 20 +++++ tests/test_rooms.py | 79 ++++++++++++++++++- 8 files changed, 233 insertions(+), 11 deletions(-) diff --git a/docs/source/configuration.md b/docs/source/configuration.md index 0f7e8115..325febac 100644 --- a/docs/source/configuration.md +++ b/docs/source/configuration.md @@ -52,6 +52,13 @@ jupyter lab --YDocExtension.file_poll_interval=2 # If None, the document will be kept in memory forever. jupyter lab --YDocExtension.document_cleanup_delay=100 +# Whether notebooks loaded from disk should be streamed progressively (default: True). +jupyter lab --YDocExtension.notebook_load_progressively=False + +# Delay notebook cell outputs larger than this size in MB during progressive loading (default: 100). +# If None, outputs are loaded with their cells. +jupyter lab --YDocExtension.notebook_output_delay_threshold_mb=50 + # The YStore class to use for storing Y updates (default: JupyterSQLiteYStore). jupyter lab --YDocExtension.ystore_class=pycrdt.store.TempFileYStore diff --git a/packages/docprovider/src/__tests__/yprovider.spec.ts b/packages/docprovider/src/__tests__/yprovider.spec.ts index 58475f6e..b83c800e 100644 --- a/packages/docprovider/src/__tests__/yprovider.spec.ts +++ b/packages/docprovider/src/__tests__/yprovider.spec.ts @@ -162,6 +162,15 @@ describe('@jupyter/docprovider', () => { await expect(provider.ready).resolves.toBeUndefined(); }); + + it('should resolve ready when websocket connects', async () => { + const provider = createProvider(); + const wsProvider = await waitForProviderConnect(provider); + + wsProvider.emit('status', { status: 'connected' }); + + await expect(provider.ready).resolves.toBeUndefined(); + }); }); }); }); diff --git a/packages/docprovider/src/yprovider.ts b/packages/docprovider/src/yprovider.ts index 88b61729..06ae6386 100644 --- a/packages/docprovider/src/yprovider.ts +++ b/packages/docprovider/src/yprovider.ts @@ -110,6 +110,7 @@ export class WebSocketProvider implements IDocumentProvider, IForkProvider { } this._yWebsocketProvider?.off('connection-close', this._onConnectionClosed); this._yWebsocketProvider?.off('sync', this._onSync); + this._yWebsocketProvider?.off('status', this._onStatus); this._yWebsocketProvider?.destroy(); this._disconnect(); Signal.clearData(this); @@ -212,11 +213,7 @@ export class WebSocketProvider implements IDocumentProvider, IForkProvider { this._yWebsocketProvider.on('sync', this._onSync); this._yWebsocketProvider.on('connection-close', this._onConnectionClosed); - this._yWebsocketProvider.on('status', ({ status }: { status: string }) => { - if (status === 'connected') { - this._attachConflictListener(); - } - }); + this._yWebsocketProvider.on('status', this._onStatus); } async connectToForkDoc(forkRoomId: string, sessionId: string): Promise { @@ -245,6 +242,7 @@ export class WebSocketProvider implements IDocumentProvider, IForkProvider { private _disconnect(): void { this._yWebsocketProvider?.off('connection-close', this._onConnectionClosed); this._yWebsocketProvider?.off('sync', this._onSync); + this._yWebsocketProvider?.off('status', this._onStatus); this._yWebsocketProvider?.destroy(); this._yWebsocketProvider = null; } @@ -420,6 +418,17 @@ export class WebSocketProvider implements IDocumentProvider, IForkProvider { } }; + private _onStatus = ({ + status + }: { + status: 'connecting' | 'connected' | 'disconnected'; + }) => { + if (status === 'connected') { + this._attachConflictListener(); + this._ready.resolve(); + } + }; + private _getCloseReasonMessage(code: 4400 | 4404 | 4500): string { switch (code) { case 4400: { diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py index ba25a12e..30dbf05b 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py @@ -76,6 +76,22 @@ class YDocExtension(ExtensionApp): Defaults to 1s, if None then the document will never be saved.""", ) + notebook_load_progressively = Bool( + True, + config=True, + help="""Whether to progressively load notebooks from disk into the shared document. + When enabled, notebook metadata and cells can be streamed to clients before the full + notebook has finished loading.""", + ) + + notebook_output_delay_threshold_mb = Float( + 100, + allow_none=True, + config=True, + help="""Notebook output size in MB above which outputs are delayed during progressive + notebook loading. Set to None to keep outputs with their cells.""", + ) + ystore_class = Type( default_value=SQLiteYStore, klass=BaseYStore, @@ -118,6 +134,10 @@ def initialize_settings(self): "collaborative_file_poll_interval": self.file_poll_interval, "collaborative_document_cleanup_delay": self.document_cleanup_delay, "collaborative_document_save_delay": self.document_save_delay, + "collaborative_notebook_load_progressively": self.notebook_load_progressively, + "collaborative_notebook_output_delay_threshold_mb": ( + self.notebook_output_delay_threshold_mb + ), "collaborative_ystore_class": self.ystore_class, "collaborative_session_store_path": self.session_store_path, } @@ -166,6 +186,10 @@ def initialize_handlers(self): { "document_cleanup_delay": self.document_cleanup_delay, "document_save_delay": self.document_save_delay, + "notebook_load_progressively": self.notebook_load_progressively, + "notebook_output_delay_threshold_mb": ( + self.notebook_output_delay_threshold_mb + ), "file_loaders": self.file_loaders, "ystore_class": ystore_class, "ywebsocket_server": self.ywebsocket_server, @@ -258,6 +282,8 @@ async def get_document( self.log, exception_handler=exception_logger, save_delay=self.document_save_delay, + notebook_load_progressively=self.notebook_load_progressively, + notebook_output_delay_threshold_mb=self.notebook_output_delay_threshold_mb, ) await room.initialize() try: diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py index a63bd379..ed83458f 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py @@ -137,6 +137,10 @@ def exception_logger(exception: Exception, log: Logger) -> bool: self.log, exception_handler=exception_logger, save_delay=self._document_save_delay, + notebook_load_progressively=self._notebook_load_progressively, + notebook_output_delay_threshold_mb=( + self._notebook_output_delay_threshold_mb + ), ) else: @@ -181,6 +185,8 @@ def initialize( room_locks: dict[str, asyncio.Lock] | None = None, document_cleanup_delay: float | None = 60.0, document_save_delay: float | None = 1.0, + notebook_load_progressively: bool = True, + notebook_output_delay_threshold_mb: float | None = 100, ) -> None: self._background_tasks = set() # File ID manager cannot be passed as argument as the extension may load after this one @@ -189,6 +195,8 @@ def initialize( self._ystore_class = ystore_class self._cleanup_delay = document_cleanup_delay self._document_save_delay = document_save_delay + self._notebook_load_progressively = notebook_load_progressively + self._notebook_output_delay_threshold_mb = notebook_output_delay_threshold_mb self._websocket_server = ywebsocket_server self._message_queue = asyncio.Queue() self._room_id = "" diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py index d432933e..76dd29ca 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py @@ -15,6 +15,7 @@ Channel, Doc, Encoder, + YSyncMessageType, ) from pycrdt.store import BaseYStore, YDocNotFound from pycrdt.websocket import YRoom @@ -40,6 +41,8 @@ def __init__( ystore: BaseYStore | None, log: Logger | None, save_delay: float | None = None, + notebook_load_progressively: bool = True, + notebook_output_delay_threshold_mb: float | None = 100, exception_handler: Callable[[Exception, Logger], bool] | None = None, ): super().__init__(ready=False, ystore=ystore, exception_handler=exception_handler, log=log) @@ -53,17 +56,21 @@ def __init__( self._logger = logger self._save_delay = save_delay + self._notebook_load_progressively = notebook_load_progressively + self._notebook_output_delay_threshold_mb = notebook_output_delay_threshold_mb self._update_lock = asyncio.Lock() self._cleaner: asyncio.Task | None = None self._saving_document: asyncio.Task | None = None self._messages: dict[str, asyncio.Lock] = {} self._background_tasks = set() + self._block_client_document_updates = False # Listen for document changes self._document.observe(self._on_document_change) self._file.observe(self.room_id, self._on_outofband_change, self._on_filepath_change) + self.on_message = self._filter_message self.on_message_error = self._handle_sync_message_error @property @@ -119,7 +126,9 @@ async def initialize(self) -> None: model = await self._file.load_content(self._file_format, self._file_type) - async with self._update_lock: + await self._update_lock.acquire() + release_update_lock = True + try: # try to apply Y updates from the YStore for this document read_from_source = True loaded_from_store = False @@ -170,7 +179,14 @@ async def initialize(self) -> None: self._room_id, self._file.path, ) - if not loaded_from_store: + if self._should_load_notebook_progressively(loaded_from_store): + self._block_client_document_updates = True + self.ready = True + self._emit(LogLevel.INFO, "initialize", "Room initialized") + self.create_task(self._finish_progressive_initialization(model["content"])) + release_update_lock = False + return + elif not loaded_from_store: await self._apply_deterministic_source_content(model["content"]) else: await self._document.aset(model["content"]) @@ -181,8 +197,37 @@ async def initialize(self) -> None: self._document.dirty = False self.ready = True self._emit(LogLevel.INFO, "initialize", "Room initialized") + finally: + if release_update_lock: + self._update_lock.release() + + def _should_load_notebook_progressively(self, loaded_from_store: bool) -> bool: + return ( + self._file_type == "notebook" + and self._notebook_load_progressively + and not loaded_from_store + ) - async def _apply_deterministic_source_content(self, content: Any) -> None: + async def _finish_progressive_initialization(self, content: Any) -> None: + try: + # Give YRoom's ready watcher a chance to subscribe before source + # transactions are replayed into the live room. + await asyncio.sleep(0) + await self._apply_deterministic_source_content(content, progressive=True) + if self.ystore: + await self.ystore.encode_state_as_update(self.ydoc) + self._document.dirty = False + except Exception as e: + msg = f"Error loading content from file: {self._file.path}\n{e!r}" + self.log.error(msg, exc_info=e) + self._emit(LogLevel.ERROR, None, msg) + finally: + self._block_client_document_updates = False + self._update_lock.release() + + async def _apply_deterministic_source_content( + self, content: Any, progressive: bool = False + ) -> None: """Load source content using a deterministic update. Rooms rebuilt from disk must recreate the same Yjs history for identical @@ -194,8 +239,19 @@ async def _apply_deterministic_source_content(self, content: Any) -> None: """ source_ydoc: Doc = Doc(client_id=0) source_document = YDOCS.get(self._file_type, YFILE)(source_ydoc) - await source_document.aset(content) - self.ydoc.apply_update(source_ydoc.get_update()) + if progressive: + subscription = source_ydoc.observe(lambda event: self.ydoc.apply_update(event.update)) + try: + await source_document.aset( + content, + progressive=True, + delay_outputs_above_mb=self._notebook_output_delay_threshold_mb, + ) + finally: + source_ydoc.unobserve(subscription) + else: + await source_document.aset(content) + self.ydoc.apply_update(source_ydoc.get_update()) def _emit(self, level: LogLevel, action: str | None = None, msg: str | None = None) -> None: data = {"level": level.value, "room": self._room_id, "path": self._file.path} @@ -235,6 +291,16 @@ async def _broadcast_updates(self): except asyncio.CancelledError: pass + def _filter_message(self, message: bytes) -> bool: + """Skip client document updates while source content is still loading.""" + return ( + self._block_client_document_updates + and len(message) > 1 + and message[0] == MessageType.SYNC + and message[1] + in (YSyncMessageType.SYNC_STEP2, YSyncMessageType.SYNC_UPDATE) + ) + async def _handle_sync_message_error( self, exc: Exception, message: bytes, channel: Channel ) -> bool: diff --git a/tests/test_app.py b/tests/test_app.py index db9a6ffd..d5094045 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -16,6 +16,8 @@ def test_default_settings(jp_serverapp): assert settings["file_poll_interval"] == 1 assert settings["document_cleanup_delay"] == 60 assert settings["document_save_delay"] == 1 + assert settings["notebook_load_progressively"] is True + assert settings["notebook_output_delay_threshold_mb"] == 100 assert settings["ystore_class"] == SQLiteYStore @@ -55,6 +57,24 @@ def test_settings_should_change_save_delay(jp_configurable_serverapp): assert settings["document_save_delay"] == 10 +def test_settings_should_change_notebook_progressive_loading(jp_configurable_serverapp): + argv = ["--YDocExtension.notebook_load_progressively=False"] + + app = jp_configurable_serverapp(argv=argv) + settings = app.web_app.settings["jupyter_server_ydoc_config"] + + assert settings["notebook_load_progressively"] is False + + +def test_settings_should_change_notebook_output_delay_threshold(jp_configurable_serverapp): + argv = ["--YDocExtension.notebook_output_delay_threshold_mb=50"] + + app = jp_configurable_serverapp(argv=argv) + settings = app.web_app.settings["jupyter_server_ydoc_config"] + + assert settings["notebook_output_delay_threshold_mb"] == 50 + + def test_settings_should_change_ystore_class(jp_configurable_serverapp): argv = ["--YDocExtension.ystore_class=jupyter_server_ydoc.stores.TempFileYStore"] diff --git a/tests/test_rooms.py b/tests/test_rooms.py index 30fa96ad..40a14ad8 100644 --- a/tests/test_rooms.py +++ b/tests/test_rooms.py @@ -6,7 +6,15 @@ import asyncio from unittest.mock import AsyncMock, patch -from jupyter_server_ydoc.utils import OutOfBandChanges +from pycrdt import YSyncMessageType +from jupyter_server_ydoc.loaders import FileLoader +from jupyter_server_ydoc.rooms import DocumentRoom +from jupyter_server_ydoc.test_utils import ( + FakeContentsManager, + FakeEventLogger, + FakeFileIDManager, +) +from jupyter_server_ydoc.utils import MessageType, OutOfBandChanges from jupyter_ydoc import YUnicode @@ -259,6 +267,75 @@ async def test_on_outofband_change_skips_aset_when_content_unchanged( await room._on_outofband_change() mock_aset.assert_not_called() + +async def test_progressive_notebook_initialize_streams_before_load_finishes(): + content = { + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5, + } + cm = FakeContentsManager({"content": content, "writable": True}) + loader = FileLoader( + "notebook-id", + FakeFileIDManager({"notebook-id": "large.ipynb"}), + cm, + poll_interval=None, + ) + room = DocumentRoom( + "test-room", + "json", + "notebook", + loader, + FakeEventLogger(), + None, + None, + save_delay=0.01, + notebook_load_progressively=True, + notebook_output_delay_threshold_mb=50, + ) + started = asyncio.Event() + resume = asyncio.Event() + + async def slow_apply_source_content(value, progressive=False): + assert value == content + assert progressive is True + assert room.ready + assert room._update_lock.locked() + started.set() + await resume.wait() + + with patch.object( + room, "_apply_deterministic_source_content", slow_apply_source_content + ): + task = asyncio.create_task(room.initialize()) + await asyncio.wait_for(started.wait(), timeout=1) + await asyncio.wait_for(task, timeout=1) + + assert room.ready + assert room._update_lock.locked() + + room._on_document_change("cells", None) + assert room._saving_document is None + assert room._save_to_disc() is None + assert room._filter_message( + bytes([MessageType.SYNC, YSyncMessageType.SYNC_UPDATE]) + ) + assert room._filter_message(bytes([MessageType.SYNC, YSyncMessageType.SYNC_STEP2])) + assert not room._filter_message( + bytes([MessageType.SYNC, YSyncMessageType.SYNC_STEP1]) + ) + assert not room._filter_message(bytes([MessageType.AWARENESS, 0])) + + resume.set() + for _ in range(20): + if not room._update_lock.locked(): + break + await asyncio.sleep(0.01) + + assert not room._update_lock.locked() + assert not room._filter_message(bytes([MessageType.SYNC, YSyncMessageType.SYNC_UPDATE])) + assert not room._document.dirty From a5a066877c447b7a9927e1111bc2754bc1c85b50 Mon Sep 17 00:00:00 2001 From: "anujkumar.singh" Date: Sat, 6 Jun 2026 02:03:44 +0530 Subject: [PATCH 02/22] Apply pre-commit formatting --- .../jupyter_server_ydoc/rooms.py | 3 +-- tests/test_rooms.py | 14 ++++---------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py index 76dd29ca..e05996eb 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py @@ -297,8 +297,7 @@ def _filter_message(self, message: bytes) -> bool: self._block_client_document_updates and len(message) > 1 and message[0] == MessageType.SYNC - and message[1] - in (YSyncMessageType.SYNC_STEP2, YSyncMessageType.SYNC_UPDATE) + and message[1] in (YSyncMessageType.SYNC_STEP2, YSyncMessageType.SYNC_UPDATE) ) async def _handle_sync_message_error( diff --git a/tests/test_rooms.py b/tests/test_rooms.py index 40a14ad8..afc0d5d7 100644 --- a/tests/test_rooms.py +++ b/tests/test_rooms.py @@ -6,7 +6,6 @@ import asyncio from unittest.mock import AsyncMock, patch -from pycrdt import YSyncMessageType from jupyter_server_ydoc.loaders import FileLoader from jupyter_server_ydoc.rooms import DocumentRoom from jupyter_server_ydoc.test_utils import ( @@ -16,6 +15,7 @@ ) from jupyter_server_ydoc.utils import MessageType, OutOfBandChanges from jupyter_ydoc import YUnicode +from pycrdt import YSyncMessageType async def test_should_initialize_document_room_without_store(rtc_create_mock_document_room): @@ -305,9 +305,7 @@ async def slow_apply_source_content(value, progressive=False): started.set() await resume.wait() - with patch.object( - room, "_apply_deterministic_source_content", slow_apply_source_content - ): + with patch.object(room, "_apply_deterministic_source_content", slow_apply_source_content): task = asyncio.create_task(room.initialize()) await asyncio.wait_for(started.wait(), timeout=1) await asyncio.wait_for(task, timeout=1) @@ -318,13 +316,9 @@ async def slow_apply_source_content(value, progressive=False): room._on_document_change("cells", None) assert room._saving_document is None assert room._save_to_disc() is None - assert room._filter_message( - bytes([MessageType.SYNC, YSyncMessageType.SYNC_UPDATE]) - ) + assert room._filter_message(bytes([MessageType.SYNC, YSyncMessageType.SYNC_UPDATE])) assert room._filter_message(bytes([MessageType.SYNC, YSyncMessageType.SYNC_STEP2])) - assert not room._filter_message( - bytes([MessageType.SYNC, YSyncMessageType.SYNC_STEP1]) - ) + assert not room._filter_message(bytes([MessageType.SYNC, YSyncMessageType.SYNC_STEP1])) assert not room._filter_message(bytes([MessageType.AWARENESS, 0])) resume.set() From cf36031554b76edca584020442b55e52340e50d1 Mon Sep 17 00:00:00 2001 From: "anujkumar.singh" Date: Thu, 11 Jun 2026 19:48:13 +0530 Subject: [PATCH 03/22] Add progressive notebook loading UI test --- projects/jupyter-server-ydoc/pyproject.toml | 5 +- ui-tests/jupyter_server_test_config.py | 5 + ui-tests/tests/progressive-loading.spec.ts | 229 ++++++++++++++++++++ 3 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 ui-tests/tests/progressive-loading.spec.ts diff --git a/projects/jupyter-server-ydoc/pyproject.toml b/projects/jupyter-server-ydoc/pyproject.toml index 8029d1c9..780239a5 100644 --- a/projects/jupyter-server-ydoc/pyproject.toml +++ b/projects/jupyter-server-ydoc/pyproject.toml @@ -29,7 +29,7 @@ authors = [ ] dependencies = [ "jupyter_server>=2.15.0,<3.0.0", - "jupyter_ydoc>=4.0.0b0,<5.0.0", + "jupyter_ydoc @ git+https://github.com/MUFFANUJ/jupyter_ydoc.git@streamNotebook", "pycrdt", "pycrdt-websocket>=0.16.2,<0.17.0", "jupyter_events>=0.11.0", @@ -60,6 +60,9 @@ test = [ [tool.hatch.version] path = "jupyter_server_ydoc/_version.py" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.build.targets.sdist] exclude = ["/.github", "/binder", "node_modules"] diff --git a/ui-tests/jupyter_server_test_config.py b/ui-tests/jupyter_server_test_config.py index c6562035..65ed5fa3 100644 --- a/ui-tests/jupyter_server_test_config.py +++ b/ui-tests/jupyter_server_test_config.py @@ -19,6 +19,11 @@ # Fast room eviction so conflict tests don't need to wait 60 seconds. c.YDocExtension.document_cleanup_delay = 1 +# Keep the delayed-output path observable in UI tests without oversized +# notebook fixtures. +c.YDocExtension.notebook_load_progressively = True +c.YDocExtension.notebook_output_delay_threshold_mb = 7 + # Force-close dead WebSocket connections quickly. Playwright's setOffline(true) # blocks network I/O without tearing down existing TCP connections, so pings are # needed to make the server detect the disconnection. The conflict test goes diff --git a/ui-tests/tests/progressive-loading.spec.ts b/ui-tests/tests/progressive-loading.spec.ts new file mode 100644 index 00000000..25bda0e7 --- /dev/null +++ b/ui-tests/tests/progressive-loading.spec.ts @@ -0,0 +1,229 @@ +// Copyright (c) Jupyter Development Team. +// Distributed under the terms of the Modified BSD License. + +import { expect, galata, test } from '@jupyterlab/galata'; +import type { IJupyterLabPageFixture } from '@jupyterlab/galata'; +import type * as nbformat from '@jupyterlab/nbformat'; +import type { BrowserContext } from '@playwright/test'; +import { rm } from 'fs/promises'; + +const NOTEBOOK_NAME = 'progressive_loading.ipynb'; +const YSTORE_DB = '/tmp/jupyter_ystore_ui_test.db'; +const YSTORE_FILES = [YSTORE_DB, `${YSTORE_DB}-shm`, `${YSTORE_DB}-wal`]; +const HEAVY_PAYLOAD_BYTES = 8 * 1024 * 1024; +const HEAVY_PAYLOAD_ALPHABET = + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; +const LARGE_COLLABORATION_MESSAGE_BYTES = 512 * 1024; +const DELAYED_OUTPUT_MARKERS = [ + 'delayed-output-1-ready', + 'delayed-output-2-ready', + 'delayed-output-3-ready' +]; + +function makeHeavyPayload(seed: number): string { + const chunks: string[] = []; + let state = seed; + + // Repeated text compresses too well over WebSocket to exercise the large + // message path reliably. + for (let i = 0; i < HEAVY_PAYLOAD_BYTES; i += 1024) { + let chunk = ''; + for (let j = 0; j < 1024; j++) { + state = (state * 1664525 + 1013904223) >>> 0; + chunk += HEAVY_PAYLOAD_ALPHABET[state % HEAVY_PAYLOAD_ALPHABET.length]; + } + chunks.push(chunk); + } + + return chunks.join(''); +} + +function heavyOutput(label: string, seed: number): nbformat.IDisplayData { + return { + output_type: 'display_data', + data: { + // Keep the payload renderable but invisible in the UI. + 'text/html': `
${label}
` + }, + metadata: {} + }; +} + +function collaborationMessageSize(message: string | Buffer): number { + return typeof message === 'string' + ? Buffer.byteLength(message) + : message.byteLength; +} + +const LARGE_OUTPUT_NOTEBOOK = galata.Notebook.makeNotebook([ + galata.Notebook.makeCell({ + cell_type: 'markdown', + id: 'intro', + source: '# Progressive loading smoke' + }), + galata.Notebook.makeCell({ + cell_type: 'code', + id: 'small-output', + source: 'print("small output")', + outputs: [ + { + output_type: 'stream', + name: 'stdout', + text: 'small-output-ready\n' + } + ], + execution_count: 1 + }), + galata.Notebook.makeCell({ + cell_type: 'code', + id: 'heavy-output-1', + source: 'display heavy output 1', + outputs: [heavyOutput(DELAYED_OUTPUT_MARKERS[0], 1)], + execution_count: 1 + }), + galata.Notebook.makeCell({ + cell_type: 'code', + id: 'heavy-output-2', + source: 'display heavy output 2', + outputs: [heavyOutput(DELAYED_OUTPUT_MARKERS[1], 2)], + execution_count: 1 + }), + galata.Notebook.makeCell({ + cell_type: 'code', + id: 'heavy-output-3', + source: 'display heavy output 3', + outputs: [heavyOutput(DELAYED_OUTPUT_MARKERS[2], 3)], + execution_count: 1 + }) +]); + +async function holdLargeCollaborationMessages(context: BrowserContext): Promise<{ + release: () => void; + heldMessageCount: () => number; +}> { + let released = false; + let heldMessages = 0; + const queuedMessages: Array<() => void> = []; + + await context.routeWebSocket( + url => url.pathname.includes('/api/collaboration/room/'), + ws => { + const server = ws.connectToServer(); + + ws.onMessage(message => { + server.send(message); + }); + server.onMessage(message => { + const forwardMessage = () => { + ws.send(message); + }; + + if ( + collaborationMessageSize(message) > LARGE_COLLABORATION_MESSAGE_BYTES + ) { + // Make the first-pass notebook state observable before outputs arrive. + if (released) { + forwardMessage(); + } else { + heldMessages++; + queuedMessages.push(forwardMessage); + } + } else { + forwardMessage(); + } + }); + } + ); + + return { + release: () => { + released = true; + for (const forwardMessage of queuedMessages.splice(0)) { + forwardMessage(); + } + }, + heldMessageCount: () => heldMessages + }; +} + +test.describe('Progressive notebook loading', () => { + test.beforeEach(async () => { + // The progressive path is skipped when the document is restored from ystore. + await Promise.all(YSTORE_FILES.map(file => rm(file, { force: true }))); + }); + + test.afterEach(async ({ request, tmpPath }) => { + const contents = galata.newContentsHelper(request); + await contents.deleteFile(`${tmpPath}/${NOTEBOOK_NAME}`).catch(() => {}); + }); + + test('renders notebook content before all delayed large outputs finish', async ({ + browser, + request, + tmpPath, + baseURL, + waitForApplication + }) => { + // Install the WebSocket route before creating the page; Galata's default + // page fixture is already initialized by the time the test body runs. + const context = await browser.newContext(); + const largeMessages = await holdLargeCollaborationMessages(context); + let page: IJupyterLabPageFixture | undefined; + + try { + page = await galata.initTestPage( + '/lab', + true, + baseURL!, + true, + galata.DEFAULT_SETTINGS, + true, + true, + await context.newPage(), + new Map(), + new Map(), + tmpPath, + waitForApplication, + new Map(), + true + ); + + // uploadContent creates a generic file; the progressive path needs an + // actual notebook model. + const response = await request.put( + `${baseURL}/api/contents/${tmpPath}/${NOTEBOOK_NAME}`, + { + headers: { 'Content-Type': 'application/json' }, + data: JSON.stringify({ + type: 'notebook', + format: 'json', + content: LARGE_OUTPUT_NOTEBOOK + }) + } + ); + expect(response.ok()).toBeTruthy(); + + await page.filebrowser.refresh(); + expect(await page.notebook.open(NOTEBOOK_NAME)).toBeTruthy(); + await expect(page.locator('.jp-Dialog:visible')).toHaveCount(0); + + const notebook = page.locator('.jp-Notebook'); + await expect(page.locator('.jp-Notebook .jp-Cell')).toHaveCount(5, { + timeout: 15000 + }); + await expect(notebook).toContainText('small-output-ready'); + expect(await notebook.textContent()).not.toContain( + DELAYED_OUTPUT_MARKERS[2] + ); + await expect.poll(largeMessages.heldMessageCount).toBeGreaterThan(0); + + largeMessages.release(); + for (const marker of DELAYED_OUTPUT_MARKERS) { + await expect(notebook).toContainText(marker, { timeout: 30000 }); + } + } finally { + await page?.close().catch(() => {}); + await context.close().catch(() => {}); + } + }); +}); From 025d7f6fccaedca7a3502f339086cca19d801b31 Mon Sep 17 00:00:00 2001 From: "anujkumar.singh" Date: Thu, 11 Jun 2026 20:17:01 +0530 Subject: [PATCH 04/22] Fix reconnect tests --- tests/test_documents.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_documents.py b/tests/test_documents.py index 50a285e9..c005a605 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -164,6 +164,7 @@ async def _create_notebook_room(notebook: dict, room_id: str) -> tuple[DocumentR None, None, None, + notebook_load_progressively=False, ) await room.initialize() return room, loader From 2c2605f3422ee75946a4efedb6d8c9f8bb3dbcf1 Mon Sep 17 00:00:00 2001 From: "anujkumar.singh" Date: Mon, 29 Jun 2026 02:39:19 +0530 Subject: [PATCH 05/22] Make progressive document loading generic --- docs/source/configuration.md | 10 +-- .../jupyter_server_ydoc/app.py | 30 ++++---- .../jupyter_server_ydoc/handlers.py | 14 ++-- .../jupyter_server_ydoc/pytest_plugin.py | 1 + .../jupyter_server_ydoc/rooms.py | 24 ++++--- tests/test_app.py | 16 ++--- tests/test_documents.py | 2 +- tests/test_rooms.py | 72 ++++++++++++++++++- ui-tests/jupyter_server_test_config.py | 4 +- 9 files changed, 121 insertions(+), 52 deletions(-) diff --git a/docs/source/configuration.md b/docs/source/configuration.md index 325febac..f4dc533c 100644 --- a/docs/source/configuration.md +++ b/docs/source/configuration.md @@ -52,12 +52,12 @@ jupyter lab --YDocExtension.file_poll_interval=2 # If None, the document will be kept in memory forever. jupyter lab --YDocExtension.document_cleanup_delay=100 -# Whether notebooks loaded from disk should be streamed progressively (default: True). -jupyter lab --YDocExtension.notebook_load_progressively=False +# Whether documents loaded from disk should be streamed progressively (default: True). +jupyter lab --YDocExtension.document_load_progressively=False -# Delay notebook cell outputs larger than this size in MB during progressive loading (default: 100). -# If None, outputs are loaded with their cells. -jupyter lab --YDocExtension.notebook_output_delay_threshold_mb=50 +# Delay document outputs larger than this size in MB during progressive loading (default: 100). +# If None, outputs are loaded with their content. +jupyter lab --YDocExtension.document_output_delay_threshold_mb=50 # The YStore class to use for storing Y updates (default: JupyterSQLiteYStore). jupyter lab --YDocExtension.ystore_class=pycrdt.store.TempFileYStore diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py index 30dbf05b..b99f7625 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py @@ -76,20 +76,20 @@ class YDocExtension(ExtensionApp): Defaults to 1s, if None then the document will never be saved.""", ) - notebook_load_progressively = Bool( + document_load_progressively = Bool( True, config=True, - help="""Whether to progressively load notebooks from disk into the shared document. - When enabled, notebook metadata and cells can be streamed to clients before the full - notebook has finished loading.""", + help="""Whether to progressively load documents from disk into the shared document. + When enabled, a shared document can stream its content to clients before the full + source file has finished loading.""", ) - notebook_output_delay_threshold_mb = Float( + document_output_delay_threshold_mb = Float( 100, allow_none=True, config=True, - help="""Notebook output size in MB above which outputs are delayed during progressive - notebook loading. Set to None to keep outputs with their cells.""", + help="""Output size in MB above which a shared document may delay loading outputs + during progressive document loading. Set to None to keep outputs with their content.""", ) ystore_class = Type( @@ -134,9 +134,9 @@ def initialize_settings(self): "collaborative_file_poll_interval": self.file_poll_interval, "collaborative_document_cleanup_delay": self.document_cleanup_delay, "collaborative_document_save_delay": self.document_save_delay, - "collaborative_notebook_load_progressively": self.notebook_load_progressively, - "collaborative_notebook_output_delay_threshold_mb": ( - self.notebook_output_delay_threshold_mb + "collaborative_document_load_progressively": self.document_load_progressively, + "collaborative_document_output_delay_threshold_mb": ( + self.document_output_delay_threshold_mb ), "collaborative_ystore_class": self.ystore_class, "collaborative_session_store_path": self.session_store_path, @@ -186,9 +186,9 @@ def initialize_handlers(self): { "document_cleanup_delay": self.document_cleanup_delay, "document_save_delay": self.document_save_delay, - "notebook_load_progressively": self.notebook_load_progressively, - "notebook_output_delay_threshold_mb": ( - self.notebook_output_delay_threshold_mb + "document_load_progressively": self.document_load_progressively, + "document_output_delay_threshold_mb": ( + self.document_output_delay_threshold_mb ), "file_loaders": self.file_loaders, "ystore_class": ystore_class, @@ -282,8 +282,8 @@ async def get_document( self.log, exception_handler=exception_logger, save_delay=self.document_save_delay, - notebook_load_progressively=self.notebook_load_progressively, - notebook_output_delay_threshold_mb=self.notebook_output_delay_threshold_mb, + document_load_progressively=self.document_load_progressively, + document_output_delay_threshold_mb=self.document_output_delay_threshold_mb, ) await room.initialize() try: diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py index ed83458f..5f3f9d32 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py @@ -137,9 +137,9 @@ def exception_logger(exception: Exception, log: Logger) -> bool: self.log, exception_handler=exception_logger, save_delay=self._document_save_delay, - notebook_load_progressively=self._notebook_load_progressively, - notebook_output_delay_threshold_mb=( - self._notebook_output_delay_threshold_mb + document_load_progressively=self._document_load_progressively, + document_output_delay_threshold_mb=( + self._document_output_delay_threshold_mb ), ) @@ -185,8 +185,8 @@ def initialize( room_locks: dict[str, asyncio.Lock] | None = None, document_cleanup_delay: float | None = 60.0, document_save_delay: float | None = 1.0, - notebook_load_progressively: bool = True, - notebook_output_delay_threshold_mb: float | None = 100, + document_load_progressively: bool = True, + document_output_delay_threshold_mb: float | None = 100, ) -> None: self._background_tasks = set() # File ID manager cannot be passed as argument as the extension may load after this one @@ -195,8 +195,8 @@ def initialize( self._ystore_class = ystore_class self._cleanup_delay = document_cleanup_delay self._document_save_delay = document_save_delay - self._notebook_load_progressively = notebook_load_progressively - self._notebook_output_delay_threshold_mb = notebook_output_delay_threshold_mb + self._document_load_progressively = document_load_progressively + self._document_output_delay_threshold_mb = document_output_delay_threshold_mb self._websocket_server = ywebsocket_server self._message_queue = asyncio.Queue() self._room_id = "" diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/pytest_plugin.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/pytest_plugin.py index dec1d1be..a1c28df8 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/pytest_plugin.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/pytest_plugin.py @@ -317,6 +317,7 @@ def _inner( store, None, save_delay, + document_load_progressively=False, ), ) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py index e05996eb..14072728 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py @@ -11,6 +11,7 @@ from jupyter_events import EventLogger from jupyter_ydoc import ydocs as YDOCS +from jupyter_ydoc.ybasedoc import YBaseDoc from pycrdt import ( Channel, Doc, @@ -41,8 +42,8 @@ def __init__( ystore: BaseYStore | None, log: Logger | None, save_delay: float | None = None, - notebook_load_progressively: bool = True, - notebook_output_delay_threshold_mb: float | None = 100, + document_load_progressively: bool = True, + document_output_delay_threshold_mb: float | None = 100, exception_handler: Callable[[Exception, Logger], bool] | None = None, ): super().__init__(ready=False, ystore=ystore, exception_handler=exception_handler, log=log) @@ -56,8 +57,8 @@ def __init__( self._logger = logger self._save_delay = save_delay - self._notebook_load_progressively = notebook_load_progressively - self._notebook_output_delay_threshold_mb = notebook_output_delay_threshold_mb + self._document_load_progressively = document_load_progressively + self._document_output_delay_threshold_mb = document_output_delay_threshold_mb self._update_lock = asyncio.Lock() self._cleaner: asyncio.Task | None = None @@ -179,7 +180,7 @@ async def initialize(self) -> None: self._room_id, self._file.path, ) - if self._should_load_notebook_progressively(loaded_from_store): + if self._should_load_document_progressively(loaded_from_store): self._block_client_document_updates = True self.ready = True self._emit(LogLevel.INFO, "initialize", "Room initialized") @@ -201,11 +202,13 @@ async def initialize(self) -> None: if release_update_lock: self._update_lock.release() - def _should_load_notebook_progressively(self, loaded_from_store: bool) -> bool: + def _should_load_document_progressively(self, loaded_from_store: bool) -> bool: + progressive_setter = getattr(type(self._document), "aset_progressively", None) return ( - self._file_type == "notebook" - and self._notebook_load_progressively + self._document_load_progressively and not loaded_from_store + and progressive_setter is not None + and progressive_setter is not YBaseDoc.aset_progressively ) async def _finish_progressive_initialization(self, content: Any) -> None: @@ -242,10 +245,9 @@ async def _apply_deterministic_source_content( if progressive: subscription = source_ydoc.observe(lambda event: self.ydoc.apply_update(event.update)) try: - await source_document.aset( + await source_document.aset_progressively( content, - progressive=True, - delay_outputs_above_mb=self._notebook_output_delay_threshold_mb, + delay_outputs_above_mb=self._document_output_delay_threshold_mb, ) finally: source_ydoc.unobserve(subscription) diff --git a/tests/test_app.py b/tests/test_app.py index d5094045..cf2afef3 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -16,8 +16,8 @@ def test_default_settings(jp_serverapp): assert settings["file_poll_interval"] == 1 assert settings["document_cleanup_delay"] == 60 assert settings["document_save_delay"] == 1 - assert settings["notebook_load_progressively"] is True - assert settings["notebook_output_delay_threshold_mb"] == 100 + assert settings["document_load_progressively"] is True + assert settings["document_output_delay_threshold_mb"] == 100 assert settings["ystore_class"] == SQLiteYStore @@ -57,22 +57,22 @@ def test_settings_should_change_save_delay(jp_configurable_serverapp): assert settings["document_save_delay"] == 10 -def test_settings_should_change_notebook_progressive_loading(jp_configurable_serverapp): - argv = ["--YDocExtension.notebook_load_progressively=False"] +def test_settings_should_change_document_progressive_loading(jp_configurable_serverapp): + argv = ["--YDocExtension.document_load_progressively=False"] app = jp_configurable_serverapp(argv=argv) settings = app.web_app.settings["jupyter_server_ydoc_config"] - assert settings["notebook_load_progressively"] is False + assert settings["document_load_progressively"] is False -def test_settings_should_change_notebook_output_delay_threshold(jp_configurable_serverapp): - argv = ["--YDocExtension.notebook_output_delay_threshold_mb=50"] +def test_settings_should_change_document_output_delay_threshold(jp_configurable_serverapp): + argv = ["--YDocExtension.document_output_delay_threshold_mb=50"] app = jp_configurable_serverapp(argv=argv) settings = app.web_app.settings["jupyter_server_ydoc_config"] - assert settings["notebook_output_delay_threshold_mb"] == 50 + assert settings["document_output_delay_threshold_mb"] == 50 def test_settings_should_change_ystore_class(jp_configurable_serverapp): diff --git a/tests/test_documents.py b/tests/test_documents.py index c005a605..d065d682 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -164,7 +164,7 @@ async def _create_notebook_room(notebook: dict, room_id: str) -> tuple[DocumentR None, None, None, - notebook_load_progressively=False, + document_load_progressively=False, ) await room.initialize() return room, loader diff --git a/tests/test_rooms.py b/tests/test_rooms.py index afc0d5d7..f9add8e0 100644 --- a/tests/test_rooms.py +++ b/tests/test_rooms.py @@ -268,7 +268,7 @@ async def test_on_outofband_change_skips_aset_when_content_unchanged( mock_aset.assert_not_called() -async def test_progressive_notebook_initialize_streams_before_load_finishes(): +async def test_progressive_document_initialize_streams_before_load_finishes(): content = { "cells": [], "metadata": {}, @@ -291,8 +291,8 @@ async def test_progressive_notebook_initialize_streams_before_load_finishes(): None, None, save_delay=0.01, - notebook_load_progressively=True, - notebook_output_delay_threshold_mb=50, + document_load_progressively=True, + document_output_delay_threshold_mb=50, ) started = asyncio.Event() resume = asyncio.Event() @@ -333,6 +333,72 @@ async def slow_apply_source_content(value, progressive=False): assert not room._document.dirty +async def test_progressive_document_initialize_uses_standard_load_for_base_fallback(): + loader = FileLoader( + "file-id", + FakeFileIDManager({"file-id": "large.txt"}), + FakeContentsManager({"content": "large text content", "writable": True}), + poll_interval=None, + ) + room = DocumentRoom( + "test-room", + "text", + "file", + loader, + FakeEventLogger(), + None, + None, + document_load_progressively=True, + document_output_delay_threshold_mb=50, + ) + + assert not room._should_load_document_progressively(loaded_from_store=False) + + +async def test_progressive_source_content_calls_generic_document_setter(): + content = "content" + + class CustomDocument: + calls: list[tuple[str, float | None]] = [] + + def __init__(self, *args, **kwargs): + pass + + async def aset(self, value): + raise AssertionError("aset should not be used for progressive source content") + + async def aset_progressively(self, value, delay_outputs_above_mb=None): + self.calls.append((value, delay_outputs_above_mb)) + + def observe(self, callback): + pass + + def unobserve(self): + pass + + cm = FakeContentsManager({"content": content, "writable": True}) + loader = FileLoader( + "file-id", + FakeFileIDManager({"file-id": "large.custom"}), + cm, + poll_interval=None, + ) + with patch.dict("jupyter_server_ydoc.rooms.YDOCS", {"custom": CustomDocument}): + room = DocumentRoom( + "test-room", + "text", + "custom", + loader, + FakeEventLogger(), + None, + None, + document_output_delay_threshold_mb=50, + ) + await room._apply_deterministic_source_content(content, progressive=True) + + assert CustomDocument.calls == [(content, 50)] + + async def test_on_outofband_change_calls_aset_when_content_changed( rtc_create_mock_document_room, ): diff --git a/ui-tests/jupyter_server_test_config.py b/ui-tests/jupyter_server_test_config.py index 65ed5fa3..3ec4ebc5 100644 --- a/ui-tests/jupyter_server_test_config.py +++ b/ui-tests/jupyter_server_test_config.py @@ -21,8 +21,8 @@ # Keep the delayed-output path observable in UI tests without oversized # notebook fixtures. -c.YDocExtension.notebook_load_progressively = True -c.YDocExtension.notebook_output_delay_threshold_mb = 7 +c.YDocExtension.document_load_progressively = True +c.YDocExtension.document_output_delay_threshold_mb = 7 # Force-close dead WebSocket connections quickly. Playwright's setOffline(true) # blocks network I/O without tearing down existing TCP connections, so pings are From ca0af02dac45b9df5b29f7da3b69c80bb1a676ac Mon Sep 17 00:00:00 2001 From: David Brochart Date: Mon, 29 Jun 2026 22:25:11 +0200 Subject: [PATCH 06/22] Rework --- docs/source/configuration.md | 6 +- .../src/__tests__/yprovider.spec.ts | 9 -- packages/docprovider/src/yprovider.ts | 19 +-- .../jupyter_server_ydoc/app.py | 18 +-- .../jupyter_server_ydoc/handlers.py | 8 +- .../jupyter_server_ydoc/rooms.py | 65 +++------ projects/jupyter-server-ydoc/pyproject.toml | 10 +- tests/test_app.py | 8 +- tests/test_rooms.py | 131 ------------------ ui-tests/jupyter_server_test_config.py | 2 +- 10 files changed, 53 insertions(+), 223 deletions(-) diff --git a/docs/source/configuration.md b/docs/source/configuration.md index f4dc533c..baf626c3 100644 --- a/docs/source/configuration.md +++ b/docs/source/configuration.md @@ -55,9 +55,9 @@ jupyter lab --YDocExtension.document_cleanup_delay=100 # Whether documents loaded from disk should be streamed progressively (default: True). jupyter lab --YDocExtension.document_load_progressively=False -# Delay document outputs larger than this size in MB during progressive loading (default: 100). -# If None, outputs are loaded with their content. -jupyter lab --YDocExtension.document_output_delay_threshold_mb=50 +# Delay notebook outputs larger than this size in MB during progressive loading (default: 100). +# If None, outputs are loaded with the inputs. +jupyter lab --YDocExtension.notebook_output_delay_threshold_mb=50 # The YStore class to use for storing Y updates (default: JupyterSQLiteYStore). jupyter lab --YDocExtension.ystore_class=pycrdt.store.TempFileYStore diff --git a/packages/docprovider/src/__tests__/yprovider.spec.ts b/packages/docprovider/src/__tests__/yprovider.spec.ts index b83c800e..58475f6e 100644 --- a/packages/docprovider/src/__tests__/yprovider.spec.ts +++ b/packages/docprovider/src/__tests__/yprovider.spec.ts @@ -162,15 +162,6 @@ describe('@jupyter/docprovider', () => { await expect(provider.ready).resolves.toBeUndefined(); }); - - it('should resolve ready when websocket connects', async () => { - const provider = createProvider(); - const wsProvider = await waitForProviderConnect(provider); - - wsProvider.emit('status', { status: 'connected' }); - - await expect(provider.ready).resolves.toBeUndefined(); - }); }); }); }); diff --git a/packages/docprovider/src/yprovider.ts b/packages/docprovider/src/yprovider.ts index 53f3454c..8d0bc6aa 100644 --- a/packages/docprovider/src/yprovider.ts +++ b/packages/docprovider/src/yprovider.ts @@ -111,7 +111,6 @@ export class WebSocketProvider implements IDocumentProvider, IForkProvider { } this._yWebsocketProvider?.off('connection-close', this._onConnectionClosed); this._yWebsocketProvider?.off('sync', this._onSync); - this._yWebsocketProvider?.off('status', this._onStatus); this._yWebsocketProvider?.destroy(); this._disconnect(); Signal.clearData(this); @@ -214,7 +213,11 @@ export class WebSocketProvider implements IDocumentProvider, IForkProvider { this._yWebsocketProvider.on('sync', this._onSync); this._yWebsocketProvider.on('connection-close', this._onConnectionClosed); - this._yWebsocketProvider.on('status', this._onStatus); + this._yWebsocketProvider.on('status', ({ status }: { status: string }) => { + if (status === 'connected') { + this._attachConflictListener(); + } + }); } async connectToForkDoc(forkRoomId: string, sessionId: string): Promise { @@ -243,7 +246,6 @@ export class WebSocketProvider implements IDocumentProvider, IForkProvider { private _disconnect(): void { this._yWebsocketProvider?.off('connection-close', this._onConnectionClosed); this._yWebsocketProvider?.off('sync', this._onSync); - this._yWebsocketProvider?.off('status', this._onStatus); this._yWebsocketProvider?.destroy(); this._yWebsocketProvider = null; } @@ -429,17 +431,6 @@ export class WebSocketProvider implements IDocumentProvider, IForkProvider { } }; - private _onStatus = ({ - status - }: { - status: 'connecting' | 'connected' | 'disconnected'; - }) => { - if (status === 'connected') { - this._attachConflictListener(); - this._ready.resolve(); - } - }; - private _getCloseReasonMessage(code: 4400 | 4404 | 4500): string { switch (code) { case 4400: { diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py index b99f7625..5c33956a 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py @@ -84,12 +84,12 @@ class YDocExtension(ExtensionApp): source file has finished loading.""", ) - document_output_delay_threshold_mb = Float( + notebook_output_delay_threshold_mb = Float( 100, allow_none=True, config=True, - help="""Output size in MB above which a shared document may delay loading outputs - during progressive document loading. Set to None to keep outputs with their content.""", + help="""Output size in MB above which a shared notebook may delay loading outputs + during progressive document loading. Set to None to keep loading outputs with the inputs.""", ) ystore_class = Type( @@ -135,8 +135,8 @@ def initialize_settings(self): "collaborative_document_cleanup_delay": self.document_cleanup_delay, "collaborative_document_save_delay": self.document_save_delay, "collaborative_document_load_progressively": self.document_load_progressively, - "collaborative_document_output_delay_threshold_mb": ( - self.document_output_delay_threshold_mb + "collaborative_notebook_output_delay_threshold_mb": ( + self.notebook_output_delay_threshold_mb ), "collaborative_ystore_class": self.ystore_class, "collaborative_session_store_path": self.session_store_path, @@ -187,8 +187,8 @@ def initialize_handlers(self): "document_cleanup_delay": self.document_cleanup_delay, "document_save_delay": self.document_save_delay, "document_load_progressively": self.document_load_progressively, - "document_output_delay_threshold_mb": ( - self.document_output_delay_threshold_mb + "notebook_output_delay_threshold_mb": ( + self.notebook_output_delay_threshold_mb ), "file_loaders": self.file_loaders, "ystore_class": ystore_class, @@ -283,12 +283,12 @@ async def get_document( exception_handler=exception_logger, save_delay=self.document_save_delay, document_load_progressively=self.document_load_progressively, - document_output_delay_threshold_mb=self.document_output_delay_threshold_mb, + notebook_output_delay_threshold_mb=self.notebook_output_delay_threshold_mb, ) - await room.initialize() try: await self.ywebsocket_server.start_room(room) self.ywebsocket_server.add_room(room_id, room) + await room.initialize() self.log.info(f"Created and started room: {room_id}") except Exception as e: self.log.error("Room %s failed to start on websocket server", room_id) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py index 5f3f9d32..c56155e9 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py @@ -138,8 +138,8 @@ def exception_logger(exception: Exception, log: Logger) -> bool: exception_handler=exception_logger, save_delay=self._document_save_delay, document_load_progressively=self._document_load_progressively, - document_output_delay_threshold_mb=( - self._document_output_delay_threshold_mb + notebook_output_delay_threshold_mb=( + self._notebook_output_delay_threshold_mb ), ) @@ -186,7 +186,7 @@ def initialize( document_cleanup_delay: float | None = 60.0, document_save_delay: float | None = 1.0, document_load_progressively: bool = True, - document_output_delay_threshold_mb: float | None = 100, + notebook_output_delay_threshold_mb: float | None = 100, ) -> None: self._background_tasks = set() # File ID manager cannot be passed as argument as the extension may load after this one @@ -196,7 +196,7 @@ def initialize( self._cleanup_delay = document_cleanup_delay self._document_save_delay = document_save_delay self._document_load_progressively = document_load_progressively - self._document_output_delay_threshold_mb = document_output_delay_threshold_mb + self._notebook_output_delay_threshold_mb = notebook_output_delay_threshold_mb self._websocket_server = ywebsocket_server self._message_queue = asyncio.Queue() self._room_id = "" diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py index 14072728..8727d61f 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py @@ -11,12 +11,10 @@ from jupyter_events import EventLogger from jupyter_ydoc import ydocs as YDOCS -from jupyter_ydoc.ybasedoc import YBaseDoc from pycrdt import ( Channel, Doc, Encoder, - YSyncMessageType, ) from pycrdt.store import BaseYStore, YDocNotFound from pycrdt.websocket import YRoom @@ -43,7 +41,7 @@ def __init__( log: Logger | None, save_delay: float | None = None, document_load_progressively: bool = True, - document_output_delay_threshold_mb: float | None = 100, + notebook_output_delay_threshold_mb: float | None = 100, exception_handler: Callable[[Exception, Logger], bool] | None = None, ): super().__init__(ready=False, ystore=ystore, exception_handler=exception_handler, log=log) @@ -58,20 +56,18 @@ def __init__( self._logger = logger self._save_delay = save_delay self._document_load_progressively = document_load_progressively - self._document_output_delay_threshold_mb = document_output_delay_threshold_mb + self._notebook_output_delay_threshold_mb = notebook_output_delay_threshold_mb self._update_lock = asyncio.Lock() self._cleaner: asyncio.Task | None = None self._saving_document: asyncio.Task | None = None self._messages: dict[str, asyncio.Lock] = {} self._background_tasks = set() - self._block_client_document_updates = False # Listen for document changes self._document.observe(self._on_document_change) self._file.observe(self.room_id, self._on_outofband_change, self._on_filepath_change) - self.on_message = self._filter_message self.on_message_error = self._handle_sync_message_error @property @@ -180,56 +176,46 @@ async def initialize(self) -> None: self._room_id, self._file.path, ) - if self._should_load_document_progressively(loaded_from_store): - self._block_client_document_updates = True - self.ready = True - self._emit(LogLevel.INFO, "initialize", "Room initialized") - self.create_task(self._finish_progressive_initialization(model["content"])) - release_update_lock = False - return - elif not loaded_from_store: - await self._apply_deterministic_source_content(model["content"]) + if not loaded_from_store: + if self._document_load_progressively: + initialized = asyncio.Event() + finish = asyncio.Event() + self.create_task(self._finish_progressive_initialization(model["content"], initialized, finish)) + await initialized.wait() + self.ready = True + await self.ydoc_observed.wait() + finish.set() + release_update_lock = False + self._emit(LogLevel.INFO, "initialize", "Room initialized") + return + else: + await self._apply_deterministic_source_content(model["content"]) else: await self._document.aset(model["content"]) if self.ystore: await self.ystore.encode_state_as_update(self.ydoc) - self._document.dirty = False self.ready = True self._emit(LogLevel.INFO, "initialize", "Room initialized") finally: if release_update_lock: self._update_lock.release() - def _should_load_document_progressively(self, loaded_from_store: bool) -> bool: - progressive_setter = getattr(type(self._document), "aset_progressively", None) - return ( - self._document_load_progressively - and not loaded_from_store - and progressive_setter is not None - and progressive_setter is not YBaseDoc.aset_progressively - ) - - async def _finish_progressive_initialization(self, content: Any) -> None: + async def _finish_progressive_initialization(self, content: Any, initialized: asyncio.Event, finish: asyncio.Event) -> None: try: - # Give YRoom's ready watcher a chance to subscribe before source - # transactions are replayed into the live room. - await asyncio.sleep(0) - await self._apply_deterministic_source_content(content, progressive=True) + await self._apply_deterministic_source_content(content, progressive=True, initialized=initialized, finish=finish) if self.ystore: await self.ystore.encode_state_as_update(self.ydoc) - self._document.dirty = False except Exception as e: msg = f"Error loading content from file: {self._file.path}\n{e!r}" self.log.error(msg, exc_info=e) self._emit(LogLevel.ERROR, None, msg) finally: - self._block_client_document_updates = False self._update_lock.release() async def _apply_deterministic_source_content( - self, content: Any, progressive: bool = False + self, content: Any, progressive: bool = False, initialized: asyncio.Event | None = None, finish: asyncio.Event | None = None ) -> None: """Load source content using a deterministic update. @@ -247,7 +233,9 @@ async def _apply_deterministic_source_content( try: await source_document.aset_progressively( content, - delay_outputs_above_mb=self._document_output_delay_threshold_mb, + initialized=initialized, + finish=finish, + delay_outputs_above_mb=self._notebook_output_delay_threshold_mb ) finally: source_ydoc.unobserve(subscription) @@ -293,15 +281,6 @@ async def _broadcast_updates(self): except asyncio.CancelledError: pass - def _filter_message(self, message: bytes) -> bool: - """Skip client document updates while source content is still loading.""" - return ( - self._block_client_document_updates - and len(message) > 1 - and message[0] == MessageType.SYNC - and message[1] in (YSyncMessageType.SYNC_STEP2, YSyncMessageType.SYNC_UPDATE) - ) - async def _handle_sync_message_error( self, exc: Exception, message: bytes, channel: Channel ) -> bool: diff --git a/projects/jupyter-server-ydoc/pyproject.toml b/projects/jupyter-server-ydoc/pyproject.toml index 780239a5..bfe183e9 100644 --- a/projects/jupyter-server-ydoc/pyproject.toml +++ b/projects/jupyter-server-ydoc/pyproject.toml @@ -29,9 +29,9 @@ authors = [ ] dependencies = [ "jupyter_server>=2.15.0,<3.0.0", - "jupyter_ydoc @ git+https://github.com/MUFFANUJ/jupyter_ydoc.git@streamNotebook", + "jupyter_ydoc @ git+https://github.com/davidbrochart/jupyter_ydoc.git@finish", "pycrdt", - "pycrdt-websocket>=0.16.2,<0.17.0", + "pycrdt-websocket>=0.16.4,<0.17.0", "jupyter_events>=0.11.0", "jupyter_server_fileid>=0.7.0,<1", "jsonschema>=4.18.0" @@ -60,9 +60,6 @@ test = [ [tool.hatch.version] path = "jupyter_server_ydoc/_version.py" -[tool.hatch.metadata] -allow-direct-references = true - [tool.hatch.build.targets.sdist] exclude = ["/.github", "/binder", "node_modules"] @@ -74,3 +71,6 @@ exclude = ["/.github", "/binder", "node_modules"] [tool.check-wheel-contents] ignore = ["W002"] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/tests/test_app.py b/tests/test_app.py index cf2afef3..3fed821c 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -17,7 +17,7 @@ def test_default_settings(jp_serverapp): assert settings["document_cleanup_delay"] == 60 assert settings["document_save_delay"] == 1 assert settings["document_load_progressively"] is True - assert settings["document_output_delay_threshold_mb"] == 100 + assert settings["notebook_output_delay_threshold_mb"] == 100 assert settings["ystore_class"] == SQLiteYStore @@ -66,13 +66,13 @@ def test_settings_should_change_document_progressive_loading(jp_configurable_ser assert settings["document_load_progressively"] is False -def test_settings_should_change_document_output_delay_threshold(jp_configurable_serverapp): - argv = ["--YDocExtension.document_output_delay_threshold_mb=50"] +def test_settings_should_change_notebook_output_delay_threshold(jp_configurable_serverapp): + argv = ["--YDocExtension.notebook_output_delay_threshold_mb=50"] app = jp_configurable_serverapp(argv=argv) settings = app.web_app.settings["jupyter_server_ydoc_config"] - assert settings["document_output_delay_threshold_mb"] == 50 + assert settings["notebook_output_delay_threshold_mb"] == 50 def test_settings_should_change_ystore_class(jp_configurable_serverapp): diff --git a/tests/test_rooms.py b/tests/test_rooms.py index f9add8e0..31f49b1b 100644 --- a/tests/test_rooms.py +++ b/tests/test_rooms.py @@ -268,137 +268,6 @@ async def test_on_outofband_change_skips_aset_when_content_unchanged( mock_aset.assert_not_called() -async def test_progressive_document_initialize_streams_before_load_finishes(): - content = { - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5, - } - cm = FakeContentsManager({"content": content, "writable": True}) - loader = FileLoader( - "notebook-id", - FakeFileIDManager({"notebook-id": "large.ipynb"}), - cm, - poll_interval=None, - ) - room = DocumentRoom( - "test-room", - "json", - "notebook", - loader, - FakeEventLogger(), - None, - None, - save_delay=0.01, - document_load_progressively=True, - document_output_delay_threshold_mb=50, - ) - started = asyncio.Event() - resume = asyncio.Event() - - async def slow_apply_source_content(value, progressive=False): - assert value == content - assert progressive is True - assert room.ready - assert room._update_lock.locked() - started.set() - await resume.wait() - - with patch.object(room, "_apply_deterministic_source_content", slow_apply_source_content): - task = asyncio.create_task(room.initialize()) - await asyncio.wait_for(started.wait(), timeout=1) - await asyncio.wait_for(task, timeout=1) - - assert room.ready - assert room._update_lock.locked() - - room._on_document_change("cells", None) - assert room._saving_document is None - assert room._save_to_disc() is None - assert room._filter_message(bytes([MessageType.SYNC, YSyncMessageType.SYNC_UPDATE])) - assert room._filter_message(bytes([MessageType.SYNC, YSyncMessageType.SYNC_STEP2])) - assert not room._filter_message(bytes([MessageType.SYNC, YSyncMessageType.SYNC_STEP1])) - assert not room._filter_message(bytes([MessageType.AWARENESS, 0])) - - resume.set() - for _ in range(20): - if not room._update_lock.locked(): - break - await asyncio.sleep(0.01) - - assert not room._update_lock.locked() - assert not room._filter_message(bytes([MessageType.SYNC, YSyncMessageType.SYNC_UPDATE])) - - assert not room._document.dirty - - -async def test_progressive_document_initialize_uses_standard_load_for_base_fallback(): - loader = FileLoader( - "file-id", - FakeFileIDManager({"file-id": "large.txt"}), - FakeContentsManager({"content": "large text content", "writable": True}), - poll_interval=None, - ) - room = DocumentRoom( - "test-room", - "text", - "file", - loader, - FakeEventLogger(), - None, - None, - document_load_progressively=True, - document_output_delay_threshold_mb=50, - ) - - assert not room._should_load_document_progressively(loaded_from_store=False) - - -async def test_progressive_source_content_calls_generic_document_setter(): - content = "content" - - class CustomDocument: - calls: list[tuple[str, float | None]] = [] - - def __init__(self, *args, **kwargs): - pass - - async def aset(self, value): - raise AssertionError("aset should not be used for progressive source content") - - async def aset_progressively(self, value, delay_outputs_above_mb=None): - self.calls.append((value, delay_outputs_above_mb)) - - def observe(self, callback): - pass - - def unobserve(self): - pass - - cm = FakeContentsManager({"content": content, "writable": True}) - loader = FileLoader( - "file-id", - FakeFileIDManager({"file-id": "large.custom"}), - cm, - poll_interval=None, - ) - with patch.dict("jupyter_server_ydoc.rooms.YDOCS", {"custom": CustomDocument}): - room = DocumentRoom( - "test-room", - "text", - "custom", - loader, - FakeEventLogger(), - None, - None, - document_output_delay_threshold_mb=50, - ) - await room._apply_deterministic_source_content(content, progressive=True) - - assert CustomDocument.calls == [(content, 50)] - - async def test_on_outofband_change_calls_aset_when_content_changed( rtc_create_mock_document_room, ): diff --git a/ui-tests/jupyter_server_test_config.py b/ui-tests/jupyter_server_test_config.py index 3ec4ebc5..d1bcad76 100644 --- a/ui-tests/jupyter_server_test_config.py +++ b/ui-tests/jupyter_server_test_config.py @@ -22,7 +22,7 @@ # Keep the delayed-output path observable in UI tests without oversized # notebook fixtures. c.YDocExtension.document_load_progressively = True -c.YDocExtension.document_output_delay_threshold_mb = 7 +c.YDocExtension.notebook_output_delay_threshold_mb = 7 # Force-close dead WebSocket connections quickly. Playwright's setOffline(true) # blocks network I/O without tearing down existing TCP connections, so pings are From 94a73d72a34ef645f67ca08b70f049bf9f27866d Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 2 Jul 2026 16:13:58 +0200 Subject: [PATCH 07/22] pre-commit --- .../jupyter_server_ydoc/app.py | 4 ++-- .../jupyter_server_ydoc/rooms.py | 22 ++++++++++++++----- tests/test_rooms.py | 10 +-------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py index 5c33956a..3af7dd20 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py @@ -88,8 +88,8 @@ class YDocExtension(ExtensionApp): 100, allow_none=True, config=True, - help="""Output size in MB above which a shared notebook may delay loading outputs - during progressive document loading. Set to None to keep loading outputs with the inputs.""", + help="""Output size in MB above which a shared notebook may delay loading outputs during + progressive document loading. Set to None to keep loading outputs with the inputs.""", ) ystore_class = Type( diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py index 8727d61f..19d5989a 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py @@ -180,7 +180,11 @@ async def initialize(self) -> None: if self._document_load_progressively: initialized = asyncio.Event() finish = asyncio.Event() - self.create_task(self._finish_progressive_initialization(model["content"], initialized, finish)) + self.create_task( + self._finish_progressive_initialization( + model["content"], initialized, finish + ) + ) await initialized.wait() self.ready = True await self.ydoc_observed.wait() @@ -202,9 +206,13 @@ async def initialize(self) -> None: if release_update_lock: self._update_lock.release() - async def _finish_progressive_initialization(self, content: Any, initialized: asyncio.Event, finish: asyncio.Event) -> None: + async def _finish_progressive_initialization( + self, content: Any, initialized: asyncio.Event, finish: asyncio.Event + ) -> None: try: - await self._apply_deterministic_source_content(content, progressive=True, initialized=initialized, finish=finish) + await self._apply_deterministic_source_content( + content, progressive=True, initialized=initialized, finish=finish + ) if self.ystore: await self.ystore.encode_state_as_update(self.ydoc) except Exception as e: @@ -215,7 +223,11 @@ async def _finish_progressive_initialization(self, content: Any, initialized: as self._update_lock.release() async def _apply_deterministic_source_content( - self, content: Any, progressive: bool = False, initialized: asyncio.Event | None = None, finish: asyncio.Event | None = None + self, + content: Any, + progressive: bool = False, + initialized: asyncio.Event | None = None, + finish: asyncio.Event | None = None, ) -> None: """Load source content using a deterministic update. @@ -235,7 +247,7 @@ async def _apply_deterministic_source_content( content, initialized=initialized, finish=finish, - delay_outputs_above_mb=self._notebook_output_delay_threshold_mb + delay_outputs_above_mb=self._notebook_output_delay_threshold_mb, ) finally: source_ydoc.unobserve(subscription) diff --git a/tests/test_rooms.py b/tests/test_rooms.py index 31f49b1b..568bb036 100644 --- a/tests/test_rooms.py +++ b/tests/test_rooms.py @@ -6,16 +6,8 @@ import asyncio from unittest.mock import AsyncMock, patch -from jupyter_server_ydoc.loaders import FileLoader -from jupyter_server_ydoc.rooms import DocumentRoom -from jupyter_server_ydoc.test_utils import ( - FakeContentsManager, - FakeEventLogger, - FakeFileIDManager, -) -from jupyter_server_ydoc.utils import MessageType, OutOfBandChanges +from jupyter_server_ydoc.utils import OutOfBandChanges from jupyter_ydoc import YUnicode -from pycrdt import YSyncMessageType async def test_should_initialize_document_room_without_store(rtc_create_mock_document_room): From 035ae700a394509d4f821905812923fe1e1d1ab3 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 2 Jul 2026 16:32:19 +0200 Subject: [PATCH 08/22] Disable pre-commit --- .github/workflows/test.yml | 80 +++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ecc138ce..0487eaef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,46 +9,46 @@ on: jobs: # Run "pre-commit run --all-files --hook-stage=manual" - pre-commit: - name: Run pre-commit hook - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.13' - - - name: Base Setup - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - - - name: Install dependencies - run: | - pip install "jupyterlab>=4.6.0b1,<5" - jlpm install - - - name: Run pre-commit - uses: pre-commit/action@v3.0.1 - with: - extra_args: --all-files --hook-stage=manual - - - name: Help message if pre-commit fail - if: ${{ failure() }} - run: | - echo "You can install pre-commit hooks to automatically run formatting" - echo "on each commit with:" - echo " pre-commit install" - echo "or you can run by hand on staged files with" - echo " pre-commit run" - echo "or after-the-fact on already committed files with" - echo " pre-commit run --all-files --hook-stage=manual" - - - name: Lint frontend - run: | - jlpm run lint:check + #pre-commit: + # name: Run pre-commit hook + # runs-on: ubuntu-latest + # timeout-minutes: 10 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: '3.13' + + # - name: Base Setup + # uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + + # - name: Install dependencies + # run: | + # pip install "jupyterlab>=4.6.0b1,<5" + # jlpm install + + # - name: Run pre-commit + # uses: pre-commit/action@v3.0.1 + # with: + # extra_args: --all-files --hook-stage=manual + + # - name: Help message if pre-commit fail + # if: ${{ failure() }} + # run: | + # echo "You can install pre-commit hooks to automatically run formatting" + # echo "on each commit with:" + # echo " pre-commit install" + # echo "or you can run by hand on staged files with" + # echo " pre-commit run" + # echo "or after-the-fact on already committed files with" + # echo " pre-commit run --all-files --hook-stage=manual" + + # - name: Lint frontend + # run: | + # jlpm run lint:check test-js: name: Test JavaScript From ce3246e948797bd36ca1e4a7fa564cca4bd833c2 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 2 Jul 2026 16:44:14 +0200 Subject: [PATCH 09/22] again --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0487eaef..5b1c7f51 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,7 +53,7 @@ jobs: test-js: name: Test JavaScript runs-on: ubuntu-latest - needs: [pre-commit] + #needs: [pre-commit] steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: test-py: name: Test Python - needs: [pre-commit] + #needs: [pre-commit] runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: @@ -188,7 +188,7 @@ jobs: make_sdist: name: Make SDist - needs: [pre-commit] + #needs: [pre-commit] runs-on: ubuntu-latest timeout-minutes: 10 steps: From 83b31873154c6d1a5f8cc2efa72388b2d1d5311e Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 2 Jul 2026 17:19:30 +0200 Subject: [PATCH 10/22] Fix test --- tests/test_documents.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_documents.py b/tests/test_documents.py index d065d682..831f895a 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -64,6 +64,7 @@ async def test_dirty( websocket, room_name = await rtc_connect_doc_client(file_format, file_type, file_path) async with websocket as ws, Provider(jupyter_ydoc.ydoc, HttpxWebsocket(ws, room_name)): + await sleep(0.1) for _ in range(2): jupyter_ydoc.dirty = True await sleep(rtc_document_save_delay * 1.5) From 2a5ac629512f6b1d02a2b283e304a32353fdc041 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 2 Jul 2026 20:02:30 +0200 Subject: [PATCH 11/22] again --- tests/test_documents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_documents.py b/tests/test_documents.py index 831f895a..56f127e1 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -64,7 +64,7 @@ async def test_dirty( websocket, room_name = await rtc_connect_doc_client(file_format, file_type, file_path) async with websocket as ws, Provider(jupyter_ydoc.ydoc, HttpxWebsocket(ws, room_name)): - await sleep(0.1) + await sleep(0.2) for _ in range(2): jupyter_ydoc.dirty = True await sleep(rtc_document_save_delay * 1.5) From 595b6e43f6654ca9532a593bdef0a4e6139def09 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 2 Jul 2026 22:33:04 +0200 Subject: [PATCH 12/22] again --- tests/test_documents.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_documents.py b/tests/test_documents.py index 56f127e1..c4800635 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -64,11 +64,15 @@ async def test_dirty( websocket, room_name = await rtc_connect_doc_client(file_format, file_type, file_path) async with websocket as ws, Provider(jupyter_ydoc.ydoc, HttpxWebsocket(ws, room_name)): - await sleep(0.2) - for _ in range(2): - jupyter_ydoc.dirty = True - await sleep(rtc_document_save_delay * 1.5) - assert not jupyter_ydoc.dirty + # the server might not be ready to receive updates from this client yet + jupyter_ydoc.dirty = True + await sleep(rtc_document_save_delay * 1.5) + # so don't expect dirty to be cleared + + # now the server should be ready, check that dirty is cleared + jupyter_ydoc.dirty = True + await sleep(rtc_document_save_delay * 1.5) + assert not jupyter_ydoc.dirty async def cleanup(jp_serverapp): From 5ac1eac5439af29a96d6bba2c493ff87d98bdc31 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 2 Jul 2026 22:51:52 +0200 Subject: [PATCH 13/22] again --- tests/test_documents.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_documents.py b/tests/test_documents.py index c4800635..4b6e701d 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -64,15 +64,15 @@ async def test_dirty( websocket, room_name = await rtc_connect_doc_client(file_format, file_type, file_path) async with websocket as ws, Provider(jupyter_ydoc.ydoc, HttpxWebsocket(ws, room_name)): - # the server might not be ready to receive updates from this client yet - jupyter_ydoc.dirty = True - await sleep(rtc_document_save_delay * 1.5) - # so don't expect dirty to be cleared - - # now the server should be ready, check that dirty is cleared - jupyter_ydoc.dirty = True - await sleep(rtc_document_save_delay * 1.5) - assert not jupyter_ydoc.dirty + for _ in range(3): + jupyter_ydoc.dirty = True + await sleep(rtc_document_save_delay * 1.5) + # the server might not be ready to receive updates from this client yet + # if it is, it should clear the dirty state + if not jupyter_ydoc.dirty: + return + else: + assert not jupyter_ydoc.dirty async def cleanup(jp_serverapp): From 09f0c50de9cc88b7ea135e74e8372d1ea4932c28 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 3 Jul 2026 11:13:49 +0200 Subject: [PATCH 14/22] - --- projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py index 19d5989a..00e96b92 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py @@ -178,6 +178,7 @@ async def initialize(self) -> None: ) if not loaded_from_store: if self._document_load_progressively: + release_update_lock = False initialized = asyncio.Event() finish = asyncio.Event() self.create_task( @@ -189,7 +190,6 @@ async def initialize(self) -> None: self.ready = True await self.ydoc_observed.wait() finish.set() - release_update_lock = False self._emit(LogLevel.INFO, "initialize", "Room initialized") return else: From e9540187c1032286324f16119638fcc9ff3c2335 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 3 Jul 2026 14:24:36 +0200 Subject: [PATCH 15/22] Run conflict tests witout progressive loading --- .github/workflows/test.yml | 1 + ...upyter_server_test_config_noprogressive.py | 44 +++++++++++++++++++ ui-tests/package.json | 2 + ui-tests/playwright.conflict.config.js | 33 ++++++++++++++ ui-tests/tests/conflict.spec.ts | 27 ++++++++++++ 5 files changed, 107 insertions(+) create mode 100644 ui-tests/jupyter_server_test_config_noprogressive.py create mode 100644 ui-tests/playwright.conflict.config.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b1c7f51..eee1914e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -306,6 +306,7 @@ jobs: run: | jlpm test TIMELINE_FEATURE=1 jlpm test:timeline + CONFLICT_FEATURE=1 jlpm test:conflict - name: Upload Playwright Test report if: always() diff --git a/ui-tests/jupyter_server_test_config_noprogressive.py b/ui-tests/jupyter_server_test_config_noprogressive.py new file mode 100644 index 00000000..b508d225 --- /dev/null +++ b/ui-tests/jupyter_server_test_config_noprogressive.py @@ -0,0 +1,44 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +"""Server configuration for conflict-resolution integration tests. + +Progressive document loading is disabled so that +_apply_deterministic_source_content fully populates the Ydoc before +any client sync, which is needed for the stale-parent RuntimeError +("block parent") to be raised on reconnect after an out-of-band change. + +!! Never use this configuration in production because it +opens the server to the world and provide access to JupyterLab +JavaScript objects through the global window variable. +""" + +from typing import Any, cast + +from jupyterlab.galata import configure_jupyter_server + +# `c` is injected by jupyter-server while loading this config file. +c = cast(Any, globals()["c"]) +configure_jupyter_server(c) # noqa + +# Fast room eviction so conflict tests don't need to wait 60 seconds. +c.YDocExtension.document_cleanup_delay = 1 + +# Disable progressive loading – the full document content must be +# deterministically loaded before the client can sync so that stale +# parent references trigger the conflict detection path. +c.YDocExtension.document_load_progressively = False + +# Force-close dead WebSocket connections quickly. Playwright's setOffline(true) +# blocks network I/O without tearing down existing TCP connections, so pings are +# needed to make the server detect the disconnection. The conflict test goes +# offline for 10 s; with interval=2 + timeout=5 the dead connection closes ≤7 s +# after going offline, leaving enough margin before the test comes back online. +c.ServerApp.websocket_ping_interval = 2 # seconds between pings +c.ServerApp.websocket_ping_timeout = 5 # close connection if no pong within 5 s + +# Use SQLiteYStore with a predictable path so conflict tests can delete +# the database during the offline period to force room recreation via +# _apply_deterministic_source_content. This simulates the production scenario +# where a room is evicted and the notebook structure changes on disk. +c.SQLiteYStore.db_path = "/tmp/jupyter_ystore_ui_test.db" diff --git a/ui-tests/package.json b/ui-tests/package.json index 59d97de3..ff94cfb0 100644 --- a/ui-tests/package.json +++ b/ui-tests/package.json @@ -5,8 +5,10 @@ "private": true, "scripts": { "start": "jupyter lab --config jupyter_server_test_config.py", + "start:conflict": "jupyter lab --config jupyter_server_test_config_noprogressive.py", "start:timeline": "jupyter lab --config jupyter_server_test_config.py --ServerApp.base_url=/api/collaboration/timeline/", "test": "npx playwright test --config=playwright.config.js", + "test:conflict": "npx playwright test --config=playwright.conflict.config.js", "test:timeline": "npx playwright test --config=playwright.timeline.config.js", "test:update": "npx playwright test --update-snapshots" }, diff --git a/ui-tests/playwright.conflict.config.js b/ui-tests/playwright.conflict.config.js new file mode 100644 index 00000000..a1e66b85 --- /dev/null +++ b/ui-tests/playwright.conflict.config.js @@ -0,0 +1,33 @@ +/* + * Copyright (c) Jupyter Development Team. + * Distributed under the terms of the Modified BSD License. + */ + +/** + * Configuration for Playwright using default from @jupyterlab/galata + */ +const baseConfig = require('@jupyterlab/galata/lib/playwright-config'); + +module.exports = { + ...baseConfig, + workers: 1, + webServer: { + command: 'jlpm start:conflict', + url: 'http://localhost:8888/lab', + timeout: 120 * 1000, + reuseExistingServer: !process.env.CI + }, + expect: { + toMatchSnapshot: { + maxDiffPixelRatio: 0.01 + } + }, + projects: [ + { + name: 'conflict-tests', + testMatch: 'tests/conflict.spec.ts', + testIgnore: '**/.ipynb_checkpoints/**', + timeout: 120 * 1000 + } + ] +}; diff --git a/ui-tests/tests/conflict.spec.ts b/ui-tests/tests/conflict.spec.ts index 210140f3..1cd15424 100644 --- a/ui-tests/tests/conflict.spec.ts +++ b/ui-tests/tests/conflict.spec.ts @@ -5,6 +5,9 @@ import { expect, galata, test } from '@jupyterlab/galata'; import { unlink } from 'fs/promises'; import type { Page, APIRequestContext } from '@playwright/test'; +const isConflictEnv = process.env.CONFLICT_FEATURE || "0"; +const isConflict = parseInt(isConflictEnv) + /** * A minimal notebook with one cell. * @@ -158,6 +161,10 @@ test.describe.serial('Conflict handling', () => { test( 'shows a conflict dialog and dismisses it', async ({ page, request, tmpPath, baseURL }) => { + if (!isConflict) { + console.log('Skipping this test.'); + return; + } const dialog = await triggerConflict( page, request, @@ -174,6 +181,10 @@ test.describe.serial('Conflict handling', () => { test( 'Revert button reloads the document from disk', async ({ page, request, tmpPath, baseURL }) => { + if (!isConflict) { + console.log('Skipping this test.'); + return; + } const dialog = await triggerConflict( page, request, @@ -207,6 +218,10 @@ test.describe.serial('Conflict handling', () => { test( 'Save As button opens the save-as dialog', async ({ page, request, tmpPath, baseURL }) => { + if (!isConflict) { + console.log('Skipping this test.'); + return; + } const dialog = await triggerConflict( page, request, @@ -229,6 +244,10 @@ test.describe.serial('Conflict handling', () => { test( 'Show Diff button opens a diff widget', async ({ page, request, tmpPath, baseURL }) => { + if (!isConflict) { + console.log('Skipping this test.'); + return; + } const dialog = await triggerConflict( page, request, @@ -249,6 +268,10 @@ test.describe.serial('Conflict handling', () => { test( 'Save Local As button in diff toolbar opens the save-as dialog', async ({ page, request, tmpPath, baseURL }) => { + if (!isConflict) { + console.log('Skipping this test.'); + return; + } const dialog = await triggerConflict( page, request, @@ -276,6 +299,10 @@ test.describe.serial('Conflict handling', () => { test( 'Revert to Remote button in diff toolbar reloads the document', async ({ page, request, tmpPath, baseURL }) => { + if (!isConflict) { + console.log('Skipping this test.'); + return; + } const dialog = await triggerConflict( page, request, From aca18d549ebc0d1f2b12dd75edee2f3ca1bf511c Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 3 Jul 2026 15:39:33 +0200 Subject: [PATCH 16/22] - --- ui-tests/playwright.conflict.config.js | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/ui-tests/playwright.conflict.config.js b/ui-tests/playwright.conflict.config.js index a1e66b85..3815ccdb 100644 --- a/ui-tests/playwright.conflict.config.js +++ b/ui-tests/playwright.conflict.config.js @@ -11,6 +11,9 @@ const baseConfig = require('@jupyterlab/galata/lib/playwright-config'); module.exports = { ...baseConfig, workers: 1, + testMatch: 'tests/conflict.spec.ts', + testIgnore: '**/.ipynb_checkpoints/**', + timeout: 120 * 1000, webServer: { command: 'jlpm start:conflict', url: 'http://localhost:8888/lab', @@ -21,13 +24,5 @@ module.exports = { toMatchSnapshot: { maxDiffPixelRatio: 0.01 } - }, - projects: [ - { - name: 'conflict-tests', - testMatch: 'tests/conflict.spec.ts', - testIgnore: '**/.ipynb_checkpoints/**', - timeout: 120 * 1000 - } - ] + } }; From 6ddee8e580127336d1263e26d5117956fde3ab5e Mon Sep 17 00:00:00 2001 From: David Brochart Date: Mon, 6 Jul 2026 08:51:02 +0200 Subject: [PATCH 17/22] Require jupyter_ydoc >=4.1.1 --- projects/jupyter-server-ydoc/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/jupyter-server-ydoc/pyproject.toml b/projects/jupyter-server-ydoc/pyproject.toml index bfe183e9..6395a692 100644 --- a/projects/jupyter-server-ydoc/pyproject.toml +++ b/projects/jupyter-server-ydoc/pyproject.toml @@ -29,7 +29,7 @@ authors = [ ] dependencies = [ "jupyter_server>=2.15.0,<3.0.0", - "jupyter_ydoc @ git+https://github.com/davidbrochart/jupyter_ydoc.git@finish", + "jupyter_ydoc >=4.1.1,<5.0.0", "pycrdt", "pycrdt-websocket>=0.16.4,<0.17.0", "jupyter_events>=0.11.0", From 91e4434d3e372efd47160693b919cb999937e422 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Mon, 6 Jul 2026 08:53:58 +0200 Subject: [PATCH 18/22] Re-enable pre-commit --- .github/workflows/test.yml | 80 +++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eee1914e..d14cddfc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,46 +9,46 @@ on: jobs: # Run "pre-commit run --all-files --hook-stage=manual" - #pre-commit: - # name: Run pre-commit hook - # runs-on: ubuntu-latest - # timeout-minutes: 10 - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - - # - name: Set up Python - # uses: actions/setup-python@v5 - # with: - # python-version: '3.13' - - # - name: Base Setup - # uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - - # - name: Install dependencies - # run: | - # pip install "jupyterlab>=4.6.0b1,<5" - # jlpm install - - # - name: Run pre-commit - # uses: pre-commit/action@v3.0.1 - # with: - # extra_args: --all-files --hook-stage=manual - - # - name: Help message if pre-commit fail - # if: ${{ failure() }} - # run: | - # echo "You can install pre-commit hooks to automatically run formatting" - # echo "on each commit with:" - # echo " pre-commit install" - # echo "or you can run by hand on staged files with" - # echo " pre-commit run" - # echo "or after-the-fact on already committed files with" - # echo " pre-commit run --all-files --hook-stage=manual" - - # - name: Lint frontend - # run: | - # jlpm run lint:check + pre-commit: + name: Run pre-commit hook + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Base Setup + uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + + - name: Install dependencies + run: | + pip install "jupyterlab>=4.6.0b1,<5" + jlpm install + + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 + with: + extra_args: --all-files --hook-stage=manual + + - name: Help message if pre-commit fail + if: ${{ failure() }} + run: | + echo "You can install pre-commit hooks to automatically run formatting" + echo "on each commit with:" + echo " pre-commit install" + echo "or you can run by hand on staged files with" + echo " pre-commit run" + echo "or after-the-fact on already committed files with" + echo " pre-commit run --all-files --hook-stage=manual" + + - name: Lint frontend + run: | + jlpm run lint:check test-js: name: Test JavaScript From 5317f567424d514d26abb36fa7506abd88399e28 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Mon, 6 Jul 2026 08:54:43 +0200 Subject: [PATCH 19/22] again --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d14cddfc..1024fdd3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,7 +53,7 @@ jobs: test-js: name: Test JavaScript runs-on: ubuntu-latest - #needs: [pre-commit] + needs: [pre-commit] steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: test-py: name: Test Python - #needs: [pre-commit] + needs: [pre-commit] runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: @@ -188,7 +188,7 @@ jobs: make_sdist: name: Make SDist - #needs: [pre-commit] + needs: [pre-commit] runs-on: ubuntu-latest timeout-minutes: 10 steps: From 12b26cdaa7919db8123e050753536b0837705ea7 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Mon, 6 Jul 2026 09:03:33 +0200 Subject: [PATCH 20/22] Bump pycrdt-websocket v0.16.4 for mypy --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7b4fa2ea..207cdf66 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: - id: mypy exclude: "(^binder/jupyter_config\\.py$)|(^scripts/bump_version\\.py$)|(/setup\\.py$)" args: ["--config-file", "pyproject.toml"] - additional_dependencies: [tornado, pytest, 'pycrdt-websocket>=0.16.0,<0.17.0'] + additional_dependencies: [tornado, pytest, 'pycrdt-websocket>=0.16.4,<0.17.0'] stages: [manual] - repo: https://github.com/python-jsonschema/check-jsonschema From a5fd7798690c44cf1118cba5f115db2ee87fd1d7 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Tue, 7 Jul 2026 09:12:18 +0200 Subject: [PATCH 21/22] Make document progressive loading opt-in --- docs/source/configuration.md | 4 ++-- projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py | 2 +- projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py | 2 +- projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py | 2 +- tests/test_app.py | 4 ++-- tests/test_rooms.py | 2 ++ 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/source/configuration.md b/docs/source/configuration.md index baf626c3..79edaf95 100644 --- a/docs/source/configuration.md +++ b/docs/source/configuration.md @@ -52,8 +52,8 @@ jupyter lab --YDocExtension.file_poll_interval=2 # If None, the document will be kept in memory forever. jupyter lab --YDocExtension.document_cleanup_delay=100 -# Whether documents loaded from disk should be streamed progressively (default: True). -jupyter lab --YDocExtension.document_load_progressively=False +# Whether documents loaded from disk should be streamed progressively (default: False). +jupyter lab --YDocExtension.document_load_progressively=True # Delay notebook outputs larger than this size in MB during progressive loading (default: 100). # If None, outputs are loaded with the inputs. diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py index 3af7dd20..da2ee4a9 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py @@ -77,7 +77,7 @@ class YDocExtension(ExtensionApp): ) document_load_progressively = Bool( - True, + False, config=True, help="""Whether to progressively load documents from disk into the shared document. When enabled, a shared document can stream its content to clients before the full diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py index c56155e9..588e0646 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py @@ -185,7 +185,7 @@ def initialize( room_locks: dict[str, asyncio.Lock] | None = None, document_cleanup_delay: float | None = 60.0, document_save_delay: float | None = 1.0, - document_load_progressively: bool = True, + document_load_progressively: bool = False, notebook_output_delay_threshold_mb: float | None = 100, ) -> None: self._background_tasks = set() diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py index 00e96b92..ad71e65b 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py @@ -40,7 +40,7 @@ def __init__( ystore: BaseYStore | None, log: Logger | None, save_delay: float | None = None, - document_load_progressively: bool = True, + document_load_progressively: bool = False, notebook_output_delay_threshold_mb: float | None = 100, exception_handler: Callable[[Exception, Logger], bool] | None = None, ): diff --git a/tests/test_app.py b/tests/test_app.py index 3fed821c..eefa4395 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -58,12 +58,12 @@ def test_settings_should_change_save_delay(jp_configurable_serverapp): def test_settings_should_change_document_progressive_loading(jp_configurable_serverapp): - argv = ["--YDocExtension.document_load_progressively=False"] + argv = ["--YDocExtension.document_load_progressively=True"] app = jp_configurable_serverapp(argv=argv) settings = app.web_app.settings["jupyter_server_ydoc_config"] - assert settings["document_load_progressively"] is False + assert settings["document_load_progressively"] def test_settings_should_change_notebook_output_delay_threshold(jp_configurable_serverapp): diff --git a/tests/test_rooms.py b/tests/test_rooms.py index 568bb036..30fa96ad 100644 --- a/tests/test_rooms.py +++ b/tests/test_rooms.py @@ -259,6 +259,8 @@ async def test_on_outofband_change_skips_aset_when_content_unchanged( await room._on_outofband_change() mock_aset.assert_not_called() + assert not room._document.dirty + async def test_on_outofband_change_calls_aset_when_content_changed( rtc_create_mock_document_room, From 99e3a24eb5f0973887a7705a83dc08766d716e9c Mon Sep 17 00:00:00 2001 From: David Brochart Date: Tue, 7 Jul 2026 09:22:32 +0200 Subject: [PATCH 22/22] Fix test --- tests/test_app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_app.py b/tests/test_app.py index eefa4395..07a4fe2d 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -16,7 +16,7 @@ def test_default_settings(jp_serverapp): assert settings["file_poll_interval"] == 1 assert settings["document_cleanup_delay"] == 60 assert settings["document_save_delay"] == 1 - assert settings["document_load_progressively"] is True + assert settings["document_load_progressively"] is False assert settings["notebook_output_delay_threshold_mb"] == 100 assert settings["ystore_class"] == SQLiteYStore @@ -63,7 +63,7 @@ def test_settings_should_change_document_progressive_loading(jp_configurable_ser app = jp_configurable_serverapp(argv=argv) settings = app.web_app.settings["jupyter_server_ydoc_config"] - assert settings["document_load_progressively"] + assert settings["document_load_progressively"] is True def test_settings_should_change_notebook_output_delay_threshold(jp_configurable_serverapp):