From 2eac78794a461986c334f26e55728e94848aa937 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Mon, 16 May 2022 14:44:10 +0200 Subject: [PATCH] Load/auto-save document from the back-end using y-py (#12360) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Load document from the back-end using y-py * Load only documents metadata when collaborative * Delay closing the room in the backend * Update Yjs * Fix notebook ycell initialization * Watch file change in the back-end and overwrite Y document * Automatically save from the back-end * Small fixes * Use ypy-websocket's WebsocketServer * Poll for file changes for now, until watchfiles is fixed * Use ypy-websocket v0.1.2 * Remove watchfiles * Rename save_document to maybe_save_document, add collab_file_poll_interval config * Workaround ypy bug * Fix for new notebook * Use jupyter_ydoc * Rename yjs_echo_ws.py->ydoc_handler.py, YjsEchoWebSocket->YDocWebSocketHandler * Update ypy-websocket and jupyter_ydoc minimum versions * Use ypy-websocket>=0.1.6 * Update jupyter_ydoc>=0.1.4 * Move WEBSOCKET_SERVER global variable to YDocWebSocketHandler class attribute * Fix tests * Update jupyterlab/staging/package.json * Rename collab_file_poll_interval to collaborative_file_poll_interval, update extension migration documentation * Set room name as file_type:file_name * Don't save file if already on disk * Pin jupyter_ydoc>=0.1.5 * Set room name as format:type:path * Disable save button * Show caption only in collaborative mode * Sync file attributes with room name * Clear dirty flag when opening document * Pin jupyter_ydoc>=0.1.7 which observes the dirty flag * Don't save when dirty flag cleared * Moves nbformat and nbformat_minor to ymeta, changes the YNotebook eve… (#2) * Moves nbformat and nbformat_minor to ymeta, changes the YNotebook event to support the new nbformat, and adds a local dirty property * Pin jupyter_ydoc>=0.1.8 * Adds a local dirty property in the DocumentModel (#3) * Removes the initialization of the dirty property from the frontend (#4) * Removes the initialization of the dirty property from the frontend * Remove setting dirty in the SharedDocument Co-authored-by: hbcarlos Co-authored-by: Frédéric Collonval --- dev_mode/package.json | 2 +- docs/source/extension/extension_migration.rst | 2 +- jupyterlab/handlers/ydoc.py | 58 ------ jupyterlab/handlers/ydoc_handler.py | 196 ++++++++++++++++++ jupyterlab/handlers/yjs_echo_ws.py | 194 ----------------- jupyterlab/labapp.py | 45 ++-- packages/docmanager-extension/src/index.tsx | 14 +- packages/docprovider/package.json | 3 +- packages/docprovider/src/mock.ts | 12 -- packages/docprovider/src/tokens.ts | 11 +- packages/docprovider/src/yprovider.ts | 92 ++------ packages/docregistry/package.json | 2 +- packages/docregistry/src/context.ts | 88 ++++++-- packages/docregistry/src/default.ts | 17 +- packages/notebook/src/model.ts | 32 ++- packages/shared-models/package.json | 2 +- packages/shared-models/src/api.ts | 10 +- packages/shared-models/src/ymodels.ts | 48 +++-- setup.cfg | 6 +- yarn.lock | 17 +- 20 files changed, 390 insertions(+), 461 deletions(-) delete mode 100644 jupyterlab/handlers/ydoc.py create mode 100644 jupyterlab/handlers/ydoc_handler.py delete mode 100644 jupyterlab/handlers/yjs_echo_ws.py diff --git a/dev_mode/package.json b/dev_mode/package.json index 2121049bbd80..64591cc5610f 100644 --- a/dev_mode/package.json +++ b/dev_mode/package.json @@ -126,7 +126,7 @@ "@lumino/widgets": "^1.31.1", "react": "^17.0.1", "react-dom": "^17.0.1", - "yjs": "^13.5.17" + "yjs": "^13.5.34" }, "dependencies": { "@jupyterlab/application": "~4.0.0-alpha.9", diff --git a/docs/source/extension/extension_migration.rst b/docs/source/extension/extension_migration.rst index 2b93adc93163..c849a82845eb 100644 --- a/docs/source/extension/extension_migration.rst +++ b/docs/source/extension/extension_migration.rst @@ -45,7 +45,7 @@ bumped their major version (following semver convention). We want to point out p in particular those with the strict null checks enabled. - ``@jupyterlab/docprovider`` from 3.x to 4.x ``WebSocketProviderWithLocks`` has been renamed to ``WebSocketProvider``. - ``acquireLock`` and ``releaseLock`` have been removed from ``IDocumentProvider``. + ``acquireLock``, ``releaseLock``, ``requestInitialContent`` and ``putInitializedState`` have been removed from ``IDocumentProvider``. ``renameAck`` is not optional anymore in ``IDocumentProvider``. - ``@jupyterlab/documentsearch`` from 3.x to 4.x ``@jupyterlab/documentsearch:plugin`` renamed ``@jupyterlab/documentsearch-extension:plugin`` diff --git a/jupyterlab/handlers/ydoc.py b/jupyterlab/handlers/ydoc.py deleted file mode 100644 index 392343df2b68..000000000000 --- a/jupyterlab/handlers/ydoc.py +++ /dev/null @@ -1,58 +0,0 @@ -import y_py as Y - - -class YBaseDoc: - def __init__(self): - self._ydoc = Y.YDoc() - - @property - def ydoc(self): - return self._ydoc - - @property - def source(self): - raise RuntimeError("Y document source generation not implemented") - - -class YFile(YBaseDoc): - def __init__(self): - super().__init__() - self._ysource = self._ydoc.get_text("source") - - @property - def source(self): - return str(self._ysource) - - -class YNotebook(YBaseDoc): - def __init__(self): - super().__init__() - self._ycells = self._ydoc.get_array("cells") - self._ymeta = self._ydoc.get_map("meta") - self._ystate = self._ydoc.get_map("state") - - @property - def source(self): - cells = self._ycells.to_json() - meta = self._ymeta.to_json() - state = self._ystate.to_json() - for cell in cells: - if "id" in cell and state["nbformat"] == 4 and state["nbformatMinor"] <= 4: - # strip cell ids if we have notebook format 4.0-4.4 - del cell["id"] - if "execution_count" in cell: - execution_count = cell["execution_count"] - if isinstance(execution_count, float): - cell["execution_count"] = int(execution_count) - if "outputs" in cell: - for output in cell["outputs"]: - if "execution_count" in output: - execution_count = output["execution_count"] - if isinstance(execution_count, float): - output["execution_count"] = int(execution_count) - return dict( - cells=cells, - metadata=meta["metadata"], - nbformat=int(state["nbformat"]), - nbformat_minor=int(state["nbformatMinor"]), - ) diff --git a/jupyterlab/handlers/ydoc_handler.py b/jupyterlab/handlers/ydoc_handler.py new file mode 100644 index 000000000000..12c1159334a3 --- /dev/null +++ b/jupyterlab/handlers/ydoc_handler.py @@ -0,0 +1,196 @@ +"""Echo WebSocket handler for real time collaboration with Yjs""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +import asyncio +from typing import Optional, Tuple + +from jupyter_server.base.handlers import JupyterHandler +from jupyter_server.utils import ensure_async +from jupyter_ydoc import ydocs as YDOCS +from tornado import web +from tornado.websocket import WebSocketHandler +from ypy_websocket.websocket_server import WebsocketServer, YRoom + +YFILE = YDOCS["file"] + +RENAME_SESSION = 127 + + +class JupyterRoom(YRoom): + def __init__(self, type): + super().__init__(ready=False) + self.type = type + self.cleaner = None + self.watcher = None + self.document = YDOCS.get(type, YFILE)(self.ydoc) + + +class JupyterWebsocketServer(WebsocketServer): + def get_room(self, path: str) -> JupyterRoom: + file_format, file_type, file_path = path.split(":", 2) + if path not in self.rooms.keys(): + self.rooms[path] = JupyterRoom(file_type) + return self.rooms[path] + + +class YDocWebSocketHandler(WebSocketHandler, JupyterHandler): + + saving_document: Optional[asyncio.Task] + websocket_server = JupyterWebsocketServer(rooms_ready=False, auto_clean_rooms=False) + + # Override max_message size to 1GB + @property + def max_message_size(self): + return 1024 * 1024 * 1024 + + def __aiter__(self): + # needed to be compatible with WebsocketServer (async for message in websocket) + return self + + async def __anext__(self): + # needed to be compatible with WebsocketServer (async for message in websocket) + message = await self._message_queue.get() + if not message: + raise StopAsyncIteration() + return message + + def get_file_info(self) -> Tuple[str]: + room_name = self.websocket_server.get_room_name(self.room) + file_format, file_type, file_path = room_name.split(":", 2) + return file_format, file_type, file_path + + def set_file_info(self, value: str) -> None: + self.websocket_server.rename_room(value, from_room=self.room) + self.path = value # needed to be compatible with WebsocketServer (websocket.path) + + async def get(self, *args, **kwargs): + if self.get_current_user() is None: + self.log.warning("Couldn't authenticate WebSocket connection") + raise web.HTTPError(403) + return await super().get(*args, **kwargs) + + async def open(self, path): + self._message_queue = asyncio.Queue() + self.room = self.websocket_server.get_room(path) + self.set_file_info(path) + self.saving_document = None + asyncio.create_task(self.websocket_server.serve(self)) + + # cancel the deletion of the room if it was scheduled + if self.room.cleaner is not None: + self.room.cleaner.cancel() + + if not self.room.ready: + file_format, file_type, file_path = self.get_file_info() + model = await ensure_async( + self.contents_manager.get(file_path, type=file_type, format=file_format) + ) + self.last_modified = model["last_modified"] + # check again if ready, because loading the file can be async + if not self.room.ready: + self.room.document.source = model["content"] + self.room.document.dirty = False + self.room.ready = True + self.room.watcher = asyncio.create_task(self.watch_file()) + # save the document when changed + self.room.document.observe(self.on_document_change) + + async def watch_file(self): + poll_interval = self.settings["collaborative_file_poll_interval"] + if not poll_interval: + self.room.watcher = None + return + while True: + await asyncio.sleep(poll_interval) + await self.maybe_load_document() + + async def maybe_load_document(self): + file_format, file_type, file_path = self.get_file_info() + model = await ensure_async( + self.contents_manager.get(file_path, content=False, type=file_type, format=file_format) + ) + # do nothing if the file was saved by us + if self.last_modified < model["last_modified"]: + model = await ensure_async( + self.contents_manager.get(file_path, type=file_type, format=file_format) + ) + self.room.document.source = model["content"] + self.last_modified = model["last_modified"] + + async def send(self, message): + # needed to be compatible with WebsocketServer (websocket.send) + self.write_message(message, binary=True) + + async def recv(self): + message = await self._message_queue.get() + return message + + def on_message(self, message): + self._message_queue.put_nowait(message) + if message[0] == RENAME_SESSION: + # The client moved the document to a different location. After receiving this message, we make the current document available under a different url. + # The other clients are automatically notified of this change because the path is shared through the Yjs document as well. + self.set_file_info(message[1:].decode("utf-8")) + self.websocket_server.rename_room(self.path, from_room=self.room) + # send rename acknowledge + self.write_message(bytes([RENAME_SESSION, 1]), binary=True) + + def on_close(self) -> bool: + # stop serving this client + self._message_queue.put_nowait(b"") + if not self.room.clients: + # keep the document for a while after every client disconnects + self.room.cleaner = asyncio.create_task(self.clean_room()) + return True + + async def clean_room(self) -> None: + await asyncio.sleep(60) + if self.room.watcher: + self.room.watcher.cancel() + self.room.document.unobserve() + self.websocket_server.delete_room(room=self.room) + + def on_document_change(self, event): + try: + dirty = event.keys["dirty"]["newValue"] + if not dirty: + # we cleared the dirty flag, nothing to save + return + except Exception: + pass + # unobserve and observe again because the structure of the document may have changed + # e.g. a new cell added to a notebook + self.room.document.unobserve() + self.room.document.observe(self.on_document_change) + if self.saving_document is not None and not self.saving_document.done(): + # the document is being saved, cancel that + self.saving_document.cancel() + self.saving_document = None + self.saving_document = asyncio.create_task(self.maybe_save_document()) + + async def maybe_save_document(self): + # save after 1 second of inactivity to prevent too frequent saving + await asyncio.sleep(1) + file_format, file_type, file_path = self.get_file_info() + model = await ensure_async( + self.contents_manager.get(file_path, type=file_type, format=file_format) + ) + if self.last_modified < model["last_modified"]: + # file changed on disk, let's revert + self.room.document.source = model["content"] + self.last_modified = model["last_modified"] + return + if model["content"] != self.room.document.source: + # don't save if not needed + # this also prevents the dirty flag from bouncing between windows of + # the same document opened as different types (e.g. notebook/text editor) + model["format"] = file_format + model["content"] = self.room.document.source + model = await ensure_async(self.contents_manager.save(model, file_path)) + self.last_modified = model["last_modified"] + self.room.document.dirty = False + + def check_origin(self, origin) -> bool: + return True diff --git a/jupyterlab/handlers/yjs_echo_ws.py b/jupyterlab/handlers/yjs_echo_ws.py deleted file mode 100644 index 9d7556ad7938..000000000000 --- a/jupyterlab/handlers/yjs_echo_ws.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Echo WebSocket handler for real time collaboration with Yjs""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import sys -import uuid -from enum import IntEnum - -import y_py as Y -from jupyter_server.base.handlers import JupyterHandler -from tornado import web -from tornado.ioloop import IOLoop -from tornado.websocket import WebSocketHandler - -# See compatibility note on `group` keyword in https://docs.python.org/3/library/importlib.metadata.html#entry-points -if sys.version_info < (3, 10): - from importlib_metadata import entry_points -else: - from importlib.metadata import entry_points - -YDOCS = {} -for ep in entry_points(group="jupyter_ydoc"): - YDOCS.update({ep.name: ep.load()}) - -YFILE = YDOCS["file"] - -ROOMS = {} - -# The y-protocol defines messages types that just need to be propagated to all other peers. -# Here, we define some additional messageTypes that the server can interpret. -# Messages that the server can't interpret should be broadcasted to all other clients. - - -class ServerMessageType(IntEnum): - # The client is asking to retrieve the initial state of the Yjs document. Return an empty buffer when nothing is available. - REQUEST_INITIALIZED_CONTENT = 127 - # The client retrieved an empty "initial content" and generated the initial state of the document. Store this. - PUT_INITIALIZED_CONTENT = 126 - # The client moved the document to a different location. After receiving this message, we make the current document available under a different url. - # The other clients are automatically notified of this change because the path is shared through the Yjs document as well. - RENAME_SESSION = 125 - - -class YjsRoom: - def __init__(self, type): - self.type = type - self.clients = {} - self.content = bytes([]) - self.ydoc = YDOCS.get(type, YFILE)() - - def get_source(self): - return self.ydoc.source - - -class YjsEchoWebSocket(WebSocketHandler, JupyterHandler): - - # Override max_message size to 1GB - @property - def max_message_size(self): - return 1024 * 1024 * 1024 - - async def get(self, *args, **kwargs): - if self.get_current_user() is None: - self.log.warning("Couldn't authenticate WebSocket connection") - raise web.HTTPError(403) - return await super().get(*args, **kwargs) - - def open(self, type_path): - # print("[YJSEchoWS]: open", type_path) - type, path = type_path.split(":", 1) - self.id = str(uuid.uuid4()) - self.room_id = path - room = ROOMS.get(self.room_id) - if room is None: - room = YjsRoom(type) - ROOMS[self.room_id] = room - room.clients[self.id] = (IOLoop.current(), self.hook_send_message, self) - # Send SyncStep1 message (based on y-protocols) - self.write_message(bytes([0, 0, 1, 0]), binary=True) - - def on_message(self, message): - # print("[YJSEchoWS]: message,", message) - room_id = self.room_id - room = ROOMS.get(room_id) - if message[0] == ServerMessageType.REQUEST_INITIALIZED_CONTENT: - # print("client requested initial content") - self.write_message( - bytes([ServerMessageType.REQUEST_INITIALIZED_CONTENT]) + room.content, binary=True - ) - elif message[0] == ServerMessageType.PUT_INITIALIZED_CONTENT: - # print("client put initialized content") - room.content = message[1:] - elif message[0] == ServerMessageType.RENAME_SESSION: - # We move the room to a different entry and also change the room_id property of each connected client - new_room_id = message[1:].decode("utf-8").split(":", 1)[1] - for _, (_, _, client) in room.clients.items(): - client.room_id = new_room_id - ROOMS.pop(room_id) - ROOMS[new_room_id] = room - # send rename acknowledge - self.write_message(bytes([ServerMessageType.RENAME_SESSION, 1]), binary=True) - # print("renamed room to " + new_room_id + ". Old room name was " + room_id) - elif room: - if message[0] == 0: # sync message - read_sync_message(self, room.ydoc.ydoc, message[1:]) - for client_id, (loop, hook_send_message, _) in room.clients.items(): - if self.id != client_id: - loop.add_callback(hook_send_message, message) - - def on_close(self): - # print("[YJSEchoWS]: close") - room = ROOMS.get(self.room_id) - room.clients.pop(self.id) - if not room.clients: - ROOMS.pop(self.room_id) - # print("[YJSEchoWS]: close room " + self.room_id) - - return True - - def check_origin(self, origin): - # print("[YJSEchoWS]: check origin") - return True - - def hook_send_message(self, msg): - self.write_message(msg, binary=True) - - -message_yjs_sync_step1 = 0 -message_yjs_sync_step2 = 1 -message_yjs_update = 2 - - -def read_sync_step1(handler, doc, encoded_state_vector): - message = Y.encode_state_as_update(doc, encoded_state_vector) - message = bytes([0, message_yjs_sync_step2] + write_var_uint(len(message)) + message) - handler.write_message(message, binary=True) - - -def read_sync_step2(doc, update): - try: - Y.apply_update(doc, update) - except Exception: - raise RuntimeError("Caught error while handling a Y update") - - -def read_sync_message(handler, doc, message): - message_type = message[0] - message = message[1:] - if message_type == message_yjs_sync_step1: - for msg in get_message(message): - read_sync_step1(handler, doc, msg) - elif message_type == message_yjs_sync_step2: - for msg in get_message(message): - read_sync_step2(doc, msg) - elif message_type == message_yjs_update: - for msg in get_message(message): - read_sync_step2(doc, msg) - else: - raise RuntimeError("Unknown message type") - - -def write_var_uint(num): - res = [] - while num > 127: - res += [128 | (127 & num)] - num >>= 7 - res += [num] - return res - - -def get_message(message): - length = len(message) - i0 = 0 - while True: - msg_len = 0 - i = 0 - while True: - byte = message[i0] - msg_len += (byte & 127) << i - i += 7 - i0 += 1 - length -= 1 - if byte < 128: - break - i1 = i0 + msg_len - msg = message[i0:i1] - length -= msg_len - yield msg - if length <= 0: - if length < 0: - raise RuntimeError("Y protocol error") - break - i0 = i1 diff --git a/jupyterlab/labapp.py b/jupyterlab/labapp.py index d2fd5ce97c0e..07b9bed13d88 100644 --- a/jupyterlab/labapp.py +++ b/jupyterlab/labapp.py @@ -19,8 +19,7 @@ WorkspaceListApp, ) from notebook_shim.shim import NotebookConfigShimMixin -from tornado import web -from traitlets import Bool, Instance, Unicode, default +from traitlets import Bool, Instance, Int, Unicode, default from ._version import __version__ from .commands import ( @@ -49,7 +48,7 @@ ExtensionManager, extensions_handler_path, ) -from .handlers.yjs_echo_ws import ROOMS, YjsEchoWebSocket +from .handlers.ydoc_handler import YDocWebSocketHandler DEV_NOTE = """You're running JupyterLab from source. If you're working on the TypeScript sources of JupyterLab, try running @@ -538,6 +537,14 @@ class LabApp(NotebookConfigShimMixin, LabServerApp): collaborative = Bool(False, config=True, help="Whether to enable collaborative mode.") + collaborative_file_poll_interval = Int( + 1, + config=True, + help="""The period in seconds to check for file changes in the back-end (relevant only when + in collaborative mode). Defaults to 1s, if 0 then file changes will only be checked when + saving changes from the front-end.""", + ) + @default("app_dir") def _default_app_dir(self): app_dir = get_app_dir() @@ -627,33 +634,6 @@ def initialize_templates(self): self.static_paths = [self.static_dir] self.template_paths = [self.templates_dir] - def initialize_settings(self): - def hook(model, path, **kwargs): - if "type" in model and model["type"] == "directory": - pass - elif "content" in model and model["content"] is not None: - # content sent through HTTP, it must not be an RTC session - if path in ROOMS: - raise web.HTTPError( - 409, - "Document content cannot be present both in RTC session and HTTP request", - ) - # keep the content sent through HTTP - else: - # no content sent through HTTP, it must be an RTC session - if path in ROOMS: - # we found the RTC session as expected - # set the document content from y-py - model["content"] = ROOMS[path].get_source() - else: - # RTC session not available, shouldn't happen - raise web.HTTPError( - 410, "Could not find an RTC session corresponding to this document" - ) - - self.serverapp.contents_manager.register_pre_save_hook(hook) - super().initialize_settings() - def initialize_handlers(self): handlers = [] @@ -686,7 +666,7 @@ def initialize_handlers(self): handlers.append(build_handler) # Yjs Echo WebSocket handler - yjs_echo_handler = (r"/api/yjs/(.*)", YjsEchoWebSocket) + yjs_echo_handler = (r"/api/yjs/(.*)", YDocWebSocketHandler) handlers.append(yjs_echo_handler) errored = False @@ -738,6 +718,9 @@ def initialize_handlers(self): # Update Jupyter Server's webapp settings with jupyterlab settings. self.serverapp.web_app.settings["page_config_data"] = page_config + self.serverapp.web_app.settings[ + "collaborative_file_poll_interval" + ] = self.collaborative_file_poll_interval # Extend Server handlers with jupyterlab handlers. self.handlers.extend(handlers) diff --git a/packages/docmanager-extension/src/index.tsx b/packages/docmanager-extension/src/index.tsx index d582f30305bc..528316c45d34 100644 --- a/packages/docmanager-extension/src/index.tsx +++ b/packages/docmanager-extension/src/index.tsx @@ -24,7 +24,7 @@ import { showErrorMessage, UseSignal } from '@jupyterlab/apputils'; -import { IChangedArgs, Time } from '@jupyterlab/coreutils'; +import { IChangedArgs, PageConfig, Time } from '@jupyterlab/coreutils'; import { DocumentManager, IDocumentManager, @@ -738,9 +738,19 @@ function addCommands( } }); + const caption = () => { + if (PageConfig.getOption('collaborative') == 'true') { + return trans.__( + 'In collaborative mode, the document is saved automatically after every change' + ); + } else { + return trans.__('Save and create checkpoint'); + } + }; + commands.addCommand(CommandIDs.save, { label: () => trans.__('Save %1', fileType(shell.currentWidget, docManager)), - caption: trans.__('Save and create checkpoint'), + caption, icon: args => (args.toolbar ? saveIcon : ''), isEnabled: isWritable, execute: () => { diff --git a/packages/docprovider/package.json b/packages/docprovider/package.json index 28fde201f2e1..e8392fed268f 100644 --- a/packages/docprovider/package.json +++ b/packages/docprovider/package.json @@ -43,8 +43,7 @@ "@jupyterlab/user": "^4.0.0-alpha.9", "@lumino/coreutils": "^1.12.0", "lib0": "^0.2.42", - "y-websocket": "^1.3.15", - "yjs": "^13.5.17" + "y-websocket": "^1.3.15" }, "devDependencies": { "@jupyterlab/testutils": "^4.0.0-alpha.9", diff --git a/packages/docprovider/src/mock.ts b/packages/docprovider/src/mock.ts index 87c925d3e566..b14503abf8c1 100644 --- a/packages/docprovider/src/mock.ts +++ b/packages/docprovider/src/mock.ts @@ -1,18 +1,6 @@ import { IDocumentProvider } from './index'; export class ProviderMock implements IDocumentProvider { - requestInitialContent(): Promise { - return Promise.resolve(false); - } - putInitializedState(): void { - /* nop */ - } - acquireLock(): Promise { - return Promise.resolve(0); - } - releaseLock(lock: number): void { - /* nop */ - } destroy(): void { /* nop */ } diff --git a/packages/docprovider/src/tokens.ts b/packages/docprovider/src/tokens.ts index 0d4eb1fd1391..0c8aa67c4b9f 100644 --- a/packages/docprovider/src/tokens.ts +++ b/packages/docprovider/src/tokens.ts @@ -12,16 +12,6 @@ export const IDocumentProviderFactory = new Token( * An interface for a document provider. */ export interface IDocumentProvider { - /** - * Resolves to true if the initial content has been initialized on the server. false otherwise. - */ - requestInitialContent(): Promise; - - /** - * Put the initialized state. - */ - putInitializedState(): void; - /** * Returns a Promise that resolves when renaming is ackownledged. */ @@ -58,6 +48,7 @@ export namespace IDocumentProviderFactory { */ path: string; contentType: string; + format: string; /** * The YNotebook. diff --git a/packages/docprovider/src/yprovider.ts b/packages/docprovider/src/yprovider.ts index 481bb752455d..a7b733d30a09 100644 --- a/packages/docprovider/src/yprovider.ts +++ b/packages/docprovider/src/yprovider.ts @@ -8,7 +8,6 @@ import { PromiseDelegate } from '@lumino/coreutils'; import * as decoding from 'lib0/decoding'; import * as encoding from 'lib0/encoding'; import { WebsocketProvider as YWebsocketProvider } from 'y-websocket'; -import * as Y from 'yjs'; import { IDocumentProvider, IDocumentProviderFactory } from './tokens'; /** @@ -32,7 +31,7 @@ export class WebSocketProvider constructor(options: WebSocketProvider.IOptions) { super( options.url, - options.contentType + ':' + options.path, + options.format + ':' + options.contentType + ':' + options.path, options.ymodel.ydoc, { awareness: options.ymodel.awareness @@ -40,30 +39,11 @@ export class WebSocketProvider ); this._path = options.path; this._contentType = options.contentType; + this._format = options.format; this._serverUrl = options.url; - // Message handler that receives the initial content - this.messageHandlers[127] = ( - encoder, - decoder, - provider, - emitSynced, - messageType - ) => { - // received initial content - const initialContent = decoding.readTailAsUint8Array(decoder); - // Apply data from server - if (initialContent.byteLength > 0) { - Y.applyUpdate(this.doc, initialContent); - } - const initialContentRequest = this._initialContentRequest; - this._initialContentRequest = null; - if (initialContentRequest) { - initialContentRequest.resolve(initialContent.byteLength > 0); - } - }; // Message handler that receives the rename acknowledge - this.messageHandlers[125] = ( + this.messageHandlers[127] = ( encoder, decoder, provider, @@ -74,9 +54,6 @@ export class WebSocketProvider decoding.readTailAsUint8Array(decoder)[0] ? true : false ); }; - this._isInitialized = false; - this._onConnectionStatus = this._onConnectionStatus.bind(this); - this.on('status', this._onConnectionStatus); const awareness = options.ymodel.awareness; const user = options.user; @@ -100,10 +77,12 @@ export class WebSocketProvider this._path = newPath; const encoder = encoding.createEncoder(); this._renameAck = new PromiseDelegate(); - encoding.write(encoder, 125); + encoding.write(encoder, 127); // writing a utf8 string to the encoder const escapedPath = unescape( - encodeURIComponent(this._contentType + ':' + newPath) + encodeURIComponent( + this._format + ':' + this._contentType + ':' + newPath + ) ); for (let i = 0; i < escapedPath.length; i++) { encoding.write( @@ -116,42 +95,18 @@ export class WebSocketProvider this.disconnectBc(); // The next time the provider connects, we should connect through a different server url this.bcChannel = - this._serverUrl + '/' + this._contentType + ':' + this._path; + this._serverUrl + + '/' + + this._format + + ':' + + this._contentType + + ':' + + this._path; this.url = this.bcChannel; this.connectBc(); } } - /** - * Resolves to true if the initial content has been initialized on the server. false otherwise. - */ - requestInitialContent(): Promise { - if (this._initialContentRequest) { - return this._initialContentRequest.promise; - } - - this._initialContentRequest = new PromiseDelegate(); - this._sendMessage(new Uint8Array([127])); - - // Resolve with true if the server doesn't respond for some reason. - // In case of a connection problem, we don't want the user to re-initialize the window. - // Instead wait for y-websocket to connect to the server. - // @todo maybe we should reload instead.. - setTimeout(() => this._initialContentRequest?.resolve(false), 1000); - return this._initialContentRequest.promise; - } - - /** - * Put the initialized state. - */ - putInitializedState(): void { - const encoder = encoding.createEncoder(); - encoding.writeVarUint(encoder, 126); - encoding.writeUint8Array(encoder, Y.encodeStateAsUpdate(this.doc)); - this._sendMessage(encoding.toUint8Array(encoder)); - this._isInitialized = true; - } - /** * Send a new message to WebSocket server. * @@ -171,27 +126,10 @@ export class WebSocketProvider send(); } - /** - * Handle a change to the connection status. - * - * @param status The connection status. - */ - private async _onConnectionStatus(status: { - status: 'connected' | 'disconnected'; - }): Promise { - if (this._isInitialized && status.status === 'connected') { - const contentIsInitialized = await this.requestInitialContent(); - if (!contentIsInitialized) { - this.putInitializedState(); - } - } - } - private _path: string; private _contentType: string; + private _format: string; private _serverUrl: string; - private _isInitialized: boolean; - private _initialContentRequest: PromiseDelegate | null = null; private _renameAck: PromiseDelegate; } diff --git a/packages/docregistry/package.json b/packages/docregistry/package.json index 9b9a06c137b8..0003351c0758 100644 --- a/packages/docregistry/package.json +++ b/packages/docregistry/package.json @@ -59,7 +59,7 @@ "@lumino/messaging": "^1.10.1", "@lumino/signaling": "^1.10.1", "@lumino/widgets": "^1.31.1", - "yjs": "^13.5.17" + "yjs": "^13.5.34" }, "devDependencies": { "@jupyterlab/testutils": "^4.0.0-alpha.9", diff --git a/packages/docregistry/src/context.ts b/packages/docregistry/src/context.ts index 96885f9981fb..e27835a7632d 100644 --- a/packages/docregistry/src/context.ts +++ b/packages/docregistry/src/context.ts @@ -78,6 +78,7 @@ export class Context< ? docProviderFactory({ path: this._path, contentType: this._factory.contentType, + format: this._factory.fileFormat!, ymodel }) : new ProviderMock(); @@ -261,18 +262,19 @@ export class Context< * @returns a promise that resolves upon initialization. */ async initialize(isNew: boolean): Promise { - const contentIsInitialized = await this._provider.requestInitialContent(); let promise; - if (isNew || contentIsInitialized) { - promise = this._save(); + if (PageConfig.getOption('collaborative') == 'true') { + promise = this._loadContext(); } else { - promise = this._revert(); + if (isNew) { + promise = this._save(); + } else { + promise = this._revert(); + } + promise = promise.then(() => { + this._model.initialize(); + }); } - // if save/revert completed successfully, we set the initialized content in the rtc server. - promise = promise.then(() => { - this._provider.putInitializedState(); - this._model.initialize(); - }); return promise; } @@ -296,10 +298,6 @@ export class Context< await this.ready; let promise: Promise; promise = this._save(); - // if save completed successfully, we set the initialized content in the rtc server. - promise = promise.then(() => { - this._provider.putInitializedState(); - }); return await promise; } @@ -498,12 +496,14 @@ export class Context< * Update our contents model, without the content. */ private _updateContentsModel(model: Contents.IModel): void { + const writable = + model.writable && PageConfig.getOption('collaborative') != 'true'; const newModel: Contents.IModel = { path: model.path, name: model.name, type: model.type, content: undefined, - writable: model.writable, + writable, created: model.created, last_modified: model.last_modified, mimetype: model.mimetype, @@ -572,17 +572,20 @@ export class Context< * Save the document contents to disk. */ private async _save(): Promise { + // if collaborative mode is enabled, saving happens in the back-end + // after each change to the document + if (PageConfig.getOption('collaborative') === 'true') { + return; + } this._saveState.emit('started'); const model = this._model; let content: PartialJSONValue = null; - if (PageConfig.getOption('collaborative') !== 'true') { - if (this._factory.fileFormat === 'json') { - content = model.toJSON(); - } else { - content = model.toString(); - if (this._lineEnding) { - content = content.replace(/\n/g, this._lineEnding); - } + if (this._factory.fileFormat === 'json') { + content = model.toJSON(); + } else { + content = model.toString(); + if (this._lineEnding) { + content = content.replace(/\n/g, this._lineEnding); } } @@ -637,6 +640,47 @@ export class Context< } } + /** + * Load the metadata of the document without the content. + */ + private _loadContext(): Promise { + const opts: Contents.IFetchOptions = { + type: this._factory.contentType, + content: false, + ...(this._factory.fileFormat !== null + ? { format: this._factory.fileFormat } + : {}) + }; + const path = this._path; + return this._manager.ready + .then(() => { + return this._manager.contents.get(path, opts); + }) + .then(contents => { + if (this.isDisposed) { + return; + } + const model = { + ...contents, + format: this._factory.fileFormat + }; + this._updateContentsModel(model); + this._model.dirty = false; + if (!this._isPopulated) { + return this._populate(); + } + }) + .catch(async err => { + const localPath = this._manager.contents.localPath(this._path); + const name = PathExt.basename(localPath); + void this._handleError( + err, + this._trans.__('File Load Error for %1', name) + ); + throw err; + }); + } + /** * Revert the document contents to disk contents. * diff --git a/packages/docregistry/src/default.ts b/packages/docregistry/src/default.ts index 88f4c6e3a72d..6f8f00af1acf 100644 --- a/packages/docregistry/src/default.ts +++ b/packages/docregistry/src/default.ts @@ -30,7 +30,6 @@ export class DocumentModel this.switchSharedModel(filemodel, true); this.value.changed.connect(this.triggerContentChange, this); - (this.sharedModel as models.YFile).dirty = false; this.sharedModel.changed.connect(this._onStateChanged, this); } @@ -52,13 +51,19 @@ export class DocumentModel * The dirty state of the document. */ get dirty(): boolean { - return this.sharedModel.dirty; + return this._dirty; } set dirty(newValue: boolean) { - if (newValue === this.dirty) { + const oldValue = this._dirty; + if (newValue === oldValue) { return; } - (this.sharedModel as models.YFile).dirty = newValue; + this._dirty = newValue; + this.triggerStateChange({ + name: 'dirty', + oldValue, + newValue + }); } /** @@ -158,7 +163,8 @@ export class DocumentModel ): void { if (changes.stateChange) { changes.stateChange.forEach(value => { - if (value.name !== 'dirty' || value.oldValue !== value.newValue) { + if (value.name !== 'dirty' || this._dirty !== value.newValue) { + this._dirty = value.newValue; this.triggerStateChange(value); } }); @@ -170,6 +176,7 @@ export class DocumentModel */ readonly sharedModel: models.ISharedFile; private _defaultLang = ''; + private _dirty = false; private _readOnly = false; private _contentChanged = new Signal(this); private _stateChanged = new Signal>(this); diff --git a/packages/notebook/src/model.ts b/packages/notebook/src/model.ts index 71aca4b2c346..7e33a2f4360d 100644 --- a/packages/notebook/src/model.ts +++ b/packages/notebook/src/model.ts @@ -132,13 +132,19 @@ export class NotebookModel implements INotebookModel { * The dirty state of the document. */ get dirty(): boolean { - return this.sharedModel.dirty; + return this._dirty; } set dirty(newValue: boolean) { - if (newValue === this.dirty) { + const oldValue = this._dirty; + if (newValue === oldValue) { return; } - (this.sharedModel as models.YNotebook).dirty = newValue; + this._dirty = newValue; + this.triggerStateChange({ + name: 'dirty', + oldValue, + newValue + }); } /** @@ -418,18 +424,23 @@ close the notebook without saving it.`, ): void { if (changes.stateChange) { changes.stateChange.forEach(value => { - if (value.name === 'nbformat') { - this._nbformat = value.newValue; - } - if (value.name === 'nbformatMinor') { - this._nbformatMinor = value.newValue; - } - if (value.name !== 'dirty' || value.oldValue !== value.newValue) { + if (value.name !== 'dirty' || this._dirty !== value.newValue) { + this._dirty = value.newValue; this.triggerStateChange(value); } }); } + if (changes.nbformatChanged) { + const change = changes.nbformatChanged; + if (change.key === 'nbformat' && change.newValue !== undefined) { + this._nbformat = change.newValue; + } + if (change.key === 'nbformat_minor' && change.newValue !== undefined) { + this._nbformatMinor = change.newValue; + } + } + if (changes.metadataChange) { const metadata = changes.metadataChange.newValue as JSONObject; this._modelDBMutex(() => { @@ -506,6 +517,7 @@ close the notebook without saving it.`, */ readonly modelDB: IModelDB; + private _dirty = false; private _readOnly = false; private _contentChanged = new Signal(this); private _stateChanged = new Signal>(this); diff --git a/packages/shared-models/package.json b/packages/shared-models/package.json index f4269aab1490..bd06f60ebdfe 100644 --- a/packages/shared-models/package.json +++ b/packages/shared-models/package.json @@ -42,7 +42,7 @@ "@lumino/disposable": "^1.10.1", "@lumino/signaling": "^1.10.1", "y-protocols": "^1.0.5", - "yjs": "^13.5.17" + "yjs": "^13.5.34" }, "devDependencies": { "@jupyterlab/testutils": "^4.0.0-alpha.9", diff --git a/packages/shared-models/src/api.ts b/packages/shared-models/src/api.ts index 681a63f835b1..8dbf34a492b8 100644 --- a/packages/shared-models/src/api.ts +++ b/packages/shared-models/src/api.ts @@ -58,11 +58,6 @@ export interface ISharedBase extends IDisposable { * This is used by, for example, docregistry to share the file-path of the edited content. */ export interface ISharedDocument extends ISharedBase { - /** - * Whether the document is saved to disk or not. - */ - readonly dirty: boolean; - /** * The changed signal. */ @@ -477,6 +472,11 @@ export type NotebookChange = { oldValue: nbformat.INotebookMetadata; newValue: nbformat.INotebookMetadata | undefined; }; + nbformatChanged?: { + key: string; + oldValue: number | undefined; + newValue: number | undefined; + }; contextChange?: MapChange; stateChange?: Array<{ name: string; diff --git a/packages/shared-models/src/ymodels.ts b/packages/shared-models/src/ymodels.ts index 5958643ef750..f4643429be4c 100644 --- a/packages/shared-models/src/ymodels.ts +++ b/packages/shared-models/src/ymodels.ts @@ -26,16 +26,6 @@ export interface IYText extends models.ISharedText { export type YCellType = YRawCell | YCodeCell | YMarkdownCell; export class YDocument implements models.ISharedDocument { - get dirty(): boolean { - return this.ystate.get('dirty'); - } - - set dirty(value: boolean) { - this.transact(() => { - this.ystate.set('dirty', value); - }, false); - } - /** * Perform a transaction. While the function f is called, all changes to the shared * document are bundled into a single event. @@ -152,7 +142,6 @@ export class YFile public static create(): YFile { const model = new YFile(); - model.dirty = false; return model; } @@ -224,27 +213,27 @@ export class YNotebook return this._ycellMapping.get(ycell) as YCellType; }); - this.ymeta.observe(this._onMetadataChanged); + this.ymeta.observe(this._onMetaChanged); this.ystate.observe(this._onStateChanged); } get nbformat(): number { - return this.ystate.get('nbformat'); + return this.ymeta.get('nbformat'); } set nbformat(value: number) { this.transact(() => { - this.ystate.set('nbformat', value); + this.ymeta.set('nbformat', value); }, false); } get nbformat_minor(): number { - return this.ystate.get('nbformatMinor'); + return this.ymeta.get('nbformat_minor'); } set nbformat_minor(value: number) { this.transact(() => { - this.ystate.set('nbformatMinor', value); + this.ymeta.set('nbformat_minor', value); }, false); } @@ -253,7 +242,7 @@ export class YNotebook */ dispose(): void { this.ycells.unobserve(this._onYCellsChanged); - this.ymeta.unobserve(this._onMetadataChanged); + this.ymeta.unobserve(this._onMetaChanged); this.ystate.unobserve(this._onStateChanged); } @@ -374,7 +363,6 @@ export class YNotebook disableDocumentWideUndoRedo: boolean ): models.ISharedNotebook { const model = new YNotebook({ disableDocumentWideUndoRedo }); - model.dirty = false; return model; } @@ -442,7 +430,7 @@ export class YNotebook /** * Handle a change to the ystate. */ - private _onMetadataChanged = (event: Y.YMapEvent) => { + private _onMetaChanged = (event: Y.YMapEvent) => { if (event.keysChanged.has('metadata')) { const change = event.changes.keys.get('metadata'); const metadataChange = { @@ -451,6 +439,26 @@ export class YNotebook }; this._changed.emit({ metadataChange }); } + + if (event.keysChanged.has('nbformat')) { + const change = event.changes.keys.get('nbformat'); + const nbformatChanged = { + key: 'nbformat', + oldValue: change?.oldValue ? change!.oldValue : undefined, + newValue: this.nbformat + }; + this._changed.emit({ nbformatChanged }); + } + + if (event.keysChanged.has('nbformat_minor')) { + const change = event.changes.keys.get('nbformat_minor'); + const nbformatChanged = { + key: 'nbformat_minor', + oldValue: change?.oldValue ? change!.oldValue : undefined, + newValue: this.nbformat_minor + }; + this._changed.emit({ nbformatChanged }); + } }; /** @@ -673,7 +681,7 @@ export class YBaseCell /** * Handle a change to the ymodel. */ - private _modelObserver = (events: Y.YEvent[]) => { + private _modelObserver = (events: Y.YEvent[]) => { const changes: models.CellChange = {}; const sourceEvent = events.find( event => event.target === this.ymodel.get('source') diff --git a/setup.cfg b/setup.cfg index de5bf80c224f..3732e3509f60 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,8 @@ install_requires = jupyter_server>=1.16.0,<2 notebook_shim>=0.1 jinja2>=3.0.3 - y-py>=0.4.6,<0.5.0 + ypy-websocket>=0.1.6 + jupyter_ydoc>=0.1.8 [options.extras_require] docs = @@ -84,9 +85,6 @@ console_scripts = jupyter-labextension = jupyterlab.labextensions:main jupyter-labhub = jupyterlab.labhubapp:main jlpm = jupyterlab.jlpmapp:main -jupyter_ydoc = - file = jupyterlab.handlers.ydoc:YFile - notebook = jupyterlab.handlers.ydoc:YNotebook [options.packages.find] exclude = ['docs*', 'examples*'] diff --git a/yarn.lock b/yarn.lock index 9480f9cde9e6..b9fb93f790b8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8534,6 +8534,13 @@ lib0@^0.2.31, lib0@^0.2.42: dependencies: isomorphic.js "^0.2.4" +lib0@^0.2.49: + version "0.2.49" + resolved "https://registry.yarnpkg.com/lib0/-/lib0-0.2.49.tgz#7addb5075063d66ea2c55749e5aeaa48e36278c8" + integrity sha512-ziwYLe/pmI9bjHsAehm4ApuVfZ+q+sbC+vO6Z5+KM+0Fe0MrTLwZSDkJ+cElnhFNQ0P6z/wVkRmc5+vTmImJ9A== + dependencies: + isomorphic.js "^0.2.4" + libnpmaccess@^4.0.1: version "4.0.3" resolved "https://registry.npmjs.org/libnpmaccess/-/libnpmaccess-4.0.3.tgz" @@ -13941,12 +13948,12 @@ yazl@2.5.1: dependencies: buffer-crc32 "~0.2.3" -yjs@^13.5.17: - version "13.5.17" - resolved "https://registry.npmjs.org/yjs/-/yjs-13.5.17.tgz" - integrity sha512-EeroWadB+/SlGuNwXaIjo75QlTlCjst3U/dLqhTkqwIXeCGl/nRTbQev+iYgWZVskD1eTCvaDc2FdrGdpKq32A== +yjs@^13.5.34: + version "13.5.34" + resolved "https://registry.yarnpkg.com/yjs/-/yjs-13.5.34.tgz#ad9ddb8b6c0806e15b289ff0eabc4f06ba238952" + integrity sha512-w/XTk5vhCzbyd6uKKJWE6rPUBf9+heOTzgq8DBkcVgBMv7oeJVFQw2sRqY0YvuLZxURd/XVD2dcNnw8qeFH7Tw== dependencies: - lib0 "^0.2.42" + lib0 "^0.2.49" yocto-queue@^0.1.0: version "0.1.0"