listen to yjs updates

app/frontend/src/common/yjs/convert.ts

@@ -74,7 +74,7 @@ export function writeAnnotationToYjs(anno: Annotation, doc: Y.Doc): void {
   return;
 }
 
-function readComponent(doc: Y.Doc, cid: ComponentId, info: YjsComponent): Component | undefined {
+export function readComponent(doc: Y.Doc, cid: ComponentId, info: YjsComponent): Component | undefined {
   const comps = yjsComponentMap(doc);
   const anchors = yjsRectangleAnchorsMap(doc);
   const verts = yjsPolychainVerticesMap(doc);

app/frontend/src/common/yjs/event.ts

@@ -0,0 +1,249 @@
+import {useEffect} from 'react';
+import * as Y from 'yjs';
+import {useYjsContext} from './context';
+import {Component as ComponentYjs, yjsComponentMap} from './docs/component';
+import {useAnnoStore} from 'state/annotate/annotation';
+import {readComponent} from './convert';
+import {RectangleAnchors, yjsRectangleAnchorsMap} from './docs/rectangle';
+import {yjsPolychainVerticesMap} from './docs/polychain';
+import {MaskComponent, Vertex} from 'type/annotation';
+import {yjsMaskMap} from './docs/mask';
+import {decodeEntityCategoryMapKey, yjsEntityCategoriesMap} from './docs/entity';
+
+export function useYjsListener() {
+  useComponentsListener();
+  useRectangleAnchorsListener();
+  usePolychainVerticesListener();
+  useMasksListener();
+  useEntityCategoriesListener();
+}
+
+function useComponentsListener() {
+  const {doc} = useYjsContext();
+  const comps = yjsComponentMap(doc);
+
+  const addComponent = useAnnoStore(s => s.addComponent);
+  const deleteComponents = useAnnoStore(s => s.deleteComponents);
+  const transferComponent = useAnnoStore(s => s.transferComponent);
+
+  useEffect(() => {
+    const fn = (e: Y.YMapEvent<ComponentYjs>) => {
+      if (e.transaction.local) {
+        return;
+      }
+
+      for (const [cid, cc] of e.changes.keys) {
+        switch (cc.action) {
+          case 'add': {
+            const info = comps.get(cid);
+            if (!info) {
+              break;
+            }
+
+            // add component
+            const {sidx, eid} = info;
+            const component = readComponent(doc, cid, info);
+            if (!component) {
+              break;
+            }
+            addComponent({sliceIndex: sidx, entityId: eid, component});
+
+            break;
+          }
+          case 'update': {
+            // transferred component
+            const comp = comps.get(cid);
+            if (!comp) {
+              return;
+            }
+
+            const {sliceIndex, entityId} = cc.oldValue;
+            transferComponent({sliceIndex, entityId, componentId: cid, targetEntityId: comp.eid});
+            break;
+          }
+          case 'delete': {
+            // delete components
+            const {sidx, eid} = cc.oldValue as ComponentYjs;
+            deleteComponents({sliceIndex: sidx, components: [[eid, cid]]});
+            break;
+          }
+        }
+      }
+    };
+    comps.observe(fn);
+    return () => comps.unobserve(fn);
+  }, [addComponent, comps, deleteComponents, doc, transferComponent]);
+}
+
+function useRectangleAnchorsListener() {
+  const {doc} = useYjsContext();
+  const comps = yjsComponentMap(doc);
+  const anchors = yjsRectangleAnchorsMap(doc);
+
+  const updateAnchors = useAnnoStore(s => s.updateRectangleAnchors);
+
+  useEffect(() => {
+    const fn = (e: Y.YMapEvent<RectangleAnchors>) => {
+      if (e.transaction.local) {
+        return;
+      }
+
+      for (const cid of e.keysChanged) {
+        const info = comps.get(cid);
+        if (info) {
+          const {sidx, eid} = info;
+          const rect = anchors.get(cid);
+          if (rect) {
+            const {topLeft, bottomRight} = rect;
+            updateAnchors({sliceIndex: sidx, entityId: eid, componentId: cid, topLeft, bottomRight});
+          }
+        } else {
+          console.warn(`rectangle ${cid} not found`);
+        }
+      }
+    };
+    anchors.observe(fn);
+    return () => anchors.unobserve(fn);
+  }, [comps, anchors, updateAnchors]);
+}
+
+function usePolychainVerticesListener() {
+  const {doc} = useYjsContext();
+  const comps = yjsComponentMap(doc);
+  const verts = yjsPolychainVerticesMap(doc);
+
+  const updateVertices = useAnnoStore(s => s.updatePolychainVertices);
+
+  useEffect(() => {
+    const fn = (e: Y.YMapEvent<Y.Array<Vertex>>) => {
+      if (e.transaction.local) {
+        return;
+      }
+
+      for (const cid of e.keysChanged) {
+        const info = comps.get(cid);
+        if (!info) {
+          console.warn(`polychain ${cid} not found`);
+          continue;
+        }
+        const vs = verts.get(cid);
+        if (!vs) {
+          console.warn(`polychain ${cid} vertices not found`);
+          continue;
+        }
+
+        const {sidx, eid} = info;
+        updateVertices({sliceIndex: sidx, entityId: eid, componentId: cid, vertices: vs.toArray()});
+      }
+    };
+    verts.observe(fn);
+    return () => verts.unobserve(fn);
+  }, [comps, updateVertices, verts]);
+
+  useEffect(() => {
+    const fn = (es: Y.YEvent<Y.Array<Vertex>>[]) => {
+      for (const e of es) {
+        if (e.transaction.local) {
+          continue;
+        }
+        if (e.path.length !== 1) {
+          continue;
+        }
+        const cid = `${e.path[0]}`;
+
+        const info = comps.get(cid);
+        if (!info) {
+          console.warn(`polychain ${cid} not found`);
+          continue;
+        }
+        const vs = verts.get(cid);
+        if (!vs) {
+          console.warn(`polychain ${cid} vertices not found`);
+          continue;
+        }
+        const {sidx, eid} = info;
+        updateVertices({sliceIndex: sidx, entityId: eid, componentId: cid, vertices: vs.toArray()});
+      }
+    };
+    verts.observeDeep(fn);
+    return () => verts.unobserveDeep(fn);
+  }, [comps, updateVertices, verts]);
+}
+
+function useMasksListener() {
+  const {doc} = useYjsContext();
+  const comps = yjsComponentMap(doc);
+  const masks = yjsMaskMap(doc);
+
+  const addComponent = useAnnoStore(s => s.addComponent);
+  useEffect(() => {
+    const fn = (e: Y.YMapEvent<MaskComponent>) => {
+      for (const [cid, cc] of e.changes.keys) {
+        switch (cc.action) {
+          case 'add':
+          case 'update': {
+            const info = comps.get(cid);
+            if (!info) {
+              console.warn('mask info not found', cid);
+              continue;
+            }
+
+            const mask = masks.get(cid);
+            if (!mask) {
+              console.warn('mask not found', cid);
+              continue;
+            }
+
+            const {sidx, eid} = info;
+            addComponent({sliceIndex: sidx, entityId: eid, component: {id: cid, ...mask}});
+            break;
+          }
+          default:
+            console.warn('not implemented', cc);
+        }
+      }
+    };
+    masks.observe(fn);
+    return () => masks.unobserve(fn);
+  }, [addComponent, comps, masks]);
+}
+
+function useEntityCategoriesListener() {
+  const {doc} = useYjsContext();
+  const cats = yjsEntityCategoriesMap(doc);
+
+  const setEntityCategory = useAnnoStore(s => s.setEntityCategory);
+  const clearEntityCategory = useAnnoStore(s => s.clearEntityCategory);
+
+  useEffect(() => {
+    const fn = (e: Y.YMapEvent<string[]>) => {
+      for (const [key, cc] of e.changes.keys) {
+        const decoded = decodeEntityCategoryMapKey(key);
+        if (!decoded) {
+          console.warn('unexpected entity category key', key);
+          continue;
+        }
+        const {eid, sidx, category} = decoded;
+
+        switch (cc.action) {
+          case 'add':
+          case 'update': {
+            const entries = cats.get(key);
+            if (!entries || entries.length === 0) {
+              console.warn('missing category entries', key);
+              continue;
+            }
+            setEntityCategory({entityId: eid, category, entries, sliceIndex: sidx});
+            break;
+          }
+          case 'delete': {
+            clearEntityCategory({entityId: eid, category, sliceIndex: sidx});
+            break;
+          }
+        }
+      }
+    };
+    cats.observe(fn);
+    return () => cats.unobserve(fn);
+  }, [cats, clearEntityCategory, setEntityCategory]);
+}

app/frontend/src/page/annotate/Panel/index.tsx

@@ -2,6 +2,7 @@ import {FC} from 'react';
 import {useParams} from 'react-router-dom';
 import {YjsProvider} from 'common/yjs/context';
 import {PanelLoad} from './Load';
+import {useYjsListener} from 'common/yjs/event';
 
 const Panel: FC = () => {
   const {videoId: id = ''} = useParams();
@@ -12,8 +13,14 @@ const Panel: FC = () => {
   return (
     <YjsProvider>
       <PanelLoad id={id} />
+      <YjsListener />
     </YjsProvider>
   );
 };
 
+function YjsListener() {
+  useYjsListener();
+  return null;
+}
+
 export default Panel;

app/frontend/src/state/annotate/annotation-broadcast.ts

@@ -223,7 +223,31 @@ function useAnnoBroadcast(): StateManipulation {
 
     deletePolychainVertex: (input: DeletePolychainVertexInput) => {
       const {componentId: cid, vertexIndex} = input;
-      verts.get(cid)?.delete(vertexIndex, 1);
+
+      const comp = comps.get(cid);
+      const vs = verts.get(cid);
+      if (!vs || comp?.type !== 'polychain') {
+        return;
+      }
+      const n = vs.length;
+      if ((comp.closed && n <= 3) || (!comp.closed && n <= 2)) {
+        deleteComponent(doc, cid, comp.type);
+        return;
+      }
+
+      doc.transact(() => {
+        if (vertexIndex > 0 || comp.closed) {
+          vs.delete(vertexIndex, 1);
+        } else {
+          // The first vertex of a polyline can NOT be bezier.
+          if (vs.get(1).bezier) {
+            const v = vs.get(1);
+            vs.delete(1, 1);
+            vs.insert(1, [{...v, bezier: undefined}]);
+          }
+          vs.delete(0, 1);
+        }
+      });
     },
 
     setPolychainVertexBezier: (input: SetPolychainVertexBezierInput) => {