cleaned code

app/frontend/src/common/annotation.ts

@@ -0,0 +1,28 @@
+import {Annotation, Component, EntityId, SliceIndex} from 'type/annotation';
+import {deepClone} from './util';
+
+export function addAnnotationComponent(
+  a: Annotation,
+  sliceIndex: SliceIndex,
+  entityId: EntityId,
+  component: Component
+) {
+  if (entityId in a.entities) {
+    const slices = a.entities[entityId].geometry.slices;
+    if (!(sliceIndex in slices)) {
+      slices[sliceIndex] = {};
+    }
+    slices[sliceIndex][component.id] = deepClone(component);
+  } else {
+    a.entities[entityId] = {
+      id: entityId,
+      geometry: {
+        slices: {
+          [sliceIndex]: {
+            [component.id]: deepClone(component),
+          },
+        },
+      },
+    };
+  }
+}

app/frontend/src/component/panel/AnnotationMonitor.tsx

@@ -1,12 +1,10 @@
 import {FC, useContext, useEffect} from 'react';
-import * as Y from 'yjs';
 import * as jsonmergepatch from 'json-merge-patch';
-import * as jsonpatch from 'fast-json-patch';
 import {produce} from 'immer';
 
 import {deleteAnnotationComponent, useStore as useAnnoStore} from 'state/annotate/annotation';
 import {useStore as useRenderStore} from 'state/annotate/render';
-import {useGetVideoAnnotationV2, usePatchVideoAnnotation} from 'state/server/annotation';
+import {usePatchVideoAnnotation} from 'state/server/annotation';
 import {useStore as useUIStore} from 'state/annotate/ui';
 
 import {ConfigContext, NutshClientContext} from 'common/context';
@@ -17,10 +15,9 @@ import {Annotation} from 'type/annotation';
 
 export const MonitorAnnotation: FC<{videoId: Video['id']}> = ({videoId}) => {
   const config = useContext(ConfigContext);
-  useSyncAnnotation({videoId});
   return (
     <>
-      {/* {!config.readonly && <SyncAnnotation videoId={videoId} />} */}
+      {!config.readonly && <SyncAnnotation videoId={videoId} />}
       <ForgetEntities />
       <CommitDraft />
     </>
@@ -135,91 +132,3 @@ function removeDraftComponents(anno: Annotation): Annotation {
     return draft;
   });
 }
-
-function useSyncAnnotation({videoId}: {videoId: Video['id']}) {
-  const {data} = useGetVideoAnnotationV2(videoId);
-
-  useEffect(() => {
-    const anno = data?.anno;
-    if (!anno) {
-      return;
-    }
-    const fn = (e: Y.YEvent<Y.Map<unknown>>[]) => {
-      console.log(e);
-    };
-    anno.observeDeep(fn);
-    return () => anno.unobserveDeep(fn);
-  }, [data?.anno]);
-
-  useEffect(() => {
-    return useAnnoStore.subscribe(
-      s => s.annotation,
-      (curr, prev) => {
-        if (!data?.anno) {
-          return;
-        }
-
-        const newPrev = produce(prev, removeDraftComponents);
-        const newCurr = produce(curr, removeDraftComponents);
-
-        const ops = jsonpatch.compare(newPrev, newCurr);
-        ops.forEach(op => updateYjsFromJsonPathOperation(data.anno, op));
-      },
-      {
-        fireImmediately: true,
-      }
-    );
-  }, [data?.anno]);
-}
-
-function updateYjsFromJsonPathOperation(anno: Y.Map<unknown>, op: jsonpatch.Operation) {
-  console.log(JSON.stringify(op));
-  switch (op.op) {
-    case 'add': {
-      if (!op.path.startsWith('/')) {
-        console.warn(`unexpected json pointer: ${op.path}`);
-        return;
-      }
-      const paths = op.path.split('/');
-      let dict = anno;
-      paths.slice(1, -1).forEach(path => {
-        if (!dict.has(path)) {
-          dict.set(path, new Y.Map());
-        }
-        dict = dict.get(path) as Y.Map<unknown>;
-      });
-      const lastKey = paths[paths.length - 1];
-      dict.set(lastKey, op.value);
-      break;
-    }
-    case 'copy':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-    case 'move':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-    case 'remove': {
-      if (!op.path.startsWith('/')) {
-        console.warn(`unexpected json pointer: ${op.path}`);
-        return;
-      }
-      const paths = op.path.split('/');
-      let dict = anno;
-      paths.slice(1, -1).forEach(path => {
-        if (!dict.has(path)) {
-          dict.set(path, new Y.Map());
-        }
-        dict = dict.get(path) as Y.Map<unknown>;
-      });
-      const lastKey = paths[paths.length - 1];
-      dict.delete(lastKey);
-      break;
-    }
-    case 'replace':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-    case 'test':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-  }
-}

app/frontend/src/component/panel/AnnotationMonitorV2.tsx

@@ -0,0 +1,98 @@
+import {FC, useEffect} from 'react';
+import * as Y from 'yjs';
+import {ComponentInfo, RectangleAnchors, annoDoc} from 'sync/doc';
+import {useStore as useAnnoStore} from 'state/annotate/annotation';
+import {Component} from 'type/annotation';
+
+export const MonitorAnnotationV2: FC = () => {
+  useSyncAnnotation();
+  return null;
+};
+
+function useSyncAnnotation() {
+  useComponentInfoMonitor();
+  useRectangleAnchorsMonitor();
+}
+
+function useComponentInfoMonitor() {
+  const comps = annoDoc.componentInfoMap();
+  const rects = annoDoc.rectangleAnchorsMap();
+
+  const addComponent = useAnnoStore(s => s.addComponent);
+  const deleteComponents = useAnnoStore(s => s.deleteComponents);
+
+  useEffect(() => {
+    const fn = (e: Y.YMapEvent<ComponentInfo>) => {
+      for (const [cid, cc] of e.changes.keys) {
+        switch (cc.action) {
+          case 'add': {
+            const info = comps.get(cid);
+            if (!info) {
+              break;
+            }
+
+            // add component
+            const {sliceIndex, entityId, type} = info;
+            switch (type) {
+              case 'rectangle': {
+                const rect = rects.get(cid);
+                if (!rect) {
+                  break;
+                }
+
+                const {topLeft, bottomRight} = rect;
+                const component: Component = {
+                  type: 'rectangle',
+                  id: cid,
+                  topLeft,
+                  bottomRight,
+                };
+                addComponent({sliceIndex, entityId, component});
+                break;
+              }
+              default:
+                // TODO(xu)
+                console.warn('not implemented');
+            }
+            break;
+          }
+          case 'delete': {
+            // delete components
+            const {sliceIndex, entityId} = cc.oldValue as ComponentInfo;
+            deleteComponents({sliceIndex, components: [[entityId, cid]]});
+            break;
+          }
+        }
+      }
+    };
+    comps.observe(fn);
+    return () => comps.unobserve(fn);
+  }, [addComponent, comps, deleteComponents, rects]);
+}
+
+function useRectangleAnchorsMonitor() {
+  const comps = annoDoc.componentInfoMap();
+  const rects = annoDoc.rectangleAnchorsMap();
+
+  const updateAnchors = useAnnoStore(s => s.updateRectangleAnchors);
+
+  useEffect(() => {
+    const fn = (e: Y.YMapEvent<RectangleAnchors>) => {
+      for (const cid of e.keysChanged) {
+        const info = comps.get(cid);
+        if (info) {
+          const {sliceIndex, entityId} = info;
+          const rect = rects.get(cid);
+          if (rect) {
+            const {topLeft, bottomRight} = rect;
+            updateAnchors({sliceIndex, entityId, componentId: cid, topLeft, bottomRight});
+          }
+        } else {
+          console.warn(`rectangle ${cid} not found`);
+        }
+      }
+    };
+    rects.observe(fn);
+    return () => rects.unobserve(fn);
+  }, [comps, rects, updateAnchors]);
+}

app/frontend/src/component/panel/layer/polychain/Draw.tsx

@@ -134,6 +134,7 @@ const LayerWithEntityId: FC<Props & {entityId: EntityId}> = ({entityId, width, h
         vertices: drawVertices,
         closed,
       },
+      broadcast: true,
     });
 
     finish();

app/frontend/src/component/panel/layer/rectangle/Draw.tsx

@@ -92,6 +92,7 @@ const LayerWithEntityId: FC<Props & {entityId: EntityId}> = ({entityId, width, h
               topLeft: limitCoordinates(min, imw, imh),
               bottomRight: limitCoordinates(max, imw, imh),
             },
+            broadcast: true,
           });
           finish();
         }}

app/frontend/src/component/panel/layer/rectangle/Edit.tsx

@@ -45,7 +45,14 @@ const Canvas: FC<{data: Data} & CanvasHTMLAttributes<HTMLCanvasElement>> = ({dat
     const x2 = Math.max(p.x, q.x);
     const y2 = Math.max(p.y, q.y);
 
-    updateAnchors({sliceIndex, entityId, componentId, topLeft: {x: x1, y: y1}, bottomRight: {x: x2, y: y2}});
+    updateAnchors({
+      sliceIndex,
+      entityId,
+      componentId,
+      topLeft: {x: x1, y: y1},
+      bottomRight: {x: x2, y: y2},
+      broadcast: true,
+    });
     finishEdit();
   }, [anchors, componentId, entityId, finishEdit, sliceIndex, updateAnchors]);
 

app/frontend/src/component/panel/menu/common.ts

@@ -56,7 +56,7 @@ export function useComponentActions(entityId: EntityId, componentId: ComponentId
     },
     {
       title: intl.get('menu.delete_hovering_component'),
-      fn: () => deleteComponents({sliceIndex, components: [[entityId, componentId]]}),
+      fn: () => deleteComponents({sliceIndex, components: [[entityId, componentId]], broadcast: true}),
       warning: intl.get('menu.warn.delete_hovering_component'),
     },
   ];
@@ -128,7 +128,7 @@ export function useEntityActions(): Action[] {
         {
           title: intl.get('menu.delete_selected_components_in_current_frame'),
           fn: () => {
-            deleteComponents({sliceIndex, components});
+            deleteComponents({sliceIndex, components, broadcast: true});
           },
           warning: intl.get('menu.warn.delete_selected_components_in_current_frame', {count: ec}),
         },

app/frontend/src/page/annotate/Panel/Loaded.tsx

@@ -5,7 +5,7 @@ import shallow from 'zustand/shallow';
 import {useStore as useRenderStore} from 'state/annotate/render';
 import {prefetchImages} from 'state/image/store';
 import {UI} from 'common/constant';
-import {MonitorAnnotation} from 'component/panel/AnnotationMonitor';
+import {MonitorAnnotationV2} from 'component/panel/AnnotationMonitorV2';
 import type {Project, Video} from 'openapi/nutsh';
 import type {ProjectSpec} from 'type/project_spec';
 import {FrameSlider} from 'component/panel/FrameSlider';
@@ -72,7 +72,7 @@ export const PanelLoaded: FC<{
         />
       </div>
       <SyncingDisableInteractionMask />
-      <MonitorAnnotation videoId={video.id} />
+      <MonitorAnnotationV2 />
     </div>
   );
 };