proof-of-conceptly added yjs

SysCV · Jan 22, 2024 · 66e791b · 66e791b
1 parent dc775ef
commit 66e791b
Show file tree

Hide file tree

Showing 8 changed files with 258 additions and 110 deletions.
diff --git a/app/frontend/src/common/annotation.ts b/app/frontend/src/common/annotation.ts
@@ -0,0 +1,28 @@
+import {Annotation, Component, EntityId, SliceIndex} from 'type/annotation';
+import {deepClone} from './util';
+
+export function addAnnotationComponent(
+  a: Annotation,
+  sliceIndex: SliceIndex,
+  entityId: EntityId,
+  component: Component
+) {
+  if (entityId in a.entities) {
+    const slices = a.entities[entityId].geometry.slices;
+    if (!(sliceIndex in slices)) {
+      slices[sliceIndex] = {};
+    }
+    slices[sliceIndex][component.id] = deepClone(component);
+  } else {
+    a.entities[entityId] = {
+      id: entityId,
+      geometry: {
+        slices: {
+          [sliceIndex]: {
+            [component.id]: deepClone(component),
+          },
+        },
+      },
+    };
+  }
+}
diff --git a/app/frontend/src/component/panel/AnnotationMonitor.tsx b/app/frontend/src/component/panel/AnnotationMonitor.tsx
@@ -4,7 +4,7 @@ import * as jsonmergepatch from 'json-merge-patch';
 import * as jsonpatch from 'fast-json-patch';
 import {produce} from 'immer';
 
-import {deleteAnnotationComponent, useStore as useAnnoStore} from 'state/annotate/annotation';
+import {AddComponentsInput, deleteAnnotationComponent, doc, useStore as useAnnoStore} from 'state/annotate/annotation';
 import {useStore as useRenderStore} from 'state/annotate/render';
 import {useGetVideoAnnotationV2, usePatchVideoAnnotation} from 'state/server/annotation';
 import {useStore as useUIStore} from 'state/annotate/ui';
@@ -13,7 +13,7 @@ import {ConfigContext, NutshClientContext} from 'common/context';
 import {ApiError} from 'openapi/nutsh';
 
 import type {Video} from 'openapi/nutsh';
-import {Annotation} from 'type/annotation';
+import {Annotation, ComponentDetail, RectangleComponent} from 'type/annotation';
 
 export const MonitorAnnotation: FC<{videoId: Video['id']}> = ({videoId}) => {
   const config = useContext(ConfigContext);
@@ -137,89 +137,141 @@ function removeDraftComponents(anno: Annotation): Annotation {
 }
 
 function useSyncAnnotation({videoId}: {videoId: Video['id']}) {
-  const {data} = useGetVideoAnnotationV2(videoId);
+  type ComponentsType = {type: ComponentDetail['type']; sliceIndex: number; entityId: string};
+  const comps = doc.getMap<ComponentsType>('components');
 
-  useEffect(() => {
-    const anno = data?.anno;
-    if (!anno) {
-      return;
-    }
-    const fn = (e: Y.YEvent<Y.Map<unknown>>[]) => {
-      console.log(e);
-    };
-    anno.observeDeep(fn);
-    return () => anno.unobserveDeep(fn);
-  }, [data?.anno]);
+  type RectanglesType = Pick<RectangleComponent, 'topLeft' | 'bottomRight'>;
+  const rects = doc.getMap<RectanglesType>('rectangles');
+
+  const updateAnchors = useAnnoStore(s => s.updateRectangleAnchors);
+  const addComponent = useAnnoStore(s => s.addComponent);
+  const deleteComponents = useAnnoStore(s => s.deleteComponents);
 
   useEffect(() => {
-    return useAnnoStore.subscribe(
-      s => s.annotation,
-      (curr, prev) => {
-        if (!data?.anno) {
-          return;
+    const fn = (e: Y.YMapEvent<ComponentsType>) => {
+      console.log(e.changes.keys);
+      for (const [cid, cc] of e.changes.keys) {
+        switch (cc.action) {
+          case 'add': {
+            const info = comps.get(cid);
+            if (info) {
+              // add component
+              const {sliceIndex, entityId, type} = info;
+              switch (type) {
+                case 'rectangle': {
+                  const rect = rects.get(cid);
+                  if (rect) {
+                    const {topLeft, bottomRight} = rect;
+                    addComponent({
+                      sliceIndex,
+                      entityId,
+                      component: {
+                        type: 'rectangle',
+                        id: cid,
+                        topLeft,
+                        bottomRight,
+                      },
+                    });
+                  }
+                  break;
+                }
+                default:
+                  // TODO(xu)
+                  console.warn('not implemented');
+              }
+            }
+            break;
+          }
+          case 'delete': {
+            console.log('hahaha');
+            const {sliceIndex, entityId} = cc.oldValue as ComponentsType;
+            deleteComponents({sliceIndex, components: [[entityId, cid]]});
+            break;
+          }
         }
-
-        const newPrev = produce(prev, removeDraftComponents);
-        const newCurr = produce(curr, removeDraftComponents);
-
-        const ops = jsonpatch.compare(newPrev, newCurr);
-        ops.forEach(op => updateYjsFromJsonPathOperation(data.anno, op));
-      },
-      {
-        fireImmediately: true,
       }
-    );
-  }, [data?.anno]);
-}
+      console.log('comps', e);
+    };
+    comps.observe(fn);
+    return () => comps.unobserve(fn);
+  }, [addComponent, comps, deleteComponents, rects]);
 
-function updateYjsFromJsonPathOperation(anno: Y.Map<unknown>, op: jsonpatch.Operation) {
-  console.log(JSON.stringify(op));
-  switch (op.op) {
-    case 'add': {
-      if (!op.path.startsWith('/')) {
-        console.warn(`unexpected json pointer: ${op.path}`);
-        return;
-      }
-      const paths = op.path.split('/');
-      let dict = anno;
-      paths.slice(1, -1).forEach(path => {
-        if (!dict.has(path)) {
-          dict.set(path, new Y.Map());
+  useEffect(() => {
+    const fn = (e: Y.YMapEvent<RectanglesType>) => {
+      for (const cid of e.keysChanged) {
+        const info = comps.get(cid);
+        if (info) {
+          const {sliceIndex, entityId} = info;
+          const rect = rects.get(cid);
+          if (rect) {
+            const {topLeft, bottomRight} = rect;
+            updateAnchors({
+              sliceIndex,
+              entityId,
+              componentId: cid,
+              topLeft,
+              bottomRight,
+            });
+          }
+        } else {
+          console.warn(`missing rectangle info ${cid}`);
         }
-        dict = dict.get(path) as Y.Map<unknown>;
-      });
-      const lastKey = paths[paths.length - 1];
-      dict.set(lastKey, op.value);
-      break;
-    }
-    case 'copy':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-    case 'move':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-    case 'remove': {
-      if (!op.path.startsWith('/')) {
-        console.warn(`unexpected json pointer: ${op.path}`);
-        return;
       }
-      const paths = op.path.split('/');
-      let dict = anno;
-      paths.slice(1, -1).forEach(path => {
-        if (!dict.has(path)) {
-          dict.set(path, new Y.Map());
-        }
-        dict = dict.get(path) as Y.Map<unknown>;
-      });
-      const lastKey = paths[paths.length - 1];
-      dict.delete(lastKey);
-      break;
-    }
-    case 'replace':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-    case 'test':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-  }
+      console.log('rects', e);
+    };
+    rects.observe(fn);
+    return () => rects.unobserve(fn);
+  }, [comps, rects, updateAnchors]);
 }
+
+// function updateYjsFromJsonPathOperation(anno: Y.Map<unknown>, op: jsonpatch.Operation) {
+//   console.log(JSON.stringify(op));
+//   switch (op.op) {
+//     case 'add': {
+//       if (!op.path.startsWith('/')) {
+//         console.warn(`unexpected json pointer: ${op.path}`);
+//         return;
+//       }
+//       const paths = op.path.split('/');
+//       let dict = anno;
+//       paths.slice(1, -1).forEach(path => {
+//         if (!dict.has(path)) {
+//           dict.set(path, new Y.Map());
+//         }
+//         dict = dict.get(path) as Y.Map<unknown>;
+//       });
+//       const lastKey = paths[paths.length - 1];
+//       dict.set(lastKey, op.value);
+//       break;
+//     }
+//     case 'copy':
+//       console.warn(`not implemented: ${JSON.stringify(op)}`);
+//       break;
+//     case 'move':
+//       console.warn(`not implemented: ${JSON.stringify(op)}`);
+//       break;
+//     case 'remove': {
+//       if (!op.path.startsWith('/')) {
+//         console.warn(`unexpected json pointer: ${op.path}`);
+//         return;
+//       }
+//       const paths = op.path.split('/');
+//       let dict = anno;
+//       paths.slice(1, -1).forEach(path => {
+//         if (!dict.has(path)) {
+//           dict.set(path, new Y.Map());
+//         }
+//         dict = dict.get(path) as Y.Map<unknown>;
+//       });
+//       const lastKey = paths[paths.length - 1];
+//       dict.delete(lastKey);
+//       break;
+//     }
+//     case 'replace':
+//       console.warn(`not implemented: ${JSON.stringify(op)}`);
+//       break;
+//     case 'test':
+//       console.warn(`not implemented: ${JSON.stringify(op)}`);
+//       break;
+//   }
+// }
diff --git a/app/frontend/src/component/panel/layer/polychain/Draw.tsx b/app/frontend/src/component/panel/layer/polychain/Draw.tsx
@@ -134,6 +134,7 @@ const LayerWithEntityId: FC<Props & {entityId: EntityId}> = ({entityId, width, h
         vertices: drawVertices,
         closed,
       },
+      broadcast: true,
     });
 
     finish();

diff --git a/app/frontend/src/component/panel/layer/rectangle/Draw.tsx b/app/frontend/src/component/panel/layer/rectangle/Draw.tsx
@@ -92,6 +92,7 @@ const LayerWithEntityId: FC<Props & {entityId: EntityId}> = ({entityId, width, h
               topLeft: limitCoordinates(min, imw, imh),
               bottomRight: limitCoordinates(max, imw, imh),
             },
+            broadcast: true,
           });
           finish();
         }}

diff --git a/app/frontend/src/component/panel/layer/rectangle/Edit.tsx b/app/frontend/src/component/panel/layer/rectangle/Edit.tsx
@@ -45,7 +45,14 @@ const Canvas: FC<{data: Data} & CanvasHTMLAttributes<HTMLCanvasElement>> = ({dat
     const x2 = Math.max(p.x, q.x);
     const y2 = Math.max(p.y, q.y);
 
-    updateAnchors({sliceIndex, entityId, componentId, topLeft: {x: x1, y: y1}, bottomRight: {x: x2, y: y2}});
+    updateAnchors({
+      sliceIndex,
+      entityId,
+      componentId,
+      topLeft: {x: x1, y: y1},
+      bottomRight: {x: x2, y: y2},
+      broadcast: true,
+    });
     finishEdit();
   }, [anchors, componentId, entityId, finishEdit, sliceIndex, updateAnchors]);
 

diff --git a/app/frontend/src/component/panel/menu/common.ts b/app/frontend/src/component/panel/menu/common.ts
@@ -56,7 +56,7 @@ export function useComponentActions(entityId: EntityId, componentId: ComponentId
     },
     {
       title: intl.get('menu.delete_hovering_component'),
-      fn: () => deleteComponents({sliceIndex, components: [[entityId, componentId]]}),
+      fn: () => deleteComponents({sliceIndex, components: [[entityId, componentId]], broadcast: true}),
       warning: intl.get('menu.warn.delete_hovering_component'),
     },
   ];
@@ -128,7 +128,7 @@ export function useEntityActions(): Action[] {
         {
           title: intl.get('menu.delete_selected_components_in_current_frame'),
           fn: () => {
-            deleteComponents({sliceIndex, components});
+            deleteComponents({sliceIndex, components, broadcast: true});
           },
           warning: intl.get('menu.warn.delete_selected_components_in_current_frame', {count: ec}),
         },
-Original file line number
+Diff line change
@@ Expand Up @@
             vertices: drawVertices,
             closed,
           },
+          broadcast: true,
         });
         finish();
@@ Expand Down @@