diff --git a/app/frontend/src/common/annotation.ts b/app/frontend/src/common/annotation.ts
new file mode 100644
index 0000000..f90bc01
--- /dev/null
+++ b/app/frontend/src/common/annotation.ts
@@ -0,0 +1,28 @@
+import {Annotation, Component, EntityId, SliceIndex} from 'type/annotation';
+import {deepClone} from './util';
+
+export function addAnnotationComponent(
+  a: Annotation,
+  sliceIndex: SliceIndex,
+  entityId: EntityId,
+  component: Component
+) {
+  if (entityId in a.entities) {
+    const slices = a.entities[entityId].geometry.slices;
+    if (!(sliceIndex in slices)) {
+      slices[sliceIndex] = {};
+    }
+    slices[sliceIndex][component.id] = deepClone(component);
+  } else {
+    a.entities[entityId] = {
+      id: entityId,
+      geometry: {
+        slices: {
+          [sliceIndex]: {
+            [component.id]: deepClone(component),
+          },
+        },
+      },
+    };
+  }
+}
diff --git a/app/frontend/src/component/panel/AnnotationMonitor.tsx b/app/frontend/src/component/panel/AnnotationMonitor.tsx
index 2c5b132..7eaf2cf 100644
--- a/app/frontend/src/component/panel/AnnotationMonitor.tsx
+++ b/app/frontend/src/component/panel/AnnotationMonitor.tsx
@@ -4,7 +4,7 @@ import * as jsonmergepatch from 'json-merge-patch';
 import * as jsonpatch from 'fast-json-patch';
 import {produce} from 'immer';
 
-import {deleteAnnotationComponent, useStore as useAnnoStore} from 'state/annotate/annotation';
+import {AddComponentsInput, deleteAnnotationComponent, doc, useStore as useAnnoStore} from 'state/annotate/annotation';
 import {useStore as useRenderStore} from 'state/annotate/render';
 import {useGetVideoAnnotationV2, usePatchVideoAnnotation} from 'state/server/annotation';
 import {useStore as useUIStore} from 'state/annotate/ui';
@@ -13,7 +13,7 @@ import {ConfigContext, NutshClientContext} from 'common/context';
 import {ApiError} from 'openapi/nutsh';
 
 import type {Video} from 'openapi/nutsh';
-import {Annotation} from 'type/annotation';
+import {Annotation, ComponentDetail, RectangleComponent} from 'type/annotation';
 
 export const MonitorAnnotation: FC<{videoId: Video['id']}> = ({videoId}) => {
   const config = useContext(ConfigContext);
@@ -137,89 +137,141 @@ function removeDraftComponents(anno: Annotation): Annotation {
 }
 
 function useSyncAnnotation({videoId}: {videoId: Video['id']}) {
-  const {data} = useGetVideoAnnotationV2(videoId);
+  type ComponentsType = {type: ComponentDetail['type']; sliceIndex: number; entityId: string};
+  const comps = doc.getMap<ComponentsType>('components');
 
-  useEffect(() => {
-    const anno = data?.anno;
-    if (!anno) {
-      return;
-    }
-    const fn = (e: Y.YEvent<Y.Map<unknown>>[]) => {
-      console.log(e);
-    };
-    anno.observeDeep(fn);
-    return () => anno.unobserveDeep(fn);
-  }, [data?.anno]);
+  type RectanglesType = Pick<RectangleComponent, 'topLeft' | 'bottomRight'>;
+  const rects = doc.getMap<RectanglesType>('rectangles');
+
+  const updateAnchors = useAnnoStore(s => s.updateRectangleAnchors);
+  const addComponent = useAnnoStore(s => s.addComponent);
+  const deleteComponents = useAnnoStore(s => s.deleteComponents);
 
   useEffect(() => {
-    return useAnnoStore.subscribe(
-      s => s.annotation,
-      (curr, prev) => {
-        if (!data?.anno) {
-          return;
+    const fn = (e: Y.YMapEvent<ComponentsType>) => {
+      console.log(e.changes.keys);
+      for (const [cid, cc] of e.changes.keys) {
+        switch (cc.action) {
+          case 'add': {
+            const info = comps.get(cid);
+            if (info) {
+              // add component
+              const {sliceIndex, entityId, type} = info;
+              switch (type) {
+                case 'rectangle': {
+                  const rect = rects.get(cid);
+                  if (rect) {
+                    const {topLeft, bottomRight} = rect;
+                    addComponent({
+                      sliceIndex,
+                      entityId,
+                      component: {
+                        type: 'rectangle',
+                        id: cid,
+                        topLeft,
+                        bottomRight,
+                      },
+                    });
+                  }
+                  break;
+                }
+                default:
+                  // TODO(xu)
+                  console.warn('not implemented');
+              }
+            }
+            break;
+          }
+          case 'delete': {
+            console.log('hahaha');
+            const {sliceIndex, entityId} = cc.oldValue as ComponentsType;
+            deleteComponents({sliceIndex, components: [[entityId, cid]]});
+            break;
+          }
         }
-
-        const newPrev = produce(prev, removeDraftComponents);
-        const newCurr = produce(curr, removeDraftComponents);
-
-        const ops = jsonpatch.compare(newPrev, newCurr);
-        ops.forEach(op => updateYjsFromJsonPathOperation(data.anno, op));
-      },
-      {
-        fireImmediately: true,
       }
-    );
-  }, [data?.anno]);
-}
+      console.log('comps', e);
+    };
+    comps.observe(fn);
+    return () => comps.unobserve(fn);
+  }, [addComponent, comps, deleteComponents, rects]);
 
-function updateYjsFromJsonPathOperation(anno: Y.Map<unknown>, op: jsonpatch.Operation) {
-  console.log(JSON.stringify(op));
-  switch (op.op) {
-    case 'add': {
-      if (!op.path.startsWith('/')) {
-        console.warn(`unexpected json pointer: ${op.path}`);
-        return;
-      }
-      const paths = op.path.split('/');
-      let dict = anno;
-      paths.slice(1, -1).forEach(path => {
-        if (!dict.has(path)) {
-          dict.set(path, new Y.Map());
+  useEffect(() => {
+    const fn = (e: Y.YMapEvent<RectanglesType>) => {
+      for (const cid of e.keysChanged) {
+        const info = comps.get(cid);
+        if (info) {
+          const {sliceIndex, entityId} = info;
+          const rect = rects.get(cid);
+          if (rect) {
+            const {topLeft, bottomRight} = rect;
+            updateAnchors({
+              sliceIndex,
+              entityId,
+              componentId: cid,
+              topLeft,
+              bottomRight,
+            });
+          }
+        } else {
+          console.warn(`missing rectangle info ${cid}`);
         }
-        dict = dict.get(path) as Y.Map<unknown>;
-      });
-      const lastKey = paths[paths.length - 1];
-      dict.set(lastKey, op.value);
-      break;
-    }
-    case 'copy':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-    case 'move':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-    case 'remove': {
-      if (!op.path.startsWith('/')) {
-        console.warn(`unexpected json pointer: ${op.path}`);
-        return;
       }
-      const paths = op.path.split('/');
-      let dict = anno;
-      paths.slice(1, -1).forEach(path => {
-        if (!dict.has(path)) {
-          dict.set(path, new Y.Map());
-        }
-        dict = dict.get(path) as Y.Map<unknown>;
-      });
-      const lastKey = paths[paths.length - 1];
-      dict.delete(lastKey);
-      break;
-    }
-    case 'replace':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-    case 'test':
-      console.warn(`not implemented: ${JSON.stringify(op)}`);
-      break;
-  }
+      console.log('rects', e);
+    };
+    rects.observe(fn);
+    return () => rects.unobserve(fn);
+  }, [comps, rects, updateAnchors]);
 }
+
+// function updateYjsFromJsonPathOperation(anno: Y.Map<unknown>, op: jsonpatch.Operation) {
+//   console.log(JSON.stringify(op));
+//   switch (op.op) {
+//     case 'add': {
+//       if (!op.path.startsWith('/')) {
+//         console.warn(`unexpected json pointer: ${op.path}`);
+//         return;
+//       }
+//       const paths = op.path.split('/');
+//       let dict = anno;
+//       paths.slice(1, -1).forEach(path => {
+//         if (!dict.has(path)) {
+//           dict.set(path, new Y.Map());
+//         }
+//         dict = dict.get(path) as Y.Map<unknown>;
+//       });
+//       const lastKey = paths[paths.length - 1];
+//       dict.set(lastKey, op.value);
+//       break;
+//     }
+//     case 'copy':
+//       console.warn(`not implemented: ${JSON.stringify(op)}`);
+//       break;
+//     case 'move':
+//       console.warn(`not implemented: ${JSON.stringify(op)}`);
+//       break;
+//     case 'remove': {
+//       if (!op.path.startsWith('/')) {
+//         console.warn(`unexpected json pointer: ${op.path}`);
+//         return;
+//       }
+//       const paths = op.path.split('/');
+//       let dict = anno;
+//       paths.slice(1, -1).forEach(path => {
+//         if (!dict.has(path)) {
+//           dict.set(path, new Y.Map());
+//         }
+//         dict = dict.get(path) as Y.Map<unknown>;
+//       });
+//       const lastKey = paths[paths.length - 1];
+//       dict.delete(lastKey);
+//       break;
+//     }
+//     case 'replace':
+//       console.warn(`not implemented: ${JSON.stringify(op)}`);
+//       break;
+//     case 'test':
+//       console.warn(`not implemented: ${JSON.stringify(op)}`);
+//       break;
+//   }
+// }
diff --git a/app/frontend/src/component/panel/layer/polychain/Draw.tsx b/app/frontend/src/component/panel/layer/polychain/Draw.tsx
index e1f4ac1..47f5b89 100644
--- a/app/frontend/src/component/panel/layer/polychain/Draw.tsx
+++ b/app/frontend/src/component/panel/layer/polychain/Draw.tsx
@@ -134,6 +134,7 @@ const LayerWithEntityId: FC<Props & {entityId: EntityId}> = ({entityId, width, h
         vertices: drawVertices,
         closed,
       },
+      broadcast: true,
     });
 
     finish();
diff --git a/app/frontend/src/component/panel/layer/rectangle/Draw.tsx b/app/frontend/src/component/panel/layer/rectangle/Draw.tsx
index 12d4c49..501c3af 100644
--- a/app/frontend/src/component/panel/layer/rectangle/Draw.tsx
+++ b/app/frontend/src/component/panel/layer/rectangle/Draw.tsx
@@ -92,6 +92,7 @@ const LayerWithEntityId: FC<Props & {entityId: EntityId}> = ({entityId, width, h
               topLeft: limitCoordinates(min, imw, imh),
               bottomRight: limitCoordinates(max, imw, imh),
             },
+            broadcast: true,
           });
           finish();
         }}
diff --git a/app/frontend/src/component/panel/layer/rectangle/Edit.tsx b/app/frontend/src/component/panel/layer/rectangle/Edit.tsx
index 4182e77..aa10f38 100644
--- a/app/frontend/src/component/panel/layer/rectangle/Edit.tsx
+++ b/app/frontend/src/component/panel/layer/rectangle/Edit.tsx
@@ -45,7 +45,14 @@ const Canvas: FC<{data: Data} & CanvasHTMLAttributes<HTMLCanvasElement>> = ({dat
     const x2 = Math.max(p.x, q.x);
     const y2 = Math.max(p.y, q.y);
 
-    updateAnchors({sliceIndex, entityId, componentId, topLeft: {x: x1, y: y1}, bottomRight: {x: x2, y: y2}});
+    updateAnchors({
+      sliceIndex,
+      entityId,
+      componentId,
+      topLeft: {x: x1, y: y1},
+      bottomRight: {x: x2, y: y2},
+      broadcast: true,
+    });
     finishEdit();
   }, [anchors, componentId, entityId, finishEdit, sliceIndex, updateAnchors]);
 
diff --git a/app/frontend/src/component/panel/menu/common.ts b/app/frontend/src/component/panel/menu/common.ts
index 74c8ceb..9c68242 100644
--- a/app/frontend/src/component/panel/menu/common.ts
+++ b/app/frontend/src/component/panel/menu/common.ts
@@ -56,7 +56,7 @@ export function useComponentActions(entityId: EntityId, componentId: ComponentId
     },
     {
       title: intl.get('menu.delete_hovering_component'),
-      fn: () => deleteComponents({sliceIndex, components: [[entityId, componentId]]}),
+      fn: () => deleteComponents({sliceIndex, components: [[entityId, componentId]], broadcast: true}),
       warning: intl.get('menu.warn.delete_hovering_component'),
     },
   ];
@@ -128,7 +128,7 @@ export function useEntityActions(): Action[] {
         {
           title: intl.get('menu.delete_selected_components_in_current_frame'),
           fn: () => {
-            deleteComponents({sliceIndex, components});
+            deleteComponents({sliceIndex, components, broadcast: true});
           },
           warning: intl.get('menu.warn.delete_selected_components_in_current_frame', {count: ec}),
         },
diff --git a/app/frontend/src/state/annotate/annotation.ts b/app/frontend/src/state/annotate/annotation.ts
index 4ee82eb..84f323c 100644
--- a/app/frontend/src/state/annotate/annotation.ts
+++ b/app/frontend/src/state/annotate/annotation.ts
@@ -1,3 +1,4 @@
+import * as Y from 'yjs';
 import create from 'zustand';
 import {temporal} from 'zundo';
 import {immer} from 'zustand/middleware/immer';
@@ -15,6 +16,7 @@ import type {
   ComponentMap,
 } from 'type/annotation';
 import {newComponentAdapter} from 'common/adapter';
+import {addAnnotationComponent} from 'common/annotation';
 
 export type State = {
   annotation: Annotation;
@@ -65,6 +67,7 @@ export type AddComponentInput = {
   sliceIndex: SliceIndex;
   entityId: EntityId;
   component: Component;
+  broadcast?: boolean;
 };
 
 export type AddComponentsInput = {
@@ -85,6 +88,7 @@ export type TransferComponentInput = {
 export type DeleteComponentsInput = {
   sliceIndex: SliceIndex;
   components: [EntityId, ComponentId][];
+  broadcast?: boolean;
 };
 
 export type SeperateComponentInput = {
@@ -123,6 +127,7 @@ export type UpdateRectangleAnchorsInput = {
   componentId: ComponentId;
   topLeft: Coordinates;
   bottomRight: Coordinates;
+  broadcast?: boolean;
 };
 
 export type PasteInput = {
@@ -159,6 +164,8 @@ export type UpdateSliceMasksInput = {
 
 const emptyAnnotation: Annotation = {entities: {}};
 
+export const doc = new Y.Doc();
+
 export const useStore = create<State>()(
   temporal(
     subscribeWithSelector(
@@ -248,8 +255,26 @@ export const useStore = create<State>()(
 
         addComponent: (input: AddComponentInput) => {
           set(s => {
-            const {sliceIndex, entityId, component} = input;
+            const {sliceIndex, entityId, component, broadcast} = input;
             addComponent(s, sliceIndex, entityId, component);
+
+            if (broadcast) {
+              switch (component.type) {
+                case 'rectangle': {
+                  const comps = doc.getMap('components');
+                  const rects = doc.getMap('rectangles');
+                  doc.transact(() => {
+                    const {id: cid, topLeft, bottomRight, type} = component;
+                    comps.set(cid, {type, sliceIndex, entityId});
+                    rects.set(cid, {topLeft, bottomRight});
+                  });
+                  break;
+                }
+                default:
+                  // TODO(xu)
+                  console.warn('not implemented');
+              }
+            }
           });
         },
 
@@ -313,10 +338,28 @@ export const useStore = create<State>()(
 
         deleteComponents: (input: DeleteComponentsInput) => {
           set(s => {
-            const {sliceIndex, components} = input;
+            const {sliceIndex, components, broadcast} = input;
 
             components.forEach(([eid, cid]) => {
-              deleteComponent(s, sliceIndex, eid, cid);
+              const deleted = deleteComponent(s, sliceIndex, eid, cid);
+
+              if (broadcast) {
+                switch (deleted?.type) {
+                  case 'rectangle': {
+                    const {id: cid} = deleted;
+                    const comps = doc.getMap('components');
+                    const rects = doc.getMap('rectangles');
+                    doc.transact(() => {
+                      comps.delete(cid);
+                      rects.delete(cid);
+                    });
+                    break;
+                  }
+                  default:
+                    // TODO(xu)
+                    console.warn('not implemented');
+                }
+              }
             });
           });
         },
@@ -394,7 +437,7 @@ export const useStore = create<State>()(
 
         updateRectangleAnchors: (input: UpdateRectangleAnchorsInput) => {
           set(s => {
-            const {sliceIndex, entityId, componentId, topLeft, bottomRight} = input;
+            const {sliceIndex, entityId, componentId, topLeft, bottomRight, broadcast} = input;
 
             const slice = getSlice(s, sliceIndex, entityId);
             if (!slice) return;
@@ -403,6 +446,11 @@ export const useStore = create<State>()(
             if (old?.type !== 'rectangle') return;
 
             slice[componentId] = {...old, topLeft, bottomRight};
+
+            if (broadcast) {
+              const rects = doc.getMap('rectangles');
+              rects.set(componentId, {topLeft, bottomRight});
+            }
           });
         },
 
@@ -456,24 +504,7 @@ export const useStore = create<State>()(
 export const useTemporalStore = create(useStore.temporal);
 
 function addComponent(s: State, sliceIndex: SliceIndex, entityId: EntityId, component: Component) {
-  if (entityId in s.annotation.entities) {
-    const slices = s.annotation.entities[entityId].geometry.slices;
-    if (!(sliceIndex in slices)) {
-      slices[sliceIndex] = {};
-    }
-    slices[sliceIndex][component.id] = deepClone(component);
-  } else {
-    s.annotation.entities[entityId] = {
-      id: entityId,
-      geometry: {
-        slices: {
-          [sliceIndex]: {
-            [component.id]: deepClone(component),
-          },
-        },
-      },
-    };
-  }
+  addAnnotationComponent(s.annotation, sliceIndex, entityId, component);
 }
 
 function deleteComponent(
diff --git a/app/frontend/src/state/server/annotation.ts b/app/frontend/src/state/server/annotation.ts
index c5cbcf0..72e0f5a 100644
--- a/app/frontend/src/state/server/annotation.ts
+++ b/app/frontend/src/state/server/annotation.ts
@@ -1,8 +1,11 @@
-import * as Y from 'yjs';
 import {createYjsProvider} from '@y-sweet/client';
 import {useQuery, useMutation} from '@tanstack/react-query';
 
 import type {NutshClient, DefaultService, Video} from 'openapi/nutsh';
+import {doc} from 'state/annotate/annotation';
+import {ComponentDetail, RectangleComponent} from 'type/annotation';
+import {addAnnotationComponent} from 'common/annotation';
+import {emptyAnnotation} from 'state/annotate/render';
 
 export const useGetVideoAnnotation = (client: NutshClient, id: Video['id']) =>
   useQuery({
@@ -28,16 +31,41 @@ export const useGetVideoAnnotationV2 = (id: Video['id']) =>
       const clientToken = await res.json();
 
       // create a Yjs document and connect it to the Y-Sweet server
-      const doc = new Y.Doc();
       createYjsProvider(doc, clientToken, {disableBc: true});
 
-      const anno = doc.getMap('annotation');
+      // reconstruct the initial annotation
+      const comps = doc.getMap<{type: ComponentDetail['type']; sliceIndex: number; entityId: string}>('components');
+      const rects = doc.getMap<Pick<RectangleComponent, 'topLeft' | 'bottomRight'>>('rectangles');
       const annoJson = await new Promise<string>(resolve => {
-        anno.observe(() => {
-          resolve(JSON.stringify(anno.toJSON()));
+        const anno = emptyAnnotation();
+
+        comps.observe(() => {
+          for (const cid of comps.keys()) {
+            const info = comps.get(cid);
+            switch (info?.type) {
+              case 'rectangle': {
+                const {sliceIndex, entityId} = info;
+                const rect = rects.get(cid);
+                if (rect) {
+                  const {topLeft, bottomRight} = rect;
+                  const comp: RectangleComponent = {type: 'rectangle', topLeft, bottomRight};
+                  addAnnotationComponent(anno, sliceIndex, entityId, {id: cid, ...comp});
+                } else {
+                  console.warn(`missing rectangle info ${cid}`);
+                }
+                break;
+              }
+              default:
+                // TODO(xu)
+                console.warn('not implemented');
+            }
+          }
+
+          // TODO(xu): return directly the anno object
+          resolve(JSON.stringify(anno));
         });
       });
 
-      return {anno, annoJson, annoVersion: ''};
+      return {annoJson, annoVersion: ''};
     },
   });