diff --git a/app/frontend/src/common/annotation.ts b/app/frontend/src/common/annotation.ts new file mode 100644 index 0000000..f90bc01 --- /dev/null +++ b/app/frontend/src/common/annotation.ts @@ -0,0 +1,28 @@ +import {Annotation, Component, EntityId, SliceIndex} from 'type/annotation'; +import {deepClone} from './util'; + +export function addAnnotationComponent( + a: Annotation, + sliceIndex: SliceIndex, + entityId: EntityId, + component: Component +) { + if (entityId in a.entities) { + const slices = a.entities[entityId].geometry.slices; + if (!(sliceIndex in slices)) { + slices[sliceIndex] = {}; + } + slices[sliceIndex][component.id] = deepClone(component); + } else { + a.entities[entityId] = { + id: entityId, + geometry: { + slices: { + [sliceIndex]: { + [component.id]: deepClone(component), + }, + }, + }, + }; + } +} diff --git a/app/frontend/src/component/panel/AnnotationMonitor.tsx b/app/frontend/src/component/panel/AnnotationMonitor.tsx index 2c5b132..39984b5 100644 --- a/app/frontend/src/component/panel/AnnotationMonitor.tsx +++ b/app/frontend/src/component/panel/AnnotationMonitor.tsx @@ -1,12 +1,10 @@ import {FC, useContext, useEffect} from 'react'; -import * as Y from 'yjs'; import * as jsonmergepatch from 'json-merge-patch'; -import * as jsonpatch from 'fast-json-patch'; import {produce} from 'immer'; import {deleteAnnotationComponent, useStore as useAnnoStore} from 'state/annotate/annotation'; import {useStore as useRenderStore} from 'state/annotate/render'; -import {useGetVideoAnnotationV2, usePatchVideoAnnotation} from 'state/server/annotation'; +import {usePatchVideoAnnotation} from 'state/server/annotation'; import {useStore as useUIStore} from 'state/annotate/ui'; import {ConfigContext, NutshClientContext} from 'common/context'; @@ -17,10 +15,9 @@ import {Annotation} from 'type/annotation'; export const MonitorAnnotation: FC<{videoId: Video['id']}> = ({videoId}) => { const config = useContext(ConfigContext); - useSyncAnnotation({videoId}); return ( <> - {/* {!config.readonly && } */} + {!config.readonly && } @@ -135,91 +132,3 @@ function removeDraftComponents(anno: Annotation): Annotation { return draft; }); } - -function useSyncAnnotation({videoId}: {videoId: Video['id']}) { - const {data} = useGetVideoAnnotationV2(videoId); - - useEffect(() => { - const anno = data?.anno; - if (!anno) { - return; - } - const fn = (e: Y.YEvent>[]) => { - console.log(e); - }; - anno.observeDeep(fn); - return () => anno.unobserveDeep(fn); - }, [data?.anno]); - - useEffect(() => { - return useAnnoStore.subscribe( - s => s.annotation, - (curr, prev) => { - if (!data?.anno) { - return; - } - - const newPrev = produce(prev, removeDraftComponents); - const newCurr = produce(curr, removeDraftComponents); - - const ops = jsonpatch.compare(newPrev, newCurr); - ops.forEach(op => updateYjsFromJsonPathOperation(data.anno, op)); - }, - { - fireImmediately: true, - } - ); - }, [data?.anno]); -} - -function updateYjsFromJsonPathOperation(anno: Y.Map, op: jsonpatch.Operation) { - console.log(JSON.stringify(op)); - switch (op.op) { - case 'add': { - if (!op.path.startsWith('/')) { - console.warn(`unexpected json pointer: ${op.path}`); - return; - } - const paths = op.path.split('/'); - let dict = anno; - paths.slice(1, -1).forEach(path => { - if (!dict.has(path)) { - dict.set(path, new Y.Map()); - } - dict = dict.get(path) as Y.Map; - }); - const lastKey = paths[paths.length - 1]; - dict.set(lastKey, op.value); - break; - } - case 'copy': - console.warn(`not implemented: ${JSON.stringify(op)}`); - break; - case 'move': - console.warn(`not implemented: ${JSON.stringify(op)}`); - break; - case 'remove': { - if (!op.path.startsWith('/')) { - console.warn(`unexpected json pointer: ${op.path}`); - return; - } - const paths = op.path.split('/'); - let dict = anno; - paths.slice(1, -1).forEach(path => { - if (!dict.has(path)) { - dict.set(path, new Y.Map()); - } - dict = dict.get(path) as Y.Map; - }); - const lastKey = paths[paths.length - 1]; - dict.delete(lastKey); - break; - } - case 'replace': - console.warn(`not implemented: ${JSON.stringify(op)}`); - break; - case 'test': - console.warn(`not implemented: ${JSON.stringify(op)}`); - break; - } -} diff --git a/app/frontend/src/component/panel/AnnotationMonitorV2.tsx b/app/frontend/src/component/panel/AnnotationMonitorV2.tsx new file mode 100644 index 0000000..20897f8 --- /dev/null +++ b/app/frontend/src/component/panel/AnnotationMonitorV2.tsx @@ -0,0 +1,98 @@ +import {FC, useEffect} from 'react'; +import * as Y from 'yjs'; +import {ComponentInfo, RectangleAnchors, annoDoc} from 'sync/doc'; +import {useStore as useAnnoStore} from 'state/annotate/annotation'; +import {Component} from 'type/annotation'; + +export const MonitorAnnotationV2: FC = () => { + useSyncAnnotation(); + return null; +}; + +function useSyncAnnotation() { + useComponentInfoMonitor(); + useRectangleAnchorsMonitor(); +} + +function useComponentInfoMonitor() { + const comps = annoDoc.componentInfoMap(); + const rects = annoDoc.rectangleAnchorsMap(); + + const addComponent = useAnnoStore(s => s.addComponent); + const deleteComponents = useAnnoStore(s => s.deleteComponents); + + useEffect(() => { + const fn = (e: Y.YMapEvent) => { + for (const [cid, cc] of e.changes.keys) { + switch (cc.action) { + case 'add': { + const info = comps.get(cid); + if (!info) { + break; + } + + // add component + const {sliceIndex, entityId, type} = info; + switch (type) { + case 'rectangle': { + const rect = rects.get(cid); + if (!rect) { + break; + } + + const {topLeft, bottomRight} = rect; + const component: Component = { + type: 'rectangle', + id: cid, + topLeft, + bottomRight, + }; + addComponent({sliceIndex, entityId, component}); + break; + } + default: + // TODO(xu) + console.warn('not implemented'); + } + break; + } + case 'delete': { + // delete components + const {sliceIndex, entityId} = cc.oldValue as ComponentInfo; + deleteComponents({sliceIndex, components: [[entityId, cid]]}); + break; + } + } + } + }; + comps.observe(fn); + return () => comps.unobserve(fn); + }, [addComponent, comps, deleteComponents, rects]); +} + +function useRectangleAnchorsMonitor() { + const comps = annoDoc.componentInfoMap(); + const rects = annoDoc.rectangleAnchorsMap(); + + const updateAnchors = useAnnoStore(s => s.updateRectangleAnchors); + + useEffect(() => { + const fn = (e: Y.YMapEvent) => { + for (const cid of e.keysChanged) { + const info = comps.get(cid); + if (info) { + const {sliceIndex, entityId} = info; + const rect = rects.get(cid); + if (rect) { + const {topLeft, bottomRight} = rect; + updateAnchors({sliceIndex, entityId, componentId: cid, topLeft, bottomRight}); + } + } else { + console.warn(`rectangle ${cid} not found`); + } + } + }; + rects.observe(fn); + return () => rects.unobserve(fn); + }, [comps, rects, updateAnchors]); +} diff --git a/app/frontend/src/component/panel/layer/polychain/Draw.tsx b/app/frontend/src/component/panel/layer/polychain/Draw.tsx index e1f4ac1..47f5b89 100644 --- a/app/frontend/src/component/panel/layer/polychain/Draw.tsx +++ b/app/frontend/src/component/panel/layer/polychain/Draw.tsx @@ -134,6 +134,7 @@ const LayerWithEntityId: FC = ({entityId, width, h vertices: drawVertices, closed, }, + broadcast: true, }); finish(); diff --git a/app/frontend/src/component/panel/layer/rectangle/Draw.tsx b/app/frontend/src/component/panel/layer/rectangle/Draw.tsx index 12d4c49..501c3af 100644 --- a/app/frontend/src/component/panel/layer/rectangle/Draw.tsx +++ b/app/frontend/src/component/panel/layer/rectangle/Draw.tsx @@ -92,6 +92,7 @@ const LayerWithEntityId: FC = ({entityId, width, h topLeft: limitCoordinates(min, imw, imh), bottomRight: limitCoordinates(max, imw, imh), }, + broadcast: true, }); finish(); }} diff --git a/app/frontend/src/component/panel/layer/rectangle/Edit.tsx b/app/frontend/src/component/panel/layer/rectangle/Edit.tsx index 4182e77..aa10f38 100644 --- a/app/frontend/src/component/panel/layer/rectangle/Edit.tsx +++ b/app/frontend/src/component/panel/layer/rectangle/Edit.tsx @@ -45,7 +45,14 @@ const Canvas: FC<{data: Data} & CanvasHTMLAttributes> = ({dat const x2 = Math.max(p.x, q.x); const y2 = Math.max(p.y, q.y); - updateAnchors({sliceIndex, entityId, componentId, topLeft: {x: x1, y: y1}, bottomRight: {x: x2, y: y2}}); + updateAnchors({ + sliceIndex, + entityId, + componentId, + topLeft: {x: x1, y: y1}, + bottomRight: {x: x2, y: y2}, + broadcast: true, + }); finishEdit(); }, [anchors, componentId, entityId, finishEdit, sliceIndex, updateAnchors]); diff --git a/app/frontend/src/component/panel/menu/common.ts b/app/frontend/src/component/panel/menu/common.ts index 74c8ceb..9c68242 100644 --- a/app/frontend/src/component/panel/menu/common.ts +++ b/app/frontend/src/component/panel/menu/common.ts @@ -56,7 +56,7 @@ export function useComponentActions(entityId: EntityId, componentId: ComponentId }, { title: intl.get('menu.delete_hovering_component'), - fn: () => deleteComponents({sliceIndex, components: [[entityId, componentId]]}), + fn: () => deleteComponents({sliceIndex, components: [[entityId, componentId]], broadcast: true}), warning: intl.get('menu.warn.delete_hovering_component'), }, ]; @@ -128,7 +128,7 @@ export function useEntityActions(): Action[] { { title: intl.get('menu.delete_selected_components_in_current_frame'), fn: () => { - deleteComponents({sliceIndex, components}); + deleteComponents({sliceIndex, components, broadcast: true}); }, warning: intl.get('menu.warn.delete_selected_components_in_current_frame', {count: ec}), }, diff --git a/app/frontend/src/page/annotate/Panel/Loaded.tsx b/app/frontend/src/page/annotate/Panel/Loaded.tsx index 414273b..68b66aa 100644 --- a/app/frontend/src/page/annotate/Panel/Loaded.tsx +++ b/app/frontend/src/page/annotate/Panel/Loaded.tsx @@ -5,7 +5,7 @@ import shallow from 'zustand/shallow'; import {useStore as useRenderStore} from 'state/annotate/render'; import {prefetchImages} from 'state/image/store'; import {UI} from 'common/constant'; -import {MonitorAnnotation} from 'component/panel/AnnotationMonitor'; +import {MonitorAnnotationV2} from 'component/panel/AnnotationMonitorV2'; import type {Project, Video} from 'openapi/nutsh'; import type {ProjectSpec} from 'type/project_spec'; import {FrameSlider} from 'component/panel/FrameSlider'; @@ -72,7 +72,7 @@ export const PanelLoaded: FC<{ /> - + ); }; diff --git a/app/frontend/src/state/annotate/annotation.ts b/app/frontend/src/state/annotate/annotation.ts index 4ee82eb..a5a79ea 100644 --- a/app/frontend/src/state/annotate/annotation.ts +++ b/app/frontend/src/state/annotate/annotation.ts @@ -15,6 +15,8 @@ import type { ComponentMap, } from 'type/annotation'; import {newComponentAdapter} from 'common/adapter'; +import {addAnnotationComponent} from 'common/annotation'; +import {syncAnnotation} from 'sync/update'; export type State = { annotation: Annotation; @@ -65,6 +67,7 @@ export type AddComponentInput = { sliceIndex: SliceIndex; entityId: EntityId; component: Component; + broadcast?: boolean; }; export type AddComponentsInput = { @@ -85,6 +88,7 @@ export type TransferComponentInput = { export type DeleteComponentsInput = { sliceIndex: SliceIndex; components: [EntityId, ComponentId][]; + broadcast?: boolean; }; export type SeperateComponentInput = { @@ -123,6 +127,7 @@ export type UpdateRectangleAnchorsInput = { componentId: ComponentId; topLeft: Coordinates; bottomRight: Coordinates; + broadcast?: boolean; }; export type PasteInput = { @@ -248,8 +253,12 @@ export const useStore = create()( addComponent: (input: AddComponentInput) => { set(s => { - const {sliceIndex, entityId, component} = input; + const {sliceIndex, entityId, component, broadcast} = input; addComponent(s, sliceIndex, entityId, component); + + if (broadcast) { + syncAnnotation.addComponent(input); + } }); }, @@ -313,11 +322,12 @@ export const useStore = create()( deleteComponents: (input: DeleteComponentsInput) => { set(s => { - const {sliceIndex, components} = input; + const {sliceIndex, components, broadcast} = input; + components.forEach(([eid, cid]) => deleteComponent(s, sliceIndex, eid, cid)); - components.forEach(([eid, cid]) => { - deleteComponent(s, sliceIndex, eid, cid); - }); + if (broadcast) { + syncAnnotation.deleteComponents(input); + } }); }, @@ -394,7 +404,7 @@ export const useStore = create()( updateRectangleAnchors: (input: UpdateRectangleAnchorsInput) => { set(s => { - const {sliceIndex, entityId, componentId, topLeft, bottomRight} = input; + const {sliceIndex, entityId, componentId, topLeft, bottomRight, broadcast} = input; const slice = getSlice(s, sliceIndex, entityId); if (!slice) return; @@ -403,6 +413,10 @@ export const useStore = create()( if (old?.type !== 'rectangle') return; slice[componentId] = {...old, topLeft, bottomRight}; + + if (broadcast) { + syncAnnotation.updateRectangleAnchors(input); + } }); }, @@ -456,24 +470,7 @@ export const useStore = create()( export const useTemporalStore = create(useStore.temporal); function addComponent(s: State, sliceIndex: SliceIndex, entityId: EntityId, component: Component) { - if (entityId in s.annotation.entities) { - const slices = s.annotation.entities[entityId].geometry.slices; - if (!(sliceIndex in slices)) { - slices[sliceIndex] = {}; - } - slices[sliceIndex][component.id] = deepClone(component); - } else { - s.annotation.entities[entityId] = { - id: entityId, - geometry: { - slices: { - [sliceIndex]: { - [component.id]: deepClone(component), - }, - }, - }, - }; - } + addAnnotationComponent(s.annotation, sliceIndex, entityId, component); } function deleteComponent( diff --git a/app/frontend/src/state/server/annotation.ts b/app/frontend/src/state/server/annotation.ts index c5cbcf0..05d95eb 100644 --- a/app/frontend/src/state/server/annotation.ts +++ b/app/frontend/src/state/server/annotation.ts @@ -1,8 +1,11 @@ -import * as Y from 'yjs'; import {createYjsProvider} from '@y-sweet/client'; import {useQuery, useMutation} from '@tanstack/react-query'; import type {NutshClient, DefaultService, Video} from 'openapi/nutsh'; +import {RectangleComponent} from 'type/annotation'; +import {addAnnotationComponent} from 'common/annotation'; +import {emptyAnnotation} from 'state/annotate/render'; +import {annoDoc} from 'sync/doc'; export const useGetVideoAnnotation = (client: NutshClient, id: Video['id']) => useQuery({ @@ -28,16 +31,47 @@ export const useGetVideoAnnotationV2 = (id: Video['id']) => const clientToken = await res.json(); // create a Yjs document and connect it to the Y-Sweet server - const doc = new Y.Doc(); - createYjsProvider(doc, clientToken, {disableBc: true}); + const doc = annoDoc.doc(); + const provider = createYjsProvider(doc, clientToken, {disableBc: true}); - const anno = doc.getMap('annotation'); + // reconstruct the initial annotation const annoJson = await new Promise(resolve => { - anno.observe(() => { - resolve(JSON.stringify(anno.toJSON())); + provider.on('synced', (isSynced: boolean) => { + if (!isSynced) { + return; + } + + const comps = annoDoc.componentInfoMap(); + const rects = annoDoc.rectangleAnchorsMap(); + + const anno = emptyAnnotation(); + for (const cid of comps.keys()) { + const info = comps.get(cid); + switch (info?.type) { + case 'rectangle': { + const rect = rects.get(cid); + if (!rect) { + console.warn(`rectangle ${cid} not found`); + break; + } + + const {sliceIndex, entityId} = info; + const {topLeft, bottomRight} = rect; + const comp: RectangleComponent = {type: 'rectangle', topLeft, bottomRight}; + addAnnotationComponent(anno, sliceIndex, entityId, {id: cid, ...comp}); + break; + } + default: + // TODO(xu) + console.warn('not implemented'); + } + } + + // TODO(xu): return directly the anno object + resolve(JSON.stringify(anno)); }); }); - return {anno, annoJson, annoVersion: ''}; + return {annoJson, annoVersion: ''}; }, }); diff --git a/app/frontend/src/sync/doc.ts b/app/frontend/src/sync/doc.ts new file mode 100644 index 0000000..137e0d3 --- /dev/null +++ b/app/frontend/src/sync/doc.ts @@ -0,0 +1,41 @@ +import * as Y from 'yjs'; +import {ComponentDetail, RectangleComponent} from 'type/annotation'; + +export type ComponentInfo = { + type: ComponentDetail['type']; + sliceIndex: number; + entityId: string; +}; + +export type RectangleAnchors = Pick; + +const mapKeys = { + componentInfo: 'componentInfo', + rectangleAnchors: 'rectangleAnchors', +}; + +export class AnnotationDoc { + private doc_: Y.Doc; + + constructor() { + this.doc_ = new Y.Doc(); + } + + doc = (): Y.Doc => { + return this.doc_; + }; + + transact = (fn: () => void) => { + this.doc_.transact(fn); + }; + + componentInfoMap = (): Y.Map => { + return this.doc_.getMap(mapKeys.componentInfo); + }; + + rectangleAnchorsMap = (): Y.Map => { + return this.doc_.getMap(mapKeys.rectangleAnchors); + }; +} + +export const annoDoc = new AnnotationDoc(); diff --git a/app/frontend/src/sync/update.ts b/app/frontend/src/sync/update.ts new file mode 100644 index 0000000..b26bb45 --- /dev/null +++ b/app/frontend/src/sync/update.ts @@ -0,0 +1,110 @@ +import { + AddComponentInput, + AddComponentsInput, + ClearEntityCategoryInput, + DeleteComponentsInput, + DeleteEntitiesInput, + DeletePolychainVertexInput, + PasteInput, + SeperateComponentInput, + SetEntityCategoryInput, + SetPolychainVertexBezierInput, + State, + TransferComponentInput, + TranslateInput, + TruncateEntitiesInput, + UpdatePolychainVerticesInput, + UpdateRectangleAnchorsInput, + UpdateSliceMasksInput, +} from 'state/annotate/annotation'; +import {annoDoc} from './doc'; + +export const syncAnnotation: Omit = { + setEntityCategory: (input: SetEntityCategoryInput) => { + console.error('not implemented', input); + }, + clearEntityCategory: (input: ClearEntityCategoryInput) => { + console.error('not implemented', input); + }, + deleteEntities: (input: DeleteEntitiesInput) => { + console.error('not implemented', input); + }, + truncateEntities: (input: TruncateEntitiesInput) => { + console.error('not implemented', input); + }, + addComponent: (input: AddComponentInput) => { + const {sliceIndex, entityId, component} = input; + switch (component.type) { + case 'rectangle': { + const comps = annoDoc.componentInfoMap(); + const rects = annoDoc.rectangleAnchorsMap(); + annoDoc.transact(() => { + const {id: cid, topLeft, bottomRight, type} = component; + comps.set(cid, {type, sliceIndex, entityId}); + rects.set(cid, {topLeft, bottomRight}); + }); + break; + } + default: + console.error('not implemented', input); + } + }, + addComponents: (input: AddComponentsInput) => { + console.error('not implemented', input); + }, + transferComponent: (input: TransferComponentInput) => { + console.error('not implemented', input); + }, + deleteComponents: (input: DeleteComponentsInput) => { + const {components} = input; + annoDoc.transact(() => { + components.forEach(([, cid]) => { + const comps = annoDoc.componentInfoMap(); + const comp = comps.get(cid); + if (!comp) { + console.warn(`component ${cid} not found`); + return; + } + comps.delete(cid); + + const {type} = comp; + switch (type) { + case 'rectangle': + annoDoc.rectangleAnchorsMap().delete(cid); + break; + default: + console.error('not implemented', input); + } + }); + }); + }, + seperateComponent: (input: SeperateComponentInput) => { + console.error('not implemented', input); + }, + commitDraftComponents: () => { + console.error('not implemented'); + }, + updatePolychainVertices: (input: UpdatePolychainVerticesInput) => { + console.error('not implemented', input); + }, + deletePolychainVertex: (input: DeletePolychainVertexInput) => { + console.error('not implemented', input); + }, + setPolychainVertexBezier: (input: SetPolychainVertexBezierInput) => { + console.error('not implemented', input); + }, + updateSliceMasks: (input: UpdateSliceMasksInput) => { + console.error('not implemented', input); + }, + updateRectangleAnchors: (input: UpdateRectangleAnchorsInput) => { + const {componentId, topLeft, bottomRight} = input; + const rects = annoDoc.rectangleAnchorsMap(); + rects.set(componentId, {topLeft, bottomRight}); + }, + paste: (input: PasteInput) => { + console.error('not implemented', input); + }, + translate: (input: TranslateInput) => { + console.error('not implemented', input); + }, +};