diff --git a/.github/workflows/pretest.yaml b/.github/workflows/pretest.yaml index b7b9dc9..e7d9334 100644 --- a/.github/workflows/pretest.yaml +++ b/.github/workflows/pretest.yaml @@ -15,8 +15,15 @@ jobs: FROM project-dev ADD . /project - # create an empty file to make `go:embed` happy - RUN cd /project && mkdir app/frontend/build && touch app/frontend/build/test && mkdir docs/build && touch docs/build/test + # create empty files to make `go:embed` happy + RUN \ + cd /project && \ + mkdir app/frontend/build && \ + touch app/frontend/build/test && \ + mkdir docs/build && \ + touch docs/build/test && \ + mkdir app/yjs-server/bin && \ + touch app/yjs-server/bin/yjs-server # generate code RUN cd /project && task openapi && task proto diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b174ce6..f5ac507 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -38,10 +38,10 @@ jobs: task frontend:build && \ task docs:build RUN \ - GOOS=linux GOARCH=amd64 task backend:build && mv build/nutsh build/nutsh-Linux-x86_64 && \ - GOOS=linux GOARCH=arm64 task backend:build && mv build/nutsh build/nutsh-Linux-arm64 && \ - GOOS=darwin GOARCH=amd64 task backend:build && mv build/nutsh build/nutsh-Darwin-x86_64 && \ - GOOS=darwin GOARCH=arm64 task backend:build && mv build/nutsh build/nutsh-Darwin-arm64 + GOOS=linux GOARCH=amd64 task yjs:build && task backend:build && mv build/nutsh build/nutsh-Linux-x86_64 && \ + GOOS=linux GOARCH=arm64 task yjs:build && task backend:build && mv build/nutsh build/nutsh-Linux-arm64 && \ + GOOS=darwin GOARCH=amd64 task yjs:build && task backend:build && mv build/nutsh build/nutsh-Darwin-x86_64 && \ + GOOS=darwin GOARCH=arm64 task yjs:build && task backend:build && mv build/nutsh build/nutsh-Darwin-arm64 RUN \ GOOS=linux GOARCH=amd64 task sam:build && mv build/nutsh-sam build/nutsh-sam-Linux-x86_64 && \ GOOS=linux GOARCH=arm64 task sam:build && mv build/nutsh-sam build/nutsh-sam-Linux-arm64 && \ diff --git a/.gitignore b/.gitignore index 12c609f..3e7c0f9 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ openapi/gen build local *.local -__pycache__ \ No newline at end of file +__pycache__ +bin \ No newline at end of file diff --git a/Taskfile.yaml b/Taskfile.yaml index d112ea8..62a81e8 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -16,6 +16,7 @@ vars: includes: frontend: task/frontend.yaml backend: task/backend.yaml + yjs: task/yjs.yaml e2e: task/e2e.yaml container: task/container.yaml deploy: task/deploy.yaml @@ -45,6 +46,7 @@ tasks: build: cmds: - rm -f build/nutsh + - task: yjs:build - task: frontend:build - task: docs:build - task: backend:build @@ -52,6 +54,7 @@ tasks: build-dev: cmds: - rm -f build/nutsh-dev + - task: yjs:build - task: frontend:build-dev - task: backend:build - mv build/nutsh build/nutsh-dev diff --git a/app/action/option.go b/app/action/option.go index 3dcb391..b67f7a3 100644 --- a/app/action/option.go +++ b/app/action/option.go @@ -7,8 +7,9 @@ var StorageOption struct { } var StartOption struct { - Frontend fs.FS - Doc fs.FS + Frontend fs.FS + YJSServer []byte + Doc fs.FS Port int Readonly bool diff --git a/app/action/start.go b/app/action/start.go index 29518a5..d03a2e6 100644 --- a/app/action/start.go +++ b/app/action/start.go @@ -3,8 +3,12 @@ package action import ( "context" "fmt" + "net" "net/http" + "net/http/httputil" + "net/url" "os" + "os/exec" "path/filepath" "github.com/labstack/echo/v4" @@ -33,30 +37,12 @@ func Start(ctx context.Context) error { e.HideBanner = true middlewares := []echo.MiddlewareFunc{ middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{ - LogURI: true, - LogStatus: true, - LogMethod: true, - LogLatency: true, - LogError: true, - LogValuesFunc: func(c echo.Context, v middleware.RequestLoggerValues) error { - // https://github.com/labstack/echo/issues/2015 - status := v.Status - if v.Error != nil { - switch e := v.Error.(type) { - case *echo.HTTPError: - status = e.Code - default: - status = http.StatusInternalServerError - } - } - zap.L().Info("request", - zap.String("method", v.Method), - zap.String("uri", v.URI), - zap.Duration("latency", v.Latency), - zap.Int("status", status), - ) - return nil - }, + LogURI: true, + LogStatus: true, + LogMethod: true, + LogLatency: true, + LogError: true, + LogValuesFunc: logValuesFunc, }), middleware.GzipWithConfig(middleware.GzipConfig{ Level: 9, @@ -68,6 +54,26 @@ func Start(ctx context.Context) error { } e.Use(middlewares...) + // proxy yjs-server ws + yjsPort := mustStartYJSServer() + e.Any("/ws/*", func(c echo.Context) error { + target := fmt.Sprintf("http://127.0.0.1:%d", yjsPort) + targetUrl, err := url.Parse(target) + if err != nil { + return err + } + + proxy := httputil.NewSingleHostReverseProxy(targetUrl) + proxy.Director = func(req *http.Request) { + req.URL.Scheme = targetUrl.Scheme + req.URL.Host = targetUrl.Host + req.URL.Path = "/" + c.Param("*") + } + + proxy.ServeHTTP(c.Response(), c.Request()) + return nil + }) + // local data if StartOption.DataDir != "" { zap.L().Info("serving local data", zap.String("dir", StartOption.DataDir)) @@ -105,11 +111,31 @@ func Start(ctx context.Context) error { return e.Start(lisAddr) } +func logValuesFunc(c echo.Context, v middleware.RequestLoggerValues) error { + // https://github.com/labstack/echo/issues/2015 + status := v.Status + if v.Error != nil { + switch e := v.Error.(type) { + case *echo.HTTPError: + status = e.Code + default: + status = http.StatusInternalServerError + } + } + zap.L().Info("request", + zap.String("method", v.Method), + zap.String("uri", v.URI), + zap.Duration("latency", v.Latency), + zap.Int("status", status), + ) + return nil +} + func createServer() (backend.Server, func(), error) { var opts []backend.Option // storage - db, err := sqlite3.New(filepath.Join(databaseDir(), "db.sqlite3")) + db, err := sqlite3.New(databasePath()) if err != nil { return nil, nil, err } @@ -190,3 +216,72 @@ func databaseDir() string { } return filepath.Join(StorageOption.Workspace, "database") } + +func databasePath() string { + return filepath.Join(databaseDir(), "db.sqlite3") +} + +func mustStartYJSServer() int { + // create a temporary file + bin, err := os.CreateTemp("", "nutsh-yjs-*") + mustOk(err) + + // write the embedded binary to the temporary file + _, err = bin.Write(StartOption.YJSServer) + mustOk(err) + mustOk(bin.Close()) + + // make the file executable + mustOk(os.Chmod(bin.Name(), 0755)) + + // prepare arguments + internalPort := mustFindFreePort() + + // set envs + envs := []string{ + fmt.Sprintf("PORT=%d", internalPort), + fmt.Sprintf("DATABASE_PATH=%s", databasePath()), + } + if StartOption.Readonly { + envs = append(envs, "READ_ONLY=true") + } + + // execute the binary in a new process + cmd := exec.Command(bin.Name()) + cmd.Env = envs + zap.L().Info("start yjs-server server", zap.Int("port", internalPort)) + + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + go func() { + defer os.Remove(bin.Name()) + mustOk(cmd.Run()) + }() + + return internalPort +} + +func mustFindFreePort() int { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + mustOk(err) + + l, err := net.ListenTCP("tcp", addr) + mustOk(err) + defer l.Close() + + return l.Addr().(*net.TCPAddr).Port +} + +func mustOk(err error) { + if err == nil { + return + } + + if os.Getenv("DEBUG") != "" { + fmt.Printf("%+v\n", err) + } else { + fmt.Printf("%v\n", err) + } + os.Exit(1) +} diff --git a/app/backend/segmentation.go b/app/backend/segmentation.go index 5adb8d1..8a878b0 100644 --- a/app/backend/segmentation.go +++ b/app/backend/segmentation.go @@ -32,7 +32,7 @@ func (s *mServer) GetOnlineSegmentation(ctx context.Context, request nutshapi.Ge return resp, nil } -func (s *mServer) getOnlineSegmentation(ctx context.Context, request nutshapi.GetOnlineSegmentationRequestObject) (nutshapi.GetOnlineSegmentationResponseObject, error) { +func (s *mServer) getOnlineSegmentation(ctx context.Context, _ nutshapi.GetOnlineSegmentationRequestObject) (nutshapi.GetOnlineSegmentationResponseObject, error) { opts := s.options store := opts.storagePublic diff --git a/app/frontend/.eslintrc.json b/app/frontend/.eslintrc.json index 81648b6..b004dfd 100644 --- a/app/frontend/.eslintrc.json +++ b/app/frontend/.eslintrc.json @@ -1,11 +1,15 @@ { - "extends": [ - "./node_modules/gts/", - "plugin:react/jsx-runtime" - ], + "extends": ["./node_modules/gts/", "plugin:react/jsx-runtime"], "plugins": ["react-hooks"], "rules": { - "react-hooks/rules-of-hooks": "error", + "react-hooks/rules-of-hooks": "error", "react-hooks/exhaustive-deps": "warn" - } -} \ No newline at end of file + }, + "overrides": [ + { + // I don't know why `eslint-disable-next-line` does not work for webpack, so I configure here as a workaround. + "files": ["src/state/annotate/annotation-broadcast.ts"], + "rules": {"@typescript-eslint/no-explicit-any": "off"} + } + ] +} diff --git a/app/frontend/craco.config.js b/app/frontend/craco.config.js index c1f278b..9e79524 100644 --- a/app/frontend/craco.config.js +++ b/app/frontend/craco.config.js @@ -1,8 +1,12 @@ // eslint-disable-next-line node/no-unpublished-require const CopyWebpackPlugin = require('copy-webpack-plugin'); +const path = require('path'); module.exports = { webpack: { + alias: { + '@@frontend': path.resolve(__dirname, 'src/'), + }, configure: config => { config.plugins.push( new CopyWebpackPlugin({ diff --git a/app/frontend/package-lock.json b/app/frontend/package-lock.json index e7441d8..b28b517 100644 --- a/app/frontend/package-lock.json +++ b/app/frontend/package-lock.json @@ -39,6 +39,8 @@ "ts-key-enum": "^2.0.12", "typescript": "^5.1.6", "uuid": "^9.0.0", + "y-websocket": "^1.5.4", + "yjs": "^13.6.12", "zundo": "^2.0.0-beta.5", "zustand": "^4.1.4" }, @@ -5308,6 +5310,22 @@ "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==" }, + "node_modules/abstract-leveldown": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", + "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", + "optional": true, + "dependencies": { + "buffer": "^5.5.0", + "immediate": "^3.2.3", + "level-concat-iterator": "~2.0.0", + "level-supports": "~1.0.0", + "xtend": "~4.0.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -5777,6 +5795,12 @@ "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" }, + "node_modules/async-limiter": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", + "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==", + "optional": true + }, "node_modules/async-validator": { "version": "4.2.5", "resolved": "https://registry.npmjs.org/async-validator/-/async-validator-4.2.5.tgz", @@ -6128,6 +6152,26 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true + }, "node_modules/batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -6299,6 +6343,30 @@ "node-int64": "^0.4.0" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true, + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -7579,6 +7647,19 @@ "node": ">= 10" } }, + "node_modules/deferred-leveldown": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz", + "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==", + "optional": true, + "dependencies": { + "abstract-leveldown": "~6.2.1", + "inherits": "^2.0.3" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/define-lazy-prop": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz", @@ -7924,6 +8005,21 @@ "node": ">= 0.8" } }, + "node_modules/encoding-down": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz", + "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==", + "optional": true, + "dependencies": { + "abstract-leveldown": "^6.2.1", + "inherits": "^2.0.3", + "level-codec": "^9.0.0", + "level-errors": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/enhanced-resolve": { "version": "5.10.0", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.10.0.tgz", @@ -7956,6 +8052,18 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/errno": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", + "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "optional": true, + "dependencies": { + "prr": "~1.0.1" + }, + "bin": { + "errno": "cli.js" + } + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -10837,6 +10945,26 @@ "node": ">=4" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true + }, "node_modules/ignore": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", @@ -10845,6 +10973,12 @@ "node": ">= 4" } }, + "node_modules/immediate": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.3.0.tgz", + "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==", + "optional": true + }, "node_modules/immer": { "version": "9.0.19", "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.19.tgz", @@ -11490,6 +11624,15 @@ "node": ">=0.10.0" } }, + "node_modules/isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==", + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/istanbul-lib-coverage": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz", @@ -13754,6 +13897,139 @@ "language-subtag-registry": "~0.3.2" } }, + "node_modules/level": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/level/-/level-6.0.1.tgz", + "integrity": "sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw==", + "optional": true, + "dependencies": { + "level-js": "^5.0.0", + "level-packager": "^5.1.0", + "leveldown": "^5.4.0" + }, + "engines": { + "node": ">=8.6.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/level" + } + }, + "node_modules/level-codec": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.2.tgz", + "integrity": "sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==", + "optional": true, + "dependencies": { + "buffer": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/level-concat-iterator": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", + "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==", + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/level-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", + "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", + "optional": true, + "dependencies": { + "errno": "~0.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/level-iterator-stream": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz", + "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==", + "optional": true, + "dependencies": { + "inherits": "^2.0.4", + "readable-stream": "^3.4.0", + "xtend": "^4.0.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/level-js": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/level-js/-/level-js-5.0.2.tgz", + "integrity": "sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg==", + "optional": true, + "dependencies": { + "abstract-leveldown": "~6.2.3", + "buffer": "^5.5.0", + "inherits": "^2.0.3", + "ltgt": "^2.1.2" + } + }, + "node_modules/level-packager": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-5.1.1.tgz", + "integrity": "sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ==", + "optional": true, + "dependencies": { + "encoding-down": "^6.3.0", + "levelup": "^4.3.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/level-supports": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", + "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", + "optional": true, + "dependencies": { + "xtend": "^4.0.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/leveldown": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.6.0.tgz", + "integrity": "sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "abstract-leveldown": "~6.2.1", + "napi-macros": "~2.0.0", + "node-gyp-build": "~4.1.0" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/levelup": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.4.0.tgz", + "integrity": "sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==", + "optional": true, + "dependencies": { + "deferred-leveldown": "~5.3.0", + "level-errors": "~2.0.0", + "level-iterator-stream": "~4.0.0", + "level-supports": "~1.0.0", + "xtend": "~4.0.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -13774,6 +14050,25 @@ "node": ">= 0.8.0" } }, + "node_modules/lib0": { + "version": "0.2.89", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.89.tgz", + "integrity": "sha512-5j19vcCjsQhvLG6mcDD+nprtJUCbmqLz5Hzt5xgi9SV6RIW/Dty7ZkVZHGBuPOADMKjQuKDvuQTH495wsmw8DQ==", + "dependencies": { + "isomorphic.js": "^0.2.4" + }, + "bin": { + "0gentesthtml": "bin/gentesthtml.js", + "0serve": "bin/0serve.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/lilconfig": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.0.6.tgz", @@ -13893,6 +14188,12 @@ "node": ">=10" } }, + "node_modules/ltgt": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", + "integrity": "sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA==", + "optional": true + }, "node_modules/magic-string": { "version": "0.25.9", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.9.tgz", @@ -14260,6 +14561,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-macros": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", + "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==", + "optional": true + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -14347,6 +14654,17 @@ "node": ">= 6.13.0" } }, + "node_modules/node-gyp-build": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", + "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -16395,6 +16713,12 @@ "node": ">= 0.10" } }, + "node_modules/prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==", + "optional": true + }, "node_modules/psl": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", @@ -20699,6 +21023,80 @@ "node": ">=0.4" } }, + "node_modules/y-leveldb": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/y-leveldb/-/y-leveldb-0.1.2.tgz", + "integrity": "sha512-6ulEn5AXfXJYi89rXPEg2mMHAyyw8+ZfeMMdOtBbV8FJpQ1NOrcgi6DTAcXof0dap84NjHPT2+9d0rb6cFsjEg==", + "optional": true, + "dependencies": { + "level": "^6.0.1", + "lib0": "^0.2.31" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "yjs": "^13.0.0" + } + }, + "node_modules/y-protocols": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.6.tgz", + "integrity": "sha512-vHRF2L6iT3rwj1jub/K5tYcTT/mEYDUppgNPXwp8fmLpui9f7Yeq3OEtTLVF012j39QnV+KEQpNqoN7CWU7Y9Q==", + "dependencies": { + "lib0": "^0.2.85" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "yjs": "^13.0.0" + } + }, + "node_modules/y-websocket": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/y-websocket/-/y-websocket-1.5.4.tgz", + "integrity": "sha512-Y3021uy0anOIHqAPyAZbNDoR05JuMEGjRNI8c+K9MHzVS8dWoImdJUjccljAznc8H2L7WkIXhRHZ1igWNRSgPw==", + "dependencies": { + "lib0": "^0.2.52", + "lodash.debounce": "^4.0.8", + "y-protocols": "^1.0.5" + }, + "bin": { + "y-websocket": "bin/server.js", + "y-websocket-server": "bin/server.js" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "optionalDependencies": { + "ws": "^6.2.1", + "y-leveldb": "^0.1.0" + }, + "peerDependencies": { + "yjs": "^13.5.6" + } + }, + "node_modules/y-websocket/node_modules/ws": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.2.tgz", + "integrity": "sha512-zmhltoSR8u1cnDsD43TX59mzoMZsLKqUweyYBAIvTngR3shc0W6aOZylZmq/7hqyVxPdi+5Ud2QInblgyE72fw==", + "optional": true, + "dependencies": { + "async-limiter": "~1.0.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -20745,6 +21143,22 @@ "node": ">=10" } }, + "node_modules/yjs": { + "version": "13.6.12", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.12.tgz", + "integrity": "sha512-KOT8ILoyVH2f/PxPadeu5kVVS055D1r3x1iFfJVJzFdnN98pVGM8H07NcKsO+fG3F7/0tf30Vnokf5YIqhU/iw==", + "dependencies": { + "lib0": "^0.2.86" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/app/frontend/package.json b/app/frontend/package.json index 8435f5e..c9e98ff 100644 --- a/app/frontend/package.json +++ b/app/frontend/package.json @@ -41,6 +41,8 @@ "ts-key-enum": "^2.0.12", "typescript": "^5.1.6", "uuid": "^9.0.0", + "y-websocket": "^1.5.4", + "yjs": "^13.6.12", "zundo": "^2.0.0-beta.5", "zustand": "^4.1.4" }, diff --git a/app/frontend/src/common/annotation.ts b/app/frontend/src/common/annotation.ts new file mode 100644 index 0000000..cc99372 --- /dev/null +++ b/app/frontend/src/common/annotation.ts @@ -0,0 +1,82 @@ +import type {Annotation, Component, Entity, EntityId, SliceIndex, Vertex} from '@@frontend/type/annotation'; +import {deepClone} from './util'; + +export function addAnnotationComponent( + a: Annotation, + sliceIndex: SliceIndex, + entityId: EntityId, + component: Component +) { + if (entityId in a.entities) { + const slices = a.entities[entityId].geometry.slices; + if (!(sliceIndex in slices)) { + slices[sliceIndex] = {}; + } + slices[sliceIndex][component.id] = deepClone(component); + } else { + a.entities[entityId] = { + id: entityId, + geometry: { + slices: { + [sliceIndex]: { + [component.id]: deepClone(component), + }, + }, + }, + }; + } +} + +export function setEntityCategory( + e: Entity, + category: string, + entries: string[], + sliceIndex: SliceIndex | undefined = undefined +) { + if (sliceIndex !== undefined) { + if (!e.sliceCategories) { + e.sliceCategories = {}; + } + if (!e.sliceCategories[sliceIndex]) { + e.sliceCategories[sliceIndex] = {}; + } + if (!e.sliceCategories[sliceIndex][category]) { + e.sliceCategories[sliceIndex][category] = {}; + } + e.sliceCategories[sliceIndex][category] = Object.fromEntries(entries.map(e => [e, true])); + } else { + if (!e.globalCategories) { + e.globalCategories = {}; + } + if (!e.globalCategories[category]) { + e.globalCategories[category] = {}; + } + e.globalCategories[category] = Object.fromEntries(entries.map(e => [e, true])); + } +} + +export function initialVertexBezier(vertexIndex: number, vertices: Vertex[]): NonNullable { + const n = vertices.length; + const i = vertexIndex; + const j = (vertexIndex + n - 1) % n; + const {x: x1, y: y1} = vertices[i].coordinates; + const {x: x2, y: y2} = vertices[j].coordinates; + const [cx1, cy1] = [(x1 * 3) / 4 + (x2 * 1) / 4, (y1 * 3) / 4 + (y2 * 1) / 4]; + const [cx2, cy2] = [(x1 * 1) / 4 + (x2 * 3) / 4, (y1 * 1) / 4 + (y2 * 3) / 4]; + + const dx = x2 - x1; + const dy = y2 - y1; + const l = Math.hypot(dx, dy); + const d = l / 4; + + return { + control1: { + x: cx2 + (d * dy) / l, + y: cy2 - (d * dx) / l, + }, + control2: { + x: cx1 - (d * dy) / l, + y: cy1 + (d * dx) / l, + }, + }; +} diff --git a/app/frontend/src/common/hook.ts b/app/frontend/src/common/hook.ts index 672e63a..e2aaf4a 100644 --- a/app/frontend/src/common/hook.ts +++ b/app/frontend/src/common/hook.ts @@ -3,7 +3,8 @@ import {v4 as uuidv4} from 'uuid'; import {deepEqual} from 'common/util'; import {EntityComponentId, useStore as useRenderStore} from 'state/annotate/render'; -import {useStore as useAnnoStore, getComponent, getSlice} from 'state/annotate/annotation'; +import {getComponent, getSlice} from 'state/annotate/annotation'; +import {useAnnoStore} from 'state/annotate/annotation-provider'; import {useStore as useUIStore} from 'state/annotate/ui'; import {useStore as useDrawPolyStore} from 'state/annotate/polychain/draw'; import {useStore as useDrawRectStore} from 'state/annotate/rectangle/draw'; diff --git a/app/frontend/src/common/mouse.ts b/app/frontend/src/common/mouse.ts new file mode 100644 index 0000000..ad3f531 --- /dev/null +++ b/app/frontend/src/common/mouse.ts @@ -0,0 +1,7 @@ +export function relativeMousePosition(event: React.MouseEvent, ele: T) { + const rect = ele.getBoundingClientRect(); + return { + x: event.clientX - rect.left, + y: event.clientY - rect.top, + }; +} diff --git a/app/frontend/src/common/render.ts b/app/frontend/src/common/render.ts index a27f99f..90ccc35 100644 --- a/app/frontend/src/common/render.ts +++ b/app/frontend/src/common/render.ts @@ -1,6 +1,6 @@ import {useCallback} from 'react'; -import {useStore as useAnnoStore} from 'state/annotate/annotation'; +import {useAnnoStore} from 'state/annotate/annotation-provider'; import {useStore as useRenderStore} from 'state/annotate/render'; import {ViewportTransform} from 'state/annotate/render/viewport'; diff --git a/app/frontend/src/common/util.ts b/app/frontend/src/common/util.ts index be18cca..b37bf5a 100644 --- a/app/frontend/src/common/util.ts +++ b/app/frontend/src/common/util.ts @@ -7,11 +7,3 @@ export function deepClone(obj: T): T { export function deepEqual(a: T, b: T): boolean { return deepCompare(a, b); } - -export function relativeMousePosition(event: React.MouseEvent, ele: T) { - const rect = ele.getBoundingClientRect(); - return { - x: event.clientX - rect.left, - y: event.clientY - rect.top, - }; -} diff --git a/app/frontend/src/common/yjs/context.tsx b/app/frontend/src/common/yjs/context.tsx new file mode 100644 index 0000000..2408540 --- /dev/null +++ b/app/frontend/src/common/yjs/context.tsx @@ -0,0 +1,16 @@ +import * as Y from 'yjs'; +import {ReactNode, createContext, useContext} from 'react'; + +interface ContextData { + doc: Y.Doc; +} + +const YjsContext = createContext(undefined!); +export function useYjsContext(): ContextData { + return useContext(YjsContext); +} + +export function YjsProvider({children}: {children: ReactNode}): JSX.Element { + const doc = new Y.Doc(); + return {children}; +} diff --git a/app/frontend/src/common/yjs/convert.ts b/app/frontend/src/common/yjs/convert.ts new file mode 100644 index 0000000..2fc4948 --- /dev/null +++ b/app/frontend/src/common/yjs/convert.ts @@ -0,0 +1,155 @@ +import * as Y from 'yjs'; +import type { + Annotation, + Component, + ComponentId, + EntityId, + PolychainComponent, + RectangleComponent, + SliceIndex, +} from '@@frontend/type/annotation'; +import {addAnnotationComponent, setEntityCategory} from '@@frontend/common/annotation'; +import {yjsComponentMap, Component as YjsComponent} from './docs/component'; +import {yjsRectangleAnchorsMap} from './docs/rectangle'; +import {yjsPolychainVerticesMap} from './docs/polychain'; +import {yjsMaskMap} from './docs/mask'; +import {encodeEntityCategoryMapKey, decodeEntityCategoryMapKey, yjsEntityCategoriesMap} from './docs/entity'; + +export function readAnnotationFromYjs(doc: Y.Doc): Annotation { + const anno: Annotation = {entities: {}}; + + // components + const comps = yjsComponentMap(doc); + for (const cid of comps.keys()) { + const info = comps.get(cid); + if (!info) { + continue; + } + const comp = readComponent(doc, cid, info); + if (!comp) { + continue; + } + const {sidx, eid} = info; + addAnnotationComponent(anno, sidx, eid, comp); + } + + // categories + const cats = yjsEntityCategoriesMap(doc); + for (const [key, entries] of cats.entries()) { + const decoded = decodeEntityCategoryMapKey(key); + if (!decoded) { + console.warn('unexpected entity category key', key); + continue; + } + const {eid, sidx, category} = decoded; + const entity = anno.entities[eid]; + if (!entity) { + console.warn('entity not found for category', key); + cats.delete(key); + continue; + } + setEntityCategory(entity, category, entries, sidx); + } + + return anno; +} + +export function writeAnnotationToYjs(anno: Annotation, doc: Y.Doc): void { + // Wrap the writing in a single transaction to make sure that only ONE `update` event is triggered. + // https://beta.yjs.dev/docs/api/transactions/#optimizing-bulk-changes + doc.transact(() => { + Object.entries(anno.entities).forEach(([eid, entity]) => { + Object.entries(entity.geometry.slices).forEach(([sidx, slice]) => { + Object.values(slice).forEach(comp => { + writeComponent(doc, comp, eid, parseInt(sidx)); + }); + }); + Object.entries(entity.globalCategories || {}).forEach(([cat, entries]) => { + writeEntityCategory(doc, Object.keys(entries), cat, eid); + }); + Object.entries(entity.sliceCategories || {}).forEach(([sidx, scats]) => { + Object.entries(scats).forEach(([cat, entries]) => { + writeEntityCategory(doc, Object.keys(entries), cat, eid, parseInt(sidx)); + }); + }); + }); + }); +} + +export function readComponent(doc: Y.Doc, cid: ComponentId, info: YjsComponent): Component | undefined { + const comps = yjsComponentMap(doc); + const anchors = yjsRectangleAnchorsMap(doc); + const verts = yjsPolychainVerticesMap(doc); + const masks = yjsMaskMap(doc); + + switch (info.type) { + case 'rectangle': { + const rect = anchors.get(cid); + if (!rect) { + console.warn(`rectangle ${cid} not found`); + comps.delete(cid); + break; + } + const {topLeft, bottomRight} = rect; + const comp: RectangleComponent = {type: 'rectangle', topLeft, bottomRight}; + return {...comp, id: cid}; + } + case 'polychain': { + const vs = verts.get(cid); + if (!vs || vs.length === 0) { + console.warn(`polychain ${cid} vertices not found or is empty`); + comps.delete(cid); + break; + } + const {closed} = info; + const comp: PolychainComponent = {type: 'polychain', vertices: vs.toArray(), closed}; + return {...comp, id: cid}; + } + case 'mask': { + const mask = masks.get(cid); + if (!mask) { + console.warn(`mask ${cid} not found`); + comps.delete(cid); + break; + } + return {...mask, id: cid}; + } + default: + console.warn('unexpected'); + } + return undefined; +} + +function writeComponent(doc: Y.Doc, comp: Component, eid: EntityId, sidx: SliceIndex): void { + const comps = yjsComponentMap(doc); + const anchors = yjsRectangleAnchorsMap(doc); + const verts = yjsPolychainVerticesMap(doc); + const masks = yjsMaskMap(doc); + + switch (comp.type) { + case 'mask': { + const {id: cid, type, ...rest} = comp; + masks.set(cid, {type, ...rest}); + comps.set(cid, {type, sidx, eid}); + break; + } + case 'polychain': { + const {id: cid, type, vertices, closed} = comp; + verts.set(cid, Y.Array.from(vertices)); + comps.set(cid, {type, sidx, eid, closed}); + break; + } + case 'rectangle': { + const {id: cid, topLeft, bottomRight, type} = comp; + anchors.set(cid, {topLeft, bottomRight}); + comps.set(cid, {type, sidx, eid}); + break; + } + } +} + +function writeEntityCategory(doc: Y.Doc, entries: string[], category: string, eid: EntityId, sidx?: SliceIndex): void { + const map = yjsEntityCategoriesMap(doc); + const key = encodeEntityCategoryMapKey({category, eid, sidx}); + map.set(key, entries); +} diff --git a/app/frontend/src/common/yjs/docs/component.ts b/app/frontend/src/common/yjs/docs/component.ts new file mode 100644 index 0000000..cf62261 --- /dev/null +++ b/app/frontend/src/common/yjs/docs/component.ts @@ -0,0 +1,29 @@ +import type {EntityId, SliceIndex} from '@@frontend/type/annotation'; +import * as Y from 'yjs'; + +export type ComponentBase = { + sidx: SliceIndex; + eid: EntityId; +}; + +export type ComponentRectangle = ComponentBase & { + type: 'rectangle'; +}; + +export type ComponentPolychain = ComponentBase & { + type: 'polychain'; + closed: boolean; +}; + +export type ComponentMask = ComponentBase & { + type: 'mask'; +}; + +export type Component = ComponentRectangle | ComponentPolychain | ComponentMask; + +// The component map stores the *properties* of components, which is set at upon creation and can not change afterwards. +// Therefore, this map represents the existence of components, namely a component exists iff it is in this map. +// The key of the map is the component id. +export function yjsComponentMap(doc: Y.Doc): Y.Map { + return doc.getMap('component'); +} diff --git a/app/frontend/src/common/yjs/docs/entity.ts b/app/frontend/src/common/yjs/docs/entity.ts new file mode 100644 index 0000000..95df377 --- /dev/null +++ b/app/frontend/src/common/yjs/docs/entity.ts @@ -0,0 +1,41 @@ +import type {EntityId, SliceIndex} from '@@frontend/type/annotation'; +import * as Y from 'yjs'; + +export type CategoryList = string[]; + +export interface EntityCategoryMapKey { + category: string; + eid: EntityId; + sidx?: SliceIndex; +} + +// entityId:categoty[:slice] -> entries +export function yjsEntityCategoriesMap(doc: Y.Doc): Y.Map { + return doc.getMap('entityCategories'); +} + +export function encodeEntityCategoryMapKey({category, eid, sidx}: EntityCategoryMapKey): string { + let key = `${eid}:${category}`; + if (sidx !== undefined) { + key += `:${sidx}`; + } + return key; +} + +export function decodeEntityCategoryMapKey(key: string): EntityCategoryMapKey | undefined { + const vs = key.split(':'); + if (vs.length === 2) { + const [eid, category] = vs; + return {category, eid}; + } else if (vs.length === 3) { + const [eid, category, sliceStr] = vs; + + const sidx = parseInt(sliceStr); + if (sidx >= 0) { + return {category, eid, sidx}; + } else { + return undefined; + } + } + return undefined; +} diff --git a/app/frontend/src/common/yjs/docs/mask.ts b/app/frontend/src/common/yjs/docs/mask.ts new file mode 100644 index 0000000..567ac8f --- /dev/null +++ b/app/frontend/src/common/yjs/docs/mask.ts @@ -0,0 +1,8 @@ +import * as Y from 'yjs'; +import type {MaskComponent} from '@@frontend/type/annotation'; + +export type Mask = MaskComponent; + +export function yjsMaskMap(doc: Y.Doc): Y.Map { + return doc.getMap('mask'); +} diff --git a/app/frontend/src/common/yjs/docs/polychain.ts b/app/frontend/src/common/yjs/docs/polychain.ts new file mode 100644 index 0000000..87c1063 --- /dev/null +++ b/app/frontend/src/common/yjs/docs/polychain.ts @@ -0,0 +1,8 @@ +import * as Y from 'yjs'; +import type {Vertex} from '@@frontend/type/annotation'; + +export type PolychainVertices = Y.Array; + +export function yjsPolychainVerticesMap(doc: Y.Doc): Y.Map { + return doc.getMap('polychainVertices'); +} diff --git a/app/frontend/src/common/yjs/docs/rectangle.ts b/app/frontend/src/common/yjs/docs/rectangle.ts new file mode 100644 index 0000000..95155c6 --- /dev/null +++ b/app/frontend/src/common/yjs/docs/rectangle.ts @@ -0,0 +1,8 @@ +import * as Y from 'yjs'; +import type {RectangleComponent} from '@@frontend/type/annotation'; + +export type RectangleAnchors = Pick; + +export function yjsRectangleAnchorsMap(doc: Y.Doc): Y.Map { + return doc.getMap('rectangleAnchors'); +} diff --git a/app/frontend/src/common/yjs/event.ts b/app/frontend/src/common/yjs/event.ts new file mode 100644 index 0000000..d55f6be --- /dev/null +++ b/app/frontend/src/common/yjs/event.ts @@ -0,0 +1,239 @@ +import {useEffect} from 'react'; +import * as Y from 'yjs'; +import {useYjsContext} from './context'; +import {Component as ComponentYjs, yjsComponentMap} from './docs/component'; +import {useAnnoStore} from 'state/annotate/annotation-provider'; +import {readComponent} from './convert'; +import {RectangleAnchors, yjsRectangleAnchorsMap} from './docs/rectangle'; +import {yjsPolychainVerticesMap} from './docs/polychain'; +import {MaskComponent, Vertex} from 'type/annotation'; +import {yjsMaskMap} from './docs/mask'; +import {decodeEntityCategoryMapKey, yjsEntityCategoriesMap} from './docs/entity'; + +export function useYjsListener() { + useComponentsListener(); + useRectangleAnchorsListener(); + usePolychainVerticesListener(); + useMasksListener(); + useEntityCategoriesListener(); +} + +function useComponentsListener() { + const {doc} = useYjsContext(); + const comps = yjsComponentMap(doc); + + const addComponent = useAnnoStore(s => s.addComponent); + const deleteComponents = useAnnoStore(s => s.deleteComponents); + const transferComponent = useAnnoStore(s => s.transferComponent); + + useEffect(() => { + const fn = (e: Y.YMapEvent) => { + for (const [cid, cc] of e.changes.keys) { + switch (cc.action) { + case 'add': { + const info = comps.get(cid); + if (!info) { + break; + } + + if (info.type === 'mask') { + // adding mask will be handled in `useMasksListener` + break; + } + + // add component + const {sidx, eid} = info; + const component = readComponent(doc, cid, info); + if (!component) { + break; + } + addComponent({sliceIndex: sidx, entityId: eid, component}); + + break; + } + case 'update': { + // transferred component + const comp = comps.get(cid); + if (!comp) { + return; + } + + const {sidx, eid} = cc.oldValue as ComponentYjs; + transferComponent({sliceIndex: sidx, entityId: eid, componentId: cid, targetEntityId: comp.eid}); + break; + } + case 'delete': { + // delete components + const {sidx, eid} = cc.oldValue as ComponentYjs; + deleteComponents({sliceIndex: sidx, components: [[eid, cid]]}); + break; + } + } + } + }; + comps.observe(fn); + return () => comps.unobserve(fn); + }, [addComponent, comps, deleteComponents, doc, transferComponent]); +} + +function useRectangleAnchorsListener() { + const {doc} = useYjsContext(); + const comps = yjsComponentMap(doc); + const anchors = yjsRectangleAnchorsMap(doc); + + const updateAnchors = useAnnoStore(s => s.updateRectangleAnchors); + + useEffect(() => { + const fn = (e: Y.YMapEvent) => { + for (const cid of e.keysChanged) { + const info = comps.get(cid); + if (info) { + const {sidx, eid} = info; + const rect = anchors.get(cid); + if (rect) { + const {topLeft, bottomRight} = rect; + updateAnchors({sliceIndex: sidx, entityId: eid, componentId: cid, topLeft, bottomRight}); + } + } else { + console.warn(`rectangle ${cid} not found`); + } + } + }; + anchors.observe(fn); + return () => anchors.unobserve(fn); + }, [comps, anchors, updateAnchors]); +} + +function usePolychainVerticesListener() { + const {doc} = useYjsContext(); + const comps = yjsComponentMap(doc); + const verts = yjsPolychainVerticesMap(doc); + + const updateVertices = useAnnoStore(s => s.updatePolychainVertices); + + useEffect(() => { + const fn = (e: Y.YMapEvent>) => { + for (const cid of e.keysChanged) { + const info = comps.get(cid); + if (!info) { + console.warn(`polychain ${cid} not found`); + continue; + } + const vs = verts.get(cid); + if (!vs) { + console.warn(`polychain ${cid} vertices not found`); + continue; + } + + const {sidx, eid} = info; + updateVertices({sliceIndex: sidx, entityId: eid, componentId: cid, vertices: vs.toArray()}); + } + }; + verts.observe(fn); + return () => verts.unobserve(fn); + }, [comps, updateVertices, verts]); + + useEffect(() => { + const fn = (es: Y.YEvent>[]) => { + for (const e of es) { + if (e.path.length !== 1) { + continue; + } + const cid = `${e.path[0]}`; + + const info = comps.get(cid); + if (!info) { + console.warn(`polychain ${cid} not found`); + continue; + } + const vs = verts.get(cid); + if (!vs) { + console.warn(`polychain ${cid} vertices not found`); + continue; + } + const {sidx, eid} = info; + updateVertices({sliceIndex: sidx, entityId: eid, componentId: cid, vertices: vs.toArray()}); + } + }; + verts.observeDeep(fn); + return () => verts.unobserveDeep(fn); + }, [comps, updateVertices, verts]); +} + +function useMasksListener() { + const {doc} = useYjsContext(); + const comps = yjsComponentMap(doc); + const masks = yjsMaskMap(doc); + + const addComponent = useAnnoStore(s => s.addComponent); + useEffect(() => { + const fn = (e: Y.YMapEvent) => { + for (const [cid, cc] of e.changes.keys) { + switch (cc.action) { + case 'add': + case 'update': { + const info = comps.get(cid); + if (!info) { + console.warn('mask info not found', cid); + continue; + } + + const mask = masks.get(cid); + if (!mask) { + console.warn('mask not found', cid); + continue; + } + + const {sidx, eid} = info; + addComponent({sliceIndex: sidx, entityId: eid, component: {id: cid, ...mask}}); + break; + } + default: + console.warn('not implemented', cc); + } + } + }; + masks.observe(fn); + return () => masks.unobserve(fn); + }, [addComponent, comps, masks]); +} + +function useEntityCategoriesListener() { + const {doc} = useYjsContext(); + const cats = yjsEntityCategoriesMap(doc); + + const setEntityCategory = useAnnoStore(s => s.setEntityCategory); + const clearEntityCategory = useAnnoStore(s => s.clearEntityCategory); + + useEffect(() => { + const fn = (e: Y.YMapEvent) => { + for (const [key, cc] of e.changes.keys) { + const decoded = decodeEntityCategoryMapKey(key); + if (!decoded) { + console.warn('unexpected entity category key', key); + continue; + } + const {eid, sidx, category} = decoded; + + switch (cc.action) { + case 'add': + case 'update': { + const entries = cats.get(key); + if (!entries || entries.length === 0) { + console.warn('missing category entries', key); + continue; + } + setEntityCategory({entityId: eid, category, entries, sliceIndex: sidx}); + break; + } + case 'delete': { + clearEntityCategory({entityId: eid, category, sliceIndex: sidx}); + break; + } + } + } + }; + cats.observe(fn); + return () => cats.unobserve(fn); + }, [cats, clearEntityCategory, setEntityCategory]); +} diff --git a/app/frontend/src/common/yjs/index.ts b/app/frontend/src/common/yjs/index.ts new file mode 100644 index 0000000..b49a14c --- /dev/null +++ b/app/frontend/src/common/yjs/index.ts @@ -0,0 +1 @@ +export {setupWSConnection, setPersistence} from 'y-websocket/bin/utils'; diff --git a/app/frontend/src/common/yjs/y-websocket.d.ts b/app/frontend/src/common/yjs/y-websocket.d.ts new file mode 100644 index 0000000..5baa3eb --- /dev/null +++ b/app/frontend/src/common/yjs/y-websocket.d.ts @@ -0,0 +1,10 @@ +declare module 'y-websocket/bin/utils' { + import type {Doc} from 'yjs'; + + export function setupWSConnection(conn: unknown, req: http.IncomingMessage, doc: {docName: string}): void; + + export function setPersistence(persistence: { + bindState: (docName: string, ydoc: Doc) => Promise; + writeState: (docName: string, ydoc: Doc) => Promise; + }): void; +} diff --git a/app/frontend/src/component/Testing.tsx b/app/frontend/src/component/Testing.tsx index 5863b5c..edf2316 100644 --- a/app/frontend/src/component/Testing.tsx +++ b/app/frontend/src/component/Testing.tsx @@ -1,20 +1,23 @@ -import {FC, useEffect} from 'react'; +import {useEffect} from 'react'; import {useStore as useRenderStore} from 'state/annotate/render'; -import {useStore as useAnnoStore} from 'state/annotate/annotation'; -export const Testing: FC = () => { +import {useAnnoStoreRaw} from 'state/annotate/annotation-provider'; + +export function Testing(): JSX.Element { + const annoStore = useAnnoStoreRaw(); + useEffect(() => { - const annoState = useAnnoStore.getState(); + const annoState = annoStore.getState(); const renderState = useRenderStore.getState(); window.testing = {annoState, renderState}; - }, []); + }, [annoStore]); useEffect( () => - useAnnoStore.subscribe(annoState => { + annoStore.subscribe(annoState => { window.testing = {...window.testing, annoState}; }), - [] + [annoStore] ); useEffect( () => @@ -23,6 +26,5 @@ export const Testing: FC = () => { }), [] ); - return <>; -}; +} diff --git a/app/frontend/src/component/panel/ActionBar.tsx b/app/frontend/src/component/panel/ActionBar.tsx index b963fbf..af3f3de 100644 --- a/app/frontend/src/component/panel/ActionBar.tsx +++ b/app/frontend/src/component/panel/ActionBar.tsx @@ -23,12 +23,12 @@ import { faChessBoard, } from '@fortawesome/free-solid-svg-icons'; import {useStore as useRenderStore} from 'state/annotate/render'; -import {useTemporalStore as useTemporalAnnoStore} from 'state/annotate/annotation'; import {useStore as useUIStore} from 'state/annotate/ui'; import {ConfigContext} from 'common/context'; import {rectFitTransform} from 'common/geometry'; import {useInvertSelection, useFocusAreas, useDrawing} from 'common/hook'; import {useCanvasSize, leftSidebarWidth} from './layout'; +import {useAnnoHistoryStore} from 'state/annotate/annotation-provider'; const ActionButton: FC<{helpCode: string; icon: React.ReactNode; hotKey?: string} & ButtonProps> = ({ helpCode, @@ -136,7 +136,10 @@ export const ActionBar: FC = ({...baseProps}) => { const focusAreas = useFocusAreas(canvasSize); // redo and undo - const {pastStates, futureStates, redo, undo} = useTemporalAnnoStore(); + const redo = useAnnoHistoryStore(s => s.redo); + const undo = useAnnoHistoryStore(s => s.undo); + const undoCount = useAnnoHistoryStore(s => s.index); + const redoCount = useAnnoHistoryStore(s => s.actions.length - s.index - 1); return (
@@ -249,14 +252,14 @@ export const ActionBar: FC = ({...baseProps}) => { helpCode="action.undo" hotKey="⌘/⌃ + Z" icon={} - disabled={isDrawing || pastStates.length === 0} + disabled={isDrawing || undoCount === 0} onClick={() => undo()} /> } - disabled={isDrawing || futureStates.length === 0} + disabled={isDrawing || redoCount === 0} onClick={() => redo()} /> > = ({...divProps}) const Loaded: FC> = ({...divProps}) => { // Reset history when a new annotation is started. - const {clear} = useTemporalAnnoStore(); + const clear = useAnnoHistoryStore(s => s.reset); useEffect(() => clear(), [clear]); const isDrawingPoly = useDrawPolyStore(s => s.vertices.length > 0); diff --git a/app/frontend/src/component/panel/AnnotationMonitor.tsx b/app/frontend/src/component/panel/AnnotationMonitor.tsx deleted file mode 100644 index 39984b5..0000000 --- a/app/frontend/src/component/panel/AnnotationMonitor.tsx +++ /dev/null @@ -1,134 +0,0 @@ -import {FC, useContext, useEffect} from 'react'; -import * as jsonmergepatch from 'json-merge-patch'; -import {produce} from 'immer'; - -import {deleteAnnotationComponent, useStore as useAnnoStore} from 'state/annotate/annotation'; -import {useStore as useRenderStore} from 'state/annotate/render'; -import {usePatchVideoAnnotation} from 'state/server/annotation'; -import {useStore as useUIStore} from 'state/annotate/ui'; - -import {ConfigContext, NutshClientContext} from 'common/context'; -import {ApiError} from 'openapi/nutsh'; - -import type {Video} from 'openapi/nutsh'; -import {Annotation} from 'type/annotation'; - -export const MonitorAnnotation: FC<{videoId: Video['id']}> = ({videoId}) => { - const config = useContext(ConfigContext); - return ( - <> - {!config.readonly && } - - - - ); -}; - -const SyncAnnotation: FC<{videoId: Video['id']}> = ({videoId}) => { - const client = useContext(NutshClientContext); - const annotationVersion = useRenderStore(s => s.annotationVersion); - const isSyncing = useRenderStore(s => s.isSyncing); - const setAnnotationVersion = useRenderStore(s => s.setAnnotationVersion); - const setIsSyncing = useRenderStore(s => s.setIsSyncing); - const setSyncError = useRenderStore(s => s.setSyncError); - - // sync with server when annotation changes - const {mutate: patchVideoAnnotation, error} = usePatchVideoAnnotation(client); - - useEffect(() => { - return useAnnoStore.subscribe( - s => s.annotation, - (curr, prev) => { - const newPrev = produce(prev, removeDraftComponents); - const newCurr = produce(curr, removeDraftComponents); - - const mergePatch = jsonmergepatch.generate(newPrev, newCurr); - if (!mergePatch) { - return; - } - console.debug('syncing annotation'); - - if (isSyncing) { - setSyncError('error.sync.conflict'); - return; - } - setIsSyncing(true); - patchVideoAnnotation( - { - videoId, - requestBody: { - json_merge_patch: JSON.stringify(mergePatch), - annotation_version: annotationVersion, - }, - }, - { - onSuccess: ({annotation_version}) => { - setAnnotationVersion(annotation_version); - setIsSyncing(false); - }, - } - ); - }, - { - fireImmediately: true, - } - ); - }, [videoId, annotationVersion, isSyncing, setAnnotationVersion, setIsSyncing, patchVideoAnnotation, setSyncError]); - - useEffect(() => { - if (error) { - if (error instanceof ApiError) { - if (error.status === 409) { - setSyncError('error.sync.conflict'); - return; - } - } - setSyncError('error.sync.unknown'); - } - }, [error, setSyncError]); - - return <>; -}; - -const ForgetEntities: FC = () => { - const forgetEntities = useRenderStore(s => s.forgetEntities); - useEffect(() => { - return useAnnoStore.subscribe( - s => s.annotation, - (curr, prev) => { - const goneEntityIds = Object.keys(prev.entities).filter(eid => !(eid in curr.entities)); - if (goneEntityIds.length > 0) { - forgetEntities(...goneEntityIds); - } - } - ); - }, [forgetEntities]); - return <>; -}; - -const CommitDraft: FC = () => { - const commitDraftComponents = useAnnoStore(s => s.commitDraftComponents); - const trackingCount = useUIStore(s => Object.keys(s.tracking).length); - useEffect(() => { - if (trackingCount === 0) { - console.debug('commit draft components'); - commitDraftComponents(); - } - }, [commitDraftComponents, trackingCount]); - return <>; -}; - -function removeDraftComponents(anno: Annotation): Annotation { - return produce(anno, draft => { - Object.values(draft.entities).forEach(entity => { - Object.entries(entity.geometry.slices).forEach(([sidx, sliceComponents]) => { - Object.values(sliceComponents).forEach(component => { - if (component.draft) { - deleteAnnotationComponent(draft, parseInt(sidx), entity.id, component.id); - } - }); - }); - }); - return draft; - }); -} diff --git a/app/frontend/src/component/panel/ContextMenuMask.tsx b/app/frontend/src/component/panel/ContextMenuMask.tsx index 696bc0a..af706e0 100644 --- a/app/frontend/src/component/panel/ContextMenuMask.tsx +++ b/app/frontend/src/component/panel/ContextMenuMask.tsx @@ -5,13 +5,14 @@ import intl from 'react-intl-universal'; import {Dropdown, DropdownProps, App, Space, Tag, MenuProps} from 'antd'; import {ExclamationCircleOutlined} from '@ant-design/icons'; import {css} from '@emotion/react'; -import {getComponent, useStore as useAnnoStore} from 'state/annotate/annotation'; +import {useAnnoStore} from 'state/annotate/annotation-provider'; import {useStore as useRenderStore} from 'state/annotate/render'; import {ComponentId, EntityId, MaskComponent, PolychainComponent} from 'type/annotation'; import {Action, useComponentActions, useEntityActions} from './menu/common'; import {useActions as usePolychainActions} from './menu/polychain'; import {useActions as useMaskActions} from './menu/mask'; +import {getComponent} from 'state/annotate/annotation'; export const ContextMenuMask: FC<{style?: CSSProperties}> = ({style}) => { const contextMenuClient = useRenderStore(s => s.mouse.contextMenuClient); diff --git a/app/frontend/src/component/panel/entity/Card.tsx b/app/frontend/src/component/panel/entity/Card.tsx index 2cdfd47..7951d45 100644 --- a/app/frontend/src/component/panel/entity/Card.tsx +++ b/app/frontend/src/component/panel/entity/Card.tsx @@ -8,7 +8,7 @@ import {css} from '@emotion/react'; import {Space, Card, Typography, Button, Popconfirm, Progress, Spin} from 'antd'; import {EditOutlined, DeleteOutlined} from '@ant-design/icons'; -import {useStore as useAnnoStore} from 'state/annotate/annotation'; +import {useAnnoStore} from 'state/annotate/annotation-provider'; import {useStore as useRenderStore} from 'state/annotate/render'; import {useStore as useUIStore} from 'state/annotate/ui'; @@ -19,6 +19,7 @@ import type {EntityId} from 'type/annotation'; import type {ProjectSpec} from 'type/project_spec'; import {ColorPalette} from './display'; import {useEntityCategories} from 'common/hook'; +import {useDeleteEntities} from 'state/annotate/annotation-broadcast'; const {Paragraph, Text} = Typography; @@ -73,7 +74,7 @@ export const EntityCard: FC<{ return Object.keys(g.slices).map(s => parseInt(s)); }, shallow); - const deleteEntities = useAnnoStore(s => s.deleteEntities); + const {deleteEntities} = useDeleteEntities(); const isSelected = useRenderStore(s => s.select.ids.has(entityId)); const isHovered = useRenderStore(s => s.mouse.hover?.entityId === entityId); diff --git a/app/frontend/src/component/panel/entity/Form.tsx b/app/frontend/src/component/panel/entity/Form.tsx index 6497fc6..3008188 100644 --- a/app/frontend/src/component/panel/entity/Form.tsx +++ b/app/frontend/src/component/panel/entity/Form.tsx @@ -2,8 +2,8 @@ import {FC} from 'react'; import intl from 'react-intl-universal'; import {Form, TreeSelect, TreeSelectProps, Space, Tag} from 'antd'; -import {useStore as useAnnoStore} from 'state/annotate/annotation'; import {useStore as useRenderStore} from 'state/annotate/render'; +import {useClearEntityCategory, useSetEntityCategory} from 'state/annotate/annotation-broadcast'; import {CategoryAbbreviation} from 'component/panel/entity/display'; @@ -30,8 +30,8 @@ export function makeTreeNode( export const EntityForm: FC<{entityId: EntityId; projectSpec: ProjectSpec}> = ({entityId, projectSpec}) => { const sidx = useRenderStore(s => s.sliceIndex); const categories = useEntityCategories(entityId, sidx); - const setEntityCategory = useAnnoStore(s => s.setEntityCategory); - const clearEntityCategory = useAnnoStore(s => s.clearEntityCategory); + const {setEntityCategory} = useSetEntityCategory(); + const {clearEntityCategory} = useClearEntityCategory(); return (
diff --git a/app/frontend/src/component/panel/entity/List.tsx b/app/frontend/src/component/panel/entity/List.tsx index d6a0979..066605e 100644 --- a/app/frontend/src/component/panel/entity/List.tsx +++ b/app/frontend/src/component/panel/entity/List.tsx @@ -5,7 +5,7 @@ import {produce} from 'immer'; import {Alert, Empty, Space, Tag, Tooltip} from 'antd'; -import {useStore as useAnnoStore, State} from 'state/annotate/annotation'; +import {useAnnoStore, State} from 'state/annotate/annotation-provider'; import {useStore as useRenderStore} from 'state/annotate/render'; import {deepEqual} from 'common/util'; diff --git a/app/frontend/src/component/panel/layer/Hover.tsx b/app/frontend/src/component/panel/layer/Hover.tsx index 109e0ef..5ec082d 100644 --- a/app/frontend/src/component/panel/layer/Hover.tsx +++ b/app/frontend/src/component/panel/layer/Hover.tsx @@ -6,7 +6,7 @@ import shallow from 'zustand/shallow'; import {Key} from 'ts-key-enum'; import {isHotkeyPressed, useHotkeys} from 'react-hotkeys-hook'; -import {useStore as useAnnoStore, getComponent} from 'state/annotate/annotation'; +import {useAnnoStore} from 'state/annotate/annotation-provider'; import {useStore as useRenderStore} from 'state/annotate/render'; import {useStore as useUIStore} from 'state/annotate/ui'; @@ -27,6 +27,8 @@ import {newComponentAdapter} from 'common/adapter'; import {ComponentProximity} from 'state/annotate/render/mouse'; import {ColorPalette} from '../entity/display'; import {editStyle} from 'common/constant'; +import {getComponent} from 'state/annotate/annotation'; +import {useAddDeleteComponents, useTransferComponent} from 'state/annotate/annotation-broadcast'; const FullSize: CSSProperties = {position: 'absolute', left: 0, top: 0, width: '100%', height: '100%'}; @@ -172,8 +174,8 @@ const TopLevelHover: FC> = ({...divProps}) => { [manipulation] ) ); - const transferComponent = useAnnoStore(s => s.transferComponent); - const addComponents = useAnnoStore(s => s.addComponents); + const {addComponents} = useAddDeleteComponents(); + const {transferComponent} = useTransferComponent(); const {isControlOrMetaPressed, isShiftPressed} = useControlMetaShiftPressed(); diff --git a/app/frontend/src/component/panel/layer/Idle.tsx b/app/frontend/src/component/panel/layer/Idle.tsx index 0bde594..6745dfa 100644 --- a/app/frontend/src/component/panel/layer/Idle.tsx +++ b/app/frontend/src/component/panel/layer/Idle.tsx @@ -4,10 +4,9 @@ import intl from 'react-intl-universal'; import {v4 as uuidv4} from 'uuid'; import {message} from 'antd'; -import {useTemporalStore as useTemporalAnnoStore} from 'state/annotate/annotation'; +import {useAnnoHistoryStore} from 'state/annotate/annotation-provider'; import {EntityComponentId, useStore as useRenderStore} from 'state/annotate/render'; import {useStore as useUIStore} from 'state/annotate/ui'; -import {useStore as useAnnoStore} from 'state/annotate/annotation'; import {useStore as useEditPolyStore} from 'state/annotate/polychain/edit'; import {useStore as useEditRectStore} from 'state/annotate/rectangle/edit'; @@ -22,6 +21,7 @@ import {editStyle, idleStyle} from 'common/constant'; import {convertRGBA2Hex, isLightBackground} from 'common/color'; import {useHotkeys} from 'react-hotkeys-hook'; import {coordinatesImageToCanvas} from 'common/geometry'; +import {usePaste} from 'state/annotate/annotation-broadcast'; export const IdleLayer: FC> = ({...divProps}) => { console.debug('render IdleLayer'); @@ -88,7 +88,7 @@ const PasteLoaded: FC<{copying: {ecids: EntityComponentId[]; sliceIndex: SliceIn copying: {ecids, sliceIndex}, }) => { const sidx = useRenderStore(s => s.sliceIndex); - const paste = useAnnoStore(s => s.paste); + const {paste} = usePaste(); useHotkeys( 'ctrl+v, meta+v', @@ -160,7 +160,8 @@ function useRenderSettings(): RenderSetting[] { } const IdleLayerTemporal: FC = () => { - const {undo, redo} = useTemporalAnnoStore(); + const undo = useAnnoHistoryStore(s => s.undo); + const redo = useAnnoHistoryStore(s => s.redo); useHotkeys('ctrl+z, meta+z', () => undo()); useHotkeys('ctrl+shift+z, meta+shift+z', () => redo()); return <>; diff --git a/app/frontend/src/component/panel/layer/Surround.tsx b/app/frontend/src/component/panel/layer/Surround.tsx index 9bd025d..d33859f 100644 --- a/app/frontend/src/component/panel/layer/Surround.tsx +++ b/app/frontend/src/component/panel/layer/Surround.tsx @@ -1,6 +1,6 @@ import {FC, CanvasHTMLAttributes, useCallback} from 'react'; -import {useStore as useAnnoStore} from 'state/annotate/annotation'; +import {useAnnoStore} from 'state/annotate/annotation-provider'; import {useStore as useRenderStore} from 'state/annotate/render'; import {useKeyPressed} from 'common/keyboard'; diff --git a/app/frontend/src/component/panel/layer/Translate.tsx b/app/frontend/src/component/panel/layer/Translate.tsx index d7c7773..a783000 100644 --- a/app/frontend/src/component/panel/layer/Translate.tsx +++ b/app/frontend/src/component/panel/layer/Translate.tsx @@ -1,7 +1,7 @@ import {FC, useCallback, useState, HTMLAttributes, useMemo, useRef} from 'react'; import shallow from 'zustand/shallow'; -import {getComponent, useStore as useAnnoStore} from 'state/annotate/annotation'; +import {useAnnoStore} from 'state/annotate/annotation-provider'; import {useStore as useRenderStore} from 'state/annotate/render'; import {TranslateData} from 'state/annotate/render/translate'; @@ -12,6 +12,8 @@ import {newComponentAdapter} from 'common/adapter'; import {createComponentSVG} from 'common/svg'; import {editStyle} from 'common/constant'; import {ColorPalette} from '../entity/display'; +import {getComponent} from 'state/annotate/annotation'; +import {useTranslate} from 'state/annotate/annotation-broadcast'; type Props = HTMLAttributes & { data: TranslateData; @@ -30,7 +32,7 @@ export const TranslateLayer: FC = ({data, ...divProps}) => { const clearSelect = useRenderStore(s => s.select.clear); const finish = useRenderStore(s => s.translate.finish); - const translate = useAnnoStore(s => s.translate); + const {translate} = useTranslate(); const ecs = useAnnoStore( useCallback( diff --git a/app/frontend/src/component/panel/layer/mask/Hover.tsx b/app/frontend/src/component/panel/layer/mask/Hover.tsx index 3cf5915..46f2e0f 100644 --- a/app/frontend/src/component/panel/layer/mask/Hover.tsx +++ b/app/frontend/src/component/panel/layer/mask/Hover.tsx @@ -3,7 +3,7 @@ import {ComponentProximity} from 'state/annotate/render/mouse'; import {useStore as useBrushCanvasStore} from 'component/segmentation/BrushCanvas'; import {useStore as useRenderStore} from 'state/annotate/render'; import {useStore as useUIStore} from 'state/annotate/ui'; -import {relativeMousePosition} from 'common/util'; +import {relativeMousePosition} from 'common/mouse'; import {coordinatesCanvasToImage} from 'common/geometry'; type Props = HTMLAttributes & { diff --git a/app/frontend/src/component/panel/layer/mask/Segment/FocusCanvas.tsx b/app/frontend/src/component/panel/layer/mask/Segment/FocusCanvas.tsx index 33b0cfc..df6a99d 100644 --- a/app/frontend/src/component/panel/layer/mask/Segment/FocusCanvas.tsx +++ b/app/frontend/src/component/panel/layer/mask/Segment/FocusCanvas.tsx @@ -4,7 +4,7 @@ import {Spin, Tag} from 'antd'; import intl from 'react-intl-universal'; import {Tensor} from 'onnxruntime-web'; import shallow from 'zustand/shallow'; -import {relativeMousePosition} from 'common/util'; +import {relativeMousePosition} from 'common/mouse'; import {coordinatesCanvasToImage, coordinatesImageToCanvas, limitGrid} from 'common/geometry'; import {drawRect} from 'common/draw'; import {SurroundStyle} from 'common/constant'; diff --git a/app/frontend/src/component/panel/layer/mask/Segment/PredictContainer.tsx b/app/frontend/src/component/panel/layer/mask/Segment/PredictContainer.tsx index 9d84e6b..943c5fe 100644 --- a/app/frontend/src/component/panel/layer/mask/Segment/PredictContainer.tsx +++ b/app/frontend/src/component/panel/layer/mask/Segment/PredictContainer.tsx @@ -6,7 +6,7 @@ import {throttle} from 'lodash'; import {Tensor} from 'onnxruntime-web'; import {FocusOpacity} from 'common/constant'; -import {relativeMousePosition} from 'common/util'; +import {relativeMousePosition} from 'common/mouse'; import {coordinatesCanvasToImage, coordinatesImageToCanvas, distance} from 'common/geometry'; import {SizedContainer} from 'component/SizedContainer'; import {Coordinates} from 'type/annotation'; diff --git a/app/frontend/src/component/panel/layer/mask/common.ts b/app/frontend/src/component/panel/layer/mask/common.ts index d76917f..c8c21c4 100644 --- a/app/frontend/src/component/panel/layer/mask/common.ts +++ b/app/frontend/src/component/panel/layer/mask/common.ts @@ -4,13 +4,14 @@ import shallow from 'zustand/shallow'; import {coordinatesImageToCanvas} from 'common/geometry'; import {ViewportTransform} from 'state/annotate/render/viewport'; import {useStore as useRenderStore} from 'state/annotate/render'; -import {UpdateSliceMasksInput, useStore as useAnnoStore} from 'state/annotate/annotation'; import {ComponentId, EntityId, MaskComponent} from 'type/annotation'; import {editStyle} from 'common/constant'; import {ColorPalette} from 'component/panel/entity/display'; import {newComponentAdapter} from 'common/adapter'; import {encodeRLE} from 'common/algorithm/rle'; import {useVisibleEntities} from 'common/render'; +import {UpdateSliceMasksInput} from 'state/annotate/annotation'; +import {useUpdateSliceMasks} from 'state/annotate/annotation-broadcast'; // WARN(hxu): consider re-drawing only the updated part when running into performance issue. export function updateImageRendering( @@ -100,7 +101,7 @@ export function useImageContext(init?: (ctx: CanvasRenderingContext2D) => void) export function useUpdateMask(imageContext: CanvasRenderingContext2D, prevMasks: EntityComponentMask[]) { const sliceIndex = useRenderStore(s => s.sliceIndex); - const updateSliceMasks = useAnnoStore(s => s.updateSliceMasks); + const {updateSliceMasks} = useUpdateSliceMasks(); return useCallback(() => { const currMasks: EntityComponentMask[] = collectLocalMasks(imageContext).map(m => { diff --git a/app/frontend/src/component/panel/layer/polychain/Draw.tsx b/app/frontend/src/component/panel/layer/polychain/Draw.tsx index e1f4ac1..8802df1 100644 --- a/app/frontend/src/component/panel/layer/polychain/Draw.tsx +++ b/app/frontend/src/component/panel/layer/polychain/Draw.tsx @@ -7,7 +7,7 @@ import intl from 'react-intl-universal'; import shallow from 'zustand/shallow'; import {v4 as uuidv4} from 'uuid'; -import {getComponent, useStore as useAnnoStore} from 'state/annotate/annotation'; +import {useAnnoStore} from 'state/annotate/annotation-provider'; import {useStore as useUIStore} from 'state/annotate/ui'; import {useStore as useRenderStore} from 'state/annotate/render'; import {useStore as useDrawStore, useTemporalStore as useTemporalDrawStore} from 'state/annotate/polychain/draw'; @@ -22,6 +22,8 @@ import type {ComponentId, Coordinates, EntityId, Vertex} from 'type/annotation'; import {ColorPalette} from 'component/panel/entity/display'; import {useHotkeys} from 'react-hotkeys-hook'; import {useKeyPressed} from 'common/keyboard'; +import {getComponent} from 'state/annotate/annotation'; +import {useAddDeleteComponents} from 'state/annotate/annotation-broadcast'; type Props = HTMLAttributes & { width: number; @@ -87,7 +89,7 @@ const LayerWithEntityId: FC = ({entityId, width, h ) ); - const addComponent = useAnnoStore(s => s.addComponent); + const {addComponent} = useAddDeleteComponents(); const drawPolychain = useDrawPolychain(transform); const drawAnnoVertex = useDrawVertex(transform); diff --git a/app/frontend/src/component/panel/layer/polychain/Edit.tsx b/app/frontend/src/component/panel/layer/polychain/Edit.tsx index 8e248a1..ff577ea 100644 --- a/app/frontend/src/component/panel/layer/polychain/Edit.tsx +++ b/app/frontend/src/component/panel/layer/polychain/Edit.tsx @@ -1,7 +1,7 @@ import {FC, useCallback, CanvasHTMLAttributes, useState} from 'react'; import shallow from 'zustand/shallow'; -import {useStore as useAnnoStore} from 'state/annotate/annotation'; +import {useAnnoStore} from 'state/annotate/annotation-provider'; import {useStore as useRenderStore} from 'state/annotate/render'; import {useStore as useEditStore} from 'state/annotate/polychain/edit'; @@ -13,6 +13,7 @@ import {useDrawPolychain, useDrawDashedLine, useDrawVertex} from 'common/render' import type {Data as EditData} from 'state/annotate/polychain/edit'; import {ColorPalette} from 'component/panel/entity/display'; import {editStyle} from 'common/constant'; +import {useUpdatePolychainVertices} from 'state/annotate/annotation-broadcast'; const Canvas: FC & {data: EditData}> = ({data, ...canvasProps}) => { const {width: imw, height: imh} = useRenderStore(s => s.sliceSize!, shallow); @@ -67,12 +68,12 @@ const Canvas: FC & {data: EditData}> = ( ) ); - const updateVertices = useAnnoStore(s => s.updatePolychainVertices); + const {updatePolychainVertices} = useUpdatePolychainVertices(); const finishEdit = useEditStore(s => s.finish); const finish = useCallback(() => { - updateVertices({sliceIndex, entityId: eid, componentId: cid, vertices}); + updatePolychainVertices({sliceIndex, entityId: eid, componentId: cid, vertices}); finishEdit(); - }, [cid, eid, finishEdit, sliceIndex, updateVertices, vertices]); + }, [cid, eid, finishEdit, sliceIndex, updatePolychainVertices, vertices]); return ( & { width: number; @@ -50,7 +50,7 @@ const LayerWithEntityId: FC = ({entityId, width, h ) ); - const addComponent = useAnnoStore(s => s.addComponent); + const {addComponent} = useAddDeleteComponents(); useHotkeys( 'esc', useCallback(() => finish(), [finish]) diff --git a/app/frontend/src/component/panel/layer/rectangle/Edit.tsx b/app/frontend/src/component/panel/layer/rectangle/Edit.tsx index 4182e77..f4c17c2 100644 --- a/app/frontend/src/component/panel/layer/rectangle/Edit.tsx +++ b/app/frontend/src/component/panel/layer/rectangle/Edit.tsx @@ -6,10 +6,10 @@ import {useDrawRect} from 'common/render'; import {editStyle} from 'common/constant'; import {coordinatesCanvasToImage, limitCoordinates} from 'common/geometry'; -import {useStore as useAnnoStore} from 'state/annotate/annotation'; import {useStore as useRenderStore} from 'state/annotate/render'; import {useStore as useEditStore, Data} from 'state/annotate/rectangle/edit'; import {ColorPalette} from 'component/panel/entity/display'; +import {useUpdateRectangleAnchors} from 'state/annotate/annotation-broadcast'; const Canvas: FC<{data: Data} & CanvasHTMLAttributes> = ({data, ...canvasProps}) => { const {width: imw, height: imh} = useRenderStore(s => s.sliceSize!, shallow); @@ -36,7 +36,7 @@ const Canvas: FC<{data: Data} & CanvasHTMLAttributes> = ({dat ); const sliceIndex = useRenderStore(s => s.sliceIndex); - const updateAnchors = useAnnoStore(s => s.updateRectangleAnchors); + const {updateRectangleAnchors} = useUpdateRectangleAnchors(); const finishEdit = useEditStore(s => s.finish); const finish = useCallback(() => { const {p, q} = anchors; @@ -45,9 +45,9 @@ const Canvas: FC<{data: Data} & CanvasHTMLAttributes> = ({dat const x2 = Math.max(p.x, q.x); const y2 = Math.max(p.y, q.y); - updateAnchors({sliceIndex, entityId, componentId, topLeft: {x: x1, y: y1}, bottomRight: {x: x2, y: y2}}); + updateRectangleAnchors({sliceIndex, entityId, componentId, topLeft: {x: x1, y: y1}, bottomRight: {x: x2, y: y2}}); finishEdit(); - }, [anchors, componentId, entityId, finishEdit, sliceIndex, updateAnchors]); + }, [anchors, componentId, entityId, finishEdit, sliceIndex, updateRectangleAnchors]); return ( s.sliceIndex); - const seperateComponent = useAnnoStore(s => s.seperateComponent); - const deleteComponents = useAnnoStore(s => s.deleteComponents); + const {separateComponent} = useSeparateComponent(); + const {deleteComponents} = useAddDeleteComponents(); const startManipulation = useRenderStore(s => s.manipulate.start); const nc = useAnnoStore( @@ -47,7 +55,7 @@ export function useComponentActions(entityId: EntityId, componentId: ComponentId fn: () => { const eid = uuidv4(); const cid = uuidv4(); - seperateComponent({sliceIndex, entityId, componentId, newEntityId: eid, newComponentId: cid}); + separateComponent({sliceIndex, entityId, componentId, newEntityId: eid, newComponentId: cid}); }, }, { @@ -71,9 +79,9 @@ export function useEntityActions(): Action[] { const selectIds = useRenderStore(s => s.select.ids); const sliceIndex = useRenderStore(s => s.sliceIndex); - const deleteComponents = useAnnoStore(s => s.deleteComponents); - const deleteEntities = useAnnoStore(s => s.deleteEntities); - const truncateEntities = useAnnoStore(s => s.truncateEntities); + const {deleteEntities} = useDeleteEntities(); + const {deleteComponents} = useAddDeleteComponents(); + const {truncateEntities} = useTruncateEntities(); // copy const copy = useRenderStore(s => s.copy); @@ -90,7 +98,7 @@ export function useEntityActions(): Action[] { }, [entities, sliceIndex, selectIds]); // paste - const paste = useAnnoStore(s => s.paste); + const {paste} = usePaste(); const actions: Action[] = [ { title: intl.get('paste'), diff --git a/app/frontend/src/component/panel/menu/mask.ts b/app/frontend/src/component/panel/menu/mask.ts index 16b36c6..f3fb91d 100644 --- a/app/frontend/src/component/panel/menu/mask.ts +++ b/app/frontend/src/component/panel/menu/mask.ts @@ -3,13 +3,13 @@ import {v4 as uuidv4} from 'uuid'; import {Component, EntityId, MaskComponent, SliceIndex} from 'type/annotation'; import {useStore as useUIStore} from 'state/annotate/ui'; import {useStore as useRenderStore} from 'state/annotate/render'; -import {useStore as useAnnoStore} from 'state/annotate/annotation'; import {Action} from './common'; import {useCallback, useContext} from 'react'; import {ConfigContext} from 'common/context'; import {expand, rleCountsFromStringCOCO, rleCountsToStringCOCO, shrink} from 'common/algorithm/rle'; import {Mask, TrackReq} from 'openapi/nutsh'; import {correctSliceUrl} from 'common/route'; +import {useAddDeleteComponents} from 'state/annotate/annotation-broadcast'; export function useActions(mask: MaskComponent, eid: EntityId): Action[] { const config = useContext(ConfigContext); @@ -22,7 +22,7 @@ export function useActions(mask: MaskComponent, eid: EntityId): Action[] { const currentSliceIndex = useRenderStore(s => s.sliceIndex); const currentSliceUrl = useRenderStore(s => correctSliceUrl(s.sliceUrls[s.sliceIndex])); const subsequentSliceUrls = useRenderStore(s => s.sliceUrls.slice(s.sliceIndex + 1).map(correctSliceUrl)); - const addComponents = useAnnoStore(s => s.addComponents); + const {addComponents} = useAddDeleteComponents(); const track = useCallback( (mask: MaskComponent) => { diff --git a/app/frontend/src/component/panel/menu/polychain.ts b/app/frontend/src/component/panel/menu/polychain.ts index 865f5fc..acf8c2d 100644 --- a/app/frontend/src/component/panel/menu/polychain.ts +++ b/app/frontend/src/component/panel/menu/polychain.ts @@ -1,14 +1,14 @@ import intl from 'react-intl-universal'; import shallow from 'zustand/shallow'; import {useStore as useRenderStore} from 'state/annotate/render'; -import {useStore as useAnnoStore} from 'state/annotate/annotation'; import {PolychainComponent} from 'type/annotation'; import {Action} from './common'; +import {useDeletePolychainVertex, useSetPolychainVertexBezier} from 'state/annotate/annotation-broadcast'; export function useActions(component: PolychainComponent): Action[] { - const setPolychainVertexBezier = useAnnoStore(s => s.setPolychainVertexBezier); - const deletePolychainVertex = useAnnoStore(s => s.deletePolychainVertex); + const {deletePolychainVertex} = useDeletePolychainVertex(); + const {setPolychainVertexBezier} = useSetPolychainVertexBezier(); const sidx = useRenderStore(s => s.sliceIndex); const hover = useRenderStore(s => s.mouse.hover, shallow); diff --git a/app/frontend/src/component/segmentation/BrushCanvas.tsx b/app/frontend/src/component/segmentation/BrushCanvas.tsx index a1c56af..410a366 100644 --- a/app/frontend/src/component/segmentation/BrushCanvas.tsx +++ b/app/frontend/src/component/segmentation/BrushCanvas.tsx @@ -6,7 +6,7 @@ import {coordinatesCanvasToImage, coordinatesImageToCanvas, distance} from 'comm import {FocusOpacity} from 'common/constant'; import {convertRGBA2Hex} from 'common/color'; import {Coordinates} from 'type/annotation'; -import {relativeMousePosition} from 'common/util'; +import {relativeMousePosition} from 'common/mouse'; import {updateImageRendering, useCanvasContext} from '../panel/layer/mask/common'; import {ViewportTransform} from 'state/annotate/render/viewport'; import {Alert, Button, Popover, Slider, Space, Tag, Tooltip, Typography, theme} from 'antd'; diff --git a/app/frontend/src/page/annotate/Panel/Load.tsx b/app/frontend/src/page/annotate/Panel/Load.tsx index c90c977..89c2d0a 100644 --- a/app/frontend/src/page/annotate/Panel/Load.tsx +++ b/app/frontend/src/page/annotate/Panel/Load.tsx @@ -2,14 +2,13 @@ import {FC, useContext, useEffect, useState} from 'react'; import intl from 'react-intl-universal'; import {Alert} from 'antd'; import {useStore as useRenderStore} from 'state/annotate/render'; -import {useStore as useAnnoStore} from 'state/annotate/annotation'; -import {useGetVideoAnnotation} from 'state/server/annotation'; +import {useAnnoStore} from 'state/annotate/annotation-provider'; import {useGetVideo} from 'state/server/video'; import {NutshClientContext} from 'common/context'; import PageLayout from 'page/Layout'; -import {mustDecodeJsonStr as mustDecodeAnnotationJsonStr} from 'type/annotation'; import type {Video} from 'openapi/nutsh'; import {PanelLoadProject} from './LoadProject'; +import {useAnnotationSync} from '@@frontend/state/server/annotation'; export const PanelLoad: FC<{id: Video['id']}> = ({id}) => { const client = useContext(NutshClientContext); @@ -17,44 +16,44 @@ export const PanelLoad: FC<{id: Video['id']}> = ({id}) => { // client state const isLoaded = useRenderStore(s => s.sliceUrls.length > 0); const startAnnotation = useRenderStore(s => s.startAnnotation); - const setAnnotation = useAnnoStore(s => s.setAnnotation); // server state - const {isFetching: isFetchingVideo, data: getVideoData} = useGetVideo(client, id); - const {isFetching: isFetchingAnno, data: getVideoAnnotationData} = useGetVideoAnnotation(client, id); + const {isFetching: isFetchingVideo, data: videoData} = useGetVideo(client, id); + + // sync + const {initial} = useAnnotationSync(id); + const setAnnotation = useAnnoStore(s => s.setAnnotation); + useEffect(() => { + if (initial) { + setAnnotation(initial); + } + }, [initial, setAnnotation]); // local state const [errorCode, setErrorCode] = useState(undefined); useEffect(() => { - if (!getVideoAnnotationData) return; - if (!getVideoData) return; + if (!videoData) return; setErrorCode(undefined); - const {annotation_json: annoJson, annotation_version: annoVersion} = getVideoAnnotationData; - const {frame_urls: frameUrls} = getVideoData.video; + const {frame_urls: frameUrls} = videoData.video; if (!frameUrls || frameUrls.length === 0) { setErrorCode('error.missing_video_frames'); return; } - try { - const annotation = annoJson ? mustDecodeAnnotationJsonStr(annoJson) : undefined; - setAnnotation(annotation); - startAnnotation(frameUrls, annoVersion); - } catch (e) { - console.error((e as Error).cause); - setErrorCode('error.invalid_annotation_json'); - } - }, [getVideoData, getVideoAnnotationData, setAnnotation, startAnnotation]); + startAnnotation(frameUrls, ''); + }, [videoData, startAnnotation]); - if (!isLoaded || !getVideoData) { + if (!isLoaded || !videoData || initial === undefined || errorCode) { return ( - + {errorCode && } ); } - return ; + // Only AFTER the annotation is initialized should we render the panel, otherwise its yjs update listener will respond + // to the initialization, causing the page to re-render frequently and impossible to load heavy annotations. + return ; }; diff --git a/app/frontend/src/page/annotate/Panel/Loaded.tsx b/app/frontend/src/page/annotate/Panel/Loaded.tsx index 414273b..9016689 100644 --- a/app/frontend/src/page/annotate/Panel/Loaded.tsx +++ b/app/frontend/src/page/annotate/Panel/Loaded.tsx @@ -5,7 +5,6 @@ import shallow from 'zustand/shallow'; import {useStore as useRenderStore} from 'state/annotate/render'; import {prefetchImages} from 'state/image/store'; import {UI} from 'common/constant'; -import {MonitorAnnotation} from 'component/panel/AnnotationMonitor'; import type {Project, Video} from 'openapi/nutsh'; import type {ProjectSpec} from 'type/project_spec'; import {FrameSlider} from 'component/panel/FrameSlider'; @@ -58,7 +57,6 @@ export const PanelLoaded: FC<{
)} - - - ); }; - -const SyncingDisableInteractionMask: FC = () => { - const isSyncing = useRenderStore(s => s.isSyncing); - return isSyncing ?
: null; -}; diff --git a/app/frontend/src/page/annotate/Panel/index.tsx b/app/frontend/src/page/annotate/Panel/index.tsx index f539c53..419ae52 100644 --- a/app/frontend/src/page/annotate/Panel/index.tsx +++ b/app/frontend/src/page/annotate/Panel/index.tsx @@ -1,6 +1,9 @@ import {FC} from 'react'; import {useParams} from 'react-router-dom'; +import {YjsProvider} from 'common/yjs/context'; import {PanelLoad} from './Load'; +import {useYjsListener} from 'common/yjs/event'; +import {AnnoProvider} from 'state/annotate/annotation-provider'; const Panel: FC = () => { const {videoId: id = ''} = useParams(); @@ -8,7 +11,19 @@ const Panel: FC = () => { return
; } - return ; + return ( + + + + + + + ); }; +function YjsListener() { + useYjsListener(); + return null; +} + export default Panel; diff --git a/app/frontend/src/page/project/Detail.tsx b/app/frontend/src/page/project/Detail.tsx index 0b8cc95..5483cf4 100644 --- a/app/frontend/src/page/project/Detail.tsx +++ b/app/frontend/src/page/project/Detail.tsx @@ -97,6 +97,7 @@ const DetailReady: FC<{project: Project; spec: ProjectSpec}> = ({project, spec}) />, config.readonly ? (