diff --git a/.golangci.yml b/.golangci.yml index d06c49ea10..57480ca690 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -25,7 +25,7 @@ linters-settings: goimports: local-prefixes: github.com/anyproto/anytype-heart govet: - check-shadowing: true + check-shadowing: false funlen: lines: 120 statements: 100 diff --git a/README.md b/README.md index 942db40d52..f406cf6ac2 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,11 @@ If you want to change the default port(9999): ---- ### Useful tools for debug +#### Debug server +Use env var ANYDEBUG=address to enable debugging HTTP server. For example: `ANYDEBUG=:6061` will start debug server on port 6061 + +You can find all endpoints in `/debug` page. For example: http://localhost:6061/debug + #### gRPC logging In order to log mw gRPC requests/responses use `ANYTYPE_GRPC_LOG` env var: - `ANYTYPE_LOG_LEVEL="grpc=DEBUG" ANYTYPE_GRPC_LOG=1` - log only method names diff --git a/app/app.go b/app/app.go deleted file mode 100644 index 9992127c30..0000000000 --- a/app/app.go +++ /dev/null @@ -1,265 +0,0 @@ -package app - -import ( - "context" - "errors" - "fmt" - "os" - "runtime" - "strings" - "sync" - "time" - - "github.com/anyproto/anytype-heart/metrics" - "github.com/anyproto/anytype-heart/pkg/lib/logging" -) - -var ( - // values of this vars will be defined while compilation - GitCommit, GitBranch, GitState, GitSummary, BuildDate string - name string -) - -var log = logging.Logger("anytype-mw-app") - -// Component is a minimal interface for a common app.Component -type Component interface { - // Init will be called first - // When returned error is not nil - app start will be aborted - Init(a *App) (err error) - // Name must return unique service name - Name() (name string) -} - -// ComponentRunnable is an interface for realizing ability to start background processes or deep configure service -type ComponentRunnable interface { - Component - // Run will be called after init stage - // Non-nil error also will be aborted app start - Run(ctx context.Context) (err error) - // Close will be called when app shutting down - // Also will be called when service return error on Init or Run stage - // Non-nil error will be printed to log - Close() (err error) -} - -type ComponentStatable interface { - StateChange(state int) -} - -// App is the central part of the application -// It contains and manages all components -type App struct { - components []Component - mu sync.RWMutex - startStat StartStat - deviceState int -} - -// Name returns app name -func (app *App) Name() string { - return name -} - -// Version return app version -func (app *App) Version() string { - return GitSummary -} - -type StartStat struct { - SpentMsPerComp map[string]int64 - SpentMsTotal int64 -} - -// StartStat returns total time spent per comp -func (app *App) StartStat() StartStat { - app.mu.Lock() - defer app.mu.Unlock() - return app.startStat -} - -// VersionDescription return the full info about the build -func (app *App) VersionDescription() string { - return VersionDescription() -} - -func Version() string { - return GitSummary -} - -func VersionDescription() string { - return fmt.Sprintf("build on %s from %s at #%s(%s)", BuildDate, GitBranch, GitCommit, GitState) -} - -// Register adds service to registry -// All components will be started in the order they were registered -func (app *App) Register(s Component) *App { - app.mu.Lock() - defer app.mu.Unlock() - for _, es := range app.components { - if s.Name() == es.Name() { - panic(fmt.Errorf("component '%s' already registered", s.Name())) - } - } - app.components = append(app.components, s) - return app -} - -// Component returns service by name -// If service with given name wasn't registered, nil will be returned -func (app *App) Component(name string) Component { - app.mu.RLock() - defer app.mu.RUnlock() - for _, s := range app.components { - if s.Name() == name { - return s - } - } - return nil -} - -// MustComponent is like Component, but it will panic if service wasn't found -func (app *App) MustComponent(name string) Component { - s := app.Component(name) - if s == nil { - panic(fmt.Errorf("component '%s' not registered", name)) - } - return s -} - -func MustComponent[i any](app *App) i { - app.mu.RLock() - defer app.mu.RUnlock() - for _, s := range app.components { - if v, ok := s.(i); ok { - return v - } - } - dummy := new(i) - panic(fmt.Errorf("component with interface %T is not found", dummy)) -} - -// ComponentNames returns all registered names -func (app *App) ComponentNames() (names []string) { - app.mu.RLock() - defer app.mu.RUnlock() - names = make([]string, len(app.components)) - for i, c := range app.components { - names[i] = c.Name() - } - return -} - -// Start starts the application -// All registered services will be initialized and started -func (app *App) Start(ctx context.Context) (err error) { - app.mu.RLock() - defer app.mu.RUnlock() - app.startStat.SpentMsPerComp = make(map[string]int64) - - closeServices := func(idx int) { - for i := idx; i >= 0; i-- { - if serviceClose, ok := app.components[i].(ComponentRunnable); ok { - if e := serviceClose.Close(); e != nil { - log.Warnf("Component '%s' close error: %v", serviceClose.Name(), e) - } - } - } - } - - for i, s := range app.components { - if err = s.Init(app); err != nil { - closeServices(i) - return fmt.Errorf("can't init service '%s': %v", s.Name(), err) - } - } - - for i, s := range app.components { - if serviceRun, ok := s.(ComponentRunnable); ok { - start := time.Now() - if err = serviceRun.Run(ctx); err != nil { - closeServices(i) - return fmt.Errorf("can't run service '%s': %v", serviceRun.Name(), err) - } - spent := time.Since(start).Milliseconds() - app.startStat.SpentMsTotal += spent - app.startStat.SpentMsPerComp[s.Name()] = spent - } - } - - stat := app.startStat - if stat.SpentMsTotal > 300 { - log.Errorf("AccountCreate app start takes %dms: %v", stat.SpentMsTotal, stat.SpentMsPerComp) - } - - var request string - if v, ok := ctx.Value(metrics.CtxKeyRequest).(string); ok { - request = v - } - - metrics.SharedClient.RecordEvent(metrics.AppStart{ - Request: request, - TotalMs: stat.SpentMsTotal, - PerCompMs: stat.SpentMsPerComp}) - log.Debugf("All components started") - return -} - -func stackAllGoroutines() []byte { - buf := make([]byte, 1024) - for { - n := runtime.Stack(buf, true) - if n < len(buf) { - return buf[:n] - } - buf = make([]byte, 2*len(buf)) - } -} - -// Close stops the application -// All components with ComponentRunnable implementation will be closed in the reversed order -func (app *App) Close() error { - log.Infof("Close components...") - app.mu.RLock() - defer app.mu.RUnlock() - done := make(chan struct{}) - go func() { - select { - case <-done: - return - case <-time.After(time.Minute): - _, _ = os.Stderr.Write([]byte("app.Close timeout\n")) - _, _ = os.Stderr.Write(stackAllGoroutines()) - panic("app.Close timeout") - } - }() - - var errs []string - for i := len(app.components) - 1; i >= 0; i-- { - if serviceClose, ok := app.components[i].(ComponentRunnable); ok { - log.Debugf("Close '%s'", serviceClose.Name()) - if e := serviceClose.Close(); e != nil { - errs = append(errs, fmt.Sprintf("Component '%s' close error: %v", serviceClose.Name(), e)) - } - } - } - close(done) - if len(errs) > 0 { - return errors.New(strings.Join(errs, "\n")) - } - log.Debugf("All components have been closed") - - return nil -} - -func (app *App) SetDeviceState(state int) { - if app == nil { - return - } - app.deviceState = state - for _, component := range app.components { - if statable, ok := component.(ComponentStatable); ok { - statable.StateChange(state) - } - } -} diff --git a/app/app_test.go b/app/app_test.go deleted file mode 100644 index 79a3c2ffc6..0000000000 --- a/app/app_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package app - -import ( - "context" - "fmt" - "sync/atomic" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestAppServiceRegistry(t *testing.T) { - app := new(App) - t.Run("Register", func(t *testing.T) { - app.Register(newTestService(testTypeRunnable, "c1", nil, nil)) - app.Register(newTestService(testTypeRunnable, "r1", nil, nil)) - app.Register(newTestService(testTypeComponent, "s1", nil, nil)) - }) - t.Run("Component", func(t *testing.T) { - assert.Nil(t, app.Component("not-registered")) - for _, name := range []string{"c1", "r1", "s1"} { - s := app.Component(name) - assert.NotNil(t, s, name) - assert.Equal(t, name, s.Name()) - } - }) - t.Run("MustComponent", func(t *testing.T) { - for _, name := range []string{"c1", "r1", "s1"} { - assert.NotPanics(t, func() { app.MustComponent(name) }, name) - } - assert.Panics(t, func() { app.MustComponent("not-registered") }) - }) - t.Run("ComponentNames", func(t *testing.T) { - names := app.ComponentNames() - assert.Equal(t, names, []string{"c1", "r1", "s1"}) - }) -} - -func TestAppStart(t *testing.T) { - t.Run("SuccessStartStop", func(t *testing.T) { - app := new(App) - seq := new(testSeq) - services := [...]iTestService{ - newTestService(testTypeRunnable, "c1", nil, seq), - newTestService(testTypeRunnable, "r1", nil, seq), - newTestService(testTypeComponent, "s1", nil, seq), - newTestService(testTypeRunnable, "c2", nil, seq), - } - for _, s := range services { - app.Register(s) - } - assert.Nil(t, app.Start(context.Background())) - assert.Nil(t, app.Close()) - - var actual []testIds - for _, s := range services { - actual = append(actual, s.Ids()) - } - - expected := []testIds{ - {1, 5, 10}, - {2, 6, 9}, - {3, 0, 0}, - {4, 7, 8}, - } - - assert.Equal(t, expected, actual) - }) - - t.Run("InitError", func(t *testing.T) { - app := new(App) - seq := new(testSeq) - expectedErr := fmt.Errorf("testError") - services := [...]iTestService{ - newTestService(testTypeRunnable, "c1", nil, seq), - newTestService(testTypeRunnable, "c2", expectedErr, seq), - } - for _, s := range services { - app.Register(s) - } - - err := app.Start(context.Background()) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), expectedErr.Error()) - - var actual []testIds - for _, s := range services { - actual = append(actual, s.Ids()) - } - - expected := []testIds{ - {1, 0, 4}, - {2, 0, 3}, - } - assert.Equal(t, expected, actual) - }) -} - -const ( - testTypeComponent int = iota - testTypeRunnable -) - -func newTestService(componentType int, name string, err error, seq *testSeq) (s iTestService) { - ts := testComponent{name: name, err: err, seq: seq} - switch componentType { - case testTypeComponent: - return &ts - case testTypeRunnable: - return &testRunnable{testComponent: ts} - } - return nil -} - -type iTestService interface { - Component - Ids() (ids testIds) -} - -type testIds struct { - initId int64 - runId int64 - closeId int64 -} - -type testComponent struct { - name string - err error - seq *testSeq - ids testIds -} - -func (t *testComponent) Init(a *App) error { - t.ids.initId = t.seq.New() - return t.err -} - -func (t *testComponent) Name() string { return t.name } - -func (t *testComponent) Ids() testIds { - return t.ids -} - -type testRunnable struct { - testComponent -} - -func (t *testRunnable) Run(context.Context) error { - t.ids.runId = t.seq.New() - return t.err -} - -func (t *testRunnable) Close() error { - t.ids.closeId = t.seq.New() - return t.err -} - -type testSeq struct { - seq int64 -} - -func (ts *testSeq) New() int64 { - return atomic.AddInt64(&ts.seq, 1) -} diff --git a/cmd/dbbenchmark/dbbenchmark.go b/cmd/dbbenchmark/dbbenchmark.go deleted file mode 100644 index 7635b4c881..0000000000 --- a/cmd/dbbenchmark/dbbenchmark.go +++ /dev/null @@ -1,251 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "math/rand" - "os" - "path/filepath" - "time" - - "github.com/gogo/protobuf/types" - dsbadgerv3 "github.com/textileio/go-ds-badger3" - - "github.com/anyproto/anytype-heart/pkg/lib/bundle" - "github.com/anyproto/anytype-heart/pkg/lib/datastore" - "github.com/anyproto/anytype-heart/pkg/lib/datastore/clientds" - "github.com/anyproto/anytype-heart/pkg/lib/datastore/noctxds" - "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" - "github.com/anyproto/anytype-heart/pkg/lib/pb/model" - "github.com/anyproto/anytype-heart/util/pbtypes" -) - -const localstoreDir string = "localstore" -const objectType string = "_otobject_type" - -type options struct { - isV3 bool - sync bool - path string -} - -func (o *options) withDatastoreVersion(isV3 bool) *options { - o.isV3 = isV3 - return o -} - -func (o *options) withDatastorePath(path string) *options { - o.path = path - return o -} - -func initObjecStore(o *options) (os objectstore.ObjectStore, closer func(), err error) { - var ds datastore.DSTxnBatching - if o.isV3 { - ds, err = initBadgerV3(o) - closer = func() { - ds.Close() - } - } else { - ds, err = initBadgerV1(o) - closer = func() { - ds.Close() - } - } - if err != nil { - return - } - - ds2 := noctxds.New(ds) - return objectstore.NewWithLocalstore(ds2), closer, nil -} - -func initBadgerV3(o *options) (*dsbadgerv3.Datastore, error) { - cfg := clientds.DefaultConfig.Localstore - cfg.SyncWrites = o.sync - localstoreDS, err := dsbadgerv3.NewDatastore(filepath.Join(o.path, localstoreDir), &cfg) - if err != nil { - return nil, err - } - return localstoreDS, nil -} - -func initBadgerV1(o *options) (*dsbadgerv3.Datastore, error) { - cfg := clientds.DefaultConfig.Localstore - cfg.SyncWrites = o.sync - localstoreDS, err := dsbadgerv3.NewDatastore(filepath.Join(o.path, localstoreDir), &cfg) - if err != nil { - return nil, err - } - return localstoreDS, nil -} - -var ( - detailsCount = flag.Int("det_count", 10, "the number of details of each object") - relationsCount = flag.Int("rel_count", 10, "the number of relations of each object") - sync = flag.Bool("s", false, "sync mode") - path = flag.String("p", "", "path to localstore") - isV3 = flag.Bool("isv3", true, "are we using badger v3") - keys = flag.Int("keys", 100000, "the number of different keys to be used") -) - -func init() { - rand.Seed(time.Now().UnixNano()) -} - -const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -const ( - letterIdxBits = 6 - letterIdxMask = 1<= 0; { - if remain == 0 { - cache, remain = rand.Int63(), letterIdxMax - } - if idx := int(cache & letterIdxMask); idx < len(letterBytes) { - b[i] = letterBytes[idx] - i-- - } - cache >>= letterIdxBits - remain-- - } - - return string(b) -} - -func genRandomIds(count, size int) []string { - buf := make([]string, 0, count) - for i := 0; i < count; i++ { - // we use _anytype_profile, so it will deem this as profile id and not check the string - // in SmartBlockTypeFromID - buf = append(buf, "_anytype_profile"+randString(size)) - } - return buf -} - -func genRandomDetails(strings []string, count int) *types.Struct { - f := make(map[string]*types.Value) - min := count - if count > len(strings) { - min = len(strings) - } - for i := 0; i < min; i++ { - f[strings[i]] = pbtypes.String(randString(60)) - } - f[bundle.RelationKeySetOf.String()] = pbtypes.String(objectType) - return &types.Struct{ - Fields: f, - } -} - -func genRandomRelations(strings []string, count int) *model.Relations { - var rels []*model.Relation - min := count - if count > len(strings) { - min = len(strings) - } - for i := 0; i < min; i++ { - rels = append(rels, []*model.Relation{ - { - Key: strings[i], - Format: model.RelationFormat_status, - Name: randString(60), - DefaultValue: nil, - SelectDict: []*model.RelationOption{ - {"id1", "option1", "red", strings[i]}, - {"id2", "option2", "red", strings[i]}, - {"id3", "option3", "red", strings[i]}, - }, - }, - { - Key: strings[i][:len(strings[i])-1], - Format: model.RelationFormat_shorttext, - Name: randString(60), - DefaultValue: nil, - }, - }...) - } - return &model.Relations{Relations: rels} -} - -func createObjects(store objectstore.ObjectStore, ids []string, detailsCount int, relationsCount int) error { - avg := float32(0) - i := float32(0) - for _, id := range ids { - details := genRandomDetails(ids, detailsCount) - start := time.Now() - err := store.UpdateObjectDetails(id, details) - if err != nil { - fmt.Println("error occurred while updating object store:", err.Error()) - return err - } - err = store.UpdateObjectSnippet(id, "snippet") - if err != nil { - fmt.Println("error occurred while updating snippet:", err.Error()) - return err - } - - taken := float32(time.Now().Sub(start).Nanoseconds()) - avg = (avg*i + taken) / (i + 1) - i += 1.0 - } - fmt.Println("avg create operation time ms", avg/1000000) - return nil -} - -func updateDetails(store objectstore.ObjectStore, ids []string, detailsCount int, relationsCount int) error { - avg := float32(0) - i := float32(0) - for _, id := range ids { - details := genRandomDetails(ids, detailsCount) - start := time.Now() - err := store.UpdateObjectDetails(id, details) - if err != nil { - fmt.Println("error occurred while updating object store:", err.Error()) - return err - } - taken := float32(time.Now().Sub(start).Nanoseconds()) - avg = (avg*i + taken) / (i + 1) - i += 1.0 - } - fmt.Println("avg update operation time ms", avg/1000000) - return nil -} - -func main() { - // go run dbbenchmark.go -p localstore -keys 3000 -det_count 200 -rel_count 10 -isv3 false -s false - // this should be read as total keys 3000, entries in details struct - 200, entries in relations - 10 - // using badger v3 - false, sync writes - false - flag.Parse() - if *path == "" { - flag.PrintDefaults() - return - } - os.RemoveAll(*path) - o := &options{ - isV3: *isV3, - path: *path, - sync: *sync, - } - store, closeDb, err := initObjecStore(o) - if err != nil { - fmt.Println("error occurred when opening object store", err.Error()) - return - } - defer closeDb() - ids := genRandomIds(*keys, 64) - err = createObjects(store, ids, *detailsCount, *relationsCount) - if err != nil { - fmt.Println(err) - return - } - err = updateDetails(store, ids, *detailsCount, *relationsCount) - if err != nil { - fmt.Println(err) - return - } -} diff --git a/cmd/debugtree/debugtree.go b/cmd/debugtree/debugtree.go index 94e0578cdb..0eb4a39015 100644 --- a/cmd/debugtree/debugtree.go +++ b/cmd/debugtree/debugtree.go @@ -102,7 +102,7 @@ func main() { if *printState { fmt.Println("Building state...") stt := time.Now() - s, err := importer.State() + s, err := importer.State(false) if err != nil { log.Fatal("can't build state:", err) } diff --git a/cmd/grpcserver/grpc.go b/cmd/grpcserver/grpc.go index d881da751e..5142d98cb8 100644 --- a/cmd/grpcserver/grpc.go +++ b/cmd/grpcserver/grpc.go @@ -48,7 +48,7 @@ var log = logging.Logger("anytype-grpc-server") func main() { var addr string var webaddr string - + app.StartWarningAfter = time.Second * 5 fmt.Printf("mw grpc: %s\n", app.VersionDescription()) if len(os.Args) > 1 { addr = os.Args[1] @@ -118,17 +118,21 @@ func main() { unaryInterceptors = append(unaryInterceptors, func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { doneCh := make(chan struct{}) start := time.Now() + + l := log.With("method", info.FullMethod) + go func() { select { case <-doneCh: case <-time.After(defaultUnaryWarningAfter): - log.With("method", info.FullMethod).With("in_progress", true).With("goroutines", debug.StackCompact(true)).With("total", defaultUnaryWarningAfter.Milliseconds()).Warnf("grpc unary request is taking too long") + l.With("in_progress", true).With("goroutines", debug.StackCompact(true)).With("total", defaultUnaryWarningAfter.Milliseconds()).Warnf("grpc unary request is taking too long") } }() + ctx = context.WithValue(ctx, metrics.CtxKeyRPC, info.FullMethod) resp, err = handler(ctx, req) close(doneCh) if time.Since(start) > defaultUnaryWarningAfter { - log.With("method", info.FullMethod).With("error", err).With("in_progress", false).With("total", time.Since(start).Milliseconds()).Warnf("grpc unary request took too long") + l.With("error", err).With("in_progress", false).With("total", time.Since(start).Milliseconds()).Warnf("grpc unary request took too long") } return }) diff --git a/core/account.go b/core/account.go index 857322863a..f189953d93 100644 --- a/core/account.go +++ b/core/account.go @@ -156,7 +156,7 @@ func (mw *Middleware) AccountCreate(cctx context.Context, req *pb.RpcAccountCrea mw.requireClientWithVersion() if mw.app, err = anytype.StartNewApp( - context.WithValue(context.Background(), metrics.CtxKeyRequest, "account_create"), + context.WithValue(context.Background(), metrics.CtxKeyEntrypoint, "account_create"), mw.clientWithVersion, comps..., ); err != nil { @@ -310,7 +310,7 @@ func (mw *Middleware) AccountSelect(cctx context.Context, req *pb.RpcAccountSele } mw.requireClientWithVersion() if mw.app, err = anytype.StartNewApp( - context.WithValue(context.Background(), metrics.CtxKeyRequest, request), + context.WithValue(context.Background(), metrics.CtxKeyEntrypoint, request), mw.clientWithVersion, comps..., ); err != nil { @@ -663,7 +663,7 @@ func (mw *Middleware) startApp(cfg *config.Config, derivationResult crypto.Deriv mw.EventSender, } - ctxWithValue := context.WithValue(context.Background(), metrics.CtxKeyRequest, "account_create") + ctxWithValue := context.WithValue(context.Background(), metrics.CtxKeyEntrypoint, "account_create") var err error if mw.app, err = anytype.StartNewApp(ctxWithValue, mw.clientWithVersion, comps...); err != nil { return err diff --git a/core/anytype/bootstrap.go b/core/anytype/bootstrap.go index 9451bff14c..c7e05a2da8 100644 --- a/core/anytype/bootstrap.go +++ b/core/anytype/bootstrap.go @@ -22,6 +22,8 @@ import ( "github.com/anyproto/any-sync/nodeconf" "github.com/anyproto/any-sync/nodeconf/nodeconfstore" "github.com/anyproto/any-sync/util/crypto" + "github.com/anyproto/anytype-heart/pkg/lib/logging" + "go.uber.org/zap" "github.com/anyproto/anytype-heart/core/anytype/config" "github.com/anyproto/anytype-heart/core/block" @@ -76,6 +78,11 @@ import ( "github.com/anyproto/anytype-heart/util/vcs" ) +var ( + log = logging.LoggerNotSugared("anytype-app") + WarningAfter = time.Second * 1 +) + func BootstrapConfig(newAccount bool, isStaging bool, createBuiltinTemplates bool) *config.Config { return config.New( config.WithDebugAddr(os.Getenv("ANYTYPE_DEBUG_ADDR")), @@ -94,12 +101,39 @@ func StartNewApp(ctx context.Context, clientWithVersion string, components ...ap Bootstrap(a, components...) metrics.SharedClient.SetAppVersion(a.Version()) metrics.SharedClient.Run() + startTime := time.Now() if err = a.Start(ctx); err != nil { metrics.SharedClient.Close() a = nil return } + totalSpent := time.Since(startTime) + l := log.With(zap.Int64("total", totalSpent.Milliseconds())) + if v, ok := ctx.Value(metrics.CtxKeyRPC).(string); ok { + l = l.With(zap.String("rpc", v)) + } + for comp, spent := range a.StartStat().SpentMsPerComp { + if spent == 0 { + continue + } + l = l.With(zap.Int64(comp, spent)) + } + l.With(zap.Int64("totalRun", a.StartStat().SpentMsTotal)) + a.IterateComponents(func(comp app.Component) { + if c, ok := comp.(ComponentLogFieldsGetter); ok { + for _, field := range c.GetLogFields() { + field.Key = comp.Name() + "_" + field.Key + l = l.With(field) + } + } + }) + + if totalSpent > WarningAfter { + l.Warn("app started") + } else { + l.Debug("app started") + } return } @@ -149,9 +183,8 @@ func Bootstrap(a *app.App, components ...app.Component) { eventService.Send, fileWatcherUpdateInterval, ) - + fileSyncService.OnUpload(syncStatusService.OnFileUpload) fileService := files.New(syncStatusService, objectStore) - indexerService := indexer.New(blockService, spaceService, fileService) a.Register(datastoreProvider). @@ -213,9 +246,14 @@ func Bootstrap(a *app.App, components ...app.Component) { Register(kanban.New()). Register(editor.NewObjectFactory(tempDirService, sbtProvider, layoutConverter)). Register(graphRenderer) - return } func MiddlewareVersion() string { return vcs.GetVCSInfo().Version() } + +type ComponentLogFieldsGetter interface { + // GetLogFields returns additional useful fields for logs to debug long app start/stop duration or something else in the future + // You don't need to provide the component name in the field's Key, because it will be added automatically + GetLogFields() []zap.Field +} diff --git a/core/block/cache.go b/core/block/cache.go index 3161aa7479..dd5406630d 100644 --- a/core/block/cache.go +++ b/core/block/cache.go @@ -370,11 +370,13 @@ func (s *Service) getDerivedObject( if newAccount { var tr objecttree.ObjectTree tr, err = space.TreeBuilder().PutTree(ctx, *payload, nil) + s.predefinedObjectWasMissing = true if err != nil { if !errors.Is(err, treestorage.ErrTreeExists) { err = fmt.Errorf("failed to put tree: %w", err) return } + s.predefinedObjectWasMissing = false // the object exists locally return s.GetAccountObject(ctx, payload.RootRawChange.Id) } diff --git a/core/block/editor/file/file.go b/core/block/editor/file/file.go index 18fea05c40..49d96041e7 100644 --- a/core/block/editor/file/file.go +++ b/core/block/editor/file/file.go @@ -154,9 +154,12 @@ func (sf *sfile) upload(s *state.State, id string, source FileSource, isSync boo if source.Path != "" { upl.SetFile(source.Path) } else if source.Url != "" { - upl.SetUrl(source.Url) + upl.SetUrl(source.Url). + SetLastModifiedDate() } else if len(source.Bytes) > 0 { - upl.SetBytes(source.Bytes).SetName(source.Name) + upl.SetBytes(source.Bytes). + SetName(source.Name). + SetLastModifiedDate() } if isSync { return upl.Upload(context.TODO()) @@ -519,7 +522,13 @@ func (dp *dropFilesProcess) addFilesWorker(wg *sync.WaitGroup, in chan *dropFile func (dp *dropFilesProcess) addFile(f *dropFileInfo) (err error) { upl := NewUploader(dp.s, dp.fileService, dp.tempDirProvider) - res := upl.SetName(f.name).AutoType(true).SetFile(f.path).Upload(context.TODO()) + + res := upl. + SetName(f.name). + AutoType(true). + SetFile(f.path). + Upload(context.TODO()) + if res.Err != nil { log.With("filePath", f.path).Errorf("upload error: %s", res.Err) f.err = fmt.Errorf("upload error: %w", res.Err) diff --git a/core/block/editor/file/uploader.go b/core/block/editor/file/uploader.go index 1abc18bad2..7b8f9582ab 100644 --- a/core/block/editor/file/uploader.go +++ b/core/block/editor/file/uploader.go @@ -58,6 +58,7 @@ type Uploader interface { SetBytes(b []byte) Uploader SetUrl(url string) Uploader SetFile(path string) Uploader + SetLastModifiedDate() Uploader SetGroupId(groupId string) Uploader AddOptions(options ...files.AddOption) Uploader AutoType(enable bool) Uploader @@ -97,17 +98,18 @@ func (ur UploadResult) ToBlock() file.Block { } type uploader struct { - service BlockService - block file.Block - getReader func(ctx context.Context) (*fileReader, error) - name string - typeDetect bool - forceType bool - smartBlockId string - fileType model.BlockContentFileType - fileStyle model.BlockContentFileStyle - opts []files.AddOption - groupId string + service BlockService + block file.Block + getReader func(ctx context.Context) (*fileReader, error) + name string + lastModifiedDate int64 + typeDetect bool + forceType bool + smartBlockID string + fileType model.BlockContentFileType + fileStyle model.BlockContentFileStyle + opts []files.AddOption + groupID string tempDirProvider core.TempDirProvider fileService files.Service @@ -148,7 +150,7 @@ func (u *uploader) SetBlock(block file.Block) Uploader { } func (u *uploader) SetGroupId(groupId string) Uploader { - u.groupId = groupId + u.groupID = groupId return u } @@ -189,7 +191,7 @@ func (u *uploader) SetUrl(url string) Uploader { if err != nil { // do nothing } - u.name = strings.Split(filepath.Base(url), "?")[0] + u.SetName(strings.Split(filepath.Base(url), "?")[0]) u.getReader = func(ctx context.Context) (*fileReader, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -250,7 +252,9 @@ func (u *uploader) SetUrl(url string) Uploader { } func (u *uploader) SetFile(path string) Uploader { - u.name = filepath.Base(path) + u.SetName(filepath.Base(path)) + u.setLastModifiedDate(path) + u.getReader = func(ctx context.Context) (*fileReader, error) { f, err := os.Open(path) if err != nil { @@ -273,13 +277,27 @@ func (u *uploader) SetFile(path string) Uploader { return u } +func (u *uploader) SetLastModifiedDate() Uploader { + u.lastModifiedDate = time.Now().Unix() + return u +} + +func (u *uploader) setLastModifiedDate(path string) { + stat, err := os.Stat(path) + if err == nil { + u.lastModifiedDate = stat.ModTime().Unix() + } else { + u.lastModifiedDate = time.Now().Unix() + } +} + func (u *uploader) AutoType(enable bool) Uploader { u.typeDetect = enable return u } func (u *uploader) AsyncUpdates(smartBlockId string) Uploader { - u.smartBlockId = smartBlockId + u.smartBlockID = smartBlockId return u } @@ -323,7 +341,7 @@ func (u *uploader) Upload(ctx context.Context) (result UploadResult) { } if fileName := buf.GetFileName(); fileName != "" { - u.name = fileName + u.SetName(fileName) } if u.block != nil { @@ -341,6 +359,7 @@ func (u *uploader) Upload(ctx context.Context) (result UploadResult) { } var opts = []files.AddOption{ files.WithName(u.name), + files.WithLastModifiedDate(u.lastModifiedDate), files.WithReader(buf), } @@ -362,6 +381,7 @@ func (u *uploader) Upload(ctx context.Context) (result UploadResult) { result.Hash = im.Hash() orig, _ := im.GetOriginalFile(context.TODO()) if orig != nil { + result.MIME = orig.Meta().Media result.Size = orig.Meta().Size } } else { @@ -389,7 +409,8 @@ func (u *uploader) Upload(ctx context.Context) (result UploadResult) { if u.block != nil { u.block.SetName(u.name). SetState(model.BlockContentFile_Done). - SetType(u.fileType).SetHash(result.Hash). + SetType(u.fileType). + SetHash(result.Hash). SetSize(result.Size). SetStyle(u.fileStyle). SetMIME(result.MIME) @@ -425,9 +446,9 @@ func (u *uploader) detectTypeByMIME(mime string) model.BlockContentFileType { } func (u *uploader) updateBlock() { - if u.smartBlockId != "" && u.block != nil { - err := u.service.DoFile(u.smartBlockId, func(f File) error { - return f.UpdateFile(u.block.Model().Id, u.groupId, func(b file.Block) error { + if u.smartBlockID != "" && u.block != nil { + err := u.service.DoFile(u.smartBlockID, func(f File) error { + return f.UpdateFile(u.block.Model().Id, u.groupID, func(b file.Block) error { b.SetModel(u.block.Copy().Model().GetFile()) return nil }) diff --git a/core/block/editor/file/uploader_test.go b/core/block/editor/file/uploader_test.go index 11a461b995..d82e401ad9 100644 --- a/core/block/editor/file/uploader_test.go +++ b/core/block/editor/file/uploader_test.go @@ -38,14 +38,16 @@ func TestUploader_Upload(t *testing.T) { defer fx.tearDown() im := fx.newImage("123") fx.fileService.EXPECT().ImageAdd(gomock.Any(), gomock.Any()).Return(im, nil) - im.EXPECT().GetOriginalFile(gomock.Any()) + im.EXPECT().GetOriginalFile(gomock.Any()).Return(fx.file, nil) b := newBlock(model.BlockContentFile_Image) fx.blockService.EXPECT().Do(gomock.Any(), gomock.Any()).Return(nil) + fx.file.EXPECT().Meta().Return(&files.FileMeta{Media: "image/jpg"}).AnyTimes() res := fx.Uploader.SetBlock(b).SetFile("./testdata/unnamed.jpg").Upload(ctx) require.NoError(t, res.Err) assert.Equal(t, res.Hash, "123") assert.Equal(t, res.Name, "unnamed.jpg") assert.Equal(t, b.Model().GetFile().Name, "unnamed.jpg") + assert.Equal(t, res.MIME, "image/jpg") }) t.Run("image type detect", func(t *testing.T) { fx := newFixture(t) @@ -164,12 +166,15 @@ func newFixture(t *testing.T) *uplFixture { fx.blockService = NewMockBlockService(fx.ctrl) fx.Uploader = file.NewUploader(fx.blockService, fx.fileService, core.NewTempDirService(nil)) + fx.file = testMock.NewMockFile(fx.ctrl) + fx.file.EXPECT().Hash().Return("123").AnyTimes() return fx } type uplFixture struct { file.Uploader blockService *MockBlockService + file *testMock.MockFile fileService *testMock.MockFileService ctrl *gomock.Controller } diff --git a/core/block/editor/smartblock/smartblock.go b/core/block/editor/smartblock/smartblock.go index 4898556752..5eef8cd2d2 100644 --- a/core/block/editor/smartblock/smartblock.go +++ b/core/block/editor/smartblock/smartblock.go @@ -907,6 +907,7 @@ func (sb *smartBlock) injectLocalDetails(s *state.State) error { if err != nil { return err } + details := storedDetails.GetDetails() var hasPendingLocalDetails bool // Consume pending details @@ -914,7 +915,7 @@ func (sb *smartBlock) injectLocalDetails(s *state.State) error { if len(pending.GetFields()) > 0 { hasPendingLocalDetails = true } - storedDetails.Details = pbtypes.StructMerge(storedDetails.GetDetails(), pending, false) + details = pbtypes.StructMerge(details, pending, false) return nil, nil }) if err != nil { @@ -925,7 +926,7 @@ func (sb *smartBlock) injectLocalDetails(s *state.State) error { // inject also derived keys, because it may be a good idea to have created date and creator cached, // so we don't need to traverse changes every time keys := append(bundle.LocalRelationsKeys, bundle.DerivedRelationsKeys...) - storedLocalScopeDetails := pbtypes.StructFilterKeys(storedDetails.GetDetails(), keys) + storedLocalScopeDetails := pbtypes.StructFilterKeys(details, keys) sbLocalScopeDetails := pbtypes.StructFilterKeys(s.LocalDetails(), keys) if pbtypes.StructEqualIgnore(sbLocalScopeDetails, storedLocalScopeDetails, nil) { return nil diff --git a/core/block/editor/state/test/buildfast_test.go b/core/block/editor/state/test/buildfast_test.go new file mode 100644 index 0000000000..0a4ac43638 --- /dev/null +++ b/core/block/editor/state/test/buildfast_test.go @@ -0,0 +1,79 @@ +package debug + +import ( + "io/ioutil" + "log" + "os" + "path/filepath" + "testing" + "time" + + "github.com/anyproto/anytype-heart/core/debug/treearchive" + "github.com/stretchr/testify/require" +) + +func TestBuildFast(t *testing.T) { + // Specify the directory you want to iterate + dir := "./testdata" + + // Read the directory + files, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatalf("Failed to read dir: %s", err) + } + + // Iterate over the files + for _, file := range files { + t.Run(file.Name(), func(t *testing.T) { + filePath := filepath.Join(dir, file.Name()) + + // open the file + f, err := os.Open(filePath) + if err != nil { + t.Fatalf("Failed to open file: %s", err) + } + defer f.Close() + + testBuildFast(t, filePath) + }) + } +} + +func testBuildFast(b *testing.T, filepath string) { + // todo: replace with less heavy tree + archive, err := treearchive.Open(filepath) + if err != nil { + require.NoError(b, err) + } + defer archive.Close() + + importer := treearchive.NewTreeImporter(archive.ListStorage(), archive.TreeStorage()) + + err = importer.Import(false, "") + if err != nil { + log.Fatal("can't import the tree", err) + } + + start := time.Now() + s, err := importer.State(false) + if err != nil { + log.Fatal("can't build state:", err) + } + b.Logf("fast build took %s", time.Since(start)) + + importer2 := treearchive.NewTreeImporter(archive.ListStorage(), archive.TreeStorage()) + + err = importer2.Import(false, "") + if err != nil { + log.Fatal("can't import the tree", err) + } + + s2, err := importer2.State(true) + if err != nil { + log.Fatal("can't build state:", err) + } + b.Logf("slow build took %s", time.Since(start)) + + require.Equal(b, s.StringDebug(), s2.StringDebug()) + +} diff --git a/core/block/editor/state/test/testdata/blocks.zip b/core/block/editor/state/test/testdata/blocks.zip new file mode 100644 index 0000000000..c740775de7 Binary files /dev/null and b/core/block/editor/state/test/testdata/blocks.zip differ diff --git a/core/block/editor/state/test/testdata/store.zip b/core/block/editor/state/test/testdata/store.zip new file mode 100644 index 0000000000..4b479d64a4 Binary files /dev/null and b/core/block/editor/state/test/testdata/store.zip differ diff --git a/core/block/editor/template.go b/core/block/editor/template.go index 5273f85521..d3919bd4d9 100644 --- a/core/block/editor/template.go +++ b/core/block/editor/template.go @@ -91,9 +91,11 @@ func (t *Template) GetNewPageState(name string) (st *state.State, err error) { // clean-up local details from the template state st.SetLocalDetails(nil) - st.SetDetail(bundle.RelationKeyName.String(), pbtypes.String(name)) - if title := st.Get(template.TitleBlockId); title != nil { - title.Model().GetText().Text = "" + if name != "" { + st.SetDetail(bundle.RelationKeyName.String(), pbtypes.String(name)) + if title := st.Get(template.TitleBlockId); title != nil { + title.Model().GetText().Text = "" + } } return } diff --git a/core/block/editor/template_test.go b/core/block/editor/template_test.go new file mode 100644 index 0000000000..5b7d64a52f --- /dev/null +++ b/core/block/editor/template_test.go @@ -0,0 +1,70 @@ +package editor + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/block/editor/smartblock" + "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" + "github.com/anyproto/anytype-heart/core/block/editor/template" + "github.com/anyproto/anytype-heart/core/block/migration" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/util/pbtypes" + "github.com/anyproto/anytype-heart/util/testMock" +) + +func NewTemplateTest(ctrl *gomock.Controller, templateName string) (*Template, error) { + sb := smarttest.New("root") + _ = sb.SetDetails(nil, []*pb.RpcObjectSetDetailsDetail{&pb.RpcObjectSetDetailsDetail{ + Key: bundle.RelationKeyName.String(), + Value: pbtypes.String(templateName), + }}, false) + objectStore := testMock.NewMockObjectStore(ctrl) + objectStore.EXPECT().GetObjectTypes(gomock.Any()).AnyTimes() + t := &Template{ + Page: &Page{ + SmartBlock: sb, + objectStore: objectStore, + }, + } + initCtx := &smartblock.InitContext{IsNewObject: true} + if err := t.Init(initCtx); err != nil { + return nil, err + } + migration.RunMigrations(t, initCtx) + if err := t.Apply(initCtx.State); err != nil { + return nil, err + } + return t, nil +} + +func TestTemplate_GetNewPageState(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + templateName := "template" + + t.Run("empty page name", func(t *testing.T) { + tmpl, err := NewTemplateTest(ctrl, templateName) + require.NoError(t, err) + + st, err := tmpl.GetNewPageState("") + require.NoError(t, err) + require.Equal(t, st.Details().Fields[bundle.RelationKeyName.String()].GetStringValue(), templateName) + require.Equal(t, st.Get(template.TitleBlockId).Model().GetText().Text, templateName) + }) + + t.Run("custom page name", func(t *testing.T) { + tmpl, err := NewTemplateTest(ctrl, templateName) + require.NoError(t, err) + + customName := "some name" + st, err := tmpl.GetNewPageState(customName) + require.NoError(t, err) + require.Equal(t, st.Details().Fields[bundle.RelationKeyName.String()].GetStringValue(), customName) + require.Equal(t, st.Get(template.TitleBlockId).Model().GetText().Text, "") + }) +} diff --git a/core/block/getblock/getblock.go b/core/block/getblock/getblock.go index e6d4efa8c4..6a36eed037 100644 --- a/core/block/getblock/getblock.go +++ b/core/block/getblock/getblock.go @@ -13,7 +13,7 @@ type Picker interface { } func Do[t any](p Picker, id string, apply func(sb t) error) error { - sb, err := p.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do"), id) + sb, err := p.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do"), id) if err != nil { return err } diff --git a/core/block/import/converter/collection.go b/core/block/import/converter/collection.go index bf67a3ac6a..05c25c12a2 100644 --- a/core/block/import/converter/collection.go +++ b/core/block/import/converter/collection.go @@ -24,7 +24,7 @@ func NewRootCollection(service *collection.Service) *RootCollection { return &RootCollection{service: service} } -func (r *RootCollection) AddObjects(collectionName string, targetObjects []string) (*Snapshot, error) { +func (r *RootCollection) MakeRootCollection(collectionName string, targetObjects []string) (*Snapshot, error) { detailsStruct := r.getCreateCollectionRequest(collectionName) _, _, st, err := r.service.CreateCollection(detailsStruct, []*model.InternalFlag{{ Value: model.InternalFlag_collectionDontIndexLinks, diff --git a/core/block/import/converter/common.go b/core/block/import/converter/common.go index 59549dd6aa..fdb5bc208d 100644 --- a/core/block/import/converter/common.go +++ b/core/block/import/converter/common.go @@ -50,6 +50,7 @@ func GetCommonDetails(sourcePath, name, emoji string) *types.Struct { bundle.RelationKeyName.String(): pbtypes.String(name), bundle.RelationKeySourceFilePath.String(): pbtypes.String(sourcePath), bundle.RelationKeyIconEmoji.String(): pbtypes.String(emoji), + bundle.RelationKeyCreatedDate.String(): pbtypes.Int64(time.Now().Unix()), // this relation will be after used in the tree header } return &types.Struct{Fields: fields} } diff --git a/core/block/import/converter/error.go b/core/block/import/converter/error.go index 94ebb14ea2..fbc1fc1311 100644 --- a/core/block/import/converter/error.go +++ b/core/block/import/converter/error.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/pkg/errors" + + "github.com/anyproto/anytype-heart/pb" ) var ErrCancel = fmt.Errorf("import is canceled") @@ -59,3 +61,33 @@ func (ce ConvertError) Error() error { func (ce ConvertError) Get(objectName string) error { return ce[objectName] } + +func (ce ConvertError) GetResultError(importType pb.RpcObjectImportRequestType) error { + if ce.IsEmpty() { + return nil + } + var countNoObjectsToImport int + for _, e := range ce { + switch { + case errors.Is(e, ErrCancel): + return errors.Wrapf(ErrCancel, "import type: %s", importType.String()) + case errors.Is(e, ErrNoObjectsToImport): + countNoObjectsToImport++ + } + } + // we return ErrNoObjectsToImport only if all paths has such error, otherwise we assume that import finished with internal code error + if countNoObjectsToImport == len(ce) { + return errors.Wrapf(ErrNoObjectsToImport, "import type: %s", importType.String()) + } + return errors.Wrapf(ce.Error(), "import type: %s", importType.String()) +} + +func (ce ConvertError) IsNoObjectToImportError(importPathsCount int) bool { + var countNoObjectsToImport int + for _, err := range ce { + if errors.Is(err, ErrNoObjectsToImport) { + countNoObjectsToImport++ + } + } + return importPathsCount == countNoObjectsToImport +} diff --git a/core/block/import/csv/collectionstrategy.go b/core/block/import/csv/collectionstrategy.go index 16328fcc97..0e244a41d3 100644 --- a/core/block/import/csv/collectionstrategy.go +++ b/core/block/import/csv/collectionstrategy.go @@ -1,8 +1,6 @@ package csv import ( - "strings" - "github.com/globalsign/mgo/bson" "github.com/gogo/protobuf/types" "github.com/google/uuid" @@ -11,7 +9,7 @@ import ( "github.com/anyproto/anytype-heart/core/block/editor/state" "github.com/anyproto/anytype-heart/core/block/editor/template" "github.com/anyproto/anytype-heart/core/block/import/converter" - "github.com/anyproto/anytype-heart/core/block/import/markdown/anymark/whitespace" + "github.com/anyproto/anytype-heart/core/block/process" "github.com/anyproto/anytype-heart/core/block/simple" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/bundle" @@ -32,18 +30,18 @@ func NewCollectionStrategy(collectionService *collection.Service) *CollectionStr return &CollectionStrategy{collectionService: collectionService} } -func (c *CollectionStrategy) CreateObjects(path string, csvTable [][]string) ([]string, []*converter.Snapshot, error) { +func (c *CollectionStrategy) CreateObjects(path string, csvTable [][]string, useFirstRowForRelations bool, progress process.Progress) (string, []*converter.Snapshot, error) { snapshots := make([]*converter.Snapshot, 0) allObjectsIDs := make([]string, 0) details := converter.GetCommonDetails(path, "", "") details.GetFields()[bundle.RelationKeyLayout.String()] = pbtypes.Float64(float64(model.ObjectType_collection)) _, _, st, err := c.collectionService.CreateCollection(details, nil) if err != nil { - return nil, nil, err + return "", nil, err } relations, relationsSnapshots := getDetailsFromCSVTable(csvTable) - objectsSnapshots := getEmptyObjects(csvTable, relations) + objectsSnapshots := getObjectsFromCSVRows(csvTable, relations, useFirstRowForRelations) targetIDs := make([]string, 0, len(objectsSnapshots)) for _, objectsSnapshot := range objectsSnapshots { targetIDs = append(targetIDs, objectsSnapshot.Id) @@ -56,9 +54,8 @@ func (c *CollectionStrategy) CreateObjects(path string, csvTable [][]string) ([] snapshots = append(snapshots, snapshot) snapshots = append(snapshots, objectsSnapshots...) snapshots = append(snapshots, relationsSnapshots...) - allObjectsIDs = append(allObjectsIDs, snapshot.Id) - - return allObjectsIDs, snapshots, nil + progress.AddDone(1) + return snapshot.Id, snapshots, nil } func getDetailsFromCSVTable(csvTable [][]string) ([]*model.Relation, []*converter.Snapshot) { @@ -66,23 +63,28 @@ func getDetailsFromCSVTable(csvTable [][]string) ([]*model.Relation, []*converte return nil, nil } relations := make([]*model.Relation, 0, len(csvTable[0])) + // first column is always a name + relations = append(relations, &model.Relation{ + Format: model.RelationFormat_shorttext, + Key: bundle.RelationKeyName.String(), + }) relationsSnapshots := make([]*converter.Snapshot, 0, len(csvTable[0])) allRelations := csvTable[0] - for _, relation := range allRelations { - if relation == "" { + for i := 1; i < len(allRelations); i++ { + if allRelations[i] == "" { continue } id := bson.NewObjectId().Hex() relations = append(relations, &model.Relation{ Format: model.RelationFormat_longtext, - Name: relation, + Name: allRelations[i], Key: id, }) relationsSnapshots = append(relationsSnapshots, &converter.Snapshot{ Id: addr.RelationKeyToIdPrefix + id, SbType: smartblock.SmartBlockTypeSubObject, Snapshot: &pb.ChangeSnapshot{Data: &model.SmartBlockSnapshotBase{ - Details: getRelationDetails(relation, id, float64(model.RelationFormat_longtext)), + Details: getRelationDetails(allRelations[i], id, float64(model.RelationFormat_longtext)), ObjectTypes: []string{bundle.TypeKeyRelation.URL()}, }}, }) @@ -100,9 +102,13 @@ func getRelationDetails(name, key string, format float64) *types.Struct { return details } -func getEmptyObjects(csvTable [][]string, relations []*model.Relation) []*converter.Snapshot { +func getObjectsFromCSVRows(csvTable [][]string, relations []*model.Relation, useFirstRowForRelations bool) []*converter.Snapshot { snapshots := make([]*converter.Snapshot, 0, len(csvTable)) - for i := 1; i < len(csvTable); i++ { + for i := 0; i < len(csvTable); i++ { + // skip first row if option is turned on + if i == 0 && useFirstRowForRelations { + continue + } st := state.NewDoc("root", map[string]simple.Block{ "root": simple.New(&model.Block{ Content: &model.BlockContentOfSmartblock{ @@ -110,31 +116,32 @@ func getEmptyObjects(csvTable [][]string, relations []*model.Relation) []*conver }, }), }).NewState() - details, relationLinks := getDetailsForObject(csvTable, relations, i) + details, relationLinks := getDetailsForObject(csvTable[i], relations, i == 0) st.SetDetails(details) - template.InitTemplate(st, template.WithTitle) st.AddRelationLinks(relationLinks...) + template.InitTemplate(st, template.WithTitle) sn := provideObjectSnapshot(st, details) snapshots = append(snapshots, sn) } return snapshots } -func getDetailsForObject(csvTable [][]string, relations []*model.Relation, i int) (*types.Struct, []*model.RelationLink) { +func getDetailsForObject(relationsValues []string, relations []*model.Relation, isFirstRowObject bool) (*types.Struct, []*model.RelationLink) { details := &types.Struct{Fields: map[string]*types.Value{}} relationLinks := make([]*model.RelationLink, 0) - for j, value := range csvTable[i] { + for j, value := range relationsValues { if len(relations) <= j { break } - name := strings.TrimSpace(whitespace.WhitespaceNormalizeString(relations[j].Name)) - if strings.EqualFold(name, "name") { - relations[j].Key = bundle.RelationKeyName.String() + relation := relations[j] + details.Fields[relation.Key] = pbtypes.String(value) + // if first row is an object and relation key is not a name, we create empty relations for this object + if isFirstRowObject && relation.Key != bundle.RelationKeyName.String() { + details.Fields[relation.Key] = pbtypes.String("") } - details.Fields[relations[j].Key] = pbtypes.String(value) relationLinks = append(relationLinks, &model.RelationLink{ - Key: relations[j].Key, - Format: relations[j].Format, + Key: relation.Key, + Format: relation.Format, }) } return details, relationLinks diff --git a/core/block/import/csv/converter.go b/core/block/import/csv/converter.go index 8c55f2afe9..3c94481258 100644 --- a/core/block/import/csv/converter.go +++ b/core/block/import/csv/converter.go @@ -14,8 +14,9 @@ import ( ) const ( - Name = "Csv" - rootCollectionName = "CSV Import" + Name = "Csv" + rootCollectionName = "CSV Import" + numberOfProgressSteps = 2 ) type Result struct { @@ -56,17 +57,17 @@ func (c *CSV) GetSnapshots(req *pb.RpcObjectImportRequest, progress process.Prog if params == nil { return nil, nil } - progress.SetProgressMessage("Start creating snapshots from files") cErr := converter.NewError() result, cancelError := c.createObjectsFromCSVFiles(req, progress, params, cErr) if !cancelError.IsEmpty() { return nil, cancelError } - if !cErr.IsEmpty() && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { + if (!cErr.IsEmpty() && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING) || + (cErr.IsNoObjectToImportError(len(params.Path))) { return nil, cErr } rootCollection := converter.NewRootCollection(c.collectionService) - rootCol, err := rootCollection.AddObjects(rootCollectionName, result.objectIDs) + rootCol, err := rootCollection.MakeRootCollection(rootCollectionName, result.objectIDs) if err != nil { cErr.Add(rootCollectionName, err) if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { @@ -76,7 +77,7 @@ func (c *CSV) GetSnapshots(req *pb.RpcObjectImportRequest, progress process.Prog if rootCol != nil { result.snapshots = append(result.snapshots, rootCol) } - progress.SetTotal(int64(len(result.objectIDs))) + progress.SetTotal(int64(len(result.snapshots))) if cErr.IsEmpty() { return &converter.Response{Snapshots: result.snapshots}, nil } @@ -92,10 +93,7 @@ func (c *CSV) createObjectsFromCSVFiles(req *pb.RpcObjectImportRequest, str := c.chooseStrategy(csvMode) result := &Result{} for _, p := range params.GetPath() { - if err := progress.TryStep(1); err != nil { - return nil, converter.NewCancelError(p, err) - } - pathResult := c.handlePath(req, p, cErr, str) + pathResult := c.getSnapshotsFromFiles(req, p, cErr, str, progress) if !cErr.IsEmpty() && req.GetMode() == pb.RpcObjectImportRequest_ALL_OR_NOTHING { return nil, nil } @@ -104,7 +102,7 @@ func (c *CSV) createObjectsFromCSVFiles(req *pb.RpcObjectImportRequest, return result, nil } -func (c *CSV) handlePath(req *pb.RpcObjectImportRequest, p string, cErr converter.ConvertError, str Strategy) *Result { +func (c *CSV) getSnapshotsFromFiles(req *pb.RpcObjectImportRequest, p string, cErr converter.ConvertError, str Strategy, progress process.Progress) *Result { params := req.GetCsvParams() s := source.GetSource(p) if s == nil { @@ -122,30 +120,37 @@ func (c *CSV) handlePath(req *pb.RpcObjectImportRequest, p string, cErr converte cErr.Add(p, converter.ErrNoObjectsToImport) return nil } - return c.handleCSVTables(req.Mode, readers, params, str, p, cErr) + return c.getSnapshots(req.Mode, readers, params, str, p, cErr, progress) } -func (c *CSV) handleCSVTables(mode pb.RpcObjectImportRequestMode, +func (c *CSV) getSnapshots(mode pb.RpcObjectImportRequestMode, readers map[string]io.ReadCloser, params *pb.RpcObjectImportRequestCsvParams, str Strategy, p string, - cErr converter.ConvertError) *Result { + cErr converter.ConvertError, + progress process.Progress) *Result { allSnapshots := make([]*converter.Snapshot, 0) allObjectsIDs := make([]string, 0) - for _, rc := range readers { + progress.SetProgressMessage("Start creating snapshots from files") + progress.SetTotal(int64(len(readers) * numberOfProgressSteps)) + for filePath, rc := range readers { + if err := progress.TryStep(1); err != nil { + cErr = converter.NewCancelError("", err) + return nil + } csvTable, err := c.getCSVTable(rc, params.GetDelimiter()) if err != nil { - cErr.Add(p, err) + cErr.Add(filePath, err) if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { return nil } continue } - if c.needToTranspose(params) && len(csvTable) != 0 { + if params.TransposeRowsAndColumns && len(csvTable) != 0 { csvTable = transpose(csvTable) } - objectsIDs, snapshots, err := str.CreateObjects(p, csvTable) + collectionID, snapshots, err := str.CreateObjects(filePath, csvTable, params.UseFirstRowForRelations, progress) if err != nil { cErr.Add(p, err) if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { @@ -153,7 +158,7 @@ func (c *CSV) handleCSVTables(mode pb.RpcObjectImportRequestMode, } continue } - allObjectsIDs = append(allObjectsIDs, objectsIDs...) + allObjectsIDs = append(allObjectsIDs, collectionID) allSnapshots = append(allSnapshots, snapshots...) } return &Result{objectIDs: allObjectsIDs, snapshots: allSnapshots} @@ -175,11 +180,6 @@ func (c *CSV) getCSVTable(rc io.ReadCloser, delimiter string) ([][]string, error return csvTable, nil } -func (c *CSV) needToTranspose(params *pb.RpcObjectImportRequestCsvParams) bool { - return (params.GetTransposeRowsAndColumns() && params.GetUseFirstRowForRelations()) || - (!params.GetUseFirstRowForRelations() && !params.GetTransposeRowsAndColumns()) -} - func (c *CSV) chooseStrategy(mode pb.RpcObjectImportRequestCsvParamsMode) Strategy { if mode == pb.RpcObjectImportRequestCsvParams_COLLECTION { return NewCollectionStrategy(c.collectionService) diff --git a/core/block/import/csv/converter_test.go b/core/block/import/csv/converter_test.go index 07eea572fb..80a1ad975b 100644 --- a/core/block/import/csv/converter_test.go +++ b/core/block/import/csv/converter_test.go @@ -1,15 +1,18 @@ package csv import ( - smartblock2 "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" + "strings" "testing" + "github.com/samber/lo" "github.com/stretchr/testify/assert" "github.com/anyproto/anytype-heart/core/block/editor/template" + "github.com/anyproto/anytype-heart/core/block/import/converter" "github.com/anyproto/anytype-heart/core/block/process" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/bundle" + sb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" "github.com/anyproto/anytype-heart/util/pbtypes" ) @@ -44,7 +47,9 @@ func TestCsv_GetSnapshots(t *testing.T) { p := process.NewProgress(pb.ModelProcess_Import) sn, err := csv.GetSnapshots(&pb.RpcObjectImportRequest{ Params: &pb.RpcObjectImportRequestParamsOfCsvParams{ - CsvParams: &pb.RpcObjectImportRequestCsvParams{Path: []string{"testdata/Journal.csv"}}, + CsvParams: &pb.RpcObjectImportRequestCsvParams{ + Path: []string{"testdata/Journal.csv"}, + UseFirstRowForRelations: true}, }, Type: pb.RpcObjectImportRequest_Csv, Mode: pb.RpcObjectImportRequest_IGNORE_ERRORS, @@ -53,7 +58,7 @@ func TestCsv_GetSnapshots(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, sn) - assert.Len(t, sn.Snapshots, 7) // test + root collection + 2 objects in Journal.csv + 3 relations (Name, Created, Tags) + assert.Len(t, sn.Snapshots, 6) // Journal.csv + root collection + 2 objects in Journal.csv + 2 relations (Created, Tags) assert.Contains(t, sn.Snapshots[0].FileName, "Journal.csv") assert.Len(t, pbtypes.GetStringList(sn.Snapshots[0].Snapshot.Data.Collections, template.CollectionStoreKey), 2) // 2 objects @@ -63,7 +68,7 @@ func TestCsv_GetSnapshots(t *testing.T) { found = true assert.NotEmpty(t, snapshot.Snapshot.Data.ObjectTypes) assert.Equal(t, snapshot.Snapshot.Data.ObjectTypes[0], bundle.TypeKeyCollection.URL()) - assert.Len(t, pbtypes.GetStringList(snapshot.Snapshot.Data.Collections, template.CollectionStoreKey), 3) // all objects + assert.Len(t, pbtypes.GetStringList(snapshot.Snapshot.Data.Collections, template.CollectionStoreKey), 1) // only Journal.csv collection } } @@ -99,6 +104,47 @@ func TestCsv_GetSnapshotsTable(t *testing.T) { assert.True(t, found) } +func TestCsv_GetSnapshotsTableUseFirstColumnForRelationsOn(t *testing.T) { + csv := CSV{} + p := process.NewProgress(pb.ModelProcess_Import) + sn, err := csv.GetSnapshots(&pb.RpcObjectImportRequest{ + Params: &pb.RpcObjectImportRequestParamsOfCsvParams{ + CsvParams: &pb.RpcObjectImportRequestCsvParams{ + Path: []string{"testdata/Journal.csv"}, + Mode: pb.RpcObjectImportRequestCsvParams_TABLE, + UseFirstRowForRelations: true, + }, + }, + Type: pb.RpcObjectImportRequest_Csv, + Mode: pb.RpcObjectImportRequest_IGNORE_ERRORS, + }, p) + + assert.Nil(t, err) + + assert.NotNil(t, sn) + assert.Len(t, sn.Snapshots, 2) // 1 page with table + root collection + assert.Contains(t, sn.Snapshots[0].FileName, "Journal.csv") + + var rowsID []string + for _, bl := range sn.Snapshots[0].Snapshot.Data.Blocks { + if blockContent, ok := bl.Content.(*model.BlockContentOfLayout); ok && blockContent.Layout.Style == model.BlockContentLayout_TableRows { + rowsID = bl.GetChildrenIds() + } + } + assert.NotNil(t, rowsID) + for _, bl := range sn.Snapshots[0].Snapshot.Data.Blocks { + if blockContent, ok := bl.Content.(*model.BlockContentOfTableRow); ok && bl.Id == rowsID[0] { + assert.True(t, blockContent.TableRow.IsHeader) + } + } + + for _, bl := range sn.Snapshots[0].Snapshot.Data.Blocks { + if strings.Contains(bl.Id, rowsID[0]) && bl.GetText() != nil { + assert.True(t, bl.GetText().Text == "Name" || bl.GetText().Text == "Created" || bl.GetText().Text == "Tags") + } + } +} + func TestCsv_GetSnapshotsSemiColon(t *testing.T) { csv := CSV{} p := process.NewProgress(pb.ModelProcess_Import) @@ -112,7 +158,7 @@ func TestCsv_GetSnapshotsSemiColon(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, sn) - assert.Len(t, sn.Snapshots, 13) // 8 objects + root collection + semicolon collection + 3 relations + assert.Len(t, sn.Snapshots, 12) // 8 objects + root collection + semicolon collection + 2 relations assert.Contains(t, sn.Snapshots[0].FileName, "semicolon.csv") assert.Len(t, pbtypes.GetStringList(sn.Snapshots[0].Snapshot.Data.Collections, template.CollectionStoreKey), 8) assert.Equal(t, sn.Snapshots[0].Snapshot.Data.ObjectTypes[0], bundle.TypeKeyCollection.URL()) @@ -127,7 +173,6 @@ func TestCsv_GetSnapshotsTranspose(t *testing.T) { Path: []string{"testdata/transpose.csv"}, Delimiter: ";", TransposeRowsAndColumns: true, - UseFirstRowForRelations: true, }, }, Type: pb.RpcObjectImportRequest_Csv, @@ -136,16 +181,104 @@ func TestCsv_GetSnapshotsTranspose(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, sn) - assert.Len(t, sn.Snapshots, 5) // 1 object + root collection + transpose collection + 2 relations + assert.Len(t, sn.Snapshots, 5) // 2 object + root collection + transpose collection + 1 relations for _, snapshot := range sn.Snapshots { - if snapshot.SbType == smartblock2.SmartBlockTypeSubObject { + if snapshot.SbType == sb.SmartBlockTypeSubObject { name := pbtypes.GetString(snapshot.Snapshot.GetData().GetDetails(), bundle.RelationKeyName.String()) assert.True(t, name == "name" || name == "price") } } } +func TestCsv_GetSnapshotsUseFirstColumnForRelationsOn(t *testing.T) { + csv := CSV{} + p := process.NewProgress(pb.ModelProcess_Import) + sn, err := csv.GetSnapshots(&pb.RpcObjectImportRequest{ + Params: &pb.RpcObjectImportRequestParamsOfCsvParams{ + CsvParams: &pb.RpcObjectImportRequestCsvParams{ + Path: []string{"testdata/Journal.csv"}, + Delimiter: ",", + UseFirstRowForRelations: true, + }, + }, + Type: pb.RpcObjectImportRequest_Csv, + Mode: pb.RpcObjectImportRequest_IGNORE_ERRORS, + }, p) + + assert.Nil(t, err) + assert.NotNil(t, sn) + assert.Len(t, sn.Snapshots, 6) // Journal.csv collection, root collection + 2 objects in Journal.csv + 2 relations (Created, Tags) + + var rowsObjects []*converter.Snapshot + for _, snapshot := range sn.Snapshots { + // only objects created from rows + if snapshot.SbType != sb.SmartBlockTypeSubObject && + !lo.Contains(snapshot.Snapshot.Data.ObjectTypes, bundle.TypeKeyCollection.URL()) { + rowsObjects = append(rowsObjects, snapshot) + } + } + + assert.Len(t, rowsObjects, 2) + + want := [][]string{ + {"Hawaii Vacation", "July 13, 2022 8:54 AM", "Special Event"}, + {"Just another day", "July 13, 2022 8:54 AM", "Daily"}, + } + assertSnapshotsHaveDetails(t, want[0], rowsObjects[0]) + assertSnapshotsHaveDetails(t, want[1], rowsObjects[1]) +} + +func assertSnapshotsHaveDetails(t *testing.T, want []string, objects *converter.Snapshot) { + for _, value := range objects.Snapshot.Data.Details.Fields { + assert.Contains(t, want, value.GetStringValue()) + } +} + +func TestCsv_GetSnapshotsUseFirstColumnForRelationsOff(t *testing.T) { + csv := CSV{} + p := process.NewProgress(pb.ModelProcess_Import) + sn, err := csv.GetSnapshots(&pb.RpcObjectImportRequest{ + Params: &pb.RpcObjectImportRequestParamsOfCsvParams{ + CsvParams: &pb.RpcObjectImportRequestCsvParams{ + Path: []string{"testdata/Journal.csv"}, + Delimiter: ",", + }, + }, + Type: pb.RpcObjectImportRequest_Csv, + Mode: pb.RpcObjectImportRequest_IGNORE_ERRORS, + }, p) + + assert.Nil(t, err) + assert.NotNil(t, sn) + assert.Len(t, sn.Snapshots, 7) // Journal.csv collection, root collection + 3 objects in Journal.csv + 2 relations (Created, Tags) + + var emptyObjects []*converter.Snapshot + for _, snapshot := range sn.Snapshots { + // only objects created from rows + if snapshot.SbType != sb.SmartBlockTypeSubObject && + !lo.Contains(snapshot.Snapshot.Data.ObjectTypes, bundle.TypeKeyCollection.URL()) { + emptyObjects = append(emptyObjects, snapshot) + } + } + + assert.Len(t, emptyObjects, 3) // first row is also an object + + for _, value := range emptyObjects[0].Snapshot.Data.Details.Fields { + assert.True(t, value.GetStringValue() == "Name" || value.GetStringValue() == "") + } + + for _, value := range emptyObjects[1].Snapshot.Data.Details.Fields { + assert.True(t, value.GetStringValue() == "Hawaii Vacation" || value.GetStringValue() == "July 13, 2022 8:54 AM" || + value.GetStringValue() == "Special Event") + } + + for _, value := range emptyObjects[2].Snapshot.Data.Details.Fields { + assert.True(t, value.GetStringValue() == "Just another day" || value.GetStringValue() == "July 13, 2022 8:54 AM" || + value.GetStringValue() == "Daily") + } +} + func TestCsv_GetSnapshotsQuotedStrings(t *testing.T) { csv := CSV{} p := process.NewProgress(pb.ModelProcess_Import) diff --git a/core/block/import/csv/strategy.go b/core/block/import/csv/strategy.go index 0598a02e5b..a0d1c36d98 100644 --- a/core/block/import/csv/strategy.go +++ b/core/block/import/csv/strategy.go @@ -2,8 +2,9 @@ package csv import ( "github.com/anyproto/anytype-heart/core/block/import/converter" + "github.com/anyproto/anytype-heart/core/block/process" ) type Strategy interface { - CreateObjects(path string, csvTable [][]string) ([]string, []*converter.Snapshot, error) + CreateObjects(path string, csvTable [][]string, useFirstRowForRelations bool, progress process.Progress) (string, []*converter.Snapshot, error) } diff --git a/core/block/import/csv/tablestrategy.go b/core/block/import/csv/tablestrategy.go index e4bbc288e3..65d8f5902e 100644 --- a/core/block/import/csv/tablestrategy.go +++ b/core/block/import/csv/tablestrategy.go @@ -6,6 +6,7 @@ import ( "github.com/anyproto/anytype-heart/core/block/editor/state" te "github.com/anyproto/anytype-heart/core/block/editor/table" "github.com/anyproto/anytype-heart/core/block/import/converter" + "github.com/anyproto/anytype-heart/core/block/process" "github.com/anyproto/anytype-heart/core/block/simple" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/bundle" @@ -21,7 +22,7 @@ func NewTableStrategy(tableEditor te.TableEditor) *TableStrategy { return &TableStrategy{tableEditor: tableEditor} } -func (c *TableStrategy) CreateObjects(path string, csvTable [][]string) ([]string, []*converter.Snapshot, error) { +func (c *TableStrategy) CreateObjects(path string, csvTable [][]string, useFirstRowForHeader bool, progress process.Progress) (string, []*converter.Snapshot, error) { st := state.NewDoc("root", map[string]simple.Block{ "root": simple.New(&model.Block{ Content: &model.BlockContentOfSmartblock{ @@ -31,9 +32,9 @@ func (c *TableStrategy) CreateObjects(path string, csvTable [][]string) ([]strin }).NewState() if len(csvTable) != 0 { - err := c.createTable(st, csvTable) + err := c.createTable(st, csvTable, useFirstRowForHeader) if err != nil { - return nil, nil, err + return "", nil, err } } @@ -52,10 +53,11 @@ func (c *TableStrategy) CreateObjects(path string, csvTable [][]string) ([]strin FileName: path, Snapshot: &pb.ChangeSnapshot{Data: sn}, } - return []string{snapshot.Id}, []*converter.Snapshot{snapshot}, nil + progress.AddDone(1) + return snapshot.Id, []*converter.Snapshot{snapshot}, nil } -func (c *TableStrategy) createTable(st *state.State, csvTable [][]string) error { +func (c *TableStrategy) createTable(st *state.State, csvTable [][]string, useFirstRowForHeader bool) error { tableID, err := c.tableEditor.TableCreate(st, pb.RpcBlockTableCreateRequest{}) if err != nil { return err @@ -65,9 +67,14 @@ func (c *TableStrategy) createTable(st *state.State, csvTable [][]string) error if err != nil { return err } - + if !useFirstRowForHeader { + err = c.createEmptyHeader(st, tableID, columnIDs) + if err != nil { + return err + } + } for i, columns := range csvTable { - rowID, err := c.createRow(st, tableID, i) + rowID, err := c.createRow(st, tableID, i == 0, useFirstRowForHeader) if err != nil { return err } @@ -80,6 +87,36 @@ func (c *TableStrategy) createTable(st *state.State, csvTable [][]string) error return nil } +func (c *TableStrategy) createEmptyHeader(st *state.State, tableID string, columnIDs []string) error { + rowID, err := c.tableEditor.RowCreate(st, pb.RpcBlockTableRowCreateRequest{ + Position: model.Block_Inner, + TargetId: tableID, + }) + if err != nil { + return err + } + for _, colID := range columnIDs { + textBlock := &model.Block{ + Id: uuid.New().String(), + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{Text: ""}, + }, + } + _, err = c.tableEditor.CellCreate(st, rowID, colID, textBlock) + if err != nil { + return err + } + } + err = c.tableEditor.RowSetHeader(st, pb.RpcBlockTableRowSetHeaderRequest{ + IsHeader: true, + TargetId: rowID, + }) + if err != nil { + return err + } + return nil +} + func (c *TableStrategy) createCells(columns []string, st *state.State, rowID string, columnIDs []string) error { for j, column := range columns { textBlock := &model.Block{ @@ -96,7 +133,7 @@ func (c *TableStrategy) createCells(columns []string, st *state.State, rowID str return nil } -func (c *TableStrategy) createRow(st *state.State, tableID string, i int) (string, error) { +func (c *TableStrategy) createRow(st *state.State, tableID string, isFirstRow bool, useFirstRowForHeader bool) (string, error) { rowID, err := c.tableEditor.RowCreate(st, pb.RpcBlockTableRowCreateRequest{ Position: model.Block_Inner, TargetId: tableID, @@ -105,7 +142,7 @@ func (c *TableStrategy) createRow(st *state.State, tableID string, i int) (strin return "", err } - if i == 0 { + if isFirstRow && useFirstRowForHeader { err = c.tableEditor.RowSetHeader(st, pb.RpcBlockTableRowSetHeaderRequest{ IsHeader: true, TargetId: rowID, diff --git a/core/block/import/csv/testdata/Journal.csv b/core/block/import/csv/testdata/Journal.csv index 56da04db55..85b85aeb2b 100644 --- a/core/block/import/csv/testdata/Journal.csv +++ b/core/block/import/csv/testdata/Journal.csv @@ -1,3 +1,3 @@ -Name,Created,Tags +Name,Created,Tags Hawaii Vacation,"July 13, 2022 8:54 AM",Special Event Just another day,"July 13, 2022 8:54 AM",Daily \ No newline at end of file diff --git a/core/block/import/html/converter.go b/core/block/import/html/converter.go index d7bda02b5f..2cc9caa277 100644 --- a/core/block/import/html/converter.go +++ b/core/block/import/html/converter.go @@ -56,12 +56,13 @@ func (h *HTML) GetSnapshots(req *pb.RpcObjectImportRequest, progress process.Pro if !cancelError.IsEmpty() { return nil, cancelError } - if !cErr.IsEmpty() && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { + if (!cErr.IsEmpty() && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING) || + (cErr.IsNoObjectToImportError(len(path))) { return nil, cErr } rootCollection := converter.NewRootCollection(h.collectionService) - rootCol, err := rootCollection.AddObjects(rootCollectionName, targetObjects) + rootCol, err := rootCollection.MakeRootCollection(rootCollectionName, targetObjects) if err != nil { cErr.Add(rootCollectionName, err) if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { diff --git a/core/block/import/importer.go b/core/block/import/importer.go index 36b5edc463..4020a8f618 100644 --- a/core/block/import/importer.go +++ b/core/block/import/importer.go @@ -87,7 +87,7 @@ func (i *Import) Init(a *app.App) (err error) { i.objectIDGetter = NewObjectIDGetter(store, coreService, i.s) fileStore := app.MustComponent[filestore.FileStore](a) relationSyncer := syncer.NewFileRelationSyncer(i.s, fileStore) - i.oc = NewCreator(i.s, objCreator, coreService, factory, store, relationSyncer) + i.oc = NewCreator(i.s, objCreator, coreService, factory, store, relationSyncer, fileStore) return nil } @@ -102,9 +102,9 @@ func (i *Import) Import(ctx *session.Context, req *pb.RpcObjectImportRequest) er if c, ok := i.converters[req.Type.String()]; ok { res, err := c.GetSnapshots(req, progress) if len(err) != 0 { - e := getResultError(err, req.Type) - if shouldReturnError(e, req) { - return e + resultErr := err.GetResultError(req.Type) + if shouldReturnError(resultErr, res, req) { + return resultErr } allErrors.Merge(err) } @@ -118,7 +118,7 @@ func (i *Import) Import(ctx *session.Context, req *pb.RpcObjectImportRequest) er } i.createObjects(ctx, res, progress, req, allErrors) - return getResultError(allErrors, req.Type) + return allErrors.GetResultError(req.Type) } if req.Type == pb.RpcObjectImportRequest_External { if req.Snapshots != nil { @@ -134,7 +134,7 @@ func (i *Import) Import(ctx *session.Context, req *pb.RpcObjectImportRequest) er } i.createObjects(ctx, res, progress, req, allErrors) if !allErrors.IsEmpty() { - return getResultError(allErrors, req.Type) + return allErrors.GetResultError(req.Type) } return nil } @@ -143,10 +143,10 @@ func (i *Import) Import(ctx *session.Context, req *pb.RpcObjectImportRequest) er return fmt.Errorf("unknown import type %s", req.Type) } -func shouldReturnError(e error, req *pb.RpcObjectImportRequest) bool { +func shouldReturnError(e error, res *converter.Response, req *pb.RpcObjectImportRequest) bool { return (e != nil && req.Mode != pb.RpcObjectImportRequest_IGNORE_ERRORS) || - e == converter.ErrNoObjectsToImport || - e == converter.ErrCancel + (errors.Is(e, converter.ErrNoObjectsToImport) && (res == nil || len(res.Snapshots) == 0)) || // return error only if we don't have object to import + errors.Is(e, converter.ErrCancel) } func (i *Import) setupProgressBar(req *pb.RpcObjectImportRequest) process.Progress { @@ -305,7 +305,7 @@ func (i *Import) getObjectID(ctx *session.Context, } else { createdTime = time.Now() } - if id, payload, err = i.objectIDGetter.Get(ctx, snapshot, snapshot.SbType, createdTime, updateExisting); err == nil { + if id, payload, err = i.objectIDGetter.Get(ctx, snapshot, snapshot.SbType, createdTime, updateExisting, oldIDToNew); err == nil { oldIDToNew[snapshot.Id] = id if snapshot.SbType == sb.SmartBlockTypeSubObject && id == "" { oldIDToNew[snapshot.Id] = snapshot.Id @@ -357,23 +357,3 @@ func (i *Import) readResultFromPool(pool *workerpool.WorkerPool, func convertType(cType string) pb.RpcObjectImportListImportResponseType { return pb.RpcObjectImportListImportResponseType(pb.RpcObjectImportListImportResponseType_value[cType]) } - -func getResultError(err converter.ConvertError, importType pb.RpcObjectImportRequestType) error { - if err.IsEmpty() { - return nil - } - var countNoObjectsToImport int - for _, e := range err { - switch { - case errors.Is(e, converter.ErrCancel): - return errors.Wrapf(converter.ErrCancel, "import type: %s", importType.String()) - case errors.Is(e, converter.ErrNoObjectsToImport): - countNoObjectsToImport++ - } - } - // we return ErrNoObjectsToImport only if all paths has such error, otherwise we assume that import finished with internal code error - if countNoObjectsToImport == len(err) { - return errors.Wrapf(converter.ErrNoObjectsToImport, importType.String()) - } - return errors.Wrapf(err.Error(), importType.String()) -} diff --git a/core/block/import/importer_test.go b/core/block/import/importer_test.go index e1fe9e880e..acbc748e64 100644 --- a/core/block/import/importer_test.go +++ b/core/block/import/importer_test.go @@ -46,7 +46,7 @@ func Test_ImportSuccess(t *testing.T) { i.oc = creator idGetter := NewMockIDGetter(ctrl) - idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) + idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) i.objectIDGetter = idGetter err := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ Params: &pb.RpcObjectImportRequestParamsOfPbParams{PbParams: &pb.RpcObjectImportRequestPbParams{Path: []string{"bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a.pb"}}}, @@ -111,7 +111,7 @@ func Test_ImportErrorFromObjectCreator(t *testing.T) { creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, "", errors.New("creator error")).Times(1) i.oc = creator idGetter := NewMockIDGetter(ctrl) - idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) + idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) i.objectIDGetter = idGetter res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ Params: &pb.RpcObjectImportRequestParamsOfPbParams{PbParams: &pb.RpcObjectImportRequestPbParams{Path: []string{"test"}}}, @@ -152,7 +152,7 @@ func Test_ImportIgnoreErrorMode(t *testing.T) { creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, "", nil).Times(1) i.oc = creator idGetter := NewMockIDGetter(ctrl) - idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) + idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) i.objectIDGetter = idGetter res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ Params: &pb.RpcObjectImportRequestParamsOfPbParams{PbParams: &pb.RpcObjectImportRequestPbParams{Path: []string{"test"}}}, @@ -195,7 +195,7 @@ func Test_ImportIgnoreErrorModeWithTwoErrorsPerFile(t *testing.T) { creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, "", errors.New("creator error")).Times(1) i.oc = creator idGetter := NewMockIDGetter(ctrl) - idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) + idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) i.objectIDGetter = idGetter res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ Params: &pb.RpcObjectImportRequestParamsOfPbParams{PbParams: &pb.RpcObjectImportRequestPbParams{Path: []string{"test"}}}, @@ -220,7 +220,7 @@ func Test_ImportExternalPlugin(t *testing.T) { creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, "", nil).Times(1) i.oc = creator idGetter := NewMockIDGetter(ctrl) - idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) + idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) i.objectIDGetter = idGetter snapshots := make([]*pb.RpcObjectImportRequestSnapshot, 0) snapshots = append(snapshots, &pb.RpcObjectImportRequestSnapshot{ @@ -357,7 +357,7 @@ func Test_ImportWebSuccess(t *testing.T) { creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, "", nil).Times(1) i.oc = creator idGetter := NewMockIDGetter(ctrl) - idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) + idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) i.objectIDGetter = idGetter parser := parsers.NewMockParser(ctrl) parser.EXPECT().MatchUrl("http://example.com").Return(true).Times(1) @@ -398,7 +398,7 @@ func Test_ImportWebFailedToCreateObject(t *testing.T) { creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, "", errors.New("error")).Times(1) i.oc = creator idGetter := NewMockIDGetter(ctrl) - idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) + idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) i.objectIDGetter = idGetter parser := parsers.NewMockParser(ctrl) parser.EXPECT().MatchUrl("http://example.com").Return(true).Times(1) @@ -425,3 +425,116 @@ func Test_ImportWebFailedToCreateObject(t *testing.T) { assert.NotNil(t, err) assert.Equal(t, "couldn't create objects", err.Error()) } + +func Test_ImportCancelError(t *testing.T) { + i := Import{} + ctrl := gomock.NewController(t) + converter := cv.NewMockConverter(ctrl) + e := cv.NewCancelError("path", fmt.Errorf("converter error")) + converter.EXPECT().GetSnapshots(gomock.Any(), gomock.Any()).Return(&cv.Response{Snapshots: nil}, e).Times(1) + i.converters = make(map[string]cv.Converter, 0) + i.converters["Notion"] = converter + res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ + Params: &pb.RpcObjectImportRequestParamsOfPbParams{PbParams: &pb.RpcObjectImportRequestPbParams{Path: []string{"test"}}}, + UpdateExistingObjects: false, + Type: 0, + Mode: pb.RpcObjectImportRequest_IGNORE_ERRORS, + }) + + assert.NotNil(t, res) + assert.True(t, errors.Is(res, cv.ErrCancel)) +} + +func Test_ImportNoObjectToImportError(t *testing.T) { + i := Import{} + ctrl := gomock.NewController(t) + converter := cv.NewMockConverter(ctrl) + e := cv.NewFromError("test", cv.ErrNoObjectsToImport) + converter.EXPECT().GetSnapshots(gomock.Any(), gomock.Any()).Return(&cv.Response{Snapshots: nil}, e).Times(1) + i.converters = make(map[string]cv.Converter, 0) + i.converters["Notion"] = converter + res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ + Params: &pb.RpcObjectImportRequestParamsOfPbParams{PbParams: &pb.RpcObjectImportRequestPbParams{Path: []string{"test"}}}, + UpdateExistingObjects: false, + Type: 0, + Mode: pb.RpcObjectImportRequest_IGNORE_ERRORS, + }) + + assert.NotNil(t, res) + assert.True(t, errors.Is(res, cv.ErrNoObjectsToImport)) +} + +func Test_ImportNoObjectToImportErrorModeAllOrNothing(t *testing.T) { + i := Import{} + ctrl := gomock.NewController(t) + converter := cv.NewMockConverter(ctrl) + e := cv.NewFromError("test", cv.ErrNoObjectsToImport) + converter.EXPECT().GetSnapshots(gomock.Any(), gomock.Any()).Return(&cv.Response{Snapshots: []*cv.Snapshot{{ + Snapshot: &pb.ChangeSnapshot{ + Data: &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{&model.Block{ + Id: "1", + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: "test", + Style: model.BlockContentText_Numbered, + }, + }, + }, + }, + }, + }, + Id: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a"}}}, e).Times(1) + i.converters = make(map[string]cv.Converter, 0) + i.converters["Notion"] = converter + res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ + Params: &pb.RpcObjectImportRequestParamsOfPbParams{PbParams: &pb.RpcObjectImportRequestPbParams{Path: []string{"test"}}}, + UpdateExistingObjects: false, + Type: 0, + Mode: pb.RpcObjectImportRequest_ALL_OR_NOTHING, + }) + + assert.NotNil(t, res) + assert.True(t, errors.Is(res, cv.ErrNoObjectsToImport)) +} + +func Test_ImportNoObjectToImportErrorIgnoreErrorsMode(t *testing.T) { + i := Import{} + ctrl := gomock.NewController(t) + e := cv.NewFromError("test", cv.ErrNoObjectsToImport) + converter := cv.NewMockConverter(ctrl) + converter.EXPECT().GetSnapshots(gomock.Any(), gomock.Any()).Return(&cv.Response{Snapshots: []*cv.Snapshot{{ + Snapshot: &pb.ChangeSnapshot{ + Data: &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{&model.Block{ + Id: "1", + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: "test", + Style: model.BlockContentText_Numbered, + }, + }, + }, + }, + }, + }, + Id: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a"}}}, e).Times(1) + i.converters = make(map[string]cv.Converter, 0) + i.converters["Notion"] = converter + creator := NewMockCreator(ctrl) + //nolint:lll + creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, "", nil).Times(1) + i.oc = creator + idGetter := NewMockIDGetter(ctrl) + idGetter.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("id", treestorage.TreeStorageCreatePayload{}, nil).Times(1) + i.objectIDGetter = idGetter + res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ + Params: &pb.RpcObjectImportRequestParamsOfPbParams{PbParams: &pb.RpcObjectImportRequestPbParams{Path: []string{"test"}}}, + UpdateExistingObjects: false, + Type: 0, + Mode: pb.RpcObjectImportRequest_IGNORE_ERRORS, + }) + + assert.NotNil(t, res) + assert.True(t, errors.Is(res, cv.ErrNoObjectsToImport)) +} diff --git a/core/block/import/markdown/anymark/anyblocks.go b/core/block/import/markdown/anymark/anyblocks.go index ca826fa01f..d9e8614b70 100644 --- a/core/block/import/markdown/anymark/anyblocks.go +++ b/core/block/import/markdown/anymark/anyblocks.go @@ -10,48 +10,45 @@ import ( ) func preprocessBlocks(blocks []*model.Block) (blocksOut []*model.Block) { - - blocksOut = []*model.Block{} - accum := []*model.Block{} - + accum := make([]*model.Block, 0) for _, b := range blocks { if t := b.GetText(); t != nil && t.Style == model.BlockContentText_Code { accum = append(accum, b) } else { if len(accum) > 0 { - blocksOut = append(blocksOut, combineCodeBlocks(accum)...) + result := combineCodeBlocks(accum) + blocksOut = append(blocksOut, result...) accum = []*model.Block{} } - blocksOut = append(blocksOut, b) } } - if len(accum) > 0 { - blocksOut = append(blocksOut, combineCodeBlocks(accum)...) + result := combineCodeBlocks(accum) + blocksOut = append(blocksOut, result...) } - for _, b := range blocks { for i, cId := range b.ChildrenIds { - if len(cId) == 0 { + if cId == "" { b.ChildrenIds = append(b.ChildrenIds[:i], b.ChildrenIds[i+1:]...) } } } - return blocksOut } -func combineCodeBlocks(accum []*model.Block) (res []*model.Block) { +func combineCodeBlocks(accum []*model.Block) []*model.Block { var ( - textArr []string - currLanguage string - resultCodeBlock []*model.Block + textArr []string + currLanguage string + resultCodeBlocks []*model.Block + currBlockID string ) if len(accum) > 0 { currLanguage = pbtypes.GetString(accum[0].GetFields(), "lang") + currBlockID = accum[0].Id } for _, b := range accum { blockLanguage := pbtypes.GetString(b.GetFields(), "lang") @@ -60,23 +57,27 @@ func combineCodeBlocks(accum []*model.Block) (res []*model.Block) { continue } if blockLanguage != currLanguage { - resultCodeBlock = append(resultCodeBlock, provideCodeBlock(textArr, currLanguage)) + codeBlock := provideCodeBlock(textArr, currLanguage, currBlockID) + resultCodeBlocks = append(resultCodeBlocks, codeBlock) textArr = []string{b.GetText().Text} currLanguage = blockLanguage + currBlockID = b.Id } } if len(textArr) > 0 { - resultCodeBlock = append(resultCodeBlock, provideCodeBlock(textArr, currLanguage)) + codeBlock := provideCodeBlock(textArr, currLanguage, currBlockID) + resultCodeBlocks = append(resultCodeBlocks, codeBlock) } - return resultCodeBlock + return resultCodeBlocks } -func provideCodeBlock(textArr []string, language string) *model.Block { +func provideCodeBlock(textArr []string, language string, id string) *model.Block { var field *types.Struct if language != "" { field = &types.Struct{Fields: map[string]*types.Value{"lang": pbtypes.String(language)}} } return &model.Block{ + Id: id, Fields: field, Content: &model.BlockContentOfText{ Text: &model.BlockContentText{ diff --git a/core/block/import/markdown/anymark/blocks_renderer.go b/core/block/import/markdown/anymark/blocks_renderer.go index 9e500fab05..3f3bc74660 100644 --- a/core/block/import/markdown/anymark/blocks_renderer.go +++ b/core/block/import/markdown/anymark/blocks_renderer.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" + "github.com/globalsign/mgo/bson" "github.com/gogo/protobuf/types" "github.com/google/uuid" @@ -203,8 +204,8 @@ func (r *blocksRenderer) AddImageBlock(source string) { } newBlock := model.Block{ + Id: bson.NewObjectId().Hex(), Content: &model.BlockContentOfFile{ - File: &model.BlockContentFile{ Name: sourceUnescaped, State: model.BlockContentFile_Empty, @@ -213,6 +214,7 @@ func (r *blocksRenderer) AddImageBlock(source string) { } r.blocks = append(r.blocks, &newBlock) + r.addChildIDToParentBlock(newBlock.Id) } func (r *blocksRenderer) AddDivider() { @@ -220,20 +222,24 @@ func (r *blocksRenderer) AddDivider() { r.marksBuffer = []*model.BlockContentTextMark{} r.textBuffer = "" - r.blocks = append(r.blocks, &model.Block{ + divider := &model.Block{ + Id: bson.NewObjectId().Hex(), Content: &model.BlockContentOfDiv{ Div: &model.BlockContentDiv{ Style: model.BlockContentDiv_Line, }, }, - }) + } + r.blocks = append(r.blocks, divider) + r.addChildIDToParentBlock(divider.Id) } func isBlockCanHaveChild(block model.Block) bool { if t := block.GetText(); t != nil { return t.Style == model.BlockContentText_Numbered || t.Style == model.BlockContentText_Marked || - t.Style == model.BlockContentText_Toggle + t.Style == model.BlockContentText_Toggle || + t.Style == model.BlockContentText_Quote } return false @@ -371,6 +377,21 @@ func (r *blocksRenderer) adjustMarkdownRange(t *model.BlockContentText, adjustNu } } +func (r *blocksRenderer) addChildIDToParentBlock(id string) { + if len(r.openedTextBlocks) > 0 { + var parentBlock *textBlock + for i := len(r.openedTextBlocks) - 1; i >= 0; i-- { + if isBlockCanHaveChild(r.openedTextBlocks[i].Block) { + parentBlock = r.openedTextBlocks[i] + break + } + } + if parentBlock != nil { + parentBlock.ChildrenIds = append(parentBlock.ChildrenIds, id) + } + } +} + func (r *blocksRenderer) ForceCloseTextBlock() { s := r.openedTextBlocks style := model.BlockContentText_Paragraph @@ -390,7 +411,3 @@ func (r *blocksRenderer) ProcessMarkdownArtifacts() { } } } - -func (r *blocksRenderer) AddQuote() { - r.blocks[len(r.blocks)-1].GetText().Style = model.BlockContentText_Quote -} diff --git a/core/block/import/markdown/anymark/blocks_test.go b/core/block/import/markdown/anymark/blocks_test.go index 12d76a4e13..4c57a02147 100644 --- a/core/block/import/markdown/anymark/blocks_test.go +++ b/core/block/import/markdown/anymark/blocks_test.go @@ -1,9 +1,15 @@ package anymark import ( + "reflect" "testing" + "github.com/globalsign/mgo/bson" + "github.com/gogo/protobuf/types" "github.com/stretchr/testify/assert" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/pbtypes" ) func TestConvertBlocks(t *testing.T) { @@ -17,3 +23,108 @@ func TestConvertBlocks(t *testing.T) { assert.NotEmpty(t, blocks) assert.NoError(t, err) } + +func TestPreprocessBlocksEmpty(t *testing.T) { + blocks := preprocessBlocks([]*model.Block{}) + assert.Empty(t, blocks) +} + +func TestPreprocessBlocksOneCodeBlock(t *testing.T) { + bl := &model.Block{ + Id: bson.NewObjectId().Hex(), + Fields: &types.Struct{Fields: map[string]*types.Value{ + "lang": pbtypes.String("Java"), + }}, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: "code", + Style: model.BlockContentText_Code, + }, + }, + } + blocks := preprocessBlocks([]*model.Block{bl}) + assert.Len(t, blocks, 1) + assert.Equal(t, blocks[0].Id, bl.Id) +} + +func TestPreprocessBlocksTwoDifferentCodeBlocks(t *testing.T) { + bl := &model.Block{ + Id: bson.NewObjectId().Hex(), + Fields: &types.Struct{Fields: map[string]*types.Value{ + "lang": pbtypes.String("java"), + }}, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: "code", + Style: model.BlockContentText_Code, + }, + }, + } + bl2 := &model.Block{ + Id: bson.NewObjectId().Hex(), + Fields: &types.Struct{Fields: map[string]*types.Value{ + "lang": pbtypes.String("go"), + }}, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: "code", + Style: model.BlockContentText_Code, + }, + }, + } + blocks := preprocessBlocks([]*model.Block{bl, bl2}) + assert.Len(t, blocks, 2) + assert.Equal(t, blocks[0].Id, bl.Id) + assert.Equal(t, blocks[1].Id, bl2.Id) + assert.True(t, reflect.DeepEqual(blocks[0].Fields, bl.Fields)) + assert.True(t, reflect.DeepEqual(blocks[1].Fields, bl2.Fields)) +} + +func TestPreprocessBlocksThreeCodeBlock(t *testing.T) { + bl := &model.Block{ + Id: bson.NewObjectId().Hex(), + Fields: &types.Struct{Fields: map[string]*types.Value{ + "lang": pbtypes.String("java"), + }}, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: "code", + Style: model.BlockContentText_Code, + }, + }, + } + bl2 := &model.Block{ + Id: bson.NewObjectId().Hex(), + Fields: &types.Struct{Fields: map[string]*types.Value{ + "lang": pbtypes.String("go"), + }}, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: "code", + Style: model.BlockContentText_Code, + }, + }, + } + bl3 := &model.Block{ + Id: bson.NewObjectId().Hex(), + Fields: &types.Struct{Fields: map[string]*types.Value{ + "lang": pbtypes.String("go"), + }}, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: "code1", + Style: model.BlockContentText_Code, + }, + }, + } + blocks := preprocessBlocks([]*model.Block{bl, bl2, bl3}) + assert.Len(t, blocks, 2) + + assert.Equal(t, blocks[0].Id, bl.Id) + assert.Equal(t, blocks[1].Id, bl2.Id) // second block is a part of first block now, because they have the same language + assert.Equal(t, blocks[0].Fields.Fields["lang"], pbtypes.String("java")) + assert.Equal(t, blocks[1].Fields.Fields["lang"], pbtypes.String("go")) + + assert.Equal(t, blocks[0].GetText().GetText(), bl.GetText().GetText()) + assert.Equal(t, blocks[1].GetText().GetText(), bl2.GetText().GetText()+"\n"+bl3.GetText().GetText()) +} diff --git a/core/block/import/markdown/anymark/renderer.go b/core/block/import/markdown/anymark/renderer.go index e1a31b76ac..197ff6a529 100644 --- a/core/block/import/markdown/anymark/renderer.go +++ b/core/block/import/markdown/anymark/renderer.go @@ -112,8 +112,10 @@ func (r *Renderer) renderBlockquote(_ util.BufWriter, node ast.Node, entering bool) (ast.WalkStatus, error) { - if !entering { - r.AddQuote() + if entering { + r.OpenNewTextBlock(model.BlockContentText_Quote, nil) + } else { + r.CloseTextBlock(model.BlockContentText_Quote) } return ast.WalkContinue, nil } diff --git a/core/block/import/markdown/anymark/table_renderer.go b/core/block/import/markdown/anymark/table_renderer.go index db52d5ee9b..67fd3c047f 100644 --- a/core/block/import/markdown/anymark/table_renderer.go +++ b/core/block/import/markdown/anymark/table_renderer.go @@ -93,6 +93,8 @@ func (r *TableRenderer) renderTable(_ util.BufWriter, _ []byte, node ast.Node, e } blocksToAdd = append(blocksToAdd, block) } + + r.blockRenderer.addChildIDToParentBlock(r.tableState.tableID) r.blockRenderer.blocks = append(r.blockRenderer.blocks, blocksToAdd...) r.blocksState = nil r.tableState.resetState() diff --git a/core/block/import/markdown/blockconverter.go b/core/block/import/markdown/blockconverter.go index 38173bcd61..4028f48624 100644 --- a/core/block/import/markdown/blockconverter.go +++ b/core/block/import/markdown/blockconverter.go @@ -52,7 +52,7 @@ func (m *mdConverter) processFiles(importPath string, mode string, allErrors ce. if s == nil { return nil } - supportedExtensions := []string{".md"} + supportedExtensions := []string{".md", ".csv"} imageFormats := []string{".jpg", ".jpeg", ".png", ".gif", ".webp"} videoFormats := []string{".mp4", ".m4v"} audioFormats := []string{".mp3", ".ogg", ".wav", ".m4a", ".flac"} diff --git a/core/block/import/markdown/import.go b/core/block/import/markdown/import.go index c92c850302..013f068ffb 100644 --- a/core/block/import/markdown/import.go +++ b/core/block/import/markdown/import.go @@ -1,7 +1,6 @@ package markdown import ( - "fmt" "net/url" "path/filepath" "regexp" @@ -79,11 +78,9 @@ func (m *Markdown) GetSnapshots(req *pb.RpcObjectImportRequest, progress process } allSnapshots = append(allSnapshots, snapshots...) } - - if len(allSnapshots) == 0 { - allErrors.Add("", fmt.Errorf("failed to get snaphots from path, no md files")) + if allErrors.IsNoObjectToImportError(len(paths)) { + return nil, allErrors } - allSnapshots, err := m.createRootCollection(allSnapshots) if err != nil { allErrors.Add(rootCollectionName, err) @@ -101,7 +98,7 @@ func (m *Markdown) GetSnapshots(req *pb.RpcObjectImportRequest, progress process func (m *Markdown) createRootCollection(allSnapshots []*converter.Snapshot) ([]*converter.Snapshot, error) { targetObjects := m.getObjectIDs(allSnapshots) rootCollection := converter.NewRootCollection(m.service) - rootCol, err := rootCollection.AddObjects(rootCollectionName, targetObjects) + rootCol, err := rootCollection.MakeRootCollection(rootCollectionName, targetObjects) if err != nil { return nil, err } @@ -114,7 +111,7 @@ func (m *Markdown) createRootCollection(allSnapshots []*converter.Snapshot) ([]* func (m *Markdown) getSnapshots(req *pb.RpcObjectImportRequest, progress process.Progress, path string, allErrors converter.ConvertError) ([]*converter.Snapshot, converter.ConvertError) { files, e := m.blockConverter.markdownToBlocks(path, req.GetMode().String()) - if !e.IsEmpty() && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { + if !e.IsEmpty() { if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { return nil, e } @@ -122,7 +119,6 @@ func (m *Markdown) getSnapshots(req *pb.RpcObjectImportRequest, progress process } if len(files) == 0 { - allErrors.Add(path, converter.ErrNoObjectsToImport) return nil, nil } progress.SetTotal(int64(numberOfStages * len(files))) diff --git a/core/block/import/mock.go b/core/block/import/mock.go index dced96008d..4a982eaa6b 100644 --- a/core/block/import/mock.go +++ b/core/block/import/mock.go @@ -79,9 +79,9 @@ func (m *MockIDGetter) EXPECT() *MockIDGetterMockRecorder { } // Get mocks base method. -func (m *MockIDGetter) Get(arg0 *session.Context, arg1 *converter.Snapshot, arg2 smartblock.SmartBlockType, arg3 time.Time, arg4 bool) (string, treestorage.TreeStorageCreatePayload, error) { +func (m *MockIDGetter) Get(arg0 *session.Context, arg1 *converter.Snapshot, arg2 smartblock.SmartBlockType, arg3 time.Time, arg4 bool, arg5 map[string]string) (string, treestorage.TreeStorageCreatePayload, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3, arg4, arg5) ret0, _ := ret[0].(string) ret1, _ := ret[1].(treestorage.TreeStorageCreatePayload) ret2, _ := ret[2].(error) @@ -89,7 +89,7 @@ func (m *MockIDGetter) Get(arg0 *session.Context, arg1 *converter.Snapshot, arg2 } // Get indicates an expected call of Get. -func (mr *MockIDGetterMockRecorder) Get(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { +func (mr *MockIDGetterMockRecorder) Get(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockIDGetter)(nil).Get), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockIDGetter)(nil).Get), arg0, arg1, arg2, arg3, arg4, arg5) } diff --git a/core/block/import/notion/api/block/block.go b/core/block/import/notion/api/block/block.go index 3cdd6b1a9e..f10d49d52c 100644 --- a/core/block/import/notion/api/block/block.go +++ b/core/block/import/notion/api/block/block.go @@ -68,14 +68,14 @@ type ChildSetter interface { } type Getter interface { - GetBlocks(req *MapRequest) *MapResponse + GetBlocks(req *NotionImportContext, pageID string) *MapResponse } const unsupportedBlockMessage = "Unsupported block" type UnsupportedBlock struct{} -func (*UnsupportedBlock) GetBlocks(req *MapRequest) *MapResponse { +func (*UnsupportedBlock) GetBlocks(*NotionImportContext, string) *MapResponse { id := bson.NewObjectId().Hex() bl := &model.Block{ Id: id, diff --git a/core/block/import/notion/api/block/columns.go b/core/block/import/notion/api/block/columns.go index 0d9f51add6..c05a9dedae 100644 --- a/core/block/import/notion/api/block/columns.go +++ b/core/block/import/notion/api/block/columns.go @@ -27,7 +27,7 @@ func (c *ColumnListBlock) SetChildren(children []interface{}) { c.ColumnList = children } -func (c *ColumnListBlock) GetBlocks(req *MapRequest) *MapResponse { +func (c *ColumnListBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { columnsList := c.ColumnList.([]interface{}) var ( resultResponse = &MapResponse{} @@ -43,20 +43,20 @@ func (c *ColumnListBlock) GetBlocks(req *MapRequest) *MapResponse { return resultResponse } -func (c *ColumnListBlock) handleColumn(req *MapRequest, notionColumn interface{}, resultResponse *MapResponse, rowBlock simple.Block) { +func (c *ColumnListBlock) handleColumn(req *NotionImportContext, notionColumn interface{}, resultResponse *MapResponse, rowBlock simple.Block) { column := c.addColumnBlocks("ct-", req, notionColumn, resultResponse) rowBlock.Model().ChildrenIds = append(rowBlock.Model().ChildrenIds, column.Model().Id) } -func (c *ColumnListBlock) handleFirstColumn(req *MapRequest, notionColumn interface{}, resultResponse *MapResponse) simple.Block { +func (c *ColumnListBlock) handleFirstColumn(req *NotionImportContext, notionColumn interface{}, resultResponse *MapResponse) simple.Block { column := c.addColumnBlocks("cd-", req, notionColumn, resultResponse) rowBlock := c.getRowBlock(strings.TrimPrefix(column.Model().Id, "cd-"), column.Model().Id) return rowBlock } -func (c *ColumnListBlock) addColumnBlocks(prefix string, req *MapRequest, notionColumn interface{}, resultResponse *MapResponse) simple.Block { +func (c *ColumnListBlock) addColumnBlocks(prefix string, req *NotionImportContext, notionColumn interface{}, resultResponse *MapResponse) simple.Block { req.Blocks = []interface{}{notionColumn} - resp := MapBlocks(req) + resp := MapBlocks(req, "") childBlocks := c.getChildBlocksForColumn(resp) id := bson.NewObjectId().Hex() column := c.getColumnBlock(id, prefix, childBlocks, resultResponse) @@ -126,9 +126,9 @@ type ColumnObject struct { Children []interface{} `json:"children"` } -func (c *ColumnBlock) GetBlocks(req *MapRequest) *MapResponse { +func (c *ColumnBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { req.Blocks = c.Column.Children - resp := MapBlocks(req) + resp := MapBlocks(req, "") return resp } diff --git a/core/block/import/notion/api/block/div.go b/core/block/import/notion/api/block/div.go index 909cd762cb..12bef3caba 100644 --- a/core/block/import/notion/api/block/div.go +++ b/core/block/import/notion/api/block/div.go @@ -11,7 +11,7 @@ type DividerBlock struct { Divider struct{} `json:"divider"` } -func (*DividerBlock) GetBlocks(*MapRequest) *MapResponse { +func (*DividerBlock) GetBlocks(*NotionImportContext, string) *MapResponse { id := bson.NewObjectId().Hex() block := &model.Block{ Id: id, diff --git a/core/block/import/notion/api/block/file.go b/core/block/import/notion/api/block/file.go index 5073e63493..b50a3ed880 100644 --- a/core/block/import/notion/api/block/file.go +++ b/core/block/import/notion/api/block/file.go @@ -11,7 +11,7 @@ type FileBlock struct { Caption []api.RichText `json:"caption"` } -func (f *FileBlock) GetBlocks(*MapRequest) *MapResponse { +func (f *FileBlock) GetBlocks(*NotionImportContext, string) *MapResponse { block, id := f.File.GetFileBlock(model.BlockContentFile_File) return &MapResponse{ Blocks: []*model.Block{block}, @@ -24,7 +24,7 @@ type ImageBlock struct { File api.FileObject `json:"image"` } -func (i *ImageBlock) GetBlocks(*MapRequest) *MapResponse { +func (i *ImageBlock) GetBlocks(*NotionImportContext, string) *MapResponse { block, id := i.File.GetFileBlock(model.BlockContentFile_Image) return &MapResponse{ Blocks: []*model.Block{block}, @@ -37,7 +37,7 @@ type PdfBlock struct { File api.FileObject `json:"pdf"` } -func (p *PdfBlock) GetBlocks(*MapRequest) *MapResponse { +func (p *PdfBlock) GetBlocks(*NotionImportContext, string) *MapResponse { block, id := p.File.GetFileBlock(model.BlockContentFile_PDF) return &MapResponse{ Blocks: []*model.Block{block}, @@ -50,7 +50,7 @@ type VideoBlock struct { File api.FileObject `json:"video"` } -func (p *VideoBlock) GetBlocks(*MapRequest) *MapResponse { +func (p *VideoBlock) GetBlocks(*NotionImportContext, string) *MapResponse { block, id := p.File.GetFileBlock(model.BlockContentFile_Video) return &MapResponse{ Blocks: []*model.Block{block}, @@ -63,7 +63,7 @@ type AudioBlock struct { File api.FileObject `json:"audio"` } -func (p *AudioBlock) GetBlocks(*MapRequest) *MapResponse { +func (p *AudioBlock) GetBlocks(*NotionImportContext, string) *MapResponse { block, id := p.File.GetFileBlock(model.BlockContentFile_Audio) return &MapResponse{ Blocks: []*model.Block{block}, diff --git a/core/block/import/notion/api/block/link.go b/core/block/import/notion/api/block/link.go index 1bcb34990a..ea129af95c 100644 --- a/core/block/import/notion/api/block/link.go +++ b/core/block/import/notion/api/block/link.go @@ -18,8 +18,8 @@ type EmbedBlock struct { Embed LinkToWeb `json:"embed"` } -func (b *EmbedBlock) GetBlocks(req *MapRequest) *MapResponse { - return b.Embed.GetBlocks(req) +func (b *EmbedBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { + return b.Embed.GetBlocks(req, "") } type LinkToWeb struct { @@ -31,11 +31,11 @@ type LinkPreviewBlock struct { LinkPreview LinkToWeb `json:"link_preview"` } -func (b *LinkPreviewBlock) GetBlocks(req *MapRequest) *MapResponse { - return b.LinkPreview.GetBlocks(req) +func (b *LinkPreviewBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { + return b.LinkPreview.GetBlocks(req, "") } -func (b *LinkToWeb) GetBlocks(*MapRequest) *MapResponse { +func (b *LinkToWeb) GetBlocks(*NotionImportContext, string) *MapResponse { id := bson.NewObjectId().Hex() to := textUtil.UTF16RuneCountString(b.URL) @@ -76,25 +76,16 @@ type ChildPage struct { Title string `json:"title"` } -func (b *ChildPageBlock) GetBlocks(req *MapRequest) *MapResponse { - bl := b.ChildPage.GetLinkToObjectBlock(req.NotionPageIdsToAnytype, req.PageNameToID) +func (b *ChildPageBlock) GetBlocks(req *NotionImportContext, pageID string) *MapResponse { + bl := b.ChildPage.GetLinkToObjectBlock(req, pageID) return &MapResponse{ Blocks: []*model.Block{bl}, BlockIDs: []string{bl.Id}, } } -func (p ChildPage) GetLinkToObjectBlock(notionIdsToAnytype, idToName map[string]string) *model.Block { - var ( - targetBlockID string - ok bool - ) - for id, name := range idToName { - if strings.EqualFold(name, p.Title) { - targetBlockID, ok = notionIdsToAnytype[id] - break - } - } +func (p ChildPage) GetLinkToObjectBlock(importContext *NotionImportContext, pageID string) *model.Block { + targetBlockID, ok := getTargetBlock(importContext.PageNameToID, importContext.ChildIDToPage, importContext.NotionPageIdsToAnytype, pageID, p.Title) id := bson.NewObjectId().Hex() if !ok { @@ -127,8 +118,8 @@ type ChildDatabaseBlock struct { ChildDatabase ChildDatabase `json:"child_database"` } -func (b *ChildDatabaseBlock) GetBlocks(req *MapRequest) *MapResponse { - bl := b.ChildDatabase.GetDataviewBlock(req.NotionDatabaseIdsToAnytype, req.DatabaseNameToID) +func (b *ChildDatabaseBlock) GetBlocks(req *NotionImportContext, pageID string) *MapResponse { + bl := b.ChildDatabase.GetDataviewBlock(req, pageID) return &MapResponse{ Blocks: []*model.Block{bl}, BlockIDs: []string{bl.Id}, @@ -139,23 +130,14 @@ type ChildDatabase struct { Title string `json:"title"` } -func (c *ChildDatabase) GetDataviewBlock(notionIdsToAnytype, idToName map[string]string) *model.Block { - var ( - targetBlockID string - ) - for id, name := range idToName { - if strings.EqualFold(name, c.Title) { - if len(notionIdsToAnytype) > 0 { - targetBlockID = notionIdsToAnytype[id] - } - break - } - } +func (c *ChildDatabase) GetDataviewBlock(importContext *NotionImportContext, pageID string) *model.Block { + targetBlockID, _ := getTargetBlock(importContext.DatabaseNameToID, + importContext.ChildIDToPage, + importContext.NotionDatabaseIdsToAnytype, + pageID, c.Title) id := bson.NewObjectId().Hex() - block := template.MakeCollectionDataviewContent() - block.Dataview.TargetObjectId = targetBlockID return &model.Block{ @@ -170,7 +152,7 @@ type LinkToPageBlock struct { LinkToPage api.Parent `json:"link_to_page"` } -func (l *LinkToPageBlock) GetBlocks(req *MapRequest) *MapResponse { +func (l *LinkToPageBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { var anytypeID string if l.LinkToPage.PageID != "" { anytypeID = req.NotionPageIdsToAnytype[l.LinkToPage.PageID] @@ -198,7 +180,7 @@ type BookmarkBlock struct { Bookmark BookmarkObject `json:"bookmark"` } -func (b *BookmarkBlock) GetBlocks(*MapRequest) *MapResponse { +func (b *BookmarkBlock) GetBlocks(*NotionImportContext, string) *MapResponse { bl, id := b.Bookmark.GetBookmarkBlock() return &MapResponse{ Blocks: []*model.Block{bl}, @@ -225,3 +207,26 @@ func (b BookmarkObject) GetBookmarkBlock() (*model.Block, string) { }, }}, id } + +func getTargetBlock(nameToID, childIDToParent, notionIDsToAnytype map[string]string, pageID, title string) (string, bool) { + var ( + targetBlockID string + ok bool + idsWithGivenName []string + ) + for id, name := range nameToID { + if strings.EqualFold(name, title) { + idsWithGivenName = append(idsWithGivenName, id) + } + } + + for _, id := range idsWithGivenName { + if parentID, exist := childIDToParent[id]; exist { + if parentID == pageID { + targetBlockID, ok = notionIDsToAnytype[id] + break + } + } + } + return targetBlockID, ok +} diff --git a/core/block/import/notion/api/block/link_test.go b/core/block/import/notion/api/block/link_test.go index f988e68107..c2f215c1f2 100644 --- a/core/block/import/notion/api/block/link_test.go +++ b/core/block/import/notion/api/block/link_test.go @@ -43,9 +43,24 @@ func Test_GetBookmarkBlock(t *testing.T) { func Test_GetLinkToObjectBlockSuccess(t *testing.T) { c := &ChildPage{Title: "title"} - nameToID := map[string]string{"id": "title"} - notionIdsToAnytype := map[string]string{"id": "anytypeId"} - bl := c.GetLinkToObjectBlock(notionIdsToAnytype, nameToID) + importContext := NewNotionImportContext() + importContext.PageNameToID = map[string]string{"id": "title"} + importContext.NotionPageIdsToAnytype = map[string]string{"id": "anytypeId"} + importContext.ChildIDToPage = map[string]string{"id": "parentID"} + bl := c.GetLinkToObjectBlock(importContext, "parentID") + assert.NotNil(t, bl) + content, ok := bl.Content.(*model.BlockContentOfLink) + assert.True(t, ok) + assert.Equal(t, content.Link.TargetBlockId, "anytypeId") +} + +func Test_GetLinkToObjectBlockTwoPagesWithSameName(t *testing.T) { + c := &ChildPage{Title: "title"} + importContext := NewNotionImportContext() + importContext.PageNameToID = map[string]string{"id": "title", "id1": "title"} + importContext.NotionPageIdsToAnytype = map[string]string{"id": "anytypeId", "id1": "anytypeId1"} + importContext.ChildIDToPage = map[string]string{"id": "parentID"} + bl := c.GetLinkToObjectBlock(importContext, "parentID") assert.NotNil(t, bl) content, ok := bl.Content.(*model.BlockContentOfLink) assert.True(t, ok) @@ -54,14 +69,15 @@ func Test_GetLinkToObjectBlockSuccess(t *testing.T) { func Test_GetLinkToObjectBlockFail(t *testing.T) { c := &ChildPage{Title: "title"} - bl := c.GetLinkToObjectBlock(nil, nil) + bl := c.GetLinkToObjectBlock(NewNotionImportContext(), "id") assert.NotNil(t, bl) content, ok := bl.Content.(*model.BlockContentOfText) assert.True(t, ok) assert.Equal(t, content.Text.Text, notFoundPageMessage) - nameToID := map[string]string{"id": "title"} - bl = c.GetLinkToObjectBlock(nameToID, nil) + importContext := NewNotionImportContext() + importContext.PageNameToID = map[string]string{"id": "title"} + bl = c.GetLinkToObjectBlock(importContext, "pageID") assert.NotNil(t, bl) content, ok = bl.Content.(*model.BlockContentOfText) assert.True(t, ok) @@ -70,9 +86,11 @@ func Test_GetLinkToObjectBlockFail(t *testing.T) { func Test_GetLinkToObjectBlockInlineCollection(t *testing.T) { c := &ChildDatabase{Title: "title"} - nameToID := map[string]string{"id": "title"} - notionIdsToAnytype := map[string]string{"id": "anytypeId"} - bl := c.GetDataviewBlock(notionIdsToAnytype, nameToID) + importContext := NewNotionImportContext() + importContext.DatabaseNameToID = map[string]string{"id": "title"} + importContext.NotionDatabaseIdsToAnytype = map[string]string{"id": "anytypeId"} + importContext.ChildIDToPage = map[string]string{"id": "parentID"} + bl := c.GetDataviewBlock(importContext, "parentID") assert.NotNil(t, bl) content, ok := bl.Content.(*model.BlockContentOfDataview) assert.True(t, ok) @@ -81,7 +99,7 @@ func Test_GetLinkToObjectBlockInlineCollection(t *testing.T) { func Test_GetLinkToObjectBlockInlineCollectionEmpty(t *testing.T) { c := &ChildDatabase{Title: "title"} - bl := c.GetDataviewBlock(nil, nil) + bl := c.GetDataviewBlock(NewNotionImportContext(), "") assert.NotNil(t, bl) content, ok := bl.Content.(*model.BlockContentOfDataview) assert.True(t, ok) diff --git a/core/block/import/notion/api/block/mapper.go b/core/block/import/notion/api/block/mapper.go index c3a9811395..949f73c503 100644 --- a/core/block/import/notion/api/block/mapper.go +++ b/core/block/import/notion/api/block/mapper.go @@ -4,8 +4,8 @@ import ( "github.com/anyproto/anytype-heart/pkg/lib/pb/model" ) -// MapRequest is a data object with all necessary structures for blocks -type MapRequest struct { +// NotionImportContext is a data object with all necessary structures for blocks +type NotionImportContext struct { Blocks []interface{} // Need all these maps for correct mapping of pages and databases from notion to anytype // for such blocks as mentions or links to objects @@ -15,27 +15,40 @@ type MapRequest struct { DatabaseNameToID map[string]string RelationsIdsToAnytypeID map[string]*model.SmartBlockSnapshotBase RelationsIdsToOptions map[string][]*model.SmartBlockSnapshotBase + ChildIDToPage map[string]string } -func (m *MapRequest) ReadRelationsMap(key string) *model.SmartBlockSnapshotBase { +func NewNotionImportContext() *NotionImportContext { + return &NotionImportContext{ + NotionPageIdsToAnytype: make(map[string]string, 0), + NotionDatabaseIdsToAnytype: make(map[string]string, 0), + PageNameToID: make(map[string]string, 0), + DatabaseNameToID: make(map[string]string, 0), + RelationsIdsToAnytypeID: make(map[string]*model.SmartBlockSnapshotBase, 0), + RelationsIdsToOptions: make(map[string][]*model.SmartBlockSnapshotBase, 0), + ChildIDToPage: make(map[string]string, 0), + } +} + +func (m *NotionImportContext) ReadRelationsMap(key string) *model.SmartBlockSnapshotBase { if snapshot, ok := m.RelationsIdsToAnytypeID[key]; ok { return snapshot } return nil } -func (m *MapRequest) WriteToRelationsMap(key string, relation *model.SmartBlockSnapshotBase) { +func (m *NotionImportContext) WriteToRelationsMap(key string, relation *model.SmartBlockSnapshotBase) { m.RelationsIdsToAnytypeID[key] = relation } -func (m *MapRequest) ReadRelationsOptionsMap(key string) []*model.SmartBlockSnapshotBase { +func (m *NotionImportContext) ReadRelationsOptionsMap(key string) []*model.SmartBlockSnapshotBase { if snapshot, ok := m.RelationsIdsToOptions[key]; ok { return snapshot } return nil } -func (m *MapRequest) WriteToRelationsOptionsMap(key string, relationOptions []*model.SmartBlockSnapshotBase) { +func (m *NotionImportContext) WriteToRelationsOptionsMap(key string, relationOptions []*model.SmartBlockSnapshotBase) { m.RelationsIdsToOptions[key] = append(m.RelationsIdsToOptions[key], relationOptions...) } @@ -51,11 +64,11 @@ func (m *MapResponse) Merge(mergedResp *MapResponse) { } } -func MapBlocks(req *MapRequest) *MapResponse { +func MapBlocks(req *NotionImportContext, pageID string) *MapResponse { resp := &MapResponse{} for _, bl := range req.Blocks { if ba, ok := bl.(Getter); ok { - textResp := ba.GetBlocks(req) + textResp := ba.GetBlocks(req, pageID) resp.Merge(textResp) continue } diff --git a/core/block/import/notion/api/block/retrieve.go b/core/block/import/notion/api/block/retrieve.go index 4cdf8e9725..34b1116c89 100644 --- a/core/block/import/notion/api/block/retrieve.go +++ b/core/block/import/notion/api/block/retrieve.go @@ -78,8 +78,8 @@ func (s *Service) GetBlocksAndChildren(ctx context.Context, return allBlocks, nil } -func (s *Service) MapNotionBlocksToAnytype(req *MapRequest) *MapResponse { - return MapBlocks(req) +func (s *Service) MapNotionBlocksToAnytype(req *NotionImportContext, pageID string) *MapResponse { + return MapBlocks(req, pageID) } func (s *Service) getBlocks(ctx context.Context, pageID, apiKey string, pagination int64) ([]interface{}, error) { diff --git a/core/block/import/notion/api/block/table.go b/core/block/import/notion/api/block/table.go index f27f3efe6a..a3f1a30f75 100644 --- a/core/block/import/notion/api/block/table.go +++ b/core/block/import/notion/api/block/table.go @@ -47,11 +47,11 @@ func (t *TableBlock) SetChildren(children []interface{}) { } } -func (t *TableBlock) GetBlocks(req *MapRequest) *MapResponse { +func (t *TableBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { columnsBlocks, columnsBlocksIDs, columnLayoutBlockID, columnLayoutBlock := t.getColumns() tableResponse := &MapResponse{} - tableRowBlocks, tableRowBlocksIDs, rowTextBlocks := t.getRows(req, columnsBlocksIDs, tableResponse) + tableRowBlocks, tableRowBlocksIDs, rowTextBlocks := t.getRows(req, columnsBlocksIDs) tableRowBlockID, tableRowLayoutBlock := t.getLayoutRowBlock(tableRowBlocksIDs) @@ -106,9 +106,8 @@ func (*TableBlock) getLayoutRowBlock(children []string) (string, *model.Block) { return tableRowBlockID, tableRowLayoutBlock } -func (t *TableBlock) getRows(req *MapRequest, - columnsBlocksIDs []string, - tableResponse *MapResponse) ([]*model.Block, []string, []*model.Block) { +func (t *TableBlock) getRows(req *NotionImportContext, + columnsBlocksIDs []string) ([]*model.Block, []string, []*model.Block) { var ( needHeader = true tableRowBlocks = make([]*model.Block, 0, len(t.Table.Children)) diff --git a/core/block/import/notion/api/block/table_test.go b/core/block/import/notion/api/block/table_test.go index 93058f1826..170416020b 100644 --- a/core/block/import/notion/api/block/table_test.go +++ b/core/block/import/notion/api/block/table_test.go @@ -23,7 +23,7 @@ func Test_TableWithOneColumnAndRow(t *testing.T) { }, } - resp := tb.GetBlocks(&MapRequest{}) + resp := tb.GetBlocks(&NotionImportContext{}, "") assert.NotNil(t, resp) assert.Len(t, resp.Blocks, 5) // table block + column block + row block + 2 empty text blocks @@ -49,7 +49,7 @@ func Test_TableWithoutContent(t *testing.T) { assert.Len(t, tb.Table.Children, 2) - resp := tb.GetBlocks(&MapRequest{}) + resp := tb.GetBlocks(&NotionImportContext{}, "") assert.NotNil(t, resp) assert.Len(t, resp.Blocks, 8) // table block + 3 * column block + 1 column layout + 1 row layout + 3 * row block @@ -84,7 +84,7 @@ func Test_TableWithDifferentText(t *testing.T) { assert.Len(t, tb.Table.Children, 2) - resp := tb.GetBlocks(&MapRequest{}) + resp := tb.GetBlocks(&NotionImportContext{}, "") assert.NotNil(t, resp) assert.Len(t, resp.Blocks, 9) // table block + 3 * column block + 1 column layout + 1 row layout + 3 * row block + 1 text block diff --git a/core/block/import/notion/api/block/tableofcontent.go b/core/block/import/notion/api/block/tableofcontent.go index 5ee8e02245..2b827d77fb 100644 --- a/core/block/import/notion/api/block/tableofcontent.go +++ b/core/block/import/notion/api/block/tableofcontent.go @@ -18,7 +18,7 @@ type TableOfContentsObject struct { Color string `json:"color"` } -func (t *TableOfContentsBlock) GetBlocks(req *MapRequest) *MapResponse { +func (t *TableOfContentsBlock) GetBlocks(*NotionImportContext, string) *MapResponse { id := bson.NewObjectId().Hex() var color string // Anytype Table Of Content doesn't support different colors of text, only background diff --git a/core/block/import/notion/api/block/text.go b/core/block/import/notion/api/block/text.go index 05653beeb2..1c9a1a64fc 100644 --- a/core/block/import/notion/api/block/text.go +++ b/core/block/import/notion/api/block/text.go @@ -15,7 +15,7 @@ type ParagraphBlock struct { Paragraph TextObjectWithChildren `json:"paragraph"` } -func (p *ParagraphBlock) GetBlocks(req *MapRequest) *MapResponse { +func (p *ParagraphBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { childResp := &MapResponse{} if p.HasChildren { mapper := ChildrenMapper(&p.Paragraph) @@ -56,7 +56,7 @@ func (h *Heading1Block) GetID() string { return h.ID } -func (h *Heading1Block) GetBlocks(req *MapRequest) *MapResponse { +func (h *Heading1Block) GetBlocks(req *NotionImportContext, _ string) *MapResponse { resp := h.Heading1.GetTextBlocks(model.BlockContentText_Header3, nil, req) if h.Heading1.IsToggleable { mapper := ChildrenMapper(&h.Heading1) @@ -83,7 +83,7 @@ func (h *Heading2Block) GetID() string { return h.ID } -func (h *Heading2Block) GetBlocks(req *MapRequest) *MapResponse { +func (h *Heading2Block) GetBlocks(req *NotionImportContext, _ string) *MapResponse { resp := h.Heading2.GetTextBlocks(model.BlockContentText_Header3, nil, req) if h.Heading2.IsToggleable { mapper := ChildrenMapper(&h.Heading2) @@ -98,7 +98,7 @@ type Heading3Block struct { Heading3 HeadingObject `json:"heading_3"` } -func (h *Heading3Block) GetBlocks(req *MapRequest) *MapResponse { +func (h *Heading3Block) GetBlocks(req *NotionImportContext, _ string) *MapResponse { resp := h.Heading3.GetTextBlocks(model.BlockContentText_Header3, nil, req) if h.Heading3.IsToggleable { mapper := ChildrenMapper(&h.Heading3) @@ -135,7 +135,7 @@ type CalloutObject struct { Icon *api.Icon `json:"icon"` } -func (c *CalloutBlock) GetBlocks(req *MapRequest) *MapResponse { +func (c *CalloutBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { childResp := &MapResponse{} if c.HasChild() { mapper := ChildrenMapper(&c.Callout) @@ -191,7 +191,7 @@ type QuoteBlock struct { Quote TextObjectWithChildren `json:"quote"` } -func (q *QuoteBlock) GetBlocks(req *MapRequest) *MapResponse { +func (q *QuoteBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { childResp := &MapResponse{} if q.HasChildren { mapper := ChildrenMapper(&q.Quote) @@ -220,7 +220,7 @@ type NumberedListBlock struct { NumberedList TextObjectWithChildren `json:"numbered_list_item"` } -func (n *NumberedListBlock) GetBlocks(req *MapRequest) *MapResponse { +func (n *NumberedListBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { childResp := &MapResponse{} if n.HasChildren { mapper := ChildrenMapper(&n.NumberedList) @@ -249,7 +249,7 @@ type ToDoBlock struct { ToDo ToDoObject `json:"to_do"` } -func (t *ToDoBlock) GetBlocks(req *MapRequest) *MapResponse { +func (t *ToDoBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { childResp := &MapResponse{} if t.HasChildren { mapper := ChildrenMapper(&t.ToDo) @@ -293,7 +293,7 @@ type BulletedListBlock struct { BulletedList TextObjectWithChildren `json:"bulleted_list_item"` } -func (b *BulletedListBlock) GetBlocks(req *MapRequest) *MapResponse { +func (b *BulletedListBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { childResp := &MapResponse{} if b.HasChildren { mapper := ChildrenMapper(&b.BulletedList) @@ -322,7 +322,7 @@ type ToggleBlock struct { Toggle TextObjectWithChildren `json:"toggle"` } -func (t *ToggleBlock) GetBlocks(req *MapRequest) *MapResponse { +func (t *ToggleBlock) GetBlocks(req *NotionImportContext, _ string) *MapResponse { childResp := &MapResponse{} if t.HasChildren { mapper := ChildrenMapper(&t.Toggle) @@ -375,7 +375,7 @@ type CodeObject struct { Language string `json:"language"` } -func (c *CodeBlock) GetBlocks(req *MapRequest) *MapResponse { +func (c *CodeBlock) GetBlocks(*NotionImportContext, string) *MapResponse { id := bson.NewObjectId().Hex() bl := &model.Block{ Id: id, @@ -411,7 +411,7 @@ type EquationBlock struct { Equation api.EquationObject `json:"equation"` } -func (e *EquationBlock) GetBlocks(req *MapRequest) *MapResponse { +func (e *EquationBlock) GetBlocks(*NotionImportContext, string) *MapResponse { bl := e.Equation.HandleEquation() return &MapResponse{ Blocks: []*model.Block{bl}, diff --git a/core/block/import/notion/api/block/text_test.go b/core/block/import/notion/api/block/text_test.go index c46cf08617..c91176cd04 100644 --- a/core/block/import/notion/api/block/text_test.go +++ b/core/block/import/notion/api/block/text_test.go @@ -24,7 +24,7 @@ func Test_GetTextBlocksTextSuccess(t *testing.T) { Color: api.RedBackGround, } - bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{}) + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &NotionImportContext{}) assert.Len(t, bl.Blocks, 1) assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph) assert.Equal(t, bl.Blocks[0].BackgroundColor, api.AnytypeRed) @@ -48,7 +48,7 @@ func Test_GetTextBlocksTextUserMention(t *testing.T) { }, } - bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{}) + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &NotionImportContext{}) assert.Len(t, bl.Blocks, 1) assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph) assert.Len(t, bl.Blocks[0].GetText().Marks.Marks, 0) @@ -70,7 +70,7 @@ func Test_GetTextBlocksTextPageMention(t *testing.T) { }, } - bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{ + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &NotionImportContext{ NotionPageIdsToAnytype: map[string]string{"notionID": "anytypeID"}, PageNameToID: map[string]string{"notionID": "Page"}, }) @@ -96,7 +96,7 @@ func Test_GetTextBlocksTextPageMentionNotFound(t *testing.T) { }, } - bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{ + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &NotionImportContext{ NotionPageIdsToAnytype: map[string]string{}, PageNameToID: map[string]string{}, }) @@ -120,7 +120,7 @@ func Test_GetTextBlocksDatabaseMention(t *testing.T) { }, } - bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{ + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &NotionImportContext{ NotionDatabaseIdsToAnytype: map[string]string{}, DatabaseNameToID: map[string]string{}, }) @@ -145,7 +145,7 @@ func Test_GetTextBlocksDatabaseMentionWithoutSource(t *testing.T) { }, } - bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{ + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &NotionImportContext{ NotionDatabaseIdsToAnytype: map[string]string{}, DatabaseNameToID: map[string]string{}, }) @@ -169,7 +169,7 @@ func Test_GetTextBlocksDateMention(t *testing.T) { }, } - bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{}) + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &NotionImportContext{}) assert.Len(t, bl.Blocks, 1) assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph) assert.Len(t, bl.Blocks[0].GetText().Marks.Marks, 1) @@ -192,7 +192,7 @@ func Test_GetTextBlocksLinkPreview(t *testing.T) { }, } - bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{}) + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &NotionImportContext{}) assert.Len(t, bl.Blocks, 1) assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph) assert.NotNil(t, bl.Blocks[0].GetText().Marks) @@ -213,7 +213,7 @@ func Test_GetTextBlocksEquation(t *testing.T) { }, } - bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{}) + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &NotionImportContext{}) assert.Len(t, bl.Blocks, 1) assert.NotNil(t, bl.Blocks[0].GetLatex()) assert.Equal(t, bl.Blocks[0].GetLatex().Text, "Equation") @@ -231,7 +231,7 @@ func Test_GetCodeBlocksSuccess(t *testing.T) { Language: "Go", }, } - bl := co.GetBlocks(&MapRequest{}) + bl := co.GetBlocks(&NotionImportContext{}, "") assert.NotNil(t, bl) assert.Len(t, bl.Blocks, 1) assert.Equal(t, bl.Blocks[0].GetText().Text, "Code") diff --git a/core/block/import/notion/api/block/textobject.go b/core/block/import/notion/api/block/textobject.go index cdae5dc6ea..312855d04a 100644 --- a/core/block/import/notion/api/block/textobject.go +++ b/core/block/import/notion/api/block/textobject.go @@ -22,7 +22,7 @@ const ( ) type ChildrenMapper interface { - MapChildren(req *MapRequest) *MapResponse + MapChildren(req *NotionImportContext) *MapResponse } type TextObject struct { @@ -30,7 +30,7 @@ type TextObject struct { Color string `json:"color"` } -func (t *TextObject) GetTextBlocks(style model.BlockContentTextStyle, childIds []string, req *MapRequest) *MapResponse { +func (t *TextObject) GetTextBlocks(style model.BlockContentTextStyle, childIds []string, req *NotionImportContext) *MapResponse { var marks []*model.BlockContentTextMark id := bson.NewObjectId().Hex() allBlocks := make([]*model.Block, 0) @@ -122,7 +122,7 @@ func (t *TextObject) handleTextType(rt api.RichText, func (t *TextObject) handleMentionType(rt api.RichText, text *strings.Builder, - req *MapRequest) []*model.BlockContentTextMark { + req *NotionImportContext) []*model.BlockContentTextMark { if rt.Mention.Type == api.UserMention { return t.handleUserMention(rt, text) } @@ -261,9 +261,9 @@ type TextObjectWithChildren struct { Children []interface{} `json:"children"` } -func (t *TextObjectWithChildren) MapChildren(req *MapRequest) *MapResponse { +func (t *TextObjectWithChildren) MapChildren(req *NotionImportContext) *MapResponse { childReq := *req childReq.Blocks = t.Children - resp := MapBlocks(&childReq) + resp := MapBlocks(&childReq, "") return resp } diff --git a/core/block/import/notion/api/database/database.go b/core/block/import/notion/api/database/database.go index 58a9e99392..cb4007e732 100644 --- a/core/block/import/notion/api/database/database.go +++ b/core/block/import/notion/api/database/database.go @@ -71,41 +71,39 @@ func (p *Database) GetObjectType() string { func (ds *Service) GetDatabase(ctx context.Context, mode pb.RpcObjectImportRequestMode, databases []Database, - progress process.Progress) (*converter.Response, *block.MapRequest, converter.ConvertError) { + progress process.Progress, + req *block.NotionImportContext) (*converter.Response, converter.ConvertError) { var ( allSnapshots = make([]*converter.Snapshot, 0) notionIdsToAnytype = make(map[string]string, 0) databaseNameToID = make(map[string]string, 0) + childIDToParent = make(map[string]string, 0) convertError = converter.ConvertError{} ) progress.SetProgressMessage("Start creating pages from notion databases") - mapRequest := &block.MapRequest{ - RelationsIdsToAnytypeID: make(map[string]*model.SmartBlockSnapshotBase, 0), - } for _, d := range databases { if err := progress.TryStep(1); err != nil { - return nil, nil, converter.NewCancelError(d.ID, err) + return nil, converter.NewCancelError(d.ID, err) } - snapshot, err := ds.makeDatabaseSnapshot(d, mapRequest, notionIdsToAnytype, databaseNameToID) + snapshot, err := ds.makeDatabaseSnapshot(d, req, notionIdsToAnytype, databaseNameToID, childIDToParent) if err != nil && mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return nil, nil, converter.NewFromError(d.ID, err) + return nil, converter.NewFromError(d.ID, err) } if err != nil { continue } allSnapshots = append(allSnapshots, snapshot...) } - mapRequest.NotionDatabaseIdsToAnytype = notionIdsToAnytype - mapRequest.DatabaseNameToID = databaseNameToID + req.NotionDatabaseIdsToAnytype = notionIdsToAnytype + req.DatabaseNameToID = databaseNameToID + req.ChildIDToPage = childIDToParent if convertError.IsEmpty() { - return &converter.Response{Snapshots: allSnapshots}, mapRequest, nil + return &converter.Response{Snapshots: allSnapshots}, nil } - return &converter.Response{Snapshots: allSnapshots}, mapRequest, convertError + return &converter.Response{Snapshots: allSnapshots}, convertError } -func (ds *Service) makeDatabaseSnapshot(d Database, - request *block.MapRequest, - notionIdsToAnytype, databaseNameToID map[string]string) ([]*converter.Snapshot, error) { +func (ds *Service) makeDatabaseSnapshot(d Database, request *block.NotionImportContext, notionIdsToAnytype, databaseNameToID, childIDToParent map[string]string) ([]*converter.Snapshot, error) { details := ds.getCollectionDetails(d) detailsStruct := &types.Struct{Fields: details} @@ -126,11 +124,17 @@ func (ds *Service) makeDatabaseSnapshot(d Database, id, converterSnapshot := ds.provideSnapshot(d, st, detailsStruct) notionIdsToAnytype[d.ID] = id databaseNameToID[d.ID] = pbtypes.GetString(converterSnapshot.Snapshot.GetData().GetDetails(), bundle.RelationKeyName.String()) + if d.Parent.DatabaseID != "" { + childIDToParent[d.ID] = d.Parent.DatabaseID + } + if d.Parent.PageID != "" { + childIDToParent[d.ID] = d.Parent.PageID + } snapshots = append(snapshots, converterSnapshot) return snapshots, nil } -func (ds *Service) handleNameProperty(d Database, request *block.MapRequest, st *state.State) *converter.Snapshot { +func (ds *Service) handleNameProperty(d Database, request *block.NotionImportContext, st *state.State) *converter.Snapshot { var ( snapshot *converter.Snapshot key = "Name" @@ -152,7 +156,7 @@ func (ds *Service) handleNameProperty(d Database, request *block.MapRequest, st return snapshot } -func (ds *Service) createRelationFromDatabaseProperty(req *block.MapRequest, +func (ds *Service) createRelationFromDatabaseProperty(req *block.NotionImportContext, databaseProperty property.DatabasePropertyHandler, key string, st *state.State) *converter.Snapshot { @@ -297,7 +301,7 @@ func (ds *Service) AddObjectsToNotionCollection(databaseSnapshots []*converter.S } rootCollection := converter.NewRootCollection(ds.collectionService) - rootCol, err := rootCollection.AddObjects(rootCollectionName, allObjects) + rootCol, err := rootCollection.MakeRootCollection(rootCollectionName, allObjects) if err != nil { return nil, err } diff --git a/core/block/import/notion/api/page/page.go b/core/block/import/notion/api/page/page.go index 9c17301e8f..fc292b8b0a 100644 --- a/core/block/import/notion/api/page/page.go +++ b/core/block/import/notion/api/page/page.go @@ -66,22 +66,13 @@ func (ds *Service) GetPages(ctx context.Context, apiKey string, mode pb.RpcObjectImportRequestMode, pages []Page, - request *block.MapRequest, - progress process.Progress) (*converter.Response, map[string]string, converter.ConvertError) { - var ( - notionPagesIdsToAnytype = make(map[string]string, 0) - ) - - progress.SetProgressMessage("Start creating pages from notion") - - convertError := ds.createIDsForPages(pages, progress, notionPagesIdsToAnytype) + notionImportContext *block.NotionImportContext, + progress process.Progress) (*converter.Response, converter.ConvertError) { + convertError := ds.fillNotionImportContext(pages, progress, notionImportContext) if convertError != nil { - return nil, nil, convertError + return nil, convertError } - progress.SetProgressMessage("Start creating blocks") - request.PageNameToID = ds.extractTitleFromPages(pages) - request.NotionPageIdsToAnytype = notionPagesIdsToAnytype numWorkers := workerPoolSize if len(pages) < workerPoolSize { numWorkers = 1 @@ -90,15 +81,15 @@ func (ds *Service) GetPages(ctx context.Context, go ds.addWorkToPool(pages, pool) - do := NewDataObject(ctx, apiKey, mode, request) + do := NewDataObject(ctx, apiKey, mode, notionImportContext) go pool.Start(do) allSnapshots, converterError := ds.readResultFromPool(pool, mode, progress) if converterError.IsEmpty() { - return &converter.Response{Snapshots: allSnapshots}, notionPagesIdsToAnytype, nil + return &converter.Response{Snapshots: allSnapshots}, nil } - return &converter.Response{Snapshots: allSnapshots}, notionPagesIdsToAnytype, converterError + return &converter.Response{Snapshots: allSnapshots}, converterError } func (ds *Service) readResultFromPool(pool *workerpool.WorkerPool, mode pb.RpcObjectImportRequestMode, progress process.Progress) ([]*converter.Snapshot, converter.ConvertError) { @@ -144,27 +135,31 @@ func (ds *Service) addWorkToPool(pages []Page, pool *workerpool.WorkerPool) { pool.CloseTask() } -func (ds *Service) extractTitleFromPages(pages []Page) map[string]string { +func (ds *Service) extractTitleFromPages(page Page) string { // Need to collect pages title and notion ids mapping for such blocks as ChildPage and ChildDatabase, // because we only get title in those blocks from API - pageNameToID := make(map[string]string, 0) - for _, p := range pages { - for _, v := range p.Properties { - if t, ok := v.(*property.TitleItem); ok { - pageNameToID[p.ID] = t.GetTitle() - } + for _, v := range page.Properties { + if t, ok := v.(*property.TitleItem); ok { + return t.GetTitle() } } - return pageNameToID + return "" } -func (ds *Service) createIDsForPages(pages []Page, progress process.Progress, notionPagesIdsToAnytype map[string]string) converter.ConvertError { +func (ds *Service) fillNotionImportContext(pages []Page, progress process.Progress, importContext *block.NotionImportContext) converter.ConvertError { for _, p := range pages { if err := progress.TryStep(1); err != nil { return converter.NewCancelError(p.ID, err) } - notionPagesIdsToAnytype[p.ID] = uuid.New().String() + importContext.NotionPageIdsToAnytype[p.ID] = uuid.New().String() + if p.Parent.PageID != "" { + importContext.ChildIDToPage[p.ID] = p.Parent.PageID + } + if p.Parent.DatabaseID != "" { + importContext.ChildIDToPage[p.ID] = p.Parent.DatabaseID + } + importContext.PageNameToID[p.ID] = ds.extractTitleFromPages(p) } return nil } diff --git a/core/block/import/notion/api/page/page_test.go b/core/block/import/notion/api/page/page_test.go index 846f5e191a..65c7293a8d 100644 --- a/core/block/import/notion/api/page/page_test.go +++ b/core/block/import/notion/api/page/page_test.go @@ -39,7 +39,7 @@ func Test_handlePagePropertiesSelect(t *testing.T) { }, } pr := property.Properties{"Select": &p} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } @@ -98,7 +98,7 @@ func Test_handlePagePropertiesLastEditedTime(t *testing.T) { LastEditedTime: "2022-10-24T22:56:00.000Z", } pr := property.Properties{"LastEditedTime": &p} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } @@ -128,7 +128,7 @@ func Test_handlePagePropertiesRichText(t *testing.T) { p := property.RichTextItem{ID: "id", Type: string(property.PropertyConfigTypeRichText)} pr := property.Properties{"RichText": &p} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } @@ -161,7 +161,7 @@ func Test_handlePagePropertiesStatus(t *testing.T) { }, } pr := property.Properties{"Status": &p} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } @@ -225,7 +225,7 @@ func Test_handlePagePropertiesNumber(t *testing.T) { Number: &num, } pr := property.Properties{"Number": &p} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } @@ -260,7 +260,7 @@ func Test_handlePagePropertiesMultiSelect(t *testing.T) { }, } pr := property.Properties{"MultiSelect": &p} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } @@ -325,7 +325,7 @@ func Test_handlePagePropertiesCheckbox(t *testing.T) { Checkbox: true, } pr := property.Properties{"Checkbox": &p} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } @@ -355,7 +355,7 @@ func Test_handlePagePropertiesEmail(t *testing.T) { Email: &email, } pr := property.Properties{"Email": &p} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } @@ -387,7 +387,7 @@ func Test_handlePagePropertiesRelation(t *testing.T) { pr := property.Properties{"Relation": &p} notionPageIdsToAnytype := map[string]string{"id": "anytypeID"} notionDatabaseIdsToAnytype := map[string]string{"id": "anytypeID"} - req := &block.MapRequest{ + req := &block.NotionImportContext{ NotionPageIdsToAnytype: notionPageIdsToAnytype, NotionDatabaseIdsToAnytype: notionDatabaseIdsToAnytype, RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, @@ -424,7 +424,7 @@ func Test_handlePagePropertiesPeople(t *testing.T) { Type: string(property.PropertyConfigTypePeople), } pr := property.Properties{"People": &p} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } @@ -462,7 +462,7 @@ func Test_handlePagePropertiesFormula(t *testing.T) { Formula: map[string]interface{}{"type": property.NumberFormula, "number": float64(1)}, } pr := property.Properties{"Formula": &p} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } @@ -491,7 +491,7 @@ func Test_handlePagePropertiesTitle(t *testing.T) { Title: []*api.RichText{{PlainText: "Title"}}, } pr := property.Properties{"Title": &p} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } @@ -542,7 +542,7 @@ func Test_handleRollupProperties(t *testing.T) { pr := property.Properties{"Rollup1": &p1, "Rollup2": &p2, "Rollup3": &p3} - req := &block.MapRequest{ + req := &block.NotionImportContext{ RelationsIdsToAnytypeID: map[string]*model.SmartBlockSnapshotBase{}, RelationsIdsToOptions: map[string][]*model.SmartBlockSnapshotBase{}, } diff --git a/core/block/import/notion/api/page/task.go b/core/block/import/notion/api/page/task.go index 07aff38b50..fb826a27c6 100644 --- a/core/block/import/notion/api/page/task.go +++ b/core/block/import/notion/api/page/task.go @@ -23,11 +23,11 @@ import ( type DataObject struct { apiKey string mode pb.RpcObjectImportRequestMode - request *block.MapRequest + request *block.NotionImportContext ctx context.Context } -func NewDataObject(ctx context.Context, apiKey string, mode pb.RpcObjectImportRequestMode, request *block.MapRequest) *DataObject { +func NewDataObject(ctx context.Context, apiKey string, mode pb.RpcObjectImportRequestMode, request *block.NotionImportContext) *DataObject { return &DataObject{apiKey: apiKey, mode: mode, request: request, ctx: ctx} } @@ -81,7 +81,7 @@ func (pt *Task) makeSnapshotFromPages( apiKey string, p Page, mode pb.RpcObjectImportRequestMode, - request *block.MapRequest) (*model.SmartBlockSnapshotBase, []*model.SmartBlockSnapshotBase, converter.ConvertError, + request *block.NotionImportContext) (*model.SmartBlockSnapshotBase, []*model.SmartBlockSnapshotBase, converter.ConvertError, ) { allErrors := converter.ConvertError{} @@ -96,13 +96,13 @@ func (pt *Task) makeSnapshotFromPages( } request.Blocks = notionBlocks - resp := pt.blockService.MapNotionBlocksToAnytype(request) + resp := pt.blockService.MapNotionBlocksToAnytype(request, pt.p.ID) snapshot := pt.provideSnapshot(resp.Blocks, details, relationLinks) return snapshot, subObjectsSnapshots, nil } -func (pt *Task) provideDetails(ctx context.Context, apiKey string, p Page, request *block.MapRequest) (map[string]*types.Value, []*model.SmartBlockSnapshotBase, []*model.RelationLink) { +func (pt *Task) provideDetails(ctx context.Context, apiKey string, p Page, request *block.NotionImportContext) (map[string]*types.Value, []*model.SmartBlockSnapshotBase, []*model.RelationLink) { details := pt.prepareDetails(p) relations, relationLinks := pt.handlePageProperties(ctx, apiKey, p.ID, p.Properties, details, request) addCoverDetail(p, details) @@ -140,7 +140,7 @@ func (pt *Task) handlePageProperties(ctx context.Context, apiKey, pageID string, p property.Properties, d map[string]*types.Value, - req *block.MapRequest) ([]*model.SmartBlockSnapshotBase, []*model.RelationLink) { + req *block.NotionImportContext) ([]*model.SmartBlockSnapshotBase, []*model.RelationLink) { relations := make([]*model.SmartBlockSnapshotBase, 0) relationsLinks := make([]*model.RelationLink, 0) for k, v := range p { @@ -158,7 +158,7 @@ func (pt *Task) handlePageProperties(ctx context.Context, func (pt *Task) retrieveRelation(ctx context.Context, apiKey, pageID, key string, propObject property.Object, - req *block.MapRequest, + req *block.NotionImportContext, details map[string]*types.Value) ([]*model.SmartBlockSnapshotBase, *model.RelationLink, error) { if err := pt.handlePagination(ctx, apiKey, pageID, propObject); err != nil { return nil, nil, err @@ -167,7 +167,7 @@ func (pt *Task) retrieveRelation(ctx context.Context, return pt.makeRelationFromProperty(req, propObject, details, key) } -func (pt *Task) makeRelationFromProperty(req *block.MapRequest, +func (pt *Task) makeRelationFromProperty(req *block.NotionImportContext, propObject property.Object, details map[string]*types.Value, name string) ([]*model.SmartBlockSnapshotBase, *model.RelationLink, error) { @@ -212,7 +212,7 @@ func (pt *Task) getRelationSnapshot(name string, propObject property.Object) (*m return rel, key } -func (pt *Task) provideRelationOptionsSnapshots(id string, propObject property.Object, req *block.MapRequest) []*model.SmartBlockSnapshotBase { +func (pt *Task) provideRelationOptionsSnapshots(id string, propObject property.Object, req *block.NotionImportContext) []*model.SmartBlockSnapshotBase { pt.relationOptCreateMutex.Lock() defer pt.relationOptCreateMutex.Unlock() subObjectsSnapshots := make([]*model.SmartBlockSnapshotBase, 0) @@ -235,7 +235,7 @@ func (pt *Task) getRelationDetails(key string, name string, propObject property. // linkRelationsIDWithAnytypeID take anytype ID based on page/database ID from Notin. // In property, we get id from Notion, so we somehow need to map this ID with anytype for correct Relation. // We use two maps notionPagesIdsToAnytype, notionDatabaseIdsToAnytype for this -func (pt *Task) handleLinkRelationsIDWithAnytypeID(propObject property.Object, req *block.MapRequest) { +func (pt *Task) handleLinkRelationsIDWithAnytypeID(propObject property.Object, req *block.NotionImportContext) { if r, ok := propObject.(*property.RelationItem); ok { for _, r := range r.Relation { if anytypeID, ok := req.NotionPageIdsToAnytype[r.ID]; ok { @@ -345,7 +345,7 @@ func isPropertyTag(pr property.Object) bool { pr.GetPropertyType() == property.PropertyConfigTypePeople } -func getRelationOptions(pr property.Object, rel string, req *block.MapRequest) []*model.SmartBlockSnapshotBase { +func getRelationOptions(pr property.Object, rel string, req *block.NotionImportContext) []*model.SmartBlockSnapshotBase { var opts []*model.SmartBlockSnapshotBase switch property := pr.(type) { case *property.StatusItem: @@ -366,7 +366,7 @@ func getRelationOptions(pr property.Object, rel string, req *block.MapRequest) [ return opts } -func peopleItemOptions(property *property.PeopleItem, rel string, req *block.MapRequest) []*model.SmartBlockSnapshotBase { +func peopleItemOptions(property *property.PeopleItem, rel string, req *block.NotionImportContext) []*model.SmartBlockSnapshotBase { peopleOptions := make([]*model.SmartBlockSnapshotBase, 0, len(property.People)) for _, po := range property.People { if po.Name == "" { @@ -386,7 +386,7 @@ func peopleItemOptions(property *property.PeopleItem, rel string, req *block.Map return peopleOptions } -func multiselectItemOptions(property *property.MultiSelectItem, rel string, req *block.MapRequest) []*model.SmartBlockSnapshotBase { +func multiselectItemOptions(property *property.MultiSelectItem, rel string, req *block.NotionImportContext) []*model.SmartBlockSnapshotBase { multiSelectOptions := make([]*model.SmartBlockSnapshotBase, 0, len(property.MultiSelect)) for _, so := range property.MultiSelect { if so.Name == "" { @@ -406,7 +406,7 @@ func multiselectItemOptions(property *property.MultiSelectItem, rel string, req return multiSelectOptions } -func selectItemOptions(property *property.SelectItem, rel string, req *block.MapRequest) *model.SmartBlockSnapshotBase { +func selectItemOptions(property *property.SelectItem, rel string, req *block.NotionImportContext) *model.SmartBlockSnapshotBase { if property.Select.Name == "" { return nil } @@ -422,7 +422,7 @@ func selectItemOptions(property *property.SelectItem, rel string, req *block.Map return optSnapshot } -func statusItemOptions(property *property.StatusItem, rel string, req *block.MapRequest) *model.SmartBlockSnapshotBase { +func statusItemOptions(property *property.StatusItem, rel string, req *block.NotionImportContext) *model.SmartBlockSnapshotBase { if property.Status.Name == "" { return nil } @@ -438,7 +438,7 @@ func statusItemOptions(property *property.StatusItem, rel string, req *block.Map return optSnapshot } -func isOptionAlreadyExist(optName, rel string, req *block.MapRequest) (bool, string) { +func isOptionAlreadyExist(optName, rel string, req *block.NotionImportContext) (bool, string) { options := req.ReadRelationsOptionsMap(rel) for _, option := range options { name := pbtypes.GetString(option.Details, bundle.RelationKeyName.String()) diff --git a/core/block/import/notion/api/search/search_test.go b/core/block/import/notion/api/search/search_test.go index 8013c9788a..a12bc6906d 100644 --- a/core/block/import/notion/api/search/search_test.go +++ b/core/block/import/notion/api/search/search_test.go @@ -2,6 +2,7 @@ package search import ( "context" + "github.com/anyproto/anytype-heart/core/block/import/notion/api/block" "net/http" "net/http/httptest" "testing" @@ -31,7 +32,7 @@ func Test_GetDatabaseSuccess(t *testing.T) { assert.Nil(t, err) ds := database.New(nil) - databases, _, ce := ds.GetDatabase(context.Background(), pb.RpcObjectImportRequest_ALL_OR_NOTHING, db, process.NewProgress(pb.ModelProcess_Import)) + databases, ce := ds.GetDatabase(context.Background(), pb.RpcObjectImportRequest_ALL_OR_NOTHING, db, process.NewProgress(pb.ModelProcess_Import), block.NewNotionImportContext()) assert.NotNil(t, databases) assert.Len(t, databases.Snapshots, 17) //1 database + 16 properties (name doesn't count) diff --git a/core/block/import/notion/converter.go b/core/block/import/notion/converter.go index 97ecb8513a..5850742eec 100644 --- a/core/block/import/notion/converter.go +++ b/core/block/import/notion/converter.go @@ -2,6 +2,7 @@ package notion import ( "context" + "errors" "fmt" "time" @@ -14,7 +15,6 @@ import ( "github.com/anyproto/anytype-heart/core/block/import/notion/api/search" "github.com/anyproto/anytype-heart/core/block/process" "github.com/anyproto/anytype-heart/pb" - "github.com/anyproto/anytype-heart/pkg/lib/pb/model" ) const ( @@ -24,6 +24,7 @@ const ( retryAmount = 5 numberOfStepsForPages = 4 // 3 cycles to get snapshots and 1 cycle to create objects numberOfStepsForDatabases = 2 // 1 cycles to get snapshots and 1 cycle to create objects + stepForSearch = 1 ) type Notion struct { @@ -53,28 +54,29 @@ func (n *Notion) GetSnapshots(req *pb.RpcObjectImportRequest, progress process.P ce.Add("/search", fmt.Errorf("failed to get pages and databases %s", err)) return nil, ce } + progress.SetTotal(int64(len(db)*numberOfStepsForDatabases+len(pages)*numberOfStepsForPages) + stepForSearch) + if err = progress.TryStep(1); err != nil { - return nil, converter.NewFromError("", err) + return nil, converter.NewFromError("", converter.ErrCancel) } if len(db) == 0 && len(pages) == 0 { return nil, converter.NewFromError("", converter.ErrNoObjectsToImport) } - progress.SetTotal(int64(len(db)*numberOfStepsForDatabases + len(pages)*numberOfStepsForPages)) - dbSnapshots, mapRequest, dbErr := n.dbService.GetDatabase(context.TODO(), req.Mode, db, progress) + notionImportContext := block.NewNotionImportContext() + dbSnapshots, dbErr := n.dbService.GetDatabase(context.TODO(), req.Mode, db, progress, notionImportContext) + if errors.Is(dbErr.GetResultError(req.Type), converter.ErrCancel) { + return nil, converter.NewFromError("", converter.ErrCancel) + } if dbErr != nil && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { ce.Merge(dbErr) return nil, ce } - r := &block.MapRequest{ - NotionDatabaseIdsToAnytype: mapRequest.NotionDatabaseIdsToAnytype, - DatabaseNameToID: mapRequest.DatabaseNameToID, - RelationsIdsToAnytypeID: mapRequest.RelationsIdsToAnytypeID, - RelationsIdsToOptions: make(map[string][]*model.SmartBlockSnapshotBase, 0), + pgSnapshots, pgErr := n.pgService.GetPages(context.TODO(), apiKey, req.Mode, pages, notionImportContext, progress) + if errors.Is(pgErr.GetResultError(req.Type), converter.ErrCancel) { + return nil, converter.NewFromError("", converter.ErrCancel) } - - pgSnapshots, notionPageIDToAnytype, pgErr := n.pgService.GetPages(context.TODO(), apiKey, req.Mode, pages, r, progress) if pgErr != nil && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { ce.Merge(pgErr) return nil, ce @@ -90,7 +92,7 @@ func (n *Notion) GetSnapshots(req *pb.RpcObjectImportRequest, progress process.P dbs = dbSnapshots.Snapshots } - n.dbService.AddPagesToCollections(dbs, pages, db, notionPageIDToAnytype, mapRequest.NotionDatabaseIdsToAnytype) + n.dbService.AddPagesToCollections(dbs, pages, db, notionImportContext.NotionPageIdsToAnytype, notionImportContext.NotionDatabaseIdsToAnytype) dbs, err = n.dbService.AddObjectsToNotionCollection(dbs, pgs) if err != nil { diff --git a/core/block/import/objectcreator.go b/core/block/import/objectcreator.go index 5728afa681..eb44373d02 100644 --- a/core/block/import/objectcreator.go +++ b/core/block/import/objectcreator.go @@ -3,6 +3,7 @@ package importer import ( "context" "fmt" + "path" "strings" "sync" @@ -26,6 +27,7 @@ import ( "github.com/anyproto/anytype-heart/pkg/lib/core" coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" "github.com/anyproto/anytype-heart/pkg/lib/localstore/addr" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/filestore" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" "github.com/anyproto/anytype-heart/util/pbtypes" @@ -46,6 +48,7 @@ type ObjectCreator struct { objectStore objectstore.ObjectStore relationSyncer syncer.RelationSyncer syncFactory *syncer.Factory + fileStore filestore.FileStore mu sync.Mutex } @@ -55,6 +58,7 @@ func NewCreator(service *block.Service, syncFactory *syncer.Factory, objectStore objectstore.ObjectStore, relationSyncer syncer.RelationSyncer, + fileStore filestore.FileStore, ) Creator { return &ObjectCreator{ service: service, @@ -63,6 +67,7 @@ func NewCreator(service *block.Service, syncFactory: syncFactory, objectStore: objectStore, relationSyncer: relationSyncer, + fileStore: fileStore, } } @@ -78,12 +83,23 @@ func (oc *ObjectCreator) Create(ctx *session.Context, newID := oldIDtoNew[sn.Id] oc.setRootBlock(snapshot, newID) - oc.setWorkspaceID(err, newID, snapshot) + oc.setWorkspaceID(newID, snapshot) st := state.NewDocFromSnapshot(newID, sn.Snapshot).(*state.State) st.SetRootId(newID) // explicitly set last modified date, because all local details removed in NewDocFromSnapshot; createdDate covered in the object header - st.SetLastModified(pbtypes.GetInt64(sn.Snapshot.Data.Details, bundle.RelationKeyLastModifiedDate.String()), oc.core.ProfileID()) + lastModifiedDate := pbtypes.GetInt64(sn.Snapshot.Data.Details, bundle.RelationKeyLastModifiedDate.String()) + createdDate := pbtypes.GetInt64(sn.Snapshot.Data.Details, bundle.RelationKeyCreatedDate.String()) + if lastModifiedDate == 0 { + if createdDate != 0 { + lastModifiedDate = createdDate + } else { + // we can't fallback to time.Now() because it will be inconsistent with the time used in object tree header. + // So instead we should EXPLICITLY set creation date to the snapshot in all importers + log.With("objectID", sn.Id).With("ext", path.Ext(sn.FileName)).Warnf("both lastModifiedDate and createdDate are not set in the imported snapshot") + } + } + st.SetLastModified(lastModifiedDate, oc.core.ProfileID()) var filesToDelete []string defer func() { // delete file in ipfs if there is error after creation @@ -127,7 +143,7 @@ func (oc *ObjectCreator) Create(ctx *session.Context, oc.setArchived(snapshot, newID) - syncErr := oc.syncFilesAndLinks(ctx, st, newID) + syncErr := oc.syncFilesAndLinks(ctx, newID) if syncErr != nil { log.With(zap.String("object id", newID)).Errorf("failed to sync %s: %s", newID, err.Error()) } @@ -207,7 +223,7 @@ func (oc *ObjectCreator) addRootBlock(snapshot *model.SmartBlockSnapshotBase, pa }) } -func (oc *ObjectCreator) setWorkspaceID(err error, newID string, snapshot *model.SmartBlockSnapshotBase) { +func (oc *ObjectCreator) setWorkspaceID(newID string, snapshot *model.SmartBlockSnapshotBase) { if oc.core.PredefinedBlocks().Account == newID { return } @@ -215,6 +231,9 @@ func (oc *ObjectCreator) setWorkspaceID(err error, newID string, snapshot *model if err != nil { log.With(zap.String("object id", newID)).Errorf("failed to get workspace id %s: %s", newID, err.Error()) } + if workspaceID == "" { + return + } if snapshot.Details != nil && snapshot.Details.Fields != nil { snapshot.Details.Fields[bundle.RelationKeyWorkspaceId.String()] = pbtypes.String(workspaceID) @@ -382,16 +401,45 @@ func (oc *ObjectCreator) setArchived(snapshot *model.SmartBlockSnapshotBase, new } } -func (oc *ObjectCreator) syncFilesAndLinks(ctx *session.Context, st *state.State, newID string) error { - return st.Iterate(func(bl simple.Block) (isContinue bool) { - s := oc.syncFactory.GetSyncer(bl) - if s != nil { - if sErr := s.Sync(ctx, newID, bl); sErr != nil { - log.With(zap.String("object id", newID)).Errorf("sync: %s", sErr) +func (oc *ObjectCreator) syncFilesAndLinks(ctx *session.Context, newID string) error { + tasks := make([]func() error, 0) + var fileHashes []string + + err := oc.service.Do(newID, func(b sb.SmartBlock) error { + st := b.NewState() + return st.Iterate(func(bl simple.Block) (isContinue bool) { + fileHashes = st.GetAllFileHashes(st.FileRelationKeys()) + s := oc.syncFactory.GetSyncer(bl) + if s != nil { + // We can't run syncer here because it will cause a deadlock, so we defer this operation + tasks = append(tasks, func() error { + return s.Sync(ctx, newID, bl) + }) } - } - return true + if bl != nil { + if file := bl.Model().GetFile(); file != nil { + fileHashes = append(fileHashes, file.Hash) + } + } + return true + }) }) + if err != nil { + return err + } + for _, task := range tasks { + if err := task(); err != nil { + log.With(zap.String("objectID", newID)).Errorf("syncer: %s", err) + } + } + + for _, hash := range fileHashes { + err = oc.fileStore.SetIsFileImported(hash, true) + if err != nil { + return fmt.Errorf("failed to set isFileImported for file %s: %s", hash, err) + } + } + return nil } func (oc *ObjectCreator) updateLinksInCollections(st *state.State, oldIDtoNew map[string]string, isNewCollection bool) { diff --git a/core/block/import/objectidgetter.go b/core/block/import/objectidgetter.go index a21a0714eb..83deae92d5 100644 --- a/core/block/import/objectidgetter.go +++ b/core/block/import/objectidgetter.go @@ -55,7 +55,8 @@ func (ou *ObjectIDGetter) Get(ctx *session.Context, sn *converter.Snapshot, sbType sb.SmartBlockType, createdTime time.Time, - getExisting bool) (string, treestorage.TreeStorageCreatePayload, error) { + getExisting bool, + oldToNewIDs map[string]string) (string, treestorage.TreeStorageCreatePayload, error) { if sbType == sb.SmartBlockTypeWorkspace { workspaceID, wErr := ou.core.GetWorkspaceIdForObject(sn.Id) if wErr == nil { @@ -72,7 +73,7 @@ func (ou *ObjectIDGetter) Get(ctx *session.Context, return id, treestorage.TreeStorageCreatePayload{}, err } if sbType == sb.SmartBlockTypeSubObject { - id, err = ou.getSubObjectID(sn) + id, err = ou.getSubObjectID(sn, oldToNewIDs) return id, treestorage.TreeStorageCreatePayload{}, err } @@ -110,9 +111,8 @@ func (ou *ObjectIDGetter) getObjectByOldAnytypeID(sn *converter.Snapshot, sbType return "", err } -func (ou *ObjectIDGetter) getSubObjectID(sn *converter.Snapshot) (string, error) { - // in case it already - ids, err := ou.getAlreadyExistingSubObject(sn) +func (ou *ObjectIDGetter) getSubObjectID(sn *converter.Snapshot, oldToNewIDs map[string]string) (string, error) { + ids, err := ou.getAlreadyExistingSubObject(sn, oldToNewIDs) if err == nil && len(ids) > 0 { return ids[0], nil } @@ -154,7 +154,7 @@ func (ou *ObjectIDGetter) getIDBySourceObject(sn *converter.Snapshot) string { return "" } -func (ou *ObjectIDGetter) getAlreadyExistingSubObject(snapshot *converter.Snapshot) ([]string, error) { +func (ou *ObjectIDGetter) getAlreadyExistingSubObject(snapshot *converter.Snapshot, oldToNewIDs map[string]string) ([]string, error) { id := pbtypes.GetString(snapshot.Snapshot.Data.Details, bundle.RelationKeyId.String()) ids, _, err := ou.objectStore.QueryObjectIDs(database.Query{ @@ -174,7 +174,7 @@ func (ou *ObjectIDGetter) getAlreadyExistingSubObject(snapshot *converter.Snapsh return ou.getExistingRelation(snapshot, ids) } if len(ids) == 0 && subObjectType == bundle.TypeKeyRelationOption.URL() { - return ou.getExistingRelationOption(snapshot, ids) + return ou.getExistingRelationOption(snapshot, ids, oldToNewIDs) } return ids, err } @@ -199,9 +199,13 @@ func (ou *ObjectIDGetter) getExistingRelation(snapshot *converter.Snapshot, ids return ids, err } -func (ou *ObjectIDGetter) getExistingRelationOption(snapshot *converter.Snapshot, ids []string) ([]string, error) { +func (ou *ObjectIDGetter) getExistingRelationOption(snapshot *converter.Snapshot, ids []string, oldToNewIDs map[string]string) ([]string, error) { name := pbtypes.GetString(snapshot.Snapshot.Data.Details, bundle.RelationKeyName.String()) key := pbtypes.GetString(snapshot.Snapshot.Data.Details, bundle.RelationKeyRelationKey.String()) + relationID := addr.RelationKeyToIdPrefix + key + if newRelationID, ok := oldToNewIDs[relationID]; ok { + key = strings.TrimPrefix(newRelationID, addr.RelationKeyToIdPrefix) + } ids, _, err := ou.objectStore.QueryObjectIDs(database.Query{ Filters: []*model.BlockContentDataviewFilter{ { diff --git a/core/block/import/pb/converter.go b/core/block/import/pb/converter.go index 15c0f2ccf1..758b295545 100644 --- a/core/block/import/pb/converter.go +++ b/core/block/import/pb/converter.go @@ -5,6 +5,7 @@ import ( "io" "path/filepath" "strings" + "time" "github.com/anyproto/any-sync/util/slice" "github.com/gogo/protobuf/jsonpb" @@ -63,12 +64,13 @@ func (p *Pb) GetSnapshots(req *pb.RpcObjectImportRequest, progress process.Progr } p.updateLinksToObjects(allSnapshots, allErrors, req.Mode) p.updateDetails(allSnapshots) - if !allErrors.IsEmpty() && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { + if (!allErrors.IsEmpty() && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING) || + allErrors.IsNoObjectToImportError(len(params.GetPath())) { return nil, allErrors } if !params.GetNoCollection() { rootCollection := converter.NewRootCollection(p.service) - rootCol, colErr := rootCollection.AddObjects(rootCollectionName, targetObjects) + rootCol, colErr := rootCollection.MakeRootCollection(rootCollectionName, targetObjects) if colErr != nil { allErrors.Add(rootCollectionName, colErr) if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { @@ -292,6 +294,11 @@ func (p *Pb) fillDetails(name string, path string, mo *pb.SnapshotWithType) { } sourceDetail := converter.GetSourceDetail(name, path) mo.Snapshot.Data.Details.Fields[bundle.RelationKeySourceFilePath.String()] = pbtypes.String(sourceDetail) + + createdDate := pbtypes.GetInt64(mo.Snapshot.Data.Details, bundle.RelationKeyCreatedDate.String()) + if createdDate == 0 { + mo.Snapshot.Data.Details.Fields[bundle.RelationKeyCreatedDate.String()] = pbtypes.Int64(time.Now().Unix()) + } } func (p *Pb) Name() string { diff --git a/core/block/import/syncer/file.go b/core/block/import/syncer/file.go index a3ecbc79ea..8671d405a6 100644 --- a/core/block/import/syncer/file.go +++ b/core/block/import/syncer/file.go @@ -46,12 +46,10 @@ func (fs *FileSyncer) Sync(ctx *session.Context, id string, b simple.Block) erro BlockId: b.Model().Id, } } - hash, err := fs.service.UploadFileBlockWithHash(ctx, id, params) - + _, err := fs.service.UploadFileBlockWithHash(ctx, id, params) if err != nil { return fmt.Errorf("failed syncing file: %s", err) } - b.Model().GetFile().Hash = hash os.Remove(b.Model().GetFile().Name) return nil } diff --git a/core/block/import/syncer/relationsyncer.go b/core/block/import/syncer/relationsyncer.go index 42c39b3aa1..105d58ce2d 100644 --- a/core/block/import/syncer/relationsyncer.go +++ b/core/block/import/syncer/relationsyncer.go @@ -3,6 +3,8 @@ package syncer import ( "strings" + "github.com/ipfs/go-cid" + "github.com/anyproto/anytype-heart/core/block" "github.com/anyproto/anytype-heart/core/block/editor/state" "github.com/anyproto/anytype-heart/pb" @@ -27,11 +29,7 @@ func NewFileRelationSyncer(service *block.Service, fileStore filestore.FileStore } func (fs *FileRelationSyncer) Sync(state *state.State, relationName string) []string { - return fs.handleFileRelation(state, relationName) -} - -func (fs *FileRelationSyncer) handleFileRelation(st *state.State, name string) []string { - allFiles := fs.getFilesFromRelations(st, name) + allFiles := fs.getFilesFromRelations(state, relationName) allFilesHashes := make([]string, 0) filesToDelete := make([]string, 0, len(allFiles)) for _, f := range allFiles { @@ -50,8 +48,7 @@ func (fs *FileRelationSyncer) handleFileRelation(st *state.State, name string) [ } } } - fs.updateFileRelationsDetails(st, name, allFilesHashes) - + fs.updateFileRelationsDetails(state, relationName, allFilesHashes) return filesToDelete } @@ -73,14 +70,20 @@ func (fs *FileRelationSyncer) uploadFile(file string) string { err error ) if strings.HasPrefix(file, "http://") || strings.HasPrefix(file, "https://") { + req := pb.RpcFileUploadRequest{Url: file} + hash, err = fs.service.UploadFile(req) + if err != nil { + logger.Errorf("file uploading %s", err) + } + } else { + _, err = cid.Decode(file) + if err == nil { + return file + } req := pb.RpcFileUploadRequest{LocalPath: file} - req.Url = file - req.LocalPath = "" hash, err = fs.service.UploadFile(req) if err != nil { logger.Errorf("file uploading %s", err) - } else { - file = hash } } return hash diff --git a/core/block/import/txt/converter.go b/core/block/import/txt/converter.go index 5536ae9ca2..fa3e7d82c6 100644 --- a/core/block/import/txt/converter.go +++ b/core/block/import/txt/converter.go @@ -54,11 +54,11 @@ func (t *TXT) GetSnapshots(req *pb.RpcObjectImportRequest, progress process.Prog if !cancelError.IsEmpty() { return nil, cancelError } - if !cErr.IsEmpty() && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { + if (!cErr.IsEmpty() && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING) || cErr.IsNoObjectToImportError(len(paths)) { return nil, cErr } rootCollection := converter.NewRootCollection(t.service) - rootCol, err := rootCollection.AddObjects(rootCollectionName, targetObjects) + rootCol, err := rootCollection.MakeRootCollection(rootCollectionName, targetObjects) if err != nil { cErr.Add(rootCollectionName, err) if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { diff --git a/core/block/import/types.go b/core/block/import/types.go index a2b96a1bcc..05f9702baf 100644 --- a/core/block/import/types.go +++ b/core/block/import/types.go @@ -37,5 +37,5 @@ type Creator interface { // IDGetter is interface for updating existing objects type IDGetter interface { //nolint:lll - Get(ctx *session.Context, cs *converter.Snapshot, sbType sb.SmartBlockType, createdTime time.Time, updateExisting bool) (string, treestorage.TreeStorageCreatePayload, error) + Get(ctx *session.Context, cs *converter.Snapshot, sbType sb.SmartBlockType, createdTime time.Time, updateExisting bool, oldIDToNew map[string]string) (string, treestorage.TreeStorageCreatePayload, error) } diff --git a/core/block/object/objectgraph/graph.go b/core/block/object/objectgraph/graph.go index 2b7bc04dd4..6e2e486466 100644 --- a/core/block/object/objectgraph/graph.go +++ b/core/block/object/objectgraph/graph.go @@ -4,6 +4,7 @@ import ( "github.com/anyproto/any-sync/app" "github.com/gogo/protobuf/types" "github.com/opentracing/opentracing-go/log" + "github.com/samber/lo" "github.com/anyproto/anytype-heart/core/relation" "github.com/anyproto/anytype-heart/core/relation/relationutils" @@ -18,6 +19,17 @@ import ( "github.com/anyproto/anytype-heart/util/pbtypes" ) +// relationsSkipList contains relations that SHOULD NOT be included in the graph. These relations of Object/File type that make no sense in the graph for user +var relationsSkipList = []bundle.RelationKey{ + bundle.RelationKeyType, + bundle.RelationKeySetOf, + bundle.RelationKeyCreator, + bundle.RelationKeyLastModifiedBy, + bundle.RelationKeyWorkspaceId, + bundle.RelationKeyIconImage, + bundle.RelationKeyCoverId, +} + type Service interface { ObjectGraph(req *pb.RpcObjectGraphRequest) ([]*types.Struct, []*pb.RpcObjectGraphEdge, error) } @@ -76,6 +88,10 @@ func (gr *Builder) ObjectGraph(req *pb.RpcObjectGraphRequest) ([]*types.Struct, return nodes, edges, nil } +func isRelationShouldBeIncludedAsEdge(rel *relationutils.Relation) bool { + return rel != nil && (rel.Format == model.RelationFormat_object || rel.Format == model.RelationFormat_file) && !lo.Contains(relationsSkipList, bundle.RelationKey(rel.Key)) +} + func (gr *Builder) extractGraph( records []database.Record, nodes []*types.Struct, @@ -92,9 +108,11 @@ func (gr *Builder) extractGraph( outgoingRelationLink := make(map[string]struct{}, 10) for k, v := range rec.Details.GetFields() { rel := relations.GetByKey(k) - if rel != nil && (rel.Format == model.RelationFormat_object || rel.Format == model.RelationFormat_file) { - edges = appendRelations(v, existedNodes, rel, edges, id, outgoingRelationLink) + if !isRelationShouldBeIncludedAsEdge(rel) { + continue } + + edges = appendRelations(v, existedNodes, rel, edges, id, outgoingRelationLink) } edges = gr.appendLinks(rec, outgoingRelationLink, existedNodes, edges, id) diff --git a/core/block/object/objectgraph/graph_test.go b/core/block/object/objectgraph/graph_test.go new file mode 100644 index 0000000000..336cfb29af --- /dev/null +++ b/core/block/object/objectgraph/graph_test.go @@ -0,0 +1,46 @@ +package objectgraph + +import ( + "testing" + + "github.com/anyproto/anytype-heart/core/relation/relationutils" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func Test_isRelationShouldBeIncludedAsEdge(t *testing.T) { + + tests := []struct { + name string + rel *relationutils.Relation + want bool + }{ + {"creator", + &relationutils.Relation{bundle.MustGetRelation(bundle.RelationKeyCreator)}, + false, + }, + {"assignee", + &relationutils.Relation{bundle.MustGetRelation(bundle.RelationKeyAssignee)}, + true, + }, + {"cover", + &relationutils.Relation{bundle.MustGetRelation(bundle.RelationKeyCoverId)}, + false, + }, + {"file relation", + &relationutils.Relation{bundle.MustGetRelation(bundle.RelationKeyTrailer)}, + true, + }, + {"custom relation", + &relationutils.Relation{&model.Relation{Name: "custom", Format: model.RelationFormat_object}}, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isRelationShouldBeIncludedAsEdge(tt.rel); got != tt.want { + t.Errorf("isRelationShouldBeIncludedAsEdge() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/core/block/service.go b/core/block/service.go index 5f23124a16..f7e0643646 100644 --- a/core/block/service.go +++ b/core/block/service.go @@ -154,6 +154,8 @@ type Service struct { syncStarted bool syncerLock sync.Mutex closing chan struct{} + + predefinedObjectWasMissing bool } func (s *Service) Name() string { @@ -195,7 +197,7 @@ func (s *Service) OpenBlock( ctx *session.Context, id string, includeRelationsAsDependentObjects bool, ) (obj *model.ObjectView, err error) { startTime := time.Now() - ob, err := s.getSmartblock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "object_open"), id) + ob, err := s.getSmartblock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "object_open"), id) if err != nil { return nil, err } @@ -259,7 +261,7 @@ func (s *Service) OpenBlock( func (s *Service) ShowBlock( ctx *session.Context, id string, includeRelationsAsDependentObjects bool, ) (obj *model.ObjectView, err error) { - cctx := context.WithValue(context.TODO(), metrics.CtxKeyRequest, "object_show") + cctx := context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "object_show") err2 := s.DoWithContext(cctx, id, func(b smartblock.SmartBlock) error { if includeRelationsAsDependentObjects { b.EnabledRelationAsDependentObjects() @@ -791,7 +793,7 @@ func (s *Service) StateFromTemplate(templateID string, name string) (st *state.S } func (s *Service) DoLinksCollection(id string, apply func(b basic.AllOperations) error) error { - sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do_links_collection"), id) + sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do_links_collection"), id) if err != nil { return err } @@ -805,7 +807,7 @@ func (s *Service) DoLinksCollection(id string, apply func(b basic.AllOperations) } func (s *Service) DoClipboard(id string, apply func(b clipboard.Clipboard) error) error { - sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do_clipboard"), id) + sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do_clipboard"), id) if err != nil { return err } @@ -819,7 +821,7 @@ func (s *Service) DoClipboard(id string, apply func(b clipboard.Clipboard) error } func (s *Service) DoText(id string, apply func(b stext.Text) error) error { - sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do_text"), id) + sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do_text"), id) if err != nil { return err } @@ -833,7 +835,7 @@ func (s *Service) DoText(id string, apply func(b stext.Text) error) error { } func (s *Service) DoFile(id string, apply func(b file.File) error) error { - sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do_file"), id) + sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do_file"), id) if err != nil { return err } @@ -847,7 +849,7 @@ func (s *Service) DoFile(id string, apply func(b file.File) error) error { } func (s *Service) DoBookmark(id string, apply func(b bookmark.Bookmark) error) error { - sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do_bookmark"), id) + sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do_bookmark"), id) if err != nil { return err } @@ -861,7 +863,7 @@ func (s *Service) DoBookmark(id string, apply func(b bookmark.Bookmark) error) e } func (s *Service) DoFileNonLock(id string, apply func(b file.File) error) error { - sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do_filenonlock"), id) + sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do_filenonlock"), id) if err != nil { return err } @@ -873,7 +875,7 @@ func (s *Service) DoFileNonLock(id string, apply func(b file.File) error) error } func (s *Service) DoHistory(id string, apply func(b basic.IHistory) error) error { - sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do_history"), id) + sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do_history"), id) if err != nil { return err } @@ -887,7 +889,7 @@ func (s *Service) DoHistory(id string, apply func(b basic.IHistory) error) error } func (s *Service) DoDataview(id string, apply func(b dataview.Dataview) error) error { - sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do_dataview"), id) + sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do_dataview"), id) if err != nil { return err } @@ -901,7 +903,7 @@ func (s *Service) DoDataview(id string, apply func(b dataview.Dataview) error) e } func (s *Service) Do(id string, apply func(b smartblock.SmartBlock) error) error { - sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do"), id) + sb, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do"), id) if err != nil { return err } @@ -916,7 +918,7 @@ type Picker interface { } func Do[t any](p Picker, id string, apply func(sb t) error) error { - sb, err := p.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do"), id) + sb, err := p.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do"), id) if err != nil { return err } @@ -985,7 +987,7 @@ func DoState2[t1, t2 any](s *Service, firstID, secondID string, f func(*state.St } func DoStateCtx[t any](p Picker, ctx *session.Context, id string, apply func(s *state.State, sb t) error, flags ...smartblock.ApplyFlag) error { - sb, err := p.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do"), id) + sb, err := p.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "do"), id) if err != nil { return err } @@ -1025,11 +1027,15 @@ func (s *Service) DoWithContext(ctx context.Context, id string, apply func(b sma func (s *Service) ObjectApplyTemplate(contextId, templateId string) error { return s.Do(contextId, func(b smartblock.SmartBlock) error { orig := b.NewState().ParentState() - ts, err := s.StateFromTemplate(templateId, pbtypes.GetString(orig.Details(), bundle.RelationKeyName.String())) + name := pbtypes.GetString(orig.Details(), bundle.RelationKeyName.String()) + ts, err := s.StateFromTemplate(templateId, name) if err != nil { return err } ts.SetRootId(contextId) + if name == "" { + orig.SetDetail(bundle.RelationKeyName.String(), ts.Details().Fields[bundle.RelationKeyName.String()]) + } ts.SetParent(orig) fromLayout, _ := orig.Layout() @@ -1110,3 +1116,11 @@ func (s *Service) replaceLink(id, oldId, newId string) error { return b.ReplaceLink(oldId, newId) }) } + +func (s *Service) GetLogFields() []zap.Field { + var fields []zap.Field + if s.predefinedObjectWasMissing { + fields = append(fields, zap.Bool("predefined_object_was_missing", true)) + } + return fields +} diff --git a/core/block/source/files.go b/core/block/source/files.go index bf93ea3db0..0c47be390e 100644 --- a/core/block/source/files.go +++ b/core/block/source/files.go @@ -53,15 +53,15 @@ func (f *file) getDetailsForFileOrImage(ctx context.Context, id string) (p *type return nil, false, err } if strings.HasPrefix(file.Info().Media, "image") { - i, err := f.fileService.ImageByHash(ctx, id) + image, err := f.fileService.ImageByHash(ctx, id) if err != nil { return nil, false, err } - d, err := i.Details(ctx) + details, err := image.Details(ctx) if err != nil { return nil, false, err } - return d, true, nil + return details, true, nil } d, err := file.Details(ctx) diff --git a/core/block/source/source.go b/core/block/source/source.go index d0a554b5f7..b1e7636ef9 100644 --- a/core/block/source/source.go +++ b/core/block/source/source.go @@ -126,6 +126,7 @@ func (s *source) Update(ot objecttree.ObjectTree) { // here it should work, because we always have the most common snapshot of the changes in tree s.lastSnapshotId = ot.Root().Id prevSnapshot := s.lastSnapshotId + // todo: check this one err := s.receiver.StateAppend(func(d state.Doc) (st *state.State, changes []*pb.ChangeContent, err error) { st, changes, sinceSnapshot, err := BuildState(d.(*state.State), ot, s.coreService.PredefinedBlocks().Profile) if prevSnapshot != s.lastSnapshotId { @@ -407,6 +408,82 @@ func BuildState(initState *state.State, ot objecttree.ReadableObjectTree, profil startId = st.ChangeId() } + var lastMigrationVersion uint32 + err = ot.IterateFrom(startId, + func(decrypted []byte) (any, error) { + ch := &pb.Change{} + err = proto.Unmarshal(decrypted, ch) + if err != nil { + return nil, err + } + return ch, nil + }, func(change *objecttree.Change) bool { + count++ + lastChange = change + // that means that we are starting from tree root + if change.Id == ot.Id() { + st = state.NewDoc(ot.Id(), nil).(*state.State) + st.SetChangeId(change.Id) + return true + } + + model := change.Model.(*pb.Change) + if model.Version > lastMigrationVersion { + lastMigrationVersion = model.Version + } + if startId == change.Id { + if st == nil { + changesAppliedSinceSnapshot = 0 + st = state.NewDocFromSnapshot(ot.Id(), model.Snapshot, state.WithChangeId(startId)).(*state.State) + return true + } else { + st = st.NewState() + } + return true + } + if model.Snapshot != nil { + changesAppliedSinceSnapshot = 0 + } else { + changesAppliedSinceSnapshot++ + } + appliedContent = append(appliedContent, model.Content...) + st.SetChangeId(change.Id) + st.ApplyChangeIgnoreErr(model.Content...) + st.AddFileKeys(model.FileKeys...) + + return true + }) + if err != nil { + return + } + _, _, err = state.ApplyStateFastOne(st) + if err != nil { + return + } + + if lastChange != nil && !st.IsTheHeaderChange() { + // todo: why do we don't need to set last modified for the header change? + st.SetLastModified(lastChange.Timestamp, profileId) + } + st.SetMigrationVersion(lastMigrationVersion) + return +} + +// BuildStateFull is deprecated, used in tests only, use BuildState instead +func BuildStateFull(initState *state.State, ot objecttree.ReadableObjectTree, profileId string) (st *state.State, appliedContent []*pb.ChangeContent, changesAppliedSinceSnapshot int, err error) { + var ( + startId string + lastChange *objecttree.Change + count int + ) + // if the state has no first change + if initState == nil { + startId = ot.Root().Id + } else { + st = initState + startId = st.ChangeId() + } + var lastMigrationVersion uint32 err = ot.IterateFrom(startId, func(decrypted []byte) (any, error) { diff --git a/core/debug/service.go b/core/debug/service.go index 6026f1fff6..29bb2abf97 100644 --- a/core/debug/service.go +++ b/core/debug/service.go @@ -5,13 +5,16 @@ import ( "context" "fmt" "io" + "net/http" "os" + "path" "path/filepath" "strings" "time" "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/commonspace/objecttreebuilder" + "github.com/go-chi/chi/v5" "github.com/gogo/protobuf/jsonpb" "github.com/anyproto/anytype-heart/core/block" @@ -41,12 +44,83 @@ type debug struct { block *block.Service store objectstore.ObjectStore clientService space.Service + + server *http.Server +} + +type Debuggable interface { + DebugRouter(r chi.Router) } func (d *debug) Init(a *app.App) (err error) { d.store = a.MustComponent(objectstore.CName).(objectstore.ObjectStore) d.clientService = a.MustComponent(space.CName).(space.Service) d.block = a.MustComponent(block.CName).(*block.Service) + + if addr, ok := os.LookupEnv("ANYDEBUG"); ok && addr != "" { + r := chi.NewRouter() + a.IterateComponents(func(c app.Component) { + if d, ok := c.(Debuggable); ok { + fmt.Println("debug router registered for component: ", c.Name()) + r.Route("/debug/"+c.Name(), d.DebugRouter) + } + }) + routes := r.Routes() + r.Get("/debug", func(w http.ResponseWriter, req *http.Request) { + err := renderLinksList(w, "/", routes) + if err != nil { + logger.Error("failed to render links list", err) + } + }) + d.server = &http.Server{ + Addr: addr, + Handler: r, + } + } + return nil +} + +func joinPath(parent string, child string) string { + parent = strings.TrimSuffix(parent, "/*") + return path.Join(parent, child) +} + +func renderLinksList(w io.Writer, path string, routes []chi.Route) error { + for _, r := range routes { + if r.SubRoutes != nil { + err := renderLinksList(w, joinPath(path, r.Pattern), r.SubRoutes.Routes()) + if err != nil { + return err + } + } else { + _, err := fmt.Fprintf(w, `%s
`, joinPath(path, r.Pattern), joinPath(path, r.Pattern)) + if err != nil { + return err + } + } + } + return nil +} + +func (d *debug) Run(ctx context.Context) error { + if d.server != nil { + go func() { + err := d.server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + logger.Error("debug server error:", err) + } + }() + } + return nil +} + +func (d *debug) Close(ctx context.Context) error { + if d.server != nil { + err := d.server.Shutdown(ctx) + if err != nil { + return fmt.Errorf("debug server shutdown: %w", err) + } + } return nil } diff --git a/core/debug/treearchive/treeimporter.go b/core/debug/treearchive/treeimporter.go index 7a5dfc469d..c3bfff3230 100644 --- a/core/debug/treearchive/treeimporter.go +++ b/core/debug/treearchive/treeimporter.go @@ -3,6 +3,7 @@ package treearchive import ( "errors" "fmt" + "github.com/anyproto/any-sync/commonspace/object/acl/list" "github.com/anyproto/anytype-heart/util/pbtypes" @@ -44,7 +45,7 @@ func (m MarshalledJsonChange) MarshalJSON() ([]byte, error) { type TreeImporter interface { ObjectTree() objecttree.ReadableObjectTree - State() (*state.State, error) + State(fullStateChain bool) (*state.State, error) // set fullStateChain to true to get full state chain, otherwise only the last state will be returned Import(fromRoot bool, beforeId string) error Json() (TreeJson, error) ChangeAt(idx int) (IdChange, error) @@ -67,10 +68,22 @@ func (t *treeImporter) ObjectTree() objecttree.ReadableObjectTree { return t.objectTree } -func (t *treeImporter) State() (*state.State, error) { - st, _, _, err := source.BuildState(nil, t.objectTree, "") - if err != nil { - return nil, err +func (t *treeImporter) State(fullStateChain bool) (*state.State, error) { + var ( + st *state.State + err error + ) + + if fullStateChain { + st, _, _, err = source.BuildStateFull(nil, t.objectTree, "") + if err != nil { + return nil, err + } + } else { + st, _, _, err = source.BuildState(nil, t.objectTree, "") + if err != nil { + return nil, err + } } if _, _, err = state.ApplyStateFast(st); err != nil { @@ -91,6 +104,7 @@ func (t *treeImporter) Import(fullTree bool, beforeId string) (err error) { IncludeBeforeId: true, BuildFullTree: fullTree, }) + return } diff --git a/core/file.go b/core/file.go index a2177146ab..85c37a5d95 100644 --- a/core/file.go +++ b/core/file.go @@ -69,7 +69,7 @@ func (mw *Middleware) FileListOffload(cctx context.Context, req *pb.RpcFileListO } fileService := app.MustComponent[files.Service](mw.app) - totalFilesOffloaded, totalBytesOffloaded, err := fileService.FileListOffload(req.OnlyIds, req.IncludeNotPinned) + totalBytesOffloaded, totalFilesOffloaded, err := fileService.FileListOffload(req.OnlyIds, req.IncludeNotPinned) if err != nil { return response(0, 0, pb.RpcFileListOffloadResponseError_UNKNOWN_ERROR, err) } diff --git a/core/files/debug.go b/core/files/debug.go new file mode 100644 index 0000000000..47cdb45126 --- /dev/null +++ b/core/files/debug.go @@ -0,0 +1,94 @@ +package files + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "github.com/go-chi/chi/v5" + "github.com/ipfs/go-cid" + + "github.com/anyproto/anytype-heart/pkg/lib/localstore" + "github.com/anyproto/anytype-heart/util/debug" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +func (s *service) DebugRouter(r chi.Router) { + r.Get("/status", debug.JSONHandler(s.debugFiles)) + r.Get("/queue", debug.JSONHandler(s.fileSync.DebugQueue)) + r.Get("/tree/{rootID}", debug.PlaintextHandler(s.printTree)) +} + +type fileDebugInfo struct { + Hash string + SyncStatus int + IsIndexed bool +} + +func (s *service) debugFiles(_ *http.Request) ([]*fileDebugInfo, error) { + hashes, err := s.fileStore.ListTargets() + if err != nil { + return nil, fmt.Errorf("list targets: %s", err) + } + result := make([]*fileDebugInfo, 0, len(hashes)) + for _, hash := range hashes { + status, err := s.fileStore.GetSyncStatus(hash) + if err == localstore.ErrNotFound { + status = -1 + err = nil + } + if err != nil { + return nil, fmt.Errorf("get status for %s: %s", hash, err) + } + + var isIndexed bool + details, err := s.objectStore.GetDetails(hash) + if err != nil && !errors.Is(err, localstore.ErrNotFound) { + return nil, fmt.Errorf("get status for %s: %s", hash, err) + } + if details != nil && !pbtypes.IsStructEmpty(details.Details) { + isIndexed = true + } + + result = append(result, &fileDebugInfo{ + Hash: hash, + SyncStatus: status, + IsIndexed: isIndexed, + }) + } + return result, nil +} + +func (s *service) printTree(w io.Writer, req *http.Request) error { + rawID := chi.URLParam(req, "rootID") + id, err := cid.Parse(rawID) + if err != nil { + return fmt.Errorf("parse cid %s: %w", rawID, err) + } + return s.printNode(req.Context(), w, id, 0) +} + +func (s *service) printNode(ctx context.Context, w io.Writer, id cid.Cid, level int) error { + node, err := s.dagService.Get(ctx, id) + if err != nil { + return fmt.Errorf("get dag node %s: %w", id.String(), err) + } + size, err := node.Size() + if err != nil { + return fmt.Errorf("get size for node %s: %w", id.String(), err) + } + _, err = fmt.Fprintln(w, strings.Repeat(" ", level), id.String(), size) + if err != nil { + return fmt.Errorf("print node %s: %w", id.String(), err) + } + for _, link := range node.Links() { + err = s.printNode(ctx, w, link.Cid, level+1) + if err != nil { + return err + } + } + return nil +} diff --git a/core/files/file.go b/core/files/file.go index 7a86edf8a0..17ebe0f7b8 100644 --- a/core/files/file.go +++ b/core/files/file.go @@ -31,10 +31,11 @@ type file struct { } type FileMeta struct { - Media string - Name string - Size int64 - Added time.Time + Media string + Name string + Size int64 + LastModifiedDate int64 + Added time.Time } func (f *file) audioDetails(ctx context.Context) (*types.Struct, error) { @@ -77,19 +78,15 @@ func (f *file) audioDetails(ctx context.Context) (*types.Struct, error) { func (f *file) Details(ctx context.Context) (*types.Struct, error) { meta := f.Meta() + commonDetails := calculateCommonDetails(f.hash, bundle.TypeKeyFile, model.ObjectType_file, f.info.LastModifiedDate) + commonDetails[bundle.RelationKeyFileMimeType.String()] = pbtypes.String(meta.Media) + commonDetails[bundle.RelationKeyName.String()] = pbtypes.String(strings.TrimSuffix(meta.Name, filepath.Ext(meta.Name))) + commonDetails[bundle.RelationKeyFileExt.String()] = pbtypes.String(strings.TrimPrefix(filepath.Ext(meta.Name), ".")) + commonDetails[bundle.RelationKeySizeInBytes.String()] = pbtypes.Float64(float64(meta.Size)) + commonDetails[bundle.RelationKeyAddedDate.String()] = pbtypes.Float64(float64(meta.Added.Unix())) + t := &types.Struct{ - Fields: map[string]*types.Value{ - bundle.RelationKeyId.String(): pbtypes.String(f.hash), - bundle.RelationKeyLayout.String(): pbtypes.Float64(float64(model.ObjectType_file)), - bundle.RelationKeyIsReadonly.String(): pbtypes.Bool(true), - bundle.RelationKeyType.String(): pbtypes.String(bundle.TypeKeyFile.URL()), - bundle.RelationKeyFileMimeType.String(): pbtypes.String(meta.Media), - bundle.RelationKeyName.String(): pbtypes.String(strings.TrimSuffix(meta.Name, filepath.Ext(meta.Name))), - bundle.RelationKeyFileExt.String(): pbtypes.String(strings.TrimPrefix(filepath.Ext(meta.Name), ".")), - bundle.RelationKeySizeInBytes.String(): pbtypes.Float64(float64(meta.Size)), - bundle.RelationKeyAddedDate.String(): pbtypes.Float64(float64(meta.Added.Unix())), - bundle.RelationKeyLastModifiedDate.String(): pbtypes.Int64(time.Now().Unix()), - }, + Fields: commonDetails, } if strings.HasPrefix(meta.Media, "video") { @@ -112,10 +109,11 @@ func (f *file) Info() *storage.FileInfo { func (f *file) Meta() *FileMeta { return &FileMeta{ - Media: f.info.Media, - Name: f.info.Name, - Size: f.info.Size_, - Added: time.Unix(f.info.Added, 0), + Media: f.info.Media, + Name: f.info.Name, + Size: f.info.Size_, + LastModifiedDate: f.info.LastModifiedDate, + Added: time.Unix(f.info.Added, 0), } } @@ -126,3 +124,18 @@ func (f *file) Hash() string { func (f *file) Reader(ctx context.Context) (io.ReadSeeker, error) { return f.node.getContentReader(ctx, f.info) } + +func calculateCommonDetails( + hash string, + typeKey bundle.TypeKey, + layout model.ObjectTypeLayout, + lastModifiedDate int64, +) map[string]*types.Value { + return map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(hash), + bundle.RelationKeyIsReadonly.String(): pbtypes.Bool(true), + bundle.RelationKeyType.String(): pbtypes.String(typeKey.URL()), + bundle.RelationKeyLayout.String(): pbtypes.Float64(float64(layout)), + bundle.RelationKeyLastModifiedDate.String(): pbtypes.Int64(lastModifiedDate), + } +} diff --git a/core/files/files.go b/core/files/files.go index 60707f196c..9132bf37b7 100644 --- a/core/files/files.go +++ b/core/files/files.go @@ -14,6 +14,7 @@ import ( "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/commonfile/fileservice" + "github.com/anyproto/any-sync/commonspace/syncstatus" "github.com/gogo/protobuf/proto" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -60,7 +61,6 @@ type Service interface { ImageAdd(ctx context.Context, options ...AddOption) (Image, error) ImageByHash(ctx context.Context, hash string) (Image, error) StoreFileKeys(fileKeys ...FileKeys) error - AddToSyncQueue(fileID string) error app.Component } @@ -126,9 +126,6 @@ func (s *service) fileAdd(ctx context.Context, opts AddOptions) (string, *storag if err != nil { return "", nil, err } - if err = s.storeChunksCount(ctx, node); err != nil { - return "", nil, fmt.Errorf("store chunks count: %w", err) - } nodeHash := node.Cid().String() if err = s.fileIndexData(ctx, node, nodeHash); err != nil { @@ -145,20 +142,6 @@ func (s *service) fileAdd(ctx context.Context, opts AddOptions) (string, *storag return nodeHash, fileInfo, nil } -func (s *service) storeChunksCount(ctx context.Context, node ipld.Node) error { - chunksCount, err := s.fileSync.FetchChunksCount(ctx, node) - if err != nil { - return fmt.Errorf("count chunks: %w", err) - } - - nodeHash := node.Cid().String() - if err = s.fileStore.SetChunksCount(nodeHash, chunksCount); err != nil { - return fmt.Errorf("store chunks count: %w", err) - } - - return nil -} - // fileRestoreKeys restores file path=>key map from the IPFS DAG using the keys in the localStore func (s *service) fileRestoreKeys(ctx context.Context, hash string) (map[string]string, error) { links, err := helpers.LinksAtCid(ctx, s.dagService, hash) @@ -288,15 +271,10 @@ func (s *service) fileAddNodeFromFiles(ctx context.Context, files []*storage.Fil if err != nil { return nil, nil, err } - - /*err = helpers.PinNode(s.node, node, false) - if err != nil { - return nil, nil, err - }*/ return node, keys, nil } -func (s *service) fileGetInfoForPath(pth string) (*storage.FileInfo, error) { +func (s *service) fileGetInfoForPath(ctx context.Context, pth string) (*storage.FileInfo, error) { if !strings.HasPrefix(pth, "/ipfs/") { return nil, fmt.Errorf("path should starts with '/dagService/...'") } @@ -312,7 +290,7 @@ func (s *service) fileGetInfoForPath(pth string) (*storage.FileInfo, error) { } if key, exists := keys.Keys["/"+strings.Join(pthParts[3:], "/")+"/"]; exists { - return s.fileInfoFromPath("", pth, key) + return s.fileInfoFromPath(ctx, "", pth, key) } return nil, fmt.Errorf("key not found") @@ -400,8 +378,8 @@ func (s *service) fileIndexLink(ctx context.Context, inode ipld.Node, fileID str return nil } -func (s *service) fileInfoFromPath(target string, path string, key string) (*storage.FileInfo, error) { - cid, r, err := helpers.DataAtPath(context.TODO(), s.commonFile, path+"/"+MetaLinkName) +func (s *service) fileInfoFromPath(ctx context.Context, target string, path string, key string) (*storage.FileInfo, error) { + cid, r, err := helpers.DataAtPath(ctx, s.commonFile, path+"/"+MetaLinkName) if err != nil { return nil, err } @@ -563,15 +541,16 @@ func (s *service) fileAddWithConfig(ctx context.Context, mill m.Mill, conf AddOp } fileInfo := &storage.FileInfo{ - Mill: mill.ID(), - Checksum: check, - Source: source, - Opts: opts, - Media: conf.Media, - Name: conf.Name, - Added: time.Now().Unix(), - Meta: pbtypes.ToStruct(res.Meta), - Size_: int64(readerWithCounter.Count()), + Mill: mill.ID(), + Checksum: check, + Source: source, + Opts: opts, + Media: conf.Media, + Name: conf.Name, + LastModifiedDate: conf.LastModifiedDate, + Added: time.Now().Unix(), + Meta: pbtypes.ToStruct(res.Meta), + Size_: int64(readerWithCounter.Count()), } var ( @@ -776,7 +755,7 @@ func (s *service) fileIndexInfo(ctx context.Context, hash string, updateIfExists key = keys["/"+index.Name+"/"] } - fileIndex, err := s.fileInfoFromPath(hash, hash+"/"+index.Name, key) + fileIndex, err := s.fileInfoFromPath(ctx, hash, hash+"/"+index.Name, key) if err != nil { return nil, fmt.Errorf("fileInfoFromPath error: %s", err.Error()) } @@ -788,7 +767,7 @@ func (s *service) fileIndexInfo(ctx context.Context, hash string, updateIfExists key = keys["/"+index.Name+"/"+link.Name+"/"] } - fileIndex, err := s.fileInfoFromPath(hash, hash+"/"+index.Name+"/"+link.Name, key) + fileIndex, err := s.fileInfoFromPath(ctx, hash, hash+"/"+index.Name+"/"+link.Name, key) if err != nil { return nil, fmt.Errorf("fileInfoFromPath error: %s", err.Error()) } @@ -805,10 +784,6 @@ func (s *service) fileIndexInfo(ctx context.Context, hash string, updateIfExists return files, nil } -func (s *service) AddToSyncQueue(fileID string) error { - return s.addToSyncQueue(fileID, false) -} - func (s *service) addToSyncQueue(fileID string, uploadedByUser bool) error { spaceID := s.spaceService.AccountId() @@ -901,11 +876,28 @@ func (s *service) FileByHash(ctx context.Context, hash string) (File, error) { log.With("cid", hash).Errorf("FileByHash: failed to retrieve from IPFS: %s", err.Error()) return nil, ErrFileNotFound } + ok, err := s.fileStore.IsFileImported(hash) + if err != nil { + return nil, fmt.Errorf("check if file is imported: %w", err) + } + if ok { + log.With("fileID", hash).Warn("file is imported, push it to uploading queue") + // If file is imported we have to sync it, so we don't set sync status to synced + err = s.fileStore.SetIsFileImported(hash, false) + if err != nil { + return nil, fmt.Errorf("set is file imported: %w", err) + } + } else { + // If file is not imported then it's definitely synced + err = s.fileStore.SetSyncStatus(hash, int(syncstatus.StatusSynced)) + if err != nil { + return nil, fmt.Errorf("set sync status: %w", err) + } + } } - if err := s.AddToSyncQueue(hash); err != nil { + if err := s.addToSyncQueue(hash, false); err != nil { return nil, fmt.Errorf("add file %s to sync queue: %w", hash, err) } - fileIndex := fileList[0] return &file{ hash: hash, diff --git a/core/files/image.go b/core/files/image.go index c91cf4c4c8..4d7216b622 100644 --- a/core/files/image.go +++ b/core/files/image.go @@ -43,7 +43,7 @@ func (i *image) GetFileForWidth(ctx context.Context, wantWidth int) (File, error } if wantWidth > 1920 { - fileIndex, err := i.service.fileGetInfoForPath("/ipfs/" + i.hash + "/0/original") + fileIndex, err := i.service.fileGetInfoForPath(ctx, "/ipfs/"+i.hash+"/0/original") if err == nil { return &file{ hash: fileIndex.Hash, @@ -54,7 +54,7 @@ func (i *image) GetFileForWidth(ctx context.Context, wantWidth int) (File, error } sizeName := getSizeForWidth(wantWidth) - fileIndex, err := i.service.fileGetInfoForPath("/ipfs/" + i.hash + "/0/" + sizeName) + fileIndex, err := i.service.fileGetInfoForPath(ctx, "/ipfs/"+i.hash+"/0/"+sizeName) if err != nil { return nil, err } @@ -69,7 +69,7 @@ func (i *image) GetFileForWidth(ctx context.Context, wantWidth int) (File, error // GetOriginalFile doesn't contains Meta func (i *image) GetOriginalFile(ctx context.Context) (File, error) { sizeName := "original" - fileIndex, err := i.service.fileGetInfoForPath("/ipfs/" + i.hash + "/0/" + sizeName) + fileIndex, err := i.service.fileGetInfoForPath(ctx, "/ipfs/"+i.hash+"/0/"+sizeName) if err == nil { return &file{ hash: fileIndex.Hash, @@ -89,7 +89,7 @@ func (i *image) GetFileForLargestWidth(ctx context.Context) (File, error) { // fallback to large size, because older image nodes don't have an original sizeName := "large" - fileIndex, err := i.service.fileGetInfoForPath("/ipfs/" + i.hash + "/0/" + sizeName) + fileIndex, err := i.service.fileGetInfoForPath(ctx, "/ipfs/"+i.hash+"/0/"+sizeName) if err != nil { return nil, err } @@ -106,7 +106,7 @@ func (i *image) Hash() string { } func (i *image) Exif(ctx context.Context) (*mill.ImageExifSchema, error) { - fileIndex, err := i.service.fileGetInfoForPath("/ipfs/" + i.hash + "/0/exif") + fileIndex, err := i.service.fileGetInfoForPath(ctx, "/ipfs/"+i.hash+"/0/exif") if err != nil { return nil, err } @@ -133,14 +133,22 @@ func (i *image) Exif(ctx context.Context) (*mill.ImageExifSchema, error) { } func (i *image) Details(ctx context.Context) (*types.Struct, error) { + imageExif, err := i.Exif(ctx) + if err != nil { + log.Errorf("failed to get exif for image: %s", err.Error()) + imageExif = &mill.ImageExifSchema{} + } + + commonDetails := calculateCommonDetails( + i.hash, + bundle.TypeKeyImage, + model.ObjectType_image, + i.extractLastModifiedDate(ctx, imageExif), + ) + commonDetails[bundle.RelationKeyIconImage.String()] = pbtypes.String(i.hash) + details := &types.Struct{ - Fields: map[string]*types.Value{ - bundle.RelationKeyId.String(): pbtypes.String(i.hash), - bundle.RelationKeyIsReadonly.String(): pbtypes.Bool(true), - bundle.RelationKeyIconImage.String(): pbtypes.String(i.hash), - bundle.RelationKeyType.String(): pbtypes.String(bundle.TypeKeyImage.URL()), - bundle.RelationKeyLayout.String(): pbtypes.Float64(float64(model.ObjectType_image)), - }, + Fields: commonDetails, } ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) @@ -168,17 +176,10 @@ func (i *image) Details(ctx context.Context) (*types.Struct, error) { details.Fields[bundle.RelationKeyFileMimeType.String()] = pbtypes.String(largest.Meta().Media) details.Fields[bundle.RelationKeySizeInBytes.String()] = pbtypes.Float64(float64(largest.Meta().Size)) details.Fields[bundle.RelationKeyAddedDate.String()] = pbtypes.Float64(float64(largest.Meta().Added.Unix())) - } - exif, err := i.Exif(ctx) - if err != nil { - log.Errorf("failed to get exif for image: %s", err.Error()) - exif = &mill.ImageExifSchema{} - } - - if !exif.Created.IsZero() { - details.Fields[bundle.RelationKeyCreatedDate.String()] = pbtypes.Float64(float64(exif.Created.Unix())) + if !imageExif.Created.IsZero() { + details.Fields[bundle.RelationKeyCreatedDate.String()] = pbtypes.Float64(float64(imageExif.Created.Unix())) } /*if exif.Latitude != 0.0 { details.Fields["latitude"] = pbtypes.Float64(exif.Latitude) @@ -186,28 +187,28 @@ func (i *image) Details(ctx context.Context) (*types.Struct, error) { if exif.Longitude != 0.0 { details.Fields["longitude"] = pbtypes.Float64(exif.Longitude) }*/ - if exif.CameraModel != "" { - details.Fields[bundle.RelationKeyCamera.String()] = pbtypes.String(exif.CameraModel) + if imageExif.CameraModel != "" { + details.Fields[bundle.RelationKeyCamera.String()] = pbtypes.String(imageExif.CameraModel) } - if exif.ExposureTime != "" { - details.Fields[bundle.RelationKeyExposure.String()] = pbtypes.String(exif.ExposureTime) + if imageExif.ExposureTime != "" { + details.Fields[bundle.RelationKeyExposure.String()] = pbtypes.String(imageExif.ExposureTime) } - if exif.FNumber != 0 { - details.Fields[bundle.RelationKeyFocalRatio.String()] = pbtypes.Float64(exif.FNumber) + if imageExif.FNumber != 0 { + details.Fields[bundle.RelationKeyFocalRatio.String()] = pbtypes.Float64(imageExif.FNumber) } - if exif.ISO != 0 { - details.Fields[bundle.RelationKeyCameraIso.String()] = pbtypes.Float64(float64(exif.ISO)) + if imageExif.ISO != 0 { + details.Fields[bundle.RelationKeyCameraIso.String()] = pbtypes.Float64(float64(imageExif.ISO)) } - if exif.Description != "" { + if imageExif.Description != "" { // use non-empty image description as an image name, because it much uglier to use file names for objects - details.Fields[bundle.RelationKeyName.String()] = pbtypes.String(exif.Description) + details.Fields[bundle.RelationKeyName.String()] = pbtypes.String(imageExif.Description) } - if exif.Artist != "" { - artistName, artistUrl := unpackArtist(exif.Artist) + if imageExif.Artist != "" { + artistName, artistURL := unpackArtist(imageExif.Artist) details.Fields[bundle.RelationKeyMediaArtistName.String()] = pbtypes.String(artistName) - if artistUrl != "" { - details.Fields[bundle.RelationKeyMediaArtistURL.String()] = pbtypes.String(artistUrl) + if artistURL != "" { + details.Fields[bundle.RelationKeyMediaArtistURL.String()] = pbtypes.String(artistURL) } } @@ -264,6 +265,27 @@ func (i *image) getFileForWidthFromCache(wantWidth int) (File, error) { return nil, ErrFileNotFound } +func (i *image) extractLastModifiedDate(ctx context.Context, imageExif *mill.ImageExifSchema) int64 { + var lastModifiedDate int64 + largest, err := i.GetFileForLargestWidth(ctx) + if err == nil { + lastModifiedDate = largest.Meta().LastModifiedDate + } + if lastModifiedDate == 0 { + lastModifiedDate = imageExif.Created.Unix() + } + + if lastModifiedDate == 0 && err == nil { + lastModifiedDate = largest.Meta().Added.Unix() + } + + if lastModifiedDate == 0 { + lastModifiedDate = time.Now().Unix() + } + + return lastModifiedDate +} + var imageWidthByName = map[string]int{ "thumb": 100, "small": 320, diff --git a/core/files/images.go b/core/files/images.go index e201907263..964963ef06 100644 --- a/core/files/images.go +++ b/core/files/images.go @@ -88,9 +88,6 @@ func (s *service) imageAdd(ctx context.Context, opts AddOptions) (string, map[in if err != nil { return "", nil, err } - if err = s.storeChunksCount(ctx, node); err != nil { - return "", nil, fmt.Errorf("store chunks count: %w", err) - } nodeHash := node.Cid().String() err = s.fileStore.AddFileKeys(filestore.FileKeys{ diff --git a/core/files/offload.go b/core/files/offload.go index 76aa48d7a0..42f3e695c2 100644 --- a/core/files/offload.go +++ b/core/files/offload.go @@ -5,11 +5,11 @@ import ( "fmt" "time" + "github.com/anyproto/any-sync/commonspace/syncstatus" "github.com/ipfs/go-cid" - "github.com/samber/lo" + "github.com/anyproto/anytype-heart/core/filestorage" "github.com/anyproto/anytype-heart/pkg/lib/localstore" - "github.com/anyproto/anytype-heart/pkg/lib/pb/storage" ) func (s *service) FileOffload(fileID string, includeNotPinned bool) (totalSize uint64, err error) { @@ -64,15 +64,10 @@ func (s *service) fileOffload(hash string) (totalSize uint64, err error) { func (s *service) FileListOffload(fileIDs []string, includeNotPinned bool) (totalBytesOffloaded uint64, totalFilesOffloaded uint64, err error) { if len(fileIDs) == 0 { - allFiles, err := s.fileStore.List() + fileIDs, err = s.fileStore.ListTargets() if err != nil { return 0, 0, fmt.Errorf("list all files: %w", err) } - - allTargets := lo.Map(allFiles, func(file *storage.FileInfo, _ int) []string { - return file.Targets - }) - fileIDs = lo.Uniq(lo.Flatten(allTargets)) } if !includeNotPinned { @@ -105,27 +100,26 @@ func (s *service) isFileDeleted(fileID string) (bool, error) { } func (s *service) keepOnlyPinnedOrDeleted(fileIDs []string) ([]string, error) { - fileStats, err := s.fileSync.FileListStats(context.Background(), s.spaceService.AccountId(), fileIDs) - if err != nil { - return nil, fmt.Errorf("files stat: %w", err) - } - - fileIDs = fileIDs[:0] - for _, fileStat := range fileStats { - if fileStat.IsPinned() { - fileIDs = append(fileIDs, fileStat.FileId) + var result []string + for _, fileID := range fileIDs { + status, err := s.fileStore.GetSyncStatus(fileID) + if err != nil && err != localstore.ErrNotFound { + return nil, fmt.Errorf("get sync status for file %s: %w", fileID, err) + } + if status == int(syncstatus.StatusSynced) { + result = append(result, fileID) continue } - isDeleted, err := s.isFileDeleted(fileStat.FileId) + isDeleted, err := s.isFileDeleted(fileID) if err != nil { - log.With("fileID", fileStat.FileId).Errorf("failed to check if file is deleted: %s", err) + log.With("fileID", fileID).Errorf("failed to check if file is deleted: %s", err) continue } if isDeleted { - fileIDs = append(fileIDs, fileStat.FileId) + result = append(result, fileID) } } - return fileIDs, nil + return result, nil } func (s *service) getAllExistingFileBlocksCids(hash string) (totalSize uint64, cids []cid.Cid, err error) { @@ -144,6 +138,7 @@ func (s *service) getAllExistingFileBlocksCids(hash string) (totalSize uint64, c // here we can be sure that the block is loaded to the blockstore, so 1s should be more than enough ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx = context.WithValue(ctx, filestorage.CtxKeyRemoteLoadDisabled, true) n, err := s.commonFile.DAGService().Get(ctx, c) if err != nil { log.Errorf("GetAllExistingFileBlocksCids: failed to get links: %s", err.Error()) diff --git a/core/files/options.go b/core/files/options.go index 18a117274e..97c7bf3987 100644 --- a/core/files/options.go +++ b/core/files/options.go @@ -16,11 +16,12 @@ import ( type AddOption func(*AddOptions) type AddOptions struct { - Reader io.ReadSeeker - Use string - Media string - Name string - Plaintext bool + Reader io.ReadSeeker + Use string + Media string + Name string + LastModifiedDate int64 + Plaintext bool } func WithReader(r io.ReadSeeker) AddOption { @@ -35,6 +36,12 @@ func WithName(name string) AddOption { } } +func WithLastModifiedDate(timestamp int64) AddOption { + return func(args *AddOptions) { + args.LastModifiedDate = timestamp + } +} + func (s *service) normalizeOptions(ctx context.Context, opts *AddOptions) error { if opts.Use != "" { ref, err := ipfspath.ParsePath(opts.Use) diff --git a/core/filestorage/filesync/filesync.go b/core/filestorage/filesync/filesync.go index 9772d04ee4..4e55d9207b 100644 --- a/core/filestorage/filesync/filesync.go +++ b/core/filestorage/filesync/filesync.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "net/http" "sync" "time" @@ -30,6 +31,7 @@ var errReachedLimit = fmt.Errorf("file upload limit has been reached") //go:generate mockgen -package mock_filesync -destination ./mock_filesync/filesync_mock.go github.com/anyproto/anytype-heart/core/filestorage/filesync FileSync type FileSync interface { AddFile(spaceId, fileId string, uploadedByUser bool) (err error) + OnUpload(func(spaceID, fileID string) error) RemoveFile(spaceId, fileId string) (err error) SpaceStat(ctx context.Context, spaceId string) (ss SpaceStat, err error) FileStat(ctx context.Context, spaceId, fileId string) (fs FileStat, err error) @@ -38,9 +40,16 @@ type FileSync interface { FetchChunksCount(ctx context.Context, node ipld.Node) (int, error) HasUpload(spaceId, fileId string) (ok bool, err error) IsFileUploadLimited(spaceId, fileId string) (ok bool, err error) + DebugQueue(*http.Request) (*QueueInfo, error) app.ComponentRunnable } +type QueueInfo struct { + UploadingQueue []*QueueItem + DiscardedQueue []*QueueItem + RemovingQueue []*QueueItem +} + type SyncStatus struct { QueueLen int } @@ -56,6 +65,7 @@ type fileSync struct { dagService ipld.DAGService fileStore filestore.FileStore sendEvent func(event *pb.Event) + onUpload func(spaceID, fileID string) error spaceStatsLock sync.Mutex spaceStats map[string]SpaceStat @@ -78,6 +88,10 @@ func (f *fileSync) Init(a *app.App) (err error) { return } +func (f *fileSync) OnUpload(callback func(spaceID, fileID string) error) { + f.onUpload = callback +} + func (f *fileSync) Name() (name string) { return CName } diff --git a/core/filestorage/filesync/filesync_test.go b/core/filestorage/filesync/filesync_test.go index f7c38201d6..01dcbc7a10 100644 --- a/core/filestorage/filesync/filesync_test.go +++ b/core/filestorage/filesync/filesync_test.go @@ -61,7 +61,7 @@ func TestFileSync_AddFile(t *testing.T) { }) return res, nil }) - fx.rpcStore.EXPECT().BindCids(gomock.Any(), spaceId, fileId, gomock.Any()).Return(nil) + // fx.rpcStore.EXPECT().BindCids(gomock.Any(), spaceId, fileId, gomock.Any()).Return(nil) fx.rpcStore.EXPECT().SpaceInfo(gomock.Any(), spaceId).Return(&fileproto.SpaceInfoResponse{LimitBytes: 2 * 1024 * 1024}, nil).AnyTimes() fx.rpcStore.EXPECT().AddToFile(gomock.Any(), spaceId, fileId, gomock.Any()).AnyTimes() require.NoError(t, fx.AddFile(spaceId, fileId, false)) @@ -239,6 +239,10 @@ func (b *badgerProvider) LocalstoreDS() (datastore.DSTxnBatching, error) { return nil, nil } +func (b *badgerProvider) LocalstoreBadger() (*badger.DB, error) { + return b.db, nil +} + func (b *badgerProvider) SpaceStorage() (*badger.DB, error) { return b.db, nil } diff --git a/core/filestorage/filesync/filesyncstore.go b/core/filestorage/filesync/filesyncstore.go index af6d8e5dab..d988720cf8 100644 --- a/core/filestorage/filesync/filesyncstore.go +++ b/core/filestorage/filesync/filesyncstore.go @@ -43,20 +43,14 @@ func newFileSyncStore(db *badger.DB) (*fileSyncStore, error) { return s, nil } -type queueItem struct { +type QueueItem struct { SpaceID string FileID string Timestamp int64 AddedByUser bool } -func (it *queueItem) less(other *queueItem) bool { - if it.AddedByUser && !other.AddedByUser { - return true - } - if !it.AddedByUser && other.AddedByUser { - return false - } +func (it *QueueItem) less(other *QueueItem) bool { return it.Timestamp < other.Timestamp } @@ -122,7 +116,7 @@ func migrateItem(txn *badger.Txn, item *badger.Item) error { if err != nil { return fmt.Errorf("get timestamp: %w", err) } - it := queueItem{ + it := QueueItem{ Timestamp: int64(timestamp), } raw, err := json.Marshal(it) @@ -146,14 +140,24 @@ func versionFromItem(it *badger.Item) (int, error) { func (s *fileSyncStore) QueueUpload(spaceId string, fileId string, addedByUser bool) (err error) { return s.db.Update(func(txn *badger.Txn) error { + logger := log.With(zap.String("fileID", fileId), zap.String("addedByUser", strconv.FormatBool(addedByUser))) ok, err := isKeyExists(txn, discardedKey(spaceId, fileId)) if err != nil { - return err + return fmt.Errorf("check discarded key: %w", err) } if ok { + logger.Info("add file to upload queue: file is in discarded queue") return nil } - + ok, err = isKeyExists(txn, uploadKey(spaceId, fileId)) + if err != nil { + return fmt.Errorf("check upload key: %w", err) + } + if ok { + logger.Info("add file to upload queue: file is already in queue, update timestamp") + } else { + logger.Info("add file to upload queue") + } raw, err := createQueueItem(addedByUser) if err != nil { return fmt.Errorf("create queue item: %w", err) @@ -163,7 +167,7 @@ func (s *fileSyncStore) QueueUpload(spaceId string, fileId string, addedByUser b } func createQueueItem(addedByUser bool) ([]byte, error) { - return json.Marshal(queueItem{ + return json.Marshal(QueueItem{ Timestamp: time.Now().UnixMilli(), AddedByUser: addedByUser, }) @@ -226,11 +230,11 @@ func (s *fileSyncStore) DoneRemove(spaceId, fileId string) (err error) { }) } -func (s *fileSyncStore) GetUpload() (it *queueItem, err error) { +func (s *fileSyncStore) GetUpload() (it *QueueItem, err error) { return s.getOne(uploadKeyPrefix) } -func (s *fileSyncStore) GetDiscardedUpload() (it *queueItem, err error) { +func (s *fileSyncStore) GetDiscardedUpload() (it *QueueItem, err error) { return s.getOne(discardedKeyPrefix) } @@ -280,12 +284,12 @@ func (s *fileSyncStore) IsFileUploadLimited(spaceId, fileId string) (ok bool, er return } -func (s *fileSyncStore) GetRemove() (it *queueItem, err error) { +func (s *fileSyncStore) GetRemove() (it *QueueItem, err error) { return s.getOne(removeKeyPrefix) } // getOne returns the oldest key from the queue with given prefix -func (s *fileSyncStore) getOne(prefix []byte) (earliest *queueItem, err error) { +func (s *fileSyncStore) getOne(prefix []byte) (earliest *QueueItem, err error) { err = s.db.View(func(txn *badger.Txn) error { it := txn.NewIterator(badger.IteratorOptions{ PrefetchSize: 100, @@ -318,6 +322,35 @@ func (s *fileSyncStore) getOne(prefix []byte) (earliest *queueItem, err error) { return } +func (s *fileSyncStore) listItemsByPrefix(prefix []byte) ([]*QueueItem, error) { + var items []*QueueItem + err := s.db.View(func(txn *badger.Txn) error { + it := txn.NewIterator(badger.IteratorOptions{ + PrefetchSize: 100, + PrefetchValues: true, + Prefix: prefix, + }) + defer it.Close() + + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() + qItem, err := getQueueItem(item) + if err != nil { + return fmt.Errorf("get queue item %s: %w", item.Key(), err) + } + fileId, spaceId := extractFileAndSpaceID(item) + qItem.FileID = fileId + qItem.SpaceID = spaceId + items = append(items, qItem) + } + return nil + }) + if err != nil { + return nil, err + } + return items, nil +} + func extractFileAndSpaceID(item *badger.Item) (string, string) { k := item.Key() idx := bytes.LastIndexByte(k, sepByte) @@ -393,8 +426,8 @@ func getTimestamp(item *badger.Item) (uint64, error) { return ts, err } -func getQueueItem(item *badger.Item) (*queueItem, error) { - var it queueItem +func getQueueItem(item *badger.Item) (*QueueItem, error) { + var it QueueItem err := item.Value(func(raw []byte) error { return json.Unmarshal(raw, &it) }) diff --git a/core/filestorage/filesync/filesyncstore_test.go b/core/filestorage/filesync/filesyncstore_test.go index ee11ac00d0..4b158afb0b 100644 --- a/core/filestorage/filesync/filesyncstore_test.go +++ b/core/filestorage/filesync/filesyncstore_test.go @@ -94,7 +94,7 @@ func TestFileSyncStore_PushBackToQueue(t *testing.T) { assert.False(t, it.AddedByUser) } -func TestFileSyncStore_PrioritizeAddedByUser(t *testing.T) { +func TestFileSyncStore_CheckSorting(t *testing.T) { fx := newStoreFixture(t) defer fx.Finish() @@ -105,8 +105,8 @@ func TestFileSyncStore_PrioritizeAddedByUser(t *testing.T) { it, err := fx.GetUpload() require.NoError(t, err) assert.Equal(t, "spaceId1", it.SpaceID) - assert.Equal(t, "fileId2", it.FileID) - assert.True(t, it.AddedByUser) + assert.Equal(t, "fileId1", it.FileID) + assert.False(t, it.AddedByUser) } func TestFileSyncStore_IsDone(t *testing.T) { @@ -126,19 +126,19 @@ func TestMigration(t *testing.T) { fx := newStoreFixture(t) defer fx.Finish() - wantUploadItem := &queueItem{ + wantUploadItem := &QueueItem{ SpaceID: "spaceId1", FileID: "fileId1", Timestamp: time.Now().UnixMilli(), AddedByUser: false, } - wantDiscardedItem := &queueItem{ + wantDiscardedItem := &QueueItem{ SpaceID: "spaceId1", FileID: "fileId2", Timestamp: time.Now().UnixMilli(), AddedByUser: false, } - wantRemoveItem := &queueItem{ + wantRemoveItem := &QueueItem{ SpaceID: "spaceId1", FileID: "fileId3", Timestamp: time.Now().UnixMilli(), diff --git a/core/filestorage/filesync/remove.go b/core/filestorage/filesync/remove.go index cb3e5261f8..240f36192c 100644 --- a/core/filestorage/filesync/remove.go +++ b/core/filestorage/filesync/remove.go @@ -45,6 +45,7 @@ func (f *fileSync) removeOperation() { log.Warn("can't remove file", zap.String("fileID", fileID), zap.Error(err)) return } + log.Warn("file removed", zap.String("fileID", fileID)) } } diff --git a/core/filestorage/filesync/stats.go b/core/filestorage/filesync/stats.go index 4f86fc3c64..fd46622001 100644 --- a/core/filestorage/filesync/stats.go +++ b/core/filestorage/filesync/stats.go @@ -3,6 +3,7 @@ package filesync import ( "context" "fmt" + "net/http" "github.com/anyproto/any-sync/commonfile/fileproto" "github.com/ipfs/go-cid" @@ -174,3 +175,24 @@ func (f *fileSync) FetchChunksCount(ctx context.Context, node ipld.Node) (int, e } return count, err } + +func (f *fileSync) DebugQueue(_ *http.Request) (*QueueInfo, error) { + var ( + info QueueInfo + err error + ) + + info.UploadingQueue, err = f.queue.listItemsByPrefix(uploadKeyPrefix) + if err != nil { + return nil, fmt.Errorf("list items from uploading queue: %w", err) + } + info.DiscardedQueue, err = f.queue.listItemsByPrefix(discardedKeyPrefix) + if err != nil { + return nil, fmt.Errorf("list items from discarded queue: %w", err) + } + info.RemovingQueue, err = f.queue.listItemsByPrefix(removeKeyPrefix) + if err != nil { + return nil, fmt.Errorf("list items from removing queue: %w", err) + } + return &info, nil +} diff --git a/core/filestorage/filesync/upload.go b/core/filestorage/filesync/upload.go index 79a1d9926e..6e41258767 100644 --- a/core/filestorage/filesync/upload.go +++ b/core/filestorage/filesync/upload.go @@ -37,7 +37,6 @@ func (f *fileSync) AddFile(spaceId, fileId string, uploadedByUser bool) (err err log.Warn("file has been deleted from store, skip upload", zap.String("fileId", fileId)) return nil } - log.Info("add file to uploading queue", zap.String("fileID", fileId)) err = f.queue.QueueUpload(spaceId, fileId, uploadedByUser) if err == nil { @@ -75,7 +74,7 @@ func (f *fileSync) addOperation() { } } -func (f *fileSync) getUpload() (*queueItem, error) { +func (f *fileSync) getUpload() (*QueueItem, error) { it, err := f.queue.GetUpload() if err == errQueueIsEmpty { return f.queue.GetDiscardedUpload() @@ -115,6 +114,15 @@ func (f *fileSync) tryToUpload() (string, error) { return fileId, err } log.Info("done upload", zap.String("fileID", fileId)) + if f.onUpload != nil { + err := f.onUpload(spaceId, fileId) + if err != nil { + log.Warn("on upload callback failed", + zap.String("fileID", fileId), + zap.String("spaceID", spaceId), + zap.Error(err)) + } + } f.updateSpaceUsageInformation(spaceId) @@ -145,6 +153,12 @@ func (f *fileSync) uploadFile(ctx context.Context, spaceId, fileId string) (err return err } + if len(blocksToUpload) == 0 { + return nil + } + + log.Info("start uploading file", zap.String("fileID", fileId), zap.Int("blocksCount", len(blocksToUpload))) + go func() { defer func() { _ = batcher.Close() @@ -188,11 +202,13 @@ func (f *fileSync) prepareToUpload(ctx context.Context, spaceId string, fileId s return nil, fmt.Errorf("select blocks to upload: %w", err) } - log.Debug("collecting blocks to upload", - zap.String("fileID", fileId), - zap.Int("blocksToUpload", len(blocksToUpload)), - zap.Int("totalBlocks", len(fileBlocks)), - ) + if len(blocksToUpload) > 0 { + log.Info("collecting blocks to upload", + zap.String("fileID", fileId), + zap.Int("blocksToUpload", len(blocksToUpload)), + zap.Int("totalBlocks", len(fileBlocks)), + ) + } stat, err := f.SpaceStat(ctx, spaceId) if err != nil { @@ -265,10 +281,11 @@ func (f *fileSync) selectBlocksToUploadAndBindExisting(ctx context.Context, spac } } - if bindErr := f.rpcStore.BindCids(ctx, spaceId, fileId, cidsToBind); bindErr != nil { - return 0, nil, fmt.Errorf("bind cids: %w", bindErr) + if len(cidsToBind) > 0 { + if bindErr := f.rpcStore.BindCids(ctx, spaceId, fileId, cidsToBind); bindErr != nil { + return 0, nil, fmt.Errorf("bind cids: %w", bindErr) + } } - return bytesToUpload, blocksToUpload, nil } diff --git a/core/filestorage/rpcstore/store.go b/core/filestorage/rpcstore/store.go index 4c2dcfdd00..6bcd75a3c0 100644 --- a/core/filestorage/rpcstore/store.go +++ b/core/filestorage/rpcstore/store.go @@ -52,13 +52,8 @@ func (s *store) Get(ctx context.Context, k cid.Cid) (b blocks.Block, err error) }, k); err != nil { return } - select { - case res := <-ready: - if res.err != nil { - return nil, res.err - } - case <-ctx.Done(): - return nil, ctx.Err() + if err := waitResult(ctx, ready); err != nil { + return nil, err } return blocks.NewBlockWithCid(data, k) } @@ -158,47 +153,17 @@ func (s *store) AddToFile(ctx context.Context, spaceID string, fileID string, bs return nil } -func (s *store) CheckAvailability(ctx context.Context, spaceID string, cids []cid.Cid) (checkResult []*fileproto.BlockAvailability, err error) { - var ready = make(chan result, 1) - // check blocks availability - ctx = context.WithValue(ctx, operationNameKey, "checkAvailability") - if err = s.cm.WriteOp(ctx, ready, func(c *client) (err error) { - checkResult, err = c.checkBlocksAvailability(ctx, spaceID, cids...) - return err - }, cid.Cid{}); err != nil { - return - } - // wait availability result - select { - case <-ctx.Done(): - return nil, ctx.Err() - case res := <-ready: - if res.err != nil { - return checkResult, err - } - } - return +func (s *store) CheckAvailability(ctx context.Context, spaceID string, cids []cid.Cid) ([]*fileproto.BlockAvailability, error) { + return writeOperation(ctx, s, "checkAvailability", func(c *client) ([]*fileproto.BlockAvailability, error) { + return c.checkBlocksAvailability(ctx, spaceID, cids...) + }) } -func (s *store) BindCids(ctx context.Context, spaceID string, fileID string, cids []cid.Cid) (err error) { - var ready = make(chan result, 1) - // check blocks availability - ctx = context.WithValue(ctx, operationNameKey, "bindCids") - if err = s.cm.WriteOp(ctx, ready, func(c *client) (err error) { - return c.bind(ctx, spaceID, fileID, cids...) - }, cid.Cid{}); err != nil { - return - } - // wait availability result - select { - case <-ctx.Done(): - return ctx.Err() - case res := <-ready: - if res.err != nil { - return res.err - } - } - return nil +func (s *store) BindCids(ctx context.Context, spaceID string, fileID string, cids []cid.Cid) error { + _, err := writeOperation(ctx, s, "bindCids", func(c *client) (interface{}, error) { + return nil, c.bind(ctx, spaceID, fileID, cids...) + }) + return err } func (s *store) Delete(ctx context.Context, c cid.Cid) error { @@ -206,60 +171,53 @@ func (s *store) Delete(ctx context.Context, c cid.Cid) error { } func (s *store) DeleteFiles(ctx context.Context, spaceId string, fileIds ...string) error { - var ready = make(chan result, 1) - ctx = context.WithValue(ctx, operationNameKey, "deleteFiles") - if err := s.cm.WriteOp(ctx, ready, func(c *client) error { - return c.delete(ctx, spaceId, fileIds...) - }, cid.Cid{}); err != nil { - return err - } - select { - case <-ctx.Done(): - return ctx.Err() - case res := <-ready: - return res.err - } + _, err := writeOperation(ctx, s, "deleteFiles", func(c *client) (interface{}, error) { + return nil, c.delete(ctx, spaceId, fileIds...) + }) + return err +} + +func (s *store) SpaceInfo(ctx context.Context, spaceId string) (*fileproto.SpaceInfoResponse, error) { + return writeOperation(ctx, s, "spaceInfo", func(c *client) (*fileproto.SpaceInfoResponse, error) { + return c.spaceInfo(ctx, spaceId) + }) +} + +func (s *store) FilesInfo(ctx context.Context, spaceId string, fileIds ...string) ([]*fileproto.FileInfo, error) { + return writeOperation(ctx, s, "filesInfo", func(c *client) ([]*fileproto.FileInfo, error) { + return c.filesInfo(ctx, spaceId, fileIds) + }) } -func (s *store) SpaceInfo(ctx context.Context, spaceId string) (info *fileproto.SpaceInfoResponse, err error) { - var ready = make(chan result, 1) - ctx = context.WithValue(ctx, operationNameKey, "spaceInfo") - if err = s.cm.WriteOp(ctx, ready, func(c *client) error { - info, err = c.spaceInfo(ctx, spaceId) - return err + +func (s *store) Close() (err error) { + return s.cm.Close() +} + +func writeOperation[T any](ctx context.Context, s *store, operationName string, fn func(c *client) (T, error)) (T, error) { + ready := make(chan result, 1) + ctx = context.WithValue(ctx, operationNameKey, operationName) + var res T + if err := s.cm.WriteOp(ctx, ready, func(c *client) error { + var opErr error + res, opErr = fn(c) + return opErr }, cid.Cid{}); err != nil { - return nil, err + return res, err } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case res := <-ready: - if res.err != nil { - return nil, res.err - } + if err := waitResult(ctx, ready); err != nil { + return res, err } - return + return res, nil } -func (s *store) FilesInfo(ctx context.Context, spaceId string, fileIds ...string) (info []*fileproto.FileInfo, err error) { - var ready = make(chan result, 1) - ctx = context.WithValue(ctx, operationNameKey, "filesInfo") - if err = s.cm.WriteOp(ctx, ready, func(c *client) error { - info, err = c.filesInfo(ctx, spaceId, fileIds) - return err - }, cid.Cid{}); err != nil { - return nil, err - } +func waitResult(ctx context.Context, ready chan result) error { select { case <-ctx.Done(): - return nil, ctx.Err() + return ctx.Err() case res := <-ready: if res.err != nil { - return nil, res.err + return res.err } } - return -} - -func (s *store) Close() (err error) { - return s.cm.Close() + return nil } diff --git a/core/indexer/full_text.go b/core/indexer/full_text.go index d0fc3a0f68..6fe07af148 100644 --- a/core/indexer/full_text.go +++ b/core/indexer/full_text.go @@ -2,9 +2,9 @@ package indexer import ( "context" + "fmt" "time" - "github.com/anyproto/anytype-heart/core/block" "github.com/anyproto/anytype-heart/metrics" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" @@ -41,6 +41,7 @@ func (i *indexer) ftLoop() { } } +// TODO maybe use two queues? One for objects, one for files func (i *indexer) runFullTextIndexer() { ids, err := i.store.ListIDsFromFullTextQueue() if err != nil { @@ -69,14 +70,12 @@ func (i *indexer) runFullTextIndexer() { func (i *indexer) prepareSearchDocument(id string) (ftDoc ftsearch.SearchDoc, err error) { // ctx := context.WithValue(context.Background(), ocache.CacheTimeout, cacheTimeout) - ctx := context.WithValue(context.Background(), metrics.CtxKeyRequest, "index_fulltext") + ctx := context.WithValue(context.Background(), metrics.CtxKeyEntrypoint, "index_fulltext") - ctx = block.CacheOptsWithRemoteLoadDisabled(ctx) info, err := i.getObjectInfo(ctx, id) if err != nil { - return + return ftDoc, fmt.Errorf("get object info: %w", err) } - sbType, err := i.typeProvider.Type(info.Id) if err != nil { sbType = smartblock.SmartBlockTypePage diff --git a/core/indexer/indexer.go b/core/indexer/indexer.go index c3a37598aa..3c38bea1ba 100644 --- a/core/indexer/indexer.go +++ b/core/indexer/indexer.go @@ -10,8 +10,9 @@ import ( "time" "github.com/anyproto/any-sync/app" + "github.com/dgraph-io/badger/v3" "github.com/gogo/protobuf/types" - ds "github.com/ipfs/go-datastore" + "go.uber.org/zap" "golang.org/x/exp/slices" "github.com/anyproto/anytype-heart/core/anytype/config" @@ -50,7 +51,7 @@ const ( ForceBundledObjectsReindexCounter int32 = 5 // reindex objects like anytypeProfile // ForceIdxRebuildCounter erases localstore indexes and reindex all type of objects // (no need to increase ForceThreadsObjectsReindexCounter & ForceFilesReindexCounter) - ForceIdxRebuildCounter int32 = 43 + ForceIdxRebuildCounter int32 = 45 // ForceFulltextIndexCounter performs fulltext indexing for all type of objects (useful when we change fulltext config) ForceFulltextIndexCounter int32 = 5 // ForceFilestoreKeysReindexCounter reindex filestore keys in all objects @@ -115,7 +116,8 @@ type indexer struct { typeProvider typeprovider.SmartBlockTypeProvider spaceService space.Service - indexedFiles *sync.Map + indexedFiles *sync.Map + reindexLogFields []zap.Field } func (i *indexer) Init(a *app.App) (err error) { @@ -329,7 +331,7 @@ func (i *indexer) indexLinkedFiles(ctx context.Context, fileHashes []string) { func (i *indexer) reindexIfNeeded() error { checksums, err := i.store.GetChecksums() - if err != nil && err != ds.ErrNotFound { + if err != nil && !errors.Is(err, badger.ErrKeyNotFound) { return err } if checksums == nil { @@ -370,7 +372,8 @@ func (i *indexer) reindexIfNeeded() error { if checksums.IdxRebuildCounter != ForceIdxRebuildCounter { flags.enableAll() } - return i.reindex(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "reindex_forced"), flags) + + return i.reindex(context.WithValue(context.TODO(), metrics.CtxKeyEntrypoint, "reindex_forced"), flags) } func (i *indexer) reindex(ctx context.Context, flags reindexFlags) (err error) { @@ -378,6 +381,15 @@ func (i *indexer) reindex(ctx context.Context, flags reindexFlags) (err error) { log.Infof("start store reindex (%s)", flags.String()) } + ctx = block.CacheOptsWithRemoteLoadDisabled(ctx) + if flags.threadObjects && flags.fileObjects { + // files will be indexed within object indexing (see indexLinkedFiles) + // because we need to do it in the background. + // otherwise it will lead to the situation when files loading called from the reindex with DisableRemoteFlag + // will be waiting for the linkedFiles background indexing without this flag + flags.fileObjects = false + } + if flags.fileKeys { err = i.fileStore.RemoveEmptyFileKeys() if err != nil { @@ -437,15 +449,9 @@ func (i *indexer) reindex(ctx context.Context, flags reindexFlags) (err error) { } start := time.Now() successfullyReindexed := i.reindexIdsIgnoreErr(ctx, ids...) - if metrics.Enabled { - metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ - ReindexType: metrics.ReindexTypeThreads, - Total: len(ids), - Success: successfullyReindexed, - SpentMs: int(time.Since(start).Milliseconds()), - IndexesRemoved: indexesWereRemoved, - }) - } + + i.logFinishedReindexStat(metrics.ReindexTypeThreads, len(ids), successfullyReindexed, time.Since(start)) + log.Infof("%d/%d objects have been successfully reindexed", successfullyReindexed, len(ids)) } else { go func() { @@ -456,14 +462,8 @@ func (i *indexer) reindex(ctx context.Context, flags reindexFlags) (err error) { } else { log.Infof("%d/%d outdated objects have been successfully reindexed", success, total) } - if metrics.Enabled && total > 0 { - metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ - ReindexType: metrics.ReindexTypeOutdatedHeads, - Total: total, - Success: success, - SpentMs: int(time.Since(start).Milliseconds()), - IndexesRemoved: indexesWereRemoved, - }) + if total > 0 { + i.logFinishedReindexStat(metrics.ReindexTypeOutdatedHeads, total, success, time.Since(start)) } }() } @@ -551,27 +551,14 @@ func (i *indexer) reindexIDsForSmartblockTypes(ctx context.Context, reindexType func (i *indexer) reindexIDs(ctx context.Context, reindexType metrics.ReindexType, indexesWereRemoved bool, ids []string) error { start := time.Now() successfullyReindexed := i.reindexIdsIgnoreErr(ctx, ids...) - if metrics.Enabled && len(ids) > 0 { - metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ - ReindexType: reindexType, - Total: len(ids), - Success: successfullyReindexed, - SpentMs: int(time.Since(start).Milliseconds()), - IndexesRemoved: indexesWereRemoved, - }) - } - msg := fmt.Sprintf("%d/%d %s have been successfully reindexed", successfullyReindexed, len(ids), reindexType) - if len(ids)-successfullyReindexed != 0 { - log.Error(msg) - } else { - log.Info(msg) - } + i.logFinishedReindexStat(reindexType, len(ids), successfullyReindexed, time.Since(start)) return nil } func (i *indexer) ensurePreinstalledObjects() error { var objects []*types.Struct + start := time.Now() for _, ot := range bundle.SystemTypes { t, err := bundle.GetTypeByUrl(ot.BundledURL()) if err != nil { @@ -588,8 +575,9 @@ func (i *indexer) ensurePreinstalledObjects() error { } objects = append(objects, (&relationutils.Relation{Relation: rel}).ToStruct()) } + ids, _, err := i.subObjectCreator.CreateSubObjectsInWorkspace(objects) + i.logFinishedReindexStat(metrics.ReindexTypeSystem, len(ids), len(ids), time.Since(start)) - _, _, err := i.subObjectCreator.CreateSubObjectsInWorkspace(objects) if errors.Is(err, editor.ErrSubObjectAlreadyExists) { return nil } @@ -652,7 +640,7 @@ func (i *indexer) reindexOutdatedThreads() (toReindex, success int, err error) { } } - ctx := context.WithValue(context.Background(), metrics.CtxKeyRequest, "reindexOutdatedThreads") + ctx := context.WithValue(context.Background(), metrics.CtxKeyEntrypoint, "reindexOutdatedThreads") success = i.reindexIdsIgnoreErr(ctx, idsToReindex...) return len(idsToReindex), success, nil } @@ -679,7 +667,6 @@ func (i *indexer) reindexDoc(ctx context.Context, id string) error { } func (i *indexer) reindexIdsIgnoreErr(ctx context.Context, ids ...string) (successfullyReindexed int) { - ctx = block.CacheOptsWithRemoteLoadDisabled(ctx) for _, id := range ids { err := i.reindexDoc(ctx, id) if err != nil { @@ -724,3 +711,30 @@ func headsHash(heads []string) string { sum := sha256.Sum256([]byte(strings.Join(heads, ","))) return fmt.Sprintf("%x", sum) } + +func (i *indexer) GetLogFields() []zap.Field { + return i.reindexLogFields +} + +func (i *indexer) logFinishedReindexStat(reindexType metrics.ReindexType, totalIds, succeedIds int, spent time.Duration) { + i.reindexLogFields = append(i.reindexLogFields, zap.Int("r_"+reindexType.String(), totalIds)) + if succeedIds < totalIds { + i.reindexLogFields = append(i.reindexLogFields, zap.Int("r_"+reindexType.String()+"_failed", totalIds-succeedIds)) + } + i.reindexLogFields = append(i.reindexLogFields, zap.Int64("r_"+reindexType.String()+"_spent", spent.Milliseconds())) + msg := fmt.Sprintf("%d/%d %s have been successfully reindexed", succeedIds, totalIds, reindexType) + if totalIds-succeedIds != 0 { + log.Error(msg) + } else { + log.Info(msg) + } + + if metrics.Enabled { + metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ + ReindexType: reindexType, + Total: totalIds, + Success: succeedIds, + SpentMs: int(spent.Milliseconds()), + }) + } +} diff --git a/core/kanban/group_tag.go b/core/kanban/group_tag.go index 564436452a..d0c993ddaf 100644 --- a/core/kanban/group_tag.go +++ b/core/kanban/group_tag.go @@ -41,7 +41,7 @@ func (t *GroupTag) InitGroups(f *database.Filters) error { }, }} - records, err := t.store.QueryRaw(f) + records, err := t.store.QueryRaw(f, 0, 0) if err != nil { return fmt.Errorf("init kanban by tag, objectStore query error: %v", err) } diff --git a/core/object.go b/core/object.go index cf41ca5964..ef3e744dd9 100644 --- a/core/object.go +++ b/core/object.go @@ -779,10 +779,10 @@ func (mw *Middleware) ObjectImport(cctx context.Context, req *pb.RpcObjectImport return response(pb.RpcObjectImportResponseError_NULL, nil) } - switch err { - case converter.ErrNoObjectsToImport: + switch { + case errors.Is(err, converter.ErrNoObjectsToImport): return response(pb.RpcObjectImportResponseError_NO_OBJECTS_TO_IMPORT, err) - case converter.ErrCancel: + case errors.Is(err, converter.ErrCancel): return response(pb.RpcObjectImportResponseError_IMPORT_IS_CANCELED, err) default: return response(pb.RpcObjectImportResponseError_INTERNAL_ERROR, err) diff --git a/core/subscription/service.go b/core/subscription/service.go index 2e9d88f3e8..8c1fcf4a94 100644 --- a/core/subscription/service.go +++ b/core/subscription/service.go @@ -159,7 +159,7 @@ func (s *service) subscribeForQuery(req pb.RpcObjectSearchSubscribeRequest, f *d sub.forceSubIds = filterDepIds } - records, err := s.objectStore.QueryRaw(f) + records, err := s.objectStore.QueryRaw(f, 0, 0) if err != nil { return nil, fmt.Errorf("objectStore query error: %v", err) } diff --git a/core/subscription/service_test.go b/core/subscription/service_test.go index 6d5294c393..101b2c84cc 100644 --- a/core/subscription/service_test.go +++ b/core/subscription/service_test.go @@ -22,7 +22,7 @@ import ( func TestService_Search(t *testing.T) { var newSub = func(fx *fixture, subId string) { - fx.store.EXPECT().QueryRaw(gomock.Any()).Return( + fx.store.EXPECT().QueryRaw(gomock.Any(), 0, 0).Return( []database.Record{ {Details: &types.Struct{Fields: map[string]*types.Value{ "id": pbtypes.String("1"), @@ -126,7 +126,7 @@ func TestService_Search(t *testing.T) { defer fx.a.Close(context.Background()) defer fx.ctrl.Finish() - fx.store.EXPECT().QueryRaw(gomock.Any()).Return( + fx.store.EXPECT().QueryRaw(gomock.Any(), 0, 0).Return( []database.Record{ {Details: &types.Struct{Fields: map[string]*types.Value{ "id": pbtypes.String("1"), @@ -177,7 +177,7 @@ func TestService_Search(t *testing.T) { defer fx.a.Close(context.Background()) defer fx.ctrl.Finish() - fx.store.EXPECT().QueryRaw(gomock.Any()).Return( + fx.store.EXPECT().QueryRaw(gomock.Any(), 0, 0).Return( []database.Record{ {Details: &types.Struct{Fields: map[string]*types.Value{ "id": pbtypes.String("1"), @@ -244,7 +244,7 @@ func TestService_Search(t *testing.T) { defer fx.a.Close(context.Background()) defer fx.ctrl.Finish() - fx.store.EXPECT().QueryRaw(gomock.Any()).Return( + fx.store.EXPECT().QueryRaw(gomock.Any(), 0, 0).Return( []database.Record{ {Details: &types.Struct{Fields: map[string]*types.Value{ "id": pbtypes.String("1"), diff --git a/core/syncstatus/debug.go b/core/syncstatus/debug.go new file mode 100644 index 0000000000..c61819d875 --- /dev/null +++ b/core/syncstatus/debug.go @@ -0,0 +1,30 @@ +package syncstatus + +import ( + "net/http" + + "github.com/go-chi/chi/v5" + + "github.com/anyproto/anytype-heart/util/debug" +) + +func (s *service) DebugRouter(r chi.Router) { + r.Get("/file_watchers", debug.JSONHandler(s.listFileWatchers)) +} + +type fileWatcherDebugInfo struct { + ID string + IsLimited bool +} + +func (s *service) listFileWatchers(_ *http.Request) ([]*fileWatcherDebugInfo, error) { + files := s.fileWatcher.list() + result := make([]*fileWatcherDebugInfo, 0, len(files)) + for _, file := range files { + result = append(result, &fileWatcherDebugInfo{ + ID: file.fileID, + IsLimited: file.isUploadLimited, + }) + } + return result, nil +} diff --git a/core/syncstatus/file_status_registry.go b/core/syncstatus/file_status_registry.go index 67907a868b..978a02ce69 100644 --- a/core/syncstatus/file_status_registry.go +++ b/core/syncstatus/file_status_registry.go @@ -53,6 +53,14 @@ func newFileStatusRegistry( } } +func (r *fileStatusRegistry) hasFileInStore(fileID string) (bool, error) { + roots, err := r.fileStore.ListByTarget(fileID) + if err != localstore.ErrNotFound && err != nil { + return false, err + } + return len(roots) > 0, nil +} + func (r *fileStatusRegistry) GetFileStatus(ctx context.Context, spaceID string, fileID string) (FileStatus, error) { key := fileWithSpace{ spaceID: spaceID, @@ -119,6 +127,8 @@ func validStatusTransition(from, to FileStatus) bool { } } +var errFileNotFound = fmt.Errorf("file is not found") + func (r *fileStatusRegistry) updateFileStatus(ctx context.Context, status fileStatus, key fileWithSpace) (fileStatus, error) { now := time.Now() if status.status == FileStatusSynced { @@ -148,6 +158,13 @@ func (r *fileStatusRegistry) updateFileStatus(ctx context.Context, status fileSt return status, nil } + ok, err := r.hasFileInStore(key.fileID) + if err != nil { + return status, fmt.Errorf("check that file is in store: %w", err) + } + if !ok { + return status, errFileNotFound + } fstat, err := r.fileSyncService.FileStat(ctx, key.spaceID, key.fileID) if err != nil { return status, fmt.Errorf("file stat: %w", err) diff --git a/core/syncstatus/file_watcher.go b/core/syncstatus/file_watcher.go index 580e5c0d32..5c5fc10573 100644 --- a/core/syncstatus/file_watcher.go +++ b/core/syncstatus/file_watcher.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "sort" "sync" "time" @@ -147,19 +148,32 @@ func (s *fileWatcher) close() { close(s.closeCh) } +func (s *fileWatcher) list() []fileWithSpace { + s.filesToWatchLock.Lock() + defer s.filesToWatchLock.Unlock() + + result := make([]fileWithSpace, 0, len(s.filesToWatch)) + for key := range s.filesToWatch { + result = append(result, key) + } + sort.Slice(result, func(i, j int) bool { + return result[i].fileID < result[j].fileID + }) + return result +} + func (s *fileWatcher) updateFileStatus(ctx context.Context, key fileWithSpace) error { status, err := s.registry.GetFileStatus(ctx, key.spaceID, key.fileID) + if err == errFileNotFound { + s.Unwatch(key.spaceID, key.fileID) + return err + } if err != nil { return fmt.Errorf("get file status: %w", err) } // Files are immutable, so we can stop watching status updates after file is synced if status == FileStatusSynced { - go func() { - err = s.Unwatch(key.spaceID, key.fileID) - if err != nil { - log.Error("unwatching file", zap.String("fileID", key.fileID), zap.Error(err)) - } - }() + s.Unwatch(key.spaceID, key.fileID) } if !key.isUploadLimited && status == FileStatusLimited { go s.moveToLimitedQueue(key) @@ -235,7 +249,16 @@ func (s *fileWatcher) Watch(spaceID, fileID string) error { return nil } -func (s *fileWatcher) Unwatch(spaceID, fileID string) error { +func (s *fileWatcher) Unwatch(spaceID, fileID string) { + go func() { + err := s.unwatch(spaceID, fileID) + if err != nil { + log.Error("unwatching file", zap.String("fileID", fileID), zap.Error(err)) + } + }() +} + +func (s *fileWatcher) unwatch(spaceID, fileID string) error { s.filesToWatchLock.Lock() defer s.filesToWatchLock.Unlock() diff --git a/core/syncstatus/linked_files_watcher.go b/core/syncstatus/linked_files_watcher.go index 952b9da1e1..39752ad4ed 100644 --- a/core/syncstatus/linked_files_watcher.go +++ b/core/syncstatus/linked_files_watcher.go @@ -11,13 +11,18 @@ import ( "github.com/anyproto/anytype-heart/space" ) +type linkedFilesSummary struct { + pinStatus pb.EventStatusThreadCafePinStatus + isUpdated bool +} + type linkedFilesWatcher struct { spaceService space.Service fileStatusRegistry *fileStatusRegistry sync.Mutex - linkedFilesSummary map[string]pb.EventStatusThreadCafePinStatus - linkedFilesCloseCh map[string]chan struct{} + linkedFilesSummaries map[string]linkedFilesSummary + linkedFilesCloseCh map[string]chan struct{} } func newLinkedFilesWatcher( @@ -25,10 +30,10 @@ func newLinkedFilesWatcher( fileStatusRegistry *fileStatusRegistry, ) *linkedFilesWatcher { return &linkedFilesWatcher{ - linkedFilesSummary: make(map[string]pb.EventStatusThreadCafePinStatus), - linkedFilesCloseCh: make(map[string]chan struct{}), - spaceService: spaceService, - fileStatusRegistry: fileStatusRegistry, + linkedFilesSummaries: make(map[string]linkedFilesSummary), + linkedFilesCloseCh: make(map[string]chan struct{}), + spaceService: spaceService, + fileStatusRegistry: fileStatusRegistry, } } @@ -42,10 +47,10 @@ func (w *linkedFilesWatcher) close() { } } -func (w *linkedFilesWatcher) GetLinkedFilesSummary(parentObjectID string) pb.EventStatusThreadCafePinStatus { +func (w *linkedFilesWatcher) GetLinkedFilesSummary(parentObjectID string) linkedFilesSummary { w.Lock() defer w.Unlock() - return w.linkedFilesSummary[parentObjectID] + return w.linkedFilesSummaries[parentObjectID] } func (w *linkedFilesWatcher) WatchLinkedFiles(parentObjectID string, filesGetter func() []string) { @@ -80,25 +85,32 @@ func (w *linkedFilesWatcher) updateLinkedFilesSummary(parentObjectID string, fil // TODO Cache linked files list? fileIDs := filesGetter() - var summary pb.EventStatusThreadCafePinStatus + var pinStatus pb.EventStatusThreadCafePinStatus for _, fileID := range fileIDs { status, err := w.fileStatusRegistry.GetFileStatus(context.Background(), w.spaceService.AccountId(), fileID) + if err == errFileNotFound { + continue + } if err != nil { log.Desugar().Error("can't get status of dependent file", zap.String("fileID", fileID), zap.Error(err)) } switch status { case FileStatusUnknown, FileStatusSyncing: - summary.Pinning++ + pinStatus.Pinning++ case FileStatusLimited: - summary.Failed++ + pinStatus.Failed++ case FileStatusSynced: - summary.Pinned++ + pinStatus.Pinned++ } } + updated := true w.Lock() - w.linkedFilesSummary[parentObjectID] = summary + if summary, exists := w.linkedFilesSummaries[parentObjectID]; exists { + updated = summary.pinStatus != pinStatus + } + w.linkedFilesSummaries[parentObjectID] = linkedFilesSummary{pinStatus: pinStatus, isUpdated: updated} w.Unlock() } diff --git a/core/syncstatus/service.go b/core/syncstatus/service.go index 2f490ab353..3c914db39f 100644 --- a/core/syncstatus/service.go +++ b/core/syncstatus/service.go @@ -29,6 +29,7 @@ const CName = "status" type Service interface { Watch(id string, fileFunc func() []string) (new bool, err error) Unwatch(id string) + OnFileUpload(spaceID string, fileID string) error app.ComponentRunnable } @@ -145,6 +146,14 @@ func (s *service) unwatch(id string) { } } +func (s *service) OnFileUpload(spaceID string, fileID string) error { + _, err := s.fileWatcher.registry.setFileStatus(fileWithSpace{spaceID: spaceID, fileID: fileID}, fileStatus{ + status: FileStatusSynced, + updatedAt: time.Now(), + }) + return err +} + func (s *service) Close(ctx context.Context) (err error) { s.unwatch(s.coreService.PredefinedBlocks().Account) s.fileWatcher.close() diff --git a/core/syncstatus/update_receiver.go b/core/syncstatus/update_receiver.go index 8f661ea0dc..b7a9c7312a 100644 --- a/core/syncstatus/update_receiver.go +++ b/core/syncstatus/update_receiver.go @@ -21,6 +21,7 @@ type updateReceiver struct { nodeConfService nodeconf.Service sync.Mutex nodeConnected bool + lastStatus map[string]pb.EventStatusThreadSyncStatus } func newUpdateReceiver( @@ -40,47 +41,53 @@ func newUpdateReceiver( subObjectsWatcher: subObjectsWatcher, nodeConfService: nodeConfService, emitter: emitter, + lastStatus: make(map[string]pb.EventStatusThreadSyncStatus), } } -func (r *updateReceiver) UpdateTree(ctx context.Context, objId string, status syncstatus.SyncStatus) (err error) { - var ( - nodeConnected bool - objStatus pb.EventStatusThreadSyncStatus - generalStatus pb.EventStatusThreadSyncStatus - ) +func (r *updateReceiver) UpdateTree(ctx context.Context, objId string, status syncstatus.SyncStatus) error { + filesSummary := r.linkedFilesWatcher.GetLinkedFilesSummary(objId) + objStatus := r.getObjectStatus(status) - nodeConnected = r.isNodeConnected() - linkedFilesSummary := r.linkedFilesWatcher.GetLinkedFilesSummary(objId) + if !r.isStatusUpdated(objId, objStatus, filesSummary) { + return nil + } + r.notify(objId, objStatus, filesSummary.pinStatus) - networkStatus := r.nodeConfService.NetworkCompatibilityStatus() - switch status { - case syncstatus.StatusUnknown: - objStatus = pb.EventStatusThread_Unknown - case syncstatus.StatusSynced: - objStatus = pb.EventStatusThread_Synced - case syncstatus.StatusNotSynced: - objStatus = pb.EventStatusThread_Syncing + if objId == r.coreService.PredefinedBlocks().Account { + r.subObjectsWatcher.ForEach(func(subObjectID string) { + r.notify(subObjectID, objStatus, filesSummary.pinStatus) + }) + } + return nil +} + +func (r *updateReceiver) isStatusUpdated(objectID string, objStatus pb.EventStatusThreadSyncStatus, filesSummary linkedFilesSummary) bool { + r.Lock() + defer r.Unlock() + if lastObjStatus, ok := r.lastStatus[objectID]; ok && objStatus == lastObjStatus && !filesSummary.isUpdated { + return false } + r.lastStatus[objectID] = objStatus + return true +} - switch networkStatus { - case nodeconf.NetworkCompatibilityStatusIncompatible: - objStatus = pb.EventStatusThread_IncompatibleVersion - default: - if !nodeConnected { - objStatus = pb.EventStatusThread_Offline - } +func (r *updateReceiver) getObjectStatus(status syncstatus.SyncStatus) pb.EventStatusThreadSyncStatus { + if r.nodeConfService.NetworkCompatibilityStatus() == nodeconf.NetworkCompatibilityStatusIncompatible { + return pb.EventStatusThread_IncompatibleVersion } - generalStatus = objStatus - r.notify(objId, objStatus, generalStatus, linkedFilesSummary) + if !r.isNodeConnected() { + return pb.EventStatusThread_Offline + } - if objId == r.coreService.PredefinedBlocks().Account { - r.subObjectsWatcher.ForEach(func(subObjectID string) { - r.notify(subObjectID, objStatus, generalStatus, linkedFilesSummary) - }) + switch status { + case syncstatus.StatusUnknown: + return pb.EventStatusThread_Unknown + case syncstatus.StatusSynced: + return pb.EventStatusThread_Synced } - return + return pb.EventStatusThread_Syncing } func (r *updateReceiver) isNodeConnected() bool { @@ -97,13 +104,13 @@ func (r *updateReceiver) UpdateNodeConnection(online bool) { func (r *updateReceiver) notify( objId string, - objStatus, generalStatus pb.EventStatusThreadSyncStatus, + objStatus pb.EventStatusThreadSyncStatus, pinStatus pb.EventStatusThreadCafePinStatus, ) { r.sendEvent(objId, &pb.EventMessageValueOfThreadStatus{ThreadStatus: &pb.EventStatusThread{ Summary: &pb.EventStatusThreadSummary{Status: objStatus}, Cafe: &pb.EventStatusThreadCafe{ - Status: generalStatus, + Status: objStatus, Files: &pinStatus, }, }}) diff --git a/docs/proto.md b/docs/proto.md index 1f8ae5ccf6..5e94612ab3 100644 --- a/docs/proto.md +++ b/docs/proto.md @@ -21232,6 +21232,7 @@ Bookmark is to keep a web-link and to preview a content. | groupRelationKey | [string](#string) | | Group view by this relationKey | | groupBackgroundColors | [bool](#bool) | | Enable backgrounds in groups | | pageLimit | [int32](#int32) | | | +| defaultTemplateId | [string](#string) | | | diff --git a/go.mod b/go.mod index 1eedb7bba4..064870f4e5 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/PuerkitoBio/goquery v1.8.1 github.com/VividCortex/ewma v1.2.0 github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786 - github.com/anyproto/any-sync v0.2.3 + github.com/anyproto/any-sync v0.2.10 github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/blevesearch/bleve/v2 v2.3.8 @@ -17,12 +17,14 @@ require ( github.com/dave/jennifer v1.6.1 github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger/v3 v3.2103.5 + github.com/dgraph-io/ristretto v0.1.1 github.com/dgtony/collections v0.1.6 github.com/dhowden/tag v0.0.0-20201120070457-d52dcb253c63 github.com/disintegration/imaging v1.6.2 github.com/dsoprea/go-exif/v3 v3.0.1 github.com/dsoprea/go-jpeg-image-structure/v2 v2.0.0-20210512043942-b434301c6836 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 + github.com/go-chi/chi/v5 v5.0.8 github.com/go-shiori/go-readability v0.0.0-20220215145315-dd6828d2f09b github.com/goccy/go-graphviz v0.1.1 github.com/gogo/protobuf v1.3.2 @@ -54,7 +56,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e github.com/kelseyhightower/envconfig v1.4.0 - github.com/libp2p/go-libp2p v0.27.5 + github.com/libp2p/go-libp2p v0.28.1 github.com/libp2p/zeroconf/v2 v2.2.0 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/magiconair/properties v1.8.7 @@ -70,7 +72,7 @@ require ( github.com/otiai10/copy v1.12.0 github.com/otiai10/opengraph/v2 v2.1.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_golang v1.16.0 github.com/pseudomuto/protoc-gen-doc v1.5.1 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd github.com/samber/lo v1.38.1 @@ -132,7 +134,6 @@ require ( github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect - github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dsoprea/go-iptc v0.0.0-20200609062250-162ae6b44feb // indirect github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd // indirect github.com/dsoprea/go-photoshop-info-format v0.0.0-20200609050348-3db9b63b202c // indirect @@ -183,13 +184,13 @@ require ( github.com/jbenet/goprocess v0.1.4 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.54 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect @@ -205,7 +206,7 @@ require ( github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/pseudomuto/protokit v0.2.0 // indirect github.com/rs/cors v1.7.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect diff --git a/go.sum b/go.sum index ae14783b3d..e304802467 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxB github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/anyproto/any-sync v0.2.3 h1:e4I1cnzBWDdqaRnkX2341yExPNih/sdvnpCXz70bw5M= -github.com/anyproto/any-sync v0.2.3/go.mod h1:U8NhzTbGCSE9Yf2CfvBTGtNPAYl6zrZT2vZypab2mck= +github.com/anyproto/any-sync v0.2.10 h1:sdXiDn4baeP40v9Mtjb9FSeyqldk7cSmF1aiTp7xaVI= +github.com/anyproto/any-sync v0.2.10/go.mod h1:ZENGpUnGbJ8NUbMkoIYPXSTX4IlXaoSko+gqzk/6onc= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= github.com/anyproto/go-chash v0.1.0/go.mod h1:0UjNQi3PDazP0fINpFYu6VKhuna+W/V+1vpXHAfNgLY= github.com/anyproto/go-ds-badger3 v0.3.1-0.20230524095230-434cf6346d9b h1:SMizb43hfILk2bpMgpTd30n6yQQdxW0ZbDti0wqfsBw= @@ -245,6 +245,8 @@ github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= +github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.0.2/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= @@ -374,8 +376,8 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gosimple/slug v1.13.1 h1:bQ+kpX9Qa6tHRaK+fZR0A0M2Kd7Pa5eHPPsb1JpHD+Q= github.com/gosimple/slug v1.13.1/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= @@ -551,10 +553,9 @@ github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= @@ -571,13 +572,13 @@ github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgx github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= -github.com/libp2p/go-libp2p v0.27.5 h1:KwA7pXKXpz8hG6Cr1fMA7UkgleogcwQj0sxl5qquWRg= -github.com/libp2p/go-libp2p v0.27.5/go.mod h1:oMfQGTb9CHnrOuSM6yMmyK2lXz3qIhnkn2+oK3B1Y2g= +github.com/libp2p/go-libp2p v0.28.1 h1:YurK+ZAI6cKfASLJBVFkpVBdl3wGhFi6fusOt725ii8= +github.com/libp2p/go-libp2p v0.28.1/go.mod h1:s3Xabc9LSwOcnv9UD4nORnXKTsWkPMkIMB/JIGXVnzk= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= -github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= +github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rBfZQ= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= @@ -616,8 +617,8 @@ github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/miolini/datacounter v1.0.3 h1:tanOZPVblGXQl7/bSZWoEM8l4KK83q24qwQLMrO/HOA= github.com/miolini/datacounter v1.0.3/go.mod h1:C45dc2hBumHjDpEU64IqPwR6TDyPVpzOqqRTN7zmBUA= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -740,8 +741,8 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -766,8 +767,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.10.0 h1:UkG7GPYkO4UZyLnyXjaWYcgOSONqwdBqFUT95ugmt6I= -github.com/prometheus/procfs v0.10.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/pseudomuto/protoc-gen-doc v1.5.1 h1:Ah259kcrio7Ix1Rhb6u8FCaOkzf9qRBqXnvAufg061w= github.com/pseudomuto/protoc-gen-doc v1.5.1/go.mod h1:XpMKYg6zkcpgfpCfQ8GcWBDRtRxOmMR5w7pz4Xo+dYM= @@ -1104,7 +1105,6 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/metrics/events.go b/metrics/events.go index 5752d2c324..20038ead85 100644 --- a/metrics/events.go +++ b/metrics/events.go @@ -4,7 +4,10 @@ import ( "fmt" ) -const CtxKeyRequest = "request" +const ( + CtxKeyEntrypoint = "entrypoint" + CtxKeyRPC = "rpc" +) type RecordAcceptEventAggregated struct { IsNAT bool @@ -62,6 +65,7 @@ const ( ReindexTypeBundledObjects ReindexTypeBundledTemplates ReindexTypeOutdatedHeads + ReindexTypeSystem ) func (t ReindexType) String() string { @@ -80,6 +84,8 @@ func (t ReindexType) String() string { return "bundled_templates" case ReindexTypeOutdatedHeads: return "outdated_heads" + case ReindexTypeSystem: + return "system" } return "unknown" } diff --git a/pkg/lib/bundle/relation.gen.go b/pkg/lib/bundle/relation.gen.go index fef96149f2..833a18c96f 100644 --- a/pkg/lib/bundle/relation.gen.go +++ b/pkg/lib/bundle/relation.gen.go @@ -9,7 +9,7 @@ import ( "github.com/anyproto/anytype-heart/pkg/lib/pb/model" ) -const RelationChecksum = "b6e52d7683a3d2ca18daf1deac443b74f95adab1529f72f1e9ec11256e74f253" +const RelationChecksum = "b4fb721c0937e69e97e0d5be67e5fab27a674afdc3a4e9a0c4323b4e7e5414ec" type RelationKey string @@ -164,6 +164,7 @@ const ( RelationKeyFileSyncStatus RelationKey = "fileSyncStatus" RelationKeyLastChangeId RelationKey = "lastChangeId" RelationKeyStarred RelationKey = "starred" + RelationKeyDefaultTemplateId RelationKey = "defaultTemplateId" ) var ( @@ -561,6 +562,20 @@ var ( ReadOnlyRelation: true, Scope: model.Relation_type, }, + RelationKeyDefaultTemplateId: { + + DataSource: model.Relation_details, + Description: "ID of template chosen as default for particular object type", + Format: model.RelationFormat_object, + Hidden: true, + Id: "_brdefaultTemplateId", + Key: "defaultTemplateId", + MaxCount: 1, + Name: "Default Template ID", + ReadOnly: false, + ReadOnlyRelation: true, + Scope: model.Relation_type, + }, RelationKeyDescription: { DataSource: model.Relation_details, diff --git a/pkg/lib/bundle/relations.json b/pkg/lib/bundle/relations.json index a719bc8bfc..a4303c3e00 100644 --- a/pkg/lib/bundle/relations.json +++ b/pkg/lib/bundle/relations.json @@ -1324,7 +1324,6 @@ { "description": "Space Dashboard object ID", "format": "object", - "hidden": false, "key": "spaceDashboardId", "maxCount": 1, "hidden": true, @@ -1390,5 +1389,15 @@ "name": "Starred", "readonly": false, "source": "details" + }, + { + "description": "ID of template chosen as default for particular object type", + "format": "object", + "key": "defaultTemplateId", + "maxCount": 1, + "hidden": true, + "name": "Default Template ID", + "readonly": false, + "source": "details" } ] diff --git a/pkg/lib/bundle/systemRelations.gen.go b/pkg/lib/bundle/systemRelations.gen.go index 86f83752be..b29074bede 100644 --- a/pkg/lib/bundle/systemRelations.gen.go +++ b/pkg/lib/bundle/systemRelations.gen.go @@ -4,7 +4,7 @@ source: pkg/lib/bundle/systemRelations.json */ package bundle -const SystemRelationsChecksum = "48cbc266c95c67dc931b23b24f596d329297f9e9709cd5c201f97a418096a292" +const SystemRelationsChecksum = "84d6c6144354f7ecb8aa415671d8821b54fa804454d17ff85f2274985a8c5705" // SystemRelations contains relations that have some special biz logic depends on them in some objects // in case EVERY object depend on the relation please add it to RequiredInternalRelations @@ -44,4 +44,5 @@ var SystemRelations = append(RequiredInternalRelations, []RelationKey{ RelationKeySizeInBytes, RelationKeySourceFilePath, RelationKeyFileSyncStatus, + RelationKeyDefaultTemplateId, }...) diff --git a/pkg/lib/bundle/systemRelations.json b/pkg/lib/bundle/systemRelations.json index eb70409266..8c2396a643 100644 --- a/pkg/lib/bundle/systemRelations.json +++ b/pkg/lib/bundle/systemRelations.json @@ -58,5 +58,6 @@ "fileExt", "sizeInBytes", "sourceFilePath", - "fileSyncStatus" + "fileSyncStatus", + "defaultTemplateId" ] diff --git a/pkg/lib/database/database.go b/pkg/lib/database/database.go index 5566d308fa..0c51297650 100644 --- a/pkg/lib/database/database.go +++ b/pkg/lib/database/database.go @@ -25,6 +25,10 @@ type Record struct { Details *types.Struct } +func (r Record) Get(key string) *types.Value { + return pbtypes.Get(r.Details, key) +} + type Query struct { FullText string Filters []*model.BlockContentDataviewFilter // filters results. apply sequentially diff --git a/pkg/lib/datastore/clientds/clientds.go b/pkg/lib/datastore/clientds/clientds.go index 47111f073f..bfbfa330d7 100644 --- a/pkg/lib/datastore/clientds/clientds.go +++ b/pkg/lib/datastore/clientds/clientds.go @@ -5,12 +5,14 @@ import ( "fmt" "os" "path/filepath" + "time" "github.com/anyproto/any-sync/app" "github.com/dgraph-io/badger/v3" "github.com/hashicorp/go-multierror" ds "github.com/ipfs/go-datastore" dsbadgerv3 "github.com/textileio/go-ds-badger3" + "go.uber.org/zap" "github.com/anyproto/anytype-heart/core/wallet" "github.com/anyproto/anytype-heart/pkg/lib/datastore" @@ -29,11 +31,13 @@ var log = logging.Logger("anytype-clientds") type clientds struct { running bool - spaceDS *dsbadgerv3.Datastore - localstoreDS *dsbadgerv3.Datastore - cfg Config - repoPath string - migrations []migration + spaceDS *dsbadgerv3.Datastore + localstoreDS *dsbadgerv3.Datastore + cfg Config + repoPath string + migrations []migration + spaceStoreWasMissing, localStoreWasMissing bool + spentOnInit time.Duration } type Config struct { @@ -80,6 +84,8 @@ func init() { } func (r *clientds) Init(a *app.App) (err error) { + // TODO: looks like we do a lot of stuff on Init here. We should consider moving it to the Run + start := time.Now() wl := a.Component(wallet.CName) if wl == nil { return fmt.Errorf("need wallet to be inited first") @@ -98,6 +104,14 @@ func (r *clientds) Init(a *app.App) (err error) { return fmt.Errorf("old repo found") } + if _, err := os.Stat(r.getRepoPath(localstoreDSDir)); os.IsNotExist(err) { + r.localStoreWasMissing = true + } + + if _, err := os.Stat(r.getRepoPath(SpaceDSDir)); os.IsNotExist(err) { + r.spaceStoreWasMissing = true + } + RemoveExpiredLocks(r.repoPath) r.localstoreDS, err = dsbadgerv3.NewDatastore(r.getRepoPath(localstoreDSDir), &r.cfg.Localstore) @@ -116,7 +130,7 @@ func (r *clientds) Init(a *app.App) (err error) { } r.running = true - + r.spentOnInit = time.Since(start) return nil } @@ -163,6 +177,13 @@ func (r *clientds) LocalstoreDS() (datastore.DSTxnBatching, error) { return r.localstoreDS, nil } +func (r *clientds) LocalstoreBadger() (*badger.DB, error) { + if !r.running { + return nil, fmt.Errorf("exact ds may be requested only after Run") + } + return r.localstoreDS.DB, nil +} + func (r *clientds) Name() (name string) { return CName } @@ -192,3 +213,11 @@ func New() datastore.Datastore { func (r *clientds) getRepoPath(dir string) string { return filepath.Join(r.repoPath, dir) } + +func (r *clientds) GetLogFields() []zap.Field { + return []zap.Field{ + zap.Bool("spaceStoreWasMissing", r.spaceStoreWasMissing), + zap.Bool("localStoreWasMissing", r.localStoreWasMissing), + zap.Int64("spentOnInit", r.spentOnInit.Milliseconds()), + } +} diff --git a/pkg/lib/datastore/datastore.go b/pkg/lib/datastore/datastore.go index 9f5475f3d2..c20f50be87 100644 --- a/pkg/lib/datastore/datastore.go +++ b/pkg/lib/datastore/datastore.go @@ -14,6 +14,7 @@ type Datastore interface { app.ComponentRunnable LocalstoreDS() (DSTxnBatching, error) SpaceStorage() (*badger.DB, error) + LocalstoreBadger() (*badger.DB, error) } type DSTxnBatching interface { diff --git a/pkg/lib/localstore/filestore/files.go b/pkg/lib/localstore/filestore/files.go index f4ea56aec3..5933814d8a 100644 --- a/pkg/lib/localstore/filestore/files.go +++ b/pkg/lib/localstore/filestore/files.go @@ -28,6 +28,7 @@ var ( filesKeysBase = dsCtx.NewKey("/" + filesPrefix + "/keys") chunksCountBase = dsCtx.NewKey("/" + filesPrefix + "/chunks_count") syncStatusBase = dsCtx.NewKey("/" + filesPrefix + "/sync_status") + isImportedBase = dsCtx.NewKey("/" + filesPrefix + "/is_imported") indexMillSourceOpts = localstore.Index{ Prefix: filesPrefix, @@ -101,6 +102,8 @@ type FileStore interface { SetChunksCount(hash string, chunksCount int) error GetSyncStatus(hash string) (int, error) SetSyncStatus(hash string, syncStatus int) error + IsFileImported(hash string) (bool, error) + SetIsFileImported(hash string, isImported bool) error } func New() FileStore { @@ -684,6 +687,24 @@ func (m *dsFileStore) SetSyncStatus(hash string, status int) error { return m.setInt(key, status) } +func (m *dsFileStore) IsFileImported(hash string) (bool, error) { + key := isImportedBase.ChildString(hash) + raw, err := m.getInt(key) + if err == localstore.ErrNotFound { + return false, nil + } + return raw == 1, err +} + +func (m *dsFileStore) SetIsFileImported(hash string, isImported bool) error { + var raw int + if isImported { + raw = 1 + } + key := isImportedBase.ChildString(hash) + return m.setInt(key, raw) +} + func (ls *dsFileStore) Close(ctx context.Context) (err error) { return nil } diff --git a/pkg/lib/localstore/ftsearch/ftsearch.go b/pkg/lib/localstore/ftsearch/ftsearch.go index c78ee90b09..cd6c85bd37 100644 --- a/pkg/lib/localstore/ftsearch/ftsearch.go +++ b/pkg/lib/localstore/ftsearch/ftsearch.go @@ -5,9 +5,11 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" "time" + "github.com/anyproto/any-sync/app" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" "github.com/blevesearch/bleve/v2/analysis/lang/en" @@ -15,8 +17,6 @@ import ( "github.com/blevesearch/bleve/v2/search/query" "github.com/samber/lo" - "github.com/anyproto/any-sync/app" - "github.com/anyproto/anytype-heart/core/wallet" "github.com/anyproto/anytype-heart/metrics" "github.com/anyproto/anytype-heart/pkg/lib/localstore/ftsearch/analyzers" @@ -141,11 +141,10 @@ func (f *ftSearch) BatchIndex(docs []SearchDoc) (err error) { func (f *ftSearch) Search(qry string) (results []string, err error) { qry = strings.ToLower(qry) - var queries = make([]query.Query, 0, 4) qry = strings.TrimSpace(qry) terms := f.getTerms(qry) - queries = append( + queries := append( getFullQueries(qry), bleve.NewMatchQuery(qry), ) @@ -262,7 +261,7 @@ func getStandardMapping() *mapping.FieldMapping { func getAllWordsFromQueryConsequently(terms []string, field string) query.Query { terms = lo.Map( terms, - func(item string, index int) string { return strings.ReplaceAll(item, "*", `\*`) }, + func(item string, index int) string { return regexp.QuoteMeta(item) }, ) qry := strings.Join(terms, ".*") regexpQuery := bleve.NewRegexpQuery(".*" + qry + ".*") diff --git a/pkg/lib/localstore/ftsearch/ftsearch_test.go b/pkg/lib/localstore/ftsearch/ftsearch_test.go index 6c8643a6e3..ae5f1f9044 100644 --- a/pkg/lib/localstore/ftsearch/ftsearch_test.go +++ b/pkg/lib/localstore/ftsearch/ftsearch_test.go @@ -36,12 +36,42 @@ func newFixture(path string, t *testing.T) *fixture { } func TestNewFTSearch(t *testing.T) { - tmpDir, _ := os.MkdirTemp("", "") - assertSearch(t, tmpDir) - assertThaiSubstrFound(t, tmpDir) - assertChineseFound(t, tmpDir) - assertFoundPartsOfTheWords(t, tmpDir) - assertFoundCaseSensitivePartsOfTheWords(t, tmpDir) + testCases := []struct { + name string + tester func(t *testing.T, tmpDir string) + }{ + { + name: "assertSearch", + tester: assertSearch, + }, + { + name: "assertThaiSubstrFound", + tester: assertThaiSubstrFound, + }, + { + name: "assertChineseFound", + tester: assertChineseFound, + }, + { + name: "assertFoundPartsOfTheWords", + tester: assertFoundPartsOfTheWords, + }, + { + name: "assertFoundCaseSensitivePartsOfTheWords", + tester: assertFoundCaseSensitivePartsOfTheWords, + }, + { + name: "assertNonEscapedQuery", + tester: assertNonEscapedQuery, + }, + } + + for _, testCase := range testCases { + tmpDir, _ := os.MkdirTemp("", "") + t.Run(testCase.name, func(t *testing.T) { + testCase.tester(t, tmpDir) + }) + } } func assertFoundCaseSensitivePartsOfTheWords(t *testing.T, tmpDir string) { @@ -67,6 +97,7 @@ func assertFoundCaseSensitivePartsOfTheWords(t *testing.T, tmpDir string) { })) validateSearch(t, ft, "Advanced", 1) + validateSearch(t, ft, "advanced", 1) validateSearch(t, ft, "Advanc", 1) validateSearch(t, ft, "advanc", 1) @@ -261,3 +292,24 @@ func givenPrefilledChineseIndex() bleve.Index { } return index } + +func assertNonEscapedQuery(t *testing.T, tmpDir string) { + fixture := newFixture(tmpDir, t) + ft := fixture.ft + require.NoError(t, ft.Index(SearchDoc{ + Id: "1", + Title: "This is the title", + Text: "two", + })) + + validateSearch(t, ft, ".*?([])", 0) + + require.NoError(t, ft.Index(SearchDoc{ + Id: "1", + Title: "This is the title", + Text: ".*?([])", + })) + validateSearch(t, ft, ".*?([])", 1) + + _ = ft.Close(nil) +} diff --git a/pkg/lib/localstore/objectstore/account_store.go b/pkg/lib/localstore/objectstore/account_store.go index 6c7a0d1abe..bad845d274 100644 --- a/pkg/lib/localstore/objectstore/account_store.go +++ b/pkg/lib/localstore/objectstore/account_store.go @@ -1,86 +1,29 @@ package objectstore import ( - "fmt" - "github.com/anyproto/any-sync/coordinator/coordinatorproto" "github.com/gogo/protobuf/proto" ) func (s *dsObjectStore) GetCurrentWorkspaceID() (string, error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return "", fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - val, err := txn.Get(currentWorkspace) - if err != nil { - return "", err - } - return string(val), nil + return getValue(s.db, currentWorkspace.Bytes(), bytesToString) } func (s *dsObjectStore) SetCurrentWorkspaceID(workspaceID string) (err error) { - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - if err := txn.Put(currentWorkspace, []byte(workspaceID)); err != nil { - return fmt.Errorf("failed to put into ds: %w", err) - } - - return txn.Commit() + return setValue(s.db, currentWorkspace.Bytes(), workspaceID) } func (s *dsObjectStore) RemoveCurrentWorkspaceID() (err error) { - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - if err := txn.Delete(currentWorkspace); err != nil { - return fmt.Errorf("failed to delete from ds: %w", err) - } - - return txn.Commit() + return deleteValue(s.db, currentWorkspace.Bytes()) } func (s *dsObjectStore) SaveAccountStatus(status *coordinatorproto.SpaceStatusPayload) (err error) { - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - b, err := status.Marshal() - if err != nil { - return err - } - - if err := txn.Put(accountStatus, b); err != nil { - return fmt.Errorf("failed to put into ds: %w", err) - } - - return txn.Commit() + return setValue(s.db, accountStatus.Bytes(), status) } -func (s *dsObjectStore) GetAccountStatus() (status *coordinatorproto.SpaceStatusPayload, err error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - status = &coordinatorproto.SpaceStatusPayload{} - if val, err := txn.Get(accountStatus); err != nil { - return nil, err - } else if err := proto.Unmarshal(val, status); err != nil { - return nil, err - } - - return status, nil +func (s *dsObjectStore) GetAccountStatus() (*coordinatorproto.SpaceStatusPayload, error) { + return getValue(s.db, accountStatus.Bytes(), func(raw []byte) (*coordinatorproto.SpaceStatusPayload, error) { + status := &coordinatorproto.SpaceStatusPayload{} + return status, proto.Unmarshal(raw, status) + }) } diff --git a/pkg/lib/localstore/objectstore/badger_helpers.go b/pkg/lib/localstore/objectstore/badger_helpers.go new file mode 100644 index 0000000000..509963628e --- /dev/null +++ b/pkg/lib/localstore/objectstore/badger_helpers.go @@ -0,0 +1,109 @@ +package objectstore + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v3" + "github.com/gogo/protobuf/proto" +) + +func (s *dsObjectStore) updateTxn(f func(txn *badger.Txn) error) error { + return retryOnConflict(func() error { + return s.db.Update(f) + }) +} + +func retryOnConflict(proc func() error) error { + for { + err := proc() + if err == nil { + return nil + } + if errors.Is(err, badger.ErrConflict) { + continue + } + return err + } +} + +func setValue(db *badger.DB, key []byte, value any) error { + return db.Update(func(txn *badger.Txn) error { + return setValueTxn(txn, key, value) + }) +} + +func setValueTxn(txn *badger.Txn, key []byte, value any) error { + raw, err := marshalValue(value) + if err != nil { + return fmt.Errorf("marshal value: %w", err) + } + return txn.Set(key, raw) +} + +func marshalValue(value any) ([]byte, error) { + if value != nil { + switch v := value.(type) { + case proto.Message: + return proto.Marshal(v) + case string: + return []byte(v), nil + default: + return nil, fmt.Errorf("unsupported type %T", v) + } + } + return nil, nil +} + +func deleteValue(db *badger.DB, key []byte) error { + return db.Update(func(txn *badger.Txn) error { + return txn.Delete(key) + }) +} + +func getValue[T any](db *badger.DB, key []byte, unmarshaler func([]byte) (T, error)) (T, error) { + var res T + txErr := db.View(func(txn *badger.Txn) error { + var err error + res, err = getValueTxn(txn, key, unmarshaler) + return err + }) + return res, txErr +} + +func getValueTxn[T any](txn *badger.Txn, key []byte, unmarshaler func([]byte) (T, error)) (T, error) { + var res T + item, err := txn.Get(key) + if err != nil { + return res, fmt.Errorf("get item: %w", err) + } + err = item.Value(func(val []byte) error { + res, err = unmarshaler(val) + return err + }) + return res, err +} + +func iterateKeysByPrefix(db *badger.DB, prefix []byte, processKeyFn func(key []byte)) error { + return db.View(func(txn *badger.Txn) error { + return iterateKeysByPrefixTx(txn, prefix, processKeyFn) + }) +} + +func iterateKeysByPrefixTx(txn *badger.Txn, prefix []byte, processKeyFn func(key []byte)) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false + opts.Prefix = prefix + iter := txn.NewIterator(opts) + defer iter.Close() + + for iter.Rewind(); iter.Valid(); iter.Next() { + key := iter.Item().Key() + processKeyFn(key) + } + return nil +} + +func isNotFound(err error) bool { + return errors.Is(err, badger.ErrKeyNotFound) +} diff --git a/pkg/lib/localstore/objectstore/delete.go b/pkg/lib/localstore/objectstore/delete.go new file mode 100644 index 0000000000..7be811e566 --- /dev/null +++ b/pkg/lib/localstore/objectstore/delete.go @@ -0,0 +1,113 @@ +package objectstore + +import ( + "errors" + "fmt" + + "github.com/dgraph-io/badger/v3" + "github.com/gogo/protobuf/types" + ds "github.com/ipfs/go-datastore" + + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +func (s *dsObjectStore) DeleteDetails(id string) error { + key := pagesDetailsBase.ChildString(id).Bytes() + return s.updateTxn(func(txn *badger.Txn) error { + s.cache.Del(key) + + for _, k := range []ds.Key{ + pagesSnippetBase.ChildString(id), + pagesDetailsBase.ChildString(id), + indexedHeadsState.ChildString(id), + } { + if err := txn.Delete(k.Bytes()); err != nil { + return fmt.Errorf("delete key %s: %w", k, err) + } + } + + return txn.Delete(key) + }) +} + +// DeleteObject removes all details, leaving only id and isDeleted +func (s *dsObjectStore) DeleteObject(id string) error { + // do not completely remove object details, so we can distinguish links to deleted and not-yet-loaded objects + err := s.UpdateObjectDetails(id, &types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(id), + bundle.RelationKeyIsDeleted.String(): pbtypes.Bool(true), // maybe we can store the date instead? + }, + }) + if err != nil && !errors.Is(err, ErrDetailsNotChanged) { + return fmt.Errorf("failed to overwrite details and relations: %w", err) + } + + return retryOnConflict(func() error { + txn := s.db.NewTransaction(true) + defer txn.Discard() + + for _, k := range []ds.Key{ + pagesSnippetBase.ChildString(id), + indexQueueBase.ChildString(id), + indexedHeadsState.ChildString(id), + } { + if err = txn.Delete(k.Bytes()); err != nil { + return err + } + } + + txn, _, err = s.removeByPrefixInTx(txn, pagesInboundLinksBase.String()+"/"+id+"/") + if err != nil { + return err + } + txn, _, err = s.removeByPrefixInTx(txn, pagesOutboundLinksBase.String()+"/"+id+"/") + if err != nil { + return err + } + err = txn.Commit() + if err != nil { + return fmt.Errorf("delete object info: %w", err) + } + + if s.fts != nil { + err = s.removeFromIndexQueue(id) + if err != nil { + log.Errorf("error removing %s from index queue: %s", id, err) + } + if err := s.fts.Delete(id); err != nil { + return err + } + } + return nil + }) +} + +func (s *dsObjectStore) removeByPrefixInTx(txn *badger.Txn, prefix string) (*badger.Txn, int, error) { + var toDelete [][]byte + err := iterateKeysByPrefixTx(txn, []byte(prefix), func(key []byte) { + toDelete = append(toDelete, key) + }) + if err != nil { + return txn, 0, fmt.Errorf("iterate keys: %w", err) + } + + var removed int + for _, key := range toDelete { + err = txn.Delete(key) + if err == badger.ErrTxnTooBig { + err = txn.Commit() + if err != nil { + return txn, removed, fmt.Errorf("commit big transaction: %w", err) + } + txn = s.db.NewTransaction(true) + err = txn.Delete(key) + } + if err != nil { + return txn, removed, fmt.Errorf("delete key %s: %w", key, err) + } + removed++ + } + return txn, removed, nil +} diff --git a/pkg/lib/localstore/objectstore/filters.go b/pkg/lib/localstore/objectstore/filters.go index 4961eac66d..dfa5422820 100644 --- a/pkg/lib/localstore/objectstore/filters.go +++ b/pkg/lib/localstore/objectstore/filters.go @@ -1,11 +1,11 @@ package objectstore import ( - "strings" - - "github.com/ipfs/go-datastore/query" + "fmt" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" + "github.com/anyproto/anytype-heart/pkg/lib/database/filter" "github.com/anyproto/anytype-heart/space/typeprovider" ) @@ -19,23 +19,31 @@ func newIdsFilter(ids []string) idsFilter { type idsFilter map[string]int -func (f idsFilter) Filter(e query.Entry) bool { - _, ok := f[extractIdFromKey(e.Key)] +func (f idsFilter) FilterObject(getter filter.Getter) bool { + id := getter.Get(bundle.RelationKeyId.String()).GetStringValue() + _, ok := f[id] return ok } -func (f idsFilter) Compare(a, b query.Entry) int { - aIndex := f[extractIdFromKey(a.Key)] - bIndex := f[extractIdFromKey(b.Key)] - if aIndex == bIndex { +func (f idsFilter) Compare(a, b filter.Getter) int { + idA := a.Get(bundle.RelationKeyId.String()).GetStringValue() + idB := b.Get(bundle.RelationKeyId.String()).GetStringValue() + aIndex := f[idA] + bIndex := f[idB] + switch { + case aIndex == bIndex: return 0 - } else if aIndex < bIndex { + case aIndex < bIndex: return -1 - } else { + default: return 1 } } +func (f idsFilter) String() string { + return "idsFilter" +} + type filterSmartblockTypes struct { smartBlockTypes []smartblock.SmartBlockType not bool @@ -50,16 +58,13 @@ func newSmartblockTypesFilter(sbtProvider typeprovider.SmartBlockTypeProvider, n } } -func (m *filterSmartblockTypes) Filter(e query.Entry) bool { - keyParts := strings.Split(e.Key, "/") - id := keyParts[len(keyParts)-1] - +func (m *filterSmartblockTypes) FilterObject(getter filter.Getter) bool { + id := getter.Get(bundle.RelationKeyId.String()).GetStringValue() t, err := m.sbtProvider.Type(id) if err != nil { log.Debugf("failed to detect smartblock type for %s: %s", id, err.Error()) return false } - for _, ot := range m.smartBlockTypes { if t == ot { return !m.not @@ -67,3 +72,7 @@ func (m *filterSmartblockTypes) Filter(e query.Entry) bool { } return m.not } + +func (m *filterSmartblockTypes) String() string { + return fmt.Sprintf("filterSmartblockTypes %v", m.smartBlockTypes) +} diff --git a/pkg/lib/localstore/objectstore/indexer_store.go b/pkg/lib/localstore/objectstore/indexer_store.go index 75b70ff50c..f49cca4665 100644 --- a/pkg/lib/localstore/objectstore/indexer_store.go +++ b/pkg/lib/localstore/objectstore/indexer_store.go @@ -1,67 +1,25 @@ package objectstore import ( - "encoding/binary" - "fmt" - "time" - "github.com/gogo/protobuf/proto" - ds "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" ) func (s *dsObjectStore) AddToIndexQueue(id string) error { - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - var buf [8]byte - size := binary.PutVarint(buf[:], time.Now().Unix()) - if err = txn.Put(indexQueueBase.ChildString(id), buf[:size]); err != nil { - return err - } - return txn.Commit() + return setValue(s.db, indexQueueBase.ChildString(id).Bytes(), nil) } func (s *dsObjectStore) removeFromIndexQueue(id string) error { - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - if err := txn.Delete(indexQueueBase.ChildString(id)); err != nil { - return fmt.Errorf("failed to remove id from full text index queue: %s", err.Error()) - } - - return txn.Commit() + return deleteValue(s.db, indexQueueBase.ChildString(id).Bytes()) } func (s *dsObjectStore) ListIDsFromFullTextQueue() ([]string, error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - res, err := txn.Query(query.Query{Prefix: indexQueueBase.String()}) - if err != nil { - return nil, fmt.Errorf("error query txn in datastore: %w", err) - } - var ids []string - for entry := range res.Next() { - ids = append(ids, extractIdFromKey(entry.Key)) - } - - err = res.Close() - if err != nil { - return nil, fmt.Errorf("close query result: %w", err) - } - return ids, nil + err := iterateKeysByPrefix(s.db, indexQueueBase.Bytes(), func(key []byte) { + ids = append(ids, extractIDFromKey(string(key))) + }) + return ids, err } func (s *dsObjectStore) RemoveIDsFromFullTextQueue(ids []string) { @@ -75,74 +33,25 @@ func (s *dsObjectStore) RemoveIDsFromFullTextQueue(ids []string) { } func (s *dsObjectStore) GetChecksums() (checksums *model.ObjectStoreChecksums, err error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - val, err := txn.Get(bundledChecksums) - if err != nil && err != ds.ErrNotFound { - return nil, fmt.Errorf("failed to get details: %w", err) - } - if err == ds.ErrNotFound { - return nil, err - } - - var objChecksum model.ObjectStoreChecksums - if err := proto.Unmarshal(val, &objChecksum); err != nil { - return nil, err - } - - return &objChecksum, nil + return getValue(s.db, bundledChecksums.Bytes(), func(raw []byte) (*model.ObjectStoreChecksums, error) { + checksums := &model.ObjectStoreChecksums{} + return checksums, proto.Unmarshal(raw, checksums) + }) } func (s *dsObjectStore) SaveChecksums(checksums *model.ObjectStoreChecksums) (err error) { - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - b, err := checksums.Marshal() - if err != nil { - return err - } - - if err := txn.Put(bundledChecksums, b); err != nil { - return fmt.Errorf("failed to put into ds: %w", err) - } - - return txn.Commit() + return setValue(s.db, bundledChecksums.Bytes(), checksums) } // GetLastIndexedHeadsHash return empty hash without error if record was not found func (s *dsObjectStore) GetLastIndexedHeadsHash(id string) (headsHash string, err error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return "", fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - if val, err := txn.Get(indexedHeadsState.ChildString(id)); err != nil && err != ds.ErrNotFound { - return "", fmt.Errorf("failed to get heads hash: %w", err) - } else if val == nil { - return "", nil - } else { - return string(val), nil + headsHash, err = getValue(s.db, indexedHeadsState.ChildString(id).Bytes(), bytesToString) + if err != nil && !isNotFound(err) { + return "", err } + return headsHash, nil } func (s *dsObjectStore) SaveLastIndexedHeadsHash(id string, headsHash string) (err error) { - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - if err := txn.Put(indexedHeadsState.ChildString(id), []byte(headsHash)); err != nil { - return fmt.Errorf("failed to put into ds: %w", err) - } - - return txn.Commit() + return setValue(s.db, indexedHeadsState.ChildString(id).Bytes(), headsHash) } diff --git a/pkg/lib/localstore/objectstore/objects.go b/pkg/lib/localstore/objectstore/objects.go index 27fa564ebe..8eff4f523a 100644 --- a/pkg/lib/localstore/objectstore/objects.go +++ b/pkg/lib/localstore/objectstore/objects.go @@ -4,23 +4,24 @@ import ( "context" "errors" "fmt" + "path" "strings" "sync" "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/coordinator/coordinatorproto" + "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/ristretto" "github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/types" ds "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" "github.com/anyproto/anytype-heart/core/relation/relationutils" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" "github.com/anyproto/anytype-heart/pkg/lib/database" + "github.com/anyproto/anytype-heart/pkg/lib/database/filter" "github.com/anyproto/anytype-heart/pkg/lib/datastore" - "github.com/anyproto/anytype-heart/pkg/lib/datastore/noctxds" - "github.com/anyproto/anytype-heart/pkg/lib/localstore" "github.com/anyproto/anytype-heart/pkg/lib/localstore/addr" "github.com/anyproto/anytype-heart/pkg/lib/localstore/ftsearch" "github.com/anyproto/anytype-heart/pkg/lib/logging" @@ -65,21 +66,14 @@ func New(sbtProvider typeprovider.SmartBlockTypeProvider) ObjectStore { } } -func NewWithLocalstore(ds noctxds.DSTxnBatching) ObjectStore { - return &dsObjectStore{ - ds: ds, - } -} - -type SourceDetailsFromId interface { +type SourceDetailsFromID interface { DetailsFromIdBasedSource(id string) (*types.Struct, error) } func (s *dsObjectStore) Init(a *app.App) (err error) { - s.dsIface = a.MustComponent(datastore.CName).(datastore.Datastore) src := a.Component("source") if src != nil { - s.sourceService = a.MustComponent("source").(SourceDetailsFromId) + s.sourceService = a.MustComponent("source").(SourceDetailsFromID) } fts := a.Component(ftsearch.CName) if fts == nil { @@ -87,6 +81,21 @@ func (s *dsObjectStore) Init(a *app.App) (err error) { } else { s.fts = fts.(ftsearch.FTSearch) } + datastoreService := a.MustComponent(datastore.CName).(datastore.Datastore) + s.db, err = datastoreService.LocalstoreBadger() + if err != nil { + return fmt.Errorf("get badger: %w", err) + } + + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 10_000_000, + MaxCost: 100_000_000, + BufferItems: 64, + }) + if err != nil { + return fmt.Errorf("init cache: %w", err) + } + s.cache = cache return nil } @@ -94,16 +103,16 @@ func (s *dsObjectStore) Name() (name string) { return CName } +// nolint: interfacebloat type ObjectStore interface { app.ComponentRunnable IndexerStore AccountStore - localstore.Indexable SubscribeForAll(callback func(rec database.Record)) Query(schema schema.Schema, q database.Query) (records []database.Record, total int, err error) - QueryRaw(f *database.Filters) (records []database.Record, err error) + QueryRaw(f *database.Filters, limit int, offset int) (records []database.Record, err error) QueryByID(ids []string) (records []database.Record, err error) QueryByIDAndSubscribeForChanges(ids []string, subscription database.Subscription) (records []database.Record, close func(), err error) QueryObjectIDs(q database.Query, objectTypes []smartblock.SmartBlockType) (ids []string, total int, err error) @@ -163,61 +172,49 @@ type AccountStore interface { var ErrNotAnObject = fmt.Errorf("not an object") type dsObjectStore struct { - // underlying storage - ds noctxds.DSTxnBatching - dsIface datastore.Datastore - sourceService SourceDetailsFromId + sourceService SourceDetailsFromID + + cache *ristretto.Cache + db *badger.DB fts ftsearch.FTSearch - // serializing page updates - l sync.Mutex + sbtProvider typeprovider.SmartBlockTypeProvider + sync.RWMutex onChangeCallback func(record database.Record) - subscriptions []database.Subscription - depSubscriptions []database.Subscription - - sbtProvider typeprovider.SmartBlockTypeProvider } -func (s *dsObjectStore) EraseIndexes() (err error) { - for _, idx := range s.Indexes() { - err = localstore.EraseIndex(idx, s.ds) - if err != nil { - return - } - } - - err = s.eraseLinks() +func (s *dsObjectStore) EraseIndexes() error { + outboundRemoved, inboundRemoved, err := s.eraseLinks() if err != nil { - log.Errorf("eraseLinks failed: %s", err.Error()) + log.Errorf("eraseLinks failed: %s", err) } - - return + log.Infof("eraseLinks: removed %d outbound links", outboundRemoved) + log.Infof("eraseLinks: removed %d inbound links", inboundRemoved) + return nil } -func (s *dsObjectStore) eraseLinks() (err error) { - n, err := removeByPrefix(s.ds, pagesOutboundLinksBase.String()) - if err != nil { - return err - } - - log.Infof("eraseLinks: removed %d outbound links", n) - n, err = removeByPrefix(s.ds, pagesInboundLinksBase.String()) - if err != nil { - return err - } - - log.Infof("eraseLinks: removed %d inbound links", n) - - return nil +func (s *dsObjectStore) eraseLinks() (outboundRemoved int, inboundRemoved int, err error) { + err = retryOnConflict(func() error { + txn := s.db.NewTransaction(true) + defer txn.Discard() + txn, outboundRemoved, err = s.removeByPrefixInTx(txn, pagesOutboundLinksBase.String()) + if err != nil { + return fmt.Errorf("remove all outbound links: %w", err) + } + txn, inboundRemoved, err = s.removeByPrefixInTx(txn, pagesInboundLinksBase.String()) + if err != nil { + return fmt.Errorf("remove all inbound links: %w", err) + } + return txn.Commit() + }) + return } func (s *dsObjectStore) Run(context.Context) (err error) { - lds, err := s.dsIface.LocalstoreDS() - s.ds = noctxds.New(lds) - return + return nil } func (s *dsObjectStore) Close(_ context.Context) (err error) { @@ -261,8 +258,8 @@ func (s *dsObjectStore) addSubscriptionIfNotExists(sub database.Subscription) (e } func (s *dsObjectStore) closeAndRemoveSubscription(subscription database.Subscription) { - s.l.Lock() - defer s.l.Unlock() + s.Lock() + defer s.Unlock() subscription.Close() for i, sub := range s.subscriptions { @@ -273,34 +270,13 @@ func (s *dsObjectStore) closeAndRemoveSubscription(subscription database.Subscri } } -func unmarshalDetails(id string, rawValue []byte) (*model.ObjectDetails, error) { - var details model.ObjectDetails - if err := proto.Unmarshal(rawValue, &details); err != nil { - return nil, err - } - - if details.Details == nil || details.Details.Fields == nil { - details.Details = &types.Struct{Fields: map[string]*types.Value{}} - } else { - pbtypes.StructDeleteEmptyFields(details.Details) - } - details.Details.Fields[database.RecordIDField] = pbtypes.ToValue(id) - return &details, nil -} - func (s *dsObjectStore) SubscribeForAll(callback func(rec database.Record)) { - s.l.Lock() + s.Lock() s.onChangeCallback = callback - s.l.Unlock() + s.Unlock() } func (s *dsObjectStore) GetRelationByID(id string) (*model.Relation, error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - det, err := s.GetDetails(id) if err != nil { return nil, err @@ -346,272 +322,226 @@ func (s *dsObjectStore) GetRelationByKey(key string) (*model.Relation, error) { return rel.Relation, nil } -func (s *dsObjectStore) DeleteDetails(id string) error { - s.l.Lock() - defer s.l.Unlock() - - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - // todo: remove all indexes with this object - for _, k := range []ds.Key{ - pagesSnippetBase.ChildString(id), - pagesDetailsBase.ChildString(id), - } { - if err = txn.Delete(k); err != nil { +func (s *dsObjectStore) GetWithLinksInfoByID(id string) (*model.ObjectInfoWithLinks, error) { + var res *model.ObjectInfoWithLinks + err := s.db.View(func(txn *badger.Txn) error { + pages, err := s.getObjectsInfo(txn, []string{id}) + if err != nil { return err } - } - - return txn.Commit() -} -// DeleteObject removes all details, leaving only id and isDeleted -func (s *dsObjectStore) DeleteObject(id string) error { - // do not completely remove object details, so we can distinguish links to deleted and not-yet-loaded objects - err := s.UpdateObjectDetails(id, &types.Struct{ - Fields: map[string]*types.Value{ - bundle.RelationKeyId.String(): pbtypes.String(id), - bundle.RelationKeyIsDeleted.String(): pbtypes.Bool(true), // maybe we can store the date instead? - }, - }) - if err != nil { - if !errors.Is(err, ErrDetailsNotChanged) { - return fmt.Errorf("failed to overwrite details and relations: %w", err) + if len(pages) == 0 { + return fmt.Errorf("page not found") } - } + page := pages[0] - s.l.Lock() - defer s.l.Unlock() - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() + inboundIds, err := findInboundLinks(txn, id) + if err != nil { + return fmt.Errorf("find inbound links: %w", err) + } + outboundsIds, err := findOutboundLinks(txn, id) + if err != nil { + return fmt.Errorf("find outbound links: %w", err) + } - // todo: remove all indexes with this object - for _, k := range []ds.Key{ - pagesSnippetBase.ChildString(id), - indexQueueBase.ChildString(id), - indexedHeadsState.ChildString(id), - } { - if err = txn.Delete(k); err != nil { + inbound, err := s.getObjectsInfo(txn, inboundIds) + if err != nil { return err } - } - _, err = removeByPrefixInTx(txn, pagesInboundLinksBase.String()+"/"+id+"/") - if err != nil { - return err - } - - _, err = removeByPrefixInTx(txn, pagesOutboundLinksBase.String()+"/"+id+"/") - if err != nil { - return err - } - - if s.fts != nil { - err = s.removeFromIndexQueue(id) + outbound, err := s.getObjectsInfo(txn, outboundsIds) if err != nil { - log.Errorf("error removing %s from index queue: %s", id, err) - } - if err := s.fts.Delete(id); err != nil { return err } - } - return txn.Commit() -} -func (s *dsObjectStore) GetWithLinksInfoByID(id string) (*model.ObjectInfoWithLinks, error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - pages, err := s.getObjectsInfo(txn, []string{id}) - if err != nil { - return nil, err - } - - if len(pages) == 0 { - return nil, fmt.Errorf("page not found") - } - page := pages[0] - - inboundIds, err := findInboundLinks(txn, id) - if err != nil { - return nil, err - } - - outboundsIds, err := findOutboundLinks(txn, id) - if err != nil { - return nil, err - } - - inbound, err := s.getObjectsInfo(txn, inboundIds) - if err != nil { - return nil, err - } - - outbound, err := s.getObjectsInfo(txn, outboundsIds) - if err != nil { - return nil, err - } - - return &model.ObjectInfoWithLinks{ - Id: id, - Info: page, - Links: &model.ObjectLinksInfo{ - Inbound: inbound, - Outbound: outbound, - }, - }, nil + res = &model.ObjectInfoWithLinks{ + Id: id, + Info: page, + Links: &model.ObjectLinksInfo{ + Inbound: inbound, + Outbound: outbound, + }, + } + return nil + }) + return res, err } func (s *dsObjectStore) GetOutboundLinksByID(id string) ([]string, error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - return findOutboundLinks(txn, id) + var links []string + err := s.db.View(func(txn *badger.Txn) error { + var err error + links, err = findOutboundLinks(txn, id) + return err + }) + return links, err } func (s *dsObjectStore) GetInboundLinksByID(id string) ([]string, error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - return findInboundLinks(txn, id) + var links []string + err := s.db.View(func(txn *badger.Txn) error { + var err error + links, err = findInboundLinks(txn, id) + return err + }) + return links, err } // GetDetails returns empty struct without errors in case details are not found // todo: get rid of this or change the name method! func (s *dsObjectStore) GetDetails(id string) (*model.ObjectDetails, error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - return getObjectDetails(txn, id) -} - -func (s *dsObjectStore) List() ([]*model.ObjectInfo, error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) + var details *model.ObjectDetails + err := s.db.View(func(txn *badger.Txn) error { + it, err := txn.Get(pagesDetailsBase.ChildString(id).Bytes()) + if err != nil { + return fmt.Errorf("get details: %w", err) + } + details, err = s.extractDetailsFromItem(it) + return err + }) + if isNotFound(err) { + return &model.ObjectDetails{ + Details: &types.Struct{Fields: map[string]*types.Value{}}, + }, nil } - defer txn.Discard() - ids, err := findByPrefix(txn, pagesDetailsBase.String()+"/", 0) if err != nil { return nil, err } + return details, nil +} - return s.getObjectsInfo(txn, ids) +func (s *dsObjectStore) List() ([]*model.ObjectInfo, error) { + var infos []*model.ObjectInfo + err := s.db.View(func(txn *badger.Txn) error { + ids, err := listIDsByPrefix(txn, pagesDetailsBase.Bytes()) + if err != nil { + return fmt.Errorf("list ids by prefix: %w", err) + } + + infos, err = s.getObjectsInfo(txn, ids) + return err + }) + return infos, err } func (s *dsObjectStore) HasIDs(ids ...string) (exists []string, err error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - for _, id := range ids { - if exist, err := hasObjectId(txn, id); err != nil { - return nil, err - } else if exist { - exists = append(exists, id) + err = s.db.View(func(txn *badger.Txn) error { + for _, id := range ids { + _, err := txn.Get(pagesDetailsBase.ChildString(id).Bytes()) + if err != nil && err != badger.ErrKeyNotFound { + return fmt.Errorf("get %s: %w", id, err) + } + if err == nil { + exists = append(exists, id) + } } - } - return exists, nil + return nil + }) + return exists, err } func (s *dsObjectStore) GetByIDs(ids ...string) ([]*model.ObjectInfo, error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - return s.getObjectsInfo(txn, ids) + var infos []*model.ObjectInfo + err := s.db.View(func(txn *badger.Txn) error { + var err error + infos, err = s.getObjectsInfo(txn, ids) + return err + }) + return infos, err } func (s *dsObjectStore) ListIds() ([]string, error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - return findByPrefix(txn, pagesDetailsBase.String()+"/", 0) + var ids []string + err := s.db.View(func(txn *badger.Txn) error { + var err error + ids, err = listIDsByPrefix(txn, pagesDetailsBase.Bytes()) + return err + }) + return ids, err } func (s *dsObjectStore) Prefix() string { return pagesPrefix } -func (s *dsObjectStore) Indexes() []localstore.Index { - return []localstore.Index{} -} - // TODO objstore: Just use dependency injection func (s *dsObjectStore) FTSearch() ftsearch.FTSearch { return s.fts } -func (s *dsObjectStore) makeFTSQuery(text string, dsq query.Query) (query.Query, error) { - if s.fts == nil { - return dsq, fmt.Errorf("fullText search not configured") +func (s *dsObjectStore) extractDetailsFromItem(it *badger.Item) (*model.ObjectDetails, error) { + key := it.Key() + if v, ok := s.cache.Get(key); ok { + return v.(*model.ObjectDetails), nil } - ids, err := s.fts.Search(text) - if err != nil { - return dsq, err - } - idsQuery := newIdsFilter(ids) - dsq.Filters = append([]query.Filter{idsQuery}, dsq.Filters...) - dsq.Orders = append([]query.Order{idsQuery}, dsq.Orders...) - return dsq, nil + return s.unmarshalDetailsFromItem(it) } -// getObjectDetails returns empty(not nil) details when not found in the DS -func getObjectDetails(txn noctxds.Txn, id string) (*model.ObjectDetails, error) { - val, err := txn.Get(pagesDetailsBase.ChildString(id)) - if err != nil { - if err != ds.ErrNotFound { - return nil, fmt.Errorf("failed to get relations: %w", err) +func (s *dsObjectStore) unmarshalDetailsFromItem(it *badger.Item) (*model.ObjectDetails, error) { + var details *model.ObjectDetails + err := it.Value(func(val []byte) error { + var err error + details, err = unmarshalDetails(detailsKeyToID(it.Key()), val) + if err != nil { + return fmt.Errorf("unmarshal details: %w", err) } - // return empty details in case not found - return &model.ObjectDetails{ - Details: &types.Struct{Fields: map[string]*types.Value{}}, - }, nil - } - - details, err := unmarshalDetails(id, val) + s.cache.Set(it.Key(), details, int64(details.Size())) + return nil + }) if err != nil { - return nil, fmt.Errorf("failed to unmarshal details: %w", err) + return nil, fmt.Errorf("get item value: %w", err) } - return details, nil } -func hasObjectId(txn noctxds.Txn, id string) (bool, error) { - if exists, err := txn.Has(pagesDetailsBase.ChildString(id)); err != nil { - return false, fmt.Errorf("failed to get details: %w", err) +func unmarshalDetails(id string, rawValue []byte) (*model.ObjectDetails, error) { + result := &model.ObjectDetails{} + if err := proto.Unmarshal(rawValue, result); err != nil { + return nil, err + } + if result.Details == nil { + result.Details = &types.Struct{Fields: map[string]*types.Value{}} + } + if result.Details.Fields == nil { + result.Details.Fields = map[string]*types.Value{} } else { - return exists, nil + pbtypes.StructDeleteEmptyFields(result.Details) + } + result.Details.Fields[database.RecordIDField] = pbtypes.ToValue(id) + return result, nil +} + +func detailsKeyToID(key []byte) string { + return path.Base(string(key)) +} + +type order struct { + filter.Order +} + +func (o order) Compare(lhs, rhs interface{}) (comp int) { + le := lhs.(database.Record) + re := rhs.(database.Record) + + if o.Order != nil { + comp = o.Order.Compare(le, re) + } + // when order isn't set or equal - sort by id + if comp == 0 { + if pbtypes.GetString(le.Details, "id") > pbtypes.GetString(re.Details, "id") { + return 1 + } + return -1 } + return comp } -func (s *dsObjectStore) getObjectInfo(txn noctxds.Txn, id string) (*model.ObjectInfo, error) { +func (o order) CalcScore(_ interface{}) float64 { + return 0 +} + +func (s *dsObjectStore) getObjectInfo(txn *badger.Txn, id string) (*model.ObjectInfo, error) { sbt, err := s.sbtProvider.Type(id) if err != nil { log.With("objectID", id).Errorf("failed to extract smartblock type %s", id) // todo rq: surpess error? @@ -622,26 +552,25 @@ func (s *dsObjectStore) getObjectInfo(txn noctxds.Txn, id string) (*model.Object } var details *types.Struct - if indexDetails, _ := sbt.Indexable(); !indexDetails { - if s.sourceService != nil { - details, err = s.sourceService.DetailsFromIdBasedSource(id) - if err != nil { - return nil, ErrObjectNotFound - } + if indexDetails, _ := sbt.Indexable(); !indexDetails && s.sourceService != nil { + details, err = s.sourceService.DetailsFromIdBasedSource(id) + if err != nil { + return nil, ErrObjectNotFound } } else { - detailsWrapped, err := getObjectDetails(txn, id) + it, err := txn.Get(pagesDetailsBase.ChildString(id).Bytes()) + if err != nil { + return nil, fmt.Errorf("get details: %w", err) + } + detailsModel, err := s.extractDetailsFromItem(it) if err != nil { return nil, err } - details = detailsWrapped.GetDetails() + details = detailsModel.Details } - - var snippet string - if val, err := txn.Get(pagesSnippetBase.ChildString(id)); err != nil && err != ds.ErrNotFound { + snippet, err := getValueTxn(txn, pagesSnippetBase.ChildString(id).Bytes(), bytesToString) + if err != nil && !isNotFound(err) { return nil, fmt.Errorf("failed to get snippet: %w", err) - } else { - snippet = string(val) } return &model.ObjectInfo{ @@ -652,12 +581,12 @@ func (s *dsObjectStore) getObjectInfo(txn noctxds.Txn, id string) (*model.Object }, nil } -func (s *dsObjectStore) getObjectsInfo(txn noctxds.Txn, ids []string) ([]*model.ObjectInfo, error) { - var objects []*model.ObjectInfo +func (s *dsObjectStore) getObjectsInfo(txn *badger.Txn, ids []string) ([]*model.ObjectInfo, error) { + objects := make([]*model.ObjectInfo, 0, len(ids)) for _, id := range ids { info, err := s.getObjectInfo(txn, id) if err != nil { - if strings.HasSuffix(err.Error(), "key not found") || err == ErrObjectNotFound || err == ErrNotAnObject { + if isNotFound(err) || err == ErrObjectNotFound || err == ErrNotAnObject { continue } return nil, err @@ -675,76 +604,21 @@ func (s *dsObjectStore) getObjectsInfo(txn noctxds.Txn, ids []string) ([]*model. } // Find to which IDs specified one has outbound links. -func findOutboundLinks(txn noctxds.Txn, id string) ([]string, error) { - return findByPrefix(txn, pagesOutboundLinksBase.String()+"/"+id+"/", 0) +func findOutboundLinks(txn *badger.Txn, id string) ([]string, error) { + return listIDsByPrefix(txn, pagesOutboundLinksBase.ChildString(id).Bytes()) } // Find from which IDs specified one has inbound links. -func findInboundLinks(txn noctxds.Txn, id string) ([]string, error) { - return findByPrefix(txn, pagesInboundLinksBase.String()+"/"+id+"/", 0) -} - -func findByPrefix(txn noctxds.Txn, prefix string, limit int) ([]string, error) { - results, err := txn.Query(query.Query{ - Prefix: prefix, - Limit: limit, - KeysOnly: true, - }) - if err != nil { - return nil, err - } - - return localstore.GetLeavesFromResults(results) -} - -// removeByPrefix query prefix and then remove keys in multiple TXs -func removeByPrefix(d noctxds.DSTxnBatching, prefix string) (int, error) { - results, err := d.Query(query.Query{ - Prefix: prefix, - KeysOnly: true, - }) - if err != nil { - return 0, err - } - var keys []ds.Key - for res := range results.Next() { - keys = append(keys, ds.NewKey(res.Key)) - } - b, err := d.Batch() - if err != nil { - return 0, err - } - var removed int - for _, key := range keys { - err := b.Delete(key) - if err != nil { - return removed, err - } - removed++ - } - - return removed, b.Commit() +func findInboundLinks(txn *badger.Txn, id string) ([]string, error) { + return listIDsByPrefix(txn, pagesInboundLinksBase.ChildString(id).Bytes()) } -func removeByPrefixInTx(txn noctxds.Txn, prefix string) (int, error) { - results, err := txn.Query(query.Query{ - Prefix: prefix, - KeysOnly: true, +func listIDsByPrefix(txn *badger.Txn, prefix []byte) ([]string, error) { + var ids []string + err := iterateKeysByPrefixTx(txn, prefix, func(key []byte) { + ids = append(ids, path.Base(string(key))) }) - if err != nil { - return 0, err - } - - var removed int - for res := range results.Next() { - err := txn.Delete(ds.NewKey(res.Key)) - if err != nil { - _ = results.Close() - return removed, err - } - removed++ - } - return removed, nil + return ids, err } func pageLinkKeys(id string, out []string) []ds.Key { @@ -764,7 +638,7 @@ func inboundLinkKey(from, to string) ds.Key { return pagesInboundLinksBase.ChildString(to).ChildString(from) } -func extractIdFromKey(key string) (id string) { +func extractIDFromKey(key string) (id string) { i := strings.LastIndexByte(key, '/') if i == -1 || len(key)-1 == i { return @@ -844,3 +718,8 @@ func (s *dsObjectStore) GetObjectTypes(urls []string) (ots []*model.ObjectType, } return } + +// bytesToString unmarshalls bytes to string +func bytesToString(b []byte) (string, error) { + return string(b), nil +} diff --git a/pkg/lib/localstore/objectstore/objects_test.go b/pkg/lib/localstore/objectstore/objects_test.go index e6d9ef02a1..b49449846f 100644 --- a/pkg/lib/localstore/objectstore/objects_test.go +++ b/pkg/lib/localstore/objectstore/objects_test.go @@ -1,7 +1,6 @@ package objectstore import ( - "context" "fmt" "math/rand" "testing" @@ -9,9 +8,6 @@ import ( "github.com/globalsign/mgo/bson" "github.com/gogo/protobuf/types" - ds "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" - "github.com/ipfs/go-datastore/sync" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -52,22 +48,6 @@ func TestDsObjectStore_UpdateLocalDetails(t *testing.T) { require.Equal(t, "2", pbtypes.GetString(recs[0].Details, "k2")) } -func TestDsObjectStore_PrefixQuery(t *testing.T) { - bds := sync.MutexWrap(ds.NewMapDatastore()) - err := bds.Put(context.Background(), ds.NewKey("/p1/abc/def/1"), []byte{}) - - require.NoError(t, err) - - res, err := bds.Query(context.Background(), query.Query{Prefix: "/p1/abc", KeysOnly: true}) - require.NoError(t, err) - - entries, err := res.Rest() - require.NoError(t, err) - require.Len(t, entries, 1) - require.Equal(t, "/p1/abc/def/1", entries[0].Key) - -} - func Test_removeByPrefix(t *testing.T) { s := newStoreFixture(t) var key = make([]byte, 32) @@ -86,18 +66,12 @@ func Test_removeByPrefix(t *testing.T) { require.NoError(t, s.UpdateObjectDetails(objId, nil)) require.NoError(t, s.UpdateObjectLinks(objId, links)) } - tx, err := s.ds.NewTransaction(false) - _, err = removeByPrefixInTx(tx, pagesInboundLinksBase.String()) - require.NotNil(t, err) - tx.Discard() - - got, err := removeByPrefix(s.ds, pagesInboundLinksBase.String()) - require.NoError(t, err) - require.Equal(t, 10*8000, got) - got, err = removeByPrefix(s.ds, pagesOutboundLinksBase.String()) + // Test huge transactions + outboundRemoved, inboundRemoved, err := s.eraseLinks() + require.Equal(t, 10*8000, outboundRemoved) + require.Equal(t, 10*8000, inboundRemoved) require.NoError(t, err) - require.Equal(t, 10*8000, got) } func TestList(t *testing.T) { diff --git a/pkg/lib/localstore/objectstore/queries.go b/pkg/lib/localstore/objectstore/queries.go index d4859f5188..8a1f399f50 100644 --- a/pkg/lib/localstore/objectstore/queries.go +++ b/pkg/lib/localstore/objectstore/queries.go @@ -3,221 +3,161 @@ package objectstore import ( "fmt" - ds "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" + "github.com/dgraph-io/badger/v3" + "github.com/huandu/skiplist" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" "github.com/anyproto/anytype-heart/pkg/lib/database" - "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/pkg/lib/database/filter" "github.com/anyproto/anytype-heart/pkg/lib/schema" "github.com/anyproto/anytype-heart/util/pbtypes" ) -// TODO: objstore: no one uses total -func (s *dsObjectStore) Query(sch schema.Schema, q database.Query) (records []database.Record, total int, err error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, 0, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - dsq, err := s.buildQuery(sch, q) +func (s *dsObjectStore) Query(sch schema.Schema, q database.Query) ([]database.Record, int, error) { + filters, err := s.buildQuery(sch, q) if err != nil { return nil, 0, fmt.Errorf("build query: %w", err) } + recs, err := s.QueryRaw(filters, q.Limit, q.Offset) + return recs, 0, err +} - res, err := txn.Query(dsq) - if err != nil { - return nil, 0, fmt.Errorf("error when querying ds: %w", err) +func (s *dsObjectStore) QueryRaw(filters *database.Filters, limit int, offset int) ([]database.Record, error) { + if filters == nil || filters.FilterObj == nil { + return nil, fmt.Errorf("filter cannot be nil or unitialized") } + skl := skiplist.New(order{filters.Order}) - var ( - results []database.Record - offset = q.Offset - ) - - // We use own limit/offset implementation in order to find out - // total number of records matching specified filters. Query - // returns this number for handy pagination on clients. - for rec := range res.Next() { - total++ + err := s.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.Prefix = pagesDetailsBase.Bytes() + iterator := txn.NewIterator(opts) + defer iterator.Close() - if offset > 0 { - offset-- - continue - } + for iterator.Rewind(); iterator.Valid(); iterator.Next() { + it := iterator.Item() + details, err := s.extractDetailsFromItem(it) + if err != nil { + return err + } - if q.Limit > 0 && len(results) >= q.Limit { - continue + rec := database.Record{Details: details.Details} + if filters.FilterObj != nil && filters.FilterObj.FilterObject(rec) { + if offset > 0 { + offset-- + continue + } + if limit > 0 && skl.Len() >= limit { + break + } + skl.Set(rec, nil) + } } + return nil + }) + if err != nil { + return nil, err + } - key := ds.NewKey(rec.Key) - keyList := key.List() - id := keyList[len(keyList)-1] - - var details *model.ObjectDetails - details, err = unmarshalDetails(id, rec.Value) - if err != nil { - total-- - log.Errorf("failed to unmarshal: %s", err.Error()) - continue - } - results = append(results, database.Record{Details: details.Details}) + records := make([]database.Record, 0, skl.Len()) + for it := skl.Front(); it != nil; it = it.Next() { + records = append(records, it.Key().(database.Record)) } - return results, total, nil + return records, nil } -func (s *dsObjectStore) buildQuery(sch schema.Schema, q database.Query) (query.Query, error) { - dsq, err := q.DSQuery(sch) +func (s *dsObjectStore) buildQuery(sch schema.Schema, q database.Query) (*database.Filters, error) { + filters, err := database.NewFilters(q, sch, s) if err != nil { - return query.Query{}, fmt.Errorf("init datastore query: %w", err) + return nil, fmt.Errorf("new filters: %w", err) } - dsq.Offset = 0 - dsq.Limit = 0 - dsq.Prefix = pagesDetailsBase.String() + "/" discardSystemObjects := newSmartblockTypesFilter(s.sbtProvider, true, []smartblock.SmartBlockType{ smartblock.SmartBlockTypeArchive, smartblock.SmartBlockTypeHome, }) - dsq.Filters = append([]query.Filter{discardSystemObjects}, dsq.Filters...) + filters.FilterObj = filter.AndFilters{filters.FilterObj, discardSystemObjects} if q.FullText != "" { - dsq, err = s.makeFTSQuery(q.FullText, dsq) + filters, err = s.makeFTSQuery(q.FullText, filters) if err != nil { - return query.Query{}, fmt.Errorf("append full text search query: %w", err) + return nil, fmt.Errorf("append full text search query: %w", err) } } - return dsq, nil + return filters, nil } -func (s *dsObjectStore) QueryRaw(f *database.Filters) (records []database.Record, err error) { - if f == nil || f.FilterObj == nil { - return nil, fmt.Errorf("filter cannot be nil or unitialized") - } - dsq := query.Query{ - Filters: []query.Filter{f}, +func (s *dsObjectStore) makeFTSQuery(text string, filters *database.Filters) (*database.Filters, error) { + if s.fts == nil { + return filters, fmt.Errorf("fullText search not configured") } - txn, err := s.ds.NewTransaction(true) + ids, err := s.fts.Search(text) if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - dsq.Prefix = pagesDetailsBase.String() + "/" - - res, err := txn.Query(dsq) - if err != nil { - return nil, fmt.Errorf("error when querying ds: %w", err) - } - - for rec := range res.Next() { - key := ds.NewKey(rec.Key) - keyList := key.List() - id := keyList[len(keyList)-1] - - var details *model.ObjectDetails - details, err = unmarshalDetails(id, rec.Value) - if err != nil { - log.Errorf("failed to unmarshal: %s", err.Error()) - continue - } - records = append(records, database.Record{Details: details.Details}) + return filters, err } - return + idsQuery := newIdsFilter(ids) + filters.FilterObj = filter.AndFilters{filters.FilterObj, idsQuery} + filters.Order = filter.SetOrder(append([]filter.Order{idsQuery}, filters.Order)) + return filters, nil } // TODO: objstore: no one uses total func (s *dsObjectStore) QueryObjectIDs(q database.Query, smartBlockTypes []smartblock.SmartBlockType) (ids []string, total int, err error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, 0, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - dsq, err := s.buildQuery(nil, q) + filters, err := s.buildQuery(nil, q) if err != nil { - return + return nil, 0, fmt.Errorf("build query: %w", err) } if len(smartBlockTypes) > 0 { - dsq.Filters = append([]query.Filter{newSmartblockTypesFilter(s.sbtProvider, false, smartBlockTypes)}, dsq.Filters...) + filters.FilterObj = filter.AndFilters{newSmartblockTypesFilter(s.sbtProvider, false, smartBlockTypes), filters.FilterObj} } - - res, err := txn.Query(dsq) + recs, err := s.QueryRaw(filters, q.Limit, q.Offset) if err != nil { - return nil, 0, fmt.Errorf("error when querying ds: %w", err) + return nil, 0, fmt.Errorf("query raw: %w", err) } - - var ( - offset = q.Offset - ) - - // We use own limit/offset implementation in order to find out - // total number of records matching specified filters. Query - // returns this number for handy pagination on clients. - for rec := range res.Next() { - if rec.Error != nil { - return nil, 0, rec.Error - } - total++ - - if offset > 0 { - offset-- - continue - } - - if q.Limit > 0 && len(ids) >= q.Limit { - continue - } - - key := ds.NewKey(rec.Key) - keyList := key.List() - id := keyList[len(keyList)-1] - ids = append(ids, id) + ids = make([]string, 0, len(recs)) + for _, rec := range recs { + ids = append(ids, pbtypes.GetString(rec.Details, bundle.RelationKeyId.String())) } - return ids, total, nil + return ids, 0, nil } func (s *dsObjectStore) QueryByID(ids []string) (records []database.Record, err error) { - txn, err := s.ds.NewTransaction(true) - if err != nil { - return nil, fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - - for _, id := range ids { - if sbt, err := s.sbtProvider.Type(id); err == nil { - if indexDetails, _ := sbt.Indexable(); !indexDetails && s.sourceService != nil { - details, err := s.sourceService.DetailsFromIdBasedSource(id) - if err != nil { - log.Errorf("QueryByIds failed to GetDetailsFromIdBasedSource id: %s", id) + err = s.db.View(func(txn *badger.Txn) error { + for _, id := range ids { + if sbt, err := s.sbtProvider.Type(id); err == nil { + if indexDetails, _ := sbt.Indexable(); !indexDetails && s.sourceService != nil { + details, err := s.sourceService.DetailsFromIdBasedSource(id) + if err != nil { + log.Errorf("QueryByIds failed to GetDetailsFromIdBasedSource id: %s", id) + continue + } + details.Fields[database.RecordIDField] = pbtypes.ToValue(id) + records = append(records, database.Record{Details: details}) continue } - details.Fields[database.RecordIDField] = pbtypes.ToValue(id) - records = append(records, database.Record{Details: details}) + } + it, err := txn.Get(pagesDetailsBase.ChildString(id).Bytes()) + if err != nil { + log.Infof("QueryByIds failed to find id: %s", id) continue } - } - v, err := txn.Get(pagesDetailsBase.ChildString(id)) - if err != nil { - log.Infof("QueryByIds failed to find id: %s", id) - continue - } - var details *model.ObjectDetails - details, err = unmarshalDetails(id, v) - if err != nil { - log.Errorf("QueryByIds failed to unmarshal id: %s", id) - continue + details, err := s.extractDetailsFromItem(it) + if err != nil { + log.Errorf("QueryByIds failed to extract details: %s", id) + continue + } + records = append(records, database.Record{Details: details.Details}) } - records = append(records, database.Record{Details: details.Details}) - } - + return nil + }) return } func (s *dsObjectStore) QueryByIDAndSubscribeForChanges(ids []string, sub database.Subscription) (records []database.Record, closeFunc func(), err error) { - s.l.Lock() - defer s.l.Unlock() + s.Lock() + defer s.Unlock() if sub == nil { err = fmt.Errorf("subscription func is nil") @@ -236,6 +176,5 @@ func (s *dsObjectStore) QueryByIDAndSubscribeForChanges(ids []string, sub databa } s.addSubscriptionIfNotExists(sub) - return } diff --git a/pkg/lib/localstore/objectstore/queries_test.go b/pkg/lib/localstore/objectstore/queries_test.go index 94eb8d058e..3957edced3 100644 --- a/pkg/lib/localstore/objectstore/queries_test.go +++ b/pkg/lib/localstore/objectstore/queries_test.go @@ -3,22 +3,22 @@ package objectstore import ( "context" "fmt" + "path/filepath" "testing" "time" "github.com/anyproto/any-sync/app" + "github.com/dgraph-io/badger/v3" "github.com/gogo/protobuf/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - dsbadgerv3 "github.com/textileio/go-ds-badger3" "github.com/anyproto/anytype-heart/core/wallet" "github.com/anyproto/anytype-heart/core/wallet/mock_wallet" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" "github.com/anyproto/anytype-heart/pkg/lib/database" - "github.com/anyproto/anytype-heart/pkg/lib/datastore/noctxds" "github.com/anyproto/anytype-heart/pkg/lib/localstore/ftsearch" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" "github.com/anyproto/anytype-heart/space/typeprovider/mock_typeprovider" @@ -30,11 +30,6 @@ type storeFixture struct { } func newStoreFixture(t *testing.T) *storeFixture { - ds, err := dsbadgerv3.NewDatastore(t.TempDir(), &dsbadgerv3.DefaultOptions) - require.NoError(t, err) - - noCtxDS := noctxds.New(ds) - typeProvider := mock_typeprovider.NewMockSmartBlockTypeProvider(t) typeProvider.EXPECT().Type(mock.Anything).Return(smartblock.SmartBlockTypePage, nil).Maybe() @@ -45,16 +40,19 @@ func newStoreFixture(t *testing.T) *storeFixture { fullText := ftsearch.New() testApp := &app.App{} testApp.Register(walletService) - err = fullText.Init(testApp) + err := fullText.Init(testApp) require.NoError(t, err) err = fullText.Run(context.Background()) require.NoError(t, err) + db, err := badger.Open(badger.DefaultOptions(filepath.Join(t.TempDir(), "badger"))) + require.NoError(t, err) + return &storeFixture{ dsObjectStore: &dsObjectStore{ - ds: noCtxDS, sbtProvider: typeProvider, fts: fullText, + db: db, }, } } @@ -578,7 +576,7 @@ func TestQueryRaw(t *testing.T) { t.Run("with nil filter expect error", func(t *testing.T) { s := newStoreFixture(t) - _, err := s.QueryRaw(nil) + _, err := s.QueryRaw(nil, 0, 0) require.Error(t, err) }) @@ -587,7 +585,7 @@ func TestQueryRaw(t *testing.T) { obj1 := makeObjectWithName("id1", "name1") s.addObjects(t, []testObject{obj1}) - _, err := s.QueryRaw(&database.Filters{}) + _, err := s.QueryRaw(&database.Filters{}, 0, 0) require.Error(t, err) }) @@ -601,7 +599,7 @@ func TestQueryRaw(t *testing.T) { flt, err := database.NewFilters(database.Query{}, nil, nil) require.NoError(t, err) - recs, err := s.QueryRaw(flt) + recs, err := s.QueryRaw(flt, 0, 0) require.NoError(t, err) assertRecordsEqual(t, []testObject{obj1, obj2, obj3}, recs) }) @@ -624,7 +622,7 @@ func TestQueryRaw(t *testing.T) { }, nil, nil) require.NoError(t, err) - recs, err := s.QueryRaw(flt) + recs, err := s.QueryRaw(flt, 0, 0) require.NoError(t, err) assertRecordsEqual(t, []testObject{obj1, obj3}, recs) }) @@ -671,13 +669,15 @@ func TestQueryById(t *testing.T) { s.sbtProvider = typeProvider obj1 := makeObjectWithName("id1", "name1") - typeProvider.EXPECT().Type("id1").Return(smartblock.SmartBlockTypePage, nil) + // TODO WHy is was ok? + // typeProvider.EXPECT().Type("id1").Return(smartblock.SmartBlockTypePage, nil) obj2 := makeObjectWithName("id2", "name2") typeProvider.EXPECT().Type("id2").Return(smartblock.SmartBlockTypePage, nil) obj3 := makeObjectWithName("id3", "name3") - typeProvider.EXPECT().Type("id3").Return(smartblock.SmartBlockTypePage, nil) + // TODO WHy is was ok? + // typeProvider.EXPECT().Type("id3").Return(smartblock.SmartBlockTypePage, nil) // obj4 is not indexable, so don't try to add it to store obj4 := makeObjectWithName("id4", "i'm special") diff --git a/pkg/lib/localstore/objectstore/update.go b/pkg/lib/localstore/objectstore/update.go index 2c283b8af9..4db2730dcd 100644 --- a/pkg/lib/localstore/objectstore/update.go +++ b/pkg/lib/localstore/objectstore/update.go @@ -1,164 +1,128 @@ package objectstore import ( - "errors" "fmt" - "github.com/anyproto/anytype-heart/util/debug" "github.com/dgraph-io/badger/v3" "github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/types" - ds "github.com/ipfs/go-datastore" - "github.com/anyproto/anytype-heart/metrics" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/database" - "github.com/anyproto/anytype-heart/pkg/lib/datastore/noctxds" - "github.com/anyproto/anytype-heart/pkg/lib/localstore" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/debug" "github.com/anyproto/anytype-heart/util/pbtypes" "github.com/anyproto/anytype-heart/util/slice" ) func (s *dsObjectStore) UpdateObjectDetails(id string, details *types.Struct) error { - s.l.Lock() - defer s.l.Unlock() - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) + if details == nil { + return nil } - defer txn.Discard() - var ( - before model.ObjectInfo - ) - - if details != nil { - exInfo, err := s.getObjectInfo(txn, id) - if err != nil { - log.Debugf("UpdateObject failed to get ex state for object %s: %s", id, err.Error()) - } - - if exInfo != nil { - before = *exInfo - } else { - // init an empty state to skip nil checks later - before = model.ObjectInfo{ - Details: &types.Struct{Fields: map[string]*types.Value{}}, - } - } + if details.Fields == nil { + return fmt.Errorf("details fields are nil") } - - err = s.updateObjectDetails(txn, id, before, details) - if err != nil { - return err + if pbtypes.GetString(details, bundle.RelationKeyWorkspaceId.String()) == "" { + log.With("objectID", id).With("stack", debug.StackCompact(false)).Warnf("workspaceId erased") } - err = txn.Commit() - if err != nil { - return err + newDetails := &model.ObjectDetails{ + Details: details, } + key := pagesDetailsBase.ChildString(id).Bytes() + txErr := s.updateTxn(func(txn *badger.Txn) error { + oldDetails, err := s.extractDetailsByKey(txn, key) + if err != nil && !isNotFound(err) { + return fmt.Errorf("extract details: %w", err) + } + if oldDetails != nil && oldDetails.Details.Equal(newDetails.Details) { + return ErrDetailsNotChanged + } + // Ensure ID is set + details.Fields[bundle.RelationKeyId.String()] = pbtypes.String(id) + s.sendUpdatesToSubscriptions(id, details) + val, err := proto.Marshal(newDetails) + if err != nil { + return fmt.Errorf("marshal details: %w", err) + } + return txn.Set(key, val) + }) + if txErr != nil { + return txErr + } + s.cache.Set(key, newDetails, int64(newDetails.Size())) return nil } -func (s *dsObjectStore) UpdateObjectLinks(id string, links []string) error { - s.l.Lock() - defer s.l.Unlock() - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) +func (s *dsObjectStore) extractDetailsByKey(txn *badger.Txn, key []byte) (*model.ObjectDetails, error) { + raw, ok := s.cache.Get(key) + if ok { + return raw.(*model.ObjectDetails), nil } - defer txn.Discard() - err = s.updateObjectLinks(txn, id, links) + it, err := txn.Get(key) if err != nil { - return err + return nil, fmt.Errorf("get item: %w", err) } - return txn.Commit() + return s.unmarshalDetailsFromItem(it) } -func (s *dsObjectStore) UpdateObjectSnippet(id string, snippet string) error { - s.l.Lock() - defer s.l.Unlock() - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() +func (s *dsObjectStore) UpdateObjectLinks(id string, links []string) error { + return s.updateTxn(func(txn *badger.Txn) error { + return s.updateObjectLinks(txn, id, links) + }) +} - if val, err := txn.Get(pagesSnippetBase.ChildString(id)); err == ds.ErrNotFound || string(val) != snippet { - if err := s.updateSnippet(txn, id, snippet); err != nil { - return err - } - } - return txn.Commit() +func (s *dsObjectStore) UpdateObjectSnippet(id string, snippet string) error { + return setValue(s.db, pagesSnippetBase.ChildString(id).Bytes(), snippet) } func (s *dsObjectStore) UpdatePendingLocalDetails(id string, proc func(details *types.Struct) (*types.Struct, error)) error { - // todo: review this method. Any other way to do this? - for { - err := s.updatePendingLocalDetails(id, proc) - if errors.Is(err, badger.ErrConflict) { - continue + return s.updateTxn(func(txn *badger.Txn) error { + key := pendingDetailsBase.ChildString(id).Bytes() + + objDetails, err := s.getPendingLocalDetails(txn, key) + if err != nil && !isNotFound(err) { + return fmt.Errorf("get pending details: %w", err) } + + oldDetails := objDetails.GetDetails() + if oldDetails == nil || oldDetails.Fields == nil { + oldDetails = &types.Struct{Fields: map[string]*types.Value{}} + } + newDetails, err := proc(oldDetails) if err != nil { - return err + return fmt.Errorf("run a modifier: %w", err) + } + if newDetails == nil { + err = txn.Delete(key) + if err != nil { + return err + } + return nil } - return nil - } -} - -func (s *dsObjectStore) updatePendingLocalDetails(id string, proc func(details *types.Struct) (*types.Struct, error)) error { - txn, err := s.ds.NewTransaction(false) - if err != nil { - return fmt.Errorf("error creating txn in datastore: %w", err) - } - defer txn.Discard() - key := pendingDetailsBase.ChildString(id) - - objDetails, err := s.getPendingLocalDetails(txn, id) - if err != nil && err != ds.ErrNotFound { - return fmt.Errorf("get pending details: %w", err) - } - details := objDetails.GetDetails() - if details == nil { - details = &types.Struct{Fields: map[string]*types.Value{}} - } - if details.Fields == nil { - details.Fields = map[string]*types.Value{} - } - details, err = proc(details) - if err != nil { - return fmt.Errorf("run a modifier: %w", err) - } - if details == nil { - err = txn.Delete(key) + if newDetails.Fields == nil { + newDetails.Fields = map[string]*types.Value{} + } + newDetails.Fields[bundle.RelationKeyId.String()] = pbtypes.String(id) + err = setValueTxn(txn, key, &model.ObjectDetails{Details: newDetails}) if err != nil { - return err + return fmt.Errorf("put pending details: %w", err) } - return txn.Commit() - } - b, err := proto.Marshal(&model.ObjectDetails{Details: details}) - if err != nil { - return err - } - err = txn.Put(key, b) - if err != nil { - return err - } - - return txn.Commit() + return nil + }) } -func (s *dsObjectStore) getPendingLocalDetails(txn noctxds.Txn, id string) (*model.ObjectDetails, error) { - val, err := txn.Get(pendingDetailsBase.ChildString(id)) - if err != nil { - return nil, err - } - return unmarshalDetails(id, val) +func (s *dsObjectStore) getPendingLocalDetails(txn *badger.Txn, key []byte) (*model.ObjectDetails, error) { + return getValueTxn(txn, key, func(raw []byte) (*model.ObjectDetails, error) { + var res model.ObjectDetails + err := proto.Unmarshal(raw, &res) + return &res, err + }) } -func (s *dsObjectStore) updateObjectLinks(txn noctxds.Txn, id string, links []string) error { +func (s *dsObjectStore) updateObjectLinks(txn *badger.Txn, id string, links []string) error { exLinks, err := findOutboundLinks(txn, id) if err != nil { log.Errorf("error while finding outbound links for %s: %s", id, err) @@ -168,15 +132,15 @@ func (s *dsObjectStore) updateObjectLinks(txn noctxds.Txn, id string, links []st removedLinks, addedLinks = slice.DifferenceRemovedAdded(exLinks, links) if len(addedLinks) > 0 { for _, k := range pageLinkKeys(id, addedLinks) { - if err := txn.Put(k, nil); err != nil { - return err + err := txn.Set(k.Bytes(), nil) + if err != nil { + return fmt.Errorf("setting link %s: %w", k, err) } } } - if len(removedLinks) > 0 { for _, k := range pageLinkKeys(id, removedLinks) { - if err := txn.Delete(k); err != nil { + if err := txn.Delete(k.Bytes()); err != nil { return err } } @@ -185,58 +149,11 @@ func (s *dsObjectStore) updateObjectLinks(txn noctxds.Txn, id string, links []st return nil } -func (s *dsObjectStore) updateObjectDetails(txn noctxds.Txn, id string, before model.ObjectInfo, details *types.Struct) error { - if details != nil { - if err := s.updateDetails(txn, id, &model.ObjectDetails{Details: before.Details}, &model.ObjectDetails{Details: details}); err != nil { - return err - } - } - - return nil -} - -func (s *dsObjectStore) updateDetails(txn noctxds.Txn, id string, oldDetails *model.ObjectDetails, newDetails *model.ObjectDetails) error { - metrics.ObjectDetailsUpdatedCounter.Inc() - detailsKey := pagesDetailsBase.ChildString(id) - - if newDetails.GetDetails().GetFields() == nil { - return fmt.Errorf("newDetails is nil") - } - - newDetails.Details.Fields[bundle.RelationKeyId.String()] = pbtypes.String(id) // always ensure we have id set - b, err := proto.Marshal(newDetails) - if err != nil { - return err - } - err = txn.Put(detailsKey, b) - if err != nil { - return err - } - - if pbtypes.GetString(newDetails.Details, bundle.RelationKeyWorkspaceId.String()) == "" { - log.With("objectID", id).With("stack", debug.StackCompact(false)).Warnf("workspaceId erased") - } - - if oldDetails.GetDetails().Equal(newDetails.GetDetails()) { - return ErrDetailsNotChanged - } - - err = localstore.UpdateIndexesWithTxn(s, txn, oldDetails, newDetails, id) - if err != nil { - return err - } - - if newDetails != nil && newDetails.Details.Fields != nil { - s.sendUpdatesToSubscriptions(id, newDetails.Details) - } - - return nil -} - -// should be called under the mutex func (s *dsObjectStore) sendUpdatesToSubscriptions(id string, details *types.Struct) { detCopy := pbtypes.CopyStruct(details) detCopy.Fields[database.RecordIDField] = pbtypes.ToValue(id) + s.RLock() + defer s.RUnlock() if s.onChangeCallback != nil { s.onChangeCallback(database.Record{ Details: detCopy, @@ -248,8 +165,3 @@ func (s *dsObjectStore) sendUpdatesToSubscriptions(id string, details *types.Str }(s.subscriptions[i]) } } - -func (s *dsObjectStore) updateSnippet(txn noctxds.Txn, id string, snippet string) error { - snippetKey := pagesSnippetBase.ChildString(id) - return txn.Put(snippetKey, []byte(snippet)) -} diff --git a/pkg/lib/localstore/stores.go b/pkg/lib/localstore/stores.go index fd0e0bcfdb..2d2e1725e0 100644 --- a/pkg/lib/localstore/stores.go +++ b/pkg/lib/localstore/stores.go @@ -5,15 +5,15 @@ import ( "fmt" "strings" - ds "github.com/anyproto/anytype-heart/pkg/lib/datastore/noctxds" - "github.com/anyproto/anytype-heart/pkg/lib/logging" - "github.com/anyproto/anytype-heart/util/slice" - "github.com/dgtony/collections/polymorph" "github.com/dgtony/collections/slices" dsCtx "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" "github.com/multiformats/go-base32" + + ds "github.com/anyproto/anytype-heart/pkg/lib/datastore/noctxds" + "github.com/anyproto/anytype-heart/pkg/lib/logging" + "github.com/anyproto/anytype-heart/util/slice" ) var ( @@ -182,6 +182,9 @@ func EraseIndex(index Index, datastore ds.DSTxnBatching) error { } keys, err := ExtractKeysFromResults(res) + if err != nil { + return fmt.Errorf("extract keys from results: %w", err) + } b, err := datastore.Batch() if err != nil { return err @@ -352,29 +355,6 @@ func GetLeavesFromResults(results query.Results) ([]string, error) { return leaves, nil } -func GetKeyPartFromResults(results query.Results, from, to int, removeDuplicates bool) ([]string, error) { - var keyParts []string - for res := range results.Next() { - if res.Error != nil { - return nil, res.Error - } - p, err := CarveKeyParts(res.Key, from, to) - if err != nil { - // should not happen, lets early-close iterator and return error - _ = results.Close() - return nil, err - } - if removeDuplicates { - if slice.FindPos(keyParts, p) >= 0 { - continue - } - } - keyParts = append(keyParts, p) - } - - return keyParts, nil -} - func ExtractKeysFromResults(results query.Results) ([]string, error) { var keys []string for res := range results.Next() { diff --git a/pkg/lib/logging/logging.go b/pkg/lib/logging/logging.go index 116b23880d..417714fa27 100644 --- a/pkg/lib/logging/logging.go +++ b/pkg/lib/logging/logging.go @@ -35,26 +35,36 @@ func LoggerNotSugared(system string) *zap.Logger { return lg.Logger } -func LevelsFromStr(s string) map[string]string { - levels := make(map[string]string) +// LevelsFromStr parses a string of the form "name1=DEBUG;prefix*=WARN;*=ERROR" into a slice of NamedLevel +// it may be useful to parse the log level from the OS env var +func LevelsFromStr(s string) (levels []logger.NamedLevel) { for _, kv := range strings.Split(s, ";") { + if kv == "" { + continue + } strings.TrimSpace(kv) parts := strings.Split(kv, "=") var key, value string if len(parts) == 1 { key = "*" - value = parts[0] - levels["*"] = parts[0] + value = strings.TrimSpace(parts[0]) } else if len(parts) == 2 { - key = parts[0] - value = parts[1] + key = strings.TrimSpace(parts[0]) + value = strings.TrimSpace(parts[1]) + } else { + fmt.Printf("invalid log level format. It should be something like `prefix*=LEVEL;*suffix=LEVEL`, where LEVEL is one of valid log levels\n") + continue } + if key == "" || value == "" { + continue + } + _, err := zap.ParseAtomicLevel(value) if err != nil { fmt.Printf("Can't parse log level %s: %s\n", parts[0], err.Error()) continue } - levels[key] = value + levels = append(levels, logger.NamedLevel{Name: key, Level: value}) } return levels } @@ -70,6 +80,6 @@ func init() { } else { registerGelfSink(&cfg) } - cfg.Levels = logger.LevelsFromStr(os.Getenv("ANYTYPE_LOG_LEVEL")) + cfg.Levels = LevelsFromStr(os.Getenv("ANYTYPE_LOG_LEVEL")) cfg.ApplyGlobal() } diff --git a/pkg/lib/logging/logging_test.go b/pkg/lib/logging/logging_test.go new file mode 100644 index 0000000000..fb888695a8 --- /dev/null +++ b/pkg/lib/logging/logging_test.go @@ -0,0 +1,80 @@ +package logging + +import ( + "reflect" + "testing" + + "github.com/anyproto/any-sync/app/logger" + "github.com/stretchr/testify/assert" +) + +func TestLevelsFromStr(t *testing.T) { + tests := []struct { + name string + arg string + want []logger.NamedLevel + }{ + { + name: "Correct Input", + arg: "name1=DEBUG;prefix*=WARN;*=ERROR", + want: []logger.NamedLevel{ + {Name: "name1", Level: "DEBUG"}, + {Name: "prefix*", Level: "WARN"}, + {Name: "*", Level: "ERROR"}, + }, + }, + { + name: "Correct Input with whitespaces", + arg: "name1 = DEBUG ; prefix* = WARN; *= ERROR", + want: []logger.NamedLevel{ + {Name: "name1", Level: "DEBUG"}, + {Name: "prefix*", Level: "WARN"}, + {Name: "*", Level: "ERROR"}, + }, + }, + { + name: "Extra semicolon", + arg: "name1=DEBUG;prefix*=WARN;*=ERROR;", + want: []logger.NamedLevel{ + {Name: "name1", Level: "DEBUG"}, + {Name: "prefix*", Level: "WARN"}, + {Name: "*", Level: "ERROR"}, + }, + }, + { + name: "Invalid level", + arg: "name1=DEBUG;prefix*=WARN;*=INVALID", + want: []logger.NamedLevel{ + {Name: "name1", Level: "DEBUG"}, + {Name: "prefix*", Level: "WARN"}, + }, + }, + { + name: "Empty", + arg: "", + want: nil, + }, + { + name: "spaces", + arg: " ", + want: nil, + }, + { + name: "invalid assignment", + arg: "a=b=c=d", + want: nil, + }, + { + name: "wtf", + arg: " ;fsg;;gf;gf;gd;gd;g;fd;dfg;;gfd----gd-gfd-g-gdf-gd-g-gd-fg-====gdf=gf==;;;==;=;=;=;=g ", + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := LevelsFromStr(tt.arg) + assert.True(t, reflect.DeepEqual(got, tt.want), "LevelsFromStr() = %v, want %v", got, tt.want) + }) + } +} diff --git a/pkg/lib/pb/model/models.pb.go b/pkg/lib/pb/model/models.pb.go index a0e7043943..1161d7dcf8 100644 --- a/pkg/lib/pb/model/models.pb.go +++ b/pkg/lib/pb/model/models.pb.go @@ -2691,6 +2691,7 @@ type BlockContentDataviewView struct { GroupRelationKey string `protobuf:"bytes,11,opt,name=groupRelationKey,proto3" json:"groupRelationKey,omitempty"` GroupBackgroundColors bool `protobuf:"varint,12,opt,name=groupBackgroundColors,proto3" json:"groupBackgroundColors,omitempty"` PageLimit int32 `protobuf:"varint,13,opt,name=pageLimit,proto3" json:"pageLimit,omitempty"` + DefaultTemplateId string `protobuf:"bytes,14,opt,name=defaultTemplateId,proto3" json:"defaultTemplateId,omitempty"` } func (m *BlockContentDataviewView) Reset() { *m = BlockContentDataviewView{} } @@ -2817,6 +2818,13 @@ func (m *BlockContentDataviewView) GetPageLimit() int32 { return 0 } +func (m *BlockContentDataviewView) GetDefaultTemplateId() string { + if m != nil { + return m.DefaultTemplateId + } + return "" +} + type BlockContentDataviewRelation struct { Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` IsVisible bool `protobuf:"varint,2,opt,name=isVisible,proto3" json:"isVisible,omitempty"` @@ -5687,329 +5695,331 @@ func init() { } var fileDescriptor_98a910b73321e591 = []byte{ - // 5149 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5b, 0x4d, 0x6c, 0x24, 0xc7, - 0x75, 0xe6, 0xfc, 0xcf, 0xbc, 0x21, 0xb9, 0xc5, 0x12, 0xb5, 0x9a, 0xb4, 0xe4, 0x0d, 0xdd, 0x91, - 0xe5, 0xf5, 0x5a, 0xe6, 0x4a, 0xfb, 0x63, 0xc9, 0x4e, 0x24, 0x99, 0x3f, 0xbb, 0x26, 0xa1, 0x5d, - 0x91, 0xee, 0xa1, 0xa8, 0x58, 0x48, 0x02, 0xd7, 0x4c, 0x17, 0x67, 0x5a, 0xec, 0xe9, 0x1a, 0x77, - 0xd7, 0x70, 0x49, 0x03, 0x01, 0x9c, 0x5f, 0x03, 0x39, 0x04, 0x46, 0x80, 0x1c, 0x03, 0x38, 0x39, - 0xe7, 0x66, 0x18, 0x49, 0x80, 0x5c, 0x03, 0x04, 0xc8, 0x21, 0x3e, 0x1a, 0x48, 0x90, 0x04, 0xd2, - 0x31, 0x87, 0x00, 0x39, 0xe7, 0x10, 0xbc, 0x57, 0xd5, 0x3d, 0x3d, 0x3f, 0xe4, 0x0e, 0x65, 0x9f, - 0xa6, 0xeb, 0xf5, 0x7b, 0xaf, 0x5f, 0x55, 0xbd, 0xfa, 0xea, 0xbd, 0x57, 0x35, 0xf0, 0xea, 0xf0, - 0xb4, 0x77, 0x37, 0x0c, 0x3a, 0x77, 0x87, 0x9d, 0xbb, 0x03, 0xe5, 0xcb, 0xf0, 0xee, 0x30, 0x56, - 0x5a, 0x25, 0xa6, 0x91, 0x6c, 0x52, 0x8b, 0xaf, 0x88, 0xe8, 0x42, 0x5f, 0x0c, 0xe5, 0x26, 0x51, - 0x9d, 0x57, 0x7a, 0x4a, 0xf5, 0x42, 0x69, 0x58, 0x3b, 0xa3, 0x93, 0xbb, 0x89, 0x8e, 0x47, 0x5d, - 0x6d, 0x98, 0xdd, 0x7f, 0x2a, 0xc1, 0xcd, 0xf6, 0x40, 0xc4, 0x7a, 0x3b, 0x54, 0xdd, 0xd3, 0x76, - 0x24, 0x86, 0x49, 0x5f, 0xe9, 0x6d, 0x91, 0x48, 0xfe, 0x3a, 0x54, 0x3b, 0x48, 0x4c, 0x5a, 0x85, - 0x8d, 0xd2, 0xed, 0xe6, 0xbd, 0xf5, 0xcd, 0x09, 0xc5, 0x9b, 0x24, 0xe1, 0x59, 0x1e, 0xfe, 0x26, - 0xd4, 0x7c, 0xa9, 0x45, 0x10, 0x26, 0xad, 0xe2, 0x46, 0xe1, 0x76, 0xf3, 0xde, 0x4b, 0x9b, 0xe6, - 0xc3, 0x9b, 0xe9, 0x87, 0x37, 0xdb, 0xf4, 0x61, 0x2f, 0xe5, 0xe3, 0xf7, 0xa1, 0x7e, 0x12, 0x84, - 0xf2, 0x7d, 0x79, 0x91, 0xb4, 0x4a, 0x57, 0xcb, 0x64, 0x8c, 0xfc, 0x3d, 0x58, 0x95, 0xe7, 0x3a, - 0x16, 0x9e, 0x0c, 0x85, 0x0e, 0x54, 0x94, 0xb4, 0xca, 0x64, 0xdd, 0x4b, 0x53, 0xd6, 0xa5, 0xef, - 0xbd, 0x29, 0x76, 0xbe, 0x01, 0x4d, 0xd5, 0xf9, 0x44, 0x76, 0xf5, 0xd1, 0xc5, 0x50, 0x26, 0xad, - 0xca, 0x46, 0xe9, 0x76, 0xc3, 0xcb, 0x93, 0xf8, 0x37, 0xa0, 0xd9, 0x55, 0x61, 0x28, 0xbb, 0x46, - 0x7f, 0xf5, 0x6a, 0xd3, 0xf2, 0xbc, 0xfc, 0x01, 0xbc, 0x18, 0xcb, 0x81, 0x3a, 0x93, 0xfe, 0x4e, - 0x46, 0xa5, 0xfe, 0xd5, 0xe9, 0x33, 0xf3, 0x5f, 0xf2, 0x2d, 0x58, 0x89, 0xad, 0x7d, 0x4f, 0x82, - 0xe8, 0x34, 0x69, 0xd5, 0xa8, 0x4b, 0x2f, 0x5f, 0xd2, 0x25, 0xe4, 0xf1, 0x26, 0x25, 0xdc, 0x7f, - 0xdf, 0x86, 0x0a, 0x4d, 0x08, 0x5f, 0x85, 0x62, 0xe0, 0xb7, 0x0a, 0x1b, 0x85, 0xdb, 0x0d, 0xaf, - 0x18, 0xf8, 0xfc, 0x2e, 0x54, 0x4f, 0x02, 0x19, 0xfa, 0xcf, 0x9d, 0x17, 0xcb, 0xc6, 0x1f, 0xc1, - 0x72, 0x2c, 0x13, 0x1d, 0x07, 0xb6, 0xff, 0x66, 0x6a, 0xbe, 0x38, 0x6f, 0xf6, 0x37, 0xbd, 0x1c, - 0xa3, 0x37, 0x21, 0x86, 0xe3, 0xdc, 0xed, 0x07, 0xa1, 0x1f, 0xcb, 0x68, 0xdf, 0x37, 0xb3, 0xd4, - 0xf0, 0xf2, 0x24, 0x7e, 0x1b, 0x6e, 0x74, 0x44, 0xf7, 0xb4, 0x17, 0xab, 0x51, 0x84, 0x43, 0xa2, - 0xe2, 0x56, 0x85, 0xcc, 0x9e, 0x26, 0xf3, 0x37, 0xa0, 0x22, 0xc2, 0xa0, 0x17, 0xd1, 0x5c, 0xac, - 0xde, 0x73, 0xe6, 0xda, 0xb2, 0x85, 0x1c, 0x9e, 0x61, 0xe4, 0x7b, 0xb0, 0x72, 0x26, 0x63, 0x1d, - 0x74, 0x45, 0x48, 0xf4, 0x56, 0x8d, 0x24, 0xdd, 0xb9, 0x92, 0xc7, 0x79, 0x4e, 0x6f, 0x52, 0x90, - 0xef, 0x03, 0x24, 0xb8, 0x40, 0xc8, 0xcf, 0x5b, 0x4d, 0x1a, 0x8c, 0x2f, 0xcf, 0x55, 0xb3, 0xa3, - 0x22, 0x2d, 0x23, 0xbd, 0xd9, 0xce, 0xd8, 0xf7, 0x96, 0xbc, 0x9c, 0x30, 0x7f, 0x0b, 0xca, 0x5a, - 0x9e, 0xeb, 0xd6, 0xea, 0x15, 0x23, 0x9a, 0x2a, 0x39, 0x92, 0xe7, 0x7a, 0x6f, 0xc9, 0x23, 0x01, - 0x14, 0xc4, 0x05, 0xd0, 0xba, 0xb1, 0x80, 0xe0, 0xe3, 0x20, 0x94, 0x28, 0x88, 0x02, 0xfc, 0x1d, - 0xa8, 0x86, 0xe2, 0x42, 0x8d, 0x74, 0x8b, 0x91, 0xe8, 0x6f, 0x5c, 0x29, 0xfa, 0x84, 0x58, 0xf7, - 0x96, 0x3c, 0x2b, 0xc4, 0x1f, 0x40, 0xc9, 0x0f, 0xce, 0x5a, 0x6b, 0x24, 0xbb, 0x71, 0xa5, 0xec, - 0x6e, 0x70, 0xb6, 0xb7, 0xe4, 0x21, 0x3b, 0xdf, 0x81, 0x7a, 0x47, 0xa9, 0xd3, 0x81, 0x88, 0x4f, - 0x5b, 0x9c, 0x44, 0xbf, 0x74, 0xa5, 0xe8, 0xb6, 0x65, 0xde, 0x5b, 0xf2, 0x32, 0x41, 0xec, 0x72, - 0xd0, 0x55, 0x51, 0xeb, 0x85, 0x05, 0xba, 0xbc, 0xdf, 0x55, 0x11, 0x76, 0x19, 0x05, 0x50, 0x30, - 0x0c, 0xa2, 0xd3, 0xd6, 0xfa, 0x02, 0x82, 0xb8, 0x76, 0x50, 0x10, 0x05, 0xd0, 0x6c, 0x5f, 0x68, - 0x71, 0x16, 0xc8, 0x67, 0xad, 0x17, 0x17, 0x30, 0x7b, 0xd7, 0x32, 0xa3, 0xd9, 0xa9, 0x20, 0x2a, - 0x49, 0x17, 0x66, 0xeb, 0xe6, 0x02, 0x4a, 0xd2, 0x35, 0x8d, 0x4a, 0x52, 0x41, 0xfe, 0x7b, 0xb0, - 0x76, 0x22, 0x85, 0x1e, 0xc5, 0xd2, 0x1f, 0xc3, 0xdc, 0x4b, 0xa4, 0x6d, 0xf3, 0xea, 0xb9, 0x9f, - 0x96, 0xda, 0x5b, 0xf2, 0x66, 0x55, 0xf1, 0x6f, 0x42, 0x25, 0x14, 0x5a, 0x9e, 0xb7, 0x5a, 0xa4, - 0xd3, 0x7d, 0x8e, 0x53, 0x68, 0x79, 0xbe, 0xb7, 0xe4, 0x19, 0x11, 0xfe, 0xdb, 0x70, 0x43, 0x8b, - 0x4e, 0x28, 0x0f, 0x4e, 0x2c, 0x43, 0xd2, 0xfa, 0x35, 0xd2, 0xf2, 0xfa, 0xd5, 0xee, 0x3c, 0x29, - 0xb3, 0xb7, 0xe4, 0x4d, 0xab, 0x41, 0xab, 0x88, 0xd4, 0x72, 0x16, 0xb0, 0x8a, 0xf4, 0xa1, 0x55, - 0x24, 0xc2, 0x9f, 0x40, 0x93, 0x1e, 0x76, 0x54, 0x38, 0x1a, 0x44, 0xad, 0x97, 0x49, 0xc3, 0xed, - 0xe7, 0x6b, 0x30, 0xfc, 0x7b, 0x4b, 0x5e, 0x5e, 0x1c, 0x27, 0x91, 0x9a, 0x9e, 0x7a, 0xd6, 0x7a, - 0x65, 0x81, 0x49, 0x3c, 0xb2, 0xcc, 0x38, 0x89, 0xa9, 0x20, 0x2e, 0xbd, 0x67, 0x81, 0xdf, 0x93, - 0xba, 0xf5, 0x85, 0x05, 0x96, 0xde, 0x47, 0xc4, 0x8a, 0x4b, 0xcf, 0x08, 0x39, 0x3f, 0x80, 0xe5, - 0x3c, 0xb8, 0x72, 0x0e, 0xe5, 0x58, 0x0a, 0x03, 0xec, 0x75, 0x8f, 0x9e, 0x91, 0x26, 0xfd, 0x40, - 0x13, 0xb0, 0xd7, 0x3d, 0x7a, 0xe6, 0x37, 0xa1, 0x6a, 0x36, 0x19, 0xc2, 0xed, 0xba, 0x67, 0x5b, - 0xc8, 0xeb, 0xc7, 0xa2, 0xd7, 0x2a, 0x1b, 0x5e, 0x7c, 0x46, 0x5e, 0x3f, 0x56, 0xc3, 0x83, 0x88, - 0x70, 0xb7, 0xee, 0xd9, 0x96, 0xf3, 0x67, 0x0f, 0xa0, 0x66, 0x0d, 0x73, 0xfe, 0xaa, 0x00, 0x55, - 0x83, 0x0b, 0xfc, 0x3d, 0xa8, 0x24, 0xfa, 0x22, 0x94, 0x64, 0xc3, 0xea, 0xbd, 0xaf, 0x2c, 0x80, - 0x25, 0x9b, 0x6d, 0x14, 0xf0, 0x8c, 0x9c, 0xeb, 0x41, 0x85, 0xda, 0xbc, 0x06, 0x25, 0x4f, 0x3d, - 0x63, 0x4b, 0x1c, 0xa0, 0x6a, 0xc6, 0x9c, 0x15, 0x90, 0xb8, 0x1b, 0x9c, 0xb1, 0x22, 0x12, 0xf7, - 0xa4, 0xf0, 0x65, 0xcc, 0x4a, 0x7c, 0x05, 0x1a, 0xe9, 0xe8, 0x26, 0xac, 0xcc, 0x19, 0x2c, 0xe7, - 0xe6, 0x2d, 0x61, 0x15, 0xe7, 0x7f, 0xcb, 0x50, 0xc6, 0x65, 0xcc, 0x5f, 0x85, 0x15, 0x2d, 0xe2, - 0x9e, 0x34, 0x91, 0xcc, 0x7e, 0xba, 0x05, 0x4e, 0x12, 0xf9, 0x3b, 0x69, 0x1f, 0x8a, 0xd4, 0x87, - 0x2f, 0x3f, 0x17, 0x1e, 0x26, 0x7a, 0x90, 0xdb, 0x4c, 0x4b, 0x8b, 0x6d, 0xa6, 0x8f, 0xa1, 0x8e, - 0xa8, 0xd4, 0x0e, 0x7e, 0x20, 0x69, 0xe8, 0x57, 0xef, 0xdd, 0x79, 0xfe, 0x27, 0xf7, 0xad, 0x84, - 0x97, 0xc9, 0xf2, 0x7d, 0x68, 0x74, 0x45, 0xec, 0x93, 0x31, 0x34, 0x5b, 0xab, 0xf7, 0xbe, 0xfa, - 0x7c, 0x45, 0x3b, 0xa9, 0x88, 0x37, 0x96, 0xe6, 0x07, 0xd0, 0xf4, 0x65, 0xd2, 0x8d, 0x83, 0x21, - 0xa1, 0x94, 0xd9, 0x52, 0xbf, 0xf6, 0x7c, 0x65, 0xbb, 0x63, 0x21, 0x2f, 0xaf, 0x81, 0xbf, 0x02, - 0x8d, 0x38, 0x83, 0xa9, 0x1a, 0xed, 0xf3, 0x63, 0x82, 0xfb, 0x16, 0xd4, 0xd3, 0xfe, 0xf0, 0x65, - 0xa8, 0xe3, 0xef, 0x07, 0x2a, 0x92, 0x6c, 0x09, 0xe7, 0x16, 0x5b, 0xed, 0x81, 0x08, 0x43, 0x56, - 0xe0, 0xab, 0x00, 0xd8, 0x7c, 0x2a, 0xfd, 0x60, 0x34, 0x60, 0x45, 0xf7, 0x37, 0x53, 0x6f, 0xa9, - 0x43, 0xf9, 0x50, 0xf4, 0x50, 0x62, 0x19, 0xea, 0x29, 0xea, 0xb2, 0x02, 0xca, 0xef, 0x8a, 0xa4, - 0xdf, 0x51, 0x22, 0xf6, 0x59, 0x91, 0x37, 0xa1, 0xb6, 0x15, 0x77, 0xfb, 0xc1, 0x99, 0x64, 0x25, - 0xf7, 0x2e, 0x34, 0x73, 0xf6, 0xa2, 0x0a, 0xfb, 0xd1, 0x06, 0x54, 0xb6, 0x7c, 0x5f, 0xfa, 0xac, - 0x80, 0x02, 0xb6, 0x83, 0xac, 0xe8, 0x7e, 0x15, 0x1a, 0xd9, 0x68, 0x21, 0x3b, 0xee, 0xbf, 0x6c, - 0x09, 0x9f, 0x90, 0xcc, 0x0a, 0xe8, 0x95, 0xfb, 0x51, 0x18, 0x44, 0x92, 0x15, 0x9d, 0xef, 0x91, - 0xab, 0xf2, 0xdf, 0x9a, 0x5c, 0x10, 0xaf, 0x3d, 0x6f, 0x83, 0x9c, 0x5c, 0x0d, 0x2f, 0xe7, 0xfa, - 0xf7, 0x24, 0x20, 0xe3, 0xea, 0x50, 0xde, 0x55, 0x3a, 0x61, 0x05, 0xe7, 0xbf, 0x8b, 0x50, 0x4f, - 0xf7, 0x45, 0xce, 0xa0, 0x34, 0x8a, 0x43, 0xeb, 0xd0, 0xf8, 0xc8, 0xd7, 0xa1, 0xa2, 0x03, 0x6d, - 0xdd, 0xb8, 0xe1, 0x99, 0x06, 0x86, 0x5c, 0xf9, 0x99, 0x2d, 0xd1, 0xbb, 0xe9, 0xa9, 0x0a, 0x06, - 0xa2, 0x27, 0xf7, 0x44, 0xd2, 0x27, 0x7f, 0x6c, 0x78, 0x63, 0x02, 0xca, 0x9f, 0x88, 0x33, 0xf4, - 0x39, 0x7a, 0x6f, 0x82, 0xb1, 0x3c, 0x89, 0xdf, 0x87, 0x32, 0x76, 0xd0, 0x3a, 0xcd, 0xaf, 0x4f, - 0x75, 0x18, 0xdd, 0xe4, 0x30, 0x96, 0x38, 0x3d, 0x9b, 0x18, 0x4a, 0x7b, 0xc4, 0xcc, 0x5f, 0x83, - 0x55, 0xb3, 0x08, 0x0f, 0x28, 0xc8, 0xde, 0xf7, 0x29, 0x18, 0x6b, 0x78, 0x53, 0x54, 0xbe, 0x85, - 0xc3, 0x29, 0xb4, 0x6c, 0xd5, 0x17, 0xf0, 0xef, 0x74, 0x70, 0x36, 0xdb, 0x28, 0xe2, 0x19, 0x49, - 0xf7, 0x21, 0x8e, 0xa9, 0xd0, 0x12, 0xa7, 0xf9, 0xd1, 0x60, 0xa8, 0x2f, 0x8c, 0xd3, 0x3c, 0x96, - 0xba, 0xdb, 0x0f, 0xa2, 0x1e, 0x2b, 0x98, 0x21, 0xc6, 0x49, 0x24, 0x96, 0x38, 0x56, 0x31, 0x2b, - 0x39, 0x0e, 0x94, 0xd1, 0x47, 0x11, 0x24, 0x23, 0x31, 0x90, 0x76, 0xa4, 0xe9, 0xd9, 0x79, 0x01, - 0xd6, 0x66, 0xb6, 0x55, 0xe7, 0x1f, 0xaa, 0xc6, 0x43, 0x50, 0x82, 0x42, 0x3a, 0x2b, 0x41, 0xd1, - 0xda, 0xb5, 0x30, 0x06, 0xb5, 0x4c, 0x62, 0xcc, 0x3b, 0x50, 0xc1, 0x8e, 0xa5, 0x10, 0xb3, 0x80, - 0xf8, 0x53, 0x64, 0xf7, 0x8c, 0x14, 0x6f, 0x41, 0xad, 0xdb, 0x97, 0xdd, 0x53, 0xe9, 0x5b, 0xac, - 0x4f, 0x9b, 0xe8, 0x34, 0xdd, 0x5c, 0x94, 0x6d, 0x1a, 0xe4, 0x12, 0x5d, 0x15, 0x3d, 0x1a, 0xa8, - 0x4f, 0x02, 0x9a, 0x57, 0x74, 0x89, 0x94, 0x90, 0xbe, 0xdd, 0x47, 0x1f, 0xb1, 0xd3, 0x36, 0x26, - 0x38, 0x8f, 0xa0, 0x42, 0xdf, 0xc6, 0x95, 0x60, 0x6c, 0x36, 0xa9, 0xe2, 0x6b, 0x8b, 0xd9, 0x6c, - 0x4d, 0x76, 0xfe, 0xb6, 0x08, 0x65, 0x6c, 0xf3, 0x3b, 0x50, 0x89, 0x45, 0xd4, 0x33, 0x13, 0x30, - 0x9b, 0x71, 0x7a, 0xf8, 0xce, 0x33, 0x2c, 0xfc, 0x3d, 0xeb, 0x8a, 0xc5, 0x05, 0x9c, 0x25, 0xfb, - 0x62, 0xde, 0x2d, 0xd7, 0xa1, 0x32, 0x14, 0xb1, 0x18, 0xd8, 0x75, 0x62, 0x1a, 0xee, 0x4f, 0x0a, - 0x50, 0x46, 0x26, 0xbe, 0x06, 0x2b, 0x6d, 0x1d, 0x07, 0xa7, 0x52, 0xf7, 0x63, 0x35, 0xea, 0xf5, - 0x8d, 0x27, 0xbd, 0x2f, 0x2f, 0x0c, 0xde, 0x18, 0x40, 0xd0, 0x22, 0x0c, 0xba, 0xac, 0x88, 0x5e, - 0xb5, 0xad, 0x42, 0x9f, 0x95, 0xf8, 0x0d, 0x68, 0x7e, 0x18, 0xf9, 0x32, 0x4e, 0xba, 0x2a, 0x96, - 0x3e, 0x2b, 0xdb, 0xd5, 0x7d, 0xca, 0x2a, 0xb4, 0x97, 0xc9, 0x73, 0x4d, 0x29, 0x0d, 0xab, 0xf2, - 0x17, 0xe0, 0xc6, 0xf6, 0x64, 0x9e, 0xc3, 0x6a, 0x88, 0x49, 0x4f, 0x65, 0x84, 0x4e, 0xc6, 0xea, - 0xc6, 0x89, 0xd5, 0x27, 0x01, 0x6b, 0xe0, 0xc7, 0xcc, 0x3a, 0x61, 0xe0, 0xfe, 0x63, 0x21, 0x45, - 0x8e, 0x15, 0x68, 0x1c, 0x8a, 0x58, 0xf4, 0x62, 0x31, 0x44, 0xfb, 0x9a, 0x50, 0x33, 0x1b, 0xe7, - 0x9b, 0x06, 0xdd, 0x4c, 0xe3, 0x9e, 0xc1, 0x46, 0xd3, 0xb8, 0xcf, 0x4a, 0xe3, 0xc6, 0x03, 0x56, - 0xc6, 0x6f, 0x7c, 0x67, 0xa4, 0xb4, 0x64, 0x15, 0xc2, 0x3a, 0xe5, 0x4b, 0x56, 0x45, 0xe2, 0x11, - 0x22, 0x0a, 0xab, 0x61, 0x9f, 0x77, 0xd0, 0x7f, 0x3a, 0xea, 0x9c, 0xd5, 0xd1, 0x0c, 0x1c, 0x46, - 0xe9, 0xb3, 0x06, 0xbe, 0xf9, 0x60, 0x34, 0xe8, 0x48, 0xec, 0x26, 0xe0, 0x9b, 0x23, 0xd5, 0xeb, - 0x85, 0x92, 0x35, 0x71, 0x0c, 0x72, 0xe0, 0xcb, 0x96, 0x09, 0x69, 0x45, 0x18, 0xaa, 0x91, 0x66, - 0x2b, 0xce, 0xcf, 0x4b, 0x50, 0xc6, 0x24, 0x05, 0xd7, 0x4e, 0x1f, 0x71, 0xc6, 0xae, 0x1d, 0x7c, - 0xce, 0x56, 0x60, 0x71, 0xbc, 0x02, 0xf9, 0x37, 0xed, 0x4c, 0x97, 0x16, 0x40, 0x59, 0x54, 0x9c, - 0x9f, 0x64, 0x0e, 0xe5, 0x41, 0x30, 0x90, 0x16, 0xeb, 0xe8, 0x19, 0x69, 0x09, 0xee, 0xc7, 0xb8, - 0x0c, 0x4a, 0x1e, 0x3d, 0xe3, 0xaa, 0x11, 0xb8, 0x2d, 0x6c, 0x69, 0x5a, 0x03, 0x25, 0x2f, 0x6d, - 0x9a, 0xd5, 0x8c, 0xa8, 0x54, 0x5b, 0x60, 0x35, 0xd3, 0xe7, 0xf3, 0x88, 0x34, 0x06, 0x83, 0xfa, - 0xe2, 0xe2, 0xb9, 0x4d, 0x62, 0xd7, 0x7a, 0xe3, 0x78, 0x03, 0xab, 0x9b, 0xd1, 0x63, 0x05, 0x9c, - 0x25, 0x5a, 0x86, 0x06, 0xcb, 0x8e, 0x03, 0x5f, 0x2a, 0x56, 0xa2, 0x0d, 0x6e, 0xe4, 0x07, 0x8a, - 0x95, 0x31, 0xa2, 0x3a, 0xdc, 0x7d, 0xcc, 0x2a, 0xee, 0x6b, 0xb9, 0xad, 0x66, 0x6b, 0xa4, 0x95, - 0x51, 0x43, 0x6e, 0x59, 0x30, 0x5e, 0xd6, 0x91, 0x3e, 0x2b, 0xba, 0x5f, 0x9f, 0x03, 0x9f, 0x2b, - 0xd0, 0xf8, 0x70, 0x18, 0x2a, 0xe1, 0x5f, 0x81, 0x9f, 0xcb, 0x00, 0xe3, 0xa4, 0xd7, 0xf9, 0xd1, - 0x17, 0xc6, 0xdb, 0x34, 0xc6, 0x98, 0x89, 0x1a, 0xc5, 0x5d, 0x49, 0xd0, 0xd0, 0xf0, 0x6c, 0x8b, - 0x7f, 0x0b, 0x2a, 0xf8, 0x3e, 0x69, 0x15, 0x09, 0x31, 0xee, 0x2c, 0x94, 0x6a, 0x6d, 0x1e, 0x07, - 0xf2, 0x99, 0x67, 0x04, 0xf9, 0xc3, 0x7c, 0xd8, 0xf1, 0x9c, 0x22, 0xd0, 0x98, 0x93, 0xdf, 0x02, - 0x10, 0x5d, 0x1d, 0x9c, 0x49, 0xd4, 0x65, 0xd7, 0x7e, 0x8e, 0xc2, 0x3d, 0x68, 0xe2, 0x92, 0x1c, - 0x1e, 0xc4, 0xb8, 0x8a, 0x5b, 0xcb, 0xa4, 0xf8, 0x8d, 0xc5, 0xcc, 0xfb, 0x76, 0x26, 0xe8, 0xe5, - 0x95, 0xf0, 0x0f, 0x61, 0xd9, 0x14, 0x98, 0xac, 0xd2, 0x15, 0x52, 0xfa, 0xe6, 0x62, 0x4a, 0x0f, - 0xc6, 0x92, 0xde, 0x84, 0x9a, 0xd9, 0xba, 0x51, 0xe5, 0xba, 0x75, 0x23, 0xdc, 0x9b, 0x8f, 0x26, - 0xf7, 0x66, 0xb3, 0x05, 0x4c, 0x51, 0xb9, 0x0b, 0xcb, 0x41, 0x32, 0x2e, 0x5b, 0x51, 0x09, 0xa3, - 0xee, 0x4d, 0xd0, 0x9c, 0x5f, 0x54, 0xa0, 0x4c, 0x43, 0x38, 0x5d, 0x82, 0xda, 0x99, 0x80, 0xea, - 0xbb, 0x8b, 0x4f, 0xf5, 0xd4, 0x4a, 0x26, 0x64, 0x28, 0xe5, 0x90, 0xe1, 0x5b, 0x50, 0x49, 0x54, - 0xac, 0xd3, 0xe9, 0x5f, 0xd0, 0x89, 0xda, 0x2a, 0xd6, 0x9e, 0x11, 0xe4, 0x8f, 0xa1, 0x76, 0x12, - 0x84, 0x1a, 0x27, 0xc5, 0x0c, 0xde, 0xeb, 0x8b, 0xe9, 0x78, 0x4c, 0x42, 0x5e, 0x2a, 0xcc, 0x9f, - 0xe4, 0x9d, 0xb1, 0x4a, 0x9a, 0x36, 0x17, 0xd3, 0x34, 0xcf, 0x47, 0xef, 0x00, 0xeb, 0xaa, 0x33, - 0x19, 0xa7, 0xef, 0xde, 0x97, 0x17, 0x76, 0xf3, 0x9d, 0xa1, 0x73, 0x07, 0xea, 0xfd, 0xc0, 0x97, - 0x18, 0xbf, 0x10, 0xc6, 0xd4, 0xbd, 0xac, 0xcd, 0xdf, 0x87, 0x3a, 0xc5, 0xfd, 0x88, 0x76, 0x8d, - 0x6b, 0x0f, 0xbe, 0x49, 0x41, 0x52, 0x05, 0xf8, 0x21, 0xfa, 0xf8, 0xe3, 0x40, 0xb7, 0xc0, 0x7c, - 0x28, 0x6d, 0xa3, 0xc1, 0xe4, 0xef, 0x79, 0x83, 0x9b, 0xc6, 0xe0, 0x69, 0x3a, 0x7f, 0x00, 0x2f, - 0x12, 0x6d, 0x6a, 0xf3, 0xc3, 0xa5, 0x86, 0x4a, 0xe7, 0xbf, 0xc4, 0x40, 0x64, 0x28, 0x7a, 0xf2, - 0x49, 0x30, 0x08, 0x74, 0x6b, 0x65, 0xa3, 0x70, 0xbb, 0xe2, 0x8d, 0x09, 0xee, 0x03, 0x0b, 0x93, - 0xb8, 0x71, 0x61, 0x7e, 0x98, 0x02, 0x5c, 0xa2, 0xcd, 0x4e, 0xf8, 0x6d, 0x11, 0x86, 0x32, 0xbe, - 0x30, 0xc9, 0xe5, 0xfb, 0x22, 0xea, 0x88, 0x88, 0x95, 0xdc, 0xdb, 0x50, 0xa6, 0x9e, 0x35, 0xa0, - 0x62, 0x92, 0x10, 0x4a, 0x48, 0x6d, 0x02, 0x42, 0xc0, 0xf8, 0x04, 0x57, 0x01, 0x2b, 0x3a, 0x7f, - 0x5f, 0x82, 0x7a, 0xda, 0x07, 0x0c, 0xc7, 0x4f, 0xe5, 0x45, 0x1a, 0x8e, 0x9f, 0xca, 0x0b, 0x8a, - 0x92, 0x92, 0xe3, 0x20, 0x09, 0x3a, 0x36, 0xea, 0xab, 0x7b, 0x63, 0x02, 0x06, 0x1a, 0xcf, 0x02, - 0x5f, 0xf7, 0xc9, 0x75, 0x2b, 0x9e, 0x69, 0xf0, 0xdb, 0x70, 0xc3, 0x17, 0x5a, 0xee, 0x47, 0xdd, - 0x70, 0xe4, 0xcb, 0x23, 0xdc, 0xa4, 0x4c, 0x16, 0x3e, 0x4d, 0xe6, 0xdf, 0x05, 0xd0, 0xc1, 0x40, - 0x3e, 0x56, 0xf1, 0x40, 0x68, 0x1b, 0x7a, 0x7f, 0xe3, 0x7a, 0xce, 0xb5, 0x79, 0x94, 0x29, 0xf0, - 0x72, 0xca, 0x50, 0x35, 0x7e, 0xcd, 0xaa, 0xae, 0x7d, 0x2e, 0xd5, 0xbb, 0x99, 0x02, 0x2f, 0xa7, - 0xcc, 0xfd, 0x1d, 0x80, 0xf1, 0x1b, 0x7e, 0x13, 0xf8, 0x53, 0x15, 0xe9, 0xfe, 0x56, 0xa7, 0x13, - 0x6f, 0xcb, 0x13, 0x15, 0xcb, 0x5d, 0x81, 0xbb, 0xcb, 0x8b, 0xb0, 0x96, 0xd1, 0xb7, 0x4e, 0xb4, - 0x8c, 0x91, 0x4c, 0x43, 0xdf, 0xee, 0xab, 0x58, 0x9b, 0xd0, 0x85, 0x1e, 0x3f, 0x6c, 0xb3, 0x12, - 0xee, 0x68, 0xfb, 0xed, 0x03, 0x56, 0x76, 0x6f, 0x03, 0x8c, 0xbb, 0x44, 0x21, 0x3e, 0x3d, 0xbd, - 0x79, 0xcf, 0x06, 0xfc, 0xd4, 0xba, 0xf7, 0x80, 0x15, 0x9c, 0xbf, 0x2b, 0x42, 0x19, 0x57, 0xbc, - 0x45, 0xa5, 0x6a, 0x86, 0x4a, 0x1b, 0xd0, 0xcc, 0xbb, 0xab, 0x99, 0xce, 0x3c, 0xe9, 0xf3, 0xe1, - 0x16, 0x7e, 0x2b, 0x8f, 0x5b, 0x6f, 0x43, 0xb3, 0x3b, 0x4a, 0xb4, 0x1a, 0x10, 0x68, 0xb7, 0x4a, - 0x84, 0x0d, 0x37, 0x67, 0xea, 0x06, 0xc7, 0x22, 0x1c, 0x49, 0x2f, 0xcf, 0xca, 0x1f, 0x42, 0xf5, - 0xc4, 0x4c, 0x8c, 0xa9, 0x1c, 0x7c, 0xe1, 0x12, 0x5c, 0xb7, 0x83, 0x6f, 0x99, 0xb1, 0x5f, 0xc1, - 0x8c, 0x53, 0xe5, 0x49, 0xee, 0x97, 0xec, 0x6a, 0xa9, 0x41, 0x69, 0x2b, 0xe9, 0xda, 0xbc, 0x53, - 0x26, 0x5d, 0x13, 0xd4, 0xee, 0x90, 0x09, 0xac, 0xe8, 0xfc, 0x6b, 0x0d, 0xaa, 0x06, 0xe7, 0xec, - 0xd8, 0x35, 0xb2, 0xb1, 0xfb, 0x0e, 0xd4, 0xd5, 0x50, 0xc6, 0x42, 0xab, 0xd8, 0x26, 0xbf, 0x0f, - 0xaf, 0x83, 0x9b, 0x9b, 0x07, 0x56, 0xd8, 0xcb, 0xd4, 0x4c, 0x4f, 0x47, 0x71, 0x76, 0x3a, 0xee, - 0x00, 0x4b, 0x21, 0xf2, 0x30, 0x46, 0x39, 0x7d, 0x61, 0x53, 0x99, 0x19, 0x3a, 0x3f, 0x82, 0x46, - 0x57, 0x45, 0x7e, 0x90, 0x25, 0xc2, 0xab, 0xf7, 0xbe, 0x7e, 0x2d, 0x0b, 0x77, 0x52, 0x69, 0x6f, - 0xac, 0x88, 0xbf, 0x0e, 0x95, 0x33, 0x9c, 0x27, 0x9a, 0x90, 0xcb, 0x67, 0xd1, 0x30, 0xf1, 0x8f, - 0xa1, 0xf9, 0xfd, 0x51, 0xd0, 0x3d, 0x3d, 0xc8, 0x17, 0x5a, 0xde, 0xbe, 0x96, 0x15, 0xdf, 0x19, - 0xcb, 0x7b, 0x79, 0x65, 0x39, 0xdf, 0xa8, 0xfd, 0x12, 0xbe, 0x51, 0x9f, 0xf5, 0x8d, 0x97, 0xa1, - 0x9e, 0x4e, 0x0e, 0xf9, 0x47, 0xe4, 0xb3, 0x25, 0x5e, 0x85, 0xe2, 0x41, 0xcc, 0x0a, 0xee, 0xff, - 0x14, 0xa0, 0x91, 0x0d, 0xcc, 0x64, 0x51, 0xe5, 0xd1, 0xf7, 0x47, 0x22, 0x64, 0x05, 0xca, 0x0a, - 0x94, 0x36, 0x2d, 0x5a, 0xbc, 0xdf, 0x8e, 0xa5, 0xd0, 0x54, 0xcb, 0x43, 0x44, 0x96, 0x49, 0xc2, - 0xca, 0x9c, 0xc3, 0xaa, 0x25, 0x1f, 0xc4, 0x86, 0xb5, 0x82, 0x49, 0x03, 0xbe, 0x4d, 0x09, 0x55, - 0x03, 0xe0, 0xa7, 0xd2, 0x24, 0x45, 0x1f, 0x28, 0x4d, 0x8d, 0x3a, 0xda, 0xb2, 0x1f, 0xb1, 0x06, - 0x7e, 0xf3, 0x03, 0xa5, 0xf7, 0x23, 0x06, 0xe3, 0x68, 0xb5, 0x99, 0x7e, 0x9e, 0x5a, 0xcb, 0x14, - 0x0b, 0x87, 0xe1, 0x7e, 0xc4, 0x56, 0xec, 0x0b, 0xd3, 0x5a, 0x45, 0x8d, 0x8f, 0xce, 0x45, 0x17, - 0xc5, 0x6f, 0xf0, 0x55, 0x00, 0x94, 0xb1, 0x6d, 0x86, 0x6b, 0xe0, 0xd1, 0x79, 0x90, 0xe8, 0x84, - 0xad, 0xb9, 0xff, 0x52, 0x80, 0x66, 0x6e, 0x12, 0x30, 0x1a, 0x26, 0x46, 0x84, 0x36, 0x13, 0x1c, - 0x7f, 0x57, 0x26, 0x5a, 0xc6, 0x7e, 0x0a, 0x5b, 0x47, 0x0a, 0x1f, 0x8b, 0xf8, 0xbd, 0x23, 0x35, - 0x50, 0x71, 0xac, 0x9e, 0xb1, 0x12, 0xb6, 0x9e, 0x88, 0x44, 0x7f, 0x24, 0xe5, 0x29, 0x2b, 0x63, - 0x57, 0x77, 0x46, 0x71, 0x2c, 0x23, 0x43, 0xa8, 0x90, 0x71, 0xf2, 0xdc, 0xb4, 0xaa, 0xa8, 0x14, - 0x99, 0x09, 0x17, 0x59, 0x8d, 0x33, 0x58, 0xb6, 0xdc, 0x86, 0x52, 0x47, 0x06, 0x64, 0x37, 0xcd, - 0x06, 0x26, 0x92, 0x26, 0x11, 0x3b, 0x38, 0xd9, 0x15, 0x17, 0xc9, 0x56, 0x4f, 0x31, 0x98, 0x26, - 0x7e, 0xa0, 0x9e, 0xb1, 0xa6, 0x33, 0x02, 0x18, 0x87, 0xa8, 0x18, 0x9a, 0xa3, 0xaf, 0x65, 0xa5, - 0x52, 0xdb, 0xe2, 0x07, 0x00, 0xf8, 0x44, 0x9c, 0x69, 0x7c, 0x7e, 0x8d, 0xb8, 0x81, 0xe4, 0xbc, - 0x9c, 0x0a, 0xe7, 0xf7, 0xa1, 0x91, 0xbd, 0xc0, 0x4c, 0x8b, 0x76, 0xf8, 0xec, 0xb3, 0x69, 0x13, - 0xf7, 0xc9, 0x20, 0xf2, 0xe5, 0x39, 0xad, 0xfd, 0x8a, 0x67, 0x1a, 0x68, 0x65, 0x3f, 0xf0, 0x7d, - 0x19, 0xa5, 0x05, 0x6d, 0xd3, 0x9a, 0x77, 0x7a, 0x58, 0x9e, 0x7b, 0x7a, 0xe8, 0xfc, 0x2e, 0x34, - 0x73, 0x31, 0xf4, 0xa5, 0xdd, 0xce, 0x19, 0x56, 0x9c, 0x34, 0xec, 0x15, 0x68, 0x28, 0x1b, 0x08, - 0x27, 0x04, 0xe0, 0x0d, 0x6f, 0x4c, 0xc0, 0x0d, 0xa6, 0x62, 0xba, 0x36, 0x1d, 0xf7, 0x3e, 0x86, - 0x2a, 0x26, 0x81, 0xa3, 0xf4, 0xe8, 0x75, 0xc1, 0xd8, 0xb2, 0x4d, 0x32, 0x7b, 0x4b, 0x9e, 0x95, - 0xe6, 0xef, 0x40, 0x49, 0x8b, 0x9e, 0xad, 0x07, 0x7d, 0x65, 0x31, 0x25, 0x47, 0xa2, 0xb7, 0xb7, - 0xe4, 0xa1, 0x1c, 0x7f, 0x02, 0xf5, 0xae, 0x4d, 0xe1, 0x2d, 0x70, 0x2d, 0x18, 0x9a, 0xa6, 0x89, - 0xff, 0xde, 0x92, 0x97, 0x69, 0xe0, 0xdf, 0x82, 0x32, 0xee, 0xf2, 0x84, 0xbc, 0x0b, 0x87, 0xdc, - 0xb8, 0x5c, 0xf6, 0x96, 0x3c, 0x92, 0xdc, 0xae, 0x41, 0x85, 0x70, 0xd2, 0x69, 0x41, 0xd5, 0xf4, - 0x75, 0x7a, 0xe4, 0x9c, 0x97, 0xa0, 0x74, 0x24, 0x7a, 0x18, 0x69, 0x05, 0x7e, 0x62, 0x33, 0x47, - 0x7c, 0x74, 0x5e, 0x1d, 0x97, 0x23, 0xf2, 0x95, 0xae, 0xc2, 0x44, 0xa5, 0xcb, 0xa9, 0x42, 0x19, - 0xbf, 0xe8, 0xbc, 0x72, 0x55, 0xd4, 0xe6, 0xbc, 0x8c, 0xf1, 0x9d, 0x96, 0xe7, 0xf3, 0x8a, 0x78, - 0xce, 0x1a, 0xdc, 0x98, 0x3a, 0xb3, 0x72, 0x6a, 0x36, 0xb8, 0x74, 0x56, 0xa0, 0x99, 0x3b, 0x85, - 0x70, 0x5e, 0x83, 0x7a, 0x7a, 0x46, 0x81, 0x41, 0x72, 0x90, 0x98, 0xea, 0x8a, 0x35, 0x2a, 0x6b, - 0x3b, 0x3f, 0x2d, 0x40, 0xd5, 0x9c, 0xf3, 0xf0, 0xed, 0xec, 0x5c, 0xb6, 0xb0, 0xc0, 0xa1, 0x80, - 0x11, 0xb2, 0x47, 0x2a, 0xd9, 0xe1, 0xec, 0x3a, 0x54, 0x42, 0x8a, 0x86, 0xed, 0x72, 0xa1, 0x46, - 0xce, 0xbb, 0x4b, 0x79, 0xef, 0x76, 0xdf, 0xca, 0x8e, 0x71, 0xd2, 0xcc, 0x9f, 0xb6, 0xfd, 0xa3, - 0x58, 0x4a, 0x93, 0xd5, 0x53, 0xb0, 0x5c, 0x24, 0x6c, 0x52, 0x83, 0xa1, 0xe8, 0x6a, 0x22, 0x94, - 0xdc, 0x13, 0xa8, 0x1f, 0xaa, 0x64, 0x1a, 0xf1, 0x6b, 0x50, 0x3a, 0x52, 0x43, 0x13, 0x30, 0x6c, - 0x2b, 0x4d, 0x01, 0x83, 0x01, 0xf8, 0x13, 0x6d, 0x8a, 0x10, 0x5e, 0xd0, 0xeb, 0x6b, 0x53, 0x60, - 0xda, 0x8f, 0x22, 0x19, 0xb3, 0x0a, 0xa2, 0xae, 0x27, 0x87, 0xa1, 0xe8, 0x4a, 0x56, 0x45, 0xd4, - 0x25, 0xfa, 0xe3, 0x20, 0x4e, 0x34, 0xab, 0xb9, 0x6f, 0x21, 0x56, 0x07, 0x3d, 0x82, 0x58, 0x7a, - 0x20, 0x55, 0x4b, 0x68, 0x10, 0x35, 0x77, 0x64, 0x84, 0xdb, 0x08, 0x9d, 0x13, 0x98, 0x83, 0x7a, - 0xfa, 0x40, 0xd1, 0xfd, 0x08, 0x56, 0x26, 0x0e, 0xf0, 0xf9, 0x3a, 0xb0, 0x09, 0x02, 0x1a, 0xba, - 0xc4, 0x5f, 0x82, 0x17, 0x26, 0xa8, 0x4f, 0x03, 0xdf, 0xa7, 0x32, 0xca, 0xf4, 0x8b, 0xb4, 0x3b, - 0xdb, 0x0d, 0xa8, 0x75, 0xcd, 0x0c, 0xb8, 0x87, 0xb0, 0x42, 0x53, 0xf2, 0x54, 0x6a, 0x71, 0x10, - 0x85, 0x17, 0xbf, 0xf4, 0x2d, 0x0b, 0xf7, 0xab, 0x50, 0xa1, 0x72, 0x26, 0x3a, 0xdf, 0x49, 0xac, - 0x06, 0xa4, 0xab, 0xe2, 0xd1, 0x33, 0x6a, 0xd7, 0xca, 0xce, 0x6b, 0x51, 0x2b, 0xf7, 0xa7, 0x0d, - 0xa8, 0x6d, 0x75, 0xbb, 0x6a, 0x14, 0xe9, 0x99, 0x2f, 0xcf, 0xab, 0x98, 0x3d, 0x84, 0xaa, 0x38, - 0x13, 0x5a, 0xc4, 0x16, 0x33, 0xa6, 0xa3, 0x03, 0xab, 0x6b, 0x73, 0x8b, 0x98, 0x3c, 0xcb, 0x8c, - 0x62, 0x5d, 0x15, 0x9d, 0x04, 0x3d, 0x0b, 0x13, 0x97, 0x89, 0xed, 0x10, 0x93, 0x67, 0x99, 0x51, - 0xcc, 0xc2, 0x5c, 0xe5, 0x4a, 0x31, 0xb3, 0xd6, 0x33, 0x54, 0xbb, 0x0b, 0xe5, 0x20, 0x3a, 0x51, - 0xf6, 0x7e, 0xcd, 0xcb, 0x97, 0x08, 0xed, 0x47, 0x27, 0xca, 0x23, 0x46, 0x47, 0x42, 0xd5, 0x18, - 0xcc, 0xbf, 0x01, 0x15, 0x3a, 0xb5, 0xb0, 0x75, 0xe2, 0x85, 0x2e, 0x44, 0x18, 0x09, 0x7e, 0x33, - 0x2d, 0x82, 0xd3, 0x78, 0x21, 0x9d, 0x9a, 0xdb, 0xf5, 0x74, 0xc8, 0x9c, 0xff, 0x2c, 0x40, 0xd5, - 0xf4, 0x90, 0xbf, 0x06, 0xab, 0x32, 0xc2, 0xa5, 0x9d, 0x02, 0x99, 0x5d, 0xd3, 0x53, 0x54, 0x0c, - 0xab, 0x2c, 0x45, 0x76, 0x46, 0x3d, 0x9b, 0x01, 0xe6, 0x49, 0xfc, 0x6d, 0x78, 0xc9, 0x34, 0x0f, - 0x63, 0x19, 0xcb, 0x50, 0x8a, 0x44, 0xee, 0xf4, 0x45, 0x14, 0xc9, 0xd0, 0x6e, 0x6b, 0x97, 0xbd, - 0xe6, 0x2e, 0x2c, 0x9b, 0x57, 0xed, 0xa1, 0xe8, 0xca, 0xc4, 0x16, 0xf5, 0x27, 0x68, 0xfc, 0x6b, - 0x50, 0xa1, 0x5b, 0x4e, 0x2d, 0xff, 0x6a, 0xe7, 0x33, 0x5c, 0x8e, 0xca, 0x70, 0x77, 0x0b, 0xc0, - 0xcc, 0x06, 0xe6, 0x03, 0x16, 0x8b, 0xbe, 0x78, 0xe5, 0xf4, 0x51, 0x66, 0x93, 0x13, 0x42, 0xfb, - 0x7c, 0x19, 0x4a, 0xc4, 0x07, 0xc4, 0x5c, 0xea, 0x7c, 0xc9, 0x9b, 0xa0, 0x39, 0x7f, 0x53, 0x82, - 0x32, 0x4e, 0x24, 0x32, 0xf7, 0xd5, 0x40, 0x66, 0xc5, 0x26, 0xe3, 0xb4, 0x13, 0x34, 0xdc, 0xd8, - 0x85, 0x39, 0xc7, 0xcb, 0xd8, 0x0c, 0x94, 0x4d, 0x93, 0x91, 0x73, 0x18, 0xab, 0x93, 0x20, 0x1c, - 0x73, 0xda, 0x10, 0x60, 0x8a, 0xcc, 0xbf, 0x0e, 0x37, 0x07, 0x22, 0x3e, 0x95, 0x9a, 0xd0, 0xe7, - 0x23, 0x15, 0x9f, 0x26, 0x38, 0x72, 0xfb, 0xbe, 0xad, 0x52, 0x5c, 0xf2, 0x16, 0xe1, 0xdc, 0x97, - 0x67, 0x01, 0x71, 0xd6, 0x89, 0x33, 0x6b, 0xa3, 0x73, 0x08, 0x33, 0x34, 0x6d, 0xab, 0xcb, 0xe4, - 0x47, 0x53, 0x54, 0x8c, 0x1e, 0xcc, 0x99, 0x7e, 0xb2, 0xef, 0x53, 0xe1, 0xa4, 0xe1, 0x8d, 0x09, - 0xfc, 0x16, 0x40, 0x4f, 0x68, 0xf9, 0x4c, 0x5c, 0x7c, 0x18, 0x87, 0x2d, 0x69, 0xca, 0x91, 0x63, - 0x0a, 0x26, 0x3d, 0xa1, 0xea, 0x8a, 0xb0, 0xad, 0x55, 0x2c, 0x7a, 0xf2, 0x50, 0xe8, 0x7e, 0xab, - 0x67, 0x92, 0x9e, 0x69, 0x3a, 0x5a, 0x8b, 0xb9, 0xfd, 0xc7, 0x2a, 0x92, 0xad, 0xbe, 0xb1, 0x36, - 0x6d, 0xa3, 0x8b, 0x8a, 0x48, 0x84, 0x17, 0x3a, 0xe8, 0xa2, 0x1d, 0x81, 0x49, 0xaf, 0x72, 0x24, - 0xf7, 0x00, 0x60, 0x3c, 0xc5, 0x88, 0xeb, 0x5b, 0x54, 0x14, 0x65, 0x4b, 0x18, 0x4b, 0x1e, 0xca, - 0xc8, 0x0f, 0xa2, 0xde, 0xae, 0x9d, 0x55, 0x56, 0x40, 0x62, 0x5b, 0x8b, 0x58, 0x4b, 0x3f, 0x23, - 0x52, 0xbc, 0x4f, 0x2d, 0xe9, 0xb3, 0x92, 0xfb, 0x7f, 0x05, 0x68, 0xe6, 0x8e, 0x04, 0x7f, 0x85, - 0xc7, 0x98, 0xb8, 0xcb, 0xe2, 0x6a, 0xc6, 0x21, 0x33, 0x33, 0x9e, 0xb5, 0x71, 0x40, 0xed, 0x89, - 0x25, 0xbe, 0x35, 0xf9, 0x61, 0x8e, 0xf2, 0xb9, 0x8e, 0x30, 0xdd, 0x7b, 0x36, 0x63, 0x6e, 0x42, - 0xed, 0xc3, 0xe8, 0x34, 0x52, 0xcf, 0x22, 0xb3, 0x7d, 0xd2, 0xb9, 0xf4, 0x44, 0x25, 0x3e, 0x3d, - 0x3a, 0x2e, 0xb9, 0x7f, 0x51, 0x9e, 0xba, 0xc2, 0xf1, 0x08, 0xaa, 0x26, 0x6a, 0xa4, 0x80, 0x66, - 0xf6, 0xcc, 0x3d, 0xcf, 0x6c, 0xab, 0xbe, 0x39, 0x92, 0x67, 0x85, 0x31, 0x9c, 0xcb, 0xee, 0x29, - 0x15, 0xe7, 0x56, 0xa7, 0x27, 0x14, 0xa5, 0x20, 0x35, 0x71, 0x55, 0x2f, 0xd3, 0xe0, 0xfc, 0x49, - 0x01, 0xd6, 0xe7, 0xb1, 0x60, 0x74, 0xd5, 0x99, 0xb8, 0x49, 0x91, 0x36, 0x79, 0x7b, 0xea, 0x82, - 0x60, 0x91, 0x7a, 0x73, 0xf7, 0x9a, 0x46, 0x4c, 0x5e, 0x17, 0x74, 0x7f, 0x5c, 0x80, 0xb5, 0x99, - 0x3e, 0xe7, 0x02, 0x0e, 0x80, 0xaa, 0xf1, 0x2c, 0x73, 0xf0, 0x9f, 0x1d, 0xc5, 0x9a, 0x92, 0x1e, - 0x21, 0x7e, 0x62, 0xce, 0xb6, 0x76, 0xcd, 0xf5, 0x52, 0x56, 0xc6, 0x48, 0x01, 0x67, 0x0d, 0x91, - 0xb4, 0x27, 0x59, 0x05, 0x33, 0x29, 0x13, 0x03, 0x59, 0x4a, 0x95, 0xb2, 0x34, 0x39, 0x18, 0x86, - 0x98, 0xcd, 0xd5, 0xe8, 0x42, 0xc1, 0x68, 0x18, 0x06, 0x5d, 0x6c, 0xd6, 0x5d, 0x0f, 0x5e, 0x98, - 0x63, 0x37, 0x59, 0x72, 0x6c, 0xad, 0x5a, 0x05, 0xd8, 0x3d, 0x4e, 0x6d, 0x61, 0x05, 0x4c, 0x6c, - 0x77, 0x8f, 0x77, 0x28, 0xb5, 0xb5, 0xc7, 0x75, 0x66, 0x4d, 0x1c, 0x63, 0xfe, 0x93, 0xb0, 0x92, - 0xfb, 0xbd, 0xf4, 0x1c, 0xcf, 0x39, 0x86, 0x15, 0x63, 0xc6, 0xa1, 0xb8, 0x08, 0x95, 0xf0, 0xf9, - 0x23, 0x58, 0x4d, 0xb2, 0x9b, 0xb8, 0x39, 0x3c, 0x9e, 0xde, 0x4e, 0xdb, 0x13, 0x4c, 0xde, 0x94, - 0x90, 0xfb, 0xe7, 0x15, 0x80, 0x83, 0xec, 0x36, 0xeb, 0x9c, 0x45, 0x37, 0x2f, 0x60, 0x98, 0x39, - 0x49, 0x28, 0x5d, 0xfb, 0x24, 0xe1, 0xed, 0x2c, 0xa4, 0x35, 0xd5, 0xaa, 0xe9, 0xeb, 0x82, 0x63, - 0x9b, 0xa6, 0x03, 0xd9, 0x89, 0x13, 0xe8, 0xca, 0xf4, 0x09, 0xf4, 0xc6, 0xec, 0x75, 0x95, 0x29, - 0x34, 0x18, 0x67, 0x88, 0xb5, 0x89, 0x0c, 0xd1, 0x81, 0x7a, 0x2c, 0x85, 0xaf, 0xa2, 0xf0, 0x22, - 0x2d, 0x58, 0xa7, 0x6d, 0x7e, 0x1f, 0x2a, 0x9a, 0xee, 0xff, 0xd6, 0xc9, 0x79, 0x9f, 0x33, 0xc6, - 0x86, 0x17, 0xa1, 0x25, 0x48, 0xec, 0x1d, 0x13, 0x83, 0xf6, 0x75, 0x2f, 0x47, 0xe1, 0x9b, 0xc0, - 0x83, 0x28, 0xd1, 0x22, 0x0c, 0xa5, 0xbf, 0x7d, 0xb1, 0x2b, 0x4f, 0xc4, 0x28, 0xd4, 0xb4, 0xc3, - 0xd4, 0xbd, 0x39, 0x6f, 0xdc, 0xcf, 0xc6, 0x77, 0xab, 0x1a, 0x50, 0xe9, 0x88, 0x24, 0xe8, 0x9a, - 0x53, 0x5c, 0xbb, 0x7d, 0x99, 0xc0, 0x5c, 0x2b, 0x5f, 0xb1, 0x22, 0x46, 0xdc, 0x89, 0xc4, 0xd8, - 0x7a, 0x15, 0x60, 0x7c, 0x5b, 0x99, 0x95, 0xd1, 0x87, 0xd3, 0x99, 0x30, 0x87, 0xb8, 0x24, 0x4a, - 0x65, 0x04, 0x3f, 0xbb, 0x1e, 0x53, 0xc3, 0x2f, 0x10, 0x46, 0xb2, 0x3a, 0xf2, 0x44, 0x4a, 0x4b, - 0x53, 0x44, 0xa1, 0xad, 0x8e, 0x01, 0xaa, 0x49, 0x2f, 0x5f, 0xb2, 0x26, 0x06, 0xc5, 0xa9, 0x52, - 0x53, 0xf9, 0x48, 0x28, 0x1d, 0x58, 0x46, 0x0f, 0x9f, 0x7c, 0xc1, 0x56, 0xd0, 0xa2, 0xf1, 0x25, - 0x68, 0xb6, 0x8a, 0xaa, 0x10, 0x5f, 0x3a, 0x22, 0x91, 0x6c, 0xdd, 0xfd, 0xcb, 0x71, 0x2f, 0xdf, - 0xc8, 0x62, 0xd7, 0x45, 0xfc, 0xe3, 0xb2, 0xe8, 0xf6, 0x11, 0xac, 0xc5, 0xf2, 0xfb, 0xa3, 0x60, - 0xe2, 0x7a, 0x64, 0xe9, 0xea, 0x03, 0xc0, 0x59, 0x09, 0xf7, 0x0c, 0xd6, 0xd2, 0xc6, 0x47, 0x81, - 0xee, 0x53, 0x4a, 0xca, 0xef, 0xe7, 0xee, 0x6f, 0x16, 0x6c, 0x30, 0x75, 0x89, 0xca, 0xf1, 0x7d, - 0xcd, 0xac, 0x2c, 0x58, 0x5c, 0xa0, 0x2c, 0xe8, 0xfe, 0x47, 0x35, 0x97, 0x95, 0x9a, 0x68, 0xde, - 0xcf, 0xa2, 0xf9, 0xd9, 0xb3, 0x85, 0x71, 0xa5, 0xaf, 0x78, 0x9d, 0x4a, 0xdf, 0xbc, 0xe3, 0xb2, - 0x6f, 0x62, 0xa8, 0x46, 0xae, 0x77, 0xbc, 0x40, 0x15, 0x73, 0x82, 0x97, 0x6f, 0xd3, 0x49, 0x81, - 0x68, 0x9b, 0xb3, 0xdc, 0xca, 0xdc, 0xdb, 0xd4, 0xf9, 0x23, 0x01, 0xcb, 0xe9, 0xe5, 0xa4, 0x72, - 0x0b, 0xb5, 0x3a, 0x6f, 0xa1, 0x62, 0x62, 0x65, 0x97, 0x70, 0xd6, 0x36, 0x45, 0x5f, 0xf3, 0x9c, - 0xaa, 0xa7, 0x6b, 0xd0, 0x75, 0x6f, 0x86, 0x8e, 0xe1, 0xc4, 0x60, 0x14, 0xea, 0xc0, 0xd6, 0x35, - 0x4d, 0x63, 0xfa, 0xc2, 0x7f, 0x63, 0xf6, 0xc2, 0xff, 0xbb, 0x00, 0x89, 0x44, 0xf7, 0xdd, 0x0d, - 0xba, 0xda, 0x9e, 0xf8, 0xde, 0xba, 0xac, 0x6f, 0xb6, 0x1a, 0x9b, 0x93, 0x40, 0xfb, 0x07, 0xe2, - 0x7c, 0x07, 0x83, 0x3e, 0x7b, 0x34, 0x95, 0xb5, 0xa7, 0xe1, 0x6b, 0x75, 0x16, 0xbe, 0xee, 0x43, - 0x25, 0xe9, 0xaa, 0xa1, 0xa4, 0x1b, 0xcb, 0x97, 0xcf, 0xef, 0x66, 0x1b, 0x99, 0x3c, 0xc3, 0x4b, - 0xb5, 0x0f, 0xdc, 0x66, 0x54, 0x4c, 0x77, 0x95, 0x1b, 0x5e, 0xda, 0x74, 0x7c, 0xa8, 0xda, 0x5a, - 0xe5, 0x9c, 0x4c, 0x91, 0xca, 0x1c, 0xc5, 0xdc, 0x5d, 0xa5, 0xec, 0x4e, 0x50, 0x29, 0x7f, 0x27, - 0x68, 0x03, 0x9a, 0x71, 0xae, 0x16, 0x6f, 0x2f, 0x82, 0xe5, 0x48, 0xee, 0xc7, 0x50, 0x21, 0x7b, - 0x70, 0x37, 0x34, 0x43, 0x69, 0x02, 0x22, 0x34, 0x9c, 0x15, 0x30, 0x05, 0x4f, 0xa4, 0x3e, 0x38, - 0x39, 0xea, 0xcb, 0xb6, 0x18, 0x48, 0x42, 0xaa, 0x22, 0x6f, 0xc1, 0xba, 0xe1, 0x4d, 0x26, 0xdf, - 0xd0, 0xb6, 0x1d, 0x06, 0x9d, 0x58, 0xc4, 0x17, 0xac, 0xec, 0xbe, 0x4b, 0x27, 0x47, 0xa9, 0xd3, - 0x34, 0xb3, 0x3f, 0x96, 0x18, 0x6c, 0xf4, 0x65, 0x8c, 0x60, 0x6b, 0xce, 0xf5, 0x6c, 0xa8, 0x6d, - 0x6e, 0x23, 0x50, 0x3c, 0xcc, 0x4a, 0xee, 0x47, 0x18, 0x77, 0x8d, 0xb7, 0xa6, 0x5f, 0xd9, 0x9a, - 0x72, 0xb7, 0x73, 0x71, 0xc7, 0xe4, 0xf5, 0x83, 0xc2, 0xa2, 0xd7, 0x0f, 0xdc, 0xf7, 0xe1, 0x86, - 0x37, 0x09, 0xac, 0xfc, 0x6d, 0xa8, 0xa9, 0x61, 0x5e, 0xcf, 0xf3, 0x7c, 0x2f, 0x65, 0x77, 0x7f, - 0x56, 0x80, 0xe5, 0xfd, 0x48, 0xcb, 0x38, 0x12, 0xe1, 0xe3, 0x50, 0xf4, 0xf8, 0x5b, 0x29, 0x12, - 0xcd, 0x4f, 0xe5, 0xf2, 0xbc, 0x93, 0xa0, 0x14, 0xda, 0x9a, 0x1c, 0x7f, 0x11, 0xd6, 0xa4, 0x1f, - 0x68, 0x15, 0x9b, 0x68, 0x2b, 0xbd, 0x05, 0xb2, 0x0e, 0xcc, 0x90, 0xdb, 0xe4, 0xf6, 0x47, 0x66, - 0x9a, 0x5b, 0xb0, 0x3e, 0x41, 0x4d, 0x43, 0xa9, 0x22, 0x7f, 0x05, 0x5a, 0xe3, 0x2d, 0x61, 0x57, - 0x45, 0x7a, 0x3f, 0xf2, 0xe5, 0x39, 0x45, 0x0a, 0xac, 0xe4, 0xfe, 0x5b, 0x16, 0xa3, 0x1c, 0xdb, - 0x3b, 0x22, 0xb1, 0x52, 0x7a, 0x5c, 0x91, 0x35, 0xad, 0xdc, 0x3f, 0x90, 0x8a, 0x0b, 0xfc, 0x03, - 0xe9, 0xdd, 0xf1, 0x3f, 0x90, 0xcc, 0x66, 0xf0, 0xea, 0xdc, 0x1d, 0x86, 0x8e, 0xb6, 0x6d, 0x8c, - 0xd8, 0x96, 0xb9, 0xbf, 0x23, 0xbd, 0x69, 0x13, 0x83, 0xf2, 0x22, 0x51, 0x97, 0x39, 0xdb, 0x7b, - 0x38, 0x7d, 0xf3, 0x75, 0xb1, 0x2b, 0x28, 0x33, 0xd1, 0x16, 0x5c, 0x3b, 0xda, 0x7a, 0x6f, 0x2a, - 0x06, 0xaf, 0xcf, 0x2d, 0xa2, 0x5c, 0xf1, 0xf7, 0x9c, 0xf7, 0xa0, 0xd6, 0x0f, 0x12, 0xad, 0xe2, - 0x0b, 0x0a, 0x64, 0x66, 0xaf, 0xb8, 0xe7, 0x46, 0x6b, 0xcf, 0x30, 0xd2, 0x7d, 0x80, 0x54, 0xca, - 0xe9, 0x01, 0x8c, 0x47, 0x71, 0x06, 0x6b, 0x3e, 0xc7, 0xdf, 0xc1, 0x6e, 0x42, 0x35, 0x19, 0x75, - 0xc6, 0x25, 0x76, 0xdb, 0x72, 0xce, 0xc1, 0x99, 0xd9, 0xa7, 0x0f, 0x65, 0x6c, 0xec, 0x43, 0xec, - 0x4d, 0x4b, 0xf1, 0xf6, 0xf3, 0x59, 0x9b, 0xbf, 0x9b, 0x9f, 0x1e, 0xe3, 0x42, 0x1b, 0x97, 0x8c, - 0x71, 0xa6, 0x39, 0x37, 0x4f, 0xce, 0x43, 0x68, 0xe6, 0xba, 0x8e, 0xf8, 0x39, 0x8a, 0x7c, 0x95, - 0x56, 0xea, 0xf0, 0xd9, 0x5c, 0xcb, 0xf7, 0xd3, 0x5a, 0x1d, 0x3d, 0xdf, 0xf9, 0x71, 0x11, 0x56, - 0x27, 0xdd, 0x85, 0x6a, 0x96, 0x06, 0xaa, 0x0e, 0x42, 0x3f, 0x97, 0x3a, 0x32, 0x7e, 0x03, 0x9a, - 0x87, 0x26, 0xda, 0x23, 0xc2, 0x1a, 0xbe, 0xda, 0x53, 0x03, 0xc9, 0x36, 0xf2, 0x17, 0x9a, 0xdf, - 0x40, 0x9c, 0x35, 0x65, 0x60, 0x36, 0xe4, 0x0d, 0x7b, 0x05, 0xec, 0x87, 0x45, 0xbe, 0x92, 0x4b, - 0x60, 0x7e, 0x52, 0xe4, 0xeb, 0x70, 0x63, 0x7b, 0x14, 0xf9, 0xa1, 0xf4, 0x33, 0xea, 0x5f, 0xe7, - 0xa9, 0x59, 0xaa, 0xf2, 0x43, 0xcc, 0x8e, 0x1a, 0xed, 0x51, 0xc7, 0xa6, 0x29, 0x7f, 0x50, 0xe6, - 0x37, 0x61, 0xcd, 0x72, 0x8d, 0x43, 0x31, 0xf6, 0x87, 0x65, 0xfe, 0x02, 0xac, 0x6e, 0x99, 0x31, - 0xb3, 0x86, 0xb2, 0x3f, 0x2a, 0xa3, 0x09, 0x74, 0x04, 0xf6, 0xc7, 0xa4, 0x27, 0x2b, 0x99, 0xb0, - 0x3f, 0x2d, 0x73, 0x0e, 0x2b, 0x4f, 0x83, 0x24, 0x09, 0xa2, 0x9e, 0xd5, 0xfd, 0xa3, 0xf2, 0x9d, - 0x9f, 0x15, 0x60, 0x75, 0x12, 0x54, 0x31, 0x48, 0x0c, 0x55, 0xd4, 0xd3, 0xe6, 0x9e, 0xf5, 0x0a, - 0x34, 0x92, 0xbe, 0x8a, 0x35, 0x35, 0xa9, 0xaa, 0x1c, 0xd1, 0xe9, 0x95, 0x49, 0xef, 0x4c, 0xb9, - 0xc9, 0x9c, 0xff, 0x6b, 0xd1, 0x63, 0x4d, 0x1c, 0x25, 0x1f, 0xbf, 0x5f, 0xce, 0x02, 0x5e, 0x3a, - 0x45, 0x4b, 0x4f, 0x29, 0x58, 0x15, 0x59, 0x47, 0x71, 0x68, 0x02, 0x5f, 0x39, 0x10, 0x41, 0x68, - 0x2e, 0x54, 0x0e, 0xfb, 0x98, 0xb9, 0x35, 0x0c, 0x55, 0x7d, 0x12, 0x98, 0xab, 0x8b, 0x76, 0x0b, - 0xf3, 0xd1, 0x8e, 0x6c, 0xfe, 0x99, 0xdc, 0xbe, 0xf3, 0xcf, 0x9f, 0xde, 0x2a, 0xfc, 0xfc, 0xd3, - 0x5b, 0x85, 0xff, 0xfa, 0xf4, 0x56, 0xe1, 0xc7, 0x9f, 0xdd, 0x5a, 0xfa, 0xf9, 0x67, 0xb7, 0x96, - 0x7e, 0xf1, 0xd9, 0xad, 0xa5, 0x8f, 0xd9, 0xf4, 0x7f, 0x31, 0x3b, 0x55, 0xf2, 0xec, 0xfb, 0xff, - 0x1f, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x03, 0x71, 0x14, 0xa6, 0x39, 0x00, 0x00, + // 5169 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x7b, 0x4d, 0x6c, 0x24, 0xc7, + 0x75, 0x3f, 0xe7, 0x7b, 0xe6, 0x0d, 0xc9, 0x2d, 0x96, 0xa8, 0xd5, 0xfc, 0x5b, 0xf2, 0xfe, 0xe9, + 0x8e, 0x2c, 0xaf, 0xd7, 0x32, 0x57, 0xda, 0x0f, 0x4b, 0x76, 0x22, 0xc9, 0xfc, 0xd8, 0x35, 0x09, + 0xed, 0x8a, 0x74, 0x0f, 0xc5, 0x8d, 0x85, 0x24, 0x70, 0xcd, 0x74, 0x71, 0xa6, 0xc5, 0x9e, 0xae, + 0x71, 0x77, 0x0d, 0x97, 0x34, 0x10, 0xc0, 0xf9, 0x72, 0x80, 0x1c, 0x02, 0x23, 0x40, 0x8e, 0x01, + 0x9c, 0x9c, 0x73, 0x33, 0x8c, 0x38, 0x40, 0xae, 0x01, 0x02, 0xe4, 0x10, 0x1f, 0x03, 0x04, 0x48, + 0x02, 0xeb, 0x98, 0x43, 0x80, 0x9c, 0x73, 0x08, 0xde, 0xab, 0xea, 0x9e, 0x9e, 0x0f, 0x72, 0x87, + 0xb2, 0x4f, 0xd3, 0xf5, 0xfa, 0xbd, 0xd7, 0xaf, 0xaa, 0x5e, 0xfd, 0xea, 0xbd, 0x57, 0x35, 0xf0, + 0xfa, 0xf0, 0xb4, 0x77, 0x37, 0x0c, 0x3a, 0x77, 0x87, 0x9d, 0xbb, 0x03, 0xe5, 0xcb, 0xf0, 0xee, + 0x30, 0x56, 0x5a, 0x25, 0xa6, 0x91, 0x6c, 0x52, 0x8b, 0xaf, 0x88, 0xe8, 0x42, 0x5f, 0x0c, 0xe5, + 0x26, 0x51, 0x9d, 0xd7, 0x7a, 0x4a, 0xf5, 0x42, 0x69, 0x58, 0x3b, 0xa3, 0x93, 0xbb, 0x89, 0x8e, + 0x47, 0x5d, 0x6d, 0x98, 0xdd, 0x7f, 0x2c, 0xc1, 0xcd, 0xf6, 0x40, 0xc4, 0x7a, 0x3b, 0x54, 0xdd, + 0xd3, 0x76, 0x24, 0x86, 0x49, 0x5f, 0xe9, 0x6d, 0x91, 0x48, 0xfe, 0x26, 0x54, 0x3b, 0x48, 0x4c, + 0x5a, 0x85, 0x8d, 0xd2, 0xed, 0xe6, 0xbd, 0xf5, 0xcd, 0x09, 0xc5, 0x9b, 0x24, 0xe1, 0x59, 0x1e, + 0xfe, 0x36, 0xd4, 0x7c, 0xa9, 0x45, 0x10, 0x26, 0xad, 0xe2, 0x46, 0xe1, 0x76, 0xf3, 0xde, 0x2b, + 0x9b, 0xe6, 0xc3, 0x9b, 0xe9, 0x87, 0x37, 0xdb, 0xf4, 0x61, 0x2f, 0xe5, 0xe3, 0xf7, 0xa1, 0x7e, + 0x12, 0x84, 0xf2, 0x43, 0x79, 0x91, 0xb4, 0x4a, 0x57, 0xcb, 0x64, 0x8c, 0xfc, 0x03, 0x58, 0x95, + 0xe7, 0x3a, 0x16, 0x9e, 0x0c, 0x85, 0x0e, 0x54, 0x94, 0xb4, 0xca, 0x64, 0xdd, 0x2b, 0x53, 0xd6, + 0xa5, 0xef, 0xbd, 0x29, 0x76, 0xbe, 0x01, 0x4d, 0xd5, 0xf9, 0x54, 0x76, 0xf5, 0xd1, 0xc5, 0x50, + 0x26, 0xad, 0xca, 0x46, 0xe9, 0x76, 0xc3, 0xcb, 0x93, 0xf8, 0x37, 0xa0, 0xd9, 0x55, 0x61, 0x28, + 0xbb, 0x46, 0x7f, 0xf5, 0x6a, 0xd3, 0xf2, 0xbc, 0xfc, 0x01, 0xbc, 0x1c, 0xcb, 0x81, 0x3a, 0x93, + 0xfe, 0x4e, 0x46, 0xa5, 0xfe, 0xd5, 0xe9, 0x33, 0xf3, 0x5f, 0xf2, 0x2d, 0x58, 0x89, 0xad, 0x7d, + 0x4f, 0x82, 0xe8, 0x34, 0x69, 0xd5, 0xa8, 0x4b, 0xaf, 0x5e, 0xd2, 0x25, 0xe4, 0xf1, 0x26, 0x25, + 0xdc, 0x3f, 0xdb, 0x81, 0x0a, 0x4d, 0x08, 0x5f, 0x85, 0x62, 0xe0, 0xb7, 0x0a, 0x1b, 0x85, 0xdb, + 0x0d, 0xaf, 0x18, 0xf8, 0xfc, 0x2e, 0x54, 0x4f, 0x02, 0x19, 0xfa, 0x2f, 0x9c, 0x17, 0xcb, 0xc6, + 0x1f, 0xc1, 0x72, 0x2c, 0x13, 0x1d, 0x07, 0xb6, 0xff, 0x66, 0x6a, 0xbe, 0x38, 0x6f, 0xf6, 0x37, + 0xbd, 0x1c, 0xa3, 0x37, 0x21, 0x86, 0xe3, 0xdc, 0xed, 0x07, 0xa1, 0x1f, 0xcb, 0x68, 0xdf, 0x37, + 0xb3, 0xd4, 0xf0, 0xf2, 0x24, 0x7e, 0x1b, 0x6e, 0x74, 0x44, 0xf7, 0xb4, 0x17, 0xab, 0x51, 0x84, + 0x43, 0xa2, 0xe2, 0x56, 0x85, 0xcc, 0x9e, 0x26, 0xf3, 0xb7, 0xa0, 0x22, 0xc2, 0xa0, 0x17, 0xd1, + 0x5c, 0xac, 0xde, 0x73, 0xe6, 0xda, 0xb2, 0x85, 0x1c, 0x9e, 0x61, 0xe4, 0x7b, 0xb0, 0x72, 0x26, + 0x63, 0x1d, 0x74, 0x45, 0x48, 0xf4, 0x56, 0x8d, 0x24, 0xdd, 0xb9, 0x92, 0xc7, 0x79, 0x4e, 0x6f, + 0x52, 0x90, 0xef, 0x03, 0x24, 0xb8, 0x40, 0xc8, 0xcf, 0x5b, 0x4d, 0x1a, 0x8c, 0x2f, 0xcf, 0x55, + 0xb3, 0xa3, 0x22, 0x2d, 0x23, 0xbd, 0xd9, 0xce, 0xd8, 0xf7, 0x96, 0xbc, 0x9c, 0x30, 0x7f, 0x07, + 0xca, 0x5a, 0x9e, 0xeb, 0xd6, 0xea, 0x15, 0x23, 0x9a, 0x2a, 0x39, 0x92, 0xe7, 0x7a, 0x6f, 0xc9, + 0x23, 0x01, 0x14, 0xc4, 0x05, 0xd0, 0xba, 0xb1, 0x80, 0xe0, 0xe3, 0x20, 0x94, 0x28, 0x88, 0x02, + 0xfc, 0x3d, 0xa8, 0x86, 0xe2, 0x42, 0x8d, 0x74, 0x8b, 0x91, 0xe8, 0x6f, 0x5c, 0x29, 0xfa, 0x84, + 0x58, 0xf7, 0x96, 0x3c, 0x2b, 0xc4, 0x1f, 0x40, 0xc9, 0x0f, 0xce, 0x5a, 0x6b, 0x24, 0xbb, 0x71, + 0xa5, 0xec, 0x6e, 0x70, 0xb6, 0xb7, 0xe4, 0x21, 0x3b, 0xdf, 0x81, 0x7a, 0x47, 0xa9, 0xd3, 0x81, + 0x88, 0x4f, 0x5b, 0x9c, 0x44, 0xbf, 0x74, 0xa5, 0xe8, 0xb6, 0x65, 0xde, 0x5b, 0xf2, 0x32, 0x41, + 0xec, 0x72, 0xd0, 0x55, 0x51, 0xeb, 0xa5, 0x05, 0xba, 0xbc, 0xdf, 0x55, 0x11, 0x76, 0x19, 0x05, + 0x50, 0x30, 0x0c, 0xa2, 0xd3, 0xd6, 0xfa, 0x02, 0x82, 0xb8, 0x76, 0x50, 0x10, 0x05, 0xd0, 0x6c, + 0x5f, 0x68, 0x71, 0x16, 0xc8, 0xe7, 0xad, 0x97, 0x17, 0x30, 0x7b, 0xd7, 0x32, 0xa3, 0xd9, 0xa9, + 0x20, 0x2a, 0x49, 0x17, 0x66, 0xeb, 0xe6, 0x02, 0x4a, 0xd2, 0x35, 0x8d, 0x4a, 0x52, 0x41, 0xfe, + 0x7b, 0xb0, 0x76, 0x22, 0x85, 0x1e, 0xc5, 0xd2, 0x1f, 0xc3, 0xdc, 0x2b, 0xa4, 0x6d, 0xf3, 0xea, + 0xb9, 0x9f, 0x96, 0xda, 0x5b, 0xf2, 0x66, 0x55, 0xf1, 0x6f, 0x42, 0x25, 0x14, 0x5a, 0x9e, 0xb7, + 0x5a, 0xa4, 0xd3, 0x7d, 0x81, 0x53, 0x68, 0x79, 0xbe, 0xb7, 0xe4, 0x19, 0x11, 0xfe, 0xdb, 0x70, + 0x43, 0x8b, 0x4e, 0x28, 0x0f, 0x4e, 0x2c, 0x43, 0xd2, 0xfa, 0x7f, 0xa4, 0xe5, 0xcd, 0xab, 0xdd, + 0x79, 0x52, 0x66, 0x6f, 0xc9, 0x9b, 0x56, 0x83, 0x56, 0x11, 0xa9, 0xe5, 0x2c, 0x60, 0x15, 0xe9, + 0x43, 0xab, 0x48, 0x84, 0x3f, 0x81, 0x26, 0x3d, 0xec, 0xa8, 0x70, 0x34, 0x88, 0x5a, 0xaf, 0x92, + 0x86, 0xdb, 0x2f, 0xd6, 0x60, 0xf8, 0xf7, 0x96, 0xbc, 0xbc, 0x38, 0x4e, 0x22, 0x35, 0x3d, 0xf5, + 0xbc, 0xf5, 0xda, 0x02, 0x93, 0x78, 0x64, 0x99, 0x71, 0x12, 0x53, 0x41, 0x5c, 0x7a, 0xcf, 0x03, + 0xbf, 0x27, 0x75, 0xeb, 0x0b, 0x0b, 0x2c, 0xbd, 0x67, 0xc4, 0x8a, 0x4b, 0xcf, 0x08, 0x39, 0x3f, + 0x80, 0xe5, 0x3c, 0xb8, 0x72, 0x0e, 0xe5, 0x58, 0x0a, 0x03, 0xec, 0x75, 0x8f, 0x9e, 0x91, 0x26, + 0xfd, 0x40, 0x13, 0xb0, 0xd7, 0x3d, 0x7a, 0xe6, 0x37, 0xa1, 0x6a, 0x36, 0x19, 0xc2, 0xed, 0xba, + 0x67, 0x5b, 0xc8, 0xeb, 0xc7, 0xa2, 0xd7, 0x2a, 0x1b, 0x5e, 0x7c, 0x46, 0x5e, 0x3f, 0x56, 0xc3, + 0x83, 0x88, 0x70, 0xb7, 0xee, 0xd9, 0x96, 0xf3, 0xf3, 0x07, 0x50, 0xb3, 0x86, 0x39, 0x7f, 0x55, + 0x80, 0xaa, 0xc1, 0x05, 0xfe, 0x01, 0x54, 0x12, 0x7d, 0x11, 0x4a, 0xb2, 0x61, 0xf5, 0xde, 0x57, + 0x16, 0xc0, 0x92, 0xcd, 0x36, 0x0a, 0x78, 0x46, 0xce, 0xf5, 0xa0, 0x42, 0x6d, 0x5e, 0x83, 0x92, + 0xa7, 0x9e, 0xb3, 0x25, 0x0e, 0x50, 0x35, 0x63, 0xce, 0x0a, 0x48, 0xdc, 0x0d, 0xce, 0x58, 0x11, + 0x89, 0x7b, 0x52, 0xf8, 0x32, 0x66, 0x25, 0xbe, 0x02, 0x8d, 0x74, 0x74, 0x13, 0x56, 0xe6, 0x0c, + 0x96, 0x73, 0xf3, 0x96, 0xb0, 0x8a, 0xf3, 0x3f, 0x65, 0x28, 0xe3, 0x32, 0xe6, 0xaf, 0xc3, 0x8a, + 0x16, 0x71, 0x4f, 0x9a, 0x48, 0x66, 0x3f, 0xdd, 0x02, 0x27, 0x89, 0xfc, 0xbd, 0xb4, 0x0f, 0x45, + 0xea, 0xc3, 0x97, 0x5f, 0x08, 0x0f, 0x13, 0x3d, 0xc8, 0x6d, 0xa6, 0xa5, 0xc5, 0x36, 0xd3, 0xc7, + 0x50, 0x47, 0x54, 0x6a, 0x07, 0x3f, 0x90, 0x34, 0xf4, 0xab, 0xf7, 0xee, 0xbc, 0xf8, 0x93, 0xfb, + 0x56, 0xc2, 0xcb, 0x64, 0xf9, 0x3e, 0x34, 0xba, 0x22, 0xf6, 0xc9, 0x18, 0x9a, 0xad, 0xd5, 0x7b, + 0x5f, 0x7d, 0xb1, 0xa2, 0x9d, 0x54, 0xc4, 0x1b, 0x4b, 0xf3, 0x03, 0x68, 0xfa, 0x32, 0xe9, 0xc6, + 0xc1, 0x90, 0x50, 0xca, 0x6c, 0xa9, 0x5f, 0x7b, 0xb1, 0xb2, 0xdd, 0xb1, 0x90, 0x97, 0xd7, 0xc0, + 0x5f, 0x83, 0x46, 0x9c, 0xc1, 0x54, 0x8d, 0xf6, 0xf9, 0x31, 0xc1, 0x7d, 0x07, 0xea, 0x69, 0x7f, + 0xf8, 0x32, 0xd4, 0xf1, 0xf7, 0x23, 0x15, 0x49, 0xb6, 0x84, 0x73, 0x8b, 0xad, 0xf6, 0x40, 0x84, + 0x21, 0x2b, 0xf0, 0x55, 0x00, 0x6c, 0x3e, 0x95, 0x7e, 0x30, 0x1a, 0xb0, 0xa2, 0xfb, 0x9b, 0xa9, + 0xb7, 0xd4, 0xa1, 0x7c, 0x28, 0x7a, 0x28, 0xb1, 0x0c, 0xf5, 0x14, 0x75, 0x59, 0x01, 0xe5, 0x77, + 0x45, 0xd2, 0xef, 0x28, 0x11, 0xfb, 0xac, 0xc8, 0x9b, 0x50, 0xdb, 0x8a, 0xbb, 0xfd, 0xe0, 0x4c, + 0xb2, 0x92, 0x7b, 0x17, 0x9a, 0x39, 0x7b, 0x51, 0x85, 0xfd, 0x68, 0x03, 0x2a, 0x5b, 0xbe, 0x2f, + 0x7d, 0x56, 0x40, 0x01, 0xdb, 0x41, 0x56, 0x74, 0xbf, 0x0a, 0x8d, 0x6c, 0xb4, 0x90, 0x1d, 0xf7, + 0x5f, 0xb6, 0x84, 0x4f, 0x48, 0x66, 0x05, 0xf4, 0xca, 0xfd, 0x28, 0x0c, 0x22, 0xc9, 0x8a, 0xce, + 0xf7, 0xc8, 0x55, 0xf9, 0x6f, 0x4d, 0x2e, 0x88, 0x37, 0x5e, 0xb4, 0x41, 0x4e, 0xae, 0x86, 0x57, + 0x73, 0xfd, 0x7b, 0x12, 0x90, 0x71, 0x75, 0x28, 0xef, 0x2a, 0x9d, 0xb0, 0x82, 0xf3, 0x5f, 0x45, + 0xa8, 0xa7, 0xfb, 0x22, 0x67, 0x50, 0x1a, 0xc5, 0xa1, 0x75, 0x68, 0x7c, 0xe4, 0xeb, 0x50, 0xd1, + 0x81, 0xb6, 0x6e, 0xdc, 0xf0, 0x4c, 0x03, 0x43, 0xae, 0xfc, 0xcc, 0x96, 0xe8, 0xdd, 0xf4, 0x54, + 0x05, 0x03, 0xd1, 0x93, 0x7b, 0x22, 0xe9, 0x93, 0x3f, 0x36, 0xbc, 0x31, 0x01, 0xe5, 0x4f, 0xc4, + 0x19, 0xfa, 0x1c, 0xbd, 0x37, 0xc1, 0x58, 0x9e, 0xc4, 0xef, 0x43, 0x19, 0x3b, 0x68, 0x9d, 0xe6, + 0xff, 0x4f, 0x75, 0x18, 0xdd, 0xe4, 0x30, 0x96, 0x38, 0x3d, 0x9b, 0x18, 0x4a, 0x7b, 0xc4, 0xcc, + 0xdf, 0x80, 0x55, 0xb3, 0x08, 0x0f, 0x28, 0xc8, 0xde, 0xf7, 0x29, 0x18, 0x6b, 0x78, 0x53, 0x54, + 0xbe, 0x85, 0xc3, 0x29, 0xb4, 0x6c, 0xd5, 0x17, 0xf0, 0xef, 0x74, 0x70, 0x36, 0xdb, 0x28, 0xe2, + 0x19, 0x49, 0xf7, 0x21, 0x8e, 0xa9, 0xd0, 0x12, 0xa7, 0xf9, 0xd1, 0x60, 0xa8, 0x2f, 0x8c, 0xd3, + 0x3c, 0x96, 0xba, 0xdb, 0x0f, 0xa2, 0x1e, 0x2b, 0x98, 0x21, 0xc6, 0x49, 0x24, 0x96, 0x38, 0x56, + 0x31, 0x2b, 0x39, 0x0e, 0x94, 0xd1, 0x47, 0x11, 0x24, 0x23, 0x31, 0x90, 0x76, 0xa4, 0xe9, 0xd9, + 0x79, 0x09, 0xd6, 0x66, 0xb6, 0x55, 0xe7, 0xef, 0xab, 0xc6, 0x43, 0x50, 0x82, 0x42, 0x3a, 0x2b, + 0x41, 0xd1, 0xda, 0xb5, 0x30, 0x06, 0xb5, 0x4c, 0x62, 0xcc, 0x7b, 0x50, 0xc1, 0x8e, 0xa5, 0x10, + 0xb3, 0x80, 0xf8, 0x53, 0x64, 0xf7, 0x8c, 0x14, 0x6f, 0x41, 0xad, 0xdb, 0x97, 0xdd, 0x53, 0xe9, + 0x5b, 0xac, 0x4f, 0x9b, 0xe8, 0x34, 0xdd, 0x5c, 0x94, 0x6d, 0x1a, 0xe4, 0x12, 0x5d, 0x15, 0x3d, + 0x1a, 0xa8, 0x4f, 0x03, 0x9a, 0x57, 0x74, 0x89, 0x94, 0x90, 0xbe, 0xdd, 0x47, 0x1f, 0xb1, 0xd3, + 0x36, 0x26, 0x38, 0x8f, 0xa0, 0x42, 0xdf, 0xc6, 0x95, 0x60, 0x6c, 0x36, 0xa9, 0xe2, 0x1b, 0x8b, + 0xd9, 0x6c, 0x4d, 0x76, 0xfe, 0xb6, 0x08, 0x65, 0x6c, 0xf3, 0x3b, 0x50, 0x89, 0x45, 0xd4, 0x33, + 0x13, 0x30, 0x9b, 0x71, 0x7a, 0xf8, 0xce, 0x33, 0x2c, 0xfc, 0x03, 0xeb, 0x8a, 0xc5, 0x05, 0x9c, + 0x25, 0xfb, 0x62, 0xde, 0x2d, 0xd7, 0xa1, 0x32, 0x14, 0xb1, 0x18, 0xd8, 0x75, 0x62, 0x1a, 0xee, + 0x4f, 0x0a, 0x50, 0x46, 0x26, 0xbe, 0x06, 0x2b, 0x6d, 0x1d, 0x07, 0xa7, 0x52, 0xf7, 0x63, 0x35, + 0xea, 0xf5, 0x8d, 0x27, 0x7d, 0x28, 0x2f, 0x0c, 0xde, 0x18, 0x40, 0xd0, 0x22, 0x0c, 0xba, 0xac, + 0x88, 0x5e, 0xb5, 0xad, 0x42, 0x9f, 0x95, 0xf8, 0x0d, 0x68, 0x7e, 0x1c, 0xf9, 0x32, 0x4e, 0xba, + 0x2a, 0x96, 0x3e, 0x2b, 0xdb, 0xd5, 0x7d, 0xca, 0x2a, 0xb4, 0x97, 0xc9, 0x73, 0x4d, 0x29, 0x0d, + 0xab, 0xf2, 0x97, 0xe0, 0xc6, 0xf6, 0x64, 0x9e, 0xc3, 0x6a, 0x88, 0x49, 0x4f, 0x65, 0x84, 0x4e, + 0xc6, 0xea, 0xc6, 0x89, 0xd5, 0xa7, 0x01, 0x6b, 0xe0, 0xc7, 0xcc, 0x3a, 0x61, 0xe0, 0xfe, 0x43, + 0x21, 0x45, 0x8e, 0x15, 0x68, 0x1c, 0x8a, 0x58, 0xf4, 0x62, 0x31, 0x44, 0xfb, 0x9a, 0x50, 0x33, + 0x1b, 0xe7, 0xdb, 0x06, 0xdd, 0x4c, 0xe3, 0x9e, 0xc1, 0x46, 0xd3, 0xb8, 0xcf, 0x4a, 0xe3, 0xc6, + 0x03, 0x56, 0xc6, 0x6f, 0x7c, 0x67, 0xa4, 0xb4, 0x64, 0x15, 0xc2, 0x3a, 0xe5, 0x4b, 0x56, 0x45, + 0xe2, 0x11, 0x22, 0x0a, 0xab, 0x61, 0x9f, 0x77, 0xd0, 0x7f, 0x3a, 0xea, 0x9c, 0xd5, 0xd1, 0x0c, + 0x1c, 0x46, 0xe9, 0xb3, 0x06, 0xbe, 0xf9, 0x68, 0x34, 0xe8, 0x48, 0xec, 0x26, 0xe0, 0x9b, 0x23, + 0xd5, 0xeb, 0x85, 0x92, 0x35, 0x71, 0x0c, 0x72, 0xe0, 0xcb, 0x96, 0x09, 0x69, 0x45, 0x18, 0xaa, + 0x91, 0x66, 0x2b, 0xce, 0x2f, 0x4a, 0x50, 0xc6, 0x24, 0x05, 0xd7, 0x4e, 0x1f, 0x71, 0xc6, 0xae, + 0x1d, 0x7c, 0xce, 0x56, 0x60, 0x71, 0xbc, 0x02, 0xf9, 0x37, 0xed, 0x4c, 0x97, 0x16, 0x40, 0x59, + 0x54, 0x9c, 0x9f, 0x64, 0x0e, 0xe5, 0x41, 0x30, 0x90, 0x16, 0xeb, 0xe8, 0x19, 0x69, 0x09, 0xee, + 0xc7, 0xb8, 0x0c, 0x4a, 0x1e, 0x3d, 0xe3, 0xaa, 0x11, 0xb8, 0x2d, 0x6c, 0x69, 0x5a, 0x03, 0x25, + 0x2f, 0x6d, 0x9a, 0xd5, 0x8c, 0xa8, 0x54, 0x5b, 0x60, 0x35, 0xd3, 0xe7, 0xf3, 0x88, 0x34, 0x06, + 0x83, 0xfa, 0xe2, 0xe2, 0xb9, 0x4d, 0x62, 0xd7, 0x7a, 0xe3, 0x78, 0x03, 0xab, 0x9b, 0xd1, 0x63, + 0x05, 0x9c, 0x25, 0x5a, 0x86, 0x06, 0xcb, 0x8e, 0x03, 0x5f, 0x2a, 0x56, 0xa2, 0x0d, 0x6e, 0xe4, + 0x07, 0x8a, 0x95, 0x31, 0xa2, 0x3a, 0xdc, 0x7d, 0xcc, 0x2a, 0xee, 0x1b, 0xb9, 0xad, 0x66, 0x6b, + 0xa4, 0x95, 0x51, 0x43, 0x6e, 0x59, 0x30, 0x5e, 0xd6, 0x91, 0x3e, 0x2b, 0xba, 0x5f, 0x9f, 0x03, + 0x9f, 0x2b, 0xd0, 0xf8, 0x78, 0x18, 0x2a, 0xe1, 0x5f, 0x81, 0x9f, 0xcb, 0x00, 0xe3, 0xa4, 0xd7, + 0xf9, 0xd9, 0x17, 0xc6, 0xdb, 0x34, 0xc6, 0x98, 0x89, 0x1a, 0xc5, 0x5d, 0x49, 0xd0, 0xd0, 0xf0, + 0x6c, 0x8b, 0x7f, 0x0b, 0x2a, 0xf8, 0x3e, 0x69, 0x15, 0x09, 0x31, 0xee, 0x2c, 0x94, 0x6a, 0x6d, + 0x1e, 0x07, 0xf2, 0xb9, 0x67, 0x04, 0xf9, 0xc3, 0x7c, 0xd8, 0xf1, 0x82, 0x22, 0xd0, 0x98, 0x93, + 0xdf, 0x02, 0x10, 0x5d, 0x1d, 0x9c, 0x49, 0xd4, 0x65, 0xd7, 0x7e, 0x8e, 0xc2, 0x3d, 0x68, 0xe2, + 0x92, 0x1c, 0x1e, 0xc4, 0xb8, 0x8a, 0x5b, 0xcb, 0xa4, 0xf8, 0xad, 0xc5, 0xcc, 0xfb, 0x76, 0x26, + 0xe8, 0xe5, 0x95, 0xf0, 0x8f, 0x61, 0xd9, 0x14, 0x98, 0xac, 0xd2, 0x15, 0x52, 0xfa, 0xf6, 0x62, + 0x4a, 0x0f, 0xc6, 0x92, 0xde, 0x84, 0x9a, 0xd9, 0xba, 0x51, 0xe5, 0xba, 0x75, 0x23, 0xdc, 0x9b, + 0x8f, 0x26, 0xf7, 0x66, 0xb3, 0x05, 0x4c, 0x51, 0xb9, 0x0b, 0xcb, 0x41, 0x32, 0x2e, 0x5b, 0x51, + 0x09, 0xa3, 0xee, 0x4d, 0xd0, 0x9c, 0x1f, 0x55, 0xa1, 0x4c, 0x43, 0x38, 0x5d, 0x82, 0xda, 0x99, + 0x80, 0xea, 0xbb, 0x8b, 0x4f, 0xf5, 0xd4, 0x4a, 0x26, 0x64, 0x28, 0xe5, 0x90, 0xe1, 0x5b, 0x50, + 0x49, 0x54, 0xac, 0xd3, 0xe9, 0x5f, 0xd0, 0x89, 0xda, 0x2a, 0xd6, 0x9e, 0x11, 0xe4, 0x8f, 0xa1, + 0x76, 0x12, 0x84, 0x1a, 0x27, 0xc5, 0x0c, 0xde, 0x9b, 0x8b, 0xe9, 0x78, 0x4c, 0x42, 0x5e, 0x2a, + 0xcc, 0x9f, 0xe4, 0x9d, 0xb1, 0x4a, 0x9a, 0x36, 0x17, 0xd3, 0x34, 0xcf, 0x47, 0xef, 0x00, 0xeb, + 0xaa, 0x33, 0x19, 0xa7, 0xef, 0x3e, 0x94, 0x17, 0x76, 0xf3, 0x9d, 0xa1, 0x73, 0x07, 0xea, 0xfd, + 0xc0, 0x97, 0x18, 0xbf, 0x10, 0xc6, 0xd4, 0xbd, 0xac, 0xcd, 0x3f, 0x84, 0x3a, 0xc5, 0xfd, 0x88, + 0x76, 0x8d, 0x6b, 0x0f, 0xbe, 0x49, 0x41, 0x52, 0x05, 0xf8, 0x21, 0xfa, 0xf8, 0xe3, 0x40, 0xb7, + 0xc0, 0x7c, 0x28, 0x6d, 0xa3, 0xc1, 0xe4, 0xef, 0x79, 0x83, 0x9b, 0xc6, 0xe0, 0x69, 0x3a, 0x7f, + 0x00, 0x2f, 0x13, 0x6d, 0x6a, 0xf3, 0xc3, 0xa5, 0x86, 0x4a, 0xe7, 0xbf, 0xc4, 0x40, 0x64, 0x28, + 0x7a, 0xf2, 0x49, 0x30, 0x08, 0x74, 0x6b, 0x65, 0xa3, 0x70, 0xbb, 0xe2, 0x8d, 0x09, 0xfc, 0x4d, + 0x58, 0xf3, 0xe5, 0x89, 0x18, 0x85, 0xfa, 0x48, 0x0e, 0x86, 0xa1, 0xd0, 0x72, 0xdf, 0x27, 0x1f, + 0x6d, 0x78, 0xb3, 0x2f, 0xdc, 0x07, 0x16, 0x54, 0x71, 0x9b, 0xc3, 0x6c, 0x32, 0x85, 0xc3, 0x44, + 0x9b, 0x7d, 0xf3, 0xdb, 0x22, 0x0c, 0x65, 0x7c, 0x61, 0x52, 0xd1, 0x0f, 0x45, 0xd4, 0x11, 0x11, + 0x2b, 0xb9, 0xb7, 0xa1, 0x4c, 0xe3, 0xd0, 0x80, 0x8a, 0x49, 0x59, 0x28, 0x7d, 0xb5, 0xe9, 0x0a, + 0xc1, 0xe8, 0x13, 0x5c, 0x33, 0xac, 0xe8, 0xfc, 0xbc, 0x04, 0xf5, 0xb4, 0xc7, 0x18, 0xbc, 0x9f, + 0xca, 0x8b, 0x34, 0x78, 0x3f, 0x95, 0x17, 0x14, 0x53, 0x25, 0xc7, 0x41, 0x12, 0x74, 0x6c, 0x8c, + 0x58, 0xf7, 0xc6, 0x04, 0x0c, 0x4b, 0x9e, 0x07, 0xbe, 0xee, 0x93, 0xa3, 0x57, 0x3c, 0xd3, 0xe0, + 0xb7, 0xe1, 0x86, 0x8f, 0xc6, 0x47, 0xdd, 0x70, 0xe4, 0xcb, 0x23, 0xdc, 0xd2, 0x4c, 0xce, 0x3e, + 0x4d, 0xe6, 0xdf, 0x05, 0xd0, 0xc1, 0x40, 0x3e, 0x56, 0xf1, 0x40, 0x68, 0x1b, 0xa8, 0x7f, 0xe3, + 0x7a, 0xae, 0xb8, 0x79, 0x94, 0x29, 0xf0, 0x72, 0xca, 0x50, 0x35, 0x7e, 0xcd, 0xaa, 0xae, 0x7d, + 0x2e, 0xd5, 0xbb, 0x99, 0x02, 0x2f, 0xa7, 0xcc, 0xfd, 0x1d, 0x80, 0xf1, 0x1b, 0x7e, 0x13, 0xf8, + 0x53, 0x15, 0xe9, 0xfe, 0x56, 0xa7, 0x13, 0x6f, 0xcb, 0x13, 0x15, 0xcb, 0x5d, 0x81, 0x7b, 0xd1, + 0xcb, 0xb0, 0x96, 0xd1, 0xb7, 0x4e, 0xb4, 0x8c, 0x91, 0x4c, 0x43, 0xdf, 0xee, 0xab, 0x58, 0x9b, + 0x40, 0x87, 0x1e, 0x3f, 0x6e, 0xb3, 0x12, 0xee, 0x7f, 0xfb, 0xed, 0x03, 0x56, 0x76, 0x6f, 0x03, + 0x8c, 0xbb, 0x44, 0x09, 0x01, 0x3d, 0xbd, 0x7d, 0xcf, 0xa6, 0x07, 0xd4, 0xba, 0xf7, 0x80, 0x15, + 0x9c, 0xbf, 0x2b, 0x42, 0x19, 0xf1, 0xc1, 0x62, 0x58, 0x35, 0xc3, 0xb0, 0x0d, 0x68, 0xe6, 0x9d, + 0xdb, 0x4c, 0x67, 0x9e, 0xf4, 0xf9, 0x50, 0x0e, 0xbf, 0x95, 0x47, 0xb9, 0x77, 0xa1, 0xd9, 0x1d, + 0x25, 0x5a, 0x0d, 0x08, 0xe2, 0x5b, 0x25, 0x42, 0x92, 0x9b, 0x33, 0x55, 0x86, 0x63, 0x11, 0x8e, + 0xa4, 0x97, 0x67, 0xe5, 0x0f, 0xa1, 0x7a, 0x62, 0x26, 0xc6, 0xd4, 0x19, 0xbe, 0x70, 0xc9, 0x2e, + 0x60, 0x07, 0xdf, 0x32, 0x63, 0xbf, 0x82, 0x19, 0xa7, 0xca, 0x93, 0xdc, 0x2f, 0xd9, 0xd5, 0x52, + 0x83, 0xd2, 0x56, 0xd2, 0xb5, 0x59, 0xaa, 0x4c, 0xba, 0x26, 0x04, 0xde, 0x21, 0x13, 0x58, 0xd1, + 0xf9, 0x97, 0x1a, 0x54, 0x0d, 0x2a, 0xda, 0xb1, 0x6b, 0x64, 0x63, 0xf7, 0x1d, 0xa8, 0xab, 0xa1, + 0x8c, 0x85, 0x56, 0xb1, 0x4d, 0x95, 0x1f, 0x5e, 0x07, 0x65, 0x37, 0x0f, 0xac, 0xb0, 0x97, 0xa9, + 0x99, 0x9e, 0x8e, 0xe2, 0xec, 0x74, 0xdc, 0x01, 0x96, 0x02, 0xea, 0x61, 0x8c, 0x72, 0xfa, 0xc2, + 0x26, 0x3e, 0x33, 0x74, 0x7e, 0x04, 0x8d, 0xae, 0x8a, 0xfc, 0x20, 0x4b, 0x9b, 0x57, 0xef, 0x7d, + 0xfd, 0x5a, 0x16, 0xee, 0xa4, 0xd2, 0xde, 0x58, 0x11, 0x7f, 0x13, 0x2a, 0x67, 0x38, 0x4f, 0x34, + 0x21, 0x97, 0xcf, 0xa2, 0x61, 0xe2, 0x9f, 0x40, 0xf3, 0xfb, 0xa3, 0xa0, 0x7b, 0x7a, 0x90, 0x2f, + 0xcb, 0xbc, 0x7b, 0x2d, 0x2b, 0xbe, 0x33, 0x96, 0xf7, 0xf2, 0xca, 0x72, 0xbe, 0x51, 0xfb, 0x15, + 0x7c, 0xa3, 0x3e, 0xeb, 0x1b, 0xaf, 0x42, 0x3d, 0x9d, 0x1c, 0xf2, 0x8f, 0xc8, 0x67, 0x4b, 0xbc, + 0x0a, 0xc5, 0x83, 0x98, 0x15, 0xdc, 0xff, 0x2e, 0x40, 0x23, 0x1b, 0x98, 0xc9, 0x12, 0xcc, 0xa3, + 0xef, 0x8f, 0x44, 0xc8, 0x0a, 0x94, 0x43, 0x28, 0x6d, 0x5a, 0xb4, 0x78, 0xbf, 0x1d, 0x4b, 0xa1, + 0xa9, 0xf2, 0x87, 0x88, 0x2c, 0x93, 0x84, 0x95, 0x39, 0x87, 0x55, 0x4b, 0x3e, 0x88, 0x0d, 0x6b, + 0x05, 0x53, 0x0c, 0x7c, 0x9b, 0x12, 0xaa, 0x06, 0xc0, 0x4f, 0xa5, 0x49, 0xa1, 0x3e, 0x52, 0x9a, + 0x1a, 0x75, 0xb4, 0x65, 0x3f, 0x62, 0x0d, 0xfc, 0xe6, 0x47, 0x4a, 0xef, 0x47, 0x0c, 0xc6, 0xb1, + 0x6d, 0x33, 0xfd, 0x3c, 0xb5, 0x96, 0x29, 0x72, 0x0e, 0xc3, 0xfd, 0x88, 0xad, 0xd8, 0x17, 0xa6, + 0xb5, 0x8a, 0x1a, 0x1f, 0x9d, 0x8b, 0x2e, 0x8a, 0xdf, 0xe0, 0xab, 0x00, 0x28, 0x63, 0xdb, 0x0c, + 0xd7, 0xc0, 0xa3, 0xf3, 0x20, 0xd1, 0x09, 0x5b, 0x73, 0xff, 0xb9, 0x00, 0xcd, 0xdc, 0x24, 0x60, + 0xec, 0x4c, 0x8c, 0x08, 0x6d, 0x26, 0x94, 0xfe, 0xae, 0x4c, 0xb4, 0x8c, 0xfd, 0x14, 0xb6, 0x8e, + 0x14, 0x3e, 0x16, 0xf1, 0x7b, 0x47, 0x6a, 0xa0, 0xe2, 0x58, 0x3d, 0x67, 0x25, 0x6c, 0x3d, 0x11, + 0x89, 0x7e, 0x26, 0xe5, 0x29, 0x2b, 0x63, 0x57, 0x77, 0x46, 0x71, 0x2c, 0x23, 0x43, 0xa8, 0x90, + 0x71, 0xf2, 0xdc, 0xb4, 0xaa, 0xa8, 0x14, 0x99, 0x09, 0x17, 0x59, 0x8d, 0x33, 0x58, 0xb6, 0xdc, + 0x86, 0x52, 0x47, 0x06, 0x64, 0x37, 0xcd, 0x06, 0xa6, 0x9d, 0x26, 0x6d, 0x3b, 0x38, 0xd9, 0x15, + 0x17, 0xc9, 0x56, 0x4f, 0x31, 0x98, 0x26, 0x7e, 0xa4, 0x9e, 0xb3, 0xa6, 0x33, 0x02, 0x18, 0x07, + 0xb4, 0x18, 0xc8, 0xa3, 0xaf, 0x65, 0x85, 0x55, 0xdb, 0xe2, 0x07, 0x00, 0xf8, 0x44, 0x9c, 0x69, + 0x34, 0x7f, 0x8d, 0x28, 0x83, 0xe4, 0xbc, 0x9c, 0x0a, 0xe7, 0xf7, 0xa1, 0x91, 0xbd, 0xc0, 0xbc, + 0x8c, 0xe2, 0x81, 0xec, 0xb3, 0x69, 0x13, 0xf7, 0xc9, 0x20, 0xf2, 0xe5, 0x39, 0xad, 0xfd, 0x8a, + 0x67, 0x1a, 0x68, 0x65, 0x3f, 0xf0, 0x7d, 0x19, 0xa5, 0xe5, 0x6f, 0xd3, 0x9a, 0x77, 0xd6, 0x58, + 0x9e, 0x7b, 0xd6, 0xe8, 0xfc, 0x2e, 0x34, 0x73, 0x11, 0xf7, 0xa5, 0xdd, 0xce, 0x19, 0x56, 0x9c, + 0x34, 0xec, 0x35, 0x68, 0x28, 0x1b, 0x36, 0x27, 0x04, 0xe0, 0x0d, 0x6f, 0x4c, 0xc0, 0x0d, 0xa6, + 0x62, 0xba, 0x36, 0x1d, 0x25, 0x3f, 0x86, 0x2a, 0xa6, 0x8c, 0xa3, 0xf4, 0xa0, 0x76, 0xc1, 0x48, + 0xb4, 0x4d, 0x32, 0x7b, 0x4b, 0x9e, 0x95, 0xe6, 0xef, 0x41, 0x49, 0x8b, 0x9e, 0xad, 0x1e, 0x7d, + 0x65, 0x31, 0x25, 0x47, 0xa2, 0xb7, 0xb7, 0xe4, 0xa1, 0x1c, 0x7f, 0x02, 0xf5, 0xae, 0x4d, 0xf8, + 0x2d, 0x70, 0x2d, 0x18, 0xc8, 0xa6, 0x65, 0x82, 0xbd, 0x25, 0x2f, 0xd3, 0xc0, 0xbf, 0x05, 0x65, + 0xdc, 0xe5, 0x09, 0x79, 0x17, 0x0e, 0xd0, 0x71, 0xb9, 0xec, 0x2d, 0x79, 0x24, 0xb9, 0x5d, 0x83, + 0x0a, 0xe1, 0xa4, 0xd3, 0x82, 0xaa, 0xe9, 0xeb, 0xf4, 0xc8, 0x39, 0xaf, 0x40, 0xe9, 0x48, 0xf4, + 0x30, 0xd2, 0x0a, 0xfc, 0xc4, 0xe6, 0x99, 0xf8, 0xe8, 0xbc, 0x3e, 0x2e, 0x5e, 0xe4, 0xeb, 0x62, + 0x85, 0x89, 0xba, 0x98, 0x53, 0x85, 0x32, 0x7e, 0xd1, 0x79, 0xed, 0xaa, 0xa8, 0xcd, 0x79, 0x15, + 0xe3, 0x3b, 0x2d, 0xcf, 0xe7, 0x95, 0xfc, 0x9c, 0x35, 0xb8, 0x31, 0x75, 0xc2, 0xe5, 0xd4, 0x6c, + 0x70, 0xe9, 0xac, 0x40, 0x33, 0x77, 0x66, 0xe1, 0xbc, 0x01, 0xf5, 0xf4, 0x44, 0x03, 0x43, 0xea, + 0x20, 0x31, 0xb5, 0x18, 0x6b, 0x54, 0xd6, 0x76, 0x7e, 0x5a, 0x80, 0xaa, 0x39, 0x15, 0xe2, 0xdb, + 0xd9, 0x29, 0x6e, 0x61, 0x81, 0x23, 0x04, 0x23, 0x64, 0x0f, 0x60, 0xb2, 0xa3, 0xdc, 0x75, 0xa8, + 0x84, 0x14, 0x3b, 0xdb, 0xe5, 0x42, 0x8d, 0x9c, 0x77, 0x97, 0xf2, 0xde, 0xed, 0xbe, 0x93, 0x1d, + 0xfa, 0xa4, 0x75, 0x02, 0xda, 0xf6, 0x8f, 0x62, 0x29, 0x4d, 0x0d, 0x80, 0x82, 0xe5, 0x22, 0x61, + 0x93, 0x1a, 0x0c, 0x45, 0x57, 0x13, 0xa1, 0xe4, 0x9e, 0x40, 0xfd, 0x50, 0x25, 0xd3, 0x88, 0x5f, + 0x83, 0xd2, 0x91, 0x1a, 0x9a, 0x80, 0x61, 0x5b, 0x69, 0x0a, 0x18, 0x0c, 0xc0, 0x9f, 0x68, 0x53, + 0xb2, 0xf0, 0x82, 0x5e, 0x5f, 0x9b, 0x72, 0xd4, 0x7e, 0x14, 0xc9, 0x98, 0x55, 0x10, 0x75, 0x3d, + 0x39, 0x0c, 0x45, 0x57, 0xb2, 0x2a, 0xa2, 0x2e, 0xd1, 0x1f, 0x07, 0x71, 0xa2, 0x59, 0xcd, 0x7d, + 0x07, 0xb1, 0x3a, 0xe8, 0x11, 0xc4, 0xd2, 0x03, 0xa9, 0x5a, 0x42, 0x83, 0xa8, 0xb9, 0x23, 0x23, + 0xdc, 0x46, 0xe8, 0x54, 0xc1, 0x1c, 0xeb, 0xd3, 0x07, 0x8a, 0xee, 0x33, 0x58, 0x99, 0x38, 0xee, + 0xe7, 0xeb, 0xc0, 0x26, 0x08, 0x68, 0xe8, 0x12, 0x7f, 0x05, 0x5e, 0x9a, 0xa0, 0x3e, 0x0d, 0x7c, + 0x9f, 0x8a, 0x2e, 0xd3, 0x2f, 0xd2, 0xee, 0x6c, 0x37, 0xa0, 0xd6, 0x35, 0x33, 0xe0, 0x1e, 0xc2, + 0x0a, 0x4d, 0xc9, 0x53, 0xa9, 0xc5, 0x41, 0x14, 0x5e, 0xfc, 0xca, 0x77, 0x32, 0xdc, 0xaf, 0x42, + 0x85, 0x8a, 0x9f, 0xe8, 0x7c, 0x27, 0xb1, 0x1a, 0x90, 0xae, 0x8a, 0x47, 0xcf, 0xa8, 0x5d, 0x2b, + 0x3b, 0xaf, 0x45, 0xad, 0xdc, 0x9f, 0x36, 0xa0, 0xb6, 0xd5, 0xed, 0xaa, 0x51, 0xa4, 0x67, 0xbe, + 0x3c, 0xaf, 0xbe, 0xf6, 0x10, 0xaa, 0xe2, 0x4c, 0x68, 0x11, 0x5b, 0xcc, 0x98, 0x8e, 0x0e, 0xac, + 0xae, 0xcd, 0x2d, 0x62, 0xf2, 0x2c, 0x33, 0x8a, 0x75, 0x55, 0x74, 0x12, 0xf4, 0x2c, 0x4c, 0x5c, + 0x26, 0xb6, 0x43, 0x4c, 0x9e, 0x65, 0x46, 0x31, 0x0b, 0x73, 0x95, 0x2b, 0xc5, 0xcc, 0x5a, 0xcf, + 0x50, 0xed, 0x2e, 0x94, 0x83, 0xe8, 0x44, 0xd9, 0xdb, 0x38, 0xaf, 0x5e, 0x22, 0xb4, 0x1f, 0x9d, + 0x28, 0x8f, 0x18, 0x1d, 0x09, 0x55, 0x63, 0x30, 0xff, 0x06, 0x54, 0xe8, 0x8c, 0xc3, 0x56, 0x95, + 0x17, 0xba, 0x3e, 0x61, 0x24, 0xf8, 0xcd, 0xb4, 0x64, 0x4e, 0xe3, 0x85, 0x74, 0x6a, 0x6e, 0xd7, + 0xd3, 0x21, 0x73, 0xfe, 0xa3, 0x00, 0x55, 0xd3, 0x43, 0xfe, 0x06, 0xac, 0xca, 0x08, 0x97, 0x76, + 0x0a, 0x64, 0x76, 0x4d, 0x4f, 0x51, 0x31, 0xac, 0xb2, 0x14, 0xd9, 0x19, 0xf5, 0x6c, 0x06, 0x98, + 0x27, 0xf1, 0x77, 0xe1, 0x15, 0xd3, 0x3c, 0x8c, 0x65, 0x2c, 0x43, 0x29, 0x12, 0xb9, 0xd3, 0x17, + 0x51, 0x24, 0x43, 0xbb, 0xad, 0x5d, 0xf6, 0x9a, 0xbb, 0xb0, 0x6c, 0x5e, 0xb5, 0x87, 0xa2, 0x2b, + 0x13, 0x7b, 0x04, 0x30, 0x41, 0xe3, 0x5f, 0x83, 0x0a, 0xdd, 0x89, 0x6a, 0xf9, 0x57, 0x3b, 0x9f, + 0xe1, 0x72, 0x54, 0x86, 0xbb, 0x5b, 0x00, 0x66, 0x36, 0x30, 0x1f, 0xb0, 0x58, 0xf4, 0xc5, 0x2b, + 0xa7, 0x8f, 0x32, 0x9b, 0x9c, 0x10, 0xda, 0xe7, 0xcb, 0x50, 0x22, 0x3e, 0x20, 0xe6, 0x52, 0xe7, + 0x4b, 0xde, 0x04, 0xcd, 0xf9, 0x9b, 0x12, 0x94, 0x71, 0x22, 0x91, 0xb9, 0xaf, 0x06, 0x32, 0x2b, + 0x4d, 0x19, 0xa7, 0x9d, 0xa0, 0xe1, 0xc6, 0x2e, 0xcc, 0xa9, 0x5f, 0xc6, 0x66, 0xa0, 0x6c, 0x9a, + 0x8c, 0x9c, 0xc3, 0x58, 0x9d, 0x04, 0xe1, 0x98, 0xd3, 0x86, 0x00, 0x53, 0x64, 0xfe, 0x75, 0xb8, + 0x39, 0x10, 0xf1, 0xa9, 0xd4, 0x84, 0x3e, 0xcf, 0x54, 0x7c, 0x9a, 0xe0, 0xc8, 0xed, 0xfb, 0xb6, + 0xa6, 0x71, 0xc9, 0x5b, 0x84, 0x73, 0x5f, 0x9e, 0x05, 0xc4, 0x59, 0x27, 0xce, 0xac, 0x8d, 0xce, + 0x21, 0xcc, 0xd0, 0xb4, 0xad, 0x2e, 0x93, 0x1f, 0x4d, 0x51, 0x31, 0x7a, 0x30, 0x37, 0x00, 0x92, + 0x7d, 0x9f, 0xca, 0x2c, 0x0d, 0x6f, 0x4c, 0xe0, 0xb7, 0x00, 0x7a, 0x42, 0xcb, 0xe7, 0xe2, 0xe2, + 0xe3, 0x38, 0x6c, 0x49, 0x53, 0xbc, 0x1c, 0x53, 0x30, 0xe9, 0x09, 0x55, 0x57, 0x84, 0x6d, 0xad, + 0x62, 0xd1, 0x93, 0x87, 0x42, 0xf7, 0x5b, 0x3d, 0x93, 0xf4, 0x4c, 0xd3, 0xd1, 0x5a, 0xcc, 0xed, + 0x3f, 0x51, 0x91, 0x6c, 0xf5, 0x8d, 0xb5, 0x69, 0x1b, 0x5d, 0x54, 0x44, 0x22, 0xbc, 0xd0, 0x41, + 0x17, 0xed, 0x08, 0x4c, 0x7a, 0x95, 0x23, 0xb9, 0x07, 0x00, 0xe3, 0x29, 0x46, 0x5c, 0xdf, 0xa2, + 0x12, 0x2a, 0x5b, 0xc2, 0x58, 0xf2, 0x50, 0x46, 0x7e, 0x10, 0xf5, 0x76, 0xed, 0xac, 0xb2, 0x02, + 0x12, 0xdb, 0x5a, 0xc4, 0x5a, 0xfa, 0x19, 0x91, 0xe2, 0x7d, 0x6a, 0x49, 0x9f, 0x95, 0xdc, 0xff, + 0x2d, 0x40, 0x33, 0x77, 0x80, 0xf8, 0x6b, 0x3c, 0xf4, 0xc4, 0x5d, 0x16, 0x57, 0x33, 0x0e, 0x99, + 0x99, 0xf1, 0xac, 0x8d, 0x03, 0x6a, 0xcf, 0x37, 0xf1, 0xad, 0xc9, 0x0f, 0x73, 0x94, 0xcf, 0x75, + 0xe0, 0xe9, 0xde, 0xb3, 0x19, 0x73, 0x13, 0x6a, 0x1f, 0x47, 0xa7, 0x91, 0x7a, 0x1e, 0x99, 0xed, + 0x93, 0x4e, 0xb1, 0x27, 0xea, 0xf6, 0xe9, 0x41, 0x73, 0xc9, 0xfd, 0x8b, 0xf2, 0xd4, 0x85, 0x8f, + 0x47, 0x50, 0x35, 0x51, 0x23, 0x05, 0x34, 0xb3, 0x27, 0xf4, 0x79, 0x66, 0x5b, 0x23, 0xce, 0x91, + 0x3c, 0x2b, 0x8c, 0xe1, 0x5c, 0x76, 0xab, 0xa9, 0x38, 0xb7, 0x96, 0x3d, 0xa1, 0x28, 0x05, 0xa9, + 0x89, 0x8b, 0x7d, 0x99, 0x06, 0xe7, 0x4f, 0x0a, 0xb0, 0x3e, 0x8f, 0x05, 0xa3, 0xab, 0xce, 0xc4, + 0xbd, 0x8b, 0xb4, 0xc9, 0xdb, 0x53, 0xd7, 0x09, 0x8b, 0xd4, 0x9b, 0xbb, 0xd7, 0x34, 0x62, 0xf2, + 0x72, 0xa1, 0xfb, 0xe3, 0x02, 0xac, 0xcd, 0xf4, 0x39, 0x17, 0x70, 0x00, 0x54, 0x8d, 0x67, 0x99, + 0x6b, 0x02, 0xd9, 0xc1, 0xad, 0x29, 0xe9, 0x11, 0xe2, 0x27, 0xe6, 0x24, 0x6c, 0xd7, 0x5c, 0x46, + 0x65, 0x65, 0x8c, 0x14, 0x70, 0xd6, 0x10, 0x49, 0x7b, 0x92, 0x55, 0x30, 0x93, 0x32, 0x31, 0x90, + 0xa5, 0x54, 0x29, 0x4b, 0xb3, 0x55, 0x44, 0x56, 0xa3, 0xeb, 0x07, 0xa3, 0x61, 0x18, 0x74, 0xb1, + 0x59, 0x77, 0x3d, 0x78, 0x69, 0x8e, 0xdd, 0x64, 0xc9, 0xb1, 0xb5, 0x6a, 0x15, 0x60, 0xf7, 0x38, + 0xb5, 0x85, 0x15, 0x30, 0xb1, 0xdd, 0x3d, 0xde, 0xa1, 0xd4, 0xd6, 0x1e, 0xee, 0x99, 0x35, 0x71, + 0x8c, 0xf9, 0x4f, 0xc2, 0x4a, 0xee, 0xf7, 0xd2, 0x53, 0x3f, 0xe7, 0x18, 0x56, 0x8c, 0x19, 0x87, + 0xe2, 0x22, 0x54, 0xc2, 0xe7, 0x8f, 0x60, 0x35, 0xc9, 0xee, 0xed, 0xe6, 0xf0, 0x78, 0x7a, 0x3b, + 0x6d, 0x4f, 0x30, 0x79, 0x53, 0x42, 0xee, 0x9f, 0x57, 0x00, 0x0e, 0xb2, 0xbb, 0xaf, 0x73, 0x16, + 0xdd, 0xbc, 0x80, 0x61, 0xe6, 0xdc, 0xa1, 0x74, 0xed, 0x73, 0x87, 0x77, 0xb3, 0x90, 0xd6, 0x54, + 0xab, 0xa6, 0x2f, 0x17, 0x8e, 0x6d, 0x9a, 0x0e, 0x64, 0x27, 0xce, 0xab, 0x2b, 0xd3, 0xe7, 0xd5, + 0x1b, 0xb3, 0x97, 0x5b, 0xa6, 0xd0, 0x60, 0x9c, 0x21, 0xd6, 0x26, 0x32, 0x44, 0x07, 0xea, 0xb1, + 0x14, 0xbe, 0x8a, 0xc2, 0x8b, 0xb4, 0xbc, 0x9d, 0xb6, 0xf9, 0x7d, 0xa8, 0x68, 0xba, 0x2d, 0x5c, + 0x27, 0xe7, 0x7d, 0xc1, 0x18, 0x1b, 0x5e, 0x84, 0x96, 0x20, 0xb1, 0x37, 0x52, 0x0c, 0xda, 0xd7, + 0xbd, 0x1c, 0x85, 0x6f, 0x02, 0x0f, 0xa2, 0x44, 0x8b, 0x30, 0x94, 0xfe, 0xf6, 0xc5, 0xae, 0xa9, + 0x52, 0xd3, 0x0e, 0x53, 0xf7, 0xe6, 0xbc, 0x71, 0x3f, 0x1b, 0xdf, 0xc4, 0x6a, 0x40, 0xa5, 0x23, + 0x92, 0xa0, 0x6b, 0xce, 0x7c, 0xed, 0xf6, 0x65, 0x02, 0x73, 0xad, 0x7c, 0xc5, 0x8a, 0x18, 0x71, + 0x27, 0x12, 0x63, 0xeb, 0x55, 0x80, 0xf1, 0xdd, 0x66, 0x56, 0x46, 0x1f, 0x4e, 0x67, 0xc2, 0x1c, + 0xf9, 0x92, 0x28, 0x95, 0x11, 0xfc, 0xec, 0x32, 0x4d, 0x0d, 0xbf, 0x40, 0x18, 0xc9, 0xea, 0xc8, + 0x13, 0x29, 0x2d, 0x4d, 0x11, 0x85, 0xb6, 0x3a, 0x06, 0xa8, 0x26, 0xbd, 0xaa, 0xc9, 0x9a, 0x18, + 0x14, 0xa7, 0x4a, 0x4d, 0xe5, 0x23, 0xa1, 0x74, 0x60, 0x19, 0x3d, 0x7c, 0xf2, 0x05, 0x5b, 0x41, + 0x8b, 0xc6, 0x57, 0xa6, 0xd9, 0x2a, 0xaa, 0x42, 0x7c, 0xe9, 0x88, 0x44, 0xb2, 0x75, 0xf7, 0x2f, + 0xc7, 0xbd, 0x7c, 0x2b, 0x8b, 0x5d, 0x17, 0xf1, 0x8f, 0xcb, 0xa2, 0xdb, 0x47, 0xb0, 0x16, 0xcb, + 0xef, 0x8f, 0x82, 0x89, 0xcb, 0x94, 0xa5, 0xab, 0x8f, 0x0b, 0x67, 0x25, 0xdc, 0x33, 0x58, 0x4b, + 0x1b, 0xcf, 0x02, 0xdd, 0xa7, 0x94, 0x94, 0xdf, 0xcf, 0xdd, 0xf6, 0x2c, 0xd8, 0x60, 0xea, 0x12, + 0x95, 0xe3, 0xdb, 0x9d, 0x59, 0x59, 0xb0, 0xb8, 0x40, 0x59, 0xd0, 0xfd, 0xf7, 0x6a, 0x2e, 0x2b, + 0x35, 0xd1, 0xbc, 0x9f, 0x45, 0xf3, 0xb3, 0x67, 0x0b, 0xe3, 0x4a, 0x5f, 0xf1, 0x3a, 0x95, 0xbe, + 0x79, 0x87, 0x6b, 0xdf, 0xc4, 0x50, 0x8d, 0x5c, 0xef, 0x78, 0x81, 0x2a, 0xe6, 0x04, 0x2f, 0xdf, + 0xa6, 0x93, 0x02, 0xd1, 0x36, 0x27, 0xbf, 0x95, 0xb9, 0x77, 0xaf, 0xf3, 0x47, 0x02, 0x96, 0xd3, + 0xcb, 0x49, 0xe5, 0x16, 0x6a, 0x75, 0xde, 0x42, 0xc5, 0xc4, 0xca, 0x2e, 0xe1, 0xac, 0x6d, 0x8a, + 0xbe, 0xe6, 0x39, 0x55, 0x4f, 0x97, 0xa6, 0xeb, 0xde, 0x0c, 0x1d, 0xc3, 0x89, 0xc1, 0x28, 0xd4, + 0x81, 0xad, 0x6b, 0x9a, 0xc6, 0xf4, 0xdf, 0x03, 0x1a, 0xb3, 0x7f, 0x0f, 0x78, 0x1f, 0x20, 0x91, + 0xe8, 0xbe, 0xbb, 0x41, 0x57, 0xdb, 0xf3, 0xe1, 0x5b, 0x97, 0xf5, 0xcd, 0x56, 0x63, 0x73, 0x12, + 0x68, 0xff, 0x40, 0x9c, 0xef, 0x60, 0xd0, 0x67, 0x0f, 0xb2, 0xb2, 0xf6, 0x34, 0x7c, 0xad, 0xce, + 0xc2, 0xd7, 0x7d, 0xa8, 0x24, 0x5d, 0x35, 0x94, 0x74, 0xbf, 0xf9, 0xf2, 0xf9, 0xdd, 0x6c, 0x23, + 0x93, 0x67, 0x78, 0xa9, 0xf6, 0x81, 0xdb, 0x8c, 0x8a, 0xe9, 0x66, 0x73, 0xc3, 0x4b, 0x9b, 0x8e, + 0x0f, 0x55, 0x5b, 0xab, 0x9c, 0x93, 0x29, 0x52, 0x99, 0xa3, 0x98, 0xbb, 0xd9, 0x94, 0xdd, 0x20, + 0x2a, 0xe5, 0x6f, 0x10, 0x6d, 0x40, 0x33, 0xce, 0xd5, 0xe2, 0xed, 0xb5, 0xb1, 0x1c, 0xc9, 0xfd, + 0x04, 0x2a, 0x64, 0x0f, 0xee, 0x86, 0x66, 0x28, 0x4d, 0x40, 0x84, 0x86, 0xb3, 0x02, 0xa6, 0xe0, + 0x89, 0xd4, 0x07, 0x27, 0x47, 0x7d, 0xd9, 0x16, 0x03, 0x49, 0x48, 0x55, 0xe4, 0x2d, 0x58, 0x37, + 0xbc, 0xc9, 0xe4, 0x1b, 0xda, 0xb6, 0xc3, 0xa0, 0x13, 0x8b, 0xf8, 0x82, 0x95, 0xdd, 0xf7, 0xe9, + 0xe4, 0x28, 0x75, 0x9a, 0x66, 0xf6, 0x37, 0x14, 0x83, 0x8d, 0xbe, 0x8c, 0x11, 0x6c, 0xcd, 0xb9, + 0x9e, 0x0d, 0xb5, 0xcd, 0xdd, 0x05, 0x8a, 0x87, 0x59, 0xc9, 0x7d, 0x86, 0x71, 0xd7, 0x78, 0x6b, + 0xfa, 0xb5, 0xad, 0x29, 0x77, 0x3b, 0x17, 0x77, 0x4c, 0x5e, 0x56, 0x28, 0x2c, 0x7a, 0x59, 0xc1, + 0xfd, 0x10, 0x6e, 0x78, 0x93, 0xc0, 0xca, 0xdf, 0x85, 0x9a, 0x1a, 0xe6, 0xf5, 0xbc, 0xc8, 0xf7, + 0x52, 0x76, 0xf7, 0x67, 0x05, 0x58, 0xde, 0x8f, 0xb4, 0x8c, 0x23, 0x11, 0x3e, 0x0e, 0x45, 0x8f, + 0xbf, 0x93, 0x22, 0xd1, 0xfc, 0x54, 0x2e, 0xcf, 0x3b, 0x09, 0x4a, 0xa1, 0xad, 0xc9, 0xf1, 0x97, + 0x61, 0x4d, 0xfa, 0x81, 0x56, 0xb1, 0x89, 0xb6, 0xd2, 0x3b, 0x23, 0xeb, 0xc0, 0x0c, 0xb9, 0x4d, + 0x6e, 0x7f, 0x64, 0xa6, 0xb9, 0x05, 0xeb, 0x13, 0xd4, 0x34, 0x94, 0x2a, 0xf2, 0xd7, 0xa0, 0x35, + 0xde, 0x12, 0x76, 0x55, 0xa4, 0xf7, 0x23, 0x5f, 0x9e, 0x53, 0xa4, 0xc0, 0x4a, 0xee, 0xbf, 0x65, + 0x31, 0xca, 0xb1, 0xbd, 0x51, 0x12, 0x2b, 0xa5, 0xc7, 0x15, 0x59, 0xd3, 0xca, 0xfd, 0x5f, 0xa9, + 0xb8, 0xc0, 0xff, 0x95, 0xde, 0x1f, 0xff, 0x5f, 0xc9, 0x6c, 0x06, 0xaf, 0xcf, 0xdd, 0x61, 0xe8, + 0x20, 0xdc, 0xc6, 0x88, 0x6d, 0x99, 0xfb, 0xf3, 0xd2, 0xdb, 0x36, 0x31, 0x28, 0x2f, 0x12, 0x75, + 0x99, 0xb3, 0xbd, 0x87, 0xd3, 0xf7, 0x64, 0x17, 0xbb, 0xb0, 0x32, 0x13, 0x6d, 0xc1, 0xb5, 0xa3, + 0xad, 0x0f, 0xa6, 0x62, 0xf0, 0xfa, 0xdc, 0x22, 0xca, 0x15, 0x7f, 0xe6, 0xf9, 0x00, 0x6a, 0xfd, + 0x20, 0xd1, 0x2a, 0xbe, 0xa0, 0x40, 0x66, 0xf6, 0x42, 0x7c, 0x6e, 0xb4, 0xf6, 0x0c, 0x23, 0xdd, + 0x1e, 0x48, 0xa5, 0x9c, 0x1e, 0xc0, 0x78, 0x14, 0x67, 0xb0, 0xe6, 0x73, 0xfc, 0x79, 0xec, 0x26, + 0x54, 0x93, 0x51, 0x67, 0x5c, 0x62, 0xb7, 0x2d, 0xe7, 0x1c, 0x9c, 0x99, 0x7d, 0xfa, 0x50, 0xc6, + 0xc6, 0x3e, 0xc4, 0xde, 0xb4, 0x14, 0x6f, 0x3f, 0x9f, 0xb5, 0xf9, 0xfb, 0xf9, 0xe9, 0x31, 0x2e, + 0xb4, 0x71, 0xc9, 0x18, 0x67, 0x9a, 0x73, 0xf3, 0xe4, 0x3c, 0x84, 0x66, 0xae, 0xeb, 0x88, 0x9f, + 0xa3, 0xc8, 0x57, 0x69, 0xa5, 0x0e, 0x9f, 0xcd, 0x25, 0x7e, 0x3f, 0xad, 0xd5, 0xd1, 0xf3, 0x9d, + 0x1f, 0x17, 0x61, 0x75, 0xd2, 0x5d, 0xa8, 0x66, 0x69, 0xa0, 0xea, 0x20, 0xf4, 0x73, 0xa9, 0x23, + 0xe3, 0x37, 0xa0, 0x79, 0x68, 0xa2, 0x3d, 0x22, 0xac, 0xe1, 0xab, 0x3d, 0x35, 0x90, 0x6c, 0x23, + 0x7f, 0xfd, 0xf9, 0x2d, 0xc4, 0x59, 0x53, 0x06, 0x66, 0x43, 0xde, 0xb0, 0x17, 0xc6, 0x7e, 0x58, + 0xe4, 0x2b, 0xb9, 0x04, 0xe6, 0x27, 0x45, 0xbe, 0x0e, 0x37, 0xb6, 0x47, 0x91, 0x1f, 0x4a, 0x3f, + 0xa3, 0xfe, 0x75, 0x9e, 0x9a, 0xa5, 0x2a, 0x3f, 0xc4, 0xec, 0xa8, 0xd1, 0x1e, 0x75, 0x6c, 0x9a, + 0xf2, 0x07, 0x65, 0x7e, 0x13, 0xd6, 0x2c, 0xd7, 0x38, 0x14, 0x63, 0x7f, 0x58, 0xe6, 0x2f, 0xc1, + 0xea, 0x96, 0x19, 0x33, 0x6b, 0x28, 0xfb, 0xa3, 0x32, 0x9a, 0x40, 0x47, 0x60, 0x7f, 0x4c, 0x7a, + 0xb2, 0x92, 0x09, 0xfb, 0x51, 0x99, 0x73, 0x58, 0x79, 0x1a, 0x24, 0x49, 0x10, 0xf5, 0xac, 0xee, + 0x3f, 0x2d, 0xdf, 0xf9, 0x59, 0x01, 0x56, 0x27, 0x41, 0x15, 0x83, 0xc4, 0x50, 0x45, 0x3d, 0x6d, + 0x6e, 0x65, 0xaf, 0x40, 0x23, 0xe9, 0xab, 0x58, 0x53, 0x93, 0xaa, 0xca, 0x11, 0x9d, 0x5e, 0x99, + 0xf4, 0xce, 0x94, 0x9b, 0xcc, 0xf9, 0xbf, 0x16, 0x3d, 0xd6, 0xc4, 0x51, 0xf2, 0xf1, 0xfb, 0xe5, + 0x2c, 0xe0, 0xa5, 0x53, 0xb4, 0xf4, 0x94, 0x82, 0x55, 0x91, 0x75, 0x14, 0x87, 0x26, 0xf0, 0x95, + 0x03, 0x11, 0x84, 0xe6, 0xfa, 0xe5, 0xb0, 0x8f, 0x99, 0x5b, 0xc3, 0x50, 0xd5, 0xa7, 0x81, 0xb9, + 0xe8, 0x68, 0xb7, 0x30, 0x1f, 0xed, 0xc8, 0xe6, 0x9f, 0xc9, 0xed, 0x3b, 0xff, 0xf4, 0xcb, 0x5b, + 0x85, 0x5f, 0xfc, 0xf2, 0x56, 0xe1, 0x3f, 0x7f, 0x79, 0xab, 0xf0, 0xe3, 0xcf, 0x6e, 0x2d, 0xfd, + 0xe2, 0xb3, 0x5b, 0x4b, 0xff, 0xfa, 0xd9, 0xad, 0xa5, 0x4f, 0xd8, 0xf4, 0x3f, 0x37, 0x3b, 0x55, + 0xf2, 0xec, 0xfb, 0xff, 0x17, 0x00, 0x00, 0xff, 0xff, 0x02, 0x3a, 0x1b, 0xb1, 0xd4, 0x39, 0x00, + 0x00, } func (m *SmartBlockSnapshotBase) Marshal() (dAtA []byte, err error) { @@ -7356,6 +7366,13 @@ func (m *BlockContentDataviewView) MarshalToSizedBuffer(dAtA []byte) (int, error _ = i var l int _ = l + if len(m.DefaultTemplateId) > 0 { + i -= len(m.DefaultTemplateId) + copy(dAtA[i:], m.DefaultTemplateId) + i = encodeVarintModels(dAtA, i, uint64(len(m.DefaultTemplateId))) + i-- + dAtA[i] = 0x72 + } if m.PageLimit != 0 { i = encodeVarintModels(dAtA, i, uint64(m.PageLimit)) i-- @@ -10445,6 +10462,10 @@ func (m *BlockContentDataviewView) Size() (n int) { if m.PageLimit != 0 { n += 1 + sovModels(uint64(m.PageLimit)) } + l = len(m.DefaultTemplateId) + if l > 0 { + n += 1 + l + sovModels(uint64(l)) + } return n } @@ -15080,6 +15101,38 @@ func (m *BlockContentDataviewView) Unmarshal(dAtA []byte) error { break } } + case 14: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DefaultTemplateId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowModels + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthModels + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthModels + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DefaultTemplateId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipModels(dAtA[iNdEx:]) diff --git a/pkg/lib/pb/model/protos/models.proto b/pkg/lib/pb/model/protos/models.proto index 4dd5354648..defb03aece 100644 --- a/pkg/lib/pb/model/protos/models.proto +++ b/pkg/lib/pb/model/protos/models.proto @@ -326,6 +326,7 @@ message Block { string groupRelationKey = 11; // Group view by this relationKey bool groupBackgroundColors = 12; // Enable backgrounds in groups int32 pageLimit = 13; + string defaultTemplateId = 14; enum Type { Table = 0; diff --git a/pkg/lib/pb/storage/file.pb.go b/pkg/lib/pb/storage/file.pb.go index 36444f9850..680c87e427 100644 --- a/pkg/lib/pb/storage/file.pb.go +++ b/pkg/lib/pb/storage/file.pb.go @@ -145,20 +145,21 @@ func (m *FileKeys) GetKeysByPath() map[string]string { } type FileInfo struct { - Mill string `protobuf:"bytes,1,opt,name=mill,proto3" json:"mill,omitempty"` - Checksum string `protobuf:"bytes,2,opt,name=checksum,proto3" json:"checksum,omitempty"` - Source string `protobuf:"bytes,3,opt,name=source,proto3" json:"source,omitempty"` - Opts string `protobuf:"bytes,4,opt,name=opts,proto3" json:"opts,omitempty"` - Hash string `protobuf:"bytes,5,opt,name=hash,proto3" json:"hash,omitempty"` - Key string `protobuf:"bytes,6,opt,name=key,proto3" json:"key,omitempty"` - Media string `protobuf:"bytes,7,opt,name=media,proto3" json:"media,omitempty"` - Name string `protobuf:"bytes,8,opt,name=name,proto3" json:"name,omitempty"` - Size_ int64 `protobuf:"varint,9,opt,name=size,proto3" json:"size,omitempty"` - Added int64 `protobuf:"varint,10,opt,name=added,proto3" json:"added,omitempty"` - Meta *types.Struct `protobuf:"bytes,11,opt,name=meta,proto3" json:"meta,omitempty"` - Targets []string `protobuf:"bytes,12,rep,name=targets,proto3" json:"targets,omitempty"` - EncMode FileInfoEncryptionMode `protobuf:"varint,13,opt,name=encMode,proto3,enum=anytype.storage.FileInfoEncryptionMode" json:"encMode,omitempty"` - MetaHash string `protobuf:"bytes,14,opt,name=metaHash,proto3" json:"metaHash,omitempty"` + Mill string `protobuf:"bytes,1,opt,name=mill,proto3" json:"mill,omitempty"` + Checksum string `protobuf:"bytes,2,opt,name=checksum,proto3" json:"checksum,omitempty"` + Source string `protobuf:"bytes,3,opt,name=source,proto3" json:"source,omitempty"` + Opts string `protobuf:"bytes,4,opt,name=opts,proto3" json:"opts,omitempty"` + Hash string `protobuf:"bytes,5,opt,name=hash,proto3" json:"hash,omitempty"` + Key string `protobuf:"bytes,6,opt,name=key,proto3" json:"key,omitempty"` + Media string `protobuf:"bytes,7,opt,name=media,proto3" json:"media,omitempty"` + Name string `protobuf:"bytes,8,opt,name=name,proto3" json:"name,omitempty"` + Size_ int64 `protobuf:"varint,9,opt,name=size,proto3" json:"size,omitempty"` + Added int64 `protobuf:"varint,10,opt,name=added,proto3" json:"added,omitempty"` + Meta *types.Struct `protobuf:"bytes,11,opt,name=meta,proto3" json:"meta,omitempty"` + Targets []string `protobuf:"bytes,12,rep,name=targets,proto3" json:"targets,omitempty"` + EncMode FileInfoEncryptionMode `protobuf:"varint,13,opt,name=encMode,proto3,enum=anytype.storage.FileInfoEncryptionMode" json:"encMode,omitempty"` + MetaHash string `protobuf:"bytes,14,opt,name=metaHash,proto3" json:"metaHash,omitempty"` + LastModifiedDate int64 `protobuf:"varint,15,opt,name=lastModifiedDate,proto3" json:"lastModifiedDate,omitempty"` } func (m *FileInfo) Reset() { *m = FileInfo{} } @@ -292,6 +293,13 @@ func (m *FileInfo) GetMetaHash() string { return "" } +func (m *FileInfo) GetLastModifiedDate() int64 { + if m != nil { + return m.LastModifiedDate + } + return 0 +} + type Directory struct { Files map[string]*FileInfo `protobuf:"bytes,1,rep,name=files,proto3" json:"files,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } @@ -577,52 +585,53 @@ func init() { } var fileDescriptor_c9351ff644be6424 = []byte{ - // 714 bytes of a gzipped FileDescriptorProto + // 736 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x55, 0x4d, 0x6f, 0xd3, 0x40, - 0x10, 0x8d, 0x6b, 0xa7, 0x49, 0x26, 0xb4, 0x8d, 0x56, 0x7c, 0x2c, 0x51, 0x15, 0xa2, 0x08, 0x24, - 0x97, 0x22, 0x1b, 0xa5, 0x12, 0x54, 0x20, 0x0e, 0x4d, 0x49, 0xa1, 0x6a, 0x4b, 0x91, 0x73, 0xe3, - 0x52, 0x39, 0xce, 0x36, 0x59, 0xe2, 0x2f, 0x79, 0x37, 0x08, 0x73, 0xe6, 0x0e, 0x07, 0xae, 0xfc, - 0x0a, 0xfe, 0x04, 0xc7, 0x1e, 0x39, 0xa2, 0xf6, 0x8f, 0xa0, 0x5d, 0x7f, 0x34, 0x6d, 0x52, 0x24, - 0x10, 0xe2, 0x36, 0x33, 0x79, 0xb3, 0xf3, 0x76, 0xde, 0xf3, 0x06, 0xee, 0x86, 0xe3, 0xa1, 0xe9, - 0xd2, 0xbe, 0x19, 0xf6, 0x4d, 0xc6, 0x83, 0xc8, 0x1e, 0x12, 0x33, 0x8c, 0x02, 0x1e, 0x30, 0xf3, - 0x98, 0xba, 0xc4, 0x90, 0x31, 0x5a, 0xb1, 0xfd, 0x98, 0xc7, 0x21, 0x31, 0x52, 0x48, 0x7d, 0x75, - 0x18, 0x04, 0x43, 0x37, 0x85, 0xf6, 0x27, 0xc7, 0x26, 0xe3, 0xd1, 0xc4, 0xe1, 0x09, 0xbc, 0xd5, - 0x05, 0xad, 0xc7, 0x49, 0x88, 0x10, 0x68, 0xbe, 0xed, 0x11, 0xac, 0x34, 0x15, 0xbd, 0x62, 0xc9, - 0x18, 0xad, 0x81, 0xe6, 0x52, 0x7f, 0x8c, 0x17, 0x9a, 0x8a, 0x5e, 0x6d, 0xdf, 0x30, 0x2e, 0x9d, - 0x6c, 0xec, 0x53, 0x7f, 0x6c, 0x49, 0x48, 0xeb, 0x8b, 0x02, 0xe5, 0x1d, 0xea, 0x92, 0x3d, 0x12, - 0x33, 0xb4, 0x0b, 0x30, 0x26, 0x31, 0xeb, 0xc4, 0xaf, 0x6d, 0x3e, 0xc2, 0x4a, 0x53, 0xd5, 0xab, - 0xed, 0xb5, 0x99, 0xee, 0x0c, 0x6e, 0xec, 0xe5, 0xd8, 0xae, 0xcf, 0xa3, 0xd8, 0x9a, 0x6a, 0xae, - 0x3f, 0x83, 0x95, 0x4b, 0x3f, 0xa3, 0x1a, 0xa8, 0x63, 0x12, 0xa7, 0x44, 0x45, 0x88, 0xae, 0x43, - 0xf1, 0x9d, 0xed, 0x4e, 0x88, 0x24, 0x5a, 0xb1, 0x92, 0xe4, 0xc9, 0xc2, 0xa6, 0xd2, 0xfa, 0xa6, - 0x26, 0xb4, 0x76, 0xfd, 0xe3, 0x40, 0x5c, 0xd1, 0xa3, 0xae, 0x9b, 0x5d, 0x51, 0xc4, 0xa8, 0x0e, - 0x65, 0x67, 0x44, 0x9c, 0x31, 0x9b, 0x78, 0x69, 0x77, 0x9e, 0xa3, 0x9b, 0xb0, 0xc8, 0x82, 0x49, - 0xe4, 0x10, 0xac, 0xca, 0x5f, 0xd2, 0x4c, 0x9c, 0x13, 0x84, 0x9c, 0x61, 0x2d, 0x39, 0x47, 0xc4, - 0xa2, 0x36, 0xb2, 0xd9, 0x08, 0x17, 0x93, 0x9a, 0x88, 0x33, 0xa2, 0x8b, 0x17, 0x88, 0x7a, 0x64, - 0x40, 0x6d, 0x5c, 0x4a, 0x88, 0xca, 0x24, 0x5f, 0x7d, 0x79, 0x6a, 0xf5, 0x08, 0x34, 0x46, 0x3f, - 0x10, 0x5c, 0x69, 0x2a, 0xba, 0x6a, 0xc9, 0x58, 0x74, 0xdb, 0x83, 0x01, 0x19, 0x60, 0x90, 0xc5, - 0x24, 0x41, 0xeb, 0xa0, 0x79, 0x84, 0xdb, 0xb8, 0x2a, 0x45, 0xba, 0x65, 0x24, 0x6a, 0x1b, 0x99, - 0xda, 0x46, 0x4f, 0xaa, 0x6d, 0x49, 0x10, 0xc2, 0x50, 0xe2, 0x76, 0x34, 0x24, 0x9c, 0xe1, 0x6b, - 0x4d, 0x55, 0xaf, 0x58, 0x59, 0x8a, 0x3a, 0x50, 0x22, 0xbe, 0x73, 0x10, 0x0c, 0x08, 0x5e, 0x6a, - 0x2a, 0xfa, 0x72, 0x5b, 0x9f, 0x2b, 0x98, 0x58, 0xa4, 0xd1, 0xf5, 0x9d, 0x28, 0x0e, 0x39, 0x0d, - 0x7c, 0x81, 0xb7, 0xb2, 0x46, 0xb1, 0x4c, 0x31, 0xe5, 0xa5, 0x58, 0xc4, 0x72, 0xb2, 0xcc, 0x2c, - 0x6f, 0xdd, 0x87, 0xe5, 0x8b, 0x6d, 0xa8, 0x0a, 0xa5, 0xad, 0x6e, 0xef, 0xe8, 0xc5, 0xf6, 0x41, - 0xad, 0x90, 0x25, 0xdb, 0x3b, 0x9d, 0x9a, 0xd2, 0xfa, 0xaa, 0x40, 0xe5, 0x39, 0x8d, 0x88, 0xc3, - 0x83, 0x28, 0x46, 0x4f, 0xa1, 0x28, 0xec, 0xcd, 0x52, 0x23, 0xdd, 0x9b, 0xe1, 0x95, 0x43, 0x25, - 0x43, 0x96, 0x98, 0x28, 0xe9, 0xa9, 0xf7, 0x00, 0xce, 0x8b, 0x73, 0xac, 0x63, 0x4e, 0x5b, 0xa7, - 0xda, 0xbe, 0x7d, 0xe5, 0xa5, 0xa7, 0x5d, 0xb5, 0x05, 0x4b, 0xf9, 0xcc, 0x7d, 0xca, 0x38, 0x7a, - 0x08, 0x45, 0xca, 0x89, 0x97, 0x51, 0xac, 0x5f, 0x4d, 0xd1, 0x4a, 0x80, 0xad, 0x4f, 0x2a, 0x68, - 0xaf, 0xc4, 0x16, 0xe6, 0x7d, 0x77, 0x35, 0x50, 0x43, 0xea, 0x4b, 0x4a, 0x65, 0x4b, 0x84, 0x68, - 0x15, 0x2a, 0xa1, 0x6b, 0x53, 0x9f, 0x93, 0xf7, 0x5c, 0xba, 0xb1, 0x6c, 0x9d, 0x17, 0x72, 0x63, - 0x6b, 0x53, 0xc6, 0xde, 0x48, 0x4d, 0x5a, 0x94, 0x8c, 0xee, 0xcc, 0x30, 0x12, 0xc3, 0x8d, 0xc3, - 0x90, 0xa7, 0xeb, 0x4a, 0x5c, 0xbc, 0x09, 0xd5, 0xb7, 0x2c, 0xf0, 0x8f, 0x98, 0x33, 0x22, 0x9e, - 0x2d, 0x9d, 0xfb, 0x1b, 0x4b, 0x81, 0xc0, 0xf6, 0x24, 0x14, 0x3d, 0x82, 0xa2, 0x78, 0x07, 0x18, - 0x2e, 0xcb, 0x79, 0xcd, 0xf9, 0xf3, 0xc4, 0x83, 0x91, 0xe9, 0x23, 0xe1, 0xf5, 0xc7, 0x50, 0xc9, - 0x49, 0xfc, 0xc9, 0x97, 0x5d, 0x3f, 0x04, 0x38, 0x3f, 0x6d, 0x4e, 0xe7, 0xfa, 0x45, 0x61, 0xaf, - 0x78, 0xbc, 0xa6, 0x44, 0xfd, 0xb8, 0x00, 0x9a, 0xa8, 0x89, 0xb3, 0x26, 0x2c, 0x13, 0x44, 0x84, - 0xff, 0x45, 0x0f, 0x31, 0xfa, 0xdf, 0xe9, 0xf1, 0xd7, 0x7b, 0xed, 0x3c, 0xf8, 0x7e, 0xda, 0x50, - 0x4e, 0x4e, 0x1b, 0xca, 0xcf, 0xd3, 0x86, 0xf2, 0xf9, 0xac, 0x51, 0x38, 0x39, 0x6b, 0x14, 0x7e, - 0x9c, 0x35, 0x0a, 0x6f, 0xd0, 0xec, 0xdf, 0x4f, 0x7f, 0x51, 0x72, 0xd8, 0xf8, 0x15, 0x00, 0x00, - 0xff, 0xff, 0x7a, 0xf0, 0xb1, 0x6b, 0x9b, 0x06, 0x00, 0x00, + 0x10, 0x8d, 0x6b, 0xa7, 0x49, 0x26, 0xb4, 0x8d, 0x56, 0x7c, 0x2c, 0x51, 0x15, 0xa2, 0x08, 0xa4, + 0xb4, 0x45, 0x0e, 0x4a, 0x25, 0xa8, 0x40, 0x1c, 0x9a, 0x36, 0x85, 0xaa, 0x2d, 0x45, 0xce, 0x8d, + 0x4b, 0xb5, 0x71, 0x36, 0xc9, 0x12, 0xc7, 0xb6, 0xbc, 0x1b, 0x84, 0x39, 0x73, 0x87, 0x03, 0x57, + 0xfe, 0x0f, 0xc7, 0x1e, 0xb9, 0x81, 0xda, 0x3f, 0x82, 0x76, 0xfd, 0xd1, 0xb4, 0x49, 0x91, 0x40, + 0x88, 0xdb, 0xcc, 0xf8, 0xcd, 0xce, 0x9b, 0x79, 0xb3, 0x6b, 0xb8, 0xef, 0x8f, 0x06, 0x0d, 0x87, + 0x75, 0x1b, 0x7e, 0xb7, 0xc1, 0x85, 0x17, 0x90, 0x01, 0x6d, 0xf8, 0x81, 0x27, 0x3c, 0xde, 0xe8, + 0x33, 0x87, 0x9a, 0xca, 0x46, 0x2b, 0xc4, 0x0d, 0x45, 0xe8, 0x53, 0x33, 0x86, 0x94, 0x57, 0x07, + 0x9e, 0x37, 0x70, 0x62, 0x68, 0x77, 0xd2, 0x6f, 0x70, 0x11, 0x4c, 0x6c, 0x11, 0xc1, 0x6b, 0x6d, + 0x30, 0x3a, 0x82, 0xfa, 0x08, 0x81, 0xe1, 0x92, 0x31, 0xc5, 0x5a, 0x55, 0xab, 0x17, 0x2c, 0x65, + 0xa3, 0x35, 0x30, 0x1c, 0xe6, 0x8e, 0xf0, 0x42, 0x55, 0xab, 0x17, 0x9b, 0xb7, 0xcc, 0x2b, 0x27, + 0x9b, 0x87, 0xcc, 0x1d, 0x59, 0x0a, 0x52, 0xfb, 0xa2, 0x41, 0x7e, 0x8f, 0x39, 0xf4, 0x80, 0x86, + 0x1c, 0xed, 0x03, 0x8c, 0x68, 0xc8, 0x5b, 0xe1, 0x6b, 0x22, 0x86, 0x58, 0xab, 0xea, 0xf5, 0x62, + 0x73, 0x6d, 0x26, 0x3b, 0x81, 0x9b, 0x07, 0x29, 0xb6, 0xed, 0x8a, 0x20, 0xb4, 0xa6, 0x92, 0xcb, + 0xcf, 0x61, 0xe5, 0xca, 0x67, 0x54, 0x02, 0x7d, 0x44, 0xc3, 0x98, 0xa8, 0x34, 0xd1, 0x4d, 0xc8, + 0xbe, 0x23, 0xce, 0x84, 0x2a, 0xa2, 0x05, 0x2b, 0x72, 0x9e, 0x2e, 0x6c, 0x69, 0xb5, 0x1f, 0x7a, + 0x44, 0x6b, 0xdf, 0xed, 0x7b, 0xb2, 0xc5, 0x31, 0x73, 0x9c, 0xa4, 0x45, 0x69, 0xa3, 0x32, 0xe4, + 0xed, 0x21, 0xb5, 0x47, 0x7c, 0x32, 0x8e, 0xb3, 0x53, 0x1f, 0xdd, 0x86, 0x45, 0xee, 0x4d, 0x02, + 0x9b, 0x62, 0x5d, 0x7d, 0x89, 0x3d, 0x79, 0x8e, 0xe7, 0x0b, 0x8e, 0x8d, 0xe8, 0x1c, 0x69, 0xcb, + 0xd8, 0x90, 0xf0, 0x21, 0xce, 0x46, 0x31, 0x69, 0x27, 0x44, 0x17, 0x2f, 0x11, 0x1d, 0xd3, 0x1e, + 0x23, 0x38, 0x17, 0x11, 0x55, 0x4e, 0x3a, 0xfa, 0xfc, 0xd4, 0xe8, 0x11, 0x18, 0x9c, 0x7d, 0xa0, + 0xb8, 0x50, 0xd5, 0xea, 0xba, 0xa5, 0x6c, 0x99, 0x4d, 0x7a, 0x3d, 0xda, 0xc3, 0xa0, 0x82, 0x91, + 0x83, 0x36, 0xc0, 0x18, 0x53, 0x41, 0x70, 0x51, 0x89, 0x74, 0xc7, 0x8c, 0xd4, 0x36, 0x13, 0xb5, + 0xcd, 0x8e, 0x52, 0xdb, 0x52, 0x20, 0x84, 0x21, 0x27, 0x48, 0x30, 0xa0, 0x82, 0xe3, 0x1b, 0x55, + 0xbd, 0x5e, 0xb0, 0x12, 0x17, 0xb5, 0x20, 0x47, 0x5d, 0xfb, 0xc8, 0xeb, 0x51, 0xbc, 0x54, 0xd5, + 0xea, 0xcb, 0xcd, 0xfa, 0x5c, 0xc1, 0xe4, 0x20, 0xcd, 0xb6, 0x6b, 0x07, 0xa1, 0x2f, 0x98, 0xe7, + 0x4a, 0xbc, 0x95, 0x24, 0xca, 0x61, 0xca, 0x2a, 0x2f, 0xe5, 0x20, 0x96, 0xa3, 0x61, 0x26, 0x3e, + 0x5a, 0x87, 0x92, 0x43, 0xb8, 0x38, 0xf2, 0x7a, 0xac, 0xcf, 0x68, 0x6f, 0x97, 0x08, 0x8a, 0x57, + 0x54, 0x1f, 0x33, 0xf1, 0xda, 0x3a, 0x2c, 0x5f, 0x2e, 0x81, 0x8a, 0x90, 0xdb, 0x6e, 0x77, 0x4e, + 0x5e, 0xec, 0x1c, 0x95, 0x32, 0x89, 0xb3, 0xb3, 0xd7, 0x2a, 0x69, 0xb5, 0xaf, 0x1a, 0x14, 0x76, + 0x59, 0x40, 0x6d, 0xe1, 0x05, 0x21, 0x7a, 0x06, 0x59, 0x79, 0x15, 0x78, 0xbc, 0x74, 0x0f, 0x66, + 0x7a, 0x48, 0xa1, 0xaa, 0x1b, 0x1e, 0x2d, 0x5c, 0x94, 0x53, 0xee, 0x00, 0x5c, 0x04, 0xe7, 0xac, + 0x59, 0x63, 0x7a, 0xcd, 0x8a, 0xcd, 0xbb, 0xd7, 0x0e, 0x68, 0x7a, 0x03, 0xb7, 0x61, 0x29, 0xad, + 0x79, 0xc8, 0xb8, 0x40, 0x8f, 0x20, 0xcb, 0x04, 0x1d, 0x27, 0x14, 0xcb, 0xd7, 0x53, 0xb4, 0x22, + 0x60, 0xed, 0x93, 0x0e, 0xc6, 0x2b, 0x39, 0x85, 0x79, 0x77, 0xb4, 0x04, 0xba, 0xcf, 0x5c, 0x45, + 0x29, 0x6f, 0x49, 0x13, 0xad, 0x42, 0xc1, 0x77, 0x08, 0x73, 0x05, 0x7d, 0x2f, 0xd4, 0xe6, 0xe6, + 0xad, 0x8b, 0x40, 0x7a, 0x09, 0x8c, 0xa9, 0x4b, 0xb0, 0x19, 0x2f, 0x74, 0x56, 0x31, 0xba, 0x37, + 0xc3, 0x48, 0x16, 0x37, 0x8f, 0x7d, 0x11, 0x8f, 0x2b, 0xda, 0xf8, 0x2d, 0x28, 0xbe, 0xe5, 0x9e, + 0x7b, 0xc2, 0xed, 0x21, 0x1d, 0x13, 0xb5, 0xe5, 0xbf, 0x59, 0x3f, 0x90, 0xd8, 0x8e, 0x82, 0xa2, + 0xc7, 0x90, 0x95, 0x6f, 0x06, 0xc7, 0x79, 0x55, 0xaf, 0x3a, 0xbf, 0x9e, 0x7c, 0x5c, 0x12, 0x7d, + 0x14, 0xbc, 0xfc, 0x04, 0x0a, 0x29, 0x89, 0x3f, 0x79, 0x05, 0xca, 0xc7, 0x00, 0x17, 0xa7, 0xcd, + 0xc9, 0xdc, 0xb8, 0x2c, 0xec, 0x35, 0x0f, 0xdd, 0x94, 0xa8, 0x1f, 0x17, 0xc0, 0x90, 0x31, 0x79, + 0xd6, 0x84, 0x27, 0x82, 0x48, 0xf3, 0xbf, 0xe8, 0x21, 0x4b, 0xff, 0x3b, 0x3d, 0xfe, 0x7a, 0xae, + 0xad, 0x87, 0xdf, 0xce, 0x2a, 0xda, 0xe9, 0x59, 0x45, 0xfb, 0x79, 0x56, 0xd1, 0x3e, 0x9f, 0x57, + 0x32, 0xa7, 0xe7, 0x95, 0xcc, 0xf7, 0xf3, 0x4a, 0xe6, 0x0d, 0x9a, 0xfd, 0x55, 0x75, 0x17, 0x15, + 0x87, 0xcd, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x3d, 0x58, 0x64, 0xbf, 0xc7, 0x06, 0x00, 0x00, } func (m *Step) Marshal() (dAtA []byte, err error) { @@ -729,6 +738,11 @@ func (m *FileInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.LastModifiedDate != 0 { + i = encodeVarintFile(dAtA, i, uint64(m.LastModifiedDate)) + i-- + dAtA[i] = 0x78 + } if len(m.MetaHash) > 0 { i -= len(m.MetaHash) copy(dAtA[i:], m.MetaHash) @@ -1225,6 +1239,9 @@ func (m *FileInfo) Size() (n int) { if l > 0 { n += 1 + l + sovFile(uint64(l)) } + if m.LastModifiedDate != 0 { + n += 1 + sovFile(uint64(m.LastModifiedDate)) + } return n } @@ -2091,6 +2108,25 @@ func (m *FileInfo) Unmarshal(dAtA []byte) error { } m.MetaHash = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 15: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LastModifiedDate", wireType) + } + m.LastModifiedDate = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFile + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LastModifiedDate |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipFile(dAtA[iNdEx:]) diff --git a/pkg/lib/pb/storage/protos/file.proto b/pkg/lib/pb/storage/protos/file.proto index 37219f604c..9a74164dec 100644 --- a/pkg/lib/pb/storage/protos/file.proto +++ b/pkg/lib/pb/storage/protos/file.proto @@ -28,6 +28,7 @@ message FileInfo { repeated string targets = 12; EncryptionMode encMode = 13; string metaHash = 14; + int64 lastModifiedDate = 15; enum EncryptionMode { AES_GCM = 0; diff --git a/space/service.go b/space/service.go index 05b7efdc9a..22745d683d 100644 --- a/space/service.go +++ b/space/service.go @@ -324,3 +324,9 @@ func (s *service) getSpaceType(header *spacesyncproto.RawSpaceHeaderWithId) (tp tp = payload.SpaceType return } + +func (s *service) GetLogFields() []zap.Field { + return []zap.Field{ + zap.Bool("newAccount", s.newAccount), + } +} diff --git a/util/debug/handler_helpers.go b/util/debug/handler_helpers.go new file mode 100644 index 0000000000..2920b77f0d --- /dev/null +++ b/util/debug/handler_helpers.go @@ -0,0 +1,52 @@ +package debug + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/anyproto/anytype-heart/pkg/lib/logging" +) + +var log = logging.Logger("debug") + +func PlaintextHandler(handlerFunc func(w io.Writer, req *http.Request) error) http.HandlerFunc { + return func(rw http.ResponseWriter, req *http.Request) { + rw.Header().Set("Content-Type", "text/plain") + + err := handlerFunc(rw, req) + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + _, err := fmt.Fprintf(rw, "error: %s", err) + if err != nil { + log.Errorf("write debug response for path %s error: %s", req.URL.Path, err) + } + return + } + } +} + +type errorResponse struct { + Error string `json:"error"` +} + +func JSONHandler[T any](handlerFunc func(req *http.Request) (T, error)) http.HandlerFunc { + return func(rw http.ResponseWriter, req *http.Request) { + rw.Header().Set("Content-Type", "application/json") + + data, err := handlerFunc(req) + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + err := json.NewEncoder(rw).Encode(errorResponse{Error: err.Error()}) + if err != nil { + log.Errorf("encode debug response for path %s error: %s", req.URL.Path, err) + } + return + } + err = json.NewEncoder(rw).Encode(data) + if err != nil { + log.Errorf("encode debug response for path %s error: %s", req.URL.Path, err) + } + } +}