From 875b243fabde4cab25f1df710d047c2b43279b5c Mon Sep 17 00:00:00 2001 From: AnastasiaShemyakinskaya Date: Fri, 9 Aug 2024 09:04:33 +0200 Subject: [PATCH 1/7] GO-3870: fix history limit Signed-off-by: AnastasiaShemyakinskaya --- core/history/history.go | 10 +- core/history/history_test.go | 192 ++++++++++++++++++++++++++++++++++- 2 files changed, 196 insertions(+), 6 deletions(-) diff --git a/core/history/history.go b/core/history/history.go index e8a83743ca..33791b0852 100644 --- a/core/history/history.go +++ b/core/history/history.go @@ -148,6 +148,9 @@ func (h *history) Versions(id domain.FullID, lastVersionId string, limit int) (r if e != nil { return nil, e } + if len(data) == 0 { + break + } if len(data[0].PreviousIds) == 0 { if data[0].Id == tree.Id() { data = data[1:] @@ -159,10 +162,6 @@ func (h *history) Versions(id domain.FullID, lastVersionId string, limit int) (r lastVersionId = tree.Root().Id includeLastId = false } - - if len(data) == 0 { - break - } } resp = reverse(resp) @@ -178,6 +177,9 @@ func (h *history) Versions(id domain.FullID, lastVersionId string, limit int) (r resp[i].GroupId = groupId } + if len(resp) > limit { + resp = resp[:limit] + } return } diff --git a/core/history/history_test.go b/core/history/history_test.go index 04779c6efb..35095ff1b1 100644 --- a/core/history/history_test.go +++ b/core/history/history_test.go @@ -27,8 +27,6 @@ import ( "github.com/anyproto/anytype-heart/util/pbtypes" ) -// todo: reimplement - type historyStub struct { changes []*objecttree.Change objectId string @@ -804,6 +802,196 @@ func TestHistory_DiffVersions(t *testing.T) { }) } +func TestHistory_Versions(t *testing.T) { + t.Run("limit 0 - 100 changes", func(t *testing.T) { + // given + objectId := "objectId" + spaceID := "spaceID" + versionId := "versionId" + + accountKeys, _ := accountdata.NewRandom() + account := accountKeys.SignKey.GetPublic() + + blDataviewId := "blDataviewId" + relationKey := "key" + + viewId := "viewId" + viewName := "view" + + viewId1 := "viewId1" + view1Name := "view1" + + viewId2 := "viewId2" + view2Name := "view2" + + blDataview := provideDataviewBlock(viewId1, view1Name, relationKey, blDataviewId) + + blSmartBlock := &model.Block{Id: objectId, Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}} + + currChange := []*objecttree.Change{ + // create block changes + provideBlockEmptyChange(objectId, account), + provideBlockCreateChange(blSmartBlock, account), + provideBlockCreateChange(blDataview, account), + + // dataview changes + provideBlockDataviewViewSetChange(blDataviewId, viewId1, view1Name, relationKey, account), + provideBlockDataviewViewSetChange(blDataviewId, viewId2, view2Name, relationKey, account), + provideBlockDataviewSourceSetChange(blDataviewId, account), + provideBlockDataviewRelationSetChange(blDataviewId, account), + provideBlockDataviewViewSetChange(blDataviewId, viewId, viewName, relationKey, account), + provideBlockDataviewViewOrderChange(blDataviewId, viewId, viewId1, account), + provideBlockDataviewViewDeleteChange(blDataviewId, viewId2, account), + provideBlockDataviewRelationDeleteChange(blDataviewId, relationKey, account), + provideBlockDataviewObjectOrderChange(blDataviewId, viewId, account), + provideBlockDataviewGroupOrderChange(blDataviewId, viewId, account), + provideBlockDataviewViewUpdateChange(blDataviewId, viewId, account), + provideBlockDataviewTargetObjectChange(blDataviewId, account), + } + + history := newFixture(t, currChange, objectId, spaceID, versionId) + + // when + resp, err := history.Versions(domain.FullID{ObjectID: objectId, SpaceID: spaceID}, versionId, 0) + + // then + assert.Nil(t, err) + assert.Len(t, resp, 14) + }) + t.Run("limit 10 - 10 changes", func(t *testing.T) { + // given + objectId := "objectId" + spaceID := "spaceID" + versionId := "versionId" + + accountKeys, _ := accountdata.NewRandom() + account := accountKeys.SignKey.GetPublic() + + blDataviewId := "blDataviewId" + relationKey := "key" + + viewId := "viewId" + viewName := "view" + + viewId1 := "viewId1" + view1Name := "view1" + + viewId2 := "viewId2" + view2Name := "view2" + + blDataview := provideDataviewBlock(viewId1, view1Name, relationKey, blDataviewId) + + blSmartBlock := &model.Block{Id: objectId, Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}} + + currChange := []*objecttree.Change{ + // create block changes + provideBlockEmptyChange(objectId, account), + provideBlockCreateChange(blSmartBlock, account), + provideBlockCreateChange(blDataview, account), + + // dataview changes + provideBlockDataviewViewSetChange(blDataviewId, viewId1, view1Name, relationKey, account), + provideBlockDataviewViewSetChange(blDataviewId, viewId2, view2Name, relationKey, account), + provideBlockDataviewSourceSetChange(blDataviewId, account), + provideBlockDataviewRelationSetChange(blDataviewId, account), + provideBlockDataviewViewSetChange(blDataviewId, viewId, viewName, relationKey, account), + provideBlockDataviewViewOrderChange(blDataviewId, viewId, viewId1, account), + provideBlockDataviewViewDeleteChange(blDataviewId, viewId2, account), + provideBlockDataviewRelationDeleteChange(blDataviewId, relationKey, account), + provideBlockDataviewObjectOrderChange(blDataviewId, viewId, account), + provideBlockDataviewGroupOrderChange(blDataviewId, viewId, account), + provideBlockDataviewViewUpdateChange(blDataviewId, viewId, account), + provideBlockDataviewTargetObjectChange(blDataviewId, account), + } + + history := newFixture(t, currChange, objectId, spaceID, versionId) + + // when + resp, err := history.Versions(domain.FullID{ObjectID: objectId, SpaceID: spaceID}, versionId, 10) + + // then + assert.Nil(t, err) + assert.Len(t, resp, 10) + }) + t.Run("number of changes equals limit", func(t *testing.T) { + // given + objectId := "objectId" + spaceID := "spaceID" + versionId := "versionId" + + accountKeys, _ := accountdata.NewRandom() + account := accountKeys.SignKey.GetPublic() + + blDataviewId := "blDataviewId" + relationKey := "key" + + viewId := "viewId" + viewName := "view" + + viewId1 := "viewId1" + view1Name := "view1" + + viewId2 := "viewId2" + view2Name := "view2" + + blDataview := provideDataviewBlock(viewId1, view1Name, relationKey, blDataviewId) + + blSmartBlock := &model.Block{Id: objectId, Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}} + + currChange := []*objecttree.Change{ + // create block changes + provideBlockEmptyChange(objectId, account), + provideBlockCreateChange(blSmartBlock, account), + provideBlockCreateChange(blDataview, account), + + // dataview changes + provideBlockDataviewViewSetChange(blDataviewId, viewId1, view1Name, relationKey, account), + provideBlockDataviewViewSetChange(blDataviewId, viewId2, view2Name, relationKey, account), + provideBlockDataviewSourceSetChange(blDataviewId, account), + provideBlockDataviewRelationSetChange(blDataviewId, account), + provideBlockDataviewViewSetChange(blDataviewId, viewId, viewName, relationKey, account), + provideBlockDataviewViewOrderChange(blDataviewId, viewId, viewId1, account), + provideBlockDataviewViewDeleteChange(blDataviewId, viewId2, account), + provideBlockDataviewRelationDeleteChange(blDataviewId, relationKey, account), + } + + history := newFixture(t, currChange, objectId, spaceID, versionId) + + // when + resp, err := history.Versions(domain.FullID{ObjectID: objectId, SpaceID: spaceID}, versionId, 10) + + // then + assert.Nil(t, err) + assert.Len(t, resp, 10) + }) + t.Run("no changes", func(t *testing.T) { + // given + objectId := "objectId" + spaceID := "spaceID" + versionId := "versionId" + + var currChange []*objecttree.Change + + history := newFixture(t, currChange, objectId, spaceID, versionId) + + ctrl := gomock.NewController(t) + spaceService := mock_space.NewMockService(t) + space := mock_clientspace.NewMockSpace(t) + treeBuilder := mock_objecttreebuilder.NewMockTreeBuilder(ctrl) + configureTreeBuilder(treeBuilder, objectId, versionId, spaceID, currChange, space, spaceService) + history.treeBuilder = treeBuilder + history.space = space + history.spaceService = spaceService + + // when + resp, err := history.Versions(domain.FullID{ObjectID: objectId, SpaceID: spaceID}, versionId, 10) + + // then + assert.Nil(t, err) + assert.Len(t, resp, 0) + }) +} + type historyFixture struct { *history space *mock_clientspace.MockSpace From 3afe976e27577a11644f95ce596d2c5af5e05a57 Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 8 Aug 2024 10:50:31 +0200 Subject: [PATCH 2/7] GO-3882: Import: do not ignore old files in link substitution mechanism (cherry picked from commit ce04ae71043fd301cd8a25a64c120a9b7cb554c2) --- core/block/import/common/common.go | 33 +++++++------------ core/block/import/common/common_test.go | 8 ++--- .../common/objectcreator/objectcreator.go | 5 ++- .../objectcreator/objectcreator_test.go | 2 +- .../block/import/common/objectcreator/task.go | 10 +----- core/block/import/importer.go | 13 +------- core/block/import/pb/converter.go | 9 ++--- 7 files changed, 23 insertions(+), 57 deletions(-) diff --git a/core/block/import/common/common.go b/core/block/import/common/common.go index 7d13c6baaa..8996539f51 100644 --- a/core/block/import/common/common.go +++ b/core/block/import/common/common.go @@ -10,7 +10,6 @@ import ( "github.com/gogo/protobuf/types" "github.com/ipfs/go-cid" - "github.com/samber/lo" "github.com/anyproto/anytype-heart/core/block/editor/state" "github.com/anyproto/anytype-heart/core/block/editor/widget" @@ -54,16 +53,16 @@ func GetCommonDetails(sourcePath, name, emoji string, layout model.ObjectTypeLay return &types.Struct{Fields: fields} } -func UpdateLinksToObjects(st *state.State, oldIDtoNew map[string]string, filesIDs []string) error { +func UpdateLinksToObjects(st *state.State, oldIDtoNew map[string]string) error { return st.Iterate(func(bl simple.Block) (isContinue bool) { // TODO I think we should use some kind of iterator by object ids switch block := bl.(type) { case link.Block: - handleLinkBlock(oldIDtoNew, block, st, filesIDs) + handleLinkBlock(oldIDtoNew, block, st) case bookmark.Block: handleBookmarkBlock(oldIDtoNew, block, st) case text.Block: - handleTextBlock(oldIDtoNew, block, st, filesIDs) + handleTextBlock(oldIDtoNew, block, st) case dataview.Block: handleDataviewBlock(block, oldIDtoNew, st) case file.Block: @@ -141,11 +140,9 @@ func handleBookmarkBlock(oldIDtoNew map[string]string, block simple.Block, st *s st.Set(simple.New(block.Model())) } -func handleLinkBlock(oldIDtoNew map[string]string, block simple.Block, st *state.State, filesIDs []string) { +func handleLinkBlock(oldIDtoNew map[string]string, block simple.Block, st *state.State) { targetBlockID := block.Model().GetLink().TargetBlockId - if lo.Contains(filesIDs, targetBlockID) { - return - } + newTarget := oldIDtoNew[targetBlockID] if newTarget == "" { if widget.IsPredefinedWidgetTargetId(targetBlockID) { @@ -195,7 +192,7 @@ func isBundledObjects(targetObjectID string) bool { return false } -func handleTextBlock(oldIDtoNew map[string]string, block simple.Block, st *state.State, filesIDs []string) { +func handleTextBlock(oldIDtoNew map[string]string, block simple.Block, st *state.State) { if iconImage := block.Model().GetText().GetIconImage(); iconImage != "" { newTarget := oldIDtoNew[iconImage] if newTarget == "" { @@ -212,9 +209,6 @@ func handleTextBlock(oldIDtoNew map[string]string, block simple.Block, st *state if mark.Type != model.BlockContentTextMark_Mention && mark.Type != model.BlockContentTextMark_Object { continue } - if lo.Contains(filesIDs, mark.Param) { - return - } if isBundledObjects(mark.Param) { return } @@ -228,7 +222,7 @@ func handleTextBlock(oldIDtoNew map[string]string, block simple.Block, st *state st.Set(simple.New(block.Model())) } -func UpdateObjectIDsInRelations(st *state.State, oldIDtoNew map[string]string, filesIDs []string) { +func UpdateObjectIDsInRelations(st *state.State, oldIDtoNew map[string]string) { rels := st.GetRelationLinks() for k, v := range st.Details().GetFields() { relLink := rels.Get(k) @@ -244,7 +238,7 @@ func UpdateObjectIDsInRelations(st *state.State, oldIDtoNew map[string]string, f continue } // For example, RelationKeySetOf is handled here - handleObjectRelation(st, oldIDtoNew, v, k, filesIDs) + handleObjectRelation(st, oldIDtoNew, v, k) } } @@ -256,28 +250,25 @@ func isLinkToObject(relLink *model.RelationLink) bool { relLink.Format == model.RelationFormat_file } -func handleObjectRelation(st *state.State, oldIDtoNew map[string]string, v *types.Value, k string, filesIDs []string) { +func handleObjectRelation(st *state.State, oldIDtoNew map[string]string, v *types.Value, k string) { if _, ok := v.GetKind().(*types.Value_StringValue); ok { objectsID := v.GetStringValue() - newObjectIDs := getNewObjectsIDForRelation([]string{objectsID}, oldIDtoNew, filesIDs) + newObjectIDs := getNewObjectsIDForRelation([]string{objectsID}, oldIDtoNew) if len(newObjectIDs) != 0 { st.SetDetail(k, pbtypes.String(newObjectIDs[0])) } return } objectsIDs := pbtypes.GetStringListValue(v) - objectsIDs = getNewObjectsIDForRelation(objectsIDs, oldIDtoNew, filesIDs) + objectsIDs = getNewObjectsIDForRelation(objectsIDs, oldIDtoNew) st.SetDetail(k, pbtypes.StringList(objectsIDs)) } -func getNewObjectsIDForRelation(objectsIDs []string, oldIDtoNew map[string]string, filesIDs []string) []string { +func getNewObjectsIDForRelation(objectsIDs []string, oldIDtoNew map[string]string) []string { for i, val := range objectsIDs { if val == "" { continue } - if lo.Contains(filesIDs, val) { - continue - } newTarget := oldIDtoNew[val] if newTarget == "" { // preserve links to bundled objects diff --git a/core/block/import/common/common_test.go b/core/block/import/common/common_test.go index d3e8feef2e..d653a6b585 100644 --- a/core/block/import/common/common_test.go +++ b/core/block/import/common/common_test.go @@ -87,7 +87,7 @@ func TestUpdateLinksToObjects(t *testing.T) { oldToNew := map[string]string{rawCid: "newFileObjectId"} // when - err = UpdateLinksToObjects(st, oldToNew, nil) + err = UpdateLinksToObjects(st, oldToNew) // then assert.Nil(t, err) @@ -109,7 +109,7 @@ func TestUpdateLinksToObjects(t *testing.T) { st := state.NewDoc("root", map[string]simple.Block{"test": simpleBlock, "root": rootSimpleBlock}).(*state.State) // when - err := UpdateLinksToObjects(st, map[string]string{}, nil) + err := UpdateLinksToObjects(st, map[string]string{}) // then assert.Nil(t, err) @@ -135,7 +135,7 @@ func TestUpdateLinksToObjects(t *testing.T) { st := state.NewDoc("root", map[string]simple.Block{"test": simpleBlock, "root": rootSimpleBlock}).(*state.State) // when - err = UpdateLinksToObjects(st, map[string]string{}, nil) + err = UpdateLinksToObjects(st, map[string]string{}) // then assert.Nil(t, err) @@ -159,7 +159,7 @@ func TestUpdateLinksToObjects(t *testing.T) { st := state.NewDoc("root", map[string]simple.Block{"test": simpleBlock, "root": rootSimpleBlock}).(*state.State) // when - err := UpdateLinksToObjects(st, map[string]string{}, nil) + err := UpdateLinksToObjects(st, map[string]string{}) // then assert.Nil(t, err) diff --git a/core/block/import/common/objectcreator/objectcreator.go b/core/block/import/common/objectcreator/objectcreator.go index 704d0404eb..a650cc957d 100644 --- a/core/block/import/common/objectcreator/objectcreator.go +++ b/core/block/import/common/objectcreator/objectcreator.go @@ -81,7 +81,6 @@ func New(service BlockService, func (oc *ObjectCreator) Create(dataObject *DataObject, sn *common.Snapshot) (*types.Struct, string, error) { snapshot := sn.Snapshot.Data oldIDtoNew := dataObject.oldIDtoNew - fileIDs := dataObject.fileIDs ctx := dataObject.ctx origin := dataObject.origin spaceID := dataObject.spaceID @@ -105,9 +104,9 @@ func (oc *ObjectCreator) Create(dataObject *DataObject, sn *common.Snapshot) (*t oc.onFinish(err, st, filesToDelete) }() - common.UpdateObjectIDsInRelations(st, oldIDtoNew, fileIDs) + common.UpdateObjectIDsInRelations(st, oldIDtoNew) - if err = common.UpdateLinksToObjects(st, oldIDtoNew, fileIDs); err != nil { + if err = common.UpdateLinksToObjects(st, oldIDtoNew); err != nil { log.With("objectID", newID).Errorf("failed to update objects ids: %s", err) } diff --git a/core/block/import/common/objectcreator/objectcreator_test.go b/core/block/import/common/objectcreator/objectcreator_test.go index 7f752bf2d2..6bfb9fc53e 100644 --- a/core/block/import/common/objectcreator/objectcreator_test.go +++ b/core/block/import/common/objectcreator/objectcreator_test.go @@ -40,7 +40,7 @@ func TestObjectCreator_Create(t *testing.T) { importedSpaceIdParticipantId := domain.NewParticipantId(importedSpaceId, identity) oldToNew := map[string]string{importedSpaceIdParticipantId: participantId} - dataObject := NewDataObject(context.Background(), oldToNew, nil, nil, objectorigin.Import(model.Import_Pb), spaceID) + dataObject := NewDataObject(context.Background(), oldToNew, nil, objectorigin.Import(model.Import_Pb), spaceID) sn := &common.Snapshot{ Id: importedSpaceIdParticipantId, SbType: coresb.SmartBlockTypeParticipant, diff --git a/core/block/import/common/objectcreator/task.go b/core/block/import/common/objectcreator/task.go index 565ad71ce8..e51c9ce6a2 100644 --- a/core/block/import/common/objectcreator/task.go +++ b/core/block/import/common/objectcreator/task.go @@ -13,7 +13,6 @@ import ( type DataObject struct { oldIDtoNew map[string]string createPayloads map[string]treestorage.TreeStorageCreatePayload - fileIDs []string ctx context.Context origin objectorigin.ObjectOrigin spaceID string @@ -27,13 +26,7 @@ type Result struct { Err error } -func NewDataObject(ctx context.Context, - oldIDtoNew map[string]string, - createPayloads map[string]treestorage.TreeStorageCreatePayload, - filesIDs []string, - origin objectorigin.ObjectOrigin, - spaceID string, -) *DataObject { +func NewDataObject(ctx context.Context, oldIDtoNew map[string]string, createPayloads map[string]treestorage.TreeStorageCreatePayload, origin objectorigin.ObjectOrigin, spaceID string) *DataObject { newIdsSet := make(map[string]struct{}, len(oldIDtoNew)) for _, newId := range oldIDtoNew { newIdsSet[newId] = struct{}{} @@ -41,7 +34,6 @@ func NewDataObject(ctx context.Context, return &DataObject{ oldIDtoNew: oldIDtoNew, createPayloads: createPayloads, - fileIDs: filesIDs, ctx: ctx, origin: origin, spaceID: spaceID, diff --git a/core/block/import/importer.go b/core/block/import/importer.go index 9648b5f19c..c4e73d6b53 100644 --- a/core/block/import/importer.go +++ b/core/block/import/importer.go @@ -312,12 +312,11 @@ func (i *Import) createObjects(ctx context.Context, if err != nil { return nil, "" } - filesIDs := i.getFilesIDs(res) numWorkers := workerPoolSize if len(res.Snapshots) < workerPoolSize { numWorkers = 1 } - do := creator.NewDataObject(ctx, oldIDToNew, createPayloads, filesIDs, origin, req.SpaceId) + do := creator.NewDataObject(ctx, oldIDToNew, createPayloads, origin, req.SpaceId) pool := workerpool.NewPool(numWorkers) progress.SetProgressMessage("Create objects") go i.addWork(res, pool) @@ -326,16 +325,6 @@ func (i *Import) createObjects(ctx context.Context, return details, oldIDToNew[res.RootCollectionID] } -func (i *Import) getFilesIDs(res *common.Response) []string { - fileIDs := make([]string, 0) - for _, snapshot := range res.Snapshots { - fileIDs = append(fileIDs, lo.Map(snapshot.Snapshot.GetFileKeys(), func(item *pb.ChangeFileKeys, index int) string { - return item.Hash - })...) - } - return fileIDs -} - func (i *Import) getIDForAllObjects(ctx context.Context, res *common.Response, allErrors *common.ConvertError, diff --git a/core/block/import/pb/converter.go b/core/block/import/pb/converter.go index 71b207d283..ade77e22ee 100644 --- a/core/block/import/pb/converter.go +++ b/core/block/import/pb/converter.go @@ -12,7 +12,6 @@ import ( "github.com/gogo/protobuf/jsonpb" "github.com/gogo/protobuf/types" "github.com/google/uuid" - "github.com/samber/lo" "github.com/anyproto/anytype-heart/core/anytype/account" "github.com/anyproto/anytype-heart/core/block/collection" @@ -463,17 +462,13 @@ func (p *Pb) shouldImportSnapshot(snapshot *common.Snapshot, needToImportWidgets func (p *Pb) updateLinksToObjects(snapshots []*common.Snapshot, allErrors *common.ConvertError, pathCount int) map[string]string { oldToNewID := make(map[string]string, len(snapshots)) - fileIDs := make([]string, 0) for _, snapshot := range snapshots { id := pbtypes.GetString(snapshot.Snapshot.Data.Details, bundle.RelationKeyId.String()) oldToNewID[id] = snapshot.Id - fileIDs = append(fileIDs, lo.Map(snapshot.Snapshot.GetFileKeys(), func(item *pb.ChangeFileKeys, index int) string { - return item.Hash - })...) } for _, snapshot := range snapshots { st := state.NewDocFromSnapshot("", snapshot.Snapshot, state.WithUniqueKeyMigration(snapshot.SbType)) - err := common.UpdateLinksToObjects(st.(*state.State), oldToNewID, fileIDs) + err := common.UpdateLinksToObjects(st.(*state.State), oldToNewID) if err != nil { allErrors.Add(err) if allErrors.ShouldAbortImport(pathCount, model.Import_Pb) { @@ -481,7 +476,7 @@ func (p *Pb) updateLinksToObjects(snapshots []*common.Snapshot, allErrors *commo } continue } - common.UpdateObjectIDsInRelations(st.(*state.State), oldToNewID, fileIDs) + common.UpdateObjectIDsInRelations(st.(*state.State), oldToNewID) // TODO Fix // converter.UpdateObjectType(oldToNewID, st.(*state.State)) p.updateObjectsIDsInCollection(st.(*state.State), oldToNewID) From aa9bc32392d9ffec5a95d11cb6aed80997fcd724 Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 8 Aug 2024 10:59:40 +0200 Subject: [PATCH 3/7] GO-3882: File migration: do not run in non-personal spaces (cherry picked from commit 00b8e75fa8c63ff83ff0e2a128d17c09e5b6545f) --- core/files/fileobject/migration.go | 12 ++++++++++++ core/files/fileobject/migration_test.go | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/core/files/fileobject/migration.go b/core/files/fileobject/migration.go index 563125e860..2cd06280cb 100644 --- a/core/files/fileobject/migration.go +++ b/core/files/fileobject/migration.go @@ -38,6 +38,9 @@ func makeMigrationItem() *migrationItem { } func (s *service) MigrateFileIdsInBlocks(st *state.State, spc source.Space) { + if !spc.IsPersonal() { + return + } st.Iterate(func(b simple.Block) (isContinue bool) { if migrator, ok := b.(simple.FileMigrator); ok { migrator.MigrateFile(func(oldId string) (newId string) { @@ -49,6 +52,9 @@ func (s *service) MigrateFileIdsInBlocks(st *state.State, spc source.Space) { } func (s *service) MigrateFileIdsInDetails(st *state.State, spc source.Space) { + if !spc.IsPersonal() { + return + } st.ModifyLinkedFilesInDetails(func(id string) string { return s.migrateFileId(spc.(clientspace.Space), st.RootId(), id) }) @@ -84,6 +90,9 @@ func (s *service) migrateFileId(space clientspace.Space, objectId string, fileId // from blocks or details will be changed to file object ids. And in case of any migration error // these files could be stuck un-migrated func (s *service) MigrateFiles(st *state.State, spc source.Space, keysChanges []*pb.ChangeFileKeys) { + if !spc.IsPersonal() { + return + } origin := objectorigin.FromDetails(st.Details()) for _, keys := range keysChanges { err := s.migrateFile(spc.(clientspace.Space), origin, keys) @@ -134,6 +143,9 @@ func (s *service) migrationQueueHandler(ctx context.Context, it *migrationItem) if err != nil { return persistentqueue.ActionDone, fmt.Errorf("get space: %w", err) } + if !space.IsPersonal() { + return persistentqueue.ActionDone, nil + } ctx = peer.CtxWithPeerId(ctx, peer.CtxResponsiblePeers) _, err = space.GetObject(ctx, it.FileObjectId) diff --git a/core/files/fileobject/migration_test.go b/core/files/fileobject/migration_test.go index c3ee57a856..f50cd07ef2 100644 --- a/core/files/fileobject/migration_test.go +++ b/core/files/fileobject/migration_test.go @@ -36,6 +36,7 @@ func TestMigrateFiles(t *testing.T) { ) space := mock_clientspace.NewMockSpace(t) spaceId := "spaceId" + space.EXPECT().IsPersonal().Return(true) space.EXPECT().Id().Return(spaceId) space.EXPECT().DeriveObjectIdWithAccountSignature(mock.Anything, mock.Anything).Return(objectId, nil) space.EXPECT().GetObject(mock.Anything, objectId).Return(nil, fmt.Errorf("not found")) @@ -86,6 +87,7 @@ func TestMigrateFiles(t *testing.T) { bb.Root(bb.ID("root")), ) space := mock_clientspace.NewMockSpace(t) + space.EXPECT().IsPersonal().Return(true) space.EXPECT().Id().Return("spaceId") space.EXPECT().DeriveObjectIdWithAccountSignature(mock.Anything, mock.Anything).Return(objectId, nil) @@ -171,6 +173,7 @@ func TestMigrateIds(t *testing.T) { ) space := mock_clientspace.NewMockSpace(t) + space.EXPECT().IsPersonal().Return(true) fx.MigrateFileIdsInBlocks(st, space) }) @@ -188,6 +191,7 @@ func TestMigrateIds(t *testing.T) { ) space := mock_clientspace.NewMockSpace(t) + space.EXPECT().IsPersonal().Return(true) fx.MigrateFileIdsInBlocks(st, space) }) @@ -205,6 +209,7 @@ func TestMigrateIds(t *testing.T) { st.SetDetailAndBundledRelation(bundle.RelationKeyAttachments, pbtypes.StringList([]string{fileId.String()})) space := mock_clientspace.NewMockSpace(t) + space.EXPECT().IsPersonal().Return(true) fx.objectStore.AddObjects(t, []objectstore.TestObject{ { @@ -253,6 +258,7 @@ func TestMigrateIds(t *testing.T) { st.SetDetailAndBundledRelation(bundle.RelationKeyAssignee, pbtypes.StringList([]string{fileId.String()})) space := mock_clientspace.NewMockSpace(t) + space.EXPECT().IsPersonal().Return(true) space.EXPECT().DeriveObjectIdWithAccountSignature(mock.Anything, mock.Anything).Return(expectedFileObjectId, nil) fx.MigrateFileIdsInBlocks(st, space) From 0ff4ae7863a704ff8ada750d632348d0eea8fbd5 Mon Sep 17 00:00:00 2001 From: Sergey Date: Wed, 14 Aug 2024 14:23:37 +0200 Subject: [PATCH 4/7] Fix file migration: move files migrated in non-personal space --- core/block/editor/files.go | 5 ++-- core/block/source/service.go | 1 + core/files/fileobject/service.go | 49 +++++++++++++++++++++++++++++++- util/debug/debug.go | 29 +++++++++++++++++++ 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/core/block/editor/files.go b/core/block/editor/files.go index 96dfa6d242..345c389c92 100644 --- a/core/block/editor/files.go +++ b/core/block/editor/files.go @@ -86,12 +86,13 @@ func (f *File) Init(ctx *smartblock.InitContext) error { f.SmartBlock.AddHook(f.reconciler.FileObjectHook(domain.FullID{SpaceID: f.SpaceID(), ObjectID: f.Id()}), smartblock.HookBeforeApply) if !ctx.IsNewObject { - err = f.fileObjectService.EnsureFileAddedToSyncQueue(domain.FullID{ObjectID: f.Id(), SpaceID: f.SpaceID()}, ctx.State.Details()) + fullId := domain.FullID{ObjectID: f.Id(), SpaceID: f.SpaceID()} + err = f.fileObjectService.EnsureFileAddedToSyncQueue(fullId, ctx.State.Details()) if err != nil { log.Errorf("failed to ensure file added to sync queue: %v", err) } f.AddHook(func(applyInfo smartblock.ApplyInfo) error { - return f.fileObjectService.EnsureFileAddedToSyncQueue(domain.FullID{ObjectID: f.Id(), SpaceID: f.SpaceID()}, applyInfo.State.Details()) + return f.fileObjectService.EnsureFileAddedToSyncQueue(fullId, applyInfo.State.Details()) }, smartblock.HookOnStateRebuild) } return nil diff --git a/core/block/source/service.go b/core/block/source/service.go index 4db5f6056b..00e78695b2 100644 --- a/core/block/source/service.go +++ b/core/block/source/service.go @@ -45,6 +45,7 @@ type Space interface { GetTypeIdByKey(ctx context.Context, key domain.TypeKey) (id string, err error) DeriveObjectID(ctx context.Context, uniqueKey domain.UniqueKey) (id string, err error) StoredIds() []string + IsPersonal() bool } type Service interface { diff --git a/core/files/fileobject/service.go b/core/files/fileobject/service.go index e6719bfc45..b4911f8e56 100644 --- a/core/files/fileobject/service.go +++ b/core/files/fileobject/service.go @@ -23,6 +23,7 @@ import ( "github.com/anyproto/anytype-heart/core/files" "github.com/anyproto/anytype-heart/core/files/fileoffloader" "github.com/anyproto/anytype-heart/core/filestorage/filesync" + "github.com/anyproto/anytype-heart/core/session" "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/bundle" @@ -80,6 +81,7 @@ type service struct { objectStore objectstore.ObjectStore spaceIdResolver idresolver.Resolver migrationQueue *persistentqueue.Queue[*migrationItem] + objectArchiver objectArchiver indexer *indexer @@ -116,6 +118,7 @@ func (s *service) Init(a *app.App) error { s.fileStore = app.MustComponent[filestore.FileStore](a) s.spaceIdResolver = app.MustComponent[idresolver.Resolver](a) s.fileOffloader = app.MustComponent[fileoffloader.Service](a) + s.objectArchiver = app.MustComponent[objectArchiver](a) cfg := app.MustComponent[configProvider](a) s.indexer = s.newIndexer() @@ -141,7 +144,11 @@ func (s *service) Init(a *app.App) error { func (s *service) Run(_ context.Context) error { go func() { - err := s.ensureNotSyncedFilesAddedToQueue() + err := s.deleteMigratedFilesInNonPersonalSpaces(context.Background()) + if err != nil { + log.Errorf("delete migrated files in non personal spaces: %v", err) + } + err = s.ensureNotSyncedFilesAddedToQueue() if err != nil { log.Errorf("ensure not synced files added to queue: %v", err) } @@ -151,6 +158,46 @@ func (s *service) Run(_ context.Context) error { return nil } +type objectArchiver interface { + SetPagesIsArchived(ctx session.Context, req pb.RpcObjectListSetIsArchivedRequest) error +} + +func (s *service) deleteMigratedFilesInNonPersonalSpaces(ctx context.Context) error { + personalSpace, err := s.spaceService.GetPersonalSpace(ctx) + if err != nil { + return err + } + + objectIds, _, err := s.objectStore.QueryObjectIDs(database.Query{ + Filters: []*model.BlockContentDataviewFilter{ + { + RelationKey: bundle.RelationKeyFileId.String(), + Condition: model.BlockContentDataviewFilter_NotEmpty, + }, + { + RelationKey: bundle.RelationKeyUniqueKey.String(), + Condition: model.BlockContentDataviewFilter_NotEmpty, + }, + { + RelationKey: bundle.RelationKeySpaceId.String(), + Condition: model.BlockContentDataviewFilter_NotEqual, + Value: pbtypes.String(personalSpace.Id()), + }, + }, + }) + if len(objectIds) > 0 { + err = s.objectArchiver.SetPagesIsArchived(nil, pb.RpcObjectListSetIsArchivedRequest{ + ObjectIds: objectIds, + IsArchived: true, + }) + if err != nil { + return err + } + } + + return nil +} + // After migrating to new sync queue we need to ensure that all not synced files are added to the queue func (s *service) ensureNotSyncedFilesAddedToQueue() error { records, err := s.objectStore.Query(database.Query{ diff --git a/util/debug/debug.go b/util/debug/debug.go index f69dbd3f19..ef991afec4 100644 --- a/util/debug/debug.go +++ b/util/debug/debug.go @@ -21,6 +21,35 @@ var ( addressPattern = regexp.MustCompile(`\+?0x[0-9a-z]*`) ) +func InlineCallStack() string { + // Allocate space for the call stack + var pcs [32]uintptr + + // Skip 3 frames: runtime.Callers, printStack, and the function calling printStack + n := runtime.Callers(2, pcs[:]) + + // Get the stack frames + frames := runtime.CallersFrames(pcs[:n]) + + var sep string + buf := &strings.Builder{} + // Iterate through the frames and print them + for { + frame, more := frames.Next() + buf.WriteString(sep) + sep = " -> " + buf.WriteString(frame.Function) + buf.WriteString(" ") + buf.WriteString(frame.File) + buf.WriteString(":") + buf.WriteString(fmt.Sprintf("%d", frame.Line)) + if !more { + break + } + } + return buf.String() +} + func ParseGoroutinesDump(trace string, pattern string) string { var sb strings.Builder From 5aeada9ea1e64db153f42f589e5be89b1a51bdfb Mon Sep 17 00:00:00 2001 From: Sergey Date: Wed, 14 Aug 2024 14:28:55 +0200 Subject: [PATCH 5/7] Fix after merge --- core/block/source/service.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/block/source/service.go b/core/block/source/service.go index 058d3c1fb9..00e78695b2 100644 --- a/core/block/source/service.go +++ b/core/block/source/service.go @@ -40,7 +40,6 @@ type accountService interface { type Space interface { Id() string - IsPersonal() bool TreeBuilder() objecttreebuilder.TreeBuilder GetRelationIdByKey(ctx context.Context, key domain.RelationKey) (id string, err error) GetTypeIdByKey(ctx context.Context, key domain.TypeKey) (id string, err error) From 9c3c6b0a627365be0e8f04601529e6255eb873ea Mon Sep 17 00:00:00 2001 From: Sergey Date: Wed, 14 Aug 2024 16:34:15 +0200 Subject: [PATCH 6/7] Fix error check --- core/files/fileobject/service.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/files/fileobject/service.go b/core/files/fileobject/service.go index 2f74d6bf6f..978986a221 100644 --- a/core/files/fileobject/service.go +++ b/core/files/fileobject/service.go @@ -192,6 +192,9 @@ func (s *service) deleteMigratedFilesInNonPersonalSpaces(ctx context.Context) er }, }, }) + if err != nil { + return err + } if len(objectIds) > 0 { err = s.objectArchiver.SetPagesIsArchived(nil, pb.RpcObjectListSetIsArchivedRequest{ ObjectIds: objectIds, From a7829d5107b5fdb3ab610ad1af1957cc2cc9878d Mon Sep 17 00:00:00 2001 From: Sergey Date: Mon, 26 Aug 2024 15:26:36 +0200 Subject: [PATCH 7/7] Fix test --- core/files/fileobject/service_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/files/fileobject/service_test.go b/core/files/fileobject/service_test.go index 659b22c579..8d22ca751e 100644 --- a/core/files/fileobject/service_test.go +++ b/core/files/fileobject/service_test.go @@ -29,6 +29,7 @@ import ( "github.com/anyproto/anytype-heart/core/filestorage" "github.com/anyproto/anytype-heart/core/filestorage/filesync" "github.com/anyproto/anytype-heart/core/filestorage/rpcstore" + "github.com/anyproto/anytype-heart/core/session" wallet2 "github.com/anyproto/anytype-heart/core/wallet" "github.com/anyproto/anytype-heart/core/wallet/mock_wallet" "github.com/anyproto/anytype-heart/pb" @@ -68,6 +69,16 @@ func (c *dummyConfig) Name() string { return "dummyConfig" } +type dummyObjectArchiver struct{} + +func (a *dummyObjectArchiver) SetPagesIsArchived(ctx session.Context, req pb.RpcObjectListSetIsArchivedRequest) error { + return nil +} + +func (a *dummyObjectArchiver) Name() string { return "dummyObjectArchiver" } + +func (a *dummyObjectArchiver) Init(_ *app.App) error { return nil } + const testResolveRetryDelay = 5 * time.Millisecond func newFixture(t *testing.T) *fixture { @@ -85,6 +96,7 @@ func newFixture(t *testing.T) *fixture { eventSender.EXPECT().Broadcast(mock.Anything).Return().Maybe() fileService := files.New() spaceService := mock_space.NewMockService(t) + spaceService.EXPECT().GetPersonalSpace(mock.Anything).Return(nil, fmt.Errorf("not needed")).Maybe() spaceIdResolver := mock_idresolver.NewMockResolver(t) svc := New(testResolveRetryDelay, testResolveRetryDelay) @@ -114,6 +126,7 @@ func newFixture(t *testing.T) *fixture { a.Register(testutil.PrepareMock(ctx, a, mock_accountservice.NewMockService(ctrl))) a.Register(testutil.PrepareMock(ctx, a, wallet)) a.Register(&config.Config{DisableFileConfig: true, NetworkMode: pb.RpcAccount_DefaultConfig, PeferYamuxTransport: true}) + a.Register(&dummyObjectArchiver{}) err = a.Start(ctx) require.NoError(t, err)