Merge pull request #1482 from anyproto/go-3273-fix-history-protocol-a…

…nd-implementation

GO-3273: fix history protocol and implementation

core/debug/service.go

@@ -165,7 +165,7 @@ func (d *debug) TreeHeads(ctx context.Context, id string) (info TreeInfo, err er
 	if err != nil {
 		return
 	}
-	tree, err := spc.TreeBuilder().BuildHistoryTree(ctx, id, objecttreebuilder.HistoryTreeOpts{})
+	tree, err := spc.TreeBuilder().BuildHistoryTree(ctx, id, objecttreebuilder.HistoryTreeOpts{Heads: []string{""}})
 	if err != nil {
 		return
 	}
@@ -187,7 +187,7 @@ func (d *debug) DumpTree(ctx context.Context, objectID string, path string, anon
 	if err != nil {
 		return
 	}
-	tree, err := spc.TreeBuilder().BuildHistoryTree(ctx, objectID, objecttreebuilder.HistoryTreeOpts{BuildFullTree: true})
+	tree, err := spc.TreeBuilder().BuildHistoryTree(ctx, objectID, objecttreebuilder.HistoryTreeOpts{})
 	if err != nil {
 		return
 	}

core/debug/treearchive/treeimporter.go

@@ -88,12 +88,15 @@ func (t *treeImporter) Import(fullTree bool, beforeId string) (err error) {
 	if err != nil {
 		return
 	}
+	var heads []string
+	if !fullTree {
+		heads = []string{beforeId}
+	}
 	t.objectTree, err = objecttree.BuildNonVerifiableHistoryTree(objecttree.HistoryTreeParams{
 		TreeStorage:     t.treeStorage,
 		AclList:         aclList,
-		BeforeId:        beforeId,
+		Heads:           heads,
 		IncludeBeforeId: true,
-		BuildFullTree:   fullTree,
 	})
 
 	return

core/history/history.go

@@ -2,15 +2,19 @@ package history
 
 import (
 	"context"
+	"encoding/hex"
 	"fmt"
 	"slices"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/anyproto/any-sync/app"
 	"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
 	"github.com/anyproto/any-sync/commonspace/objecttreebuilder"
 	"github.com/gogo/protobuf/proto"
 	"github.com/samber/lo"
+	"github.com/zeebo/blake3"
 
 	"github.com/anyproto/anytype-heart/core/block/cache"
 	smartblock2 "github.com/anyproto/anytype-heart/core/block/editor/smartblock"
@@ -37,8 +41,14 @@ const versionGroupInterval = time.Minute * 5
 
 var log = logging.Logger("anytype-mw-history")
 
+var hashersPool = &sync.Pool{
+	New: func() any {
+		return blake3.New()
+	},
+}
+
 func New() History {
-	return new(history)
+	return &history{heads: make(map[string]string, 0)}
 }
 
 type History interface {
@@ -54,6 +64,7 @@ type history struct {
 	picker       cache.ObjectGetter
 	objectStore  objectstore.ObjectStore
 	spaceService space.Service
+	heads        map[string]string
 }
 
 func (h *history) Init(a *app.App) (err error) {
@@ -121,6 +132,8 @@ func (h *history) Show(id domain.FullID, versionID string) (bs *model.ObjectView
 }
 
 func (h *history) Versions(id domain.FullID, lastVersionId string, limit int) (resp []*pb.RpcHistoryVersion, err error) {
+	hasher := hashersPool.Get().(*blake3.Hasher)
+	defer hashersPool.Put(hasher)
 	if limit <= 0 {
 		limit = 100
 	}
@@ -134,6 +147,7 @@ func (h *history) Versions(id domain.FullID, lastVersionId string, limit int) (r
 	}
 
 	for len(resp) < limit {
+		curHeads := make(map[string]struct{})
 		tree, _, e := h.treeWithId(id, lastVersionId, includeLastId)
 		if e != nil {
 			return nil, e
@@ -142,12 +156,7 @@ func (h *history) Versions(id domain.FullID, lastVersionId string, limit int) (r
 
 		e = tree.IterateFrom(tree.Root().Id, source.UnmarshalChange, func(c *objecttree.Change) (isContinue bool) {
 			participantId := domain.NewParticipantId(id.SpaceID, c.Identity.Account())
-			data = append(data, &pb.RpcHistoryVersion{
-				Id:          c.Id,
-				PreviousIds: c.PreviousIds,
-				AuthorId:    participantId,
-				Time:        c.Timestamp,
-			})
+			data = h.fillVersionData(c, curHeads, participantId, data, hasher)
 			return true
 		})
 		if e != nil {
@@ -188,6 +197,40 @@ func (h *history) Versions(id domain.FullID, lastVersionId string, limit int) (r
 	return
 }
 
+func (h *history) retrieveHeads(versionId string) []string {
+	if heads, ok := h.heads[versionId]; ok {
+		return strings.Split(heads, " ")
+	}
+	return []string{versionId}
+}
+
+func (h *history) fillVersionData(change *objecttree.Change, curHeads map[string]struct{}, participantId string, data []*pb.RpcHistoryVersion, hasher *blake3.Hasher) []*pb.RpcHistoryVersion {
+	curHeads[change.Id] = struct{}{}
+	for _, previousId := range change.PreviousIds {
+		delete(curHeads, previousId)
+	}
+	version := &pb.RpcHistoryVersion{
+		Id:          change.Id,
+		PreviousIds: change.PreviousIds,
+		AuthorId:    participantId,
+		Time:        change.Timestamp,
+	}
+	if len(curHeads) > 1 {
+		var combinedHeads string
+		for head := range curHeads {
+			combinedHeads += head + " "
+		}
+		combinedHeads = strings.TrimSpace(combinedHeads)
+		hasher.Reset()
+		// nolint: errcheck
+		hasher.Write([]byte(combinedHeads)) // it never returns an error
+		hashSum := hex.EncodeToString(hasher.Sum(nil))
+		h.heads[hashSum] = combinedHeads
+		version.Id = hashSum
+	}
+	return append(data, version)
+}
+
 func (h *history) DiffVersions(req *pb.RpcHistoryDiffVersionsRequest) ([]*pb.EventMessage, *model.ObjectView, error) {
 	id := domain.FullID{
 		ObjectID: req.ObjectId,
@@ -468,14 +511,15 @@ func (h *history) SetVersion(id domain.FullID, versionId string) (err error) {
 	})
 }
 
-func (h *history) treeWithId(id domain.FullID, beforeId string, includeBeforeId bool) (ht objecttree.HistoryTree, sbt smartblock.SmartBlockType, err error) {
+func (h *history) treeWithId(id domain.FullID, versionId string, includeBeforeId bool) (ht objecttree.HistoryTree, sbt smartblock.SmartBlockType, err error) {
+	heads := h.retrieveHeads(versionId)
 	spc, err := h.spaceService.Get(context.Background(), id.SpaceID)
 	if err != nil {
 		return
 	}
 	ht, err = spc.TreeBuilder().BuildHistoryTree(context.Background(), id.ObjectID, objecttreebuilder.HistoryTreeOpts{
-		BeforeId: beforeId,
-		Include:  includeBeforeId,
+		Heads:   heads,
+		Include: includeBeforeId,
 	})
 	if err != nil {
 		return

core/history/history_test.go

@@ -990,6 +990,111 @@ func TestHistory_Versions(t *testing.T) {
 		assert.Nil(t, err)
 		assert.Len(t, resp, 0)
 	})
+	t.Run("changes from parallel editing", func(t *testing.T) {
+		// given
+		objectId := "objectId"
+		spaceID := "spaceID"
+		versionId := "versionId"
+
+		accountKeys, _ := accountdata.NewRandom()
+		account := accountKeys.SignKey.GetPublic()
+
+		ch := &objecttree.Change{
+			Id:          "id",
+			PreviousIds: []string{"id2"},
+			Identity:    account,
+			Model: &pb.Change{
+				Content: []*pb.ChangeContent{
+					{
+						Value: &pb.ChangeContentValueOfBlockUpdate{
+							BlockUpdate: &pb.ChangeBlockUpdate{
+								Events: []*pb.EventMessage{
+									{
+										Value: &pb.EventMessageValueOfBlockSetText{
+											BlockSetText: &pb.EventBlockSetText{
+												Id: "blockId",
+												Text: &pb.EventBlockSetTextText{
+													Value: "new text",
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		ch1 := &objecttree.Change{
+			Identity:    account,
+			Id:          "id1",
+			PreviousIds: []string{"id2"},
+			Model: &pb.Change{
+				Content: []*pb.ChangeContent{
+					{
+						Value: &pb.ChangeContentValueOfBlockUpdate{
+							BlockUpdate: &pb.ChangeBlockUpdate{
+								Events: []*pb.EventMessage{
+									{
+										Value: &pb.EventMessageValueOfBlockSetText{
+											BlockSetText: &pb.EventBlockSetText{
+												Id: "blockId",
+												Text: &pb.EventBlockSetTextText{
+													Value: "some text",
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		ch2 := &objecttree.Change{
+			Id:       "id2",
+			Identity: account,
+			Model: &pb.Change{
+				Content: []*pb.ChangeContent{
+					{
+						Value: &pb.ChangeContentValueOfBlockUpdate{
+							BlockUpdate: &pb.ChangeBlockUpdate{
+								Events: []*pb.EventMessage{
+									{
+										Value: &pb.EventMessageValueOfBlockSetText{
+											BlockSetText: &pb.EventBlockSetText{
+												Id: "blockId",
+												Text: &pb.EventBlockSetTextText{
+													Value: "new text some text",
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		currChange := []*objecttree.Change{
+			ch2, ch, ch1,
+		}
+
+		history := newFixture(t, currChange, objectId, spaceID, versionId)
+
+		// when
+		resp, err := history.Versions(domain.FullID{ObjectID: objectId, SpaceID: spaceID}, versionId, 10)
+
+		// then
+		assert.Nil(t, err)
+		assert.Len(t, resp, 3)
+	})
 }
 
 type historyFixture struct {
@@ -1010,6 +1115,7 @@ func newFixture(t *testing.T, expectedChanges []*objecttree.Change, objectId, sp
 	history := &history{
 		objectStore:  objectstore.NewStoreFixture(t),
 		spaceService: spaceService,
+		heads:        map[string]string{},
 	}
 	return &historyFixture{
 		history:     history,
@@ -1035,6 +1141,7 @@ func newFixtureDiffVersions(t *testing.T,
 	history := &history{
 		objectStore:  objectstore.NewStoreFixture(t),
 		spaceService: spaceService,
+		heads:        map[string]string{},
 	}
 	return &historyFixture{
 		history:     history,
@@ -1050,8 +1157,8 @@ func configureTreeBuilder(treeBuilder *mock_objecttreebuilder.MockTreeBuilder,
 	spaceService *mock_space.MockService,
 ) {
 	treeBuilder.EXPECT().BuildHistoryTree(context.Background(), objectId, objecttreebuilder.HistoryTreeOpts{
-		BeforeId: currVersionId,
-		Include:  true,
+		Heads:   []string{currVersionId},
+		Include: true,
 	}).Return(&historyStub{
 		objectId: objectId,
 		changes:  expectedChanges,

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/VividCortex/ewma v1.2.0
 	github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786
 	github.com/anyproto/any-store v0.0.1
-	github.com/anyproto/any-sync v0.4.28
+	github.com/anyproto/any-sync v0.4.29
 	github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438
 	github.com/anyproto/tantivy-go v0.0.8
 	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
@@ -87,6 +87,7 @@ require (
 	github.com/vektra/mockery/v2 v2.42.2
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/yuin/goldmark v1.7.4
+	github.com/zeebo/blake3 v0.2.3
 	go.uber.org/atomic v1.11.0
 	go.uber.org/mock v0.4.0
 	go.uber.org/multierr v1.11.0
@@ -250,7 +251,6 @@ require (
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	github.com/zeebo/blake3 v0.2.3 // indirect
 	github.com/zeebo/errs v1.3.0 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
 	go.opencensus.io v0.24.0 // indirect

go.sum

@@ -85,8 +85,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/anyproto/any-store v0.0.1 h1:RiZi3cHVSpIebcNeHYOUaDShH7E9Ahto33zdnqCaUHM=
 github.com/anyproto/any-store v0.0.1/go.mod h1:JV7dJ8+xF1VaCQgFQsD8SZSNlxxTGFbDiF8llkB2ILc=
-github.com/anyproto/any-sync v0.4.28 h1:FM6X50oBqODGtcRIHkAK/rn0Mn48WYzH/9D2wzPYmQQ=
-github.com/anyproto/any-sync v0.4.28/go.mod h1:0SIpsBQU1t2njK9U6ue1HMdMCV5tFtSawsUe556FkkU=
+github.com/anyproto/any-sync v0.4.29 h1:zLi88h6z7818QC5s7A6bvOyAdrogbbM5GAWe30GPiRU=
+github.com/anyproto/any-sync v0.4.29/go.mod h1:0SIpsBQU1t2njK9U6ue1HMdMCV5tFtSawsUe556FkkU=
 github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580 h1:Ba80IlCCxkZ9H1GF+7vFu/TSpPvbpDCxXJ5ogc4euYc=
 github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580/go.mod h1:T/uWAYxrXdaXw64ihI++9RMbKTCpKd/yE9+saARew7k=
 github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY=