From 472f40fd4288a1602be3718866e7b433db8fcdcf Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Thu, 18 Nov 2021 23:07:28 -0600 Subject: [PATCH] Add DumpArraySlabs, DumpMapSlabs, and more tests DumpArraySlabs returns a slice of slab details, including id, size, count, elements, child headers. DumpMapSlabs returns a slice of slab details, including id, size, firstKey, elements, child headers. Array.String returns array data similar to Go's slice string representation. OrderedMap.String returns map data similar to Go's map string representation. Add more tests. --- array.go | 56 ++--- array_debug.go | 46 ++-- array_test.go | 83 ++++++- map.go | 141 +++--------- map_debug.go | 92 +++++--- map_test.go | 578 ++++++++++++++++++++++++++++++++++++++++++++++--- 6 files changed, 767 insertions(+), 229 deletions(-) diff --git a/array.go b/array.go index 67c1b7cb..f0cd9264 100644 --- a/array.go +++ b/array.go @@ -798,25 +798,17 @@ func (a *ArrayDataSlab) PopIterate(storage SlabStorage, fn ArrayPopIterationFunc } func (a *ArrayDataSlab) String() string { - var elements []Storable - if len(a.elements) <= 6 { - elements = a.elements - } else { - elements = append(elements, a.elements[:3]...) - elements = append(elements, a.elements[len(a.elements)-3:]...) - } - var elemsStr []string - for _, e := range elements { + for _, e := range a.elements { elemsStr = append(elemsStr, fmt.Sprint(e)) } - if len(a.elements) > 6 { - elemsStr = append(elemsStr, "") - copy(elemsStr[4:], elemsStr[3:]) - elemsStr[3] = "..." - } - return fmt.Sprintf("[%s]", strings.Join(elemsStr, " ")) + return fmt.Sprintf("ArrayDataSlab id:%s size:%d count:%d elements: [%s]", + a.header.id, + a.header.size, + a.header.count, + strings.Join(elemsStr, " "), + ) } func newArrayMetaDataSlabFromData( @@ -1793,7 +1785,13 @@ func (a *ArrayMetaDataSlab) String() string { for _, h := range a.childrenHeaders { elemsStr = append(elemsStr, fmt.Sprintf("{id:%s size:%d count:%d}", h.id, h.size, h.count)) } - return fmt.Sprintf("[%s]", strings.Join(elemsStr, " ")) + + return fmt.Sprintf("ArrayMetaDataSlab id:%s size:%d count:%d children: [%s]", + a.header.id, + a.header.size, + a.header.count, + strings.Join(elemsStr, " "), + ) } func NewArray(storage SlabStorage, address Address, typeInfo TypeInfo) (*Array, error) { @@ -2120,30 +2118,24 @@ func (a *Array) Type() TypeInfo { } func (a *Array) String() string { - if a.root.IsData() { - return a.root.String() + iterator, err := a.Iterator() + if err != nil { + return err.Error() } - meta := a.root.(*ArrayMetaDataSlab) - return a.string(meta) -} -func (a *Array) string(meta *ArrayMetaDataSlab) string { var elemsStr []string - - for _, h := range meta.childrenHeaders { - child, err := getArraySlab(a.Storage, h.id) + for { + v, err := iterator.Next() if err != nil { return err.Error() } - if child.IsData() { - data := child.(*ArrayDataSlab) - elemsStr = append(elemsStr, data.String()) - } else { - meta := child.(*ArrayMetaDataSlab) - elemsStr = append(elemsStr, a.string(meta)) + if v == nil { + break } + elemsStr = append(elemsStr, fmt.Sprintf("%s", v)) } - return strings.Join(elemsStr, " ") + + return fmt.Sprintf("[%s]", strings.Join(elemsStr, " ")) } func getArraySlab(storage SlabStorage, id StorageID) (ArraySlab, error) { diff --git a/array_debug.go b/array_debug.go index 28233a18..49d63e12 100644 --- a/array_debug.go +++ b/array_debug.go @@ -20,8 +20,10 @@ package atree import ( "bytes" + "errors" "fmt" "reflect" + "strings" "github.com/fxamacker/cbor/v2" ) @@ -97,6 +99,16 @@ func GetArrayStats(a *Array) (ArrayStats, error) { // PrintArray prints array slab data to stdout. func PrintArray(a *Array) { + dumps, err := DumpArraySlabs(a) + if err != nil { + fmt.Println(err) + return + } + fmt.Println(strings.Join(dumps, "\n")) +} + +func DumpArraySlabs(a *Array) ([]string, error) { + var dumps []string nextLevelIDs := []StorageID{a.StorageID()} @@ -113,20 +125,12 @@ func PrintArray(a *Array) { slab, err := getArraySlab(a.Storage, id) if err != nil { - fmt.Println(err) - return + return nil, err } if slab.IsData() { dataSlab := slab.(*ArrayDataSlab) - fmt.Printf( - "level %d leaf (id:%s size:%d count:%d next:%s): %s\n", - level+1, - dataSlab.header.id, - dataSlab.header.size, - dataSlab.header.count, - dataSlab.next, - dataSlab) + dumps = append(dumps, fmt.Sprintf("level %d, %s", level+1, dataSlab)) childStorables := dataSlab.ChildStorables() for _, e := range childStorables { @@ -137,20 +141,12 @@ func PrintArray(a *Array) { } else { meta := slab.(*ArrayMetaDataSlab) - fmt.Printf( - "level %d meta (id:%s size:%d count:%d) children: %s\n", - level+1, - meta.header.id, - meta.header.size, - meta.header.count, - meta, - ) + dumps = append(dumps, fmt.Sprintf("level %d, %s", level+1, meta)) for _, storable := range slab.ChildStorables() { id, ok := storable.(StorageIDStorable) if !ok { - fmt.Printf("metadata slab's child storables are not of type StorageIDStorable") - return + return nil, errors.New("metadata slab's child storables are not of type StorageIDStorable") } nextLevelIDs = append(nextLevelIDs, StorageID(id)) } @@ -163,15 +159,15 @@ func PrintArray(a *Array) { for _, id := range overflowIDs { slab, found, err := a.Storage.Retrieve(id) if err != nil { - fmt.Println(err.Error()) - return + return nil, err } if !found { - fmt.Printf("slab %s not found\n", id) - return + return nil, NewSlabNotFoundErrorf(id, "slab not found during array slab dump") } - fmt.Printf("overflow: (id %s) %s\n", id, slab) + dumps = append(dumps, fmt.Sprintf("overflow: %s", slab)) } + + return dumps, nil } type TypeInfoComparator func(TypeInfo, TypeInfo) bool diff --git a/array_test.go b/array_test.go index cee389d3..88f7559d 100644 --- a/array_test.go +++ b/array_test.go @@ -2164,7 +2164,7 @@ func TestArrayString(t *testing.T) { }) t.Run("large", func(t *testing.T) { - const arraySize = 190 + const arraySize = 120 typeInfo := testTypeInfo{42} storage := newTestPersistentStorage(t) @@ -2178,10 +2178,83 @@ func TestArrayString(t *testing.T) { require.NoError(t, err) } - wantArrayString := `[0 1 2 ... 51 52 53] [54 55 56 ... 97 98 99] [100 101 102 ... 187 188 189]` - require.Equal(t, wantArrayString, array.String()) + want := `[0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119]` + require.Equal(t, want, array.String()) + }) +} + +func TestArraySlabDump(t *testing.T) { + SetThreshold(256) + defer SetThreshold(1024) - wantMetaDataSlabString := `[{id:0x102030405060708.2 size:213 count:54} {id:0x102030405060708.3 size:205 count:46} {id:0x102030405060708.4 size:381 count:90}]` - require.Equal(t, wantMetaDataSlabString, array.root.String()) + t.Run("small", func(t *testing.T) { + const arraySize = 6 + + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + array, err := NewArray(storage, address, typeInfo) + require.NoError(t, err) + + for i := uint64(0); i < arraySize; i++ { + err := array.Append(Uint64Value(i)) + require.NoError(t, err) + } + + want := []string{ + "level 1, ArrayDataSlab id:0x102030405060708.1 size:23 count:6 elements: [0 1 2 3 4 5]", + } + dumps, err := DumpArraySlabs(array) + require.NoError(t, err) + require.Equal(t, want, dumps) + }) + + t.Run("large", func(t *testing.T) { + const arraySize = 120 + + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + array, err := NewArray(storage, address, typeInfo) + require.NoError(t, err) + + for i := uint64(0); i < arraySize; i++ { + err := array.Append(Uint64Value(i)) + require.NoError(t, err) + } + + want := []string{ + "level 1, ArrayMetaDataSlab id:0x102030405060708.1 size:52 count:120 children: [{id:0x102030405060708.2 size:213 count:54} {id:0x102030405060708.3 size:285 count:66}]", + "level 2, ArrayDataSlab id:0x102030405060708.2 size:213 count:54 elements: [0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53]", + "level 2, ArrayDataSlab id:0x102030405060708.3 size:285 count:66 elements: [54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119]", + } + + dumps, err := DumpArraySlabs(array) + require.NoError(t, err) + require.Equal(t, want, dumps) + }) + + t.Run("overflow", func(t *testing.T) { + + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + array, err := NewArray(storage, address, typeInfo) + require.NoError(t, err) + + err = array.Append(NewStringValue(strings.Repeat("a", int(MaxInlineArrayElementSize)))) + require.NoError(t, err) + + want := []string{ + "level 1, ArrayDataSlab id:0x102030405060708.1 size:24 count:1 elements: [StorageIDStorable({[1 2 3 4 5 6 7 8] [0 0 0 0 0 0 0 2]})]", + "overflow: &{0x102030405060708.2 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}", + } + + dumps, err := DumpArraySlabs(array) + require.NoError(t, err) + require.Equal(t, want, dumps) }) } diff --git a/map.go b/map.go index e589bec0..bb5bc612 100644 --- a/map.go +++ b/map.go @@ -788,7 +788,7 @@ func (e *inlineCollisionGroup) PopIterate(storage SlabStorage, fn MapPopIteratio } func (e *inlineCollisionGroup) String() string { - return "inline [" + e.elements.String() + "]" + return "inline[" + e.elements.String() + "]" } func newExternalCollisionGroupFromData(cborDec *cbor.StreamDecoder, decodeStorable StorableDecoder) (*externalCollisionGroup, error) { @@ -957,7 +957,7 @@ func (e *externalCollisionGroup) PopIterate(storage SlabStorage, fn MapPopIterat } func (e *externalCollisionGroup) String() string { - return fmt.Sprintf("external group(%d)", e.id) + return fmt.Sprintf("external(%s)", e.id) } func newElementsFromData(cborDec *cbor.StreamDecoder, decodeStorable StorableDecoder) (elements, error) { @@ -1640,23 +1640,8 @@ func (e *hkeyElements) PopIterate(storage SlabStorage, fn MapPopIterationFunc) e func (e *hkeyElements) String() string { var s []string - s = append(s, fmt.Sprintf("(level %v)", e.level)) - if len(e.elems) <= 6 { - for i := 0; i < len(e.elems); i++ { - s = append(s, fmt.Sprintf("%d:%s", e.hkeys[i], e.elems[i].String())) - } - return strings.Join(s, " ") - } - - for i := 0; i < 3; i++ { - s = append(s, fmt.Sprintf("%d:%s", e.hkeys[i], e.elems[i].String())) - } - - s = append(s, "...") - - elemLength := len(e.elems) - for i := elemLength - 3; i < elemLength; i++ { + for i := 0; i < len(e.elems); i++ { s = append(s, fmt.Sprintf("%d:%s", e.hkeys[i], e.elems[i].String())) } @@ -1824,65 +1809,11 @@ func (e *singleElements) Element(i int) (element, error) { } func (e *singleElements) Merge(elems elements) error { - mElems, ok := elems.(*singleElements) - if !ok { - return NewSlabMergeError(fmt.Errorf("cannot merge elements of different types (%T, %T)", e, elems)) - } - - e.elems = append(e.elems, mElems.elems...) - e.size += mElems.size - - // Set merged elements to nil to prevent memory leak - for i := 0; i < len(mElems.elems); i++ { - mElems.elems[i] = nil - } - - return nil + return NewNotApplicableError("singleElements", "elements", "Merge") } func (e *singleElements) Split() (elements, elements, error) { - - // This computes the ceil of split to give the first slab more elements. - dataSize := e.Size() - singleElementsPrefixSize - midPoint := (dataSize + 1) >> 1 - - leftSize := uint32(0) - leftCount := 0 - for i, elem := range e.elems { - elemSize := elem.Size() - if leftSize+elemSize >= midPoint { - // i is mid point element. Place i on the small side. - if leftSize <= dataSize-leftSize-elemSize { - leftSize += elemSize - leftCount = i + 1 - } else { - leftCount = i - } - break - } - // left slab size < midPoint - leftSize += elemSize - } - - rightCount := len(e.elems) - leftCount - - // Create right slab elements - rightElements := &singleElements{level: e.level} - - rightElements.elems = make([]*singleElement, rightCount) - copy(rightElements.elems, e.elems[leftCount:]) - - rightElements.size = dataSize - leftSize + singleElementsPrefixSize - - e.elems = e.elems[:leftCount] - e.size = leftSize + singleElementsPrefixSize - - // NOTE: prevent memory leak - for i := leftCount; i < len(e.elems); i++ { - e.elems[i] = nil - } - - return e, rightElements, nil + return nil, nil, NewNotApplicableError("singleElements", "elements", "Split") } func (e *singleElements) LendToRight(re elements) error { @@ -1943,23 +1874,8 @@ func (e *singleElements) PopIterate(storage SlabStorage, fn MapPopIterationFunc) func (e *singleElements) String() string { var s []string - s = append(s, fmt.Sprintf("(level %v)", e.level)) - - if len(e.elems) <= 6 { - for i := 0; i < len(e.elems); i++ { - s = append(s, fmt.Sprintf(":%s", e.elems[i].String())) - } - return strings.Join(s, " ") - } - for i := 0; i < 3; i++ { - s = append(s, fmt.Sprintf(":%s", e.elems[i].String())) - } - - s = append(s, "...") - - elemLength := len(e.elems) - for i := elemLength - 3; i < elemLength; i++ { + for i := 0; i < len(e.elems); i++ { s = append(s, fmt.Sprintf(":%s", e.elems[i].String())) } @@ -2452,7 +2368,12 @@ func (m *MapDataSlab) PopIterate(storage SlabStorage, fn MapPopIterationFunc) er } func (m *MapDataSlab) String() string { - return fmt.Sprintf("{%s}", m.elements.String()) + return fmt.Sprintf("MapDataSlab id:%s size:%d firstkey:%d elements: [%s]", + m.header.id, + m.header.size, + m.header.firstKey, + m.elements.String(), + ) } func newMapMetaDataSlabFromData( @@ -3301,11 +3222,17 @@ func (m *MapMetaDataSlab) PopIterate(storage SlabStorage, fn MapPopIterationFunc } func (m *MapMetaDataSlab) String() string { - var hStr []string + var elemsStr []string for _, h := range m.childrenHeaders { - hStr = append(hStr, fmt.Sprintf("%+v", h)) + elemsStr = append(elemsStr, fmt.Sprintf("{id:%s size:%d firstKey:%d}", h.id, h.size, h.firstKey)) } - return strings.Join(hStr, " ") + + return fmt.Sprintf("MapMetaDataSlab id:%s size:%d firstKey:%d children: [%s]", + m.header.id, + m.header.size, + m.header.firstKey, + strings.Join(elemsStr, " "), + ) } func NewMap(storage SlabStorage, address Address, digestBuilder DigesterBuilder, typeInfo TypeInfo) (*OrderedMap, error) { @@ -3612,30 +3539,24 @@ func (m *OrderedMap) Type() TypeInfo { } func (m *OrderedMap) String() string { - if m.root.IsData() { - return m.root.String() + iterator, err := m.Iterator() + if err != nil { + return err.Error() } - meta := m.root.(*MapMetaDataSlab) - return m.string(meta) -} -func (m *OrderedMap) string(meta *MapMetaDataSlab) string { var elemsStr []string - - for _, h := range meta.childrenHeaders { - child, err := getMapSlab(m.Storage, h.id) + for { + k, v, err := iterator.Next() if err != nil { return err.Error() } - if child.IsData() { - data := child.(*MapDataSlab) - elemsStr = append(elemsStr, data.String()) - } else { - meta := child.(*MapMetaDataSlab) - elemsStr = append(elemsStr, m.string(meta)) + if k == nil { + break } + elemsStr = append(elemsStr, fmt.Sprintf("%s:%s", k, v)) } - return strings.Join(elemsStr, " ") + + return fmt.Sprintf("[%s]", strings.Join(elemsStr, " ")) } func getMapSlab(storage SlabStorage, id StorageID) (MapSlab, error) { diff --git a/map_debug.go b/map_debug.go index ff0d3ad9..e7a8135f 100644 --- a/map_debug.go +++ b/map_debug.go @@ -20,10 +20,11 @@ package atree import ( "bytes" - "container/list" + "errors" "fmt" "reflect" "sort" + "strings" "github.com/fxamacker/cbor/v2" ) @@ -135,71 +136,108 @@ func GetMapStats(m *OrderedMap) (MapStats, error) { } func PrintMap(m *OrderedMap) { - nextLevelIDs := list.New() - nextLevelIDs.PushBack(m.root.Header().id) + dumps, err := DumpMapSlabs(m) + if err != nil { + fmt.Println(err) + return + } + fmt.Println(strings.Join(dumps, "\n")) +} - collisionSlabIDs := list.New() +func DumpMapSlabs(m *OrderedMap) ([]string, error) { + var dumps []string + + nextLevelIDs := []StorageID{m.StorageID()} + + var overflowIDs []StorageID + var collisionSlabIDs []StorageID level := 0 - for nextLevelIDs.Len() > 0 { + for len(nextLevelIDs) > 0 { ids := nextLevelIDs - nextLevelIDs = list.New() + nextLevelIDs = []StorageID(nil) - for e := ids.Front(); e != nil; e = e.Next() { - id := e.Value.(StorageID) + for _, id := range ids { slab, err := getMapSlab(m.Storage, id) if err != nil { - fmt.Println(err) - return + return nil, err } if slab.IsData() { dataSlab := slab.(*MapDataSlab) - fmt.Printf("level %d, leaf (%+v): %s\n", level+1, dataSlab.header, dataSlab.String()) + dumps = append(dumps, fmt.Sprintf("level %d, %s", level+1, dataSlab)) for i := 0; i < int(dataSlab.elements.Count()); i++ { elem, err := dataSlab.elements.Element(i) if err != nil { - fmt.Println(err) - return + return nil, err } if group, ok := elem.(elementGroup); ok { if !group.Inline() { extSlab := group.(*externalCollisionGroup) - collisionSlabIDs.PushBack(extSlab.id) + collisionSlabIDs = append(collisionSlabIDs, extSlab.id) } } } + childStorables := dataSlab.ChildStorables() + for _, e := range childStorables { + if id, ok := e.(StorageIDStorable); ok { + overflowIDs = append(overflowIDs, StorageID(id)) + } + } + } else { meta := slab.(*MapMetaDataSlab) - fmt.Printf("level %d, meta (%+v) headers: [", level+1, meta.header) - for _, h := range meta.childrenHeaders { - fmt.Printf("%+v ", h) - nextLevelIDs.PushBack(h.id) + dumps = append(dumps, fmt.Sprintf("level %d, %s", level+1, meta)) + + for _, storable := range slab.ChildStorables() { + id, ok := storable.(StorageIDStorable) + if !ok { + return nil, errors.New("metadata slab's child storables are not of type StorageIDStorable") + } + nextLevelIDs = append(nextLevelIDs, StorageID(id)) } - fmt.Println("]") } } level++ } - if collisionSlabIDs.Len() > 0 { - for e := collisionSlabIDs.Front(); e != nil; e = e.Next() { - id := e.Value.(StorageID) + for _, id := range collisionSlabIDs { + slab, err := getMapSlab(m.Storage, id) + if err != nil { + return nil, err + } + dumps = append(dumps, fmt.Sprintf("collision: %s", slab.String())) + } - slab, err := getMapSlab(m.Storage, id) - if err != nil { - fmt.Println(err.Error()) - return + // overflowIDs include collisionSlabIDs + for _, id := range overflowIDs { + found := false + for _, cid := range collisionSlabIDs { + if id == cid { + found = true + break } - fmt.Printf("collision slab: (id %d) %s\n", id, slab.String()) } + if found { + continue + } + slab, found, err := m.Storage.Retrieve(id) + if err != nil { + return nil, err + } + if !found { + return nil, NewSlabNotFoundErrorf(id, "slab not found during map slab dump") + } + dumps = append(dumps, fmt.Sprintf("overflow: %s", slab)) } + + return dumps, nil } func ValidMap(m *OrderedMap, typeInfo TypeInfo, tic TypeInfoComparator, hip HashInputProvider) error { diff --git a/map_test.go b/map_test.go index df5e0dfb..1356acc0 100644 --- a/map_test.go +++ b/map_test.go @@ -347,6 +347,107 @@ func TestMapSetAndGet(t *testing.T) { verifyMap(t, storage, typeInfo, address, m, keyValues, nil, false) }) + + t.Run("unique keys with hash collision", func(t *testing.T) { + + SetThreshold(256) + defer SetThreshold(1024) + + const ( + mapSize = 1024 + keyStringSize = 16 + ) + + r := newRand(t) + + digesterBuilder := &mockDigesterBuilder{} + keyValues := make(map[Value]Value, mapSize) + i := uint64(0) + for len(keyValues) < mapSize { + k := NewStringValue(randStr(r, keyStringSize)) + v := Uint64Value(i) + keyValues[k] = v + i++ + + digests := []Digest{ + Digest(i % 10), + } + digesterBuilder.On("Digest", k).Return(mockDigester{digests}) + } + + typeInfo := testTypeInfo{42} + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + storage := newTestPersistentStorage(t) + + m, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + for k, v := range keyValues { + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + verifyMap(t, storage, typeInfo, address, m, keyValues, nil, false) + }) + + t.Run("replicate keys with hash collision", func(t *testing.T) { + SetThreshold(256) + defer SetThreshold(1024) + + const ( + mapSize = 1024 + keyStringSize = 16 + ) + + r := newRand(t) + + digesterBuilder := &mockDigesterBuilder{} + keyValues := make(map[Value]Value, mapSize) + i := uint64(0) + for len(keyValues) < mapSize { + k := NewStringValue(randStr(r, keyStringSize)) + v := Uint64Value(i) + keyValues[k] = v + i++ + + digests := []Digest{ + Digest(1 % 10), + } + digesterBuilder.On("Digest", k).Return(mockDigester{digests}) + } + + typeInfo := testTypeInfo{42} + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + storage := newTestPersistentStorage(t) + + m, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + for k, v := range keyValues { + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // Overwrite values + for k, v := range keyValues { + oldValue := v.(Uint64Value) + newValue := Uint64Value(uint64(oldValue) + mapSize) + + existingStorable, err := m.Set(compare, hashInputProvider, k, newValue) + require.NoError(t, err) + require.NotNil(t, existingStorable) + + existingValue, err := existingStorable.StoredValue(storage) + require.NoError(t, err) + valueEqual(t, typeInfoComparator, oldValue, existingValue) + + keyValues[k] = newValue + } + + verifyMap(t, storage, typeInfo, address, m, keyValues, nil, false) + }) } func TestMapHas(t *testing.T) { @@ -400,6 +501,36 @@ func TestMapHas(t *testing.T) { } } +func testMapRemoveElement(t *testing.T, m *OrderedMap, k Value, expectedV Value) { + + removedKeyStorable, removedValueStorable, err := m.Remove(compare, hashInputProvider, k) + require.NoError(t, err) + + removedKey, err := removedKeyStorable.StoredValue(m.Storage) + require.NoError(t, err) + valueEqual(t, typeInfoComparator, k, removedKey) + + removedValue, err := removedValueStorable.StoredValue(m.Storage) + require.NoError(t, err) + valueEqual(t, typeInfoComparator, expectedV, removedValue) + + if id, ok := removedKeyStorable.(StorageIDStorable); ok { + err = m.Storage.Remove(StorageID(id)) + require.NoError(t, err) + } + + if id, ok := removedValueStorable.(StorageIDStorable); ok { + err = m.Storage.Remove(StorageID(id)) + require.NoError(t, err) + } + + // Remove the same key for the second time. + removedKeyStorable, removedValueStorable, err = m.Remove(compare, hashInputProvider, k) + require.Error(t, err, KeyNotFoundError{}) + require.Nil(t, removedKeyStorable) + require.Nil(t, removedValueStorable) +} + func TestMapRemove(t *testing.T) { SetThreshold(512) @@ -460,32 +591,7 @@ func TestMapRemove(t *testing.T) { // Remove all elements for k, v := range tc.keyValues { - removedKeyStorable, removedValueStorable, err := m.Remove(compare, hashInputProvider, k) - require.NoError(t, err) - - removedKey, err := removedKeyStorable.StoredValue(storage) - require.NoError(t, err) - valueEqual(t, typeInfoComparator, k, removedKey) - - removedValue, err := removedValueStorable.StoredValue(storage) - require.NoError(t, err) - valueEqual(t, typeInfoComparator, v, removedValue) - - if id, ok := removedKeyStorable.(StorageIDStorable); ok { - err = storage.Remove(StorageID(id)) - require.NoError(t, err) - } - - if id, ok := removedValueStorable.(StorageIDStorable); ok { - err = storage.Remove(StorageID(id)) - require.NoError(t, err) - } - - // Remove the same key for the second time. - removedKeyStorable, removedValueStorable, err = m.Remove(compare, hashInputProvider, k) - require.Error(t, err, KeyNotFoundError{}) - require.Nil(t, removedKeyStorable) - require.Nil(t, removedValueStorable) + testMapRemoveElement(t, m, k, v) count-- @@ -497,6 +603,187 @@ func TestMapRemove(t *testing.T) { verifyEmptyMap(t, storage, typeInfo, address, m) }) } + + t.Run("collision", func(t *testing.T) { + // Test: + // - data slab refers to an external slab containing elements with hash collision + // - last collision element is inlined after all other collision elements are removed + // - data slab overflows with inlined colllision element + // - data slab splits + + SetThreshold(512) + defer SetThreshold(1024) + + const ( + numOfElementsBeforeCollision = 54 + numOfElementsWithCollision = 10 + numOfElementsAfterCollision = 1 + ) + + digesterBuilder := &mockDigesterBuilder{} + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + r := newRand(t) + + m, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + nextDigest := Digest(0) + + nonCollisionKeyValues := make(map[Value]Value) + for i := 0; i < numOfElementsBeforeCollision; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + nonCollisionKeyValues[k] = v + + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{nextDigest}}) + nextDigest++ + + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + collisionKeyValues := make(map[Value]Value) + for len(collisionKeyValues) < numOfElementsWithCollision { + k := NewStringValue(randStr(r, int(MaxInlineMapKeyOrValueSize)-2)) + v := NewStringValue(randStr(r, int(MaxInlineMapKeyOrValueSize)-2)) + collisionKeyValues[k] = v + + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{nextDigest}}) + } + + for k, v := range collisionKeyValues { + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + nextDigest++ + k := Uint64Value(nextDigest) + v := Uint64Value(nextDigest) + nonCollisionKeyValues[k] = v + + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{nextDigest}}) + + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + + count := len(nonCollisionKeyValues) + len(collisionKeyValues) + + // Remove all collision elements + for k, v := range collisionKeyValues { + + testMapRemoveElement(t, m, k, v) + + count-- + + require.True(t, typeInfoComparator(typeInfo, m.Type())) + require.Equal(t, address, m.Address()) + require.Equal(t, uint64(count), m.Count()) + } + + verifyMap(t, storage, typeInfo, address, m, nonCollisionKeyValues, nil, false) + + // Remove remaining elements + for k, v := range nonCollisionKeyValues { + + testMapRemoveElement(t, m, k, v) + + count-- + + require.True(t, typeInfoComparator(typeInfo, m.Type())) + require.Equal(t, address, m.Address()) + require.Equal(t, uint64(count), m.Count()) + } + + verifyEmptyMap(t, storage, typeInfo, address, m) + }) + + t.Run("collision with data root", func(t *testing.T) { + // Test: + // - data slab refers to an external slab containing elements with hash collision + // - last collision element is inlined after all other collision elements are removed + // - data slab overflows with inlined colllision element + // - data slab splits + + SetThreshold(512) + defer SetThreshold(1024) + + const ( + numOfElementsWithCollision = 10 + numOfElementsWithoutCollision = 35 + ) + + digesterBuilder := &mockDigesterBuilder{} + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + r := newRand(t) + + m, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + collisionKeyValues := make(map[Value]Value) + for len(collisionKeyValues) < numOfElementsWithCollision { + k := NewStringValue(randStr(r, int(MaxInlineMapKeyOrValueSize)-2)) + v := NewStringValue(randStr(r, int(MaxInlineMapKeyOrValueSize)-2)) + collisionKeyValues[k] = v + + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{0}}) + } + + for k, v := range collisionKeyValues { + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + nonCollisionKeyValues := make(map[Value]Value) + for i := 0; i < numOfElementsWithoutCollision; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + nonCollisionKeyValues[k] = v + + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{Digest(i) + 1}}) + + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + count := len(nonCollisionKeyValues) + len(collisionKeyValues) + + // Remove all collision elements + for k, v := range collisionKeyValues { + + testMapRemoveElement(t, m, k, v) + + count-- + + require.True(t, typeInfoComparator(typeInfo, m.Type())) + require.Equal(t, address, m.Address()) + require.Equal(t, uint64(count), m.Count()) + } + + verifyMap(t, storage, typeInfo, address, m, nonCollisionKeyValues, nil, false) + + // Remove remaining elements + for k, v := range nonCollisionKeyValues { + + testMapRemoveElement(t, m, k, v) + + count-- + + require.True(t, typeInfoComparator(typeInfo, m.Type())) + require.Equal(t, address, m.Address()) + require.Equal(t, uint64(count), m.Count()) + } + + verifyEmptyMap(t, storage, typeInfo, address, m) + }) } func TestMapIterate(t *testing.T) { @@ -2246,13 +2533,32 @@ func TestMapStoredValue(t *testing.T) { require.Nil(t, existingStorable) } - value, err := m.root.StoredValue(storage) + rootID := m.StorageID() + + slabIterator, err := storage.SlabIterator() require.NoError(t, err) - m2, ok := value.(*OrderedMap) - require.True(t, ok) + for { + id, slab := slabIterator() + + if id == StorageIDUndefined { + break + } + + value, err := slab.StoredValue(storage) + + if id == rootID { + require.NoError(t, err) + + m2, ok := value.(*OrderedMap) + require.True(t, ok) - verifyMap(t, storage, typeInfo, address, m2, keyValues, nil, false) + verifyMap(t, storage, typeInfo, address, m2, keyValues, nil, false) + } else { + require.Error(t, err) + require.Nil(t, value) + } + } } func TestMapPopIterate(t *testing.T) { @@ -3125,3 +3431,215 @@ func TestMapMaxInlineElement(t *testing.T) { verifyMap(t, storage, typeInfo, address, m, keyValues, nil, false) } + +func TestMapString(t *testing.T) { + + SetThreshold(256) + defer SetThreshold(1024) + + t.Run("small", func(t *testing.T) { + const mapSize = 3 + + digesterBuilder := &mockDigesterBuilder{} + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + m, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + for i := uint64(0); i < mapSize; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{Digest(i)}}) + + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + want := `[0:0 1:1 2:2]` + require.Equal(t, want, m.String()) + }) + + t.Run("large", func(t *testing.T) { + const mapSize = 30 + + digesterBuilder := &mockDigesterBuilder{} + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + m, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + for i := uint64(0); i < mapSize; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{Digest(i)}}) + + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + want := `[0:0 1:1 2:2 3:3 4:4 5:5 6:6 7:7 8:8 9:9 10:10 11:11 12:12 13:13 14:14 15:15 16:16 17:17 18:18 19:19 20:20 21:21 22:22 23:23 24:24 25:25 26:26 27:27 28:28 29:29]` + require.Equal(t, want, m.String()) + }) +} + +func TestMapSlabDump(t *testing.T) { + + SetThreshold(256) + defer SetThreshold(1024) + + t.Run("small", func(t *testing.T) { + const mapSize = 3 + + digesterBuilder := &mockDigesterBuilder{} + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + m, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + for i := uint64(0); i < mapSize; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{Digest(i)}}) + + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + want := []string{ + "level 1, MapDataSlab id:0x102030405060708.1 size:67 firstkey:0 elements: [0:0:0 1:1:1 2:2:2]", + } + dumps, err := DumpMapSlabs(m) + require.NoError(t, err) + require.Equal(t, want, dumps) + }) + + t.Run("large", func(t *testing.T) { + const mapSize = 30 + + digesterBuilder := &mockDigesterBuilder{} + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + m, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + for i := uint64(0); i < mapSize; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{Digest(i)}}) + + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + want := []string{ + "level 1, MapMetaDataSlab id:0x102030405060708.1 size:60 firstKey:0 children: [{id:0x102030405060708.2 size:233 firstKey:0} {id:0x102030405060708.3 size:305 firstKey:13}]", + "level 2, MapDataSlab id:0x102030405060708.2 size:233 firstkey:0 elements: [0:0:0 1:1:1 2:2:2 3:3:3 4:4:4 5:5:5 6:6:6 7:7:7 8:8:8 9:9:9 10:10:10 11:11:11 12:12:12]", + "level 2, MapDataSlab id:0x102030405060708.3 size:305 firstkey:13 elements: [13:13:13 14:14:14 15:15:15 16:16:16 17:17:17 18:18:18 19:19:19 20:20:20 21:21:21 22:22:22 23:23:23 24:24:24 25:25:25 26:26:26 27:27:27 28:28:28 29:29:29]", + } + dumps, err := DumpMapSlabs(m) + require.NoError(t, err) + require.Equal(t, want, dumps) + }) + + t.Run("inline collision", func(t *testing.T) { + const mapSize = 30 + + digesterBuilder := &mockDigesterBuilder{} + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + m, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + for i := uint64(0); i < mapSize; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{Digest(i % 10)}}) + + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + want := []string{ + "level 1, MapMetaDataSlab id:0x102030405060708.1 size:60 firstKey:0 children: [{id:0x102030405060708.2 size:255 firstKey:0} {id:0x102030405060708.3 size:263 firstKey:5}]", + "level 2, MapDataSlab id:0x102030405060708.2 size:255 firstkey:0 elements: [0:inline[:0:0 :10:10 :20:20] 1:inline[:1:1 :11:11 :21:21] 2:inline[:2:2 :12:12 :22:22] 3:inline[:3:3 :13:13 :23:23] 4:inline[:4:4 :14:14 :24:24]]", + "level 2, MapDataSlab id:0x102030405060708.3 size:263 firstkey:5 elements: [5:inline[:5:5 :15:15 :25:25] 6:inline[:6:6 :16:16 :26:26] 7:inline[:7:7 :17:17 :27:27] 8:inline[:8:8 :18:18 :28:28] 9:inline[:9:9 :19:19 :29:29]]", + } + dumps, err := DumpMapSlabs(m) + require.NoError(t, err) + require.Equal(t, want, dumps) + }) + + t.Run("external collision", func(t *testing.T) { + const mapSize = 30 + + digesterBuilder := &mockDigesterBuilder{} + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + m, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + for i := uint64(0); i < mapSize; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{Digest(i % 2)}}) + + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + want := []string{ + "level 1, MapDataSlab id:0x102030405060708.1 size:80 firstkey:0 elements: [0:external(0x102030405060708.2) 1:external(0x102030405060708.3)]", + "collision: MapDataSlab id:0x102030405060708.2 size:141 firstkey:0 elements: [:0:0 :2:2 :4:4 :6:6 :8:8 :10:10 :12:12 :14:14 :16:16 :18:18 :20:20 :22:22 :24:24 :26:26 :28:28]", + "collision: MapDataSlab id:0x102030405060708.3 size:141 firstkey:0 elements: [:1:1 :3:3 :5:5 :7:7 :9:9 :11:11 :13:13 :15:15 :17:17 :19:19 :21:21 :23:23 :25:25 :27:27 :29:29]", + } + dumps, err := DumpMapSlabs(m) + require.NoError(t, err) + require.Equal(t, want, dumps) + }) + + t.Run("overflow", func(t *testing.T) { + + digesterBuilder := &mockDigesterBuilder{} + typeInfo := testTypeInfo{42} + storage := newTestPersistentStorage(t) + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + m, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + k := NewStringValue(strings.Repeat("a", int(MaxInlineMapKeyOrValueSize))) + v := NewStringValue(strings.Repeat("b", int(MaxInlineMapKeyOrValueSize))) + digesterBuilder.On("Digest", k).Return(mockDigester{d: []Digest{Digest(0)}}) + + existingStorable, err := m.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + + want := []string{ + "level 1, MapDataSlab id:0x102030405060708.1 size:69 firstkey:0 elements: [0:StorageIDStorable({[1 2 3 4 5 6 7 8] [0 0 0 0 0 0 0 2]}):StorageIDStorable({[1 2 3 4 5 6 7 8] [0 0 0 0 0 0 0 3]})]", + "overflow: &{0x102030405060708.2 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}", + "overflow: &{0x102030405060708.3 bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb}", + } + dumps, err := DumpMapSlabs(m) + require.NoError(t, err) + require.Equal(t, want, dumps) + }) +}