From 0d63aaf0e9ea7a0b471e5a3e0be16b559fb5c42f Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Tue, 21 May 2024 10:38:38 -0500 Subject: [PATCH] Check mutation of elements from readonly map iterator This commit returns ReadOnlyIteratorElementMutationError when elements of readonly map iterator are mutated. Also, a callback can be provided by the caller to log or debug such mutations with more context. As always, mutation of elements from readonly iterators are not guaranteed to persist. Instead of relying on other projects not to mutate readonly elements, this commit returns ReadOnlyIteratorElementMutationError when elements from readonly iterators are mutated. This commit also adds readonly iterator functions that receive mutation callbacks. Callbacks are useful for logging, etc. with more context when mutation occurs. Mutation handling is the same with or without callbacks. If needed, other projects using atree can choose to panic in the callback when mutation is detected. If elements from readonly iterators are mutated: - those changes are not guaranteed to persist. - mutation functions of child containers return ReadOnlyIteratorElementMutationError. - ReadOnlyMapIteratorMutationCallback are called if provided --- errors.go | 18 + map.go | 165 ++++++++- map_test.go | 881 +++++++++++++++++++++++++++++++++++++++++++++++ storable_test.go | 58 ++++ 4 files changed, 1111 insertions(+), 11 deletions(-) diff --git a/errors.go b/errors.go index 80c51359..6ea2298a 100644 --- a/errors.go +++ b/errors.go @@ -456,6 +456,24 @@ func (e *MapElementCountError) Error() string { return e.msg } +// ReadOnlyIteratorElementMutationError is the error returned when readonly iterator element is mutated. +type ReadOnlyIteratorElementMutationError struct { + containerValueID ValueID + elementValueID ValueID +} + +// NewReadOnlyIteratorElementMutationError creates ReadOnlyIteratorElementMutationError. +func NewReadOnlyIteratorElementMutationError(containerValueID, elementValueID ValueID) error { + return NewFatalError(&ReadOnlyIteratorElementMutationError{ + containerValueID: containerValueID, + elementValueID: elementValueID, + }) +} + +func (e *ReadOnlyIteratorElementMutationError) Error() string { + return fmt.Sprintf("element (%s) cannot be mutated because it is from readonly iterator of container (%s)", e.elementValueID, e.containerValueID) +} + func wrapErrorAsExternalErrorIfNeeded(err error) error { return wrapErrorfAsExternalErrorIfNeeded(err, "") } diff --git a/map.go b/map.go index b89474cd..e4531e10 100644 --- a/map.go +++ b/map.go @@ -5746,14 +5746,37 @@ func (i *mutableMapIterator) NextValue() (Value, error) { return v, nil } +type ReadOnlyMapIteratorMutationCallback func(mutatedValue Value) + type readOnlyMapIterator struct { - m *OrderedMap - nextDataSlabID SlabID - elemIterator *mapElementIterator + m *OrderedMap + nextDataSlabID SlabID + elemIterator *mapElementIterator + keyMutationCallback ReadOnlyMapIteratorMutationCallback + valueMutationCallback ReadOnlyMapIteratorMutationCallback } +// defaultReadOnlyMapIteratorMutatinCallback is no-op. +var defaultReadOnlyMapIteratorMutatinCallback ReadOnlyMapIteratorMutationCallback = func(Value) {} + var _ MapIterator = &readOnlyMapIterator{} +func (i *readOnlyMapIterator) setMutationCallback(key, value Value) { + if k, ok := key.(mutableValueNotifier); ok { + k.setParentUpdater(func() (found bool, err error) { + i.keyMutationCallback(key) + return true, NewReadOnlyIteratorElementMutationError(i.m.ValueID(), k.ValueID()) + }) + } + + if v, ok := value.(mutableValueNotifier); ok { + v.setParentUpdater(func() (found bool, err error) { + i.valueMutationCallback(value) + return true, NewReadOnlyIteratorElementMutationError(i.m.ValueID(), v.ValueID()) + }) + } +} + func (i *readOnlyMapIterator) Next() (key Value, value Value, err error) { if i.elemIterator == nil { if i.nextDataSlabID == SlabIDUndefined { @@ -5786,6 +5809,8 @@ func (i *readOnlyMapIterator) Next() (key Value, value Value, err error) { return nil, nil, wrapErrorfAsExternalErrorIfNeeded(err, "failed to get map value's stored value") } + i.setMutationCallback(key, value) + return key, value, nil } @@ -5821,6 +5846,8 @@ func (i *readOnlyMapIterator) NextKey() (key Value, err error) { return nil, wrapErrorfAsExternalErrorIfNeeded(err, "failed to get map key's stored value") } + i.setMutationCallback(key, nil) + return key, nil } @@ -5856,6 +5883,8 @@ func (i *readOnlyMapIterator) NextValue() (value Value, err error) { return nil, wrapErrorfAsExternalErrorIfNeeded(err, "failed to get map value's stored value") } + i.setMutationCallback(nil, value) + return value, nil } @@ -5932,9 +5961,31 @@ func (m *OrderedMap) Iterator(comparator ValueComparator, hip HashInputProvider) } // ReadOnlyIterator returns readonly iterator for map elements. -// If elements are mutated, those changes are not guaranteed to persist. -// NOTE: Use readonly iterator if mutation is not needed for better performance. +// If elements are mutated: +// - those changes are not guaranteed to persist. +// - mutation functions of child containers return ReadOnlyIteratorElementMutationError. +// NOTE: +// Use readonly iterator if mutation is not needed for better performance. +// If callback is needed (e.g. for logging mutation, etc.), use ReadOnlyIteratorWithMutationCallback(). func (m *OrderedMap) ReadOnlyIterator() (MapIterator, error) { + return m.ReadOnlyIteratorWithMutationCallback(nil, nil) +} + +// ReadOnlyIteratorWithMutationCallback returns readonly iterator for map elements. +// keyMutatinCallback and valueMutationCallback are useful for logging, etc. with +// more context when mutation occurs. Mutation handling here is the same with or +// without these callbacks. +// If elements are mutated: +// - those changes are not guaranteed to persist. +// - mutation functions of child containers return ReadOnlyIteratorElementMutationError. +// - keyMutatinCallback and valueMutationCallback are called if provided +// NOTE: +// Use readonly iterator if mutation is not needed for better performance. +// If callback isn't needed, use ReadOnlyIterator(). +func (m *OrderedMap) ReadOnlyIteratorWithMutationCallback( + keyMutatinCallback ReadOnlyMapIteratorMutationCallback, + valueMutationCallback ReadOnlyMapIteratorMutationCallback, +) (MapIterator, error) { if m.Count() == 0 { return emptyReadOnlyMapIterator, nil } @@ -5945,6 +5996,14 @@ func (m *OrderedMap) ReadOnlyIterator() (MapIterator, error) { return nil, err } + if keyMutatinCallback == nil { + keyMutatinCallback = defaultReadOnlyMapIteratorMutatinCallback + } + + if valueMutationCallback == nil { + valueMutationCallback = defaultReadOnlyMapIteratorMutatinCallback + } + return &readOnlyMapIterator{ m: m, nextDataSlabID: dataSlab.next, @@ -5952,6 +6011,8 @@ func (m *OrderedMap) ReadOnlyIterator() (MapIterator, error) { storage: m.Storage, elements: dataSlab.elements, }, + keyMutationCallback: keyMutatinCallback, + valueMutationCallback: valueMutationCallback, }, nil } @@ -5987,8 +6048,36 @@ func (m *OrderedMap) Iterate(comparator ValueComparator, hip HashInputProvider, return iterateMap(iterator, fn) } -func (m *OrderedMap) IterateReadOnly(fn MapEntryIterationFunc) error { - iterator, err := m.ReadOnlyIterator() +// IterateReadOnly iterates readonly map elements. +// If elements are mutated: +// - those changes are not guaranteed to persist. +// - mutation functions of child containers return ReadOnlyIteratorElementMutationError. +// NOTE: +// Use readonly iterator if mutation is not needed for better performance. +// If callback is needed (e.g. for logging mutation, etc.), use IterateReadOnlyWithMutationCallback(). +func (m *OrderedMap) IterateReadOnly( + fn MapEntryIterationFunc, +) error { + return m.IterateReadOnlyWithMutationCallback(fn, nil, nil) +} + +// IterateReadOnlyWithMutationCallback iterates readonly map elements. +// keyMutatinCallback and valueMutationCallback are useful for logging, etc. with +// more context when mutation occurs. Mutation handling here is the same with or +// without these callbacks. +// If elements are mutated: +// - those changes are not guaranteed to persist. +// - mutation functions of child containers return ReadOnlyIteratorElementMutationError. +// - keyMutatinCallback/valueMutationCallback is called if provided +// NOTE: +// Use readonly iterator if mutation is not needed for better performance. +// If callback isn't needed, use IterateReadOnly(). +func (m *OrderedMap) IterateReadOnlyWithMutationCallback( + fn MapEntryIterationFunc, + keyMutatinCallback ReadOnlyMapIteratorMutationCallback, + valueMutationCallback ReadOnlyMapIteratorMutationCallback, +) error { + iterator, err := m.ReadOnlyIteratorWithMutationCallback(keyMutatinCallback, valueMutationCallback) if err != nil { // Don't need to wrap error as external error because err is already categorized by OrderedMap.ReadOnlyIterator(). return err @@ -6028,8 +6117,35 @@ func (m *OrderedMap) IterateKeys(comparator ValueComparator, hip HashInputProvid return iterateMapKeys(iterator, fn) } -func (m *OrderedMap) IterateReadOnlyKeys(fn MapElementIterationFunc) error { - iterator, err := m.ReadOnlyIterator() +// IterateReadOnlyKeys iterates readonly map keys. +// If keys are mutated: +// - those changes are not guaranteed to persist. +// - mutation functions of key containers return ReadOnlyIteratorElementMutationError. +// NOTE: +// Use readonly iterator if mutation is not needed for better performance. +// If callback is needed (e.g. for logging mutation, etc.), use IterateReadOnlyKeysWithMutationCallback(). +func (m *OrderedMap) IterateReadOnlyKeys( + fn MapElementIterationFunc, +) error { + return m.IterateReadOnlyKeysWithMutationCallback(fn, nil) +} + +// IterateReadOnlyKeysWithMutationCallback iterates readonly map keys. +// keyMutatinCallback is useful for logging, etc. with more context +// when mutation occurs. Mutation handling here is the same with or +// without this callback. +// If keys are mutated: +// - those changes are not guaranteed to persist. +// - mutation functions of key containers return ReadOnlyIteratorElementMutationError. +// - keyMutatinCallback is called if provided +// NOTE: +// Use readonly iterator if mutation is not needed for better performance. +// If callback isn't needed, use IterateReadOnlyKeys(). +func (m *OrderedMap) IterateReadOnlyKeysWithMutationCallback( + fn MapElementIterationFunc, + keyMutatinCallback ReadOnlyMapIteratorMutationCallback, +) error { + iterator, err := m.ReadOnlyIteratorWithMutationCallback(keyMutatinCallback, nil) if err != nil { // Don't need to wrap error as external error because err is already categorized by OrderedMap.ReadOnlyIterator(). return err @@ -6069,8 +6185,35 @@ func (m *OrderedMap) IterateValues(comparator ValueComparator, hip HashInputProv return iterateMapValues(iterator, fn) } -func (m *OrderedMap) IterateReadOnlyValues(fn MapElementIterationFunc) error { - iterator, err := m.ReadOnlyIterator() +// IterateReadOnlyValues iterates readonly map values. +// If values are mutated: +// - those changes are not guaranteed to persist. +// - mutation functions of child containers return ReadOnlyIteratorElementMutationError. +// NOTE: +// Use readonly iterator if mutation is not needed for better performance. +// If callback is needed (e.g. for logging mutation, etc.), use IterateReadOnlyValuesWithMutationCallback(). +func (m *OrderedMap) IterateReadOnlyValues( + fn MapElementIterationFunc, +) error { + return m.IterateReadOnlyValuesWithMutationCallback(fn, nil) +} + +// IterateReadOnlyValuesWithMutationCallback iterates readonly map values. +// valueMutationCallback is useful for logging, etc. with more context +// when mutation occurs. Mutation handling here is the same with or +// without this callback. +// If values are mutated: +// - those changes are not guaranteed to persist. +// - mutation functions of child containers return ReadOnlyIteratorElementMutationError. +// - keyMutatinCallback is called if provided +// NOTE: +// Use readonly iterator if mutation is not needed for better performance. +// If callback isn't needed, use IterateReadOnlyValues(). +func (m *OrderedMap) IterateReadOnlyValuesWithMutationCallback( + fn MapElementIterationFunc, + valueMutationCallback ReadOnlyMapIteratorMutationCallback, +) error { + iterator, err := m.ReadOnlyIteratorWithMutationCallback(nil, valueMutationCallback) if err != nil { // Don't need to wrap error as external error because err is already categorized by OrderedMap.ReadOnlyIterator(). return err diff --git a/map_test.go b/map_test.go index 344ccdd2..f5361205 100644 --- a/map_test.go +++ b/map_test.go @@ -1306,6 +1306,887 @@ func TestReadOnlyMapIterate(t *testing.T) { }) } +func TestMutateElementFromReadOnlyMapIterator(t *testing.T) { + + SetThreshold(256) + defer SetThreshold(1024) + + typeInfo := testTypeInfo{42} + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + storage := newTestPersistentStorage(t) + digesterBuilder := newBasicDigesterBuilder() + + var mutationError *ReadOnlyIteratorElementMutationError + + t.Run("mutate inlined map key from IterateReadOnly", func(t *testing.T) { + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // child map key {} + childMapKey, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + require.False(t, childMapKey.Inlined()) + + // parent map {{}: 0} + existingStorable, err := parentMap.Set(compare, hashInputProvider, NewHashableMap(childMapKey), Uint64Value(0)) + require.NoError(t, err) + require.Nil(t, existingStorable) + require.True(t, childMapKey.Inlined()) + + // Iterate elements and modify key + var keyMutationCallbackCalled, valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyWithMutationCallback( + func(k Value, v Value) (resume bool, err error) { + c, ok := k.(*OrderedMap) + require.True(t, ok) + require.True(t, c.Inlined()) + + existingStorable, err := c.Set(compare, hashInputProvider, Uint64Value(0), Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingStorable) + + return true, err + }, + func(k Value) { + keyMutationCallbackCalled = true + require.Equal(t, childMapKey.ValueID(), k.(mutableValueNotifier).ValueID()) + }, + func(v Value) { + valueMutationCallbackCalled = true + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, keyMutationCallbackCalled) + require.False(t, valueMutationCallbackCalled) + }) + + t.Run("mutate inlined map value from IterateReadOnly", func(t *testing.T) { + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // child map {} + childMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + require.False(t, childMap.Inlined()) + + // parent map {0: {}} + existingStorable, err := parentMap.Set(compare, hashInputProvider, Uint64Value(0), childMap) + require.NoError(t, err) + require.Nil(t, existingStorable) + require.True(t, childMap.Inlined()) + + // Iterate elements and modify value + var keyMutationCallbackCalled, valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyWithMutationCallback( + func(k Value, v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.True(t, c.Inlined()) + + existingStorable, err := c.Set(compare, hashInputProvider, Uint64Value(0), Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingStorable) + + return true, err + }, + func(k Value) { + keyMutationCallbackCalled = true + }, + func(v Value) { + valueMutationCallbackCalled = true + require.Equal(t, childMap.ValueID(), v.(mutableValueNotifier).ValueID()) + }) + + require.ErrorAs(t, err, &mutationError) + require.False(t, keyMutationCallbackCalled) + require.True(t, valueMutationCallbackCalled) + }) + + t.Run("mutate inlined map key from IterateReadOnlyKeys", func(t *testing.T) { + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // child map key {} + childMapKey, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + require.False(t, childMapKey.Inlined()) + + // parent map {{}: 0} + existingStorable, err := parentMap.Set(compare, hashInputProvider, NewHashableMap(childMapKey), Uint64Value(0)) + require.NoError(t, err) + require.Nil(t, existingStorable) + require.True(t, childMapKey.Inlined()) + + // Iterate and modify key + var keyMutationCallbackCalled bool + err = parentMap.IterateReadOnlyKeysWithMutationCallback( + func(v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.True(t, c.Inlined()) + + existingStorable, err := c.Set(compare, hashInputProvider, Uint64Value(0), Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingStorable) + + return true, err + }, + func(v Value) { + keyMutationCallbackCalled = true + require.Equal(t, childMapKey.ValueID(), v.(mutableValueNotifier).ValueID()) + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, keyMutationCallbackCalled) + }) + + t.Run("mutate inlined map value from IterateReadOnlyValues", func(t *testing.T) { + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // child map {} + childMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + require.False(t, childMap.Inlined()) + + // parent map {0: {}} + existingStorable, err := parentMap.Set(compare, hashInputProvider, Uint64Value(0), childMap) + require.NoError(t, err) + require.Nil(t, existingStorable) + require.True(t, childMap.Inlined()) + + // Iterate and modify value + var valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyValuesWithMutationCallback( + func(v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.True(t, c.Inlined()) + + existingStorable, err := c.Set(compare, hashInputProvider, Uint64Value(1), Uint64Value(1)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingStorable) + + return true, err + }, + func(v Value) { + valueMutationCallbackCalled = true + require.Equal(t, childMap.ValueID(), v.(mutableValueNotifier).ValueID()) + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, valueMutationCallbackCalled) + }) + + t.Run("mutate not inlined map key from IterateReadOnly", func(t *testing.T) { + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // child map key {} + childMapKey, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + require.False(t, childMapKey.Inlined()) + + // Inserting elements into childMapKey so it can't be inlined + const size = 20 + for i := 0; i < size; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + existingStorable, err := childMapKey.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // parent map {{...}: 0} + existingStorable, err := parentMap.Set(compare, hashInputProvider, NewHashableMap(childMapKey), Uint64Value(0)) + require.NoError(t, err) + require.Nil(t, existingStorable) + require.False(t, childMapKey.Inlined()) + + // Iterate elements and modify key + var keyMutationCallbackCalled, valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyWithMutationCallback( + func(k Value, v Value) (resume bool, err error) { + c, ok := k.(*OrderedMap) + require.True(t, ok) + require.False(t, c.Inlined()) + + existingKeyStorable, existingValueStorable, err := c.Remove(compare, hashInputProvider, Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingKeyStorable) + require.Nil(t, existingValueStorable) + + return true, err + }, + func(k Value) { + keyMutationCallbackCalled = true + require.Equal(t, childMapKey.ValueID(), k.(mutableValueNotifier).ValueID()) + }, + func(v Value) { + valueMutationCallbackCalled = true + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, keyMutationCallbackCalled) + require.False(t, valueMutationCallbackCalled) + }) + + t.Run("mutate not inlined map value from IterateReadOnly", func(t *testing.T) { + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // child map {} + childMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + require.False(t, childMap.Inlined()) + + // parent map {0: {}} + existingStorable, err := parentMap.Set(compare, hashInputProvider, Uint64Value(0), childMap) + require.NoError(t, err) + require.Nil(t, existingStorable) + require.True(t, childMap.Inlined()) + + // Inserting elements into childMap until it is no longer inlined + for i := 0; childMap.Inlined(); i++ { + k := Uint64Value(i) + v := Uint64Value(i) + existingStorable, err := childMap.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // Iterate elements and modify value + var keyMutationCallbackCalled, valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyWithMutationCallback( + func(k Value, v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.False(t, c.Inlined()) + + existingKeyStorable, existingValueStorable, err := c.Remove(compare, hashInputProvider, Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingKeyStorable) + require.Nil(t, existingValueStorable) + + return true, err + }, + func(k Value) { + keyMutationCallbackCalled = true + }, + func(v Value) { + valueMutationCallbackCalled = true + require.Equal(t, childMap.ValueID(), v.(mutableValueNotifier).ValueID()) + }) + + require.ErrorAs(t, err, &mutationError) + require.False(t, keyMutationCallbackCalled) + require.True(t, valueMutationCallbackCalled) + }) + + t.Run("mutate not inlined map key from IterateReadOnlyKeys", func(t *testing.T) { + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // child map key {} + childMapKey, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + require.False(t, childMapKey.Inlined()) + + // Inserting elements into childMap so it can't be inlined. + const size = 20 + for i := 0; i < size; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + existingStorable, err := childMapKey.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // parent map {{...}: 0} + existingStorable, err := parentMap.Set(compare, hashInputProvider, NewHashableMap(childMapKey), Uint64Value(0)) + require.NoError(t, err) + require.Nil(t, existingStorable) + require.False(t, childMapKey.Inlined()) + + // Iterate and modify key + var keyMutationCallbackCalled bool + err = parentMap.IterateReadOnlyKeysWithMutationCallback( + func(v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.False(t, c.Inlined()) + + existingKeyStorable, existingValueStorable, err := c.Remove(compare, hashInputProvider, Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingKeyStorable) + require.Nil(t, existingValueStorable) + + return true, err + }, + func(v Value) { + keyMutationCallbackCalled = true + require.Equal(t, childMapKey.ValueID(), v.(mutableValueNotifier).ValueID()) + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, keyMutationCallbackCalled) + }) + + t.Run("mutate not inlined map value from IterateReadOnlyValues", func(t *testing.T) { + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // child map {} + childMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + require.False(t, childMap.Inlined()) + + // parent map {0: {}} + existingStorable, err := parentMap.Set(compare, hashInputProvider, Uint64Value(0), childMap) + require.NoError(t, err) + require.Nil(t, existingStorable) + require.True(t, childMap.Inlined()) + + // Inserting elements into childMap until it is no longer inlined + for i := 0; childMap.Inlined(); i++ { + k := Uint64Value(i) + v := Uint64Value(i) + existingStorable, err := childMap.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // Iterate and modify value + var valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyValuesWithMutationCallback( + func(v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.False(t, c.Inlined()) + + existingKeyStorable, existingValueStorable, err := c.Remove(compare, hashInputProvider, Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingKeyStorable) + require.Nil(t, existingValueStorable) + + return true, err + }, + func(v Value) { + valueMutationCallbackCalled = true + require.Equal(t, childMap.ValueID(), v.(mutableValueNotifier).ValueID()) + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, valueMutationCallbackCalled) + }) + + t.Run("mutate inlined map key in collision from IterateReadOnly", func(t *testing.T) { + digesterBuilder := &mockDigesterBuilder{} + + // childMapKey1 {} + childMapKey1, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + // childMapKey2 {} + childMapKey2, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // parentMap {{}:0, {}:1} with all elements in the same collision group + for i, m := range []*OrderedMap{childMapKey1, childMapKey2} { + k := NewHashableMap(m) + v := Uint64Value(i) + + digests := []Digest{Digest(0)} + digesterBuilder.On("Digest", k).Return(mockDigester{digests}) + // This is needed because Digest is called again on OrderedMap when inserting collision element. + digesterBuilder.On("Digest", m).Return(mockDigester{digests}) + + existingStorable, err := parentMap.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // Iterate element and modify key + var keyMutationCallbackCalled, valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyWithMutationCallback( + func(k Value, v Value) (resume bool, err error) { + c, ok := k.(*OrderedMap) + require.True(t, ok) + require.True(t, c.Inlined()) + + existingStorable, err := c.Set(compare, hashInputProvider, Uint64Value(0), Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingStorable) + + return true, err + }, + func(k Value) { + keyMutationCallbackCalled = true + vid := k.(mutableValueNotifier).ValueID() + require.True(t, childMapKey1.ValueID() == vid || childMapKey2.ValueID() == vid) + }, + func(v Value) { + valueMutationCallbackCalled = true + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, keyMutationCallbackCalled) + require.False(t, valueMutationCallbackCalled) + }) + + t.Run("mutate inlined map value in collision from IterateReadOnly", func(t *testing.T) { + digesterBuilder := &mockDigesterBuilder{} + + // childMap1 {} + childMap1, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + // childMap2 {} + childMap2, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // parentMap {0: {}, 1:{}} with all elements in the same collision group + for i, m := range []*OrderedMap{childMap1, childMap2} { + k := Uint64Value(i) + + digests := []Digest{Digest(0)} + digesterBuilder.On("Digest", k).Return(mockDigester{digests}) + + existingStorable, err := parentMap.Set(compare, hashInputProvider, k, m) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // Iterate elements and modify values + var keyMutationCallbackCalled, valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyWithMutationCallback( + func(k Value, v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.True(t, c.Inlined()) + + existingStorable, err := c.Set(compare, hashInputProvider, Uint64Value(1), Uint64Value(1)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingStorable) + + return true, err + }, + func(k Value) { + keyMutationCallbackCalled = true + }, + func(v Value) { + valueMutationCallbackCalled = true + vid := v.(mutableValueNotifier).ValueID() + require.True(t, childMap1.ValueID() == vid || childMap2.ValueID() == vid) + }) + + require.ErrorAs(t, err, &mutationError) + require.False(t, keyMutationCallbackCalled) + require.True(t, valueMutationCallbackCalled) + }) + + t.Run("mutate inlined map key in collision from IterateReadOnlyKeys", func(t *testing.T) { + digesterBuilder := &mockDigesterBuilder{} + + // childMapKey1 {} + childMapKey1, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + // childMapKey2 {} + childMapKey2, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // parentMap {{}: 0, {}: 1} with all elements in the same collision group + for i, m := range []*OrderedMap{childMapKey1, childMapKey2} { + k := NewHashableMap(m) + v := Uint64Value(i) + + digests := []Digest{Digest(0)} + digesterBuilder.On("Digest", k).Return(mockDigester{digests}) + // This is needed because Digest is called again on OrderedMap when inserting collision element. + digesterBuilder.On("Digest", m).Return(mockDigester{digests}) + + existingStorable, err := parentMap.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // Iterate and modify keys + var keyMutationCallbackCalled bool + err = parentMap.IterateReadOnlyKeysWithMutationCallback( + func(v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.True(t, c.Inlined()) + + existingStorable, err := c.Set(compare, hashInputProvider, Uint64Value(0), Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingStorable) + + return true, err + }, + func(v Value) { + keyMutationCallbackCalled = true + vid := v.(mutableValueNotifier).ValueID() + require.True(t, childMapKey1.ValueID() == vid || childMapKey2.ValueID() == vid) + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, keyMutationCallbackCalled) + }) + + t.Run("mutate inlined map value in collision from IterateReadOnlyValues", func(t *testing.T) { + digesterBuilder := &mockDigesterBuilder{} + + // childMap1 {} + childMap1, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + // childMap2 {} + childMap2, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // parentMap {0: {}, 1:{}} with all elements in the same collision group + for i, m := range []*OrderedMap{childMap1, childMap2} { + k := Uint64Value(i) + + digests := []Digest{Digest(0)} + digesterBuilder.On("Digest", k).Return(mockDigester{digests}) + + existingStorable, err := parentMap.Set(compare, hashInputProvider, k, m) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // Iterate and modify values + var valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyValuesWithMutationCallback( + func(v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.True(t, c.Inlined()) + + existingStorable, err := c.Set(compare, hashInputProvider, Uint64Value(0), Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingStorable) + + return true, err + }, + func(v Value) { + valueMutationCallbackCalled = true + vid := v.(mutableValueNotifier).ValueID() + require.True(t, childMap1.ValueID() == vid || childMap2.ValueID() == vid) + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, valueMutationCallbackCalled) + }) + + t.Run("mutate not inlined map key in collision from IterateReadOnly", func(t *testing.T) { + digesterBuilder := &mockDigesterBuilder{} + + // childMapKey1 {} + childMapKey1, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + const size = 20 + for i := 0; i < size; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + + existingStorable, err := childMapKey1.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // childMapKey2 {} + childMapKey2, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + for i := 0; i < size; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + + existingStorable, err := childMapKey2.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // parentMap {0: {}, 1:{}} with all elements in the same collision group + for i, m := range []*OrderedMap{childMapKey1, childMapKey2} { + k := NewHashableMap(m) + v := Uint64Value(i) + + digests := []Digest{Digest(0)} + digesterBuilder.On("Digest", k).Return(mockDigester{digests}) + // This is needed because Digest is called again on OrderedMap when inserting collision element. + digesterBuilder.On("Digest", m).Return(mockDigester{digests}) + + existingStorable, err := parentMap.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // Iterate elements and modify keys + var keyMutationCallbackCalled, valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyWithMutationCallback( + func(k Value, v Value) (resume bool, err error) { + c, ok := k.(*OrderedMap) + require.True(t, ok) + require.False(t, c.Inlined()) + + existingKeyStorable, existingValueStorable, err := c.Remove(compare, hashInputProvider, Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingKeyStorable) + require.Nil(t, existingValueStorable) + + return true, err + }, + func(k Value) { + keyMutationCallbackCalled = true + vid := k.(mutableValueNotifier).ValueID() + require.True(t, childMapKey1.ValueID() == vid || childMapKey2.ValueID() == vid) + }, + func(v Value) { + valueMutationCallbackCalled = true + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, keyMutationCallbackCalled) + require.False(t, valueMutationCallbackCalled) + }) + + t.Run("mutate not inlined map value in collision from IterateReadOnly", func(t *testing.T) { + digesterBuilder := &mockDigesterBuilder{} + + // childMap1 {} + childMap1, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + // childMap2 {} + childMap2, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // parentMap {0: {}, 1:{}} with all elements in the same collision group + for i, m := range []*OrderedMap{childMap1, childMap2} { + k := Uint64Value(i) + + digests := []Digest{Digest(0)} + digesterBuilder.On("Digest", k).Return(mockDigester{digests}) + + existingStorable, err := parentMap.Set(compare, hashInputProvider, k, m) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + for i := 0; childMap1.Inlined(); i++ { + k := Uint64Value(i) + v := Uint64Value(i) + + existingStorable, err := childMap1.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + for i := 0; childMap2.Inlined(); i++ { + k := Uint64Value(i) + v := Uint64Value(i) + + existingStorable, err := childMap2.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // Iterate elements and modify values + var keyMutationCallbackCalled, valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyWithMutationCallback( + func(k Value, v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.False(t, c.Inlined()) + + existingKeyStorable, existingValueStorable, err := c.Remove(compare, hashInputProvider, Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingKeyStorable) + require.Nil(t, existingValueStorable) + + return true, err + }, + func(k Value) { + keyMutationCallbackCalled = true + }, + func(v Value) { + valueMutationCallbackCalled = true + vid := v.(mutableValueNotifier).ValueID() + require.True(t, childMap1.ValueID() == vid || childMap2.ValueID() == vid) + }) + + require.ErrorAs(t, err, &mutationError) + require.False(t, keyMutationCallbackCalled) + require.True(t, valueMutationCallbackCalled) + }) + + t.Run("mutate not inlined map key in collision from IterateReadOnlyKeys", func(t *testing.T) { + digesterBuilder := &mockDigesterBuilder{} + + // childMapKey1 {} + childMapKey1, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + size := 20 + for i := 0; i < size; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + + existingStorable, err := childMapKey1.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // childMapKey2 {} + childMapKey2, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + for i := 0; i < size; i++ { + k := Uint64Value(i) + v := Uint64Value(i) + + existingStorable, err := childMapKey2.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // parentMap {0: {}, 1:{}} with all elements in the same collision group + for i, m := range []*OrderedMap{childMapKey1, childMapKey2} { + k := NewHashableMap(m) + v := Uint64Value(i) + + digests := []Digest{Digest(0)} + digesterBuilder.On("Digest", k).Return(mockDigester{digests}) + // This is needed because Digest is called again on OrderedMap when inserting collision element. + digesterBuilder.On("Digest", m).Return(mockDigester{digests}) + + existingStorable, err := parentMap.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // Iterate and modify keys + var keyMutationCallbackCalled bool + err = parentMap.IterateReadOnlyKeysWithMutationCallback( + func(v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.False(t, c.Inlined()) + + existingKeyStorable, existingValueStorable, err := c.Remove(compare, hashInputProvider, Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingKeyStorable) + require.Nil(t, existingValueStorable) + + return true, err + }, + func(v Value) { + keyMutationCallbackCalled = true + vid := v.(mutableValueNotifier).ValueID() + require.True(t, childMapKey1.ValueID() == vid || childMapKey2.ValueID() == vid) + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, keyMutationCallbackCalled) + }) + + t.Run("mutate not inlined map value in collision from IterateReadOnlyValues", func(t *testing.T) { + digesterBuilder := &mockDigesterBuilder{} + + // childMap1 {} + childMap1, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + // childMap2 {} + childMap2, err := NewMap(storage, address, NewDefaultDigesterBuilder(), typeInfo) + require.NoError(t, err) + + parentMap, err := NewMap(storage, address, digesterBuilder, typeInfo) + require.NoError(t, err) + + // parentMap {0: {}, 1:{}} with all elements in the same collision group + for i, m := range []*OrderedMap{childMap1, childMap2} { + k := Uint64Value(i) + + digests := []Digest{Digest(0)} + digesterBuilder.On("Digest", k).Return(mockDigester{digests}) + + existingStorable, err := parentMap.Set(compare, hashInputProvider, k, m) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + for i := 0; childMap1.Inlined(); i++ { + k := Uint64Value(i) + v := Uint64Value(i) + + existingStorable, err := childMap1.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + for i := 0; childMap2.Inlined(); i++ { + k := Uint64Value(i) + v := Uint64Value(i) + + existingStorable, err := childMap2.Set(compare, hashInputProvider, k, v) + require.NoError(t, err) + require.Nil(t, existingStorable) + } + + // Iterate and modify values + var valueMutationCallbackCalled bool + err = parentMap.IterateReadOnlyValuesWithMutationCallback( + func(v Value) (resume bool, err error) { + c, ok := v.(*OrderedMap) + require.True(t, ok) + require.False(t, c.Inlined()) + + existingKeyStorable, existingValueStorable, err := c.Remove(compare, hashInputProvider, Uint64Value(0)) + require.ErrorAs(t, err, &mutationError) + require.Nil(t, existingKeyStorable) + require.Nil(t, existingValueStorable) + + return true, err + }, + func(v Value) { + valueMutationCallbackCalled = true + vid := v.(mutableValueNotifier).ValueID() + require.True(t, childMap1.ValueID() == vid || childMap2.ValueID() == vid) + }) + + require.ErrorAs(t, err, &mutationError) + require.True(t, valueMutationCallbackCalled) + }) +} + func TestMutableMapIterate(t *testing.T) { t.Run("empty", func(t *testing.T) { diff --git a/storable_test.go b/storable_test.go index 4f705ab6..912207dc 100644 --- a/storable_test.go +++ b/storable_test.go @@ -34,6 +34,7 @@ const ( cborTagUInt32Value = 163 cborTagUInt64Value = 164 cborTagSomeValue = 165 + cborTagHashableMap = 166 ) type HashableValue interface { @@ -634,6 +635,19 @@ func compare(storage SlabStorage, value Value, storable Storable) (bool, error) } return compare(storage, v.Value, other.Storable) + + case *HashableMap: + other, err := storable.StoredValue(storage) + if err != nil { + return false, err + } + + otherMap, ok := other.(*OrderedMap) + if !ok { + return false, nil + } + + return v.m.ValueID() == otherMap.ValueID(), nil } return false, fmt.Errorf("value %T not supported for comparison", value) @@ -784,3 +798,47 @@ func (*mutableStorable) Encode(*Encoder) error { // no-op for testing return nil } + +type HashableMap struct { + m *OrderedMap +} + +var _ Value = &HashableMap{} +var _ HashableValue = &HashableMap{} + +func NewHashableMap(m *OrderedMap) *HashableMap { + return &HashableMap{m} +} + +func (v *HashableMap) Storable(storage SlabStorage, address Address, maxInlineSize uint64) (Storable, error) { + return v.m.Storable(storage, address, maxInlineSize) +} + +func (v *HashableMap) HashInput(scratch []byte) ([]byte, error) { + const ( + cborTypeByteString = 0x40 + + valueIDLength = len(ValueID{}) + cborTagNumSize = 2 + cborByteStringHeadSize = 1 + cborByteStringSize = valueIDLength + hashInputSize = cborTagNumSize + cborByteStringHeadSize + cborByteStringSize + ) + + var buf []byte + if len(scratch) >= hashInputSize { + buf = scratch[:hashInputSize] + } else { + buf = make([]byte, hashInputSize) + } + + // CBOR tag number + buf[0], buf[1] = 0xd8, cborTagHashableMap + + // CBOR byte string head + buf[2] = cborTypeByteString | byte(valueIDLength) + + vid := v.m.ValueID() + copy(buf[3:], vid[:]) + return buf, nil +}