Skip to content

Commit

Permalink
Use encoded type info to deduplicate extra data
Browse files Browse the repository at this point in the history
Currently, we use TypeInfo.Identifier() to deduplicate extra data
and type info.  However, TypeInfo.Identifier() is implemented in
another package and we can't enforce its uniqueness for different
types.  If TypeInfo.Identifier() returns same ID for different
types, different types is wrongly deduplicated.

This commit uses encoded type info via TypeInfo.Encode() to
deduplicate extra data.  This prevents differently encoded type
info from being deduplicated by mistake.

This commit also uses sync.Pool to reuse buffer for type info
encoding.
  • Loading branch information
fxamacker committed Apr 2, 2024
1 parent 92714ca commit 83c99b3
Show file tree
Hide file tree
Showing 4 changed files with 169 additions and 80 deletions.
7 changes: 4 additions & 3 deletions array.go
Original file line number Diff line number Diff line change
Expand Up @@ -720,15 +720,16 @@ func (a *ArrayDataSlab) encodeAsInlined(enc *Encoder) error {
fmt.Errorf("failed to encode standalone array data slab as inlined"))
}

extraDataIndex := enc.inlinedExtraData().addArrayExtraData(a.extraData)
extraDataIndex, err := enc.inlinedExtraData().addArrayExtraData(a.extraData)
if err != nil {
return NewEncodingError(err)
}

if extraDataIndex > maxInlinedExtraDataIndex {
return NewEncodingError(
fmt.Errorf("failed to encode inlined array data slab: extra data index %d exceeds limit %d", extraDataIndex, maxInlinedExtraDataIndex))
}

var err error

// Encode tag number and array head of 3 elements
err = enc.CBOR.EncodeRawBytes([]byte{
// tag number
Expand Down
41 changes: 40 additions & 1 deletion cmd/stress/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,16 @@
package main

import (
"bytes"
"fmt"
"math"
"math/rand"
"reflect"
"sync"
"time"

"github.com/fxamacker/cbor/v2"

"github.com/onflow/atree"
)

Expand Down Expand Up @@ -540,5 +544,40 @@ func (v mapValue) Storable(atree.SlabStorage, atree.Address, uint64) (atree.Stor
}

var typeInfoComparator = func(a atree.TypeInfo, b atree.TypeInfo) bool {
return a.Identifier() == b.Identifier()
aID, _ := getEncodedTypeInfo(a)
bID, _ := getEncodedTypeInfo(b)
return aID == bID
}

func getEncodedTypeInfo(ti atree.TypeInfo) (string, error) {
b := getTypeIDBuffer()
defer putTypeIDBuffer(b)

enc := cbor.NewStreamEncoder(b)
err := ti.Encode(enc)
if err != nil {
return "", err
}
enc.Flush()

return b.String(), nil
}

const defaultTypeIDBufferSize = 256

var typeIDBufferPool = sync.Pool{
New: func() interface{} {
e := new(bytes.Buffer)
e.Grow(defaultTypeIDBufferSize)
return e
},
}

func getTypeIDBuffer() *bytes.Buffer {
return typeIDBufferPool.Get().(*bytes.Buffer)
}

func putTypeIDBuffer(e *bytes.Buffer) {
e.Reset()
typeIDBufferPool.Put(e)
}
14 changes: 8 additions & 6 deletions map.go
Original file line number Diff line number Diff line change
Expand Up @@ -3007,14 +3007,15 @@ func (m *MapDataSlab) encodeAsInlined(enc *Encoder) error {

func (m *MapDataSlab) encodeAsInlinedMap(enc *Encoder) error {

extraDataIndex := enc.inlinedExtraData().addMapExtraData(m.extraData)
extraDataIndex, err := enc.inlinedExtraData().addMapExtraData(m.extraData)
if err != nil {
return NewEncodingError(err)
}

if extraDataIndex > maxInlinedExtraDataIndex {
return NewEncodingError(fmt.Errorf("extra data index %d exceeds limit %d", extraDataIndex, maxInlinedExtraDataIndex))
}

var err error

// Encode tag number and array head of 3 elements
err = enc.CBOR.EncodeRawBytes([]byte{
// tag number
Expand Down Expand Up @@ -3067,7 +3068,10 @@ func encodeAsInlinedCompactMap(
values []Storable,
) error {

extraDataIndex, cachedKeys := enc.inlinedExtraData().addCompactMapExtraData(extraData, hkeys, keys)
extraDataIndex, cachedKeys, err := enc.inlinedExtraData().addCompactMapExtraData(extraData, hkeys, keys)
if err != nil {
return NewEncodingError(err)
}

if len(keys) != len(cachedKeys) {
return NewEncodingError(fmt.Errorf("number of elements %d is different from number of elements in cached compact map type %d", len(keys), len(cachedKeys)))
Expand All @@ -3078,8 +3082,6 @@ func encodeAsInlinedCompactMap(
return NewEncodingError(fmt.Errorf("extra data index %d exceeds limit %d", extraDataIndex, maxInlinedExtraDataIndex))
}

var err error

// Encode tag number and array head of 3 elements
err = enc.CBOR.EncodeRawBytes([]byte{
// tag number
Expand Down
Loading

0 comments on commit 83c99b3

Please sign in to comment.