Skip to content

Commit

Permalink
Deduplicate inlined dict type info to reduce RAM
Browse files Browse the repository at this point in the history 
This change deduplicates Cadence dictionary type and composite
type info, resulting in reduced memory and also persistent storage.

More specifically, this encodes inlined atree slab extra data
section as two-element array:
- array of deduplicated type info
- array of deduplicated extra data with type info index
  • Loading branch information
@fxamacker
fxamacker committed Feb 28, 2024
1 parent 2fbf860 commit 83d8870
Show file tree
Hide file tree
Showing 7 changed files with 464 additions and 111 deletions.
10 changes: 6 additions & 4 deletions array.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,13 +301,13 @@ func (a *ArrayExtraData) isExtraData() bool {
// Encode encodes extra data as CBOR array:
//
// [type info]
func (a *ArrayExtraData) Encode(enc *Encoder) error {
func (a *ArrayExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error {
err := enc.CBOR.EncodeArrayHead(arrayExtraDataLength)
if err != nil {
return NewEncodingError(err)
}

err = a.TypeInfo.Encode(enc.CBOR)
err = encodeTypeInfo(enc, a.TypeInfo)
if err != nil {
// Wrap err as external error (if needed) because err is returned by TypeInfo interface.
return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode type info")
Expand Down Expand Up @@ -840,7 +840,8 @@ func (a *ArrayDataSlab) Encode(enc *Encoder) error {

// Encode extra data
if a.extraData != nil {
err = a.extraData.Encode(enc)
// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
err = a.extraData.Encode(enc, defaultEncodeTypeInfo)
if err != nil {
// err is already categorized by ArrayExtraData.Encode().
return err
Expand Down Expand Up @@ -1738,7 +1739,8 @@ func (a *ArrayMetaDataSlab) Encode(enc *Encoder) error {

// Encode extra data if present
if a.extraData != nil {
err = a.extraData.Encode(enc)
// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
err = a.extraData.Encode(enc, defaultEncodeTypeInfo)
if err != nil {
// Don't need to wrap because err is already categorized by ArrayExtraData.Encode().
return err
Expand Down
17 changes: 16 additions & 1 deletion array_debug.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -861,12 +861,27 @@ func hasInlinedComposite(data []byte) (bool, error) {

// Parse inlined extra data to find compact map extra data.
dec := cbor.NewStreamDecoder(bytes.NewBuffer(data))

count, err := dec.DecodeArrayHead()
if err != nil {
return false, NewDecodingError(err)
}
if count != inlinedExtraDataArrayCount {
return false, NewDecodingError(fmt.Errorf("failed to decode inlined extra data, expect %d elements, got %d elements", inlinedExtraDataArrayCount, count))
}

for i := uint64(0); i < count; i++ {
// Skip element 0 (inlined type info)
err = dec.Skip()
if err != nil {
return false, NewDecodingError(err)
}

// Decoding element 1 (inlined extra data)
extraDataCount, err := dec.DecodeArrayHead()
if err != nil {
return false, NewDecodingError(err)
}
for i := uint64(0); i < extraDataCount; i++ {
tagNum, err := dec.DecodeTagNumber()
if err != nil {
return false, NewDecodingError(err)
Expand Down
86 changes: 71 additions & 15 deletions array_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3184,11 +3184,19 @@ func TestArrayEncodeDecode(t *testing.T) {
0x18, 0x2a,

// inlined extra data
0x82,
// element 0: array of type info
0x81,
// inlined array extra data
// type info
0x18, 0x2b,
// element 1: array of extra data
0x81,
// array extra data
0xd8, 0xf7,
0x81,
0x18, 0x2b,
// array type info ref
0xd8, 0xf6,
0x00,

// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x02,
Expand Down Expand Up @@ -3266,14 +3274,22 @@ func TestArrayEncodeDecode(t *testing.T) {

// inlined extra data
0x82,
// element 0: array of inlined type info
0x82,
0x18, 0x2c,
0x18, 0x2b,
// element 1: array of inlined extra data
0x82,
// inlined array extra data
0xd8, 0xf7,
0x81,
0x18, 0x2c,
0xd8, 0xf6,
0x00,
// inlined array extra data
0xd8, 0xf7,
0x81,
0x18, 0x2b,
0xd8, 0xf6,
0x01,

// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x02,
Expand Down Expand Up @@ -3355,13 +3371,21 @@ func TestArrayEncodeDecode(t *testing.T) {

// inlined extra data
0x82,
// element 0: array of inlined type info
0x82,
0x18, 0x2c,
0x18, 0x2b,
// element 1: array of inlined extra data
0x82,
// inlined array extra data
0xd8, 0xf7,
0x81,
0x18, 0x2c,
0xd8, 0xf6,
0x00,
0xd8, 0xf7,
0x81,
0x18, 0x2b,
0xd8, 0xf6,
0x01,

// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x02,
Expand Down Expand Up @@ -3454,23 +3478,35 @@ func TestArrayEncodeDecode(t *testing.T) {
0x18, 0x2a,

// inlined extra data
0x82,
// element 0: array of inlined type info
0x84,
0x18, 0x2c,
0x18, 0x2b,
0x18, 0x2e,
0x18, 0x2d,
// element 1: array of inlined extra data
0x84,
// typeInfo3
0xd8, 0xf7,
0x81,
0x18, 0x2c,
0xd8, 0xf6,
0x00,
// typeInfo2
0xd8, 0xf7,
0x81,
0x18, 0x2b,
0xd8, 0xf6,
0x01,
// typeInfo5
0xd8, 0xf7,
0x81,
0x18, 0x2e,
0xd8, 0xf6,
0x02,
// typeInfo4
0xd8, 0xf7,
0x81,
0x18, 0x2d,
0xd8, 0xf6,
0x03,

// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x02,
Expand Down Expand Up @@ -3595,11 +3631,17 @@ func TestArrayEncodeDecode(t *testing.T) {
// array data slab flag
0x00,
// inlined extra data
0x82,
// element 0: array of inlined type info
0x81,
0x18, 0x2b,
// element 1: array of inlined extra data
0x81,
// inlined array extra data
0xd8, 0xf7,
0x81,
0x18, 0x2b,
0xd8, 0xf6,
0x00,
// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x0b,
// CBOR encoded array elements
Expand Down Expand Up @@ -3744,13 +3786,21 @@ func TestArrayEncodeDecode(t *testing.T) {
0x00,
// inlined extra data
0x82,
// element 0: array of inlined extra data
0x82,
0x18, 0x2c,
0x18, 0x2b,
// element 1: array of inlined extra data
0x82,
// inlined array extra data
0xd8, 0xf7,
0x81,
0x18, 0x2c,
0xd8, 0xf6,
0x00,
0xd8, 0xf7,
0x81,
0x18, 0x2b,
0xd8, 0xf6,
0x01,
// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x0b,
// CBOR encoded array elements
Expand Down Expand Up @@ -4064,12 +4114,18 @@ func TestArrayEncodeDecode(t *testing.T) {
// array data slab flag (has pointer)
0x40,

// inlined array of extra data
// inlined extra data
0x82,
// element 0: array of type info
0x81,
0x18, 0x2c,
// element 1: array of extra data
0x81,
// type info
0xd8, 0xf7,
0x81,
0x18, 0x2c,
0xd8, 0xf6,
0x00,

// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x0b,
Expand Down
10 changes: 6 additions & 4 deletions map.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,14 +497,14 @@ func (m *MapExtraData) isExtraData() bool {
// Encode encodes extra data as CBOR array:
//
// [type info, count, seed]
func (m *MapExtraData) Encode(enc *Encoder) error {
func (m *MapExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error {

err := enc.CBOR.EncodeArrayHead(mapExtraDataLength)
if err != nil {
return NewEncodingError(err)
}

err = m.TypeInfo.Encode(enc.CBOR)
err = encodeTypeInfo(enc, m.TypeInfo)
if err != nil {
// Wrap err as external error (if needed) because err is returned by TypeInfo interface.
return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode type info")
Expand Down Expand Up @@ -2917,7 +2917,8 @@ func (m *MapDataSlab) Encode(enc *Encoder) error {

// Encode extra data
if m.extraData != nil {
err = m.extraData.Encode(enc)
// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
err = m.extraData.Encode(enc, defaultEncodeTypeInfo)
if err != nil {
// Don't need to wrap error as external error because err is already categorized by MapExtraData.Encode().
return err
Expand Down Expand Up @@ -3918,7 +3919,8 @@ func (m *MapMetaDataSlab) Encode(enc *Encoder) error {

// Encode extra data if present
if m.extraData != nil {
err = m.extraData.Encode(enc)
// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
err = m.extraData.Encode(enc, defaultEncodeTypeInfo)
if err != nil {
// Don't need to wrap error as external error because err is already categorized by MapExtraData.Encode().
return err
Expand Down
Loading

0 comments on commit 83d8870

Please sign in to comment.