Merge pull request #369 from onflow/fxamacker/deduplicate-dict-type-info

Reduce RAM and persistent storage by deduplicating inlined dict type info

array.go

@@ -298,16 +298,20 @@ func (a *ArrayExtraData) isExtraData() bool {
 	return true
 }
 
+func (a *ArrayExtraData) Type() TypeInfo {
+	return a.TypeInfo
+}
+
 // Encode encodes extra data as CBOR array:
 //
 //	[type info]
-func (a *ArrayExtraData) Encode(enc *Encoder) error {
+func (a *ArrayExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error {
 	err := enc.CBOR.EncodeArrayHead(arrayExtraDataLength)
 	if err != nil {
 		return NewEncodingError(err)
 	}
 
-	err = a.TypeInfo.Encode(enc.CBOR)
+	err = encodeTypeInfo(enc, a.TypeInfo)
 	if err != nil {
 		// Wrap err as external error (if needed) because err is returned by TypeInfo interface.
 		return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode type info")
@@ -840,7 +844,8 @@ func (a *ArrayDataSlab) Encode(enc *Encoder) error {
 
 	// Encode extra data
 	if a.extraData != nil {
-		err = a.extraData.Encode(enc)
+		// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
+		err = a.extraData.Encode(enc, defaultEncodeTypeInfo)
 		if err != nil {
 			// err is already categorized by ArrayExtraData.Encode().
 			return err
@@ -1729,7 +1734,8 @@ func (a *ArrayMetaDataSlab) Encode(enc *Encoder) error {
 
 	// Encode extra data if present
 	if a.extraData != nil {
-		err = a.extraData.Encode(enc)
+		// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
+		err = a.extraData.Encode(enc, defaultEncodeTypeInfo)
 		if err != nil {
 			// Don't need to wrap because err is already categorized by ArrayExtraData.Encode().
 			return err

array_debug.go

@@ -861,12 +861,27 @@ func hasInlinedComposite(data []byte) (bool, error) {
 
 	// Parse inlined extra data to find compact map extra data.
 	dec := cbor.NewStreamDecoder(bytes.NewBuffer(data))
+
 	count, err := dec.DecodeArrayHead()
 	if err != nil {
 		return false, NewDecodingError(err)
 	}
+	if count != inlinedExtraDataArrayCount {
+		return false, NewDecodingError(fmt.Errorf("failed to decode inlined extra data, expect %d elements, got %d elements", inlinedExtraDataArrayCount, count))
+	}
 
-	for i := uint64(0); i < count; i++ {
+	// Skip element 0 (inlined type info)
+	err = dec.Skip()
+	if err != nil {
+		return false, NewDecodingError(err)
+	}
+
+	// Decoding element 1 (inlined extra data)
+	extraDataCount, err := dec.DecodeArrayHead()
+	if err != nil {
+		return false, NewDecodingError(err)
+	}
+	for i := uint64(0); i < extraDataCount; i++ {
 		tagNum, err := dec.DecodeTagNumber()
 		if err != nil {
 			return false, NewDecodingError(err)

array_test.go

@@ -3185,10 +3185,15 @@ func TestArrayEncodeDecode(t *testing.T) {
 				0x18, 0x2a,
 
 				// inlined extra data
+				0x82,
+				// element 0: array of type info
+				0x80,
+				// element 1: array of extra data
 				0x81,
-				// inlined array extra data
+				// array extra data
 				0xd8, 0xf7,
 				0x81,
+				// array type info ref
 				0x18, 0x2b,
 
 				// CBOR encoded array head (fixed size 3 byte)
@@ -3267,6 +3272,10 @@ func TestArrayEncodeDecode(t *testing.T) {
 
 				// inlined extra data
 				0x82,
+				// element 0: array of inlined type info
+				0x80,
+				// element 1: array of inlined extra data
+				0x82,
 				// inlined array extra data
 				0xd8, 0xf7,
 				0x81,
@@ -3356,6 +3365,10 @@ func TestArrayEncodeDecode(t *testing.T) {
 
 				// inlined extra data
 				0x82,
+				// element 0: array of inlined type info
+				0x80,
+				// element 1: array of inlined extra data
+				0x82,
 				// inlined array extra data
 				0xd8, 0xf7,
 				0x81,
@@ -3455,6 +3468,10 @@ func TestArrayEncodeDecode(t *testing.T) {
 				0x18, 0x2a,
 
 				// inlined extra data
+				0x82,
+				// element 0: array of inlined type info
+				0x80,
+				// element 1: array of inlined extra data
 				0x84,
 				// typeInfo3
 				0xd8, 0xf7,
@@ -3596,6 +3613,10 @@ func TestArrayEncodeDecode(t *testing.T) {
 				// array data slab flag
 				0x00,
 				// inlined extra data
+				0x82,
+				// element 0: array of inlined type info
+				0x80,
+				// element 1: array of inlined extra data
 				0x81,
 				// inlined array extra data
 				0xd8, 0xf7,
@@ -3745,6 +3766,10 @@ func TestArrayEncodeDecode(t *testing.T) {
 				0x00,
 				// inlined extra data
 				0x82,
+				// element 0: array of inlined extra data
+				0x80,
+				// element 1: array of inlined extra data
+				0x82,
 				// inlined array extra data
 				0xd8, 0xf7,
 				0x81,
@@ -4065,7 +4090,11 @@ func TestArrayEncodeDecode(t *testing.T) {
 				// array data slab flag (has pointer)
 				0x40,
 
-				// inlined array of extra data
+				// inlined extra data
+				0x82,
+				// element 0: array of type info
+				0x80,
+				// element 1: array of extra data
 				0x81,
 				// type info
 				0xd8, 0xf7,

map.go

@@ -494,17 +494,21 @@ func (m *MapExtraData) isExtraData() bool {
 	return true
 }
 
+func (m *MapExtraData) Type() TypeInfo {
+	return m.TypeInfo
+}
+
 // Encode encodes extra data as CBOR array:
 //
 //	[type info, count, seed]
-func (m *MapExtraData) Encode(enc *Encoder) error {
+func (m *MapExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error {
 
 	err := enc.CBOR.EncodeArrayHead(mapExtraDataLength)
 	if err != nil {
 		return NewEncodingError(err)
 	}
 
-	err = m.TypeInfo.Encode(enc.CBOR)
+	err = encodeTypeInfo(enc, m.TypeInfo)
 	if err != nil {
 		// Wrap err as external error (if needed) because err is returned by TypeInfo interface.
 		return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode type info")
@@ -2916,7 +2920,8 @@ func (m *MapDataSlab) Encode(enc *Encoder) error {
 
 	// Encode extra data
 	if m.extraData != nil {
-		err = m.extraData.Encode(enc)
+		// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
+		err = m.extraData.Encode(enc, defaultEncodeTypeInfo)
 		if err != nil {
 			// Don't need to wrap error as external error because err is already categorized by MapExtraData.Encode().
 			return err
@@ -3909,7 +3914,8 @@ func (m *MapMetaDataSlab) Encode(enc *Encoder) error {
 
 	// Encode extra data if present
 	if m.extraData != nil {
-		err = m.extraData.Encode(enc)
+		// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
+		err = m.extraData.Encode(enc, defaultEncodeTypeInfo)
 		if err != nil {
 			// Don't need to wrap error as external error because err is already categorized by MapExtraData.Encode().
 			return err