From 30dad00da2bfa82e14bcde1cbd677aa3c6861cc6 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Thu, 11 Apr 2024 15:02:45 -0500 Subject: [PATCH 1/6] Add feature to fix refs to non-existent registers In testnet, broken references seem to have resulted from a bug that was fixed 2 years ago by https://github.com/onflow/cadence/pull/1565. A broken reference is a `StorageID` referencing a non-existent register. So far, only 10 registers in testnet (none on mainnet) were found to contain broken references. This commit adds a feature to enable migration programs in onflow/flow-go to fix broken references in maps. `FixLoadedBrokenReferences()` traverses loaded slabs and replaces broken map (if any) with empty map having the same StorageID and also removes all slabs in the old map. Limitations: - only fix broken references in map (this is intentional) - only traverse loaded slabs in deltas and cache IMPORTANT: This should not be used to silently fix unknown problems. It should only be used by migration programs to fix known problems which were determined to be appropriate to fix in this manner. --- storage.go | 233 +++++++++ storage_test.go | 1274 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1507 insertions(+) diff --git a/storage.go b/storage.go index 3f79d40e..9b4f5a5d 100644 --- a/storage.go +++ b/storage.go @@ -21,6 +21,7 @@ package atree import ( "bytes" "encoding/binary" + "errors" "fmt" "sort" "strings" @@ -1088,3 +1089,235 @@ func storeSlab(storage SlabStorage, slab Slab) error { } return nil } + +// FixLoadedBrokenReferences traverses loaded slabs and fixes broken references in maps. +// A broken reference is a SlabID referencing a non-existent slab. +// To fix a map containing broken references, this function replaces broken map with +// empty map having the same SlabID and also removes all slabs in the old map. +// Limitations: +// - only fix broken references in map +// - only traverse loaded slabs in deltas and cache +// NOTE: The intended use case is to enable migration programs in onflow/flow-go to +// fix broken references. As of April 2024, only 10 registers in testnet (not mainnet) +// were found to have broken references and they seem to have resulted from a bug +// that was fixed 2 years ago by https://github.com/onflow/cadence/pull/1565. +func (s *PersistentSlabStorage) FixLoadedBrokenReferences() ([]SlabID, error) { + + // parentOf is used to find root slab from non-root slab. + // Broken reference can be in non-root slab, and we need SlabID of root slab + // to replace broken map by creating an empty new map with same SlabID. + parentOf := make(map[SlabID]SlabID) + + getRootSlabID := func(id SlabID) SlabID { + for { + parentID, ok := parentOf[id] + if ok { + id = parentID + } else { + return id + } + } + } + + hasBrokenReferenceInSlab := func(id SlabID, slab Slab) bool { + if slab == nil { + return false + } + + var isMetaDataSlab bool + + switch slab.(type) { + case *ArrayMetaDataSlab, *MapMetaDataSlab: + isMetaDataSlab = true + } + + var foundBrokenRef bool + for _, childStorable := range slab.ChildStorables() { + + slabIDStorable, ok := childStorable.(SlabIDStorable) + if !ok { + continue + } + + childID := SlabID(slabIDStorable) + + // Track parent-child relationship of root slabs and non-root slabs. + if isMetaDataSlab { + parentOf[childID] = id + } + + if s.existIfLoaded(childID) { + continue + } + + foundBrokenRef = true + + if !isMetaDataSlab { + return true + } + } + + return foundBrokenRef + } + + var brokenStorageIDs []SlabID + + // Iterate delta slabs. + for id, slab := range s.deltas { + if hasBrokenReferenceInSlab(id, slab) { + brokenStorageIDs = append(brokenStorageIDs, id) + } + } + + // Iterate cache slabs. + for id, slab := range s.cache { + if _, ok := s.deltas[id]; ok { + continue + } + if hasBrokenReferenceInSlab(id, slab) { + brokenStorageIDs = append(brokenStorageIDs, id) + } + } + + if len(brokenStorageIDs) == 0 { + return nil, nil + } + + rootSlabStorageIDsWithBrokenData := make(map[SlabID]struct{}) + var errs []error + + // Find StorageIDs of root slab for slabs containing broken references. + for _, id := range brokenStorageIDs { + rootID := getRootSlabID(id) + if rootID == SlabIDUndefined { + errs = append(errs, fmt.Errorf("failed to get root slab id for slab %s", id)) + continue + } + rootSlabStorageIDsWithBrokenData[rootID] = struct{}{} + } + + for rootSlabID := range rootSlabStorageIDsWithBrokenData { + rootSlab := s.RetrieveIfLoaded(rootSlabID) + if rootSlab == nil { + errs = append(errs, fmt.Errorf("failed to retrieve loaded root slab %s", rootSlabID)) + continue + } + + switch rootSlab := rootSlab.(type) { + case MapSlab: + if rootSlab.ExtraData() == nil { + errs = append(errs, fmt.Errorf("failed to fix broken references because slab %s isn't root slab", rootSlab.SlabID())) + continue + } + + err := s.fixBrokenReferencesInMap(rootSlab) + if err != nil { + errs = append(errs, err) + } + + default: + // IMPORTANT: Only handle map slabs for now. DO NOT silently fix currently unknown problems. + errs = append(errs, fmt.Errorf("failed to fix broken references in non-map slab %s (%T)", rootSlab.SlabID(), rootSlab)) + } + } + + return brokenStorageIDs, errors.Join(errs...) +} + +// fixBrokenReferencesInMap replaces replaces broken map with empty map +// having the same SlabID and also removes all slabs in the old map. +func (s *PersistentSlabStorage) fixBrokenReferencesInMap(old MapSlab) error { + id := old.SlabID() + + // Create an empty map with the same StorgeID, type, and seed as the old map. + new := &MapDataSlab{ + header: MapSlabHeader{ + slabID: id, + size: mapRootDataSlabPrefixSize + hkeyElementsPrefixSize, + }, + extraData: &MapExtraData{ + TypeInfo: old.ExtraData().TypeInfo, + Seed: old.ExtraData().Seed, + }, + elements: newHkeyElements(0), + } + + // Store new empty map with the same SlabID. + err := s.Store(id, new) + if err != nil { + return err + } + + // Remove all slabs and references in old map. + references, _, err := s.getAllChildReferences(old) + if err != nil { + return err + } + + for _, childID := range references { + err = s.Remove(childID) + if err != nil { + return err + } + } + + return nil +} + +func (s *PersistentSlabStorage) existIfLoaded(id SlabID) bool { + // Check deltas. + if slab, ok := s.deltas[id]; ok { + return slab != nil + } + + // Check read cache. + if slab, ok := s.cache[id]; ok { + return slab != nil + } + + return false +} + +// getAllChildReferences returns child references of given slab (all levels). +func (s *PersistentSlabStorage) getAllChildReferences(slab Slab) ( + references []SlabID, + brokenReferences []SlabID, + err error, +) { + childStorables := slab.ChildStorables() + + for len(childStorables) > 0 { + + var nextChildStorables []Storable + + for _, childStorable := range childStorables { + + slabIDStorable, ok := childStorable.(SlabIDStorable) + if !ok { + continue + } + + childID := SlabID(slabIDStorable) + + childSlab, ok, err := s.Retrieve(childID) + if err != nil { + return nil, nil, err + } + if !ok { + brokenReferences = append(brokenReferences, childID) + continue + } + + references = append(references, childID) + + nextChildStorables = append( + nextChildStorables, + childSlab.ChildStorables()..., + ) + } + + childStorables = nextChildStorables + } + + return references, brokenReferences, nil +} diff --git a/storage_test.go b/storage_test.go index 2b90bb11..9852cd64 100644 --- a/storage_test.go +++ b/storage_test.go @@ -1240,3 +1240,1277 @@ func (s slowStorable) Encode(encoder *Encoder) error { runtime.KeepAlive(n) return s.Uint8Value.Encode(encoder) } + +func TestFixLoadedBrokenReferences(t *testing.T) { + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + t.Run("healthy", func(t *testing.T) { + + // Create a health storage with arrays and maps + mapMetaDataRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + mapDataNonRootID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + mapDataNonRootID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} + nestedArrayID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}} + + emptyMapDataRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 5}} + + mapDataRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 6}} + + emptyArrayDataRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 7}} + + arrayDataRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 8}} + + arrayMetaDataRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 9}} + arrayDataNonRootID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 10}} + arrayDataNonRootID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 11}} + nestedArrayID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 12}} + + rootIDs := []SlabID{ + mapMetaDataRootID, + emptyMapDataRootID, + mapDataRootID, + emptyArrayDataRootID, + arrayDataRootID, + arrayMetaDataRootID, + } + + data := map[SlabID][]byte{ + // root map metadata slab + // metadata slab + mapMetaDataRootID: { + // extra data + // version + 0x00, + // flag: root + map meta + 0x89, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info: "map" + 0x18, 0x2A, + // count: 8 + 0x08, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + meta + 0x89, + // child header count + 0x00, 0x02, + // child header 1 (storage id, first key, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x02, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0xfe, + }, + + // data slab + mapDataNonRootID1: { + // version + 0x00, + // flag: map data + 0x08, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // hkey: 1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // hkey: 2 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + // hkey: 3 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] + 0x82, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] + 0x82, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] + 0x82, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] + 0x82, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + }, + + // data slab + mapDataNonRootID2: { + // version + 0x00, + // flag: has pointer + map data + 0x48, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 4 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // hkey: 5 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + // hkey: 6 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // hkey: 7 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] + 0x82, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] + 0x82, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] + 0x82, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + // element: [hhhhhhhhhhhhhhhhhhhhhh:SlabID(1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,4)] + 0x82, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + }, + + // array data slab + nestedArrayID: { + // extra data + // version + 0x00, + // flag: root + array data + 0x80, + // extra data (CBOR encoded array of 1 elements) + 0x81, + // type info + 0x18, 0x2b, + + // version + 0x00, + // flag: root + array data + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xa4, 0x00, + }, + + // empty map + emptyMapDataRootID: { + // extra data + // version + 0x00, + // flag: root + map data + 0x88, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info + 0x18, 0x2a, + // count: 0 + 0x00, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + map data + 0x88, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // elements (array of 0 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + + // root map data slab + mapDataRootID: { + // extra data + // version + 0x00, + // flag: root + map data + 0x88, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info + 0x18, 0x2a, + // count: 1 + 0x01, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + map data + 0x88, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // elements (array of 1 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // element: [uint64(0):uint64(0)] + 0x82, 0xd8, 0xa4, 0x00, 0xd8, 0xa4, 0x00, + }, + + // empty array + emptyArrayDataRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x00, + }, + + // root array data slab + arrayDataRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xa4, 0x00, + }, + + // root array metadata slab + // (metadata slab) headers: [{id:2 size:228 count:9} {id:3 size:270 count:11} ] + arrayMetaDataRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x81, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array meta data slab flag + 0x81, + // child header count + 0x00, 0x02, + // child header 1 (storage id, count, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, + 0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0xe4, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x01, 0x0e, + }, + + // (data slab) next: 3, data: [aaaaaaaaaaaaaaaaaaaaaa ... aaaaaaaaaaaaaaaaaaaaaa] + arrayDataNonRootID1: { + // version + 0x00, + // array data slab flag + 0x00, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x09, + // CBOR encoded array elements + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + }, + + // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... SlabID(...)] + arrayDataNonRootID2: { + // version + 0x00, + // array data slab flag + 0x40, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x0b, + // CBOR encoded array elements + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, + }, + + // (data slab) next: 0, data: [0] + nestedArrayID2: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2b, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xa4, 0x00, + }, + } + + storage := newTestPersistentStorageWithData(t, data) + + // Load data in storage + for id := range data { + _, found, err := storage.Retrieve(id) + require.NoError(t, err) + require.True(t, found) + } + + // Check health before fixing broken reference + rootIDSet, err := CheckStorageHealth(storage, -1) + require.NoError(t, err) + require.Equal(t, len(rootIDs), len(rootIDSet)) + + for _, rootID := range rootIDs { + _, found := rootIDSet[rootID] + require.True(t, found) + } + + // Fix broken reference + fixedIDs, err := storage.FixLoadedBrokenReferences() + require.NoError(t, err) + require.Equal(t, 0, len(fixedIDs)) + + // No data is modified during fixing broken reference + require.Equal(t, 0, len(storage.deltas)) + + // Check health after fixing broken reference + rootIDSet, err = CheckStorageHealth(storage, -1) + require.NoError(t, err) + require.Equal(t, len(rootIDs), len(rootIDSet)) + + }) + + t.Run("broken root map data slab", func(t *testing.T) { + + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + + brokenRefSlabID := rootID + + data := map[SlabID][]byte{ + rootID: { + // extra data + // version + 0x00, + // flag: root + map data + 0x88, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info + 0x18, 0x2a, + // count: 1 + 0x01, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + map data + 0x88, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // elements (array of 1 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // element: [SlabID(0x0.1):uint64(0)] + 0x82, + 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0xd8, 0xa4, 0x00, + }, + } + + fixedData := map[SlabID][]byte{ + rootID: { + // version + 0x10, + // flag: root + map data + 0x88, + + // extra data + // CBOR encoded array of 3 elements + 0x83, + // type info + 0x18, 0x2a, + // count: 0 + 0x00, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x59, 0x00, 0x00, + + // elements (array of 0 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x99, 0x00, 0x00, + }, + } + + storage := newTestPersistentStorageWithData(t, data) + + // Load data in storage + for id := range data { + _, found, err := storage.Retrieve(id) + require.NoError(t, err) + require.True(t, found) + } + + // Check health before fixing broken reference + _, err := CheckStorageHealth(storage, -1) + require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") + + // Fix broken reference + fixedIDs, err := storage.FixLoadedBrokenReferences() + require.NoError(t, err) + require.Equal(t, 1, len(fixedIDs)) + require.Equal(t, brokenRefSlabID, fixedIDs[0]) + require.Equal(t, 1, len(storage.deltas)) + + // Check health after fixing broken reference + rootIDs, err := CheckStorageHealth(storage, -1) + require.NoError(t, err) + require.Equal(t, 1, len(rootIDs)) + + _, ok := rootIDs[rootID] + require.True(t, ok) + + // Save data in storage + err = storage.FastCommit(runtime.NumCPU()) + require.NoError(t, err) + require.Equal(t, 0, len(storage.deltas)) + + // Check encoded data + baseStorage := storage.baseStorage.(*InMemBaseStorage) + require.Equal(t, 1, len(baseStorage.segments)) + + savedData, found, err := baseStorage.Retrieve(rootID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, fixedData[rootID], savedData) + }) + + t.Run("broken non-root map data slab", func(t *testing.T) { + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootDataID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootDataID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + brokenRefSlabID := nonRootDataID2 + + // Expected serialized slab data with storage id + data := map[SlabID][]byte{ + + // metadata slab + rootID: { + // extra data + // version + 0x00, + // flag: root + map meta + 0x89, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info: "map" + 0x18, 0x2A, + // count: 8 + 0x08, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + meta + 0x89, + // child header count + 0x00, 0x02, + // child header 1 (storage id, first key, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x02, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0xfe, + }, + + // data slab + nonRootDataID1: { + // version + 0x00, + // flag: map data + 0x08, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // hkey: 1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // hkey: 2 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + // hkey: 3 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] + 0x82, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] + 0x82, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] + 0x82, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] + 0x82, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + }, + + // data slab + nonRootDataID2: { + // version + 0x00, + // flag: has pointer + map data + 0x48, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 4 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // hkey: 5 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + // hkey: 6 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // hkey: 7 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] + 0x82, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] + 0x82, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] + 0x82, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + // element: [hhhhhhhhhhhhhhhhhhhhhh:SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] + 0x82, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + }, + } + + fixedData := map[SlabID][]byte{ + rootID: { + // version + 0x10, + // flag: root + map data + 0x88, + + // extra data + // CBOR encoded array of 3 elements + 0x83, + // type info + 0x18, 0x2a, + // count: 0 + 0x00, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x59, 0x00, 0x00, + + // elements (array of 0 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x99, 0x00, 0x00, + }, + } + + storage := newTestPersistentStorageWithData(t, data) + + // Load data in storage + for id := range data { + _, found, err := storage.Retrieve(id) + require.NoError(t, err) + require.True(t, found) + } + + // Check health before fixing broken reference + _, err := CheckStorageHealth(storage, -1) + require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") + + // Fix broken reference + fixedIDs, err := storage.FixLoadedBrokenReferences() + require.NoError(t, err) + require.Equal(t, 1, len(fixedIDs)) + require.Equal(t, brokenRefSlabID, fixedIDs[0]) + require.Equal(t, 3, len(storage.deltas)) + + // Check health after fixing broken reference + rootIDs, err := CheckStorageHealth(storage, -1) + require.NoError(t, err) + require.Equal(t, 1, len(rootIDs)) + + _, ok := rootIDs[rootID] + require.True(t, ok) + + // Save data in storage + err = storage.FastCommit(runtime.NumCPU()) + require.NoError(t, err) + require.Equal(t, 0, len(storage.deltas)) + + // Check encoded data + baseStorage := storage.baseStorage.(*InMemBaseStorage) + require.Equal(t, 1, len(baseStorage.segments)) + + savedData, found, err := baseStorage.Retrieve(rootID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, fixedData[rootID], savedData) + }) + + t.Run("multiple data slabs with broken reference in the same map", func(t *testing.T) { + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootDataID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootDataID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + brokenRefSlabID := []SlabID{nonRootDataID1, nonRootDataID2} + + data := map[SlabID][]byte{ + + // metadata slab + rootID: { + // extra data + // version + 0x00, + // flag: root + map meta + 0x89, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info: "map" + 0x18, 0x2A, + // count: 8 + 0x08, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + meta + 0x89, + // child header count + 0x00, 0x02, + // child header 1 (storage id, first key, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x02, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0xfe, + }, + + // data slab + nonRootDataID1: { + // version + 0x00, + // flag: map data + 0x08, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // hkey: 1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // hkey: 2 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + // hkey: 3 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] + 0x82, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] + 0x82, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] + 0x82, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + // element: [dddddddddddddddddddddd:SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] + 0x82, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + }, + + // data slab + nonRootDataID2: { + // version + 0x00, + // flag: has pointer + map data + 0x48, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 4 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // hkey: 5 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + // hkey: 6 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // hkey: 7 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] + 0x82, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] + 0x82, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] + 0x82, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + // element: [hhhhhhhhhhhhhhhhhhhhhh:SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2)] + 0x82, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + }, + } + + fixedData := map[SlabID][]byte{ + rootID: { + // version + 0x10, + // flag: root + map data + 0x88, + + // extra data + // CBOR encoded array of 3 elements + 0x83, + // type info + 0x18, 0x2a, + // count: 0 + 0x00, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x59, 0x00, 0x00, + + // elements (array of 0 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x99, 0x00, 0x00, + }, + } + + storage := newTestPersistentStorageWithData(t, data) + + // Load data in storage + for id := range data { + _, found, err := storage.Retrieve(id) + require.NoError(t, err) + require.True(t, found) + } + + // Check health before fixing broken reference + _, err := CheckStorageHealth(storage, -1) + require.ErrorContains(t, err, "slab not found during slab iteration") + + // Fix broken reference + fixedIDs, err := storage.FixLoadedBrokenReferences() + require.NoError(t, err) + require.Equal(t, len(brokenRefSlabID), len(fixedIDs)) + require.Equal(t, 3, len(storage.deltas)) + + for _, expected := range brokenRefSlabID { + var found bool + for _, actual := range fixedIDs { + if actual == expected { + found = true + break + } + } + require.True(t, found) + } + + // Check health after fixing broken reference + rootIDs, err := CheckStorageHealth(storage, -1) + require.NoError(t, err) + require.Equal(t, 1, len(rootIDs)) + + _, ok := rootIDs[rootID] + require.True(t, ok) + + // Save data in storage + err = storage.FastCommit(runtime.NumCPU()) + require.NoError(t, err) + require.Equal(t, 0, len(storage.deltas)) + + // Check encoded data + baseStorage := storage.baseStorage.(*InMemBaseStorage) + require.Equal(t, 1, len(baseStorage.segments)) + + savedData, found, err := baseStorage.Retrieve(rootID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, fixedData[rootID], savedData) + }) + + t.Run("broken reference in nested container", func(t *testing.T) { + parentContainerRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootDataID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootDataID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} + nestedContainerRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}} + + brokenRefSlabID := []SlabID{nestedContainerRootID} + + data := map[SlabID][]byte{ + + // metadata slab + parentContainerRootID: { + // extra data + // version + 0x00, + // flag: root + map meta + 0x89, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info: "map" + 0x18, 0x2A, + // count: 8 + 0x08, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + meta + 0x89, + // child header count + 0x00, 0x02, + // child header 1 (storage id, first key, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x02, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0xfe, + }, + + // data slab + nonRootDataID1: { + // version + 0x00, + // flag: map data + 0x08, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // hkey: 1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // hkey: 2 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + // hkey: 3 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] + 0x82, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] + 0x82, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] + 0x82, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] + 0x82, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + }, + + // data slab + nonRootDataID2: { + // version + 0x00, + // flag: has pointer + map data + 0x48, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 4 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // hkey: 5 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + // hkey: 6 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // hkey: 7 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] + 0x82, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] + 0x82, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] + 0x82, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + // element: [hhhhhhhhhhhhhhhhhhhhhh:SlabID(1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,4)] + 0x82, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + }, + + // map data slab + nestedContainerRootID: { + // extra data + // version + 0x00, + // flag: root + map data + 0x88, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info + 0x18, 0x2a, + // count: 1 + 0x01, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + map data + 0x88, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // elements (array of 1 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // element: [uint64(0):SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] + 0x82, + 0xd8, 0xa4, 0x00, + 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + }, + } + + fixedData := map[SlabID][]byte{ + // map data slab + nestedContainerRootID: { + // version + 0x10, + // flag: root + map data + 0x88, + + // extra data + // CBOR encoded array of 3 elements + 0x83, + // type info + 0x18, 0x2a, + // count: 0 + 0x00, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x59, 0x00, 0x00, + + // elements (array of 0 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x99, 0x00, 0x00, + }, + } + + storage := newTestPersistentStorageWithData(t, data) + + // Load data in storage + for id := range data { + _, found, err := storage.Retrieve(id) + require.NoError(t, err) + require.True(t, found) + } + + // Check health before fixing broken reference + _, err := CheckStorageHealth(storage, -1) + require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") + + // Fix broken reference + fixedIDs, err := storage.FixLoadedBrokenReferences() + require.NoError(t, err) + require.Equal(t, len(brokenRefSlabID), len(fixedIDs)) + require.Equal(t, brokenRefSlabID[0], fixedIDs[0]) + require.Equal(t, 1, len(storage.deltas)) + + // Check health after fixing broken reference + rootIDs, err := CheckStorageHealth(storage, -1) + require.NoError(t, err) + require.Equal(t, 1, len(rootIDs)) + + _, ok := rootIDs[parentContainerRootID] + require.True(t, ok) + + // Save data in storage + err = storage.FastCommit(runtime.NumCPU()) + require.NoError(t, err) + require.Equal(t, 0, len(storage.deltas)) + + // Check encoded data + baseStorage := storage.baseStorage.(*InMemBaseStorage) + require.Equal(t, 4, len(baseStorage.segments)) + + savedData, found, err := baseStorage.Retrieve(nestedContainerRootID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, fixedData[nestedContainerRootID], savedData) + }) +} From 2a71edc4f15c73c07b73f2340281ab45d2402f9c Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Thu, 11 Apr 2024 15:45:40 -0500 Subject: [PATCH 2/6] Bump Go to 1.20 in coverage.yml --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 88b4907a..0ec246bc 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-go@v4 with: - go-version: '1.19' + go-version: '1.20' check-latest: true - name: Get dependencies From 66477156165abfe3c46ac869272f4aeaf020d229 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Mon, 15 Apr 2024 09:23:01 -0500 Subject: [PATCH 3/6] Update storage.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Müller --- storage.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/storage.go b/storage.go index 9b4f5a5d..fb0f206c 100644 --- a/storage.go +++ b/storage.go @@ -1229,6 +1229,8 @@ func (s *PersistentSlabStorage) FixLoadedBrokenReferences() ([]SlabID, error) { func (s *PersistentSlabStorage) fixBrokenReferencesInMap(old MapSlab) error { id := old.SlabID() + oldExtraData := old.ExtraData() + // Create an empty map with the same StorgeID, type, and seed as the old map. new := &MapDataSlab{ header: MapSlabHeader{ @@ -1236,8 +1238,8 @@ func (s *PersistentSlabStorage) fixBrokenReferencesInMap(old MapSlab) error { size: mapRootDataSlabPrefixSize + hkeyElementsPrefixSize, }, extraData: &MapExtraData{ - TypeInfo: old.ExtraData().TypeInfo, - Seed: old.ExtraData().Seed, + TypeInfo: oldExtraData.TypeInfo, + Seed: oldExtraData.Seed, }, elements: newHkeyElements(0), } From 3efd6bed83271ff396879fc0fb60e43c4c3522c3 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:25:22 -0500 Subject: [PATCH 4/6] Allow callers to skip fixing a broken reference Currently, calls to this function can be limited by migration programs by specifying the 9 testnet accounts affected by 10 registers with broken references on testnet. This commit allows callers to implement additional restrictions, so calling FixLoadedBrokenReferences for the affected 9 testnet accounts can be even more limited at the callers discretion. In practice, this change is not expected to produce different migration results because full migration tests before this change correctly fixed the 10 known registers in the 9 testnet accounts. This commit added predicate func(old Value) bool to PersistentSlabStorage.FixLoadedBrokenReferences() to control whether to fix a atree.Value containing broken references. Also modified PersistentSlabStorage.FixLoadedBrokenReferences() to return fixed storage IDs and skipped storage IDs. Both returned values are of type map[StorageID][]StorageID, with key as root slab ID and value as all slab IDs containing broken references connected to the root. Also added more tests and improved existing tests. --- storage.go | 51 +++-- storage_test.go | 514 +++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 520 insertions(+), 45 deletions(-) diff --git a/storage.go b/storage.go index fb0f206c..e8cd23ff 100644 --- a/storage.go +++ b/storage.go @@ -1101,7 +1101,11 @@ func storeSlab(storage SlabStorage, slab Slab) error { // fix broken references. As of April 2024, only 10 registers in testnet (not mainnet) // were found to have broken references and they seem to have resulted from a bug // that was fixed 2 years ago by https://github.com/onflow/cadence/pull/1565. -func (s *PersistentSlabStorage) FixLoadedBrokenReferences() ([]SlabID, error) { +func (s *PersistentSlabStorage) FixLoadedBrokenReferences(needToFix func(old Value) bool) ( + fixedSlabIDs map[SlabID][]SlabID, // key: root slab ID, value: slab IDs containing broken refs + skippedSlabIDs map[SlabID][]SlabID, // key: root slab ID, value: slab IDs containing broken refs + err error, +) { // parentOf is used to find root slab from non-root slab. // Broken reference can be in non-root slab, and we need SlabID of root slab @@ -1160,12 +1164,12 @@ func (s *PersistentSlabStorage) FixLoadedBrokenReferences() ([]SlabID, error) { return foundBrokenRef } - var brokenStorageIDs []SlabID + var brokenSlabIDs []SlabID // Iterate delta slabs. for id, slab := range s.deltas { if hasBrokenReferenceInSlab(id, slab) { - brokenStorageIDs = append(brokenStorageIDs, id) + brokenSlabIDs = append(brokenSlabIDs, id) } } @@ -1175,28 +1179,28 @@ func (s *PersistentSlabStorage) FixLoadedBrokenReferences() ([]SlabID, error) { continue } if hasBrokenReferenceInSlab(id, slab) { - brokenStorageIDs = append(brokenStorageIDs, id) + brokenSlabIDs = append(brokenSlabIDs, id) } } - if len(brokenStorageIDs) == 0 { - return nil, nil + if len(brokenSlabIDs) == 0 { + return nil, nil, nil } - rootSlabStorageIDsWithBrokenData := make(map[SlabID]struct{}) + rootSlabIDsWithBrokenData := make(map[SlabID][]SlabID) var errs []error - // Find StorageIDs of root slab for slabs containing broken references. - for _, id := range brokenStorageIDs { + // Find SlabIDs of root slab for slabs containing broken references. + for _, id := range brokenSlabIDs { rootID := getRootSlabID(id) if rootID == SlabIDUndefined { errs = append(errs, fmt.Errorf("failed to get root slab id for slab %s", id)) continue } - rootSlabStorageIDsWithBrokenData[rootID] = struct{}{} + rootSlabIDsWithBrokenData[rootID] = append(rootSlabIDsWithBrokenData[rootID], id) } - for rootSlabID := range rootSlabStorageIDsWithBrokenData { + for rootSlabID, brokenSlabIDs := range rootSlabIDsWithBrokenData { rootSlab := s.RetrieveIfLoaded(rootSlabID) if rootSlab == nil { errs = append(errs, fmt.Errorf("failed to retrieve loaded root slab %s", rootSlabID)) @@ -1205,14 +1209,23 @@ func (s *PersistentSlabStorage) FixLoadedBrokenReferences() ([]SlabID, error) { switch rootSlab := rootSlab.(type) { case MapSlab: - if rootSlab.ExtraData() == nil { - errs = append(errs, fmt.Errorf("failed to fix broken references because slab %s isn't root slab", rootSlab.SlabID())) + value, err := rootSlab.StoredValue(s) + if err != nil { + errs = append(errs, fmt.Errorf("failed to convert slab %s into value", rootSlab.SlabID())) continue } - err := s.fixBrokenReferencesInMap(rootSlab) - if err != nil { - errs = append(errs, err) + if needToFix(value) { + err := s.fixBrokenReferencesInMap(rootSlab) + if err != nil { + errs = append(errs, err) + continue + } + } else { + if skippedSlabIDs == nil { + skippedSlabIDs = make(map[SlabID][]SlabID) + } + skippedSlabIDs[rootSlabID] = brokenSlabIDs } default: @@ -1221,7 +1234,11 @@ func (s *PersistentSlabStorage) FixLoadedBrokenReferences() ([]SlabID, error) { } } - return brokenStorageIDs, errors.Join(errs...) + for id := range skippedSlabIDs { + delete(rootSlabIDsWithBrokenData, id) + } + + return rootSlabIDsWithBrokenData, skippedSlabIDs, errors.Join(errs...) } // fixBrokenReferencesInMap replaces replaces broken map with empty map diff --git a/storage_test.go b/storage_test.go index 9852cd64..7dadeb04 100644 --- a/storage_test.go +++ b/storage_test.go @@ -1665,10 +1665,27 @@ func TestFixLoadedBrokenReferences(t *testing.T) { require.True(t, found) } + var fixedRootIDs map[SlabID][]SlabID + var skippedRootIDs map[SlabID][]SlabID + + // Don't fix any broken references + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool { + return false + }) + require.NoError(t, err) + require.Equal(t, 0, len(fixedRootIDs)) + require.Equal(t, 0, len(skippedRootIDs)) + + // No data is modified because no fix happened + require.Equal(t, 0, len(storage.deltas)) + // Fix broken reference - fixedIDs, err := storage.FixLoadedBrokenReferences() + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool { + return true + }) require.NoError(t, err) - require.Equal(t, 0, len(fixedIDs)) + require.Equal(t, 0, len(fixedRootIDs)) + require.Equal(t, 0, len(skippedRootIDs)) // No data is modified during fixing broken reference require.Equal(t, 0, len(storage.deltas)) @@ -1684,7 +1701,9 @@ func TestFixLoadedBrokenReferences(t *testing.T) { rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} - brokenRefSlabID := rootID + brokenRefs := map[SlabID][]SlabID{ + rootID: {rootID}, + } data := map[SlabID][]byte{ rootID: { @@ -1777,11 +1796,36 @@ func TestFixLoadedBrokenReferences(t *testing.T) { _, err := CheckStorageHealth(storage, -1) require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") - // Fix broken reference - fixedIDs, err := storage.FixLoadedBrokenReferences() + var fixedRootIDs map[SlabID][]SlabID + var skippedRootIDs map[SlabID][]SlabID + + // Don't fix any broken references + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool { + return false + }) + require.NoError(t, err) + require.Equal(t, 0, len(fixedRootIDs)) + require.Equal(t, len(brokenRefs), len(skippedRootIDs)) + + for rootID, slabIDsWithBrokenRef := range brokenRefs { + require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) + } + + // No data is modified because no fix happened + require.Equal(t, 0, len(storage.deltas)) + + // Fix broken references + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool { + return true + }) require.NoError(t, err) - require.Equal(t, 1, len(fixedIDs)) - require.Equal(t, brokenRefSlabID, fixedIDs[0]) + require.Equal(t, len(brokenRefs), len(fixedRootIDs)) + + for rootID, slabIDsWithBrokenRef := range brokenRefs { + require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) + } + + require.Equal(t, 0, len(skippedRootIDs)) require.Equal(t, 1, len(storage.deltas)) // Check health after fixing broken reference @@ -1812,7 +1856,9 @@ func TestFixLoadedBrokenReferences(t *testing.T) { nonRootDataID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} nonRootDataID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - brokenRefSlabID := nonRootDataID2 + brokenRefs := map[SlabID][]SlabID{ + rootID: {nonRootDataID2}, + } // Expected serialized slab data with storage id data := map[SlabID][]byte{ @@ -1995,11 +2041,36 @@ func TestFixLoadedBrokenReferences(t *testing.T) { _, err := CheckStorageHealth(storage, -1) require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") + var fixedRootIDs map[SlabID][]SlabID + var skippedRootIDs map[SlabID][]SlabID + + // Don't fix any broken references + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool { + return false + }) + require.NoError(t, err) + require.Equal(t, 0, len(fixedRootIDs)) + require.Equal(t, len(brokenRefs), len(skippedRootIDs)) + + for rootID, slabIDsWithBrokenRef := range brokenRefs { + require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) + } + + // No data is modified because no fix happened + require.Equal(t, 0, len(storage.deltas)) + // Fix broken reference - fixedIDs, err := storage.FixLoadedBrokenReferences() + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool { + return true + }) require.NoError(t, err) - require.Equal(t, 1, len(fixedIDs)) - require.Equal(t, brokenRefSlabID, fixedIDs[0]) + require.Equal(t, 1, len(fixedRootIDs)) + + for rootID, slabIDsWithBrokenRef := range brokenRefs { + require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) + } + + require.Equal(t, 0, len(skippedRootIDs)) require.Equal(t, 3, len(storage.deltas)) // Check health after fixing broken reference @@ -2030,7 +2101,9 @@ func TestFixLoadedBrokenReferences(t *testing.T) { nonRootDataID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} nonRootDataID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - brokenRefSlabID := []SlabID{nonRootDataID1, nonRootDataID2} + brokenRefs := map[SlabID][]SlabID{ + rootID: {nonRootDataID1, nonRootDataID2}, + } data := map[SlabID][]byte{ @@ -2212,23 +2285,38 @@ func TestFixLoadedBrokenReferences(t *testing.T) { _, err := CheckStorageHealth(storage, -1) require.ErrorContains(t, err, "slab not found during slab iteration") + var fixedRootIDs map[SlabID][]SlabID + var skippedRootIDs map[SlabID][]SlabID + + // Don't fix any broken references + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool { + return false + }) + require.NoError(t, err) + require.Equal(t, 0, len(fixedRootIDs)) + require.Equal(t, len(brokenRefs), len(skippedRootIDs)) + + for rootID, slabIDsWithBrokenRef := range brokenRefs { + require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) + } + + // No data is modified because no fix happened + require.Equal(t, 0, len(storage.deltas)) + // Fix broken reference - fixedIDs, err := storage.FixLoadedBrokenReferences() + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool { + return true + }) require.NoError(t, err) - require.Equal(t, len(brokenRefSlabID), len(fixedIDs)) - require.Equal(t, 3, len(storage.deltas)) + require.Equal(t, len(brokenRefs), len(fixedRootIDs)) - for _, expected := range brokenRefSlabID { - var found bool - for _, actual := range fixedIDs { - if actual == expected { - found = true - break - } - } - require.True(t, found) + for rootID, slabIDsWithBrokenRef := range brokenRefs { + require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) } + require.Equal(t, 0, len(skippedRootIDs)) + require.Equal(t, 3, len(storage.deltas)) + // Check health after fixing broken reference rootIDs, err := CheckStorageHealth(storage, -1) require.NoError(t, err) @@ -2258,7 +2346,9 @@ func TestFixLoadedBrokenReferences(t *testing.T) { nonRootDataID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} nestedContainerRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}} - brokenRefSlabID := []SlabID{nestedContainerRootID} + brokenRefs := map[SlabID][]SlabID{ + nestedContainerRootID: {nestedContainerRootID}, + } data := map[SlabID][]byte{ @@ -2484,11 +2574,36 @@ func TestFixLoadedBrokenReferences(t *testing.T) { _, err := CheckStorageHealth(storage, -1) require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration") + var fixedRootIDs map[SlabID][]SlabID + var skippedRootIDs map[SlabID][]SlabID + + // Don't fix any broken references + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool { + return false + }) + require.NoError(t, err) + require.Equal(t, 0, len(fixedRootIDs)) + require.Equal(t, len(brokenRefs), len(skippedRootIDs)) + + for rootID, slabIDsWithBrokenRef := range brokenRefs { + require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) + } + + // No data is modified because no fix happened + require.Equal(t, 0, len(storage.deltas)) + // Fix broken reference - fixedIDs, err := storage.FixLoadedBrokenReferences() + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool { + return true + }) require.NoError(t, err) - require.Equal(t, len(brokenRefSlabID), len(fixedIDs)) - require.Equal(t, brokenRefSlabID[0], fixedIDs[0]) + require.Equal(t, len(brokenRefs), len(fixedRootIDs)) + + for rootID, slabIDsWithBrokenRef := range brokenRefs { + require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID]) + } + + require.Equal(t, 0, len(skippedRootIDs)) require.Equal(t, 1, len(storage.deltas)) // Check health after fixing broken reference @@ -2513,4 +2628,347 @@ func TestFixLoadedBrokenReferences(t *testing.T) { require.True(t, found) require.Equal(t, fixedData[nestedContainerRootID], savedData) }) + + t.Run("selectively fix maps", func(t *testing.T) { + rootID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootDataID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootDataID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} // containing broken ref + + rootID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}} // containing broken ref + + rootIDs := []SlabID{rootID1, rootID2} + + brokenRefs := map[SlabID][]SlabID{ + rootID1: {nonRootDataID2}, + rootID2: {rootID2}, + } + + data := map[SlabID][]byte{ + // metadata slab + rootID1: { + // extra data + // version + 0x00, + // flag: root + map meta + 0x89, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info: "map" + 0x18, 0x2A, + // count: 8 + 0x08, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + meta + 0x89, + // child header count + 0x00, 0x02, + // child header 1 (storage id, first key, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x02, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0xfe, + }, + + // data slab + nonRootDataID1: { + // version + 0x00, + // flag: map data + 0x08, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // hkey: 1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // hkey: 2 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + // hkey: 3 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] + 0x82, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] + 0x82, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] + 0x82, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] + 0x82, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + }, + + // data slab + nonRootDataID2: { + // version + 0x00, + // flag: has pointer + map data + 0x48, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 4 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // hkey: 5 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + // hkey: 6 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // hkey: 7 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] + 0x82, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] + 0x82, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] + 0x82, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + // element: [hhhhhhhhhhhhhhhhhhhhhh:SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] + 0x82, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + }, + + // map data slab + rootID2: { + // extra data + // version + 0x00, + // flag: root + map data + 0x88, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info + 0x18, 0x2a, + // count: 1 + 0x01, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x4a, + + // version + 0x00, + // flag: root + map data + 0x88, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // elements (array of 1 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // element: [uint64(0):SlabID(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1)] + 0x82, + 0xd8, 0xa4, 0x00, + 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + }, + } + + fixedData := map[SlabID][]byte{ + rootID1: { + // version + 0x10, + // flag: root + map data + 0x88, + + // extra data + // CBOR encoded array of 3 elements + 0x83, + // type info + 0x18, 0x2a, + // count: 0 + 0x00, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x59, 0x00, 0x00, + + // elements (array of 0 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x99, 0x00, 0x00, + }, + + rootID2: { + // version + 0x10, + // flag: root + map data + 0x88, + + // extra data + // CBOR encoded array of 3 elements + 0x83, + // type info + 0x18, 0x2a, + // count: 0 + 0x00, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x4a, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x59, 0x00, 0x00, + + // elements (array of 0 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x99, 0x00, 0x00, + }, + } + + storage := newTestPersistentStorageWithData(t, data) + + // Load data in storage + for id := range data { + _, found, err := storage.Retrieve(id) + require.NoError(t, err) + require.True(t, found) + } + + // Check health before fixing broken reference + _, err := CheckStorageHealth(storage, -1) + require.ErrorContains(t, err, "slab not found during slab iteration") + + var fixedRootIDs map[SlabID][]SlabID + var skippedRootIDs map[SlabID][]SlabID + + // Don't fix any broken references + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool { + return false + }) + require.NoError(t, err) + require.Equal(t, 0, len(fixedRootIDs)) + require.Equal(t, len(brokenRefs), len(skippedRootIDs)) + + for rootID, slabIDsWithBrokenRef := range brokenRefs { + require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID]) + } + + // No data is modified because no fix happened + require.Equal(t, 0, len(storage.deltas)) + + // Only fix one map with broken reference + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(v Value) bool { + m, ok := v.(*OrderedMap) + require.True(t, ok) + return rootID1 == m.SlabID() + }) + require.NoError(t, err) + require.Equal(t, 1, len(fixedRootIDs)) + require.Equal(t, brokenRefs[rootID1], fixedRootIDs[rootID1]) + require.Equal(t, 1, len(skippedRootIDs)) + require.Equal(t, brokenRefs[rootID2], skippedRootIDs[rootID2]) + require.Equal(t, 3, len(storage.deltas)) + + // Save data in storage + err = storage.FastCommit(runtime.NumCPU()) + require.NoError(t, err) + require.Equal(t, 0, len(storage.deltas)) + + // Check health after only fixing one map with broken reference + _, err = CheckStorageHealth(storage, -1) + require.ErrorContains(t, err, "slab not found during slab iteration") + + // Fix remaining map with broken reference + fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(v Value) bool { + return true + }) + require.NoError(t, err) + require.Equal(t, 1, len(fixedRootIDs)) + require.Equal(t, brokenRefs[rootID2], fixedRootIDs[rootID2]) + require.Equal(t, 0, len(skippedRootIDs)) + require.Equal(t, 1, len(storage.deltas)) + + // Check health after fixing remaining maps with broken reference + returnedRootIDs, err := CheckStorageHealth(storage, -1) + require.NoError(t, err) + require.Equal(t, len(rootIDs), len(returnedRootIDs)) + + // Save data in storage + err = storage.FastCommit(runtime.NumCPU()) + require.NoError(t, err) + require.Equal(t, 0, len(storage.deltas)) + + // Check encoded data + baseStorage := storage.baseStorage.(*InMemBaseStorage) + require.Equal(t, 2, len(baseStorage.segments)) + + savedData, found, err := baseStorage.Retrieve(rootID1) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, fixedData[rootID1], savedData) + + savedData, found, err = baseStorage.Retrieve(rootID2) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, fixedData[rootID2], savedData) + }) } From b362f829eaad900199a708c47bee2ec2a16f674d Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:19:27 -0500 Subject: [PATCH 5/6] Add PersistentSlabStorage.GetAllChildReferences() This commit adds GetAllChildReferences() which essentially wraps an existing internal function. --- storage.go | 17 + storage_test.go | 1488 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1505 insertions(+) diff --git a/storage.go b/storage.go index e8cd23ff..e8843fac 100644 --- a/storage.go +++ b/storage.go @@ -1297,6 +1297,23 @@ func (s *PersistentSlabStorage) existIfLoaded(id SlabID) bool { return false } +// GetAllChildReferences returns child references of given slab (all levels), +// including nested container and theirs child references. +func (s *PersistentSlabStorage) GetAllChildReferences(id StorageID) ( + references []StorageID, + brokenReferences []StorageID, + err error, +) { + slab, found, err := s.Retrieve(id) + if err != nil { + return nil, nil, err + } + if !found { + return nil, nil, NewSlabNotFoundErrorf(id, fmt.Sprintf("failed to get root slab by id %s", id)) + } + return s.getAllChildReferences(slab) +} + // getAllChildReferences returns child references of given slab (all levels). func (s *PersistentSlabStorage) getAllChildReferences(slab Slab) ( references []SlabID, diff --git a/storage_test.go b/storage_test.go index 7dadeb04..74ff3a25 100644 --- a/storage_test.go +++ b/storage_test.go @@ -2972,3 +2972,1491 @@ func TestFixLoadedBrokenReferences(t *testing.T) { require.Equal(t, fixedData[rootID2], savedData) }) } + +func TestGetAllChildReferencesFromArray(t *testing.T) { + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + t.Run("empty", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + + expectedRefIDs := []StorageID{} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + rootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x00, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root data slab without refs", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + + expectedRefIDs := []StorageID{} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + rootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xa4, 0x00, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root data slab with ref to nested element", func(t *testing.T) { + parentRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + + expectedRefIDs := []StorageID{childRootID} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + parentRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + }, + + childRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xa4, 0x00, + }, + } + + testGetAllChildReferences(t, data, parentRootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root data slab with broken ref", func(t *testing.T) { + parentRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + + expectedRefIDs := []StorageID{} + expectedBrokenRefIDs := []StorageID{childRootID} + + data := map[StorageID][]byte{ + parentRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + }, + } + + testGetAllChildReferences(t, data, parentRootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root metadata slab", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + expectedRefIDs := []StorageID{nonRootID1, nonRootID2} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + rootID: { + // extra data + // version + 0x00, + // extra data flag + 0x81, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array meta data slab flag + 0x81, + // child header count + 0x00, 0x02, + // child header 1 (storage id, count, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0xe4, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x01, 0x0e, + }, + + // (data slab) next: 3, data: [aaaaaaaaaaaaaaaaaaaaaa ... aaaaaaaaaaaaaaaaaaaaaa] + nonRootID1: { + // version + 0x00, + // array data slab flag + 0x00, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x09, + // CBOR encoded array elements + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + }, + + // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... StorageID(...)] + nonRootID2: { + // version + 0x00, + // array data slab flag + 0x40, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x0b, + // CBOR encoded array elements + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root metadata slab with broken ref to first data slab", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + expectedRefIDs := []StorageID{nonRootID2} + expectedBrokenRefIDs := []StorageID{nonRootID1} + + data := map[StorageID][]byte{ + rootID: { + // extra data + // version + 0x00, + // extra data flag + 0x81, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array meta data slab flag + 0x81, + // child header count + 0x00, 0x02, + // child header 1 (storage id, count, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0xe4, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x01, 0x0e, + }, + + // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... StorageID(...)] + nonRootID2: { + // version + 0x00, + // array data slab flag + 0x40, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x0b, + // CBOR encoded array elements + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root metadata slab with ref", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 4}} + + expectedRefIDs := []StorageID{nonRootID1, nonRootID2, childRootID} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + rootID: { + // extra data + // version + 0x00, + // extra data flag + 0x81, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array meta data slab flag + 0x81, + // child header count + 0x00, 0x02, + // child header 1 (storage id, count, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0xe4, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x01, 0x0e, + }, + + // (data slab) next: 3, data: [aaaaaaaaaaaaaaaaaaaaaa ... aaaaaaaaaaaaaaaaaaaaaa] + nonRootID1: { + // version + 0x00, + // array data slab flag + 0x00, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x09, + // CBOR encoded array elements + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + }, + + // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... StorageID(...)] + nonRootID2: { + // version + 0x00, + // array data slab flag + 0x40, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x0b, + // CBOR encoded array elements + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + }, + + childRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xa4, 0x00, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root metadata slab with broken ref to nested element", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 4}} + + expectedRefIDs := []StorageID{nonRootID1, nonRootID2} + expectedBrokenRefIDs := []StorageID{childRootID} + + data := map[StorageID][]byte{ + rootID: { + // extra data + // version + 0x00, + // extra data flag + 0x81, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array meta data slab flag + 0x81, + // child header count + 0x00, 0x02, + // child header 1 (storage id, count, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0xe4, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x01, 0x0e, + }, + + // (data slab) next: 3, data: [aaaaaaaaaaaaaaaaaaaaaa ... aaaaaaaaaaaaaaaaaaaaaa] + nonRootID1: { + // version + 0x00, + // array data slab flag + 0x00, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x09, + // CBOR encoded array elements + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + }, + + // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... StorageID(...)] + nonRootID2: { + // version + 0x00, + // array data slab flag + 0x40, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x0b, + // CBOR encoded array elements + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("3-level of nested containers", func(t *testing.T) { + parentRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + gchildRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + expectedRefIDs := []StorageID{childRootID, gchildRootID} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + parentRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + }, + + childRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + }, + + gchildRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xa4, 0x00, + }, + } + + testGetAllChildReferences(t, data, parentRootID, expectedRefIDs, expectedBrokenRefIDs) + }) +} + +func TestGetAllChildReferencesFromMap(t *testing.T) { + address := Address{1, 2, 3, 4, 5, 6, 7, 8} + + t.Run("empty", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + + expectedRefIDs := []StorageID{} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + rootID: { + // extra data + // version + 0x00, + // flag: root + map data + 0x88, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info + 0x18, 0x2a, + // count: 0 + 0x00, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + map data + 0x88, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // elements (array of 0 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root data slab without refs", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + + expectedRefIDs := []StorageID{} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + rootID: { + // extra data + // version + 0x00, + // flag: root + map data + 0x88, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info + 0x18, 0x2a, + // count: 1 + 0x01, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + map data + 0x88, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // elements (array of 1 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // element: [uint64(0):uint64(0)] + 0x82, 0xd8, 0xa4, 0x00, 0xd8, 0xa4, 0x00, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root data slab with ref", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + + expectedRefIDs := []StorageID{childRootID} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + rootID: { + // extra data + // version + 0x00, + // flag: root + map data + 0x88, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info + 0x18, 0x2a, + // count: 1 + 0x01, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + map data + 0x88, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // elements (array of 1 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // element: [uint64(0):uint64(0)] + 0x82, + 0xd8, 0xa4, 0x00, + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + }, + + childRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xa4, 0x00, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root data slab with broken ref", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + + expectedRefIDs := []StorageID{} + expectedBrokenRefIDs := []StorageID{childRootID} + + data := map[StorageID][]byte{ + rootID: { + // extra data + // version + 0x00, + // flag: root + map data + 0x88, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info + 0x18, 0x2a, + // count: 1 + 0x01, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + map data + 0x88, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // elements (array of 1 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // element: [uint64(0):uint64(0)] + 0x82, + 0xd8, 0xa4, 0x00, + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root metadata slab", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + expectedRefIDs := []StorageID{nonRootID1, nonRootID2} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + // metadata slab + rootID: { + // extra data + // version + 0x00, + // flag: root + map meta + 0x89, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info: "map" + 0x18, 0x2A, + // count: 8 + 0x08, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + meta + 0x89, + // child header count + 0x00, 0x02, + // child header 1 (storage id, first key, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x02, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0xfe, + }, + + // data slab + nonRootID1: { + // version + 0x00, + // flag: map data + 0x08, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // hkey: 1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // hkey: 2 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + // hkey: 3 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] + 0x82, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] + 0x82, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] + 0x82, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] + 0x82, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + }, + + // data slab + nonRootID2: { + // version + 0x00, + // flag: has pointer + map data + 0x48, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 4 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // hkey: 5 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + // hkey: 6 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // hkey: 7 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] + 0x82, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] + 0x82, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] + 0x82, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + // element: [hhhhhhhhhhhhhhhhhhhhhh:hhhhhhhhhhhhhhhhhhhhhh] + 0x82, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root metadata slab with broken ref to first data slab", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + expectedRefIDs := []StorageID{nonRootID2} + expectedBrokenRefIDs := []StorageID{nonRootID1} + + data := map[StorageID][]byte{ + // metadata slab + rootID: { + // extra data + // version + 0x00, + // flag: root + map meta + 0x89, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info: "map" + 0x18, 0x2A, + // count: 8 + 0x08, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + meta + 0x89, + // child header count + 0x00, 0x02, + // child header 1 (storage id, first key, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x02, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0xfe, + }, + + // data slab + nonRootID2: { + // version + 0x00, + // flag: has pointer + map data + 0x48, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 4 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // hkey: 5 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + // hkey: 6 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // hkey: 7 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] + 0x82, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] + 0x82, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] + 0x82, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + // element: [hhhhhhhhhhhhhhhhhhhhhh:hhhhhhhhhhhhhhhhhhhhhh] + 0x82, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root metadata slab with ref", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 4}} + + expectedRefIDs := []StorageID{nonRootID1, nonRootID2, childRootID} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + // metadata slab + rootID: { + // extra data + // version + 0x00, + // flag: root + map meta + 0x89, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info: "map" + 0x18, 0x2A, + // count: 8 + 0x08, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + meta + 0x89, + // child header count + 0x00, 0x02, + // child header 1 (storage id, first key, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x02, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0xfe, + }, + + // data slab + nonRootID1: { + // version + 0x00, + // flag: map data + 0x08, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // hkey: 1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // hkey: 2 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + // hkey: 3 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] + 0x82, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] + 0x82, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] + 0x82, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] + 0x82, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + }, + + // data slab + nonRootID2: { + // version + 0x00, + // flag: has pointer + map data + 0x48, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 4 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // hkey: 5 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + // hkey: 6 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // hkey: 7 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] + 0x82, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] + 0x82, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] + 0x82, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + // element: [hhhhhhhhhhhhhhhhhhhhhh:hhhhhhhhhhhhhhhhhhhhhh] + 0x82, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + }, + + childRootID: { + // extra data + // version + 0x00, + // flag: root + map data + 0x88, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info + 0x18, 0x2a, + // count: 0 + 0x00, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + map data + 0x88, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // elements (array of 0 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("root metadata slab with broken ref to nested element", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 4}} + + expectedRefIDs := []StorageID{nonRootID1, nonRootID2} + expectedBrokenRefIDs := []StorageID{childRootID} + + data := map[StorageID][]byte{ + // metadata slab + rootID: { + // extra data + // version + 0x00, + // flag: root + map meta + 0x89, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info: "map" + 0x18, 0x2A, + // count: 8 + 0x08, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + meta + 0x89, + // child header count + 0x00, 0x02, + // child header 1 (storage id, first key, size) + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x02, + // child header 2 + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0xfe, + }, + + // data slab + nonRootID1: { + // version + 0x00, + // flag: map data + 0x08, + // next storage id + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // hkey: 1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // hkey: 2 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + // hkey: 3 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [aaaaaaaaaaaaaaaaaaaaaa:aaaaaaaaaaaaaaaaaaaaaa] + 0x82, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + // element: [bbbbbbbbbbbbbbbbbbbbbb:bbbbbbbbbbbbbbbbbbbbbb] + 0x82, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + 0x76, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, + // element: [cccccccccccccccccccccc:cccccccccccccccccccccc] + 0x82, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + 0x76, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, + // element: [dddddddddddddddddddddd:dddddddddddddddddddddd] + 0x82, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + 0x76, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, 0x64, + }, + + // data slab + nonRootID2: { + // version + 0x00, + // flag: has pointer + map data + 0x48, + // next storage id + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 4) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // hkey: 4 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // hkey: 5 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + // hkey: 6 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // hkey: 7 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + + // elements (array of 4 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + // element: [eeeeeeeeeeeeeeeeeeeeee:eeeeeeeeeeeeeeeeeeeeee] + 0x82, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + 0x76, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, 0x65, + // element: [ffffffffffffffffffffff:ffffffffffffffffffffff] + 0x82, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, + // element: [gggggggggggggggggggggg:gggggggggggggggggggggg] + 0x82, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + 0x76, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, 0x67, + // element: [hhhhhhhhhhhhhhhhhhhhhh:hhhhhhhhhhhhhhhhhhhhhh] + 0x82, + 0x76, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, 0x68, + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) + + t.Run("3-level containers", func(t *testing.T) { + rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + gchildRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + + expectedRefIDs := []StorageID{childRootID, gchildRootID} + expectedBrokenRefIDs := []StorageID{} + + data := map[StorageID][]byte{ + rootID: { + // extra data + // version + 0x00, + // flag: root + map data + 0x88, + // extra data (CBOR encoded array of 3 elements) + 0x83, + // type info + 0x18, 0x2a, + // count: 1 + 0x01, + // seed + 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, + + // version + 0x00, + // flag: root + map data + 0x88, + + // the following encoded data is valid CBOR + + // elements (array of 3 elements) + 0x83, + + // level: 0 + 0x00, + + // hkeys (byte string of length 8 * 1) + 0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + // hkey: 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // elements (array of 1 elements) + // each element is encoded as CBOR array of 2 elements (key, value) + 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + // element: [uint64(0):uint64(0)] + 0x82, + 0xd8, 0xa4, 0x00, + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + }, + + childRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + }, + + gchildRootID: { + // extra data + // version + 0x00, + // extra data flag + 0x80, + // array of extra data + 0x81, + // type info + 0x18, 0x2a, + + // version + 0x00, + // array data slab flag + 0x80, + // CBOR encoded array head (fixed size 3 byte) + 0x99, 0x00, 0x01, + // CBOR encoded array elements + 0xd8, 0xa4, 0x00, + }, + } + + testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs) + }) +} + +func testGetAllChildReferences( + t *testing.T, + data map[StorageID][]byte, + rootID StorageID, + expectedRefIDs []StorageID, + expectedBrokenRefIDs []StorageID, +) { + storage := newTestPersistentStorageWithData(t, data) + + refs, brokenRefs, err := storage.GetAllChildReferences(rootID) + require.NoError(t, err) + + require.Equal(t, len(expectedRefIDs), len(refs)) + require.ElementsMatch(t, expectedRefIDs, refs) + + require.Equal(t, len(expectedBrokenRefIDs), len(brokenRefs)) + require.ElementsMatch(t, expectedBrokenRefIDs, brokenRefs) +} From 8febf26a11258c0dd0e0dd3adba27f40b6740390 Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:25:35 -0500 Subject: [PATCH 6/6] Fix tests --- storage.go | 6 +- storage_test.go | 216 ++++++++++++++++++++++++------------------------ 2 files changed, 111 insertions(+), 111 deletions(-) diff --git a/storage.go b/storage.go index e8843fac..394ecf23 100644 --- a/storage.go +++ b/storage.go @@ -1299,9 +1299,9 @@ func (s *PersistentSlabStorage) existIfLoaded(id SlabID) bool { // GetAllChildReferences returns child references of given slab (all levels), // including nested container and theirs child references. -func (s *PersistentSlabStorage) GetAllChildReferences(id StorageID) ( - references []StorageID, - brokenReferences []StorageID, +func (s *PersistentSlabStorage) GetAllChildReferences(id SlabID) ( + references []SlabID, + brokenReferences []SlabID, err error, ) { slab, found, err := s.Retrieve(id) diff --git a/storage_test.go b/storage_test.go index 74ff3a25..1b407fb4 100644 --- a/storage_test.go +++ b/storage_test.go @@ -2977,12 +2977,12 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { address := Address{1, 2, 3, 4, 5, 6, 7, 8} t.Run("empty", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} - expectedRefIDs := []StorageID{} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ rootID: { // extra data // version @@ -3007,12 +3007,12 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { }) t.Run("root data slab without refs", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} - expectedRefIDs := []StorageID{} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ rootID: { // extra data // version @@ -3039,13 +3039,13 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { }) t.Run("root data slab with ref to nested element", func(t *testing.T) { - parentRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + parentRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} - expectedRefIDs := []StorageID{childRootID} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{childRootID} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ parentRootID: { // extra data // version @@ -3093,13 +3093,13 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { }) t.Run("root data slab with broken ref", func(t *testing.T) { - parentRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + parentRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} - expectedRefIDs := []StorageID{} - expectedBrokenRefIDs := []StorageID{childRootID} + expectedRefIDs := []SlabID{} + expectedBrokenRefIDs := []SlabID{childRootID} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ parentRootID: { // extra data // version @@ -3126,14 +3126,14 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { }) t.Run("root metadata slab", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} - nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - expectedRefIDs := []StorageID{nonRootID1, nonRootID2} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{nonRootID1, nonRootID2} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ rootID: { // extra data // version @@ -3183,7 +3183,7 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, }, - // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... StorageID(...)] + // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... SlabID(...)] nonRootID2: { // version 0x00, @@ -3212,14 +3212,14 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { }) t.Run("root metadata slab with broken ref to first data slab", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} - nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - expectedRefIDs := []StorageID{nonRootID2} - expectedBrokenRefIDs := []StorageID{nonRootID1} + expectedRefIDs := []SlabID{nonRootID2} + expectedBrokenRefIDs := []SlabID{nonRootID1} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ rootID: { // extra data // version @@ -3247,7 +3247,7 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { 0x00, 0x00, 0x01, 0x0e, }, - // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... StorageID(...)] + // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... SlabID(...)] nonRootID2: { // version 0x00, @@ -3276,16 +3276,16 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { }) t.Run("root metadata slab with ref", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} - nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 4}} + childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}} - expectedRefIDs := []StorageID{nonRootID1, nonRootID2, childRootID} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{nonRootID1, nonRootID2, childRootID} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ rootID: { // extra data // version @@ -3335,7 +3335,7 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, }, - // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... StorageID(...)] + // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... SlabID(...)] nonRootID2: { // version 0x00, @@ -3385,16 +3385,16 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { }) t.Run("root metadata slab with broken ref to nested element", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} - nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 4}} + childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}} - expectedRefIDs := []StorageID{nonRootID1, nonRootID2} - expectedBrokenRefIDs := []StorageID{childRootID} + expectedRefIDs := []SlabID{nonRootID1, nonRootID2} + expectedBrokenRefIDs := []SlabID{childRootID} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ rootID: { // extra data // version @@ -3444,7 +3444,7 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { 0x76, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, }, - // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... StorageID(...)] + // (data slab) next: 0, data: [aaaaaaaaaaaaaaaaaaaaaa ... SlabID(...)] nonRootID2: { // version 0x00, @@ -3473,14 +3473,14 @@ func TestGetAllChildReferencesFromArray(t *testing.T) { }) t.Run("3-level of nested containers", func(t *testing.T) { - parentRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} - gchildRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + parentRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + gchildRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - expectedRefIDs := []StorageID{childRootID, gchildRootID} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{childRootID, gchildRootID} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ parentRootID: { // extra data // version @@ -3553,12 +3553,12 @@ func TestGetAllChildReferencesFromMap(t *testing.T) { address := Address{1, 2, 3, 4, 5, 6, 7, 8} t.Run("empty", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} - expectedRefIDs := []StorageID{} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ rootID: { // extra data // version @@ -3600,12 +3600,12 @@ func TestGetAllChildReferencesFromMap(t *testing.T) { }) t.Run("root data slab without refs", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} - expectedRefIDs := []StorageID{} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ rootID: { // extra data // version @@ -3651,13 +3651,13 @@ func TestGetAllChildReferencesFromMap(t *testing.T) { }) t.Run("root data slab with ref", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} - expectedRefIDs := []StorageID{childRootID} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{childRootID} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ rootID: { // extra data // version @@ -3726,13 +3726,13 @@ func TestGetAllChildReferencesFromMap(t *testing.T) { }) t.Run("root data slab with broken ref", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} - expectedRefIDs := []StorageID{} - expectedBrokenRefIDs := []StorageID{childRootID} + expectedRefIDs := []SlabID{} + expectedBrokenRefIDs := []SlabID{childRootID} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ rootID: { // extra data // version @@ -3780,14 +3780,14 @@ func TestGetAllChildReferencesFromMap(t *testing.T) { }) t.Run("root metadata slab", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} - nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - expectedRefIDs := []StorageID{nonRootID1, nonRootID2} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{nonRootID1, nonRootID2} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ // metadata slab rootID: { // extra data @@ -3924,14 +3924,14 @@ func TestGetAllChildReferencesFromMap(t *testing.T) { }) t.Run("root metadata slab with broken ref to first data slab", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} - nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - expectedRefIDs := []StorageID{nonRootID2} - expectedBrokenRefIDs := []StorageID{nonRootID1} + expectedRefIDs := []SlabID{nonRootID2} + expectedBrokenRefIDs := []SlabID{nonRootID1} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ // metadata slab rootID: { // extra data @@ -4019,16 +4019,16 @@ func TestGetAllChildReferencesFromMap(t *testing.T) { }) t.Run("root metadata slab with ref", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} - nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 4}} + childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}} - expectedRefIDs := []StorageID{nonRootID1, nonRootID2, childRootID} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{nonRootID1, nonRootID2, childRootID} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ // metadata slab rootID: { // extra data @@ -4200,16 +4200,16 @@ func TestGetAllChildReferencesFromMap(t *testing.T) { }) t.Run("root metadata slab with broken ref to nested element", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - nonRootID1 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} - nonRootID2 := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + nonRootID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + nonRootID2 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 4}} + childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 4}} - expectedRefIDs := []StorageID{nonRootID1, nonRootID2} - expectedBrokenRefIDs := []StorageID{childRootID} + expectedRefIDs := []SlabID{nonRootID1, nonRootID2} + expectedBrokenRefIDs := []SlabID{childRootID} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ // metadata slab rootID: { // extra data @@ -4345,14 +4345,14 @@ func TestGetAllChildReferencesFromMap(t *testing.T) { }) t.Run("3-level containers", func(t *testing.T) { - rootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 1}} - childRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 2}} - gchildRootID := StorageID{Address: address, Index: StorageIndex{0, 0, 0, 0, 0, 0, 0, 3}} + rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}} + childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}} + gchildRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 3}} - expectedRefIDs := []StorageID{childRootID, gchildRootID} - expectedBrokenRefIDs := []StorageID{} + expectedRefIDs := []SlabID{childRootID, gchildRootID} + expectedBrokenRefIDs := []SlabID{} - data := map[StorageID][]byte{ + data := map[SlabID][]byte{ rootID: { // extra data // version @@ -4444,10 +4444,10 @@ func TestGetAllChildReferencesFromMap(t *testing.T) { func testGetAllChildReferences( t *testing.T, - data map[StorageID][]byte, - rootID StorageID, - expectedRefIDs []StorageID, - expectedBrokenRefIDs []StorageID, + data map[SlabID][]byte, + rootID SlabID, + expectedRefIDs []SlabID, + expectedBrokenRefIDs []SlabID, ) { storage := newTestPersistentStorageWithData(t, data)