Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix migration filter for old unreferenced slabs (atree inlining feature branch) #396

Merged
merged 1 commit into from
Apr 29, 2024
Merged

Fix migration filter for old unreferenced slabs (atree inlining feature branch) #396

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 48 additions & 23 deletions storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1128,40 +1128,60 @@ func (s *PersistentSlabStorage) FixLoadedBrokenReferences(needToFix func(old Val
return false
}

var isMetaDataSlab bool

switch slab.(type) {
case *ArrayMetaDataSlab, *MapMetaDataSlab:
isMetaDataSlab = true
}
case *ArrayMetaDataSlab, *MapMetaDataSlab: // metadata slabs
var foundBrokenRef bool

var foundBrokenRef bool
for _, childStorable := range slab.ChildStorables() {
for _, childStorable := range slab.ChildStorables() {

slabIDStorable, ok := childStorable.(SlabIDStorable)
if !ok {
continue
}
if slabIDStorable, ok := childStorable.(SlabIDStorable); ok {

childID := SlabID(slabIDStorable)
childID := SlabID(slabIDStorable)

// Track parent-child relationship of root slabs and non-root slabs.
if isMetaDataSlab {
parentOf[childID] = id
}
// Track parent-child relationship of root slabs and non-root slabs.
parentOf[childID] = id

if s.existIfLoaded(childID) {
continue
if !s.existIfLoaded(childID) {
foundBrokenRef = true
}

// Continue with remaining child storables to track parent-child relationship.
}
}

foundBrokenRef = true
return foundBrokenRef

default: // data slabs
childStorables := slab.ChildStorables()

for len(childStorables) > 0 {

var nextChildStorables []Storable

for _, childStorable := range childStorables {

if slabIDStorable, ok := childStorable.(SlabIDStorable); ok {

if !isMetaDataSlab {
return true
if !s.existIfLoaded(SlabID(slabIDStorable)) {
return true
}

continue
}

// Append child storables of this childStorable to
// handle nested SlabIDStorable, such as Cadence SomeValue.
nextChildStorables = append(
nextChildStorables,
childStorable.ChildStorables()...,
)
}

childStorables = nextChildStorables
}
}

return foundBrokenRef
return false
}
}

var brokenSlabIDs []SlabID
Expand Down Expand Up @@ -1330,6 +1350,11 @@ func (s *PersistentSlabStorage) getAllChildReferences(slab Slab) (

slabIDStorable, ok := childStorable.(SlabIDStorable)
if !ok {
nextChildStorables = append(
nextChildStorables,
childStorable.ChildStorables()...,
)

continue
}

Expand Down
283 changes: 283 additions & 0 deletions storage_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1851,6 +1851,160 @@ func TestFixLoadedBrokenReferences(t *testing.T) {
require.Equal(t, fixedData[rootID], savedData)
})

t.Run("broken nested storable in root map data slab", func(t *testing.T) {

rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}}

brokenRefs := map[SlabID][]SlabID{
rootID: {rootID},
}

data := map[SlabID][]byte{
rootID: {
// extra data
// version
0x00,
// flag: root + map data
0x88,
// extra data (CBOR encoded array of 3 elements)
0x83,
// type info
0x18, 0x2a,
// count: 1
0x01,
// seed
0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49,

// version
0x00,
// flag: root + map data
0x88,

// the following encoded data is valid CBOR

// elements (array of 3 elements)
0x83,

// level: 0
0x00,

// hkeys (byte string of length 8 * 1)
0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08,
// hkey: 0
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,

// elements (array of 1 elements)
// each element is encoded as CBOR array of 2 elements (key, value)
0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
// element: [uint64(0):SomeValue(SlabID(0x0.1))]
0x82,
0xd8, 0xa4, 0x00,
0xd8, cborTagSomeValue, 0xd8, 0xff, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
},
}

fixedData := map[SlabID][]byte{
rootID: {
// version
0x10,
// flag: root + map data
0x88,

// extra data
// CBOR encoded array of 3 elements
0x83,
// type info
0x18, 0x2a,
// count: 0
0x00,
// seed
0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49,

// the following encoded data is valid CBOR

// elements (array of 3 elements)
0x83,

// level: 0
0x00,

// hkeys (byte string of length 8 * 1)
0x59, 0x00, 0x00,

// elements (array of 0 elements)
// each element is encoded as CBOR array of 2 elements (key, value)
0x99, 0x00, 0x00,
},
}

storage := newTestPersistentStorageWithData(t, data)

// Load data in storage
for id := range data {
_, found, err := storage.Retrieve(id)
require.NoError(t, err)
require.True(t, found)
}

// Check health before fixing broken reference
_, err := CheckStorageHealth(storage, -1)
require.ErrorContains(t, err, "slab (0x0.1) not found: slab not found during slab iteration")

var fixedRootIDs map[SlabID][]SlabID
var skippedRootIDs map[SlabID][]SlabID

// Don't fix any broken references
fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool {
return false
})
require.NoError(t, err)
require.Equal(t, 0, len(fixedRootIDs))
require.Equal(t, len(brokenRefs), len(skippedRootIDs))

for rootID, slabIDsWithBrokenRef := range brokenRefs {
require.ElementsMatch(t, slabIDsWithBrokenRef, skippedRootIDs[rootID])
}

// No data is modified because no fix happened
require.Equal(t, 0, len(storage.deltas))

// Fix broken references
fixedRootIDs, skippedRootIDs, err = storage.FixLoadedBrokenReferences(func(_ Value) bool {
return true
})
require.NoError(t, err)
require.Equal(t, len(brokenRefs), len(fixedRootIDs))

for rootID, slabIDsWithBrokenRef := range brokenRefs {
require.ElementsMatch(t, slabIDsWithBrokenRef, fixedRootIDs[rootID])
}

require.Equal(t, 0, len(skippedRootIDs))
require.Equal(t, 1, len(storage.deltas))

// Check health after fixing broken reference
rootIDs, err := CheckStorageHealth(storage, -1)
require.NoError(t, err)
require.Equal(t, 1, len(rootIDs))

_, ok := rootIDs[rootID]
require.True(t, ok)

// Save data in storage
err = storage.FastCommit(runtime.NumCPU())
require.NoError(t, err)
require.Equal(t, 0, len(storage.deltas))

// Check encoded data
baseStorage := storage.baseStorage.(*InMemBaseStorage)
require.Equal(t, 1, len(baseStorage.segments))

savedData, found, err := baseStorage.Retrieve(rootID)
require.NoError(t, err)
require.True(t, found)
require.Equal(t, fixedData[rootID], savedData)
})

t.Run("broken non-root map data slab", func(t *testing.T) {
rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}}
nonRootDataID1 := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}}
Expand Down Expand Up @@ -3092,6 +3246,60 @@ func TestGetAllChildReferencesFromArray(t *testing.T) {
testGetAllChildReferences(t, data, parentRootID, expectedRefIDs, expectedBrokenRefIDs)
})

t.Run("root data slab with ref in nested storable", func(t *testing.T) {
parentRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}}
childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}}

expectedRefIDs := []SlabID{childRootID}
expectedBrokenRefIDs := []SlabID{}

data := map[SlabID][]byte{
parentRootID: {
// extra data
// version
0x00,
// extra data flag
0x80,
// array of extra data
0x81,
// type info
0x18, 0x2a,

// version
0x00,
// array data slab flag
0x80,
// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x01,
// CBOR encoded array elements
0xd8, cborTagSomeValue, 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
},

childRootID: {
// extra data
// version
0x00,
// extra data flag
0x80,
// array of extra data
0x81,
// type info
0x18, 0x2a,

// version
0x00,
// array data slab flag
0x80,
// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x01,
// CBOR encoded array elements
0xd8, 0xa4, 0x00,
},
}

testGetAllChildReferences(t, data, parentRootID, expectedRefIDs, expectedBrokenRefIDs)
})

t.Run("root data slab with broken ref", func(t *testing.T) {
parentRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}}
childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}}
Expand Down Expand Up @@ -3725,6 +3933,81 @@ func TestGetAllChildReferencesFromMap(t *testing.T) {
testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs)
})

t.Run("root data slab with ref in nested storable", func(t *testing.T) {
rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}}
childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}}

expectedRefIDs := []SlabID{childRootID}
expectedBrokenRefIDs := []SlabID{}

data := map[SlabID][]byte{
rootID: {
// extra data
// version
0x00,
// flag: root + map data
0x88,
// extra data (CBOR encoded array of 3 elements)
0x83,
// type info
0x18, 0x2a,
// count: 1
0x01,
// seed
0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49,

// version
0x00,
// flag: root + map data
0x88,

// the following encoded data is valid CBOR

// elements (array of 3 elements)
0x83,

// level: 0
0x00,

// hkeys (byte string of length 8 * 1)
0x5b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08,
// hkey: 0
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,

// elements (array of 1 elements)
// each element is encoded as CBOR array of 2 elements (key, value)
0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
// element: [uint64(0):uint64(0)]
0x82,
0xd8, 0xa4, 0x00,
0xd8, cborTagSomeValue, 0xd8, 0xff, 0x50, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
},

childRootID: {
// extra data
// version
0x00,
// extra data flag
0x80,
// array of extra data
0x81,
// type info
0x18, 0x2a,

// version
0x00,
// array data slab flag
0x80,
// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x01,
// CBOR encoded array elements
0xd8, 0xa4, 0x00,
},
}

testGetAllChildReferences(t, data, rootID, expectedRefIDs, expectedBrokenRefIDs)
})

t.Run("root data slab with broken ref", func(t *testing.T) {
rootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 1}}
childRootID := SlabID{address: address, index: SlabIndex{0, 0, 0, 0, 0, 0, 0, 2}}
Expand Down
Loading