Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve migrations #3144

Merged
merged 18 commits into from
Mar 6, 2024
Merged

Improve migrations #3144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
e21ea84
check storage health after migrating
turbolent Feb 29, 2024
f9b1a87
implement AccountLinkValue.ChildStorables for atree storage health check
turbolent Feb 29, 2024
1700a85
fix test case: transfer value to account before storing in storage map
turbolent Mar 4, 2024
3f78538
enable atree value and storage validation where possible
turbolent Mar 5, 2024
d371793
improve tests: enable atree storage health and value validation where…
turbolent Mar 5, 2024
407233e
add DictionaryValue InsertWithoutTransfer and RemoveWithoutTransfer
turbolent Mar 5, 2024
15b7ac3
fix dictionary value case in storage migration: always remove old val…
turbolent Mar 5, 2024
33b0c52
fix dictionary value case in static type and entitlements migration
turbolent Mar 5, 2024
e9c82ad
Merge branch 'master' into bastian/improve-migration-2
turbolent Mar 5, 2024
5db6a8c
refactor dictionary creation into function
turbolent Mar 5, 2024
e841f2b
add test for DictionaryValue.NewWithType
turbolent Mar 5, 2024
67e5130
add ArrayValue.InsertWithoutTransfer and RemoveWithoutTransfer
turbolent Mar 5, 2024
792bf1b
add ArrayValue.NewWithType
turbolent Mar 5, 2024
a301d13
fix migration of arrays: migrate without transfer, remove old data if…
turbolent Mar 5, 2024
c6d4238
add CompositeValue.SetMemberWithoutTransfer
turbolent Mar 5, 2024
d10955b
add test for migration of nested containers
turbolent Mar 5, 2024
40f8e68
fix composite field migration: set member without transfer
turbolent Mar 5, 2024
d855aee
only remove existing value if there is a new value
turbolent Mar 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 25 additions & 13 deletions migrations/capcons/migration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -514,16 +514,12 @@ func testPathCapabilityValueMigration(
err = migration.Commit()
require.NoError(t, err)

// Check migrated capabilities
// Assert

assert.Equal(t,
expectedMigrations,
reporter.migrations,
)
assert.Equal(t,
expectedErrors,
reporter.errors,
)
assert.Equal(t,
expectedPathMigrations,
reporter.pathCapabilityMigrations,
Expand All @@ -532,6 +528,13 @@ func testPathCapabilityValueMigration(
expectedMissingCapabilityIDs,
reporter.missingCapabilityIDs,
)
require.Equal(t,
expectedErrors,
reporter.errors,
)

err = storage.CheckHealth()
require.NoError(t, err)

if len(expectedMissingCapabilityIDs) == 0 {

Expand Down Expand Up @@ -1354,10 +1357,6 @@ func testLinkMigration(
expectedMigrations,
reporter.migrations,
)
assert.Empty(t,
expectedErrors,
reporter.errors,
)
assert.Equal(t,
expectedLinkMigrations,
reporter.linkMigrations,
Expand All @@ -1370,6 +1369,13 @@ func testLinkMigration(
expectedMissingTargets,
reporter.missingTargets,
)
require.Equal(t,
expectedErrors,
reporter.errors,
)

err = storage.CheckHealth()
require.NoError(t, err)
}

func TestLinkMigration(t *testing.T) {
Expand Down Expand Up @@ -2073,20 +2079,22 @@ func TestPublishedPathCapabilityValueMigration(t *testing.T) {
err = migration.Commit()
require.NoError(t, err)

// Check migrated capabilities
// Assert

assert.Equal(t,
expectedMigrations,
reporter.migrations,
)
assert.Empty(t, reporter.errors)
assert.Equal(t,
expectedPathMigrations,
reporter.pathCapabilityMigrations,
)
require.Nil(t, reporter.missingCapabilityIDs)

// Check
require.Empty(t, reporter.errors)

err = storage.CheckHealth()
require.NoError(t, err)

// language=cadence
checkScript := `
Expand Down Expand Up @@ -2320,7 +2328,7 @@ func TestUntypedPathCapabilityValueMigration(t *testing.T) {
err = migration.Commit()
require.NoError(t, err)

// Check migrated capabilities
// Assert

assert.Equal(t,
expectedMigrations,
Expand All @@ -2331,8 +2339,12 @@ func TestUntypedPathCapabilityValueMigration(t *testing.T) {
reporter.pathCapabilityMigrations,
)
require.Nil(t, reporter.missingCapabilityIDs)

require.Empty(t, reporter.errors)

err = storage.CheckHealth()
require.NoError(t, err)

// Check

// language=cadence
Expand Down
39 changes: 10 additions & 29 deletions migrations/entitlements/migration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,16 +241,10 @@ func ConvertValueToEntitlements(
return nil, nil
}

iterator := v.Iterator(inter, interpreter.EmptyLocationRange)

return interpreter.NewArrayValueWithIterator(
return v.NewWithType(
inter,
interpreter.EmptyLocationRange,
entitledElementType.(interpreter.ArrayStaticType),
v.GetOwner(),
uint64(v.Count()),
func() interpreter.Value {
return iterator.Next(inter, interpreter.EmptyLocationRange)
},
), nil

case *interpreter.DictionaryValue:
Expand All @@ -261,29 +255,16 @@ func ConvertValueToEntitlements(
return nil, err
}

if entitledElementType != nil {
var keysAndValues []interpreter.Value

iterator := v.Iterator()
for {
keyValue, value := iterator.Next(inter)
if keyValue == nil {
break
}

keysAndValues = append(keysAndValues, keyValue)
keysAndValues = append(keysAndValues, value)
}

return interpreter.NewDictionaryValueWithAddress(
inter,
interpreter.EmptyLocationRange,
entitledElementType.(*interpreter.DictionaryStaticType),
v.GetOwner(),
keysAndValues...,
), nil
if entitledElementType == nil {
return nil, nil
}

return v.NewWithType(
inter,
interpreter.EmptyLocationRange,
entitledElementType.(*interpreter.DictionaryStaticType),
), nil

case *interpreter.IDCapabilityValue:
borrowType := v.BorrowType

Expand Down
78 changes: 67 additions & 11 deletions migrations/entitlements/migration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -725,7 +725,10 @@ func convertEntireTestValue(

// Assert

assert.Len(t, reporter.errors, 0)
require.Empty(t, reporter.errors)

err = storage.CheckHealth()
require.NoError(t, err)

if migratedValue == nil {
return v
Expand Down Expand Up @@ -1317,8 +1320,12 @@ func TestConvertToEntitledValue(t *testing.T) {
},
}

test := func(t *testing.T, testCase testCase, valueGenerator valueGenerator, typeGenerator typeGenerator) {

test := func(
t *testing.T,
testCase testCase,
valueGenerator valueGenerator,
typeGenerator typeGenerator,
) {
input := valueGenerator.wrap(typeGenerator.wrap(testCase.Input))
if input == nil {
return
Expand All @@ -1328,6 +1335,9 @@ func TestConvertToEntitledValue(t *testing.T) {

convertedValue := convertEntireTestValue(t, inter, storage, testAddress, input)

err := storage.CheckHealth()
require.NoError(t, err)

switch convertedValue := convertedValue.(type) {
case interpreter.EquatableValue:
require.True(t,
Expand Down Expand Up @@ -1488,11 +1498,20 @@ func TestMigrateSimpleContract(t *testing.T) {
}

for name, testCase := range testCases {
transferredValue := testCase.storedValue.Transfer(
inter,
interpreter.EmptyLocationRange,
atree.Address(account),
false,
nil,
nil,
)

inter.WriteStored(
account,
storageIdentifier,
interpreter.StringStorageMapKey(name),
testCase.storedValue,
transferredValue,
)
}

Expand Down Expand Up @@ -1521,7 +1540,10 @@ func TestMigrateSimpleContract(t *testing.T) {

// Assert

assert.Len(t, reporter.errors, 0)
require.Empty(t, reporter.errors)

err = storage.CheckHealth()
require.NoError(t, err)

storageMap := storage.GetStorageMap(account, storageIdentifier, false)
require.NotNil(t, storageMap)
Expand Down Expand Up @@ -1705,7 +1727,11 @@ func TestMigratePublishedValue(t *testing.T) {

// Assert

assert.Len(t, reporter.errors, 0)
require.Empty(t, reporter.errors)

err = storage.CheckHealth()
require.NoError(t, err)

assert.Equal(t,
map[struct {
interpreter.StorageKey
Expand Down Expand Up @@ -1959,7 +1985,11 @@ func TestMigratePublishedValueAcrossTwoAccounts(t *testing.T) {

// Assert

assert.Len(t, reporter.errors, 0)
require.Empty(t, reporter.errors)

err = storage.CheckHealth()
require.NoError(t, err)

assert.Equal(t,
map[struct {
interpreter.StorageKey
Expand Down Expand Up @@ -2408,7 +2438,11 @@ func TestMigrateArrayOfValues(t *testing.T) {

// Assert

assert.Len(t, reporter.errors, 0)
require.Empty(t, reporter.errors)

err = storage.CheckHealth()
require.NoError(t, err)

assert.Equal(t,
map[struct {
interpreter.StorageKey
Expand Down Expand Up @@ -2655,7 +2689,11 @@ func TestMigrateDictOfValues(t *testing.T) {

// Assert

assert.Len(t, reporter.errors, 0)
require.Empty(t, reporter.errors)

err = storage.CheckHealth()
require.NoError(t, err)

assert.Equal(t,
map[struct {
interpreter.StorageKey
Expand Down Expand Up @@ -2974,7 +3012,11 @@ func TestMigrateCapConsAcrossTwoAccounts(t *testing.T) {

// Assert

assert.Len(t, reporter.errors, 0)
require.Empty(t, reporter.errors)

err = storage.CheckHealth()
require.NoError(t, err)

assert.Len(t, reporter.migrated, 1)

// TODO: assert
Expand Down Expand Up @@ -3063,7 +3105,8 @@ func TestRehash(t *testing.T) {
nil,
utils.TestLocation,
&interpreter.Config{
Storage: storage,
Storage: storage,
// NOTE: disabled, because encoded and decoded values are expected to not match
AtreeValueValidationEnabled: false,
AtreeStorageValidationEnabled: true,
},
Expand Down Expand Up @@ -3154,6 +3197,9 @@ func TestRehash(t *testing.T) {

err := storage.Commit(inter, false)
require.NoError(t, err)

err = storage.CheckHealth()
require.NoError(t, err)
})

t.Run("migrate", func(t *testing.T) {
Expand Down Expand Up @@ -3200,6 +3246,13 @@ func TestRehash(t *testing.T) {
err := migration.Commit()
require.NoError(t, err)

// Assert

err = storage.CheckHealth()
require.NoError(t, err)

assert.Empty(t, reporter.errors)

require.Equal(t,
map[struct {
interpreter.StorageKey
Expand All @@ -3221,6 +3274,9 @@ func TestRehash(t *testing.T) {

storage, inter := newStorageAndInterpreter(t)

err := storage.CheckHealth()
require.NoError(t, err)

storageMap := storage.GetStorageMap(
testAddress,
common.PathDomainStorage.Identifier(),
Expand Down
Loading
Loading