Lum Network v1.6.3 (#64)

* Added epoch unbondings export / import at genesis time

* chore(dev-deps): update ledger-cosmos-go from v0.12.2 to v0.12.4 and ledger-go from v0.14.1 to v0.14.3

* Temporary context on ICA callbacks

* [LUM-856] Restoring ICA should restore blocked entities (#62)

* Fix epoch unittests

* Add new entity restoration procedure in case of channel restore

---------

Co-authored-by: Fabrice Bascoulergue <[email protected]>

* Revert "Temporary context on ICA callbacks"

This reverts commit 05862f1.

* Update hooks.go

* Added missing log

* Update hooks to allow for operation rollback

* Introduce temporary cache ctx for rollback operation

* Improve comments, add logs

---------

Co-authored-by: Segfault <[email protected]>
Co-authored-by: Zakaria Lounes <[email protected]>
Co-authored-by: Fabrice Bascoulergue <[email protected]>

go.mod

@@ -63,7 +63,7 @@ require (
 	github.com/cosmos/go-bip39 v1.0.0 // indirect
 	github.com/cosmos/gogogateway v1.2.0 // indirect
 	github.com/cosmos/iavl v0.20.0 // indirect
-	github.com/cosmos/ledger-cosmos-go v0.12.2 // indirect
+	github.com/cosmos/ledger-cosmos-go v0.12.4 // indirect
 	github.com/cosmos/rosetta-sdk-go v0.10.0 // indirect
 	github.com/creachadair/taskgroup v0.4.2 // indirect
 	github.com/danieljoos/wincred v1.1.2 // indirect
@@ -152,8 +152,8 @@ require (
 	github.com/tendermint/go-amino v0.16.0 // indirect
 	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
-	github.com/zondax/hid v0.9.1 // indirect
-	github.com/zondax/ledger-go v0.14.1 // indirect
+	github.com/zondax/hid v0.9.2 // indirect
+	github.com/zondax/ledger-go v0.14.3 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/crypto v0.11.0 // indirect

go.sum

@@ -400,8 +400,8 @@ github.com/cosmos/ibc-go/v7 v7.2.0 h1:dx0DLUl7rxdyZ8NiT6UsrbzKOJx/w7s+BOaewFRH6c
 github.com/cosmos/ibc-go/v7 v7.2.0/go.mod h1:OOcjKIRku/j1Xs1RgKK0yvKRrJ5iFuZYMetR1n3yMlc=
 github.com/cosmos/ics23/go v0.10.0 h1:iXqLLgp2Lp+EdpIuwXTYIQU+AiHj9mOC2X9ab++bZDM=
 github.com/cosmos/ics23/go v0.10.0/go.mod h1:ZfJSmng/TBNTBkFemHHHj5YY7VAU/MBU980F4VU1NG0=
-github.com/cosmos/ledger-cosmos-go v0.12.2 h1:/XYaBlE2BJxtvpkHiBm97gFGSGmYGKunKyF3nNqAXZA=
-github.com/cosmos/ledger-cosmos-go v0.12.2/go.mod h1:ZcqYgnfNJ6lAXe4HPtWgarNEY+B74i+2/8MhZw4ziiI=
+github.com/cosmos/ledger-cosmos-go v0.12.4 h1:drvWt+GJP7Aiw550yeb3ON/zsrgW0jgh5saFCr7pDnw=
+github.com/cosmos/ledger-cosmos-go v0.12.4/go.mod h1:fjfVWRf++Xkygt9wzCsjEBdjcf7wiiY35fv3ctT+k4M=
 github.com/cosmos/rosetta-sdk-go v0.10.0 h1:E5RhTruuoA7KTIXUcMicL76cffyeoyvNybzUGSKFTcM=
 github.com/cosmos/rosetta-sdk-go v0.10.0/go.mod h1:SImAZkb96YbwvoRkzSMQB6noNJXFgWl/ENIznEoYQI4=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
@@ -1160,10 +1160,10 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo=
-github.com/zondax/hid v0.9.1/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM=
-github.com/zondax/ledger-go v0.14.1 h1:Pip65OOl4iJ84WTpA4BKChvOufMhhbxED3BaihoZN4c=
-github.com/zondax/ledger-go v0.14.1/go.mod h1:fZ3Dqg6qcdXWSOJFKMG8GCTnD7slO/RL2feOQv8K320=
+github.com/zondax/hid v0.9.2 h1:WCJFnEDMiqGF64nlZz28E9qLVZ0KSJ7xpc5DLEyma2U=
+github.com/zondax/hid v0.9.2/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM=
+github.com/zondax/ledger-go v0.14.3 h1:wEpJt2CEcBJ428md/5MgSLsXLBos98sBOyxNmCjfUCw=
+github.com/zondax/ledger-go v0.14.3/go.mod h1:IKKaoxupuB43g4NxeQmbLXv7T9AlQyie1UpHb342ycI=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
 go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=

proto/lum/network/millions/genesis.proto

@@ -29,4 +29,6 @@ message GenesisState {
   repeated Prize prizes = 9 [ (gogoproto.nullable) = false ];
   repeated Withdrawal withdrawals = 10 [ (gogoproto.nullable) = false ];
   repeated EpochTracker epoch_trackers = 11 [ (gogoproto.nullable) = false ];
+  repeated EpochUnbonding epoch_unbondings = 12
+      [ (gogoproto.nullable) = false ];
 }

x/millions/genesis.go

@@ -47,6 +47,9 @@ func InitGenesis(ctx sdk.Context, k keeper.Keeper, genState types.GenesisState)
 		k.SetEpochTracker(ctx, epochTracker, types.WithdrawalTrackerType)
 	}
 
+	for _, epochUnbonding := range genState.EpochUnbondings {
+		k.SetEpochUnbonding(ctx, epochUnbonding)
+	}
 }
 
 func ExportGenesis(ctx sdk.Context, k keeper.Keeper) *types.GenesisState {
@@ -63,5 +66,6 @@ func ExportGenesis(ctx sdk.Context, k keeper.Keeper) *types.GenesisState {
 		Prizes:           k.ListPrizes(ctx),
 		Withdrawals:      k.ListWithdrawals(ctx),
 		EpochTrackers:    k.ListEpochTrackers(ctx),
+		EpochUnbondings:  k.ListEpochUnbondings(ctx),
 	}
 }

x/millions/genesis_test.go

@@ -109,6 +109,9 @@ var testGenesis = millionstypes.GenesisState{
 	EpochTrackers: []millionstypes.EpochTracker{
 		{EpochTrackerType: epochstypes.DAY_EPOCH, EpochIdentifier: epochstypes.DAY_EPOCH, NextEpochNumber: uint64(2), PreviousEpochNumber: uint64(0), NextEpochStartTime: future},
 	},
+	EpochUnbondings: []millionstypes.EpochUnbonding{
+		{PoolId: 1, EpochIdentifier: "test", EpochNumber: 1, TotalAmount: sdk.NewCoin("ulum", sdk.NewInt(1000000)), WithdrawalIds: []uint64{1, 2}, WithdrawalIdsCount: 2},
+	},
 }
 
 func TestInitGenesis(t *testing.T) {
@@ -298,6 +301,16 @@ func TestInitGenesis(t *testing.T) {
 		require.Equal(t, testGenesis.EpochTrackers[i].NextEpochStartTime, epochTracker.NextEpochStartTime)
 	}
 
+	// Make sure epoch unbondings have been imported
+	epochUnbondings := app.MillionsKeeper.ListEpochUnbondings(ctx)
+	require.Len(t, epochUnbondings, len(testGenesis.EpochUnbondings))
+	for i, epochUnbonding := range epochUnbondings {
+		require.Equal(t, testGenesis.EpochUnbondings[i].EpochIdentifier, epochUnbonding.EpochIdentifier)
+		require.Equal(t, testGenesis.EpochUnbondings[i].EpochNumber, epochUnbonding.EpochNumber)
+		require.Equal(t, testGenesis.EpochUnbondings[i].WithdrawalIds, epochUnbonding.WithdrawalIds)
+		require.Equal(t, testGenesis.EpochUnbondings[i].WithdrawalIdsCount, epochUnbonding.WithdrawalIdsCount)
+		require.Equal(t, testGenesis.EpochUnbondings[i].TotalAmount.Amount.Int64(), epochUnbonding.TotalAmount.Amount.Int64())
+	}
 }
 
 func TestExportGenesis(t *testing.T) {
@@ -407,4 +420,14 @@ func TestExportGenesis(t *testing.T) {
 		require.Equal(t, testGenesis.EpochTrackers[i].NextEpochNumber, epochTracker.NextEpochNumber)
 		require.Equal(t, testGenesis.EpochTrackers[i].NextEpochStartTime, epochTracker.NextEpochStartTime)
 	}
+
+	// Test epoch unbondings export
+	require.Len(t, exportGenesis.EpochUnbondings, len(testGenesis.EpochUnbondings))
+	for i, epochUnbonding := range testGenesis.EpochUnbondings {
+		require.Equal(t, testGenesis.EpochUnbondings[i].EpochIdentifier, epochUnbonding.EpochIdentifier)
+		require.Equal(t, testGenesis.EpochUnbondings[i].EpochNumber, epochUnbonding.EpochNumber)
+		require.Equal(t, testGenesis.EpochUnbondings[i].WithdrawalIds, epochUnbonding.WithdrawalIds)
+		require.Equal(t, testGenesis.EpochUnbondings[i].WithdrawalIdsCount, epochUnbonding.WithdrawalIdsCount)
+		require.Equal(t, testGenesis.EpochUnbondings[i].TotalAmount.Amount.Int64(), epochUnbonding.TotalAmount.Amount.Int64())
+	}
 }

x/millions/keeper/hooks.go

@@ -22,6 +22,7 @@ func (k Keeper) BeforeEpochStart(ctx sdk.Context, epochInfo epochstypes.EpochInf
 }
 
 func (k Keeper) processEpochUnbondings(ctx sdk.Context, epochInfo epochstypes.EpochInfo, logger log.Logger) (successCount, errorCount, skippedCount int) {
+	// Update the epoch tracker
 	epochTracker, err := k.UpdateEpochTracker(ctx, epochInfo, types.WithdrawalTrackerType)
 	if err != nil {
 		logger.Error(
@@ -31,33 +32,85 @@ func (k Keeper) processEpochUnbondings(ctx sdk.Context, epochInfo epochstypes.Ep
 		return
 	}
 
+	// Create temporary context
+	cacheCtx, writeCache := ctx.CacheContext()
+
 	// Get epoch unbondings
-	epochUnbondings := k.GetEpochUnbondings(ctx, epochTracker.EpochNumber)
+	epochUnbondings := k.GetEpochUnbondings(cacheCtx, epochTracker.EpochNumber)
+
+	// For each unbonding, try to proceed the undelegation otherwise, rollback the entire operation
 	for _, epochUnbonding := range epochUnbondings {
-		success, err := k.processEpochUnbonding(ctx, epochUnbonding, epochTracker, logger)
+		success, err := k.processEpochUnbonding(cacheCtx, epochUnbonding, epochTracker, logger)
 		if err != nil {
 			logger.Error(
 				fmt.Sprintf("Error processing epoch unbonding: %v", err),
 				"pool_id", epochUnbonding.GetPoolId(),
 				"epoch_number", epochUnbonding.GetEpochNumber(),
 			)
 			errorCount++
+			break
 		} else if success {
 			successCount++
 		} else {
 			skippedCount++
 		}
 	}
 
-	if successCount+errorCount > 0 {
+	// If there was an error, we are supposed to cancel the entire operation
+	if errorCount > 0 {
+		// Log out the critical error
+		logger.Error(
+			"epoch unbonding undelegate processed with errors, cache not written",
+			"nbr_success", successCount,
+			"nbr_error", errorCount,
+			"nbr_skipped", skippedCount,
+		)
+
+		// Allocate new cache context
+		rollbackTmpCacheCtx, writeRollbackCache := ctx.CacheContext()
+
+		// Get epoch unbondings
+		epochUnbondings = k.GetEpochUnbondings(rollbackTmpCacheCtx, epochTracker.EpochNumber)
+
+		// Rollback the operations and put everything back up in the next epoch
+		// We explicitly use a custom tailored cache for this operation
+		for _, epochUnbonding := range epochUnbondings {
+			for _, wid := range epochUnbonding.WithdrawalIds {
+				withdrawal, err := k.GetPoolWithdrawal(rollbackTmpCacheCtx, epochUnbonding.PoolId, wid)
+				if err != nil {
+					logger.Error(fmt.Sprintf("Failure in getting the pool withdrawals for pool %d, err: %v", epochUnbonding.PoolId, err))
+					return 0, 1, 0
+				}
+				if err := k.AddEpochUnbonding(rollbackTmpCacheCtx, withdrawal, false); err != nil {
+					logger.Error(fmt.Sprintf("Failure in adding the withdrawal to epoch unbonding for pool %d, err: %v", epochUnbonding.PoolId, err))
+					return 0, 1, 0
+				}
+			}
+
+			if err := k.RemoveEpochUnbonding(rollbackTmpCacheCtx, epochUnbonding); err != nil {
+				logger.Error(fmt.Sprintf("Failure in removing the epoch unbonding for pool %d, err: %v", epochUnbonding.PoolId, err))
+				return 0, 1, 0
+			}
+		}
+
+		// Write the rollback cache
+		writeRollbackCache()
 		logger.Info(
-			"epoch unbonding undelegate started",
+			"epoch unbonding processed with failure, rollback done",
+			"nbr_success", successCount,
+			"nbr_error", errorCount,
+			"nbr_skipped", skippedCount,
+		)
+	} else {
+		// Write the operation succeed cache
+		writeCache()
+		logger.Info(
+			"epoch unbonding undelegate processed",
 			"nbr_success", successCount,
 			"nbr_error", errorCount,
 			"nbr_skipped", skippedCount,
 		)
 	}
-
 	return successCount, errorCount, skippedCount
 }
 

x/millions/keeper/keeper_epoch.go

@@ -395,6 +395,14 @@ func (k Keeper) GetEpochUnbondings(ctx sdk.Context, epochID uint64) (epochUnbond
 	return
 }
 
+// SetEpochUnbonding sets the internal epoch unbondings
+func (k Keeper) SetEpochUnbonding(ctx sdk.Context, epochUnbonding types.EpochUnbonding) {
+	store := ctx.KVStore(k.storeKey)
+	key := types.GetEpochUnbondingsKey(epochUnbonding.GetEpochNumber())
+	encodedEpochUnbonding := k.cdc.MustMarshal(&epochUnbonding)
+	store.Set(key, encodedEpochUnbonding)
+}
+
 func (k Keeper) ListEpochTrackers(ctx sdk.Context) (list []types.EpochTracker) {
 	store := ctx.KVStore(k.storeKey)
 	iterator := sdk.KVStorePrefixIterator(store, types.EpochTrackerPrefix)
@@ -408,3 +416,17 @@ func (k Keeper) ListEpochTrackers(ctx sdk.Context) (list []types.EpochTracker) {
 
 	return
 }
+
+func (k Keeper) ListEpochUnbondings(ctx sdk.Context) (list []types.EpochUnbonding) {
+	store := ctx.KVStore(k.storeKey)
+	iterator := sdk.KVStorePrefixIterator(store, types.EpochUnbondingPrefix)
+	defer iterator.Close()
+
+	for ; iterator.Valid(); iterator.Next() {
+		var val types.EpochUnbonding
+		k.cdc.MustUnmarshal(iterator.Value(), &val)
+		list = append(list, val)
+	}
+
+	return
+}

x/millions/keeper/keeper_epoch_test.go

@@ -255,9 +255,12 @@ func (suite *KeeperTestSuite) TestEpoch_AddEpochUnbonding() {
 	suite.Require().Equal([]uint64{1, 2}, epochUnbondingPool.WithdrawalIds)
 	suite.Require().Equal(uint64(2), epochUnbondingPool.WithdrawalIdsCount)
 
-	// Same withdrawalID to the epoch unbonding should faild
+	// Same withdrawalID to the epoch unbonding should do nothing
 	err = app.MillionsKeeper.AddEpochUnbonding(ctx, withdrawals[0], false)
-	suite.Require().ErrorIs(millionstypes.ErrEntityOverride, err)
+	suite.Require().NoError(err)
+	epochUnbondingPoolAfter, err := app.MillionsKeeper.GetEpochPoolUnbonding(ctx, epochTracker.EpochNumber+frequency, 1)
+	suite.Require().NoError(err)
+	suite.Require().Equal(epochUnbondingPool.WithdrawalIds, epochUnbondingPoolAfter.WithdrawalIds)
 
 	// Trigger new deposits for new pool
 	for i := 0; i < 3; i++ {

x/millions/keeper/keeper_withdrawal.go

@@ -82,13 +82,12 @@ func (k Keeper) OnUndelegateWithdrawalsOnRemoteZoneCompleted(ctx sdk.Context, po
 			if err := k.AddEpochUnbonding(ctx, withdrawal, true); err != nil {
 				return err
 			}
-			continue
+		} else {
+			// Set the unbondingEndsAt and add withdrawal to matured queue
+			withdrawal.UnbondingEndsAt = unbondingEndsAt
+			k.UpdateWithdrawalStatus(ctx, withdrawal.PoolId, withdrawal.WithdrawalId, types.WithdrawalState_IcaUnbonding, unbondingEndsAt, false)
+			k.addWithdrawalToMaturedQueue(ctx, withdrawal)
 		}
-
-		// Set the unbondingEndsAt and add withdrawal to matured queue
-		withdrawal.UnbondingEndsAt = unbondingEndsAt
-		k.UpdateWithdrawalStatus(ctx, withdrawal.PoolId, withdrawal.WithdrawalId, types.WithdrawalState_IcaUnbonding, unbondingEndsAt, false)
-		k.addWithdrawalToMaturedQueue(ctx, withdrawal)
 	}
 
 	if isError {

x/millions/keeper/msg_server_pool.go

@@ -47,6 +47,7 @@ func (k msgServer) RestoreInterchainAccounts(goCtx context.Context, msg *types.M
 		if err := k.ICAControllerKeeper.RegisterInterchainAccount(ctx, pool.GetConnectionId(), icaDepositPortName, appVersion); err != nil {
 			return nil, errorsmod.Wrapf(types.ErrFailedToRestorePool, fmt.Sprintf("Unable to trigger deposit account registration, err: %s", err.Error()))
 		}
+		k.restoreICADepositEntities(ctx, pool.GetPoolId())
 		// Exit to prevent a double channel registration which might create unpredictable behaviours
 		// The registration of the ICA PrizePool will either be automatically triggered once the ICA Deposit gets ACK (see keeper.OnSetupPoolICACompleted)
 		// or can be restored by calling this method again
@@ -65,7 +66,54 @@ func (k msgServer) RestoreInterchainAccounts(goCtx context.Context, msg *types.M
 		if err := k.ICAControllerKeeper.RegisterInterchainAccount(ctx, pool.GetConnectionId(), icaPrizePoolPortName, appVersion); err != nil {
 			return nil, errorsmod.Wrapf(types.ErrFailedToRestorePool, fmt.Sprintf("Unable to trigger prizepool account registration, err: %s", err.Error()))
 		}
+		k.restoreICAPrizePoolEntities(ctx, pool.GetPoolId())
 	}
 
 	return &types.MsgRestoreInterchainAccountsResponse{}, nil
 }
+
+// restoreICADepositEntities restores all blocked entities by putting them in error state instead of locked state
+// thus leaving users the opportunity to retry the operation
+func (k msgServer) restoreICADepositEntities(ctx sdk.Context, poolID uint64) {
+	// Restore deposits ICA locked operations on ICADeposit account
+	deposits := k.Keeper.ListPoolDeposits(ctx, poolID)
+	for _, d := range deposits {
+		if d.State == types.DepositState_IcaDelegate {
+			d.ErrorState = d.State
+			d.State = types.DepositState_Failure
+			k.Keeper.setPoolDeposit(ctx, &d)
+		}
+	}
+	// Restore withdrawals ICA locked operations on ICADeposit account
+	withdrawals := k.Keeper.ListPoolWithdrawals(ctx, poolID)
+	for _, w := range withdrawals {
+		if w.State == types.WithdrawalState_IcaUndelegate {
+			w.ErrorState = w.State
+			w.State = types.WithdrawalState_Failure
+			k.Keeper.setPoolWithdrawal(ctx, w)
+		}
+	}
+	// Restore draws ICA locked operations on ICADeposit account
+	draws := k.Keeper.ListPoolDraws(ctx, poolID)
+	for _, d := range draws {
+		if d.State == types.DrawState_IcaWithdrawRewards {
+			d.ErrorState = d.State
+			d.State = types.DrawState_Failure
+			k.Keeper.SetPoolDraw(ctx, d)
+		}
+	}
+}
+
+// restoreICAPrizePoolEntities restores all blocked entities by putting them in error state instead of locked state
+// thus leaving users the opportunity to retry the operation
+func (k msgServer) restoreICAPrizePoolEntities(ctx sdk.Context, poolID uint64) {
+	// Restore draws ICQ locked operations on ICAPrizePool account
+	draws := k.Keeper.ListPoolDraws(ctx, poolID)
+	for _, d := range draws {
+		if d.State == types.DrawState_IcqBalance {
+			d.ErrorState = d.State
+			d.State = types.DrawState_Failure
+			k.Keeper.SetPoolDraw(ctx, d)
+		}
+	}
+}

x/millions/keeper/msg_server_withdrawal.go

@@ -130,6 +130,13 @@ func (k msgServer) WithdrawDepositRetry(goCtx context.Context, msg *types.MsgWit
 		if err := k.TransferWithdrawalToRecipient(ctx, withdrawal.PoolId, withdrawal.WithdrawalId); err != nil {
 			return nil, err
 		}
+	} else if withdrawal.ErrorState == types.WithdrawalState_IcaUndelegate {
+		// Only possible following a restore account which put entities in this error state
+		// otherwise the retry is automatically triggered by the ICA callback
+		err := k.AddEpochUnbonding(ctx, withdrawal, true)
+		if err != nil {
+			return nil, err
+		}
 	} else {
 		return nil, errorsmod.Wrapf(
 			types.ErrInvalidWithdrawalState,