From 6b4f78640a0d6b5dfe9f6e82f59048398a74e00b Mon Sep 17 00:00:00 2001 From: Elias Van Ootegem Date: Mon, 10 Jun 2024 14:28:40 +0100 Subject: [PATCH 1/7] fix: underflow on win socialisation check Signed-off-by: Elias Van Ootegem --- core/settlement/engine.go | 5 +- core/settlement/engine_test.go | 100 +++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/core/settlement/engine.go b/core/settlement/engine.go index 0eba081daad..be3a486f122 100644 --- a/core/settlement/engine.go +++ b/core/settlement/engine.go @@ -345,7 +345,10 @@ func (e *Engine) SettleMTM(ctx context.Context, markPrice *num.Uint, positions [ } // no need for this lock anymore e.mu.Unlock() - delta := num.UintZero().Sub(lossTotal, winTotal) + delta := num.UintZero() + if lossTotal.GT(winTotal) { + delta.Sub(lossTotal, winTotal) + } // make sure largests share is never nil if largestShare == nil { largestShare = &mtmTransfer{ diff --git a/core/settlement/engine_test.go b/core/settlement/engine_test.go index 50756694fe2..43bcade79b3 100644 --- a/core/settlement/engine_test.go +++ b/core/settlement/engine_test.go @@ -84,6 +84,7 @@ func TestMTMWinDistribution(t *testing.T) { t.Run("A MTM loss party with a loss of value 1, with several parties needing a win", testMTMWinOneExcess) t.Run("Distribute win excess in a scenario where no transfer amount is < 1", testMTMWinNoZero) t.Run("Distribute loss excess in a scenario where no transfer amount is < 1", testMTMWinWithZero) + t.Run("A MTM case where win total > loss total", testMTMWinGTLoss) } func testSettlingAFundingPeriod(t *testing.T) { @@ -1395,6 +1396,105 @@ func testMTMWinWithZero(t *testing.T) { require.NotEmpty(t, transfers) } +func testMTMWinGTLoss(t *testing.T) { + // cheat by setting the factor to some specific value, makes it easier to create a scenario where win/loss amounts don't match + engine := getTestEngineWithFactor(t, 5) + defer engine.Finish() + + price := num.NewUint(100000) + one := num.NewUint(1) + two := num.Sum(one, one) + ctx := context.Background() + + initPos := []testPos{ + { + price: price.Clone(), + party: "party-1", + size: 10, + }, + { + price: price.Clone(), + party: "party-2", + size: -4, + }, + { + price: price.Clone(), + party: "party-3", + size: -1, + }, + { + price: price.Clone(), + party: "party-4", + size: -5, + }, + { + price: price.Clone(), + party: "party-6", + size: -1, + }, + { + price: price.Clone(), + party: "party-7", + size: 1, + }, + } + init := make([]events.MarketPosition, 0, len(initPos)) + for _, p := range initPos { + init = append(init, p) + } + + somePrice := num.Sum(price, two) + newPrice := num.Sum(price, one) + newParty := testPos{ + size: 3, + price: newPrice.Clone(), + party: "party-5", + } + + trades := []*types.Trade{ + { + Size: 1, + Buyer: newParty.party, + Seller: initPos[0].party, + Price: somePrice.Clone(), + }, + { + Size: 1, + Buyer: newParty.party, + Seller: initPos[3].party, + Price: somePrice.Clone(), + }, + { + Size: 1, + Buyer: newParty.party, + Seller: initPos[0].party, + Price: newPrice.Clone(), + }, + } + updates := make([]events.MarketPosition, 0, len(initPos)+1) + for _, trade := range trades { + for i, p := range initPos { + if p.party == trade.Seller { + p.size -= int64(trade.Size) + initPos[i] = p + } + p.price = trade.Price.Clone() + } + } + for _, p := range initPos { + updates = append(updates, p) + } + updates = append(updates, newParty) + engine.Update(init) + for _, trade := range trades { + engine.AddTrade(trade) + } + transfers := engine.SettleMTM(ctx, newPrice.Clone(), updates) + require.NotEmpty(t, transfers) + // just a single transfer of 2 + require.True(t, transfers[0].Transfer().Amount.Amount.EQ(two)) +} + // {{{. func (te *testEngine) getExpiryPositions(positions ...posValue) []events.MarketPosition { te.positions = make([]*mocks.MockMarketPosition, 0, len(positions)) From e77c1ab15a5037f4230507f34fb848986852d2d5 Mon Sep 17 00:00:00 2001 From: Elias Van Ootegem Date: Mon, 10 Jun 2024 17:31:53 +0100 Subject: [PATCH 2/7] chore: add migration to fix bad PnL Signed-off-by: Elias Van Ootegem --- datanode/sqlstore/migrations/0111_fix_pnl.sql | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 datanode/sqlstore/migrations/0111_fix_pnl.sql diff --git a/datanode/sqlstore/migrations/0111_fix_pnl.sql b/datanode/sqlstore/migrations/0111_fix_pnl.sql new file mode 100644 index 00000000000..5477a8e6798 --- /dev/null +++ b/datanode/sqlstore/migrations/0111_fix_pnl.sql @@ -0,0 +1,26 @@ +-- +goose Up + +WITH updated_pnl AS ( + SELECT DISTINCT ON (pc.party_id) pc.party_id AS pid, pc.market_id AS mid, pc.loss - ph.loss AS correct_loss, pc.loss_socialisation_amount - ph.loss_socialisation_amount as correct_loss_soc, + pc.realised_pnl - ph.realised_pnl as correct_pnl, + pc.pending_realised_pnl - ph.pending_realised_pnl as correct_ppnl, + pc.pending_open_volume + FROM public.positions_current AS pc + JOIN public.positions AS ph + ON pc.party_id = ph.party_id + AND pc.market_id = ph.market_id + WHERE pc.party_id IN ('\x947a700141e3d175304ee176d0beecf9ee9f462e09330e33c386952caf21f679', '\x15a8f372e255c6fa596a0b3acd62bc3be63b65188c23d33fc350f38ef52902e3', '\xaa1ce33b0b31a2e0f0a947ba83f64fa4a7e5d977fffb82c278c3b33fb0498113', '\x6527ffdd223ef2b4695ad90d832adc5493e9b8e25ad3185e67d873767f1f275e') + AND ph.vega_time >= '2024-06-08 19:38:49.89053+00' + AND pc.market_id = '\xe63a37edae8b74599d976f5dedbf3316af82579447f7a08ae0495a021fd44d13' + ORDER BY pc.party_id, ph.vega_time ASC +) +UPDATE positions_current pu +SET loss = updated_pnl.correct_loss, + loss_socialisation_amount = updated_pnl.correct_loss_soc, + realised_pnl = updated_pnl.correct_pnl, + pending_realised_pnl = updated_pnl.correct_ppnl +FROM updated_pnl +WHERE pu.party_id = updated_pnl.pid AND pu.market_id = updated_pnl.mid + +-- +goose Down +-- nothing From 9c1e0e6e9859b089db8c7205b34d66655ca517e7 Mon Sep 17 00:00:00 2001 From: Jeremy Letang Date: Tue, 11 Jun 2024 14:20:25 +0100 Subject: [PATCH 3/7] chore: fix migration --- datanode/sqlstore/migrations/0111_fix_pnl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datanode/sqlstore/migrations/0111_fix_pnl.sql b/datanode/sqlstore/migrations/0111_fix_pnl.sql index 5477a8e6798..e66fa64f332 100644 --- a/datanode/sqlstore/migrations/0111_fix_pnl.sql +++ b/datanode/sqlstore/migrations/0111_fix_pnl.sql @@ -20,7 +20,7 @@ SET loss = updated_pnl.correct_loss, realised_pnl = updated_pnl.correct_pnl, pending_realised_pnl = updated_pnl.correct_ppnl FROM updated_pnl -WHERE pu.party_id = updated_pnl.pid AND pu.market_id = updated_pnl.mid +WHERE pu.party_id = updated_pnl.pid AND pu.market_id = updated_pnl.mid; -- +goose Down -- nothing From 8b47858652b280304c01c8b9894d73a1337816b0 Mon Sep 17 00:00:00 2001 From: Jeremy Letang Date: Tue, 11 Jun 2024 14:28:42 +0100 Subject: [PATCH 4/7] chore: rename migration Signed-off-by: Jeremy Letang --- .../sqlstore/migrations/{0111_fix_pnl.sql => 0112_fix_pnl.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename datanode/sqlstore/migrations/{0111_fix_pnl.sql => 0112_fix_pnl.sql} (100%) diff --git a/datanode/sqlstore/migrations/0111_fix_pnl.sql b/datanode/sqlstore/migrations/0112_fix_pnl.sql similarity index 100% rename from datanode/sqlstore/migrations/0111_fix_pnl.sql rename to datanode/sqlstore/migrations/0112_fix_pnl.sql From 73d72ac5f8be5cd291b8ee67bb087697a0ebda83 Mon Sep 17 00:00:00 2001 From: Elias Van Ootegem Date: Wed, 12 Jun 2024 10:03:35 +0100 Subject: [PATCH 5/7] test: fix network history tests Signed-off-by: Elias Van Ootegem --- datanode/networkhistory/service_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/datanode/networkhistory/service_test.go b/datanode/networkhistory/service_test.go index 5a42d055cce..29dd0a385e2 100644 --- a/datanode/networkhistory/service_test.go +++ b/datanode/networkhistory/service_test.go @@ -379,12 +379,12 @@ func TestMain(t *testing.M) { log.Infof("%s", goldenSourceHistorySegment[4000].HistorySegmentID) log.Infof("%s", goldenSourceHistorySegment[5000].HistorySegmentID) - panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[1000].HistorySegmentID, "QmRHcfKhp4XHAaXDKwEqKVrhTCJPT53pQ7RQEoAj423CA3", snapshots) - panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[2000].HistorySegmentID, "QmZPvVJj9P3yQxr9S4Ck5NFp1ncKmQN1HQzLt9h8A7Gxoz", snapshots) - panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[2500].HistorySegmentID, "QmPnhU8dxWJoMRUwwJZ35YnS9SV7PFFSE8PJuB5u9iscwR", snapshots) - panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[3000].HistorySegmentID, "Qma4K4eug73JcwP9v5Cgrf4L9LkQ3W5H5TKQEb8zdqjpXM", snapshots) - panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[4000].HistorySegmentID, "QmX3P7SKGNXma8be9c3Q94K6ycMuc67oRhAJAREntxzXUx", snapshots) - panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[5000].HistorySegmentID, "QmUbgGpmRZi8mc3nmkGewwksSiSCR3gyhoa4JXa3ca9RxT", snapshots) + panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[1000].HistorySegmentID, "Qmf5QzhfokNqFE7N5Kf1oUmrzA94tTJNRWxB9ULShf1uzV", snapshots) + panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[2000].HistorySegmentID, "QmbsusmxWeUxCHdJRznLbtcevgKhF4dkvL9yFnhpJS3wQa", snapshots) + panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[2500].HistorySegmentID, "QmPfUnXkuBzmMjB2rcdtGMNhdd6mc3cvYMfS9rnMXjfu6L", snapshots) + panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[3000].HistorySegmentID, "QmRwtm3pUuUYG69tmbssFkYd2DafLELBeDfxuEjqjmXct8", snapshots) + panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[4000].HistorySegmentID, "QmQmC4xrdSJMiJ9MLgWdU1mptVv9TdSsYyosusd8dCt8A4", snapshots) + panicIfHistorySegmentIdsNotEqual(goldenSourceHistorySegment[5000].HistorySegmentID, "QmU3zDAHiQYWLyeYzwwCAynoh8A7g38ZaLpdV2h5pgkuL5", snapshots) }, postgresRuntimePath, sqlFs) if exitCode != 0 { From 1ea2d31c2370ef006bb1d065387cb5d22ecd2d24 Mon Sep 17 00:00:00 2001 From: Elias Van Ootegem Date: Wed, 12 Jun 2024 10:31:16 +0100 Subject: [PATCH 6/7] fix: remove public from migration script Signed-off-by: Elias Van Ootegem --- datanode/sqlstore/migrations/0112_fix_pnl.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datanode/sqlstore/migrations/0112_fix_pnl.sql b/datanode/sqlstore/migrations/0112_fix_pnl.sql index e66fa64f332..059c598506e 100644 --- a/datanode/sqlstore/migrations/0112_fix_pnl.sql +++ b/datanode/sqlstore/migrations/0112_fix_pnl.sql @@ -5,8 +5,8 @@ WITH updated_pnl AS ( pc.realised_pnl - ph.realised_pnl as correct_pnl, pc.pending_realised_pnl - ph.pending_realised_pnl as correct_ppnl, pc.pending_open_volume - FROM public.positions_current AS pc - JOIN public.positions AS ph + FROM positions_current AS pc + JOIN positions AS ph ON pc.party_id = ph.party_id AND pc.market_id = ph.market_id WHERE pc.party_id IN ('\x947a700141e3d175304ee176d0beecf9ee9f462e09330e33c386952caf21f679', '\x15a8f372e255c6fa596a0b3acd62bc3be63b65188c23d33fc350f38ef52902e3', '\xaa1ce33b0b31a2e0f0a947ba83f64fa4a7e5d977fffb82c278c3b33fb0498113', '\x6527ffdd223ef2b4695ad90d832adc5493e9b8e25ad3185e67d873767f1f275e') From f1834e513a6136017de1289a9f72d75459b8a028 Mon Sep 17 00:00:00 2001 From: Elias Van Ootegem Date: Wed, 12 Jun 2024 11:33:32 +0100 Subject: [PATCH 7/7] fix: add missing adjustment field to migration Signed-off-by: Elias Van Ootegem --- datanode/sqlstore/migrations/0112_fix_pnl.sql | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/datanode/sqlstore/migrations/0112_fix_pnl.sql b/datanode/sqlstore/migrations/0112_fix_pnl.sql index 059c598506e..e9bed15d97e 100644 --- a/datanode/sqlstore/migrations/0112_fix_pnl.sql +++ b/datanode/sqlstore/migrations/0112_fix_pnl.sql @@ -1,10 +1,10 @@ -- +goose Up WITH updated_pnl AS ( - SELECT DISTINCT ON (pc.party_id) pc.party_id AS pid, pc.market_id AS mid, pc.loss - ph.loss AS correct_loss, pc.loss_socialisation_amount - ph.loss_socialisation_amount as correct_loss_soc, - pc.realised_pnl - ph.realised_pnl as correct_pnl, - pc.pending_realised_pnl - ph.pending_realised_pnl as correct_ppnl, - pc.pending_open_volume + SELECT DISTINCT ON (pc.party_id) pc.party_id AS pid, pc.market_id AS mid, pc.loss - ph.loss AS correct_loss, pc.loss_socialisation_amount - ph.loss_socialisation_amount AS correct_loss_soc, + pc.realised_pnl - ph.realised_pnl AS correct_pnl, + pc.pending_realised_pnl - ph.pending_realised_pnl AS correct_ppnl, + pc.adjustment - ph.adjustment AS correct_adj FROM positions_current AS pc JOIN positions AS ph ON pc.party_id = ph.party_id @@ -14,13 +14,14 @@ WITH updated_pnl AS ( AND pc.market_id = '\xe63a37edae8b74599d976f5dedbf3316af82579447f7a08ae0495a021fd44d13' ORDER BY pc.party_id, ph.vega_time ASC ) -UPDATE positions_current pu +UPDATE positions_current SET loss = updated_pnl.correct_loss, loss_socialisation_amount = updated_pnl.correct_loss_soc, realised_pnl = updated_pnl.correct_pnl, - pending_realised_pnl = updated_pnl.correct_ppnl + pending_realised_pnl = updated_pnl.correct_ppnl, + adjustment = updated_pnl.correct_adj FROM updated_pnl -WHERE pu.party_id = updated_pnl.pid AND pu.market_id = updated_pnl.mid; +WHERE party_id = updated_pnl.pid AND market_id = updated_pnl.mid; -- +goose Down -- nothing