From 310e16c1643a918c9a0f897e334a310f817adf92 Mon Sep 17 00:00:00 2001 From: wwestgarth Date: Fri, 19 Jul 2024 14:32:44 +0100 Subject: [PATCH] fix: separate on-book and off-book volume in auction uncrossing calcs to ensure exact order extraction --- core/matching/cached_orderbook.go | 2 +- core/matching/can_uncross_test.go | 4 +- core/matching/indicative_price_and_volume.go | 86 +++++++++++++++----- core/matching/orderbook.go | 33 ++++---- 4 files changed, 87 insertions(+), 38 deletions(-) diff --git a/core/matching/cached_orderbook.go b/core/matching/cached_orderbook.go index c8a0dc5dddf..8707e974864 100644 --- a/core/matching/cached_orderbook.go +++ b/core/matching/cached_orderbook.go @@ -200,7 +200,7 @@ func (b *CachedOrderBook) GetIndicativePriceAndVolume() (*num.Uint, uint64, type volume, cachedVolOk := b.cache.GetIndicativeVolume() side, cachedSideOk := b.cache.GetIndicativeUncrossingSide() if !cachedPriceOk || !cachedVolOk || !cachedSideOk { - price, volume, side = b.OrderBook.GetIndicativePriceAndVolume() + price, volume, side, _ = b.OrderBook.GetIndicativePriceAndVolume() b.cache.SetIndicativePrice(price.Clone()) b.cache.SetIndicativeVolume(volume) b.cache.SetIndicativeUncrossingSide(side) diff --git a/core/matching/can_uncross_test.go b/core/matching/can_uncross_test.go index 6449aac813b..f07d434b293 100644 --- a/core/matching/can_uncross_test.go +++ b/core/matching/can_uncross_test.go @@ -80,7 +80,7 @@ func TestBidAndAskPresentAfterAuction(t *testing.T) { assert.NoError(t, err) } - indicativePrice, indicativeVolume, indicativeSide := book.GetIndicativePriceAndVolume() + indicativePrice, indicativeVolume, indicativeSide, _ := book.GetIndicativePriceAndVolume() assert.Equal(t, indicativePrice.Uint64(), uint64(1975)) assert.Equal(t, int(indicativeVolume), 5) assert.Equal(t, indicativeSide, types.SideBuy) @@ -143,7 +143,7 @@ func TestBidAndAskPresentAfterAuctionInverse(t *testing.T) { assert.NoError(t, err) } - indicativePrice, indicativeVolume, indicativeSide := book.GetIndicativePriceAndVolume() + indicativePrice, indicativeVolume, indicativeSide, _ := book.GetIndicativePriceAndVolume() assert.Equal(t, indicativePrice.Uint64(), uint64(1950)) assert.Equal(t, int(indicativeVolume), 5) assert.Equal(t, indicativeSide, types.SideBuy) diff --git a/core/matching/indicative_price_and_volume.go b/core/matching/indicative_price_and_volume.go index 6bba28205a3..3483af832fc 100644 --- a/core/matching/indicative_price_and_volume.go +++ b/core/matching/indicative_price_and_volume.go @@ -53,7 +53,8 @@ type ipvPriceLevel struct { } type ipvVolume struct { - volume uint64 + volume uint64 + offbookVolume uint64 // how much of the above total volume is coming from AMMs } type ipvGeneratedOffbook struct { @@ -131,13 +132,15 @@ func (ipv *IndicativePriceAndVolume) buildInitialOffbookShape(offbook OffbookSou // expand all AMM's into orders within the crossed region and add them to the price-level cache buys, sells := offbook.OrderbookShape(min, max, nil) - for _, o := range buys { + for i := len(buys) - 1; i >= 0; i-- { + o := buys[i] mpl, ok := mplm[*o.Price] if !ok { - mpl = ipvPriceLevel{price: o.Price, buypl: ipvVolume{0}, sellpl: ipvVolume{0}} + mpl = ipvPriceLevel{price: o.Price, buypl: ipvVolume{0, 0}, sellpl: ipvVolume{0, 0}} } // increment the volume at this level mpl.buypl.volume += o.Size + mpl.buypl.offbookVolume += o.Size mplm[*o.Price] = mpl if ipv.generated[o.Party] == nil { @@ -149,10 +152,11 @@ func (ipv *IndicativePriceAndVolume) buildInitialOffbookShape(offbook OffbookSou for _, o := range sells { mpl, ok := mplm[*o.Price] if !ok { - mpl = ipvPriceLevel{price: o.Price, buypl: ipvVolume{0}, sellpl: ipvVolume{0}} + mpl = ipvPriceLevel{price: o.Price, buypl: ipvVolume{0, 0}, sellpl: ipvVolume{0, 0}} } mpl.sellpl.volume += o.Size + mpl.sellpl.offbookVolume += o.Size mplm[*o.Price] = mpl if ipv.generated[o.Party] == nil { @@ -170,10 +174,10 @@ func (ipv *IndicativePriceAndVolume) removeOffbookShape(party string) { // remove all the old volume for the AMM's for _, o := range orders.buy { - ipv.RemoveVolumeAtPrice(o.Price, o.Size, o.Side) + ipv.RemoveVolumeAtPrice(o.Price, o.Size, o.Side, true) } for _, o := range orders.sell { - ipv.RemoveVolumeAtPrice(o.Price, o.Size, o.Side) + ipv.RemoveVolumeAtPrice(o.Price, o.Size, o.Side, true) } // clear it out the saved generated orders for the offbook shape @@ -184,8 +188,10 @@ func (ipv *IndicativePriceAndVolume) addOffbookShape(party *string, minPrice, ma // recalculate new orders for the shape and add the volume in buys, sells := ipv.offbook.OrderbookShape(minPrice, maxPrice, party) - for _, o := range buys { - ipv.AddVolumeAtPrice(o.Price, o.Size, o.Side) + // add buys backwards so that the best-bid is first + for i := len(buys) - 1; i >= 0; i-- { + o := buys[i] + ipv.AddVolumeAtPrice(o.Price, o.Size, o.Side, true) if ipv.generated[o.Party] == nil { ipv.generated[o.Party] = &ipvGeneratedOffbook{} @@ -193,8 +199,9 @@ func (ipv *IndicativePriceAndVolume) addOffbookShape(party *string, minPrice, ma ipv.generated[o.Party].add(o) } + // add buys fowards so that the best-ask is first for _, o := range sells { - ipv.AddVolumeAtPrice(o.Price, o.Size, o.Side) + ipv.AddVolumeAtPrice(o.Price, o.Size, o.Side, true) if ipv.generated[o.Party] == nil { ipv.generated[o.Party] = &ipvGeneratedOffbook{} @@ -223,7 +230,7 @@ func (ipv *IndicativePriceAndVolume) buildInitialCumulativeLevels(buy, sell *Ord mplm := map[num.Uint]ipvPriceLevel{} for i := len(buy.levels) - 1; i >= 0; i-- { - mplm[*buy.levels[i].price] = ipvPriceLevel{price: buy.levels[i].price.Clone(), buypl: ipvVolume{buy.levels[i].volume}, sellpl: ipvVolume{0}} + mplm[*buy.levels[i].price] = ipvPriceLevel{price: buy.levels[i].price.Clone(), buypl: ipvVolume{buy.levels[i].volume, 0}, sellpl: ipvVolume{0, 0}} } // now we add all the sells @@ -232,10 +239,10 @@ func (ipv *IndicativePriceAndVolume) buildInitialCumulativeLevels(buy, sell *Ord for i := len(sell.levels) - 1; i >= 0; i-- { price := sell.levels[i].price.Clone() if mpl, ok := mplm[*price]; ok { - mpl.sellpl = ipvVolume{sell.levels[i].volume} + mpl.sellpl = ipvVolume{sell.levels[i].volume, 0} mplm[*price] = mpl } else { - mplm[*price] = ipvPriceLevel{price: price, sellpl: ipvVolume{sell.levels[i].volume}, buypl: ipvVolume{0}} + mplm[*price] = ipvPriceLevel{price: price, sellpl: ipvVolume{sell.levels[i].volume, 0}, buypl: ipvVolume{0, 0}} } } @@ -254,16 +261,22 @@ func (ipv *IndicativePriceAndVolume) buildInitialCumulativeLevels(buy, sell *Ord sort.Slice(ipv.levels, func(i, j int) bool { return ipv.levels[i].price.GT(ipv.levels[j].price) }) } -func (ipv *IndicativePriceAndVolume) incrementLevelVolume(idx int, volume uint64, side types.Side) { +func (ipv *IndicativePriceAndVolume) incrementLevelVolume(idx int, volume uint64, side types.Side, isOffbook bool) { switch side { case types.SideBuy: ipv.levels[idx].buypl.volume += volume + if isOffbook { + ipv.levels[idx].buypl.offbookVolume += volume + } case types.SideSell: ipv.levels[idx].sellpl.volume += volume + if isOffbook { + ipv.levels[idx].sellpl.offbookVolume += volume + } } } -func (ipv *IndicativePriceAndVolume) AddVolumeAtPrice(price *num.Uint, volume uint64, side types.Side) { +func (ipv *IndicativePriceAndVolume) AddVolumeAtPrice(price *num.Uint, volume uint64, side types.Side, isOffbook bool) { if price.GTE(ipv.lastMinPrice) || price.LTE(ipv.lastMaxPrice) { // the new price added is in the range, that will require // to recompute the whole range when we call GetCumulativePriceLevels @@ -275,25 +288,31 @@ func (ipv *IndicativePriceAndVolume) AddVolumeAtPrice(price *num.Uint, volume ui }) if i < len(ipv.levels) && ipv.levels[i].price.EQ(price) { // we found the price level, let's add the volume there, and we are done - ipv.incrementLevelVolume(i, volume, side) + ipv.incrementLevelVolume(i, volume, side, isOffbook) } else { ipv.levels = append(ipv.levels, ipvPriceLevel{}) copy(ipv.levels[i+1:], ipv.levels[i:]) ipv.levels[i] = ipvPriceLevel{price: price.Clone()} - ipv.incrementLevelVolume(i, volume, side) + ipv.incrementLevelVolume(i, volume, side, isOffbook) } } -func (ipv *IndicativePriceAndVolume) decrementLevelVolume(idx int, volume uint64, side types.Side) { +func (ipv *IndicativePriceAndVolume) decrementLevelVolume(idx int, volume uint64, side types.Side, isOffbook bool) { switch side { case types.SideBuy: ipv.levels[idx].buypl.volume -= volume + if isOffbook { + ipv.levels[idx].buypl.offbookVolume -= volume + } case types.SideSell: ipv.levels[idx].sellpl.volume -= volume + if isOffbook { + ipv.levels[idx].sellpl.offbookVolume -= volume + } } } -func (ipv *IndicativePriceAndVolume) RemoveVolumeAtPrice(price *num.Uint, volume uint64, side types.Side) { +func (ipv *IndicativePriceAndVolume) RemoveVolumeAtPrice(price *num.Uint, volume uint64, side types.Side, isOffbook bool) { if price.GTE(ipv.lastMinPrice) || price.LTE(ipv.lastMaxPrice) { // the new price added is in the range, that will require // to recompute the whole range when we call GetCumulativePriceLevels @@ -305,7 +324,7 @@ func (ipv *IndicativePriceAndVolume) RemoveVolumeAtPrice(price *num.Uint, volume }) if i < len(ipv.levels) && ipv.levels[i].price.EQ(price) { // we found the price level, let's add the volume there, and we are done - ipv.decrementLevelVolume(i, volume, side) + ipv.decrementLevelVolume(i, volume, side, isOffbook) } else { ipv.log.Panic("cannot remove volume from a non-existing level", logging.String("side", side.String()), @@ -379,6 +398,7 @@ func (ipv *IndicativePriceAndVolume) GetCumulativePriceLevels(maxPrice, minPrice var ( cumulativeVolumeSell, cumulativeVolumeBuy, maxTradable uint64 + cumulativeOffbookSell, cumulativeOffbookBuy uint64 // here the caching buf is already allocated, we can just resize it // based on the required length cumulativeVolumes = ipv.buf[:len(rangedLevels)] @@ -404,18 +424,23 @@ func (ipv *IndicativePriceAndVolume) GetCumulativePriceLevels(maxPrice, minPrice // if we had a price level in the buy side, use it if rangedLevels[j].buypl.volume > 0 { cumulativeVolumeBuy += rangedLevels[j].buypl.volume + cumulativeOffbookBuy += rangedLevels[j].buypl.offbookVolume cumulativeVolumes[j].bidVolume = rangedLevels[j].buypl.volume } // same same if rangedLevels[i].sellpl.volume > 0 { cumulativeVolumeSell += rangedLevels[i].sellpl.volume + cumulativeOffbookSell += rangedLevels[i].sellpl.offbookVolume cumulativeVolumes[i].askVolume = rangedLevels[i].sellpl.volume } // this will always erase the previous values cumulativeVolumes[j].cumulativeBidVolume = cumulativeVolumeBuy + cumulativeVolumes[j].cumulativeBidOffbook = cumulativeOffbookBuy + cumulativeVolumes[i].cumulativeAskVolume = cumulativeVolumeSell + cumulativeVolumes[i].cumulativeAskOffbook = cumulativeOffbookSell // we just do that // price | sell | buy | vol | ibuy | isell @@ -448,7 +473,11 @@ func (ipv *IndicativePriceAndVolume) GetCumulativePriceLevels(maxPrice, minPrice // ExtractOffbookOrders returns the cached expanded orders of AMM's in the crossed region of the given side. These // are the order that we will send in aggressively to uncrossed the book. -func (ipv *IndicativePriceAndVolume) ExtractOffbookOrders(price *num.Uint, side types.Side) ([]*types.Order, uint64) { +func (ipv *IndicativePriceAndVolume) ExtractOffbookOrders(price *num.Uint, side types.Side, target uint64) []*types.Order { + if target == 0 { + return []*types.Order{} + } + var volume uint64 orders := []*types.Order{} // the ipv keeps track of all the expand AMM orders in the crossed region @@ -469,7 +498,22 @@ func (ipv *IndicativePriceAndVolume) ExtractOffbookOrders(price *num.Uint, side } orders = append(orders, o) volume += o.Size + + // if we're extracted enough we can stop now + if volume == target { + return orders + } } } - return orders, volume + + if volume != target { + ipv.log.Panic("Failed to extract AMM orders for uncrossing", + logging.BigUint("price", price), + logging.Uint64("volume", volume), + logging.Uint64("extracted-volume", volume), + logging.Uint64("target-volume", target), + ) + } + + return orders } diff --git a/core/matching/orderbook.go b/core/matching/orderbook.go index 0aa42e06788..694304015a0 100644 --- a/core/matching/orderbook.go +++ b/core/matching/orderbook.go @@ -86,6 +86,10 @@ type CumulativeVolumeLevel struct { cumulativeBidVolume uint64 cumulativeAskVolume uint64 maxTradableAmount uint64 + + // keep track of how much of the cumulative volume is from AMMs + cumulativeBidOffbook uint64 + cumulativeAskOffbook uint64 } func (b *OrderBook) Hash() []byte { @@ -345,7 +349,7 @@ func (b *OrderBook) canUncross(requireTrades bool) bool { if buyMatch && sellMatch { return true } - _, v, _ := b.GetIndicativePriceAndVolume() + _, v, _, _ := b.GetIndicativePriceAndVolume() // no buy orders remaining on the book after uncrossing, it buyMatches exactly vol := uint64(0) if !buyMatch { @@ -394,14 +398,14 @@ func (b *OrderBook) canUncross(requireTrades bool) bool { } // GetIndicativePriceAndVolume Calculates the indicative price and volume of the order book without modifying the order book state. -func (b *OrderBook) GetIndicativePriceAndVolume() (retprice *num.Uint, retvol uint64, retside types.Side) { +func (b *OrderBook) GetIndicativePriceAndVolume() (retprice *num.Uint, retvol uint64, retside types.Side, offbookVolume uint64) { // Generate a set of price level pairs with their maximum tradable volumes cumulativeVolumes, maxTradableAmount, err := b.buildCumulativePriceLevels() if err != nil { if b.log.GetLevel() <= logging.DebugLevel { b.log.Debug("could not get cumulative price levels", logging.Error(err)) } - return num.UintZero(), 0, types.SideUnspecified + return num.UintZero(), 0, types.SideUnspecified, 0 } // Pull out all prices that match that volume @@ -432,14 +436,17 @@ func (b *OrderBook) GetIndicativePriceAndVolume() (retprice *num.Uint, retvol ui if ordersToFill == 0 { // Buys fill exactly, uncross from the buy side uncrossSide = types.SideBuy + offbookVolume = value.cumulativeBidOffbook break } else if ordersToFill < 0 { // Buys are not exact, uncross from the sell side uncrossSide = types.SideSell + offbookVolume = value.cumulativeAskOffbook break } } - return uncrossPrice, maxTradableAmount, uncrossSide + + return uncrossPrice, maxTradableAmount, uncrossSide, offbookVolume } // GetIndicativePrice Calculates the indicative price of the order book without modifying the order book state. @@ -474,7 +481,7 @@ func (b *OrderBook) GetIndicativePrice() (retprice *num.Uint) { func (b *OrderBook) GetIndicativeTrades() ([]*types.Trade, error) { // Get the uncrossing price and which side has the most volume at that price - price, volume, uncrossSide := b.GetIndicativePriceAndVolume() + price, volume, uncrossSide, offbookVolume := b.GetIndicativePriceAndVolume() // If we have no uncrossing price, we have nothing to do if price.IsZero() && volume == 0 { @@ -497,8 +504,7 @@ func (b *OrderBook) GetIndicativeTrades() ([]*types.Trade, error) { } // extract uncrossing orders from all AMMs - var offbookVolume uint64 - uncrossOrders, offbookVolume = b.indicativePriceAndVolume.ExtractOffbookOrders(price, uncrossSide) + uncrossOrders = b.indicativePriceAndVolume.ExtractOffbookOrders(price, uncrossSide, offbookVolume) // the remaining volume should now come from the orderbook volume -= offbookVolume @@ -545,7 +551,7 @@ func (b *OrderBook) buildCumulativePriceLevels() ([]CumulativeVolumeLevel, uint6 // if removeOrders is set to true then matched orders get removed from the book. func (b *OrderBook) uncrossBook() ([]*types.OrderConfirmation, error) { // Get the uncrossing price and which side has the most volume at that price - price, volume, uncrossSide := b.GetIndicativePriceAndVolume() + price, volume, uncrossSide, offbookVolume := b.GetIndicativePriceAndVolume() // If we have no uncrossing price, we have nothing to do if price.IsZero() && volume == 0 { @@ -567,8 +573,7 @@ func (b *OrderBook) uncrossBook() ([]*types.OrderConfirmation, error) { } // extract uncrossing orders from all AMMs - var offbookVolume uint64 - uncrossOrders, offbookVolume := b.indicativePriceAndVolume.ExtractOffbookOrders(price, uncrossSide) + uncrossOrders := b.indicativePriceAndVolume.ExtractOffbookOrders(price, uncrossSide, offbookVolume) // the remaining volume should now come from the orderbook volume -= offbookVolume @@ -851,12 +856,12 @@ func (b *OrderBook) AmendOrder(originalOrder, amendedOrder *types.Order) error { if volumeChange < 0 { b.indicativePriceAndVolume.RemoveVolumeAtPrice( - amendedOrder.Price, uint64(-volumeChange), amendedOrder.Side) + amendedOrder.Price, uint64(-volumeChange), amendedOrder.Side, false) } if volumeChange > 0 { b.indicativePriceAndVolume.AddVolumeAtPrice( - amendedOrder.Price, uint64(volumeChange), amendedOrder.Side) + amendedOrder.Price, uint64(volumeChange), amendedOrder.Side, false) } return nil @@ -1008,7 +1013,7 @@ func (b *OrderBook) SubmitOrder(order *types.Order) (*types.OrderConfirmation, e // also add it to the indicative price and volume if in auction if b.auction { b.indicativePriceAndVolume.AddVolumeAtPrice( - order.Price, order.TrueRemaining(), order.Side) + order.Price, order.TrueRemaining(), order.Side, false) } } @@ -1095,7 +1100,7 @@ func (b *OrderBook) DeleteOrder( // cancel the order if it expires it if b.auction { b.indicativePriceAndVolume.RemoveVolumeAtPrice( - dorder.Price, dorder.TrueRemaining(), dorder.Side) + dorder.Price, dorder.TrueRemaining(), dorder.Side, false) } return dorder, err }