From a1713e7a61259bbb8a3ff2f844fa42c3818ba926 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 12:46:59 +0100 Subject: [PATCH 01/12] add protocol fee query --- src/models/order_rewards_schema.py | 3 + src/sql/orderbook/order_rewards.sql | 119 ++++++++++++++++++++++++++-- 2 files changed, 115 insertions(+), 7 deletions(-) diff --git a/src/models/order_rewards_schema.py b/src/models/order_rewards_schema.py index f2a5ea63..78f8e2ed 100644 --- a/src/models/order_rewards_schema.py +++ b/src/models/order_rewards_schema.py @@ -26,6 +26,9 @@ def from_pdf_to_dune_records(cls, rewards_df: DataFrame) -> list[dict[str, Any]] "surplus_fee": str(row["surplus_fee"]), "amount": float(row["amount"]), "quote_solver": row["quote_solver"], + "protocol_fee": row["protocol_fee"], + "protocol_fee_token": row["protocol_fee_token"], + "protocol_fee_native_price": row["protocol_fee_native_price"], }, } for row in rewards_df.to_dict(orient="records") diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index efe117d3..9fdd4bcb 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -19,7 +19,107 @@ with trade_hashes as (SELECT settlement.solver, on settlement.tx_from = auction_transaction.tx_from and settlement.tx_nonce = auction_transaction.tx_nonce where block_number > {{start_block}} and block_number <= {{end_block}}), - +order_surplus AS ( + SELECT + at.auction_id, + t.order_uid, + o.sell_token, + o.buy_token, + t.sell_amount, -- the total amount the user sends + t.buy_amount, -- the total amount the user receives + oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price + o.kind, + CASE + WHEN o.kind = 'sell' + THEN t.buy_amount - t.sell_amount * o.buy_amount / (o.sell_amount + o.fee_amount) + WHEN o.kind = 'buy' + THEN t.buy_amount * (o.sell_amount + o.fee_amount) / o.buy_amount - t.sell_amount + END AS surplus, + CASE + WHEN o.kind = 'sell' + THEN o.buy_token + WHEN o.kind = 'buy' + THEN o.sell_token + END AS surplus_token + FROM settlements s -- links block_number and log_index to tx_from and tx_nonce + JOIN auction_transaction at -- links auction_id to tx_from and tx_nonce + ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce + JOIN settlement_scores ss -- contains block_deadline + ON at.auction_id = ss.auction_id + JOIN trades t -- contains traded amounts + ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block + JOIN orders o -- contains tokens and limit amounts + ON t.order_uid = o.uid + JOIN order_execution oe -- contains surplus fee + ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id + WHERE s.block_number > {{start_block}} + AND s.block_number <= {{end_block}} +) +,order_observations AS ( + SELECT + os.auction_id, + os.order_uid, + os.sell_amount, + os.buy_amount, + os.sell_token, + os.observed_fee, + os.surplus, + os.surplus_token, + CASE + WHEN fp.kind = 'surplus' + THEN + CASE + WHEN os.kind = 'sell' + THEN + CASE + WHEN fp.max_volume_factor = 1 -- this is done to avoid a division by zero errors + -- We assume that the case surplus_factor != 1 always. In + -- that case reconstructing the protocol fee would be + -- impossible anyways. This query will return a division by + -- zero error in that case. + THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus + ELSE + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + END + WHEN os.kind = 'buy' + THEN + CASE + WHEN fp.max_volume_factor = 1 + THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus + ELSE + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + END + END + WHEN fp.kind = 'volume' + THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount + END AS protocol_fee, + CASE + WHEN fp.kind = 'surplus' + THEN os.surplus_token + WHEN fp.kind = 'volume' + THEN os.sell_token + END AS protocol_fee_token + FROM order_surplus os + JOIN fee_policies fp -- contains protocol fee policy + ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid +) +,order_observations_prices AS ( + SELECT + oo.order_uid, + oo.auction_id, + oo.protocol_fee, + oo.protocol_fee_token, + ap.price / pow(10, 18) as protocol_fee_native_price + FROM order_observations oo + JOIN auction_prices ap-- contains price: protocol fee token + ON oo.auction_id = ap.auction_id AND oo.protocol_fee_token = ap.token +), winning_quotes as (SELECT concat('0x', encode(oq.solver, 'hex')) as quote_solver, oq.order_uid FROM trades t @@ -28,10 +128,9 @@ with trade_hashes as (SELECT settlement.solver, WHERE ((o.kind = 'sell' AND o.buy_amount <= oq.buy_amount) OR (o.kind='buy' AND o.sell_amount >= oq.sell_amount)) AND o.partially_fillable='f' - AND block_number > {{start_block}} - AND block_number <= {{end_block}} + AND block_number > 19000000 + AND block_number <= 19500000 AND oq.solver != '\x0000000000000000000000000000000000000000') - -- Most efficient column order for sorting would be having tx_hash or order_uid first select block_number, concat('0x', encode(trade_hashes.order_uid, 'hex')) as order_uid, @@ -39,10 +138,16 @@ select block_number, quote_solver, concat('0x', encode(tx_hash, 'hex')) as tx_hash, coalesce(surplus_fee, 0)::text as surplus_fee, - coalesce(reward, 0.0) as amount + coalesce(reward, 0.0) as amount, + coalesce(cast(protocol_fee as numeric(78, 0)), 0)::text as protocol_fee, + concat('0x', encode(protocol_fee_token, 'hex')) as protocol_fee_token, + coalesce(protocol_fee_native_price, 0.0) as protocol_fee_native_price from trade_hashes left outer join order_execution o on trade_hashes.order_uid = o.order_uid and trade_hashes.auction_id = o.auction_id - left outer join winning_quotes wq - on trade_hashes.order_uid = wq.order_uid; + left outer join winning_quotes wq + on trade_hashes.order_uid = wq.order_uid + left outer join order_observations_prices oop + on trade_hashes.order_uid = oop.order_uid + and trade_hashes.auction_id = oop.auction_id; From 5c63fc8aedfc8f12c9b531ca7f83a2c569f026bb Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 13:16:16 +0100 Subject: [PATCH 02/12] adapt test for adding protocol fee to order_rewards --- tests/unit/test_order_rewards_schema.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/test_order_rewards_schema.py b/tests/unit/test_order_rewards_schema.py index c9edf367..75da8882 100644 --- a/tests/unit/test_order_rewards_schema.py +++ b/tests/unit/test_order_rewards_schema.py @@ -16,6 +16,9 @@ def test_order_rewards_transformation(self): "quote_solver": ["0x21", None, "0x22"], "surplus_fee": [12345678910111213, 0, 0], "amount": [40.70410, 39.00522, 0], + "protocol_fee": [1000000000000000, 123123123123123, 0], + "protocol_fee_token": ["0x91", "0x92", None], + "protocol_fee_native_price": [1.0, 0.1, 0.0], } ) @@ -30,6 +33,9 @@ def test_order_rewards_transformation(self): "surplus_fee": "12345678910111213", "amount": 40.70410, "quote_solver": "0x21", + "protocol_fee": "1000000000000000", + "protocol_fee_token": "0x91", + "protocol_fee_native_price": 1.0, }, }, { @@ -41,6 +47,9 @@ def test_order_rewards_transformation(self): "surplus_fee": "0", "amount": 39.00522, "quote_solver": None, + "protocol_fee": "123123123123123", + "protocol_fee_token": "0x92", + "protocol_fee_native_price": 0.1, }, }, { @@ -52,6 +61,9 @@ def test_order_rewards_transformation(self): "surplus_fee": "0", "amount": 0.0, "quote_solver": "0x22", + "protocol_fee": "0", + "protocol_fee_token": None, + "protocol_fee_native_price": 0.0, }, }, ], From 415cc894c40d26bdca5806978241e48359f9003c Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 14:43:44 +0100 Subject: [PATCH 03/12] fix tests for uploading order rewards --- src/models/order_rewards_schema.py | 4 ++-- tests/unit/test_order_rewards_schema.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/models/order_rewards_schema.py b/src/models/order_rewards_schema.py index 78f8e2ed..683b747d 100644 --- a/src/models/order_rewards_schema.py +++ b/src/models/order_rewards_schema.py @@ -26,9 +26,9 @@ def from_pdf_to_dune_records(cls, rewards_df: DataFrame) -> list[dict[str, Any]] "surplus_fee": str(row["surplus_fee"]), "amount": float(row["amount"]), "quote_solver": row["quote_solver"], - "protocol_fee": row["protocol_fee"], + "protocol_fee": str(row["protocol_fee"]), "protocol_fee_token": row["protocol_fee_token"], - "protocol_fee_native_price": row["protocol_fee_native_price"], + "protocol_fee_native_price": float(row["protocol_fee_native_price"]), }, } for row in rewards_df.to_dict(orient="records") diff --git a/tests/unit/test_order_rewards_schema.py b/tests/unit/test_order_rewards_schema.py index 75da8882..4ac5d358 100644 --- a/tests/unit/test_order_rewards_schema.py +++ b/tests/unit/test_order_rewards_schema.py @@ -27,8 +27,8 @@ def test_order_rewards_transformation(self): { "block_number": 1, "order_uid": "0x01", - "solver": "0x51", "tx_hash": "0x71", + "solver": "0x51", "data": { "surplus_fee": "12345678910111213", "amount": 40.70410, @@ -41,8 +41,8 @@ def test_order_rewards_transformation(self): { "block_number": 2, "order_uid": "0x02", - "solver": "0x52", "tx_hash": "0x72", + "solver": "0x52", "data": { "surplus_fee": "0", "amount": 39.00522, @@ -55,8 +55,8 @@ def test_order_rewards_transformation(self): { "block_number": 3, "order_uid": "0x03", - "solver": "0x53", "tx_hash": "0x73", + "solver": "0x53", "data": { "surplus_fee": "0", "amount": 0.0, From 46aeab2dcc1a0842ff67bc59158cae7b9d3f8cb4 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 16:10:17 +0100 Subject: [PATCH 04/12] add protocol fee and fee correction to batch rewards query --- src/fetch/orderbook.py | 1 + src/models/batch_rewards_schema.py | 1 + src/sql/orderbook/batch_rewards.sql | 157 ++++++++++++++++++++-- tests/integration/test_fetch_orderbook.py | 7 + tests/unit/test_batch_rewards_schema.py | 3 + 5 files changed, 157 insertions(+), 12 deletions(-) diff --git a/src/fetch/orderbook.py b/src/fetch/orderbook.py index 06f65603..834c4631 100644 --- a/src/fetch/orderbook.py +++ b/src/fetch/orderbook.py @@ -97,6 +97,7 @@ def get_batch_rewards(cls, block_range: BlockRange) -> DataFrame: open_query("orderbook/batch_rewards.sql") .replace("{{start_block}}", str(block_range.block_from)) .replace("{{end_block}}", str(block_range.block_to)) + .replace("{{EPSILON}}", "10000000000000000") ) data_types = { # According to this: https://stackoverflow.com/a/11548224 diff --git a/src/models/batch_rewards_schema.py b/src/models/batch_rewards_schema.py index a3bc5bab..f22a6c22 100644 --- a/src/models/batch_rewards_schema.py +++ b/src/models/batch_rewards_schema.py @@ -32,6 +32,7 @@ def from_pdf_to_dune_records(cls, rewards_df: DataFrame) -> list[dict[str, Any]] "execution_cost": int(row["execution_cost"]), "surplus": int(row["surplus"]), "fee": int(row["fee"]), + "protocol_fee": int(row["protocol_fee"]), "winning_score": int(row["winning_score"]), "reference_score": int(row["reference_score"]), "participating_solvers": row["participating_solvers"], diff --git a/src/sql/orderbook/batch_rewards.sql b/src/sql/orderbook/batch_rewards.sql index affa000b..f5828817 100644 --- a/src/sql/orderbook/batch_rewards.sql +++ b/src/sql/orderbook/batch_rewards.sql @@ -31,13 +31,140 @@ WITH observed_settlements AS (SELECT WHERE block_deadline > {{start_block}} AND block_deadline <= {{end_block}} GROUP BY ss.auction_id), +-- protocol fees: +order_surplus AS ( + SELECT + ss.winner as solver, + at.auction_id, + s.tx_hash, + t.order_uid, + o.sell_token, + o.buy_token, + t.sell_amount, -- the total amount the user sends + t.buy_amount, -- the total amount the user receives + oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price + o.kind, + CASE + WHEN o.kind = 'sell' + THEN t.buy_amount - t.sell_amount * o.buy_amount / (o.sell_amount + o.fee_amount) + WHEN o.kind = 'buy' + THEN t.buy_amount * (o.sell_amount + o.fee_amount) / o.buy_amount - t.sell_amount + END AS surplus, + CASE + WHEN o.kind = 'sell' + THEN o.buy_token + WHEN o.kind = 'buy' + THEN o.sell_token + END AS surplus_token + FROM settlements s -- links block_number and log_index to tx_from and tx_nonce + JOIN auction_transaction at -- links auction_id to tx_from and tx_nonce + ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce + JOIN settlement_scores ss -- contains block_deadline + ON at.auction_id = ss.auction_id + JOIN trades t -- contains traded amounts + ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block + JOIN orders o -- contains tokens and limit amounts + ON t.order_uid = o.uid + JOIN order_execution oe -- contains surplus fee + ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id + WHERE ss.block_deadline >= {{start_block}} + AND ss.block_deadline <= {{end_block}} +) +,order_observations AS ( + SELECT + os.auction_id, + os.solver, + os.tx_hash, + os.sell_amount, + os.buy_amount, + os.sell_token, + os.observed_fee, + os.surplus, + os.surplus_token, + CASE + WHEN fp.kind = 'surplus' + THEN + CASE + WHEN os.kind = 'sell' + THEN + CASE + WHEN fp.max_volume_factor = 1 -- this is done to avoid a division by zero errors + -- We assume that the case surplus_factor != 1 always. In + -- that case reconstructing the protocol fee would be + -- impossible anyways. This query will return a division by + -- zero error in that case. + THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus + ELSE + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + END + WHEN os.kind = 'buy' + THEN + CASE + WHEN fp.max_volume_factor = 1 + THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus + ELSE + LEAST( + fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus + ) + END + END + WHEN fp.kind = 'volume' + THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount + END AS protocol_fee, + CASE + WHEN fp.kind = 'surplus' + THEN os.surplus_token + WHEN fp.kind = 'volume' + THEN os.sell_token + END AS protocol_fee_token + FROM order_surplus os + JOIN fee_policies fp -- contains protocol fee policy + ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid +) +,order_observations_prices AS ( + SELECT + oo.solver, + oo.tx_hash, + oo.surplus, + oo.protocol_fee, + CASE + WHEN oo.sell_token != oo.protocol_fee_token + THEN (oo.sell_amount - oo.observed_fee) / oo.buy_amount * oo.protocol_fee + ELSE oo.protocol_fee + END AS network_fee_correction, + oo.sell_token as network_fee_token, + ap_surplus.price / pow(10, 18) as surplus_token_price, + ap_protocol.price / pow(10, 18) as protocol_fee_token_price, + ap_sell.price / pow(10, 18) as network_fee_token_price + FROM order_observations oo + JOIN auction_prices ap_sell -- contains price: sell token + ON oo.auction_id = ap_sell.auction_id AND oo.sell_token = ap_sell.token + JOIN auction_prices ap_surplus -- contains price: surplus token + ON oo.auction_id = ap_surplus.auction_id AND oo.surplus_token = ap_surplus.token + JOIN auction_prices ap_protocol -- contains price: protocol fee token + ON oo.auction_id = ap_protocol.auction_id AND oo.protocol_fee_token = ap_protocol.token +), +batch_protocol_fees AS ( + SELECT + solver, + tx_hash, + -- sum(surplus * surplus_token_price) as surplus, + sum(protocol_fee * protocol_fee_token_price) as protocol_fee, + sum(network_fee_correction * network_fee_token_price) as network_fee_correction + FROM order_observations_prices oop + group by solver, tx_hash +), reward_data AS (SELECT -- observations - tx_hash, + os.tx_hash, ss.auction_id, -- TODO - Assuming that `solver == winner` when both not null -- We will need to monitor that `solver == winner`! - coalesce(solver, winner) as solver, + coalesce(os.solver, winner) as solver, block_number as settlement_block, block_deadline, case @@ -53,7 +180,10 @@ WITH observed_settlements AS (SELECT winning_score, reference_score, -- auction_participation - participating_solvers + participating_solvers, + -- protocol_fees + coalesce(cast(protocol_fee as numeric(78, 0)), 0) as protocol_fee, + coalesce(cast(network_fee_correction as numeric(78, 0)), 0) as network_fee_correction FROM settlement_scores ss -- If there are reported scores, -- there will always be a record of auction participants @@ -61,24 +191,26 @@ WITH observed_settlements AS (SELECT ON ss.auction_id = ap.auction_id -- outer joins made in order to capture non-existent settlements. LEFT OUTER JOIN observed_settlements os - ON os.auction_id = ss.auction_id), + ON os.auction_id = ss.auction_id + LEFT OUTER JOIN batch_protocol_fees bpf + ON bpf.tx_hash = os.tx_hash), reward_per_auction as (SELECT tx_hash, + auction_id, settlement_block, block_deadline, solver, execution_cost, surplus, - fee, - surplus + fee - reference_score as uncapped_payment_eth, - -- Uncapped Reward = CLAMP_[-E, E + exec_cost](uncapped_payment_eth) - LEAST(GREATEST(-10000000000000000, surplus + fee - reference_score), - 10000000000000000 + execution_cost) as capped_payment, + protocol_fee + fee - network_fee_correction as fee,-- total fee for ranking + protocol_fee, -- the protocol fee + surplus + protocol_fee + fee - network_fee_correction - reference_score as uncapped_payment_eth, + -- Uncapped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth) + LEAST(GREATEST(-{{EPSILON}}, surplus + protocol_fee + fee - network_fee_correction - reference_score), + {{EPSILON}} + execution_cost) as capped_payment, winning_score, reference_score, - participating_solvers + participating_solvers as participating_solvers FROM reward_data) - - SELECT settlement_block as block_number, block_deadline, case @@ -89,6 +221,7 @@ SELECT settlement_block as block_number, execution_cost::text as execution_cost, surplus::text as surplus, fee::text as fee, + protocol_fee::text as protocol_fee, uncapped_payment_eth::text as uncapped_payment_eth, capped_payment::text as capped_payment, winning_score::text as winning_score, diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index 11ca924e..4f6cb159 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -36,6 +36,9 @@ def test_get_order_rewards(self): ], "surplus_fee": ["0", "0"], "amount": [40.70410, 39.00522], + "protocol_fee": ["0", "0"], + "protocol_fee_token": [None, None], + "protocol_fee_native_price": ["0.0", "0.0"], } ) @@ -74,6 +77,10 @@ def test_get_batch_rewards(self): "10350680045815651", "0", ], + "protocol_fee": [ + "0", + "0", + ], "uncapped_payment_eth": [ "7232682540629268", "82825156151734420", diff --git a/tests/unit/test_batch_rewards_schema.py b/tests/unit/test_batch_rewards_schema.py index cfdaaca1..4c06499b 100644 --- a/tests/unit/test_batch_rewards_schema.py +++ b/tests/unit/test_batch_rewards_schema.py @@ -29,6 +29,7 @@ def test_order_rewards_transformation(self): 1000000000000000, max_uint, ], + "protocol_fee": [2000000000000000, 0], "uncapped_payment_eth": [0, -10 * ONE_ETH], "capped_payment": [-1000000000000000, -1000000000000000], "winning_score": [123456 * ONE_ETH, 6789 * ONE_ETH], @@ -61,6 +62,7 @@ def test_order_rewards_transformation(self): "execution_cost": 9999000000000000000000, "fee": 1000000000000000, "participating_solvers": ["0x51", "0x52", "0x53"], + "protocol_fee": 2000000000000000, "reference_score": 1000000000000000000, "surplus": 2000000000000000000, "uncapped_payment_eth": 0, @@ -84,6 +86,7 @@ def test_order_rewards_transformation(self): "0x55", "0x56", ], + "protocol_fee": 0, "reference_score": 2000000000000000000, "surplus": 3000000000000000000, "uncapped_payment_eth": -10000000000000000000, From 78f7e60c142e4e9827ad3a8b0f63ffd983a33843 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 16:40:52 +0100 Subject: [PATCH 05/12] black fix --- src/models/order_rewards_schema.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/models/order_rewards_schema.py b/src/models/order_rewards_schema.py index 683b747d..44411f4a 100644 --- a/src/models/order_rewards_schema.py +++ b/src/models/order_rewards_schema.py @@ -28,7 +28,9 @@ def from_pdf_to_dune_records(cls, rewards_df: DataFrame) -> list[dict[str, Any]] "quote_solver": row["quote_solver"], "protocol_fee": str(row["protocol_fee"]), "protocol_fee_token": row["protocol_fee_token"], - "protocol_fee_native_price": float(row["protocol_fee_native_price"]), + "protocol_fee_native_price": float( + row["protocol_fee_native_price"] + ), }, } for row in rewards_df.to_dict(orient="records") From 30c5b85c6d284febcf03a01d6af4ca6e684fc430 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 16:48:52 +0100 Subject: [PATCH 06/12] sync protocol_fee_token = NULL when there is no protocol fee --- src/sql/orderbook/order_rewards.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index 9fdd4bcb..1fdc5ce9 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -140,7 +140,9 @@ select block_number, coalesce(surplus_fee, 0)::text as surplus_fee, coalesce(reward, 0.0) as amount, coalesce(cast(protocol_fee as numeric(78, 0)), 0)::text as protocol_fee, - concat('0x', encode(protocol_fee_token, 'hex')) as protocol_fee_token, + CASE WHEN protocol_fee_token is not NULL + THEN concat('0x', encode(protocol_fee_token, 'hex')) + END as protocol_fee_token, coalesce(protocol_fee_native_price, 0.0) as protocol_fee_native_price from trade_hashes left outer join order_execution o From 76431e018cc726faf5b1c752c0a256ad41f6cffb Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 19 Jan 2024 17:03:08 +0100 Subject: [PATCH 07/12] fix tests --- tests/integration/test_fetch_orderbook.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index 4f6cb159..97d2fd08 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -38,7 +38,7 @@ def test_get_order_rewards(self): "amount": [40.70410, 39.00522], "protocol_fee": ["0", "0"], "protocol_fee_token": [None, None], - "protocol_fee_native_price": ["0.0", "0.0"], + "protocol_fee_native_price": [0.0, 0.0], } ) @@ -80,6 +80,7 @@ def test_get_batch_rewards(self): "protocol_fee": [ "0", "0", + "0", ], "uncapped_payment_eth": [ "7232682540629268", From 70401cecd47e9266c0acb0fa33f698861ea12d12 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Sat, 20 Jan 2024 10:11:18 +0100 Subject: [PATCH 08/12] small changes - fix small bug in sql query - add comment for eth cap --- src/fetch/orderbook.py | 2 +- src/sql/orderbook/batch_rewards.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fetch/orderbook.py b/src/fetch/orderbook.py index 834c4631..19cc0660 100644 --- a/src/fetch/orderbook.py +++ b/src/fetch/orderbook.py @@ -97,7 +97,7 @@ def get_batch_rewards(cls, block_range: BlockRange) -> DataFrame: open_query("orderbook/batch_rewards.sql") .replace("{{start_block}}", str(block_range.block_from)) .replace("{{end_block}}", str(block_range.block_to)) - .replace("{{EPSILON}}", "10000000000000000") + .replace("{{EPSILON}}", "10000000000000000") # ETH cap for payment (in WEI) ) data_types = { # According to this: https://stackoverflow.com/a/11548224 diff --git a/src/sql/orderbook/batch_rewards.sql b/src/sql/orderbook/batch_rewards.sql index f5828817..de393065 100644 --- a/src/sql/orderbook/batch_rewards.sql +++ b/src/sql/orderbook/batch_rewards.sql @@ -67,7 +67,7 @@ order_surplus AS ( ON t.order_uid = o.uid JOIN order_execution oe -- contains surplus fee ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id - WHERE ss.block_deadline >= {{start_block}} + WHERE ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}} ) ,order_observations AS ( From 210a4fe4fb1d59042258075462ec07a21210c859 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Sat, 20 Jan 2024 10:11:49 +0100 Subject: [PATCH 09/12] add integration test for protocol fee uses one of the test batches on staging --- tests/integration/test_fetch_orderbook.py | 61 +++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index 97d2fd08..94fdcf26 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -134,6 +134,67 @@ def test_get_batch_rewards(self): ) self.assertIsNone(pd.testing.assert_frame_equal(expected, rewards_df)) + def test_get_batch_rewards_with_protocol_fees(self): + block_number = 19034333 + block_range = BlockRange(block_number, block_number + 9) + rewards_df = OrderbookFetcher.get_batch_rewards(block_range) + print(rewards_df) + expected = pd.DataFrame( + { + "block_number": pd.Series([19034331, 19034333], dtype="Int64"), + "block_deadline": [19034340, 19034341], + "tx_hash": [ + "0x60cfcecab62c2fc03596be4e9a9c7c1113d41a10eb9f821da04b096aacb7e73d", + "0xc725ce0a051955d5c6c98e039cb52a72f96b56b5ea3e95b23ff92c746c0ae4c3", + ], + "solver": [ + "0x8616dcdfcecbde13ccd89eac358dc5abda79ec31", + "0x01246d541e732d7f15d164331711edff217e4665", + ], + "execution_cost": [ + "6874093717444341", + "3885032282790366", + ], + "surplus": [ + "1917140833491803", + "45868496778113149", + ], + "fee": [ + "5746294767802878", + "6875724468968459", + ], + "protocol_fee": [ + "0", + "463318149273870", + ], + "uncapped_payment_eth": [ + "7613796432942231", + "52744221247081608", + ], + "capped_payment": [ + "7613796432942231", + "13885032282790366", + ], + "winning_score": [ + "280137833843581", + "47509854752448587", + ], + "reference_score": [ + "49639168352450", + "0", + ], + "participating_solvers": [ + [ + "0x01246d541e732d7f15d164331711edff217e4665", + "0x2456a4c1241e43e11b0b8f80e31c940bebd9090f", + "0x8616dcdfcecbde13ccd89eac358dc5abda79ec31", + ], + ["0x01246d541e732d7f15d164331711edff217e4665"], + ], + }, + ) + self.assertIsNone(pd.testing.assert_frame_equal(expected, rewards_df)) + if __name__ == "__main__": unittest.main() From ca65ccbd9611ead9eb0486a1f6e69eecbc864ef6 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 22 Jan 2024 11:44:30 +0100 Subject: [PATCH 10/12] rename tables --- src/sql/orderbook/batch_rewards.sql | 30 ++++++++++++++--------------- src/sql/orderbook/order_rewards.sql | 22 ++++++++++----------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/sql/orderbook/batch_rewards.sql b/src/sql/orderbook/batch_rewards.sql index de393065..758ae0eb 100644 --- a/src/sql/orderbook/batch_rewards.sql +++ b/src/sql/orderbook/batch_rewards.sql @@ -70,7 +70,7 @@ order_surplus AS ( WHERE ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}} ) -,order_observations AS ( +,order_protocol_fee AS ( SELECT os.auction_id, os.solver, @@ -125,28 +125,28 @@ order_surplus AS ( JOIN fee_policies fp -- contains protocol fee policy ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid ) -,order_observations_prices AS ( +,order_protocol_fee_prices AS ( SELECT - oo.solver, - oo.tx_hash, - oo.surplus, - oo.protocol_fee, + opf.solver, + opf.tx_hash, + opf.surplus, + opf.protocol_fee, CASE - WHEN oo.sell_token != oo.protocol_fee_token - THEN (oo.sell_amount - oo.observed_fee) / oo.buy_amount * oo.protocol_fee - ELSE oo.protocol_fee + WHEN opf.sell_token != opf.protocol_fee_token + THEN (opf.sell_amount - opf.observed_fee) / opf.buy_amount * opf.protocol_fee + ELSE opf.protocol_fee END AS network_fee_correction, - oo.sell_token as network_fee_token, + opf.sell_token as network_fee_token, ap_surplus.price / pow(10, 18) as surplus_token_price, ap_protocol.price / pow(10, 18) as protocol_fee_token_price, ap_sell.price / pow(10, 18) as network_fee_token_price - FROM order_observations oo + FROM order_protocol_fee opf JOIN auction_prices ap_sell -- contains price: sell token - ON oo.auction_id = ap_sell.auction_id AND oo.sell_token = ap_sell.token + ON opf.auction_id = ap_sell.auction_id AND opf.sell_token = ap_sell.token JOIN auction_prices ap_surplus -- contains price: surplus token - ON oo.auction_id = ap_surplus.auction_id AND oo.surplus_token = ap_surplus.token + ON opf.auction_id = ap_surplus.auction_id AND opf.surplus_token = ap_surplus.token JOIN auction_prices ap_protocol -- contains price: protocol fee token - ON oo.auction_id = ap_protocol.auction_id AND oo.protocol_fee_token = ap_protocol.token + ON opf.auction_id = ap_protocol.auction_id AND opf.protocol_fee_token = ap_protocol.token ), batch_protocol_fees AS ( SELECT @@ -155,7 +155,7 @@ batch_protocol_fees AS ( -- sum(surplus * surplus_token_price) as surplus, sum(protocol_fee * protocol_fee_token_price) as protocol_fee, sum(network_fee_correction * network_fee_token_price) as network_fee_correction - FROM order_observations_prices oop + FROM order_protocol_fee_prices group by solver, tx_hash ), reward_data AS (SELECT diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index 1fdc5ce9..d1277eac 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -55,7 +55,7 @@ order_surplus AS ( WHERE s.block_number > {{start_block}} AND s.block_number <= {{end_block}} ) -,order_observations AS ( +,order_protocol_fee AS ( SELECT os.auction_id, os.order_uid, @@ -109,16 +109,16 @@ order_surplus AS ( JOIN fee_policies fp -- contains protocol fee policy ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid ) -,order_observations_prices AS ( +,order_protocol_fee_prices AS ( SELECT - oo.order_uid, - oo.auction_id, - oo.protocol_fee, - oo.protocol_fee_token, + opf.order_uid, + opf.auction_id, + opf.protocol_fee, + opf.protocol_fee_token, ap.price / pow(10, 18) as protocol_fee_native_price - FROM order_observations oo + FROM order_protocol_fee opf JOIN auction_prices ap-- contains price: protocol fee token - ON oo.auction_id = ap.auction_id AND oo.protocol_fee_token = ap.token + ON opf.auction_id = ap.auction_id AND opf.protocol_fee_token = ap.token ), winning_quotes as (SELECT concat('0x', encode(oq.solver, 'hex')) as quote_solver, oq.order_uid @@ -150,6 +150,6 @@ from trade_hashes and trade_hashes.auction_id = o.auction_id left outer join winning_quotes wq on trade_hashes.order_uid = wq.order_uid - left outer join order_observations_prices oop - on trade_hashes.order_uid = oop.order_uid - and trade_hashes.auction_id = oop.auction_id; + left outer join order_protocol_fee_prices opfp + on trade_hashes.order_uid = opfp.order_uid + and trade_hashes.auction_id = opfp.auction_id; From 0cc12f12e142c9d60a5658420ec4384124f2b09a Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 22 Jan 2024 11:45:28 +0100 Subject: [PATCH 11/12] fix block range --- src/sql/orderbook/order_rewards.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index d1277eac..584139d8 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -128,8 +128,8 @@ order_surplus AS ( WHERE ((o.kind = 'sell' AND o.buy_amount <= oq.buy_amount) OR (o.kind='buy' AND o.sell_amount >= oq.sell_amount)) AND o.partially_fillable='f' - AND block_number > 19000000 - AND block_number <= 19500000 + AND block_number > {{start_block}} + AND block_number <= {{end_block}} AND oq.solver != '\x0000000000000000000000000000000000000000') -- Most efficient column order for sorting would be having tx_hash or order_uid first select block_number, From 8dfd3082970fd8517f1d83172642f5978fb01312 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 22 Jan 2024 11:52:48 +0100 Subject: [PATCH 12/12] compute network fee instead of protocol fee --- src/models/batch_rewards_schema.py | 2 +- src/sql/orderbook/batch_rewards.sql | 4 ++-- tests/integration/test_fetch_orderbook.py | 18 +++++++++--------- tests/unit/test_batch_rewards_schema.py | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/models/batch_rewards_schema.py b/src/models/batch_rewards_schema.py index f22a6c22..7f481c15 100644 --- a/src/models/batch_rewards_schema.py +++ b/src/models/batch_rewards_schema.py @@ -31,8 +31,8 @@ def from_pdf_to_dune_records(cls, rewards_df: DataFrame) -> list[dict[str, Any]] "capped_payment": int(row["capped_payment"]), "execution_cost": int(row["execution_cost"]), "surplus": int(row["surplus"]), - "fee": int(row["fee"]), "protocol_fee": int(row["protocol_fee"]), + "fee": int(row["network_fee"]), "winning_score": int(row["winning_score"]), "reference_score": int(row["reference_score"]), "participating_solvers": row["participating_solvers"], diff --git a/src/sql/orderbook/batch_rewards.sql b/src/sql/orderbook/batch_rewards.sql index 758ae0eb..c89ac44c 100644 --- a/src/sql/orderbook/batch_rewards.sql +++ b/src/sql/orderbook/batch_rewards.sql @@ -201,8 +201,8 @@ batch_protocol_fees AS ( solver, execution_cost, surplus, - protocol_fee + fee - network_fee_correction as fee,-- total fee for ranking protocol_fee, -- the protocol fee + fee - network_fee_correction as network_fee,-- the network fee surplus + protocol_fee + fee - network_fee_correction - reference_score as uncapped_payment_eth, -- Uncapped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth) LEAST(GREATEST(-{{EPSILON}}, surplus + protocol_fee + fee - network_fee_correction - reference_score), @@ -220,8 +220,8 @@ SELECT settlement_block as block_number, concat('0x', encode(solver, 'hex')) as solver, execution_cost::text as execution_cost, surplus::text as surplus, - fee::text as fee, protocol_fee::text as protocol_fee, + network_fee::text as network_fee, uncapped_payment_eth::text as uncapped_payment_eth, capped_payment::text as capped_payment, winning_score::text as winning_score, diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index 94fdcf26..2502b388 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -72,16 +72,16 @@ def test_get_batch_rewards(self): "104011002982952097", "0", ], - "fee": [ - "7751978767036064", - "10350680045815651", - "0", - ], "protocol_fee": [ "0", "0", "0", ], + "network_fee": [ + "7751978767036064", + "10350680045815651", + "0", + ], "uncapped_payment_eth": [ "7232682540629268", "82825156151734420", @@ -159,14 +159,14 @@ def test_get_batch_rewards_with_protocol_fees(self): "1917140833491803", "45868496778113149", ], - "fee": [ - "5746294767802878", - "6875724468968459", - ], "protocol_fee": [ "0", "463318149273870", ], + "network_fee": [ + "5746294767802878", + "6412406319694589", + ], "uncapped_payment_eth": [ "7613796432942231", "52744221247081608", diff --git a/tests/unit/test_batch_rewards_schema.py b/tests/unit/test_batch_rewards_schema.py index 4c06499b..e7f7a171 100644 --- a/tests/unit/test_batch_rewards_schema.py +++ b/tests/unit/test_batch_rewards_schema.py @@ -25,11 +25,11 @@ def test_order_rewards_transformation(self): ], "execution_cost": [9999 * ONE_ETH, 1], "surplus": [2 * ONE_ETH, 3 * ONE_ETH], - "fee": [ + "protocol_fee": [2000000000000000, 0], + "network_fee": [ 1000000000000000, max_uint, ], - "protocol_fee": [2000000000000000, 0], "uncapped_payment_eth": [0, -10 * ONE_ETH], "capped_payment": [-1000000000000000, -1000000000000000], "winning_score": [123456 * ONE_ETH, 6789 * ONE_ETH],