From dce2fa17b006db8e32cdc8be36f0a8091b94db7b Mon Sep 17 00:00:00 2001 From: jeff-dude <102681548+jeff-dude@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:45:35 -0500 Subject: [PATCH] revert 0x enhancements: v1 fills and deduped (#7186) * Revert "zeroex - macro for v1 fills (#7161)" This reverts commit 91241e616f5d77c7952d4723f3bebc583ffbfdea. * Revert "zeroex - macro for deduped fills (#7156)" This reverts commit 30b95c51798cd7c8da243cfad67e0f703449ae33. --- .../_project/zeroex/zeroex_v1_api_fills.sql | 577 ------------------ .../zeroex/base/zeroex_base_api_fills.sql | 251 ++++++-- .../base/zeroex_base_api_fills_deduped.sql | 102 ++++ .../zeroex/base/zeroex_base_schema.yml | 2 +- .../base/zeroex_v1_base_deduped_trades.sql | 30 - .../ethereum/zeroex_ethereum_api_fills.sql | 570 +++++++++++++---- .../zeroex_ethereum_api_fills_deduped.sql} | 30 +- .../ethereum/zeroex_ethereum_schema.yml | 6 +- .../zeroex_v1_ethereum_deduped_trades.sql | 30 - .../zeroex/zeroex_api_fills_deduped.sql | 4 +- 10 files changed, 800 insertions(+), 802 deletions(-) delete mode 100644 dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_api_fills.sql create mode 100644 dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql delete mode 100644 dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql rename dbt_subprojects/dex/{macros/models/_project/zeroex/zeroex_v1_deduped_trades.sql => models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills_deduped.sql} (80%) delete mode 100644 dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql diff --git a/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_api_fills.sql b/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_api_fills.sql deleted file mode 100644 index 662d3bbd4cb..00000000000 --- a/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_api_fills.sql +++ /dev/null @@ -1,577 +0,0 @@ -{% macro zeroex_evt_fills_txs(blockchain,zeroex_v3_start_date) %} -{%- set table_prefix = 'zeroex_v3_' + blockchain -%} - -SELECT - v3.evt_tx_hash AS tx_hash, - CASE - WHEN takerAddress = 0x63305728359c088a52b0b0eeec235db4d31a67fc THEN takerAddress - ELSE NULL - END AS affiliate_address, - NULL AS is_gasless, - evt_block_time AS block_time -FROM {{ source(table_prefix, 'Exchange_evt_Fill') }} v3 -WHERE ( -- nuo - v3.takerAddress = 0x63305728359c088a52b0b0eeec235db4d31a67fc - OR -- contains a bridge order - ( - v3.feeRecipientAddress = 0x1000000000000000000000000000000000000011 - AND bytearray_substring(v3.makerAssetData, 1, 4) = 0xdc1600f3 - ) -) -{% if is_incremental() %} -AND {{ incremental_predicate('evt_block_time') }} -{% else %} -AND evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) -{% endif %} -{% endmacro %} - - -{% macro zeroex_v1_txs(blockchain,zeroex_v3_start_date) %} -SELECT - tr.tx_hash, - CASE - WHEN bytearray_position(INPUT, 0x869584cd) <> 0 THEN - SUBSTRING(INPUT FROM (bytearray_position(INPUT, 0x869584cd) + 16) FOR 20) - WHEN bytearray_position(INPUT, 0xfbc019a7) <> 0 THEN - SUBSTRING(INPUT FROM (bytearray_position(INPUT, 0xfbc019a7) + 16) FOR 20) - END AS affiliate_address, - CASE - WHEN (varbinary_position(input, 0x3d8d4082) <> 0 OR varbinary_position(input, 0x4f948110) <> 0) - THEN 1 - END AS is_gasless, - block_time -FROM {{ source(blockchain, 'traces') }} tr -WHERE tr.to IN ( - 0x61935cbdd02287b511119ddb11aeb42f1593b7ef, -- exchange contract - 0x6958f5e95332d93d21af0d7b9ca85b8212fee0a5, -- forwarder address - 0x4aa817c6f383c8e8ae77301d18ce48efb16fd2be, -- forwarder address - 0x4ef40d1bf0983899892946830abf99eca2dbc5ce, -- forwarder address - 0xdef1c0ded9bec7f1a1670819833240f027b25eff -- exchange proxy -) -AND ( - bytearray_position(INPUT, 0x869584cd) <> 0 - OR bytearray_position(INPUT, 0xfbc019a7) <> 0 -) -{% if is_incremental() %} -AND {{ incremental_predicate('block_time') }} -{% else %} -AND block_time >= cast('{{zeroex_v3_start_date}}' as date) -{% endif %} -{% endmacro %} - - -{% macro v3_fills_no_bridge(blockchain,zeroex_v3_start_date) %} -{%- set table_prefix = 'zeroex_v3_' + blockchain -%} - -SELECT - fills.evt_tx_hash AS tx_hash, - fills.evt_index, - fills.contract_address, - evt_block_time AS block_time, - fills.makerAddress AS maker, - fills.takerAddress AS taker, - bytearray_substring(fills.takerAssetData, 17, 20) AS taker_token, - bytearray_substring(fills.makerAssetData, 17, 20) AS maker_token, - CAST(fills.takerAssetFilledAmount AS int256) AS taker_token_amount_raw, - CAST(fills.makerAssetFilledAmount AS int256) AS maker_token_amount_raw, - 'Fill' AS type, - COALESCE(zeroex_tx.affiliate_address, fills.feeRecipientAddress) AS affiliate_address, - (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, - (fills.feeRecipientAddress = 0x86003b044f70dac0abc80ac8957305b6370893ed) AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'Exchange_evt_Fill') }} fills -INNER JOIN zeroex_tx ON zeroex_tx.tx_hash = fills.evt_tx_hash AND fills.evt_block_time = zeroex_tx.block_time -WHERE - -- Exclude bridge orders - (bytearray_substring(makerAssetData, 1, 4) <> 0xdc1600f3) - AND - ( - -- Include transactions with a matching tx_hash in zeroex_tx or specific feeRecipientAddress - zeroex_tx.tx_hash IS NOT NULL - OR fills.feeRecipientAddress = 0x86003b044f70dac0abc80ac8957305b6370893ed - ) - - {% if is_incremental() %} - AND {{ incremental_predicate('evt_block_time') }} - {% endif %} - {% if not is_incremental() %} - AND evt_block_time >= CAST('{{zeroex_v3_start_date}}' AS DATE) - {% endif %} -{% endmacro %} - - -{% macro v4_rfq_fills_no_bridge(blockchain, zeroex_v4_start_date) %} -{%- set table_prefix = 'zeroex_' + blockchain -%} -SELECT - fills.evt_tx_hash AS tx_hash, - fills.evt_index, - fills.contract_address, - fills.evt_block_time AS block_time, - fills.maker, - fills.taker, - fills.takerToken AS taker_token, - fills.makerToken AS maker_token, - CAST(fills.takerTokenFilledAmount AS int256) AS taker_token_amount_raw, - CAST(fills.makerTokenFilledAmount AS int256) AS maker_token_amount_raw, - 'RfqOrderFilled' AS type, - zeroex_tx.affiliate_address, - (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'ExchangeProxy_evt_RfqOrderFilled') }} fills -LEFT JOIN zeroex_tx - ON zeroex_tx.tx_hash = fills.evt_tx_hash - AND fills.evt_block_time = zeroex_tx.block_time - -{% if is_incremental() %} -WHERE {{ incremental_predicate('evt_block_time') }} -{% else %} -WHERE evt_block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - - -{% macro v4_limit_fills_no_bridge(blockchain, zeroex_v4_start_date) %} -{%- set table_prefix = 'zeroex_' + blockchain -%} -SELECT - fills.evt_tx_hash AS tx_hash, - fills.evt_index, - fills.contract_address, - fills.evt_block_time AS block_time, - fills.maker, - fills.taker, - fills.takerToken AS taker_token, - fills.makerToken AS maker_token, - CAST(fills.takerTokenFilledAmount AS int256) AS taker_token_amount_raw, - CAST(fills.makerTokenFilledAmount AS int256) AS maker_token_amount_raw, - 'LimitOrderFilled' AS type, - COALESCE(zeroex_tx.affiliate_address, fills.feeRecipient) AS affiliate_address, - (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, - (fills.feeRecipient IN ( - 0x9b858be6e3047d88820f439b240deac2418a2551, - 0x86003b044f70dac0abc80ac8957305b6370893ed, - 0x5bc2419a087666148bfbe1361ae6c06d240c6131 - )) AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'ExchangeProxy_evt_LimitOrderFilled') }} fills -LEFT JOIN zeroex_tx - ON zeroex_tx.tx_hash = fills.evt_tx_hash - AND fills.evt_block_time = zeroex_tx.block_time - -{% if is_incremental() %} -WHERE {{ incremental_predicate('evt_block_time') }} -{% else %} -WHERE evt_block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - -{% macro otc_fills(blockchain, zeroex_v4_start_date) %} -{%- set table_prefix = 'zeroex_' + blockchain -%} -SELECT - fills.evt_tx_hash AS tx_hash, - fills.evt_index, - fills.contract_address, - fills.evt_block_time AS block_time, - fills.maker, - 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, - fills.takerToken AS taker_token, - fills.makerToken AS maker_token, - CAST(fills.takerTokenFilledAmount AS int256) AS taker_token_amount_raw, - CAST(fills.makerTokenFilledAmount AS int256) AS maker_token_amount_raw, - 'OtcOrderFilled' AS type, - zeroex_tx.affiliate_address, - (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'ExchangeProxy_evt_OtcOrderFilled') }} fills -LEFT JOIN zeroex_tx - ON zeroex_tx.tx_hash = fills.evt_tx_hash - AND fills.evt_block_time = zeroex_tx.block_time -{% if is_incremental() %} -WHERE {{ incremental_predicate('evt_block_time') }} -{% else %} -WHERE evt_block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - - -{% macro ERC20BridgeTransfer(blockchain, zeroex_v3_start_date) %} -SELECT - logs.tx_hash, - logs.index AS evt_index, - logs.contract_address, - logs.block_time, - bytearray_substring(data, 141, 20) AS maker, - bytearray_substring(data, 173, 20) AS taker, - bytearray_substring(data, 13, 20) AS taker_token, - bytearray_substring(data, 45, 20) AS maker_token, - bytearray_to_int256(bytearray_substring(data, 77, 20)) AS taker_token_amount_raw, - bytearray_to_int256(bytearray_substring(data, 109, 20)) AS maker_token_amount_raw, - 'ERC20BridgeTransfer' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(blockchain, 'logs') }} logs -JOIN zeroex_tx - ON zeroex_tx.tx_hash = logs.tx_hash - AND logs.block_time = zeroex_tx.block_time -WHERE topic0 = 0x349fc08071558d8e3aa92dec9396e4e9f2dfecd6bb9065759d1932e7da43b8a9 - -{% if is_incremental() %} -AND {{ incremental_predicate('logs.block_time') }} -{% else %} -AND logs.block_time >= CAST('{{zeroex_v3_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - - -{% macro BridgeFill(blockchain, zeroex_v4_start_date) %} -SELECT - logs.tx_hash, - logs.index AS evt_index, - logs.contract_address, - logs.block_time, - bytearray_substring(data, 13, 20) AS maker, - 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, - bytearray_substring(data, 45, 20) AS taker_token, - bytearray_substring(data, 77, 20) AS maker_token, - bytearray_to_int256(bytearray_substring(data, 109, 20)) AS taker_token_amount_raw, - bytearray_to_int256(bytearray_substring(data, 141, 20)) AS maker_token_amount_raw, - 'BridgeFill' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(blockchain, 'logs') }} logs -JOIN zeroex_tx - ON zeroex_tx.tx_hash = logs.tx_hash - AND logs.block_time = zeroex_tx.block_time -WHERE topic0 = 0xff3bc5e46464411f331d1b093e1587d2d1aa667f5618f98a95afc4132709d3a9 - AND contract_address = 0x22f9dcf4647084d6c31b2765f6910cd85c178c18 - -{% if is_incremental() %} -AND {{ incremental_predicate('logs.block_time') }} -{% else %} -AND logs.block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - -{% macro NewBridgeFill(blockchain, zeroex_v4_start_date) %} -SELECT - logs.tx_hash, - logs.index AS evt_index, - logs.contract_address, - logs.block_time, - bytearray_substring(data, 13, 20) AS maker, - 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, - bytearray_substring(data, 45, 20) AS taker_token, - bytearray_substring(data, 77, 20) AS maker_token, - bytearray_to_int256(bytearray_substring(data, 109, 20)) AS taker_token_amount_raw, - bytearray_to_int256(bytearray_substring(data, 141, 20)) AS maker_token_amount_raw, - 'NewBridgeFill' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(blockchain, 'logs') }} logs -JOIN zeroex_tx - ON zeroex_tx.tx_hash = logs.tx_hash - AND logs.block_time = zeroex_tx.block_time -WHERE topic0 = 0xe59e71a14fe90157eedc866c4f8c767d3943d6b6b2e8cd64dddcc92ab4c55af8 - AND contract_address = 0x22f9dcf4647084d6c31b2765f6910cd85c178c18 - -{% if is_incremental() %} -AND {{ incremental_predicate('logs.block_time') }} -{% else %} -AND logs.block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - -{% macro direct_PLP(blockchain, zeroex_v3_start_date) %} -{%- set table_prefix = 'zeroex_' + blockchain -%} -SELECT - plp.evt_tx_hash, - plp.evt_index AS evt_index, - plp.contract_address, - plp.evt_block_time AS block_time, - provider AS maker, - recipient AS taker, - inputToken AS taker_token, - outputToken AS maker_token, - CAST(inputTokenAmount AS int256) AS taker_token_amount_raw, - CAST(outputTokenAmount AS int256) AS maker_token_amount_raw, - 'LiquidityProviderSwap' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'ExchangeProxy_evt_LiquidityProviderSwap') }} plp -JOIN zeroex_tx - ON zeroex_tx.tx_hash = plp.evt_tx_hash - AND plp.evt_block_time = zeroex_tx.block_time - -{% if is_incremental() %} -WHERE {{ incremental_predicate('evt_block_time') }} -{% else %} -WHERE evt_block_time >= CAST('{{zeroex_v3_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - -{% macro direct_uniswapv2(blockchain, zeroex_v3_start_date) %} -{%- set table_prefix = 'uniswap_v2_' + blockchain -%} -SELECT - swap.evt_tx_hash AS tx_hash, - swap.evt_index, - swap.contract_address, - swap.evt_block_time AS block_time, - swap.contract_address AS maker, - 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, - CASE - WHEN swap.amount0In > swap.amount0Out THEN pair.token0 - ELSE pair.token1 - END AS taker_token, - CASE - WHEN swap.amount0In > swap.amount0Out THEN pair.token1 - ELSE pair.token0 - END AS maker_token, - CASE - WHEN swap.amount0In > swap.amount0Out THEN - CASE - WHEN swap.amount0In >= swap.amount0Out THEN CAST(swap.amount0In - swap.amount0Out AS int256) - ELSE CAST(0 AS int256) - END - ELSE - CASE - WHEN swap.amount1In >= swap.amount1Out THEN CAST(swap.amount1In - swap.amount1Out AS int256) - ELSE CAST(0 AS int256) - END - END AS taker_token_amount_raw, - CASE - WHEN swap.amount0In > swap.amount0Out THEN - CASE - WHEN swap.amount1Out >= swap.amount1In THEN CAST(swap.amount1Out - swap.amount1In AS int256) - ELSE CAST(0 AS int256) - END - ELSE - CASE - WHEN swap.amount0Out >= swap.amount0In THEN CAST(swap.amount0Out - swap.amount0In AS int256) - ELSE CAST(0 AS int256) - END - END AS maker_token_amount_raw, - 'Uniswap V2 Direct' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'Pair_evt_Swap') }} swap -LEFT JOIN {{ source(table_prefix, 'Factory_evt_PairCreated') }} pair - ON pair.pair = swap.contract_address -JOIN zeroex_tx - ON zeroex_tx.tx_hash = swap.evt_tx_hash - AND swap.evt_block_time = zeroex_tx.block_time -WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff - {% if is_incremental() %} - AND {{ incremental_predicate('swap.evt_block_time') }} - {% else %} - AND swap.evt_block_time >= CAST('{{zeroex_v3_start_date}}' AS DATE) - {% endif %} -{% endmacro %} - -{% macro direct_sushiswap(blockchain, zeroex_v3_start_date) %} -{%- set table_prefix = 'sushi_' + blockchain -%} -SELECT - swap.evt_tx_hash AS tx_hash, - swap.evt_index, - swap.contract_address, - swap.evt_block_time AS block_time, - swap.contract_address AS maker, - LAST_VALUE(swap.to) OVER (PARTITION BY swap.evt_tx_hash ORDER BY swap.evt_index) AS taker, - CASE - WHEN swap.amount0In > swap.amount0Out THEN pair.token0 - ELSE pair.token1 - END AS taker_token, - CASE - WHEN swap.amount0In > swap.amount0Out THEN pair.token1 - ELSE pair.token0 - END AS maker_token, - CASE - WHEN swap.amount0In > swap.amount0Out THEN - CASE - WHEN swap.amount0In >= swap.amount0Out THEN CAST(swap.amount0In - swap.amount0Out AS int256) - ELSE CAST(0 AS int256) - END - ELSE - CASE - WHEN swap.amount1In >= swap.amount1Out THEN CAST(swap.amount1In - swap.amount1Out AS int256) - ELSE CAST(0 AS int256) - END - END AS taker_token_amount_raw, - CASE - WHEN swap.amount0In > swap.amount0Out THEN - CASE - WHEN swap.amount1Out >= swap.amount1In THEN CAST(swap.amount1Out - swap.amount1In AS int256) - ELSE CAST(0 AS int256) - END - ELSE - CASE - WHEN swap.amount0Out >= swap.amount0In THEN CAST(swap.amount0Out - swap.amount0In AS int256) - ELSE CAST(0 AS int256) - END - END AS maker_token_amount_raw, - 'Sushiswap Direct' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'Pair_evt_Swap') }} swap -LEFT JOIN {{ source(table_prefix, 'Factory_evt_PairCreated') }} pair - ON pair.pair = swap.contract_address -JOIN zeroex_tx - ON zeroex_tx.tx_hash = swap.evt_tx_hash - AND swap.evt_block_time = zeroex_tx.block_time -WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff - {% if is_incremental() %} - AND {{ incremental_predicate('swap.evt_block_time') }} - {% else %} - AND swap.evt_block_time >= CAST('{{zeroex_v3_start_date}}' AS DATE) - {% endif %} -{% endmacro %} - - -{% macro direct_uniswapv3(blockchain,zeroex_v4_start_date) %} -{%- set table_prefix = 'uniswap_v3_' + blockchain -%} - SELECT - swap.evt_tx_hash AS tx_hash, - swap.evt_index, - swap.contract_address, - swap.evt_block_time AS block_time, - swap.contract_address AS maker, - 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, - CASE - WHEN amount0 < CAST(0 AS int256) THEN pair.token1 - ELSE pair.token0 - END AS taker_token, - CASE - WHEN amount0 < CAST(0 AS int256) THEN pair.token0 - ELSE pair.token1 - END AS maker_token, - CASE - WHEN amount0 < CAST(0 AS int256) THEN ABS(swap.amount1) - ELSE ABS(swap.amount0) - END AS taker_token_amount_raw, - CASE - WHEN amount0 < CAST(0 AS int256) THEN ABS(swap.amount0) - ELSE ABS(swap.amount1) - END AS maker_token_amount_raw, - 'Uniswap V3 Direct' AS type, - zeroex_tx.affiliate_address AS affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless - FROM {{ source(table_prefix, 'Pair_evt_Swap') }} swap - LEFT JOIN {{ source(table_prefix, 'Factory_evt_PoolCreated') }} pair - ON pair.pool = swap.contract_address - JOIN zeroex_tx - ON zeroex_tx.tx_hash = swap.evt_tx_hash - AND swap.evt_block_time = zeroex_tx.block_time - WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff - {% if is_incremental() %} - AND {{ incremental_predicate('swap.evt_block_time') }} - {% endif %} - {% if not is_incremental() %} - AND swap.evt_block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) - {% endif %} -{% endmacro %} - -{% macro trade_details(blockchain, zeroex_v3_start_date) %} -WITH results AS ( - SELECT - all_tx.tx_hash, - tx.block_number, - all_tx.evt_index, - all_tx.contract_address, - all_tx.block_time, - CAST(DATE_TRUNC('day', all_tx.block_time) AS DATE) AS block_date, - CAST(DATE_TRUNC('month', all_tx.block_time) AS DATE) AS block_month, - maker, - CASE - WHEN is_gasless = 1 AND VARBINARY_POSITION(data, 0x3a46c4e1) <> 0 THEN VARBINARY_SUBSTRING(data, 81, 20) - WHEN is_gasless = 1 AND VARBINARY_POSITION(data, 0xa98fcbf1) <> 0 THEN VARBINARY_SUBSTRING(data, 81, 20) - WHEN is_gasless = 1 AND VARBINARY_POSITION(data, 0x3d8d4082) <> 0 THEN VARBINARY_SUBSTRING(data, 177, 20) - WHEN taker = 0xdef1c0ded9bec7f1a1670819833240f027b25eff THEN tx."from" - ELSE taker - END AS taker, - taker_token, - taker_token AS token_sold_address, - ts.symbol AS taker_symbol, - maker_token, - maker_token AS token_bought_address, - ms.symbol AS maker_symbol, - CASE - WHEN LOWER(ts.symbol) > LOWER(ms.symbol) THEN CONCAT(ms.symbol, '-', ts.symbol) - ELSE CONCAT(ts.symbol, '-', ms.symbol) - END AS token_pair, - taker_token_amount_raw / POW(10, tp.decimals) AS taker_token_amount, - taker_token_amount_raw / POW(10, tp.decimals) AS token_sold_amount, - CAST(taker_token_amount_raw AS UINT256) AS taker_token_amount_raw, - maker_token_amount_raw / POW(10, mp.decimals) AS maker_token_amount, - maker_token_amount_raw / POW(10, mp.decimals) AS token_bought_amount, - CAST(maker_token_amount_raw AS UINT256) AS maker_token_amount_raw, - all_tx.type, - MAX(affiliate_address) OVER (PARTITION BY all_tx.tx_hash) AS affiliate_address, - swap_flag, - matcha_limit_order_flag, - tx."from" AS tx_from, - tx.to AS tx_to, - '{{ blockchain }}' AS blockchain - FROM all_tx - INNER JOIN {{ source(blockchain, 'transactions') }} tx - ON all_tx.tx_hash = tx.hash AND all_tx.block_time = tx.block_time - {% if is_incremental() %} - AND {{ incremental_predicate('tx.block_time') }} - {% endif %} - {% if not is_incremental() %} - AND tx.block_time >= CAST('{{ zeroex_v3_start_date }}' AS DATE) - {% endif %} - LEFT JOIN {{ source('prices', 'usd') }} tp - ON DATE_TRUNC('minute', all_tx.block_time) = tp.minute - AND CASE - WHEN all_tx.taker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 - ELSE all_tx.taker_token - END = tp.contract_address - AND tp.blockchain = '{{ blockchain }}' - {% if is_incremental() %} - AND {{ incremental_predicate('tp.minute') }} - {% endif %} - {% if not is_incremental() %} - AND tp.minute >= CAST('{{ zeroex_v3_start_date }}' AS DATE) - {% endif %} - LEFT JOIN {{ source('prices', 'usd') }} mp - ON DATE_TRUNC('minute', all_tx.block_time) = mp.minute - AND CASE - WHEN all_tx.maker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 - ELSE all_tx.maker_token - END = mp.contract_address - AND mp.blockchain = '{{ blockchain }}' - {% if is_incremental() %} - AND {{ incremental_predicate('mp.minute') }} - {% endif %} - {% if not is_incremental() %} - AND mp.minute >= CAST('{{ zeroex_v3_start_date }}' AS DATE) - {% endif %} - LEFT JOIN {{ source('tokens', 'erc20') }} ts - ON ts.contract_address = taker_token AND ts.blockchain = '{{ blockchain }}' - LEFT JOIN {{ source('tokens', 'erc20') }} ms - ON ms.contract_address = maker_token AND ms.blockchain = '{{ blockchain }}' -), - -results_usd AS ( - {{ - add_amount_usd( - trades_cte = 'results' - ) - }} -) -select * from results_usd -{% endmacro %} diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills.sql b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills.sql index c5c5d327ec1..fc3f6009fc9 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills.sql @@ -19,65 +19,228 @@ {% set zeroex_v3_start_date = '2019-12-01' %} {% set zeroex_v4_start_date = '2021-01-06' %} -{% set blockchain = 'base' %} + +-- Test Query here: https://dune.com/queries/2834419 WITH zeroex_tx AS ( - SELECT tx_hash, - block_time as block_time, + SELECT tx_hash, max(affiliate_address) as affiliate_address, - max(is_gasless) as is_gasless + is_gasless FROM ( - {{ - zeroex_v1_txs( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date, - ) - }} + SELECT tr.tx_hash, + CASE + WHEN bytearray_position(INPUT, 0x869584cd ) <> 0 THEN SUBSTRING(INPUT + FROM (bytearray_position(INPUT, 0x869584cd) + 16) + FOR 20) + WHEN bytearray_position(INPUT, 0xfbc019a7) <> 0 THEN SUBSTRING(INPUT + FROM (bytearray_position(INPUT, 0xfbc019a7 ) + 16) + FOR 20) + END AS affiliate_address, + case when (varbinary_position(input,0x3d8d4082) <> 0 or varbinary_position(input,0x4f948110) <> 0 ) then 1 else 0 end as is_gasless + FROM {{ source('base', 'traces') }} tr + WHERE tr.to IN ( + -- exchange contract + 0xdef1c0ded9bec7f1a1670819833240f027b25eff, + -- forwarder addresses + 0x6958f5e95332d93d21af0d7b9ca85b8212fee0a5, + 0x4aa817c6f383c8e8ae77301d18ce48efb16fd2be, + 0x4ef40d1bf0983899892946830abf99eca2dbc5ce, + -- exchange proxy + 0xdef189deaef76e379df891899eb5a00a94cbc250 + ) + AND ( + bytearray_position(INPUT, 0x869584cd ) <> 0 + OR bytearray_position(INPUT, 0xfbc019a7 ) <> 0 + ) + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} ) temp - group by tx_hash, block_time + group by tx_hash , is_gasless + ), -ERC20BridgeTransfer as ( - {{ - ERC20BridgeTransfer( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} + +ERC20BridgeTransfer AS ( + SELECT + logs.tx_hash, + logs.block_number AS block_number, + INDEX AS evt_index, + logs.contract_address, + block_time AS block_time, + bytearray_substring(DATA, 142, 20) AS maker, + bytearray_substring(DATA, 172, 20) AS taker, + bytearray_substring(DATA, 14, 20) AS taker_token, + bytearray_substring(DATA, 45, 20) AS maker_token, + bytearray_to_uint256(bytearray_substring(DATA, 77, 20)) AS taker_token_amount_raw, + bytearray_to_uint256(bytearray_substring(DATA, 110, 20)) AS maker_token_amount_raw, + 'ERC20BridgeTransfer' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('base', 'logs') }} logs + INNER JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash + WHERE topic0 = 0x349fc08071558d8e3aa92dec9396e4e9f2dfecd6bb9065759d1932e7da43b8a9 + + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + ), -BridgeFill as ( - {{ - BridgeFill( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +BridgeFill AS ( + SELECT + logs.tx_hash, + logs.block_number AS block_number, + INDEX AS evt_index, + logs.contract_address, + block_time AS block_time, + bytearray_substring(DATA, 13, 20) AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, + bytearray_substring(DATA, 45, 20) AS taker_token, + bytearray_substring(DATA, 77, 20) AS maker_token, + bytearray_to_uint256(bytearray_substring(DATA, 109, 20)) AS taker_token_amount_raw, + bytearray_to_uint256(bytearray_substring(DATA, 141, 20)) AS maker_token_amount_raw, + 'BridgeFill' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('base', 'logs') }} logs + INNER JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash + WHERE topic0 = 0xff3bc5e46464411f331d1b093e1587d2d1aa667f5618f98a95afc4132709d3a9 + AND contract_address = 0xdb6f1920a889355780af7570773609bd8cb1f498 + + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), -NewBridgeFill as ( - {{ - NewBridgeFill( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +NewBridgeFill AS ( + SELECT + logs.tx_hash as tx_hash, + logs.block_number AS block_number, + INDEX AS evt_index, + logs.contract_address, + block_time AS block_time, + bytearray_substring(DATA, 13, 20) AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, + bytearray_substring(DATA, 45, 20) AS taker_token, + bytearray_substring(DATA, 77, 20) AS maker_token, + bytearray_to_uint256(bytearray_substring(DATA, 109, 20)) AS taker_token_amount_raw, + bytearray_to_uint256(bytearray_substring(DATA, 141, 20)) AS maker_token_amount_raw, + 'BridgeFill' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('base' ,'logs') }} logs + INNER JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash + WHERE topic0 = 0xe59e71a14fe90157eedc866c4f8c767d3943d6b6b2e8cd64dddcc92ab4c55af8 + AND contract_address = 0xdb6f1920a889355780af7570773609bd8cb1f498 + + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), + all_tx AS ( + SELECT * FROM ERC20BridgeTransfer - UNION ALL - SELECT * + UNION ALL SELECT * FROM BridgeFill UNION ALL SELECT * FROM NewBridgeFill -), -tbl_trade_details AS ( - {{ - trade_details( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} + + ) -select * from tbl_trade_details -order by block_time desc \ No newline at end of file + +SELECT + all_tx.tx_hash, + all_tx.block_number, + all_tx.evt_index, + all_tx.contract_address, + all_tx.block_time, + cast(date_trunc('day', all_tx.block_time) AS date) AS block_date, + cast(date_trunc('month', all_tx.block_time) AS date) AS block_month, + maker, + CASE + WHEN is_gasless = 1 then case when (varbinary_substring(data,177,19) ) = 0x00000000000000000000000000000000000000 then varbinary_substring(data,81,20) else (varbinary_substring(data,177,20) ) end + WHEN taker = 0xdef1c0ded9bec7f1a1670819833240f027b25eff THEN tx."from" + ELSE taker + END AS taker, -- fix the user masked by ProxyContract issue + taker_token, + ts.symbol AS taker_symbol, + maker_token, + ms.symbol AS maker_symbol, + CASE WHEN lower(ts.symbol) > lower(ms.symbol) THEN concat(ms.symbol, '-', ts.symbol) ELSE concat(ts.symbol, '-', ms.symbol) END AS token_pair, + taker_token_amount_raw / pow(10, tp.decimals) AS taker_token_amount, + taker_token_amount_raw, + maker_token_amount_raw / pow(10, mp.decimals) AS maker_token_amount, + maker_token_amount_raw, + all_tx.type, + max(affiliate_address) over (partition by all_tx.tx_hash) as affiliate_address, + swap_flag, + matcha_limit_order_flag, + CASE WHEN maker_token IN (0xd9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca,0x2ae3f1ec7f1f5012cfeab0185bfc7aa3cf0dec22,0xeb466342c4d449bc9f53a865d5cb90586f405215 ) + THEN (all_tx.maker_token_amount_raw / pow(10, mp.decimals)) * mp.price + WHEN taker_token IN (0xd9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca,0x2ae3f1ec7f1f5012cfeab0185bfc7aa3cf0dec22,0xeb466342c4d449bc9f53a865d5cb90586f405215,0xd9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca ) + THEN (all_tx.taker_token_amount_raw / pow(10, tp.decimals)) * tp.price + ELSE COALESCE((all_tx.maker_token_amount_raw / pow(10, mp.decimals)) * mp.price, (all_tx.taker_token_amount_raw / pow(10, tp.decimals)) * tp.price) + END AS volume_usd, + tx."from" AS tx_from, + tx.to AS tx_to, + 'base' AS blockchain +FROM all_tx +INNER JOIN {{ source('base', 'transactions')}} tx ON all_tx.tx_hash = tx.hash + +{% if is_incremental() %} +AND {{ incremental_predicate('tx.block_time') }} +{% endif %} +{% if not is_incremental() %} +AND tx.block_time >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT JOIN {{ source('prices', 'usd') }} tp ON date_trunc('minute', all_tx.block_time) = tp.minute +AND CASE + WHEN all_tx.taker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 + ELSE all_tx.taker_token + END = tp.contract_address +AND tp.blockchain = 'base' + +{% if is_incremental() %} +AND {{ incremental_predicate('tp.minute') }} +{% endif %} +{% if not is_incremental() %} +AND tp.minute >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT JOIN {{ source('prices', 'usd') }} mp ON DATE_TRUNC('minute', all_tx.block_time) = mp.minute +AND CASE + WHEN all_tx.maker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 + ELSE all_tx.maker_token + END = mp.contract_address +AND mp.blockchain = 'base' + +{% if is_incremental() %} +AND {{ incremental_predicate('mp.minute') }} +{% endif %} +{% if not is_incremental() %} +AND mp.minute >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT OUTER JOIN {{ source('tokens', 'erc20') }} ts ON ts.contract_address = taker_token and ts.blockchain = 'base' +LEFT OUTER JOIN {{ source('tokens', 'erc20') }} ms ON ms.contract_address = maker_token and ms.blockchain = 'base' \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql new file mode 100644 index 00000000000..e7f7dd0ddbd --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql @@ -0,0 +1,102 @@ +{{ config( + + schema = 'zeroex_base', + alias = 'api_fills_deduped', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['block_date', 'tx_hash', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{% set zeroex_v3_start_date = '2019-12-01' %} + +WITH fills_with_tx_fill_number +AS +( + SELECT row_number() OVER ( partition BY tx_hash ORDER BY evt_index ASC ) AS tx_fill_number + , * + FROM {{ ref('zeroex_base_api_fills') }} + WHERE 1=1 + AND swap_flag = true + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} +) +, fills_first_last +AS +( + SELECT CASE + WHEN a.taker_token = c.maker_token AND a.taker_token_amount_raw = c.maker_token_amount_raw THEN 0 + ELSE 1 END AS taker_consider_flag--from + , CASE + WHEN a.maker_token = b.taker_token AND a.maker_token_amount_raw = b.taker_token_amount_raw THEN 0 + ELSE 1 END AS maker_consider_flag + , SUM(CASE + WHEN a.maker_token = b.taker_token AND a.maker_token_amount_raw = b.taker_token_amount_raw THEN 0 + ELSE 1 END) OVER (PARTITION BY a.tx_hash ORDER BY a.evt_index DESC) hop_count + , a.* + FROM fills_with_tx_fill_number a + LEFT JOIN fills_with_tx_fill_number b ON (a.tx_hash = b.tx_hash AND a.tx_fill_number = b.tx_fill_number - 1) + LEFT JOIN fills_with_tx_fill_number c ON (a.tx_hash = c.tx_hash AND a.tx_fill_number = c.tx_fill_number + 1) +) +, deduped_bridge_fills +AS +( + SELECT tx_hash + , MAX(evt_index) AS evt_index + , MAX(affiliate_address) AS affiliate_address + , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_token END ) AS taker_token + , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token END ) AS maker_token + , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_symbol END ) AS taker_symbol + , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_symbol END ) AS maker_symbol + , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_token_amount END) AS taker_token_amount + , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token_amount END) AS maker_token_amount + , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_token_amount_raw END) AS taker_token_amount_raw + , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token_amount_raw END) AS maker_token_amount_raw + , COUNT(*) AS fills_within + FROM fills_first_last a + GROUP BY tx_hash,hop_count +) +SELECT a.blockchain + , '0x API' as project + , cast('1' as varchar(10)) as version + + , a.block_date + , a.block_month + , a.block_time + , b.taker_symbol AS taker_symbol + , b.maker_symbol AS maker_symbol + , CASE + WHEN LOWER(b.taker_symbol) > LOWER(b.maker_symbol) + THEN CONCAT(COALESCE(b.maker_symbol, ''), '-', COALESCE(b.taker_symbol, '')) + ELSE CONCAT(COALESCE(b.taker_symbol, ''), '-', COALESCE(b.maker_symbol, '')) + END AS token_pair + , b.taker_token_amount + , b.maker_token_amount + , b.taker_token_amount_raw AS taker_token_amount_raw + , b.maker_token_amount_raw AS maker_token_amount_raw + , a.volume_usd + , b.taker_token + , b.maker_token + , a.taker + , a.maker + , a.affiliate_address + , a.tx_hash + , a.tx_from + , a.tx_to + , b.evt_index + , ARRAY[-1] AS trace_address + , a.type + , a.swap_flag + , b.fills_within + , a.contract_address +FROM fills_with_tx_fill_number a +INNER JOIN deduped_bridge_fills b + ON a.tx_hash = b.tx_hash AND a.evt_index = b.evt_index \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml index a0212b1a160..a6c1b0f74b8 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml +++ b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml @@ -82,7 +82,7 @@ models: name: blockchain description: "Blockchain which the aggregator project is deployed" - - name: zeroex_v1_base_deduped_trades + - name: zeroex_base_api_fills_deduped meta: blockchain: base project: zeroex diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql deleted file mode 100644 index 5713bc0b849..00000000000 --- a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql +++ /dev/null @@ -1,30 +0,0 @@ -{{ config( - schema = 'zeroex_v1_base', - alias = 'deduped_trades', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['block_date', 'tx_hash', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] - ) -}} - -{% set zeroex_v3_start_date = '2019-12-01' %} -{% set blockchain = 'base' %} - -WITH -deduped_trades as ( - {{ - zeroex_v1_deduped_trades( - blockchain = blockchain, - start_date = zeroex_v3_start_date - - ) - }} - -) -select - * -from deduped_trades diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills.sql b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills.sql index 459792679f8..814f58dca9c 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills.sql @@ -13,121 +13,405 @@ {% set zeroex_v3_start_date = '2019-12-01' %} {% set zeroex_v4_start_date = '2021-01-06' %} -{% set blockchain = 'ethereum' %} +-- Test Query here: https://dune.com/queries/1330551 WITH zeroex_tx AS ( - SELECT tx_hash, - block_time as block_time, + SELECT tx_hash, + block_time as block_time, max(affiliate_address) as affiliate_address, - max(is_gasless) as is_gasless - + max(is_gasless) as is_gasless FROM ( - {{ - zeroex_evt_fills_txs( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date, - ) - }} - - union - - {{ - zeroex_v1_txs( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date, - ) - }} + SELECT v3.evt_tx_hash AS tx_hash, + CASE + WHEN takerAddress = 0x63305728359c088a52b0b0eeec235db4d31a67fc THEN takerAddress + ELSE NULL + END AS affiliate_address, + null as is_gasless, + evt_block_time as block_time + + FROM {{ source('zeroex_v3_ethereum', 'Exchange_evt_Fill') }} v3 + WHERE ( -- nuo + v3.takerAddress = 0x63305728359c088a52b0b0eeec235db4d31a67fc + OR -- contains a bridge order + ( + v3.feeRecipientAddress = 0x1000000000000000000000000000000000000011 + AND bytearray_substring(v3.makerAssetData, 1, 4) = 0xdc1600f3 + ) + ) + + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + AND evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + + UNION ALL + SELECT tr.tx_hash, + CASE + WHEN bytearray_position(INPUT, 0x869584cd ) <> 0 THEN SUBSTRING(INPUT + FROM (bytearray_position(INPUT, 0x869584cd) + 16) + FOR 20) + WHEN bytearray_position(INPUT, 0xfbc019a7) <> 0 THEN SUBSTRING(INPUT + FROM (bytearray_position(INPUT, 0xfbc019a7 ) + 16) + FOR 20) + END AS affiliate_address, + case when (varbinary_position(input,0x3d8d4082) <> 0 or varbinary_position(input,0x4f948110) <> 0 ) then 1 end as is_gasless, + block_time + FROM {{ source('ethereum', 'traces') }} tr + WHERE tr.to IN ( + -- exchange contract + 0x61935cbdd02287b511119ddb11aeb42f1593b7ef, + -- forwarder addresses + 0x6958f5e95332d93d21af0d7b9ca85b8212fee0a5, + 0x4aa817c6f383c8e8ae77301d18ce48efb16fd2be, + 0x4ef40d1bf0983899892946830abf99eca2dbc5ce, + -- exchange proxy + 0xdef1c0ded9bec7f1a1670819833240f027b25eff + ) + AND ( + bytearray_position(INPUT, 0x869584cd ) <> 0 + OR bytearray_position(INPUT, 0xfbc019a7 ) <> 0 + ) + + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} ) temp group by tx_hash, block_time + ), -v3_fills_no_bridge as ( - {{ - v3_fills_no_bridge( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} +v3_fills_no_bridge AS ( + SELECT + fills.evt_tx_hash AS tx_hash, + fills.evt_index, + fills.contract_address, + evt_block_time AS block_time, + fills.makerAddress AS maker, + fills.takerAddress AS taker, + bytearray_substring(fills.takerAssetData, 17, 20) AS taker_token, + bytearray_substring(fills.makerAssetData, 17, 20) AS maker_token, + cast(fills.takerAssetFilledAmount as int256) AS taker_token_amount_raw, + cast(fills.makerAssetFilledAmount as int256)AS maker_token_amount_raw, + 'Fill' AS type, + COALESCE(zeroex_tx.affiliate_address, fills.feeRecipientAddress) AS affiliate_address, + (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, + (fills.feeRecipientAddress = 0x86003b044f70dac0abc80ac8957305b6370893ed) AS matcha_limit_order_flag, + is_gasless + FROM {{ source('zeroex_v3_ethereum', 'Exchange_evt_Fill') }} fills + INNER JOIN zeroex_tx ON zeroex_tx.tx_hash = fills.evt_tx_hash and fills.evt_block_time = zeroex_tx.block_time + WHERE (bytearray_substring(makerAssetData, 1, 4) <> 0xdc1600f3) + AND (zeroex_tx.tx_hash IS NOT NULL + OR fills.feeRecipientAddress = 0x86003b044f70dac0abc80ac8957305b6370893ed) + + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + AND evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + ), -v4_rfq_fills_no_bridge as ( - {{ - v4_rfq_fills_no_bridge( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +v4_rfq_fills_no_bridge AS ( + SELECT + fills.evt_tx_hash AS tx_hash, + fills.evt_index, + fills.contract_address, + fills.evt_block_time AS block_time, + fills.maker AS maker, + fills.taker AS taker, + fills.takerToken AS taker_token, + fills.makerToken AS maker_token, + cast(fills.takerTokenFilledAmount as int256) AS taker_token_amount_raw, + cast(fills.makerTokenFilledAmount as int256) AS maker_token_amount_raw, + 'RfqOrderFilled' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('zeroex_ethereum', 'ExchangeProxy_evt_RfqOrderFilled') }} fills + LEFT JOIN zeroex_tx ON zeroex_tx.tx_hash = fills.evt_tx_hash and fills.evt_block_time = zeroex_tx.block_time + + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + WHERE evt_block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), -v4_limit_fills_no_bridge as ( - {{ - v4_limit_fills_no_bridge( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +v4_limit_fills_no_bridge AS ( + SELECT + fills.evt_tx_hash AS tx_hash, + fills.evt_index, + fills.contract_address, + fills.evt_block_time AS block_time, + fills.maker AS maker, + fills.taker AS taker, + fills.takerToken AS taker_token, + fills.makerToken AS maker_token, + cast(fills.takerTokenFilledAmount as int256) AS taker_token_amount_raw, + cast(fills.makerTokenFilledAmount as int256) AS maker_token_amount_raw, + 'LimitOrderFilled' AS type, + COALESCE(zeroex_tx.affiliate_address, fills.feeRecipient) AS affiliate_address, + (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, + (fills.feeRecipient in + (0x9b858be6e3047d88820f439b240deac2418a2551,0x86003b044f70dac0abc80ac8957305b6370893ed,0x5bc2419a087666148bfbe1361ae6c06d240c6131)) + AS matcha_limit_order_flag, + is_gasless + FROM {{ source('zeroex_ethereum', 'ExchangeProxy_evt_LimitOrderFilled') }} fills + LEFT JOIN zeroex_tx ON zeroex_tx.tx_hash = fills.evt_tx_hash and fills.evt_block_time = zeroex_tx.block_time + + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + WHERE evt_block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), -otc_fills as ( - {{ - otc_fills( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +otc_fills AS ( + SELECT + fills.evt_tx_hash AS tx_hash, + fills.evt_index, + fills.contract_address, + fills.evt_block_time AS block_time, + fills.maker AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, + fills.takerToken AS taker_token, + fills.makerToken AS maker_token, + cast(fills.takerTokenFilledAmount as int256) AS taker_token_amount_raw, + cast(fills.makerTokenFilledAmount as int256) AS maker_token_amount_raw, + 'OtcOrderFilled' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('zeroex_ethereum', 'ExchangeProxy_evt_OtcOrderFilled') }} fills + LEFT JOIN zeroex_tx ON zeroex_tx.tx_hash = fills.evt_tx_hash and fills.evt_block_time = zeroex_tx.block_time + + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + WHERE evt_block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} + ), -ERC20BridgeTransfer as ( - {{ - ERC20BridgeTransfer( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} +ERC20BridgeTransfer AS ( + SELECT + logs.tx_hash, + INDEX AS evt_index, + logs.contract_address, + logs.block_time AS block_time, + bytearray_substring(DATA, 141, 20) AS maker, + bytearray_substring(DATA, 173, 20) AS taker, + bytearray_substring(DATA, 13, 20) AS taker_token, + bytearray_substring(DATA, 45, 20) AS maker_token, + bytearray_to_int256(bytearray_substring(DATA, 77, 20)) AS taker_token_amount_raw, + bytearray_to_int256(bytearray_substring(DATA, 109, 20)) AS maker_token_amount_raw, + 'ERC20BridgeTransfer' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('ethereum', 'logs') }} logs + JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash and logs.block_time = zeroex_tx.block_time + WHERE topic0 = 0x349fc08071558d8e3aa92dec9396e4e9f2dfecd6bb9065759d1932e7da43b8a9 + + {% if is_incremental() %} + AND {{ incremental_predicate('logs.block_time') }} + {% endif %} + {% if not is_incremental() %} + AND logs.block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + ), -BridgeFill as ( - {{ - BridgeFill( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +BridgeFill AS ( + SELECT + logs.tx_hash, + INDEX AS evt_index, + logs.contract_address, + logs.block_time AS block_time, + bytearray_substring(DATA, 13, 20) AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, + bytearray_substring(DATA, 45, 20) AS taker_token, + bytearray_substring(DATA, 77, 20) AS maker_token, + bytearray_to_int256(bytearray_substring(DATA, 109, 20)) AS taker_token_amount_raw, + bytearray_to_int256(bytearray_substring(DATA, 141, 20)) AS maker_token_amount_raw, + 'BridgeFill' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('ethereum', 'logs') }} logs + JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash and logs.block_time = zeroex_tx.block_time + WHERE topic0 = 0xff3bc5e46464411f331d1b093e1587d2d1aa667f5618f98a95afc4132709d3a9 + AND contract_address = 0x22f9dcf4647084d6c31b2765f6910cd85c178c18 + + {% if is_incremental() %} + AND {{ incremental_predicate('logs.block_time') }} + {% endif %} + {% if not is_incremental() %} + AND logs.block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), -NewBridgeFill as ( - {{ - NewBridgeFill( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +NewBridgeFill AS ( + SELECT + logs.tx_hash, + INDEX AS evt_index, + logs.contract_address, + logs.block_time AS block_time, + bytearray_substring(DATA, 13, 20) AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, + bytearray_substring(DATA, 45, 20) AS taker_token, + bytearray_substring(DATA, 77, 20) AS maker_token, + bytearray_to_int256(bytearray_substring(DATA, 109, 20)) AS taker_token_amount_raw, + bytearray_to_int256(bytearray_substring(DATA, 141, 20)) AS maker_token_amount_raw, + 'NewBridgeFill' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('ethereum' ,'logs') }} logs + JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash and logs.block_time = zeroex_tx.block_time + WHERE topic0 = 0xe59e71a14fe90157eedc866c4f8c767d3943d6b6b2e8cd64dddcc92ab4c55af8 + AND contract_address = 0x22f9dcf4647084d6c31b2765f6910cd85c178c18 + + {% if is_incremental() %} + AND {{ incremental_predicate('logs.block_time') }} + {% endif %} + {% if not is_incremental() %} + AND logs.block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), -direct_PLP as ( - {{ - direct_PLP( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} +direct_PLP AS ( + SELECT + plp.evt_tx_hash, + plp.evt_index AS evt_index, + plp.contract_address, + plp.evt_block_time AS block_time, + provider AS maker, + recipient AS taker, + inputToken AS taker_token, + outputToken AS maker_token, + cast(inputTokenAmount as int256) AS taker_token_amount_raw, + cast(outputTokenAmount as int256)AS maker_token_amount_raw, + 'LiquidityProviderSwap' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('zeroex_ethereum', 'ExchangeProxy_evt_LiquidityProviderSwap') }} plp + JOIN zeroex_tx ON zeroex_tx.tx_hash = plp.evt_tx_hash and plp.evt_block_time = zeroex_tx.block_time + + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + WHERE evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + ), -direct_uniswapv2 as ( - {{ - direct_uniswapv2( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} +direct_uniswapv2 AS ( + SELECT + swap.evt_tx_hash AS tx_hash, + swap.evt_index, + swap.contract_address, + swap.evt_block_time AS block_time, + swap.contract_address AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff as taker, + CASE WHEN swap.amount0In > swap.amount0Out THEN pair.token0 ELSE pair.token1 END AS taker_token, + CASE WHEN swap.amount0In > swap.amount0Out THEN pair.token1 ELSE pair.token0 END AS maker_token, + CASE WHEN swap.amount0In > swap.amount0Out THEN + CASE WHEN swap.amount0In >= swap.amount0Out THEN cast(swap.amount0In - swap.amount0Out as int256) ELSE cast(0 as int256) END ELSE + CASE WHEN swap.amount1In >= swap.amount1Out THEN cast(swap.amount1In - swap.amount1Out as int256) ELSE cast(0 as int256) END END AS taker_token_amount_raw, + CASE WHEN swap.amount0In > swap.amount0Out THEN + CASE WHEN swap.amount1Out >= swap.amount1In THEN cast(swap.amount1Out - swap.amount1In as int256) ELSE cast(0 as int256) END ELSE + CASE WHEN swap.amount0Out >= swap.amount0In THEN cast(swap.amount0Out - swap.amount0In as int256) ELSE cast(0 as int256) END END AS maker_token_amount_raw, + 'Uniswap V2 Direct' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('uniswap_v2_ethereum', 'Pair_evt_Swap') }} swap + LEFT JOIN {{ source('uniswap_v2_ethereum', 'Factory_evt_PairCreated') }} pair ON pair.pair = swap.contract_address + JOIN zeroex_tx ON zeroex_tx.tx_hash = swap.evt_tx_hash and swap.evt_block_time = zeroex_tx.block_time + WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff + + {% if is_incremental() %} + AND {{ incremental_predicate('swap.evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + AND swap.evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + ), -direct_sushiswap as ( - {{ - direct_sushiswap( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} +direct_sushiswap AS ( + SELECT + swap.evt_tx_hash AS tx_hash, + swap.evt_index, + swap.contract_address, + swap.evt_block_time AS block_time, + swap.contract_address AS maker, + LAST_VALUE(swap.to) OVER (PARTITION BY swap.evt_tx_hash ORDER BY swap.evt_index) AS taker, + CASE WHEN swap.amount0In > swap.amount0Out THEN pair.token0 ELSE pair.token1 END AS taker_token, + CASE WHEN swap.amount0In > swap.amount0Out THEN pair.token1 ELSE pair.token0 END AS maker_token, + CASE WHEN swap.amount0In > swap.amount0Out THEN + CASE WHEN swap.amount0In >= swap.amount0Out THEN cast(swap.amount0In - swap.amount0Out as int256) ELSE cast(0 as int256) END ELSE + CASE WHEN swap.amount1In >= swap.amount1Out THEN cast(swap.amount1In - swap.amount1Out as int256) ELSE cast(0 as int256) END END AS taker_token_amount_raw, + CASE WHEN swap.amount0In > swap.amount0Out THEN + CASE WHEN swap.amount1Out >= swap.amount1In THEN cast(swap.amount1Out - swap.amount1In as int256) ELSE cast(0 as int256) END ELSE + CASE WHEN swap.amount0Out >= swap.amount0In THEN cast(swap.amount0Out - swap.amount0In as int256) ELSE cast(0 as int256) END END AS maker_token_amount_raw, + + 'Sushiswap Direct' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('sushi_ethereum', 'Pair_evt_Swap') }} swap + LEFT JOIN {{ source('sushi_ethereum', 'Factory_evt_PairCreated') }} pair ON pair.pair = swap.contract_address + JOIN zeroex_tx ON zeroex_tx.tx_hash = swap.evt_tx_hash and swap.evt_block_time = zeroex_tx.block_time + WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff + + {% if is_incremental() %} + AND {{ incremental_predicate('swap.evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + AND swap.evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} ), -direct_uniswapv3 as ( - {{ - direct_uniswapv3( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +direct_uniswapv3 AS ( + SELECT + swap.evt_tx_hash AS tx_hash, + swap.evt_index, + swap.contract_address, + swap.evt_block_time AS block_time, + swap.contract_address AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff as taker, + CASE WHEN amount0 < cast(0 as int256) THEN pair.token1 ELSE pair.token0 END AS taker_token, + CASE WHEN amount0 < cast(0 as int256) THEN pair.token0 ELSE pair.token1 END AS maker_token, + CASE WHEN amount0 < cast(0 as int256) THEN ABS(swap.amount1) ELSE ABS(swap.amount0) END AS taker_token_amount_raw, + CASE WHEN amount0 < cast(0 as int256) THEN ABS(swap.amount0) ELSE ABS(swap.amount1) END AS maker_token_amount_raw, + 'Uniswap V3 Direct' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('uniswap_v3_ethereum', 'Pair_evt_Swap') }} swap + LEFT JOIN {{ source('uniswap_v3_ethereum', 'Factory_evt_PoolCreated') }} pair ON pair.pool = swap.contract_address + JOIN zeroex_tx ON zeroex_tx.tx_hash = swap.evt_tx_hash and swap.evt_block_time = zeroex_tx.block_time + WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff + + {% if is_incremental() %} + AND {{ incremental_predicate('swap.evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + AND swap.evt_block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} + ), all_tx AS ( SELECT * @@ -152,14 +436,86 @@ all_tx AS ( FROM v4_limit_fills_no_bridge UNION ALL SELECT * FROM otc_fills -), -tbl_trade_details AS ( - {{ - trade_details( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} ) -select * from tbl_trade_details -order by block_time desc \ No newline at end of file + +SELECT + all_tx.tx_hash, + tx.block_number, + all_tx.evt_index, + all_tx.contract_address, + all_tx.block_time, + cast(date_trunc('day', all_tx.block_time) AS date) AS block_date, + cast(date_trunc('month', all_tx.block_time) AS date) AS block_month, + maker, + CASE + when is_gasless = 1 and varbinary_position (data, 0x3a46c4e1) <> 0 then varbinary_substring(data,81,20) + when is_gasless = 1 and varbinary_position (data, 0xa98fcbf1) <> 0 then varbinary_substring(data,81,20) + when is_gasless = 1 and varbinary_position (data, 0x3d8d4082) <> 0 then varbinary_substring(data,177,20) + + WHEN taker = 0xdef1c0ded9bec7f1a1670819833240f027b25eff THEN tx."from" + ELSE taker + END AS taker, -- fix the user masked by ProxyContract issue + taker_token, + ts.symbol AS taker_symbol, + maker_token, + ms.symbol AS maker_symbol, + CASE WHEN lower(ts.symbol) > lower(ms.symbol) THEN concat(ms.symbol, '-', ts.symbol) ELSE concat(ts.symbol, '-', ms.symbol) END AS token_pair, + taker_token_amount_raw / pow(10, tp.decimals) AS taker_token_amount, + cast(taker_token_amount_raw as uint256) as taker_token_amount_raw, + maker_token_amount_raw / pow(10, mp.decimals) AS maker_token_amount, + cast(maker_token_amount_raw as uint256) as maker_token_amount_raw, + all_tx.type, + max(affiliate_address) over (partition by all_tx.tx_hash) as affiliate_address, + swap_flag, + matcha_limit_order_flag, + CASE WHEN maker_token IN (0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,0xdac17f958d2ee523a2206206994597c13d831ec7, + 0x4fabb145d64652a948d72533023f6e7a623c7c53,0x6b175474e89094c44da98b954eedeac495271d0f,0xae7ab96520de3a18e5e111b5eaab095312d7fe84) AND mp.price IS NOT NULL + THEN (all_tx.maker_token_amount_raw / pow(10, mp.decimals)) * mp.price + WHEN taker_token IN (0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,0xdac17f958d2ee523a2206206994597c13d831ec7, + 0x4fabb145d64652a948d72533023f6e7a623c7c53,0x6b175474e89094c44da98b954eedeac495271d0f,0xae7ab96520de3a18e5e111b5eaab095312d7fe84) AND tp.price IS NOT NULL + THEN (all_tx.taker_token_amount_raw / pow(10, tp.decimals)) * tp.price + ELSE COALESCE((all_tx.maker_token_amount_raw / pow(10, mp.decimals)) * mp.price, (all_tx.taker_token_amount_raw / pow(10, tp.decimals)) * tp.price) + END AS volume_usd, + tx."from" AS tx_from, + tx.to AS tx_to, + 'ethereum' AS blockchain +FROM all_tx +INNER JOIN {{ source('ethereum', 'transactions')}} tx ON all_tx.tx_hash = tx.hash and all_tx.block_time = tx.block_time + +{% if is_incremental() %} +AND {{ incremental_predicate('tx.block_time') }} +{% endif %} +{% if not is_incremental() %} +AND tx.block_time >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT JOIN {{ source('prices', 'usd') }} tp ON date_trunc('minute', all_tx.block_time) = tp.minute +AND CASE + WHEN all_tx.taker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 + ELSE all_tx.taker_token + END = tp.contract_address +AND tp.blockchain = 'ethereum' + +{% if is_incremental() %} +AND {{ incremental_predicate('tp.minute') }} +{% endif %} +{% if not is_incremental() %} +AND tp.minute >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT JOIN {{ source('prices', 'usd') }} mp ON DATE_TRUNC('minute', all_tx.block_time) = mp.minute +AND CASE + WHEN all_tx.maker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 + ELSE all_tx.maker_token + END = mp.contract_address +AND mp.blockchain = 'ethereum' + +{% if is_incremental() %} +AND {{ incremental_predicate('mp.minute') }} +{% endif %} +{% if not is_incremental() %} +AND mp.minute >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT OUTER JOIN {{ source('tokens', 'erc20') }} ts ON ts.contract_address = taker_token and ts.blockchain = 'ethereum' +LEFT OUTER JOIN {{ source('tokens', 'erc20') }} ms ON ms.contract_address = maker_token and ms.blockchain = 'ethereum' diff --git a/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_deduped_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills_deduped.sql similarity index 80% rename from dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_deduped_trades.sql rename to dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills_deduped.sql index 76f95860aaa..9a23ab938c9 100644 --- a/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_deduped_trades.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills_deduped.sql @@ -1,20 +1,36 @@ -{% macro zeroex_v1_deduped_trades(blockchain, start_date) %} +{{ config( -{% set table_name = 'zeroex_' ~ blockchain ~ '_api_fills' %} + schema = 'zeroex_ethereum', + alias = 'api_fills_deduped', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['block_date', 'tx_hash', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{% set zeroex_v3_start_date = '2019-12-01' %} +--the code used to create the data for the insertion into the dex.trades table +--this code is also the deduped version for the fills tables. +--only the data for 0x API fills. +--dependent on:zeroex_ethereum.api_fills. WITH fills_with_tx_fill_number AS ( SELECT row_number() OVER ( partition BY tx_hash ORDER BY evt_index ASC ) AS tx_fill_number , * - FROM {{ ref(table_name) }} + FROM {{ ref('zeroex_ethereum_api_fills') }} WHERE 1=1 AND swap_flag = true {% if is_incremental() %} AND {{ incremental_predicate('block_time') }} {% endif %} {% if not is_incremental() %} - AND block_time >= DATE '{{start_date}}' + AND block_time >= cast('{{zeroex_v3_start_date}}' as date) {% endif %} ) , fills_first_last @@ -50,7 +66,7 @@ AS , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token_amount_raw END) AS maker_token_amount_raw , COUNT(*) AS fills_within FROM fills_first_last a - GROUP BY tx_hash + GROUP BY tx_hash,hop_count ) SELECT a.blockchain , '0x API' as project @@ -82,6 +98,4 @@ SELECT a.blockchain , a.contract_address FROM fills_with_tx_fill_number a INNER JOIN deduped_bridge_fills b - ON a.tx_hash = b.tx_hash AND a.evt_index = b.evt_index -{% endmacro %} - + ON a.tx_hash = b.tx_hash AND a.evt_index = b.evt_index \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml index 134e8bc863f..1337fcc8435 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml +++ b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml @@ -5,7 +5,7 @@ models: meta: blockchain: ethereum project: zeroex - contributors: danning.sui, bakabhai993, hosuke, rantum + contributors: danning.sui, bakabhai993, hosuke config: tags: ['ethereum','0x','dex_aggregator','dex','aggregator'] description: > @@ -106,11 +106,11 @@ models: name: blockchain - - name: zeroex_v1_ethereum_deduped_trades + - name: zeroex_ethereum_api_fills_deduped meta: blockchain: ethereum project: zeroex - contributors: bakabhai993, rantum + contributors: bakabhai993 config: tags: ['ethereum','0x','dex_aggregator','dex','aggregator'] description: > diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql deleted file mode 100644 index 5b0debbdab0..00000000000 --- a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql +++ /dev/null @@ -1,30 +0,0 @@ -{{ config( - schema = 'zeroex_v1_ethereum', - alias = 'deduped_trades', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['block_date', 'tx_hash', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] - ) -}} - -{% set zeroex_v3_start_date = '2019-12-01' %} -{% set blockchain = 'ethereum' %} - -WITH -deduped_trades as ( - {{ - zeroex_v1_deduped_trades( - blockchain = blockchain, - start_date = zeroex_v3_start_date - - ) - }} - -) -select - * -from deduped_trades diff --git a/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql b/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql index 2b00a9a3f86..07f52a7d2f9 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql @@ -12,9 +12,9 @@ {% set zeroex_models = [ ref('zeroex_arbitrum_api_fills_deduped') ,ref('zeroex_avalanche_c_api_fills_deduped') - ,ref('zeroex_v1_base_deduped_trades') + ,ref('zeroex_base_api_fills_deduped') ,ref('zeroex_celo_api_fills_deduped') - ,ref('zeroex_v1_ethereum_deduped_trades') + ,ref('zeroex_ethereum_api_fills_deduped') ,ref('zeroex_fantom_api_fills_deduped') ,ref('zeroex_optimism_api_fills_deduped') ,ref('zeroex_polygon_api_fills_deduped')