diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills_deduped.sql b/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_deduped_trades.sql similarity index 80% rename from dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills_deduped.sql rename to dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_deduped_trades.sql index 9a23ab938c9..76f95860aaa 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills_deduped.sql +++ b/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_deduped_trades.sql @@ -1,36 +1,20 @@ -{{ config( +{% macro zeroex_v1_deduped_trades(blockchain, start_date) %} - schema = 'zeroex_ethereum', - alias = 'api_fills_deduped', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['block_date', 'tx_hash', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] - ) -}} - -{% set zeroex_v3_start_date = '2019-12-01' %} ---the code used to create the data for the insertion into the dex.trades table ---this code is also the deduped version for the fills tables. ---only the data for 0x API fills. ---dependent on:zeroex_ethereum.api_fills. +{% set table_name = 'zeroex_' ~ blockchain ~ '_api_fills' %} WITH fills_with_tx_fill_number AS ( SELECT row_number() OVER ( partition BY tx_hash ORDER BY evt_index ASC ) AS tx_fill_number , * - FROM {{ ref('zeroex_ethereum_api_fills') }} + FROM {{ ref(table_name) }} WHERE 1=1 AND swap_flag = true {% if is_incremental() %} AND {{ incremental_predicate('block_time') }} {% endif %} {% if not is_incremental() %} - AND block_time >= cast('{{zeroex_v3_start_date}}' as date) + AND block_time >= DATE '{{start_date}}' {% endif %} ) , fills_first_last @@ -66,7 +50,7 @@ AS , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token_amount_raw END) AS maker_token_amount_raw , COUNT(*) AS fills_within FROM fills_first_last a - GROUP BY tx_hash,hop_count + GROUP BY tx_hash ) SELECT a.blockchain , '0x API' as project @@ -98,4 +82,6 @@ SELECT a.blockchain , a.contract_address FROM fills_with_tx_fill_number a INNER JOIN deduped_bridge_fills b - ON a.tx_hash = b.tx_hash AND a.evt_index = b.evt_index \ No newline at end of file + ON a.tx_hash = b.tx_hash AND a.evt_index = b.evt_index +{% endmacro %} + diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql deleted file mode 100644 index e7f7dd0ddbd..00000000000 --- a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql +++ /dev/null @@ -1,102 +0,0 @@ -{{ config( - - schema = 'zeroex_base', - alias = 'api_fills_deduped', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['block_date', 'tx_hash', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] - ) -}} - -{% set zeroex_v3_start_date = '2019-12-01' %} - -WITH fills_with_tx_fill_number -AS -( - SELECT row_number() OVER ( partition BY tx_hash ORDER BY evt_index ASC ) AS tx_fill_number - , * - FROM {{ ref('zeroex_base_api_fills') }} - WHERE 1=1 - AND swap_flag = true - {% if is_incremental() %} - AND {{ incremental_predicate('block_time') }} - {% endif %} - {% if not is_incremental() %} - AND block_time >= cast('{{zeroex_v3_start_date}}' as date) - {% endif %} -) -, fills_first_last -AS -( - SELECT CASE - WHEN a.taker_token = c.maker_token AND a.taker_token_amount_raw = c.maker_token_amount_raw THEN 0 - ELSE 1 END AS taker_consider_flag--from - , CASE - WHEN a.maker_token = b.taker_token AND a.maker_token_amount_raw = b.taker_token_amount_raw THEN 0 - ELSE 1 END AS maker_consider_flag - , SUM(CASE - WHEN a.maker_token = b.taker_token AND a.maker_token_amount_raw = b.taker_token_amount_raw THEN 0 - ELSE 1 END) OVER (PARTITION BY a.tx_hash ORDER BY a.evt_index DESC) hop_count - , a.* - FROM fills_with_tx_fill_number a - LEFT JOIN fills_with_tx_fill_number b ON (a.tx_hash = b.tx_hash AND a.tx_fill_number = b.tx_fill_number - 1) - LEFT JOIN fills_with_tx_fill_number c ON (a.tx_hash = c.tx_hash AND a.tx_fill_number = c.tx_fill_number + 1) -) -, deduped_bridge_fills -AS -( - SELECT tx_hash - , MAX(evt_index) AS evt_index - , MAX(affiliate_address) AS affiliate_address - , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_token END ) AS taker_token - , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token END ) AS maker_token - , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_symbol END ) AS taker_symbol - , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_symbol END ) AS maker_symbol - , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_token_amount END) AS taker_token_amount - , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token_amount END) AS maker_token_amount - , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_token_amount_raw END) AS taker_token_amount_raw - , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token_amount_raw END) AS maker_token_amount_raw - , COUNT(*) AS fills_within - FROM fills_first_last a - GROUP BY tx_hash,hop_count -) -SELECT a.blockchain - , '0x API' as project - , cast('1' as varchar(10)) as version - - , a.block_date - , a.block_month - , a.block_time - , b.taker_symbol AS taker_symbol - , b.maker_symbol AS maker_symbol - , CASE - WHEN LOWER(b.taker_symbol) > LOWER(b.maker_symbol) - THEN CONCAT(COALESCE(b.maker_symbol, ''), '-', COALESCE(b.taker_symbol, '')) - ELSE CONCAT(COALESCE(b.taker_symbol, ''), '-', COALESCE(b.maker_symbol, '')) - END AS token_pair - , b.taker_token_amount - , b.maker_token_amount - , b.taker_token_amount_raw AS taker_token_amount_raw - , b.maker_token_amount_raw AS maker_token_amount_raw - , a.volume_usd - , b.taker_token - , b.maker_token - , a.taker - , a.maker - , a.affiliate_address - , a.tx_hash - , a.tx_from - , a.tx_to - , b.evt_index - , ARRAY[-1] AS trace_address - , a.type - , a.swap_flag - , b.fills_within - , a.contract_address -FROM fills_with_tx_fill_number a -INNER JOIN deduped_bridge_fills b - ON a.tx_hash = b.tx_hash AND a.evt_index = b.evt_index \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml index a6c1b0f74b8..a0212b1a160 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml +++ b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml @@ -82,7 +82,7 @@ models: name: blockchain description: "Blockchain which the aggregator project is deployed" - - name: zeroex_base_api_fills_deduped + - name: zeroex_v1_base_deduped_trades meta: blockchain: base project: zeroex diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql new file mode 100644 index 00000000000..5713bc0b849 --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql @@ -0,0 +1,30 @@ +{{ config( + schema = 'zeroex_v1_base', + alias = 'deduped_trades', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['block_date', 'tx_hash', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{% set zeroex_v3_start_date = '2019-12-01' %} +{% set blockchain = 'base' %} + +WITH +deduped_trades as ( + {{ + zeroex_v1_deduped_trades( + blockchain = blockchain, + start_date = zeroex_v3_start_date + + ) + }} + +) +select + * +from deduped_trades diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml index 161329dd166..134e8bc863f 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml +++ b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml @@ -106,11 +106,11 @@ models: name: blockchain - - name: zeroex_ethereum_api_fills_deduped + - name: zeroex_v1_ethereum_deduped_trades meta: blockchain: ethereum project: zeroex - contributors: bakabhai993 + contributors: bakabhai993, rantum config: tags: ['ethereum','0x','dex_aggregator','dex','aggregator'] description: > diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql new file mode 100644 index 00000000000..5b0debbdab0 --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql @@ -0,0 +1,30 @@ +{{ config( + schema = 'zeroex_v1_ethereum', + alias = 'deduped_trades', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['block_date', 'tx_hash', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{% set zeroex_v3_start_date = '2019-12-01' %} +{% set blockchain = 'ethereum' %} + +WITH +deduped_trades as ( + {{ + zeroex_v1_deduped_trades( + blockchain = blockchain, + start_date = zeroex_v3_start_date + + ) + }} + +) +select + * +from deduped_trades diff --git a/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql b/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql index 07f52a7d2f9..2b00a9a3f86 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql @@ -12,9 +12,9 @@ {% set zeroex_models = [ ref('zeroex_arbitrum_api_fills_deduped') ,ref('zeroex_avalanche_c_api_fills_deduped') - ,ref('zeroex_base_api_fills_deduped') + ,ref('zeroex_v1_base_deduped_trades') ,ref('zeroex_celo_api_fills_deduped') - ,ref('zeroex_ethereum_api_fills_deduped') + ,ref('zeroex_v1_ethereum_deduped_trades') ,ref('zeroex_fantom_api_fills_deduped') ,ref('zeroex_optimism_api_fills_deduped') ,ref('zeroex_polygon_api_fills_deduped')