zeroex - macro for deduped fills (#7156)

* deduped macro * add eth model via macro * revise ref table name * cast to date * remove/replace old model * update source method * source name * source name * table name * update macro, add base model * update source --------- Co-authored-by: Huang Geyang <[email protected]> Co-authored-by: jeff-dude <[email protected]>
duneanalytics · Nov 25, 2024 · 30b95c5 · 30b95c5
1 parent 241e6a3
commit 30b95c5
Show file tree

Hide file tree

Showing 7 changed files with 73 additions and 129 deletions.
diff --git a/...eum/zeroex_ethereum_api_fills_deduped.sql → ...oject/zeroex/zeroex_v1_deduped_trades.sql b/...eum/zeroex_ethereum_api_fills_deduped.sql → ...oject/zeroex/zeroex_v1_deduped_trades.sql
@@ -1,36 +1,20 @@
-{{  config(
+{% macro zeroex_v1_deduped_trades(blockchain, start_date) %}
 
-        schema = 'zeroex_ethereum',
-        alias = 'api_fills_deduped',
-        materialized='incremental',
-        partition_by = ['block_month'],
-        unique_key = ['block_date', 'tx_hash', 'evt_index'],
-        on_schema_change='sync_all_columns',
-        file_format ='delta',
-        incremental_strategy='merge',
-        incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
-    )
-}}
-
-{% set zeroex_v3_start_date = '2019-12-01' %}
---the code used to create the data for the insertion into the dex.trades table
---this code is also the deduped version for the fills tables.
---only the data for 0x API fills.
---dependent on:zeroex_ethereum.api_fills.
+{% set table_name = 'zeroex_' ~ blockchain ~ '_api_fills' %}
 
 WITH fills_with_tx_fill_number
 AS
 (
     SELECT   row_number() OVER ( partition BY tx_hash ORDER BY evt_index ASC ) AS tx_fill_number
            , *
-    FROM {{ ref('zeroex_ethereum_api_fills') }}
+    FROM {{ ref(table_name) }}
     WHERE 1=1
     AND swap_flag = true
     {% if is_incremental() %}
     AND {{ incremental_predicate('block_time') }}
     {% endif %}
     {% if not is_incremental() %}
-    AND block_time >= cast('{{zeroex_v3_start_date}}' as date)
+    AND block_time >= DATE '{{start_date}}'
     {% endif %}
 )
 , fills_first_last
@@ -66,7 +50,7 @@ AS
         , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token_amount_raw END)  AS maker_token_amount_raw
         , COUNT(*)                                                                          AS fills_within
     FROM fills_first_last a
-    GROUP BY  tx_hash,hop_count
+    GROUP BY  tx_hash
 )
 SELECT  a.blockchain
       , '0x API'  as project
@@ -98,4 +82,6 @@ SELECT  a.blockchain
       , a.contract_address
 FROM fills_with_tx_fill_number a
 INNER JOIN deduped_bridge_fills b
-    ON a.tx_hash = b.tx_hash AND a.evt_index = b.evt_index
+    ON a.tx_hash = b.tx_hash AND a.evt_index = b.evt_index
+{% endmacro %}
+
diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql
diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml
@@ -82,7 +82,7 @@ models:
         name: blockchain
         description: "Blockchain which the aggregator project is deployed"
 
-  - name: zeroex_base_api_fills_deduped
+  - name: zeroex_v1_base_deduped_trades
     meta:
       blockchain: base
       project: zeroex

diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql
@@ -0,0 +1,30 @@
+{{  config(
+        schema = 'zeroex_v1_base',
+        alias = 'deduped_trades',
+        materialized='incremental',
+        partition_by = ['block_month'],
+        unique_key = ['block_date', 'tx_hash', 'evt_index'],
+        on_schema_change='sync_all_columns',
+        file_format ='delta',
+        incremental_strategy='merge',
+        incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
+    )
+}}
+
+{% set zeroex_v3_start_date = '2019-12-01' %}
+{% set blockchain = 'base' %}
+
+WITH 
+deduped_trades as (
+    {{
+        zeroex_v1_deduped_trades(
+            blockchain = blockchain,
+            start_date = zeroex_v3_start_date
+
+        )
+    }}
+
+)
+select 
+    *
+from deduped_trades 
diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml
@@ -106,11 +106,11 @@ models:
         name: blockchain
 
 
-  - name: zeroex_ethereum_api_fills_deduped
+  - name: zeroex_v1_ethereum_deduped_trades
     meta:
       blockchain: ethereum
       project: zeroex
-      contributors: bakabhai993
+      contributors: bakabhai993, rantum
     config:
       tags: ['ethereum','0x','dex_aggregator','dex','aggregator']
     description: >

diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql
@@ -0,0 +1,30 @@
+{{  config(
+        schema = 'zeroex_v1_ethereum',
+        alias = 'deduped_trades',
+        materialized='incremental',
+        partition_by = ['block_month'],
+        unique_key = ['block_date', 'tx_hash', 'evt_index'],
+        on_schema_change='sync_all_columns',
+        file_format ='delta',
+        incremental_strategy='merge',
+        incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
+    )
+}}
+
+{% set zeroex_v3_start_date = '2019-12-01' %}
+{% set blockchain = 'ethereum' %}
+
+WITH 
+deduped_trades as (
+    {{
+        zeroex_v1_deduped_trades(
+            blockchain = blockchain,
+            start_date = zeroex_v3_start_date
+
+        )
+    }}
+
+)
+select 
+    *
+from deduped_trades 
diff --git a/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql b/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql
@@ -12,9 +12,9 @@
 {% set zeroex_models = [  
   ref('zeroex_arbitrum_api_fills_deduped')
   ,ref('zeroex_avalanche_c_api_fills_deduped')
-  ,ref('zeroex_base_api_fills_deduped')
+  ,ref('zeroex_v1_base_deduped_trades')
   ,ref('zeroex_celo_api_fills_deduped')
-  ,ref('zeroex_ethereum_api_fills_deduped')
+  ,ref('zeroex_v1_ethereum_deduped_trades')
   ,ref('zeroex_fantom_api_fills_deduped')
   ,ref('zeroex_optimism_api_fills_deduped')
   ,ref('zeroex_polygon_api_fills_deduped')