Remove duplicates from transfers_ethereum.erc20 originating in `erc…

…20_ethereum.evt_transfer` (#1164) * union all to union + tx type on unique key
duneanalytics · Jun 15, 2022 · dcbea17 · dcbea17
1 parent 950174d
commit dcbea17
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 15 deletions.
diff --git a/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20.sql b/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20.sql
@@ -3,8 +3,8 @@
 with
     sent_transfers as (
         select
-            evt_tx_hash || '-' || evt_index || '-' || to as unique_tx_id,
-            to as wallet_address,
+            'send' || '-' || evt_tx_hash || '-' || evt_index || '-' || `to` as unique_transfer_id,
+            `to` as wallet_address,
             contract_address as token_address,
             evt_block_time,
             value as amount_raw
@@ -14,20 +14,20 @@ with
 
     ,
     received_transfers as (
-        select evt_tx_hash || '-' || evt_index || '-' || to as unique_tx_id,
-        from
-            as wallet_address,
-            contract_address as token_address,
-            evt_block_time, 
-            - value as amount_raw
+        select
+        'receive' || '-' || evt_tx_hash || '-' || evt_index || '-' || `from` as unique_transfer_id,
+        `from` as wallet_address,
+        contract_address as token_address,
+        evt_block_time,
+        - value as amount_raw
         from
             {{ source('erc20_ethereum', 'evt_transfer') }}
     )
 
     ,
     deposited_weth as (
         select
-            evt_tx_hash || '-' || evt_index as unique_tx_id,
+            'deposit' || '-' || evt_tx_hash || '-' || evt_index || '-' || dst as unique_transfer_id,
             dst as wallet_address,
             contract_address as token_address,
             evt_block_time,
@@ -39,7 +39,7 @@ with
     ,
     withdrawn_weth as (
         select
-            evt_tx_hash || '-' || evt_index as unique_tx_id,
+            'withdrawn' || '-' || evt_tx_hash || '-' || evt_index || '-' || src as unique_transfer_id,
             src as wallet_address,
             contract_address as token_address,
             evt_block_time,
@@ -48,8 +48,14 @@ with
             {{ source('zeroex_ethereum', 'weth9_evt_withdrawal') }}
     )
 
-select 'ethereum' as blockchain, wallet_address, token_address, evt_block_time, amount_raw
+select unique_transfer_id, 'ethereum' as blockchain, wallet_address, token_address, evt_block_time, amount_raw
 from sent_transfers
-union all
-select 'ethereum' as blockchain, wallet_address, token_address, evt_block_time, amount_raw
+union
+select unique_transfer_id, 'ethereum' as blockchain, wallet_address, token_address, evt_block_time, amount_raw
 from received_transfers
+union
+select unique_transfer_id, 'ethereum' as blockchain, wallet_address, token_address, evt_block_time, amount_raw
+from deposited_weth
+union
+select unique_transfer_id, 'ethereum' as blockchain, wallet_address, token_address, evt_block_time, amount_raw
+from withdrawn_weth
diff --git a/spellbook/models/transfers/ethereum/transfers_ethereum_schema.yml b/spellbook/models/transfers/ethereum/transfers_ethereum_schema.yml
@@ -11,8 +11,10 @@ models:
       tags: ['transfers', 'ethereum', 'erc20', 'soispoke', 'dot2dotseurat']
     description: "ERC20 Token Transfers on Ethereum. This table is updated every 15 minutes."
     columns:
-      - name: unique_tx_id
-        description: "Unique tx ID (testing)"
+      - name: unique_transfer_id
+        description: "Unique transfer ID (used for testing for duplicates)"
+        tests:
+          - unique
       - &blockchain
         name: blockchain
         description: "Blockchain"