From 0e7e8c402225fcec6d3e65fd608906cb867f8e26 Mon Sep 17 00:00:00 2001 From: Jacob Sharples <45126111+JacobSharples@users.noreply.github.com> Date: Fri, 24 Jan 2025 21:43:39 -0500 Subject: [PATCH 01/11] Create raydium_v5_base_trades.sql --- .../dex/raydium/raydium_v5_base_trades.sql | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql diff --git a/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql new file mode 100644 index 00000000000..f3348c3be34 --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql @@ -0,0 +1,108 @@ + {{ + config( + schema = 'raydium_v5', -- Raydium CP-Swap (or CPMM). Similar logic to AMM (raydium_v4 table) + alias = 'base_trades', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['tx_id', 'outer_instruction_index', 'inner_instruction_index', 'tx_index','block_month'], + pre_hook='{{ enforce_join_distribution("PARTITIONED") }}' + ) +}} + +{% set project_start_date = '2024-05-16' %} --grabbed program deployed at time (account created at). + + WITH + all_swaps as ( + SELECT + sp.call_block_time as block_time + , sp.call_block_slot as block_slot + , 'raydium' as project + , 5 as version + , 'solana' as blockchain + , case when sp.call_is_inner = False then 'direct' + else sp.call_outer_executing_account + end as trade_source + -- -- token bought is always the second instruction (transfer) in the inner instructions + , trs_2.amount as token_bought_amount_raw + , trs_1.amount as token_sold_amount_raw + , account_poolState as pool_id --p.pool_id + , sp.call_tx_signer as trader_id + , sp.call_tx_id as tx_id + , sp.call_outer_instruction_index as outer_instruction_index + , COALESCE(sp.call_inner_instruction_index, 0) as inner_instruction_index + , sp.call_tx_index as tx_index + , COALESCE(trs_2.token_mint_address, cast(null as varchar)) as token_bought_mint_address + , COALESCE(trs_1.token_mint_address, cast(null as varchar)) as token_sold_mint_address + , trs_2.from_token_account as token_bought_vault + , trs_1.to_token_account as token_sold_vault + FROM ( + SELECT account_poolState, call_is_inner, call_outer_instruction_index, call_inner_instruction_index, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, call_tx_index + FROM {{ source('raydium_cp_solana', 'raydium_cp_swap_call_swapBaseOutput') }} + UNION ALL + SELECT account_poolState, call_is_inner, call_outer_instruction_index, call_inner_instruction_index, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, call_tx_index + FROM {{ source('raydium_cp_solana', 'raydium_cp_swap_call_swapBaseInput') }} + ) sp + INNER JOIN {{ ref('tokens_solana_transfers') }} trs_1 + ON trs_1.tx_id = sp.call_tx_id + AND trs_1.block_time = sp.call_block_time + AND trs_1.outer_instruction_index = sp.call_outer_instruction_index + AND ((sp.call_is_inner = false AND (trs_1.inner_instruction_index = 1 OR trs_1.inner_instruction_index = 2)) + OR (sp.call_is_inner = true AND (trs_1.inner_instruction_index = sp.call_inner_instruction_index + 1 OR trs_1.inner_instruction_index = sp.call_inner_instruction_index + 2)) + ) + AND trs_1.token_version = 'spl_token' + {% if is_incremental() %} + AND {{incremental_predicate('trs_1.block_time')}} + {% else %} + AND trs_1.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + INNER JOIN {{ ref('tokens_solana_transfers') }} trs_2 + ON trs_2.tx_id = sp.call_tx_id + AND trs_2.block_time = sp.call_block_time + AND trs_2.outer_instruction_index = sp.call_outer_instruction_index + AND ((sp.call_is_inner = false AND (trs_2.inner_instruction_index = 2 OR trs_2.inner_instruction_index = 3)) + OR (sp.call_is_inner = true AND (trs_2.inner_instruction_index = sp.call_inner_instruction_index + 2 OR trs_2.inner_instruction_index = sp.call_inner_instruction_index + 3)) + ) + AND trs_2.token_version = 'spl_token' + {% if is_incremental() %} + AND {{incremental_predicate('trs_2.block_time')}} + {% else %} + AND trs_2.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_2 ON tk_2.address = trs_2.from_token_account + AND tk_2.account_type = 'fungible' + WHERE 1=1 + and trs_1.token_mint_address != trs_2.token_mint_address --gets rid of dupes from the OR statement in transfer joins + and tk_2.token_balance_owner = 'GpMZbSM2GgvTKHJirzeGfMFoaZ8UR2X7F4v8vHTvxFbL' --raydium pool v4 authority. makes sure we don't accidently catch some fee transfer or something after the swap. should add for lifinity too later. + {% if is_incremental() %} + AND {{incremental_predicate('sp.call_block_time')}} + {% else %} + AND sp.call_block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + +SELECT + tb.blockchain + , tb.project + , tb.version + , CAST(date_trunc('month', tb.block_time) AS DATE) as block_month + , tb.block_time + , tb.block_slot + , tb.trade_source + , tb.token_bought_amount_raw + , tb.token_sold_amount_raw + , cast(null as double) as fee_tier + , tb.token_sold_mint_address + , tb.token_bought_mint_address + , tb.token_sold_vault + , tb.token_bought_vault + , tb.pool_id as project_program_id + , '675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8' as project_main_id + , tb.trader_id + , tb.tx_id + , tb.outer_instruction_index + , tb.inner_instruction_index + , tb.tx_index +FROM all_swaps tb From d6fa76f23f3229af90c0e790160f77daea5ad06f Mon Sep 17 00:00:00 2001 From: Jacob Sharples <45126111+JacobSharples@users.noreply.github.com> Date: Fri, 24 Jan 2025 22:02:05 -0500 Subject: [PATCH 02/11] Update raydium_v5_base_trades.sql --- .../models/_sector/dex/raydium/raydium_v5_base_trades.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql index f3348c3be34..d079b9c364b 100644 --- a/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql @@ -99,7 +99,7 @@ SELECT , tb.token_sold_vault , tb.token_bought_vault , tb.pool_id as project_program_id - , '675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8' as project_main_id + , 'CPMMoo8L3F4NbTegBCKVNunggL7H1ZpdTHKxQB5qKP1C' as project_main_id , tb.trader_id , tb.tx_id , tb.outer_instruction_index From 8d26107f59446c111645f640fa52ce7865047ffe Mon Sep 17 00:00:00 2001 From: Jacob Sharples <45126111+JacobSharples@users.noreply.github.com> Date: Fri, 24 Jan 2025 22:55:36 -0500 Subject: [PATCH 03/11] Update sources.yml --- .../models/_sector/dex/raydium/sources.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/raydium/sources.yml b/dbt_subprojects/solana/models/_sector/dex/raydium/sources.yml index 685cd1a056d..381adce0ee4 100644 --- a/dbt_subprojects/solana/models/_sector/dex/raydium/sources.yml +++ b/dbt_subprojects/solana/models/_sector/dex/raydium/sources.yml @@ -1,4 +1,4 @@ -version: 2 +version: 3 sources: - name: raydium_clmm_solana @@ -26,4 +26,16 @@ sources: - name: raydium_amm_call_swapBaseOut loaded_at_field: call_block_time - name: raydium_amm_call_swapBaseIn - loaded_at_field: call_block_time \ No newline at end of file + loaded_at_field: call_block_time + - name: raydium_cpmm_solana + description: "raydium_cpmm_solana v5 decoded tables" + freshness: # default freshness + warn_after: { count: 12, period: hour } + error_after: { count: 24, period: hour } + tables: + - name: raydium_cp_swap_call_initialize + loaded_at_field: call_block_time + - name: raydium_cp_swap_call_swapBaseOutput + loaded_at_field: call_block_time + - name: raydium_cp_swap_call_swapBaseInput + loaded_at_field: call_block_time From 21ebee7014144129f6d0556f68dd99c502879fb7 Mon Sep 17 00:00:00 2001 From: Jacob Sharples <45126111+JacobSharples@users.noreply.github.com> Date: Fri, 24 Jan 2025 22:57:45 -0500 Subject: [PATCH 04/11] Update schema.yml --- .../models/_sector/dex/raydium/schema.yml | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/raydium/schema.yml b/dbt_subprojects/solana/models/_sector/dex/raydium/schema.yml index b7f4a788ce3..ca22d27cf0b 100644 --- a/dbt_subprojects/solana/models/_sector/dex/raydium/schema.yml +++ b/dbt_subprojects/solana/models/_sector/dex/raydium/schema.yml @@ -1,4 +1,4 @@ -version: 2 +version: 3 models: - name: raydium_v3_base_trades @@ -162,3 +162,55 @@ models: - *outer_instruction_index - *inner_instruction_index - *tx_index + +- name: raydium_v5_base_trades + meta: + blockchain: solana + contributors: [ 0xsharples ] + config: + tags: [ 'solana','dex' ] + description: > + all raw raydium_v5 constant product amm dex trades on Solana + data_tests: + - check_columns_solana_dex_trades + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [ 'tx_id', 'outer_instruction_index', 'inner_instruction_index', 'tx_index', 'block_month' ] + + - name: raydium_v5_trades + meta: + blockchain: solana + contributors: [ 0xsharples ] + config: + tags: ['solana','dex'] + description: > + all raydium_v5 constant product amm dex trades on Solana + columns: + - *blockchain + - *project + - *version + - *block_month + - *block_time + - *block_slot + - *trade_source + - *token_bought_symbol + - *token_sold_symbol + - *token_pair + - *token_bought_amount + - *token_sold_amount + - *token_bought_amount_raw + - *token_sold_amount_raw + - *amount_usd + - *fee_tier + - *fee_usd + - *token_bought_mint_address + - *token_sold_mint_address + - *token_bought_vault + - *token_sold_vault + - *project_program_id + - *project_main_id + - *trader_id + - *tx_id + - *outer_instruction_index + - *inner_instruction_index + - *tx_index + From 1ffbaa3cad012d176780a4d1afe5e75f82b21e86 Mon Sep 17 00:00:00 2001 From: Jacob Sharples <45126111+JacobSharples@users.noreply.github.com> Date: Fri, 24 Jan 2025 23:01:13 -0500 Subject: [PATCH 05/11] Create raydium_v5_trades.sql --- .../_sector/dex/raydium/raydium_v5_trades.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_trades.sql diff --git a/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_trades.sql b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_trades.sql new file mode 100644 index 00000000000..0620bbe8047 --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_trades.sql @@ -0,0 +1,13 @@ + {{ + config( + schema = 'raydium_v5', + alias = 'trades', + materialized = 'view', + post_hook='{{ expose_spells(\'["solana"]\', + "project", + "raydium", + \'["0xsharples"]\') }}') +}} + +select * from {{ref('dex_solana_trades')}} +where project = 'raydium' and version = 5 From 34463c68b44a375e3169b13883dfb26f9cd19b3b Mon Sep 17 00:00:00 2001 From: Jacob Sharples <45126111+JacobSharples@users.noreply.github.com> Date: Sat, 25 Jan 2025 10:23:42 -0500 Subject: [PATCH 06/11] Update schema.yml --- dbt_subprojects/solana/models/_sector/dex/raydium/schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/raydium/schema.yml b/dbt_subprojects/solana/models/_sector/dex/raydium/schema.yml index ca22d27cf0b..9608e5251c7 100644 --- a/dbt_subprojects/solana/models/_sector/dex/raydium/schema.yml +++ b/dbt_subprojects/solana/models/_sector/dex/raydium/schema.yml @@ -163,7 +163,7 @@ models: - *inner_instruction_index - *tx_index -- name: raydium_v5_base_trades + - name: raydium_v5_base_trades meta: blockchain: solana contributors: [ 0xsharples ] From d72f33651617df6d162c8a83ffe6c4ea44ff6d3d Mon Sep 17 00:00:00 2001 From: Jacob Sharples <45126111+JacobSharples@users.noreply.github.com> Date: Sat, 25 Jan 2025 10:57:13 -0500 Subject: [PATCH 07/11] Update raydium_v5_base_trades.sql --- .../models/_sector/dex/raydium/raydium_v5_base_trades.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql index d079b9c364b..31457fda396 100644 --- a/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql @@ -1,6 +1,6 @@ {{ config( - schema = 'raydium_v5', -- Raydium CP-Swap (or CPMM). Similar logic to AMM (raydium_v4 table) + schema = 'raydium_v5', alias = 'base_trades', partition_by = ['block_month'], materialized = 'incremental', From c1dc647868efae32d60e106b55d9ec5ab3022790 Mon Sep 17 00:00:00 2001 From: Jacob Sharples <45126111+JacobSharples@users.noreply.github.com> Date: Sat, 25 Jan 2025 11:13:14 -0500 Subject: [PATCH 08/11] Update sources.yml --- dbt_subprojects/solana/models/_sector/dex/raydium/sources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/raydium/sources.yml b/dbt_subprojects/solana/models/_sector/dex/raydium/sources.yml index 381adce0ee4..8ae62e458fd 100644 --- a/dbt_subprojects/solana/models/_sector/dex/raydium/sources.yml +++ b/dbt_subprojects/solana/models/_sector/dex/raydium/sources.yml @@ -27,7 +27,7 @@ sources: loaded_at_field: call_block_time - name: raydium_amm_call_swapBaseIn loaded_at_field: call_block_time - - name: raydium_cpmm_solana + - name: raydium_cp_solana description: "raydium_cpmm_solana v5 decoded tables" freshness: # default freshness warn_after: { count: 12, period: hour } From 0698ab8f2ff6bffed0113cb5bd0e63d4aee727ce Mon Sep 17 00:00:00 2001 From: jeff-dude Date: Tue, 28 Jan 2025 13:13:39 -0500 Subject: [PATCH 09/11] add to cross-project base trades --- .../solana/models/_sector/dex/dex_solana_base_trades.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql index d3dba42ca27..769727b86b1 100644 --- a/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql @@ -14,6 +14,7 @@ {% set solana_dexes = [ ref('raydium_v3_base_trades') , ref('raydium_v4_base_trades') + , ref('raydium_v5_base_trades') , ref('orca_whirlpool_base_trades') , ref('orca_whirlpool_v2_base_trades') , ref('phoenix_v1_base_trades') From b5c51746f65417c8b13ef8fa0f24108b47fdae65 Mon Sep 17 00:00:00 2001 From: jeff-dude Date: Tue, 28 Jan 2025 13:14:53 -0500 Subject: [PATCH 10/11] force short runtimes for CI --- .../solana/models/_sector/dex/dex_solana_base_trades.sql | 2 +- .../models/_sector/dex/raydium/raydium_v5_base_trades.sql | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql index 769727b86b1..f903a5ef98e 100644 --- a/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql @@ -53,7 +53,7 @@ SELECT , tx_index FROM {{ dex }} -{% if is_incremental() %} +{% if is_incremental() or true %} WHERE {{incremental_predicate('block_time')}} {% endif %} diff --git a/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql index 31457fda396..86b7ef6071e 100644 --- a/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql @@ -53,7 +53,7 @@ OR (sp.call_is_inner = true AND (trs_1.inner_instruction_index = sp.call_inner_instruction_index + 1 OR trs_1.inner_instruction_index = sp.call_inner_instruction_index + 2)) ) AND trs_1.token_version = 'spl_token' - {% if is_incremental() %} + {% if is_incremental() or true %} AND {{incremental_predicate('trs_1.block_time')}} {% else %} AND trs_1.block_time >= TIMESTAMP '{{project_start_date}}' @@ -66,7 +66,7 @@ OR (sp.call_is_inner = true AND (trs_2.inner_instruction_index = sp.call_inner_instruction_index + 2 OR trs_2.inner_instruction_index = sp.call_inner_instruction_index + 3)) ) AND trs_2.token_version = 'spl_token' - {% if is_incremental() %} + {% if is_incremental() or true %} AND {{incremental_predicate('trs_2.block_time')}} {% else %} AND trs_2.block_time >= TIMESTAMP '{{project_start_date}}' @@ -76,7 +76,7 @@ WHERE 1=1 and trs_1.token_mint_address != trs_2.token_mint_address --gets rid of dupes from the OR statement in transfer joins and tk_2.token_balance_owner = 'GpMZbSM2GgvTKHJirzeGfMFoaZ8UR2X7F4v8vHTvxFbL' --raydium pool v4 authority. makes sure we don't accidently catch some fee transfer or something after the swap. should add for lifinity too later. - {% if is_incremental() %} + {% if is_incremental() or true %} AND {{incremental_predicate('sp.call_block_time')}} {% else %} AND sp.call_block_time >= TIMESTAMP '{{project_start_date}}' From cf2ba8d998737f91614847a09efac6a5c1e40103 Mon Sep 17 00:00:00 2001 From: jeff-dude Date: Tue, 28 Jan 2025 13:35:03 -0500 Subject: [PATCH 11/11] Revert "force short runtimes for CI" This reverts commit b5c51746f65417c8b13ef8fa0f24108b47fdae65. --- .../solana/models/_sector/dex/dex_solana_base_trades.sql | 2 +- .../models/_sector/dex/raydium/raydium_v5_base_trades.sql | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql index f903a5ef98e..769727b86b1 100644 --- a/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql @@ -53,7 +53,7 @@ SELECT , tx_index FROM {{ dex }} -{% if is_incremental() or true %} +{% if is_incremental() %} WHERE {{incremental_predicate('block_time')}} {% endif %} diff --git a/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql index 86b7ef6071e..31457fda396 100644 --- a/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/raydium/raydium_v5_base_trades.sql @@ -53,7 +53,7 @@ OR (sp.call_is_inner = true AND (trs_1.inner_instruction_index = sp.call_inner_instruction_index + 1 OR trs_1.inner_instruction_index = sp.call_inner_instruction_index + 2)) ) AND trs_1.token_version = 'spl_token' - {% if is_incremental() or true %} + {% if is_incremental() %} AND {{incremental_predicate('trs_1.block_time')}} {% else %} AND trs_1.block_time >= TIMESTAMP '{{project_start_date}}' @@ -66,7 +66,7 @@ OR (sp.call_is_inner = true AND (trs_2.inner_instruction_index = sp.call_inner_instruction_index + 2 OR trs_2.inner_instruction_index = sp.call_inner_instruction_index + 3)) ) AND trs_2.token_version = 'spl_token' - {% if is_incremental() or true %} + {% if is_incremental() %} AND {{incremental_predicate('trs_2.block_time')}} {% else %} AND trs_2.block_time >= TIMESTAMP '{{project_start_date}}' @@ -76,7 +76,7 @@ WHERE 1=1 and trs_1.token_mint_address != trs_2.token_mint_address --gets rid of dupes from the OR statement in transfer joins and tk_2.token_balance_owner = 'GpMZbSM2GgvTKHJirzeGfMFoaZ8UR2X7F4v8vHTvxFbL' --raydium pool v4 authority. makes sure we don't accidently catch some fee transfer or something after the swap. should add for lifinity too later. - {% if is_incremental() or true %} + {% if is_incremental() %} AND {{incremental_predicate('sp.call_block_time')}} {% else %} AND sp.call_block_time >= TIMESTAMP '{{project_start_date}}'