Merge branch 'duneanalytics:main' into main

.github/workflows/dbt_run.yml

@@ -48,6 +48,9 @@ jobs:
       - name: dbt seed
         run: "dbt seed $PROFILE --select @state:modified --exclude tag:prod_exclude tag:remove --state . --project-dir ${PROJECT_DIR}"
 
+      - name: list downstream models
+        run: "./scripts/list_modified_downstream.sh"
+
       - name: dbt run initial model(s)
         run: "dbt -x run $PROFILE --select state:modified --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"
 

CONTRIBUTING.md

@@ -101,7 +101,7 @@ models:
   - name: mock_table
     columns:
       - name: tx_id
-        tests:
+        data_tests:
           - unique
           - not_null
 ```

Pipfile

@@ -7,7 +7,7 @@ name = "pypi"
 numpy = "2.0.12"
 pre-commit = "2.20.0"
 pytest = "7.1.3"
-dbt-trino = "1.7.0"
+dbt-trino = "1.8.2"
 
 [requires]
-python_version = "3.9"
+python_version = "3.9"

README.md

@@ -200,7 +200,7 @@ models:
     columns:
       - name: tx_hash
         description: "Table primary key: a transaction hash (tx_hash) is a unique identifier for a transaction."
-        tests:
+        data_tests:
           - unique
           - not_null
 

dbt_subprojects/daily_spellbook/dbt_project.yml

@@ -3,6 +3,7 @@
 name: "daily_spellbook"
 version: "1.0.0"
 config-version: 2
+require-dbt-version: ">=1.8.0"
 
 quoting:
   database: false
@@ -33,7 +34,7 @@ clean-targets: # directories to be removed by `dbt clean`
 
 # Configuring tests
 # Full documentation: https://docs.getdbt.com/reference/test-configs
-tests:
+data_tests:
   daily_spellbook:
     +store_failures: true  # store failures for all tests
 

dbt_subprojects/daily_spellbook/models/_metrics/dune_index/_schema.yml

@@ -8,7 +8,7 @@ models:
     config:
       tags: ['metrics', 'dune', 'index', 'daily']
     description: "Combine transactions, transfers and fees index values to get a daily dune index value"
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - blockchain

dbt_subprojects/daily_spellbook/models/_metrics/dune_index/metrics_dune_index_stats.sql

@@ -42,10 +42,10 @@ with source as (
     select
         c.blockchain
         , c.last_1_days_dune_index as last_1_days_dune_index_contribution
-        , (c.last_1_days_dune_index / t.total_cross_chain_last_1_days_dune_index) * 100 as last_1_days_dune_index_contribution_percent
+        , c.last_1_days_dune_index / t.total_cross_chain_last_1_days_dune_index as last_1_days_dune_index_contribution_percent
         , t.total_cross_chain_last_1_days_dune_index
         , p.previous_1_days_dune_index as previous_1_days_dune_index_contribution
-        , ((c.last_1_days_dune_index - coalesce(p.previous_1_days_dune_index, 0)) / coalesce(p.previous_1_days_dune_index, 1)) * 100 AS daily_percent_change
+        , (c.last_1_days_dune_index - coalesce(p.previous_1_days_dune_index, 0)) / coalesce(p.previous_1_days_dune_index, 1) AS daily_percent_change
     from
         current_day as c
     left join previous_day as p
@@ -83,10 +83,10 @@ with source as (
     select
         c.blockchain
         , c.last_7_days_dune_index as last_7_days_dune_index_contribution
-        , (c.last_7_days_dune_index / t.total_cross_chain_last_7_days_dune_index) * 100 as last_7_days_dune_index_contribution_percent
+        , c.last_7_days_dune_index / t.total_cross_chain_last_7_days_dune_index as last_7_days_dune_index_contribution_percent
         , t.total_cross_chain_last_7_days_dune_index
         , p.previous_7_days_dune_index as previous_7_days_dune_index_contribution
-        , ((c.last_7_days_dune_index - coalesce(p.previous_7_days_dune_index, 0)) / coalesce(p.previous_7_days_dune_index, 1)) * 100 AS weekly_percent_change
+        , (c.last_7_days_dune_index - coalesce(p.previous_7_days_dune_index, 0)) / coalesce(p.previous_7_days_dune_index, 1) AS weekly_percent_change
     from
         current_week as c
     left join previous_week as p
@@ -124,10 +124,10 @@ with source as (
     select
         c.blockchain
         , c.last_30_days_dune_index as last_30_days_dune_index_contribution
-        , (c.last_30_days_dune_index / t.total_cross_chain_last_30_days_dune_index) * 100 as last_30_days_dune_index_contribution_percent
+        , c.last_30_days_dune_index / t.total_cross_chain_last_30_days_dune_index as last_30_days_dune_index_contribution_percent
         , t.total_cross_chain_last_30_days_dune_index
         , p.previous_30_days_dune_index as previous_30_days_dune_index_contribution
-        , ((c.last_30_days_dune_index - coalesce(p.previous_30_days_dune_index, 0)) / coalesce(p.previous_30_days_dune_index, 1)) * 100 AS monthly_percent_change
+        , (c.last_30_days_dune_index - coalesce(p.previous_30_days_dune_index, 0)) / coalesce(p.previous_30_days_dune_index, 1) AS monthly_percent_change
     from
         current_month as c
     left join previous_month as p

dbt_subprojects/daily_spellbook/models/_metrics/fees/_schema.yml

@@ -8,7 +8,7 @@ models:
     config:
       tags: ['metrics', 'fees', 'gas', 'daily']
     description: "Sum of total fees spent per day across all chains available in raw gas.fees table"
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - blockchain
@@ -27,7 +27,7 @@ models:
     config:
       tags: ['metrics', 'fees', 'gas', 'index', 'daily']
     description: "Each day, per chain, compare the adoption of activity based on total gas fees relative to the baseline expectation"
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - blockchain

dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_fees_index_daily.sql

@@ -9,7 +9,7 @@
         )
 }}
 
-{% set baseline_date = '2018-01-01' %}
+{% set baseline_date = '2015-08-21' %}
 
 with baseline as (
     select

dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_gas_fees_stats.sql

@@ -40,9 +40,9 @@ with source as (
         c.blockchain
         , c.last_1_days_gas_fees_usd
         , p.previous_1_days_gas_fees_usd
-        , ((c.last_1_days_gas_fees_usd - coalesce(p.previous_1_days_gas_fees_usd, 0)) / coalesce(p.previous_1_days_gas_fees_usd, 1)) * 100 AS daily_percent_change
+        , (c.last_1_days_gas_fees_usd - coalesce(p.previous_1_days_gas_fees_usd, 0)) / coalesce(p.previous_1_days_gas_fees_usd, 1) AS daily_percent_change
         , t.total_cross_chain_last_1_days_gas_fees_usd
-        , (c.last_1_days_gas_fees_usd / t.total_cross_chain_last_1_days_gas_fees_usd) * 100 AS percent_total_last_1_days_fees_usd
+        , c.last_1_days_gas_fees_usd / t.total_cross_chain_last_1_days_gas_fees_usd AS percent_total_last_1_days_fees_usd
     from
         current_day as c
     left join previous_day as p
@@ -81,9 +81,9 @@ with source as (
         c.blockchain
         , c.last_7_days_gas_fees_usd
         , p.previous_7_days_gas_fees_usd
-        , ((c.last_7_days_gas_fees_usd - coalesce(p.previous_7_days_gas_fees_usd, 0)) / coalesce(p.previous_7_days_gas_fees_usd, 1)) * 100 AS weekly_percent_change
+        , (c.last_7_days_gas_fees_usd - coalesce(p.previous_7_days_gas_fees_usd, 0)) / coalesce(p.previous_7_days_gas_fees_usd, 1) AS weekly_percent_change
         , t.total_cross_chain_last_7_days_gas_fees_usd
-        , (c.last_7_days_gas_fees_usd / t.total_cross_chain_last_7_days_gas_fees_usd) * 100 AS percent_total_last_7_days_gas_fees_usd
+        , c.last_7_days_gas_fees_usd / t.total_cross_chain_last_7_days_gas_fees_usd AS percent_total_last_7_days_gas_fees_usd
     from
         current_week as c
     left join previous_week as p
@@ -122,9 +122,9 @@ with source as (
         c.blockchain
         , c.last_30_days_gas_fees_usd
         , p.previous_30_days_gas_fees_usd
-        , ((c.last_30_days_gas_fees_usd - coalesce(p.previous_30_days_gas_fees_usd, 0)) / coalesce(p.previous_30_days_gas_fees_usd, 1)) * 100 AS monthly_percent_change
+        , (c.last_30_days_gas_fees_usd - coalesce(p.previous_30_days_gas_fees_usd, 0)) / coalesce(p.previous_30_days_gas_fees_usd, 1) AS monthly_percent_change
         , t.total_cross_chain_last_30_days_gas_fees_usd
-        , (c.last_30_days_gas_fees_usd / t.total_cross_chain_last_30_days_gas_fees_usd) * 100 AS percent_total_last_30_days_gas_fees_usd
+        , c.last_30_days_gas_fees_usd / t.total_cross_chain_last_30_days_gas_fees_usd AS percent_total_last_30_days_gas_fees_usd
     from
         current_month as c
     left join previous_month as p

dbt_subprojects/daily_spellbook/models/_metrics/transactions/_schema.yml

@@ -8,7 +8,7 @@ models:
     config:
       tags: ['metrics', 'transactions', 'daily']
     description: "Sum of total tx's per day across all chains"
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - blockchain
@@ -27,7 +27,7 @@ models:
     config:
       tags: ['metrics', 'transactions', 'index', 'daily']
     description: "Each day, per chain, compare the adoption of total transactions relative to the baseline expectation"
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - blockchain

dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_daily.sql

@@ -34,7 +34,7 @@ with raw_tx as (
         {{ ref('metrics_net_transfers') }}
     where
         1 = 1
-        and net_transfer_amount_usd >= 1 --only include tx's where transfer value is greater at least $1
+        and net_transfer_amount_usd >= 1 --only include tx's where transfer value is at least $1
         {% if is_incremental() %}
         and {{ incremental_predicate('block_date') }}
         {% endif %}

dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_index_daily.sql

@@ -9,7 +9,7 @@
         )
 }}
 
-{% set baseline_date = '2018-01-01' %}
+{% set baseline_date = '2015-08-21' %}
 
 with baseline as (
     select

dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_stats.sql

@@ -40,9 +40,9 @@ with source as (
         c.blockchain
         , c.last_1_days_tx_count
         , p.previous_1_days_tx_count
-        , ((cast(c.last_1_days_tx_count as double) - coalesce(cast(p.previous_1_days_tx_count as double), 0)) / coalesce(cast(p.previous_1_days_tx_count as double), 1)) * 100 AS daily_percent_change
+        , (cast(c.last_1_days_tx_count as double) - coalesce(cast(p.previous_1_days_tx_count as double), 0)) / coalesce(cast(p.previous_1_days_tx_count as double), 1) AS daily_percent_change
         , t.total_cross_chain_last_1_days_tx_count
-        , (cast(c.last_1_days_tx_count as double) / cast(t.total_cross_chain_last_1_days_tx_count as double)) * 100 AS percent_total_last_1_days_tx_count
+        , cast(c.last_1_days_tx_count as double) / cast(t.total_cross_chain_last_1_days_tx_count as double) AS percent_total_last_1_days_tx_count
     from
         current_day as c
     left join previous_day as p
@@ -81,9 +81,9 @@ with source as (
         c.blockchain
         , c.last_7_days_tx_count
         , p.previous_7_days_tx_count
-        , ((cast(c.last_7_days_tx_count as double) - coalesce(cast(p.previous_7_days_tx_count as double), 0)) / coalesce(cast(p.previous_7_days_tx_count as double), 1)) * 100 AS weekly_percent_change
+        , (cast(c.last_7_days_tx_count as double) - coalesce(cast(p.previous_7_days_tx_count as double), 0)) / coalesce(cast(p.previous_7_days_tx_count as double), 1) AS weekly_percent_change
         , t.total_cross_chain_last_7_days_tx_count
-        , (cast(c.last_7_days_tx_count as double) / cast(t.total_cross_chain_last_7_days_tx_count as double)) * 100 AS percent_total_last_7_days_tx_count
+        , cast(c.last_7_days_tx_count as double) / cast(t.total_cross_chain_last_7_days_tx_count as double) AS percent_total_last_7_days_tx_count
     from
         current_week as c
     left join previous_week as p
@@ -122,9 +122,9 @@ with source as (
         c.blockchain
         , c.last_30_days_tx_count
         , p.previous_30_days_tx_count
-        , ((cast(c.last_30_days_tx_count as double) - coalesce(cast(p.previous_30_days_tx_count as double), 0)) / coalesce(cast(p.previous_30_days_tx_count as double), 1)) * 100 AS monthly_percent_change
+        , (cast(c.last_30_days_tx_count as double) - coalesce(cast(p.previous_30_days_tx_count as double), 0)) / coalesce(cast(p.previous_30_days_tx_count as double), 1) AS monthly_percent_change
         , t.total_cross_chain_last_30_days_tx_count
-        , (cast(c.last_30_days_tx_count as double) / cast(t.total_cross_chain_last_30_days_tx_count as double)) * 100 AS percent_total_last_30_days_tx_count
+        , cast(c.last_30_days_tx_count as double) / cast(t.total_cross_chain_last_30_days_tx_count as double) AS percent_total_last_30_days_tx_count
     from
         current_month as c
     left join previous_month as p

dbt_subprojects/daily_spellbook/models/_metrics/transfers/_schema.yml

@@ -18,7 +18,7 @@ models:
       with the positive net received (aka received) it adds up to 0 since the net sent should equal the net received. 
       For example, if I give you $1 that's -1 sent (from me) and +1 received (by you) which adds to 0. 
       This query counts the received to show that $1 was effectively transfered."
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - blockchain
@@ -32,7 +32,7 @@ models:
       tags: ['metrics', 'transfers', 'daily']
     description: |
       "Sum of total transfer amount per day across all chain."
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - blockchain
@@ -51,7 +51,7 @@ models:
     config:
       tags: ['metrics', 'transfers', 'index', 'daily']
     description: "Each day, per chain, compare the adoption of total transfers relative to the baseline expectation"
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - blockchain

dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_net_transfers.sql

@@ -122,6 +122,8 @@ select
     , sum(net_transfer_amount_usd) as net_transfer_amount_usd
 from
     net_transfers
+where
+    net_transfer_amount_usd > 0
 group by
     blockchain
     , block_date

dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_daily.sql

@@ -19,7 +19,6 @@ from
     {{ ref('metrics_net_transfers') }}
 where
     1 = 1
-    and net_transfer_amount_usd > 0
     {% if is_incremental() %}
     and {{ incremental_predicate('block_date') }}
     {% endif %}

dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_index_daily.sql

@@ -9,7 +9,7 @@
         )
 }}
 
-{% set baseline_date = '2018-01-01' %}
+{% set baseline_date = '2015-08-21' %}
 
 with baseline as (
     select

dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_stats.sql

@@ -40,9 +40,9 @@ with source as (
         c.blockchain
         , c.last_1_days_net_transfer_amount_usd
         , p.previous_1_days_net_transfer_amount_usd
-        , ((cast(c.last_1_days_net_transfer_amount_usd as double) - coalesce(cast(p.previous_1_days_net_transfer_amount_usd as double), 0)) / coalesce(cast(p.previous_1_days_net_transfer_amount_usd as double), 1)) * 100 AS daily_percent_change
+        , (cast(c.last_1_days_net_transfer_amount_usd as double) - coalesce(cast(p.previous_1_days_net_transfer_amount_usd as double), 0)) / coalesce(cast(p.previous_1_days_net_transfer_amount_usd as double), 1) AS daily_percent_change
         , t.total_cross_chain_last_1_days_net_transfer_amount_usd
-        , (cast(c.last_1_days_net_transfer_amount_usd as double) / cast(t.total_cross_chain_last_1_days_net_transfer_amount_usd as double)) * 100 AS percent_total_last_1_days_net_transfer_amount_usd
+        , cast(c.last_1_days_net_transfer_amount_usd as double) / cast(t.total_cross_chain_last_1_days_net_transfer_amount_usd as double) AS percent_total_last_1_days_net_transfer_amount_usd
     from
         current_day as c
     left join previous_day as p
@@ -81,9 +81,9 @@ with source as (
         c.blockchain
         , c.last_7_days_net_transfer_amount_usd
         , p.previous_7_days_net_transfer_amount_usd
-        , ((cast(c.last_7_days_net_transfer_amount_usd as double) - coalesce(cast(p.previous_7_days_net_transfer_amount_usd as double), 0)) / coalesce(cast(p.previous_7_days_net_transfer_amount_usd as double), 1)) * 100 AS weekly_percent_change
+        , (cast(c.last_7_days_net_transfer_amount_usd as double) - coalesce(cast(p.previous_7_days_net_transfer_amount_usd as double), 0)) / coalesce(cast(p.previous_7_days_net_transfer_amount_usd as double), 1) AS weekly_percent_change
         , t.total_cross_chain_last_7_days_net_transfer_amount_usd
-        , (cast(c.last_7_days_net_transfer_amount_usd as double) / cast(t.total_cross_chain_last_7_days_net_transfer_amount_usd as double)) * 100 AS percent_total_last_7_days_net_transfer_amount_usd
+        , cast(c.last_7_days_net_transfer_amount_usd as double) / cast(t.total_cross_chain_last_7_days_net_transfer_amount_usd as double) AS percent_total_last_7_days_net_transfer_amount_usd
     from
         current_week as c
     left join previous_week as p
@@ -122,9 +122,9 @@ with source as (
         c.blockchain
         , c.last_30_days_net_transfer_amount_usd
         , p.previous_30_days_net_transfer_amount_usd
-        , ((cast(c.last_30_days_net_transfer_amount_usd as double) - coalesce(cast(p.previous_30_days_net_transfer_amount_usd as double), 0)) / coalesce(cast(p.previous_30_days_net_transfer_amount_usd as double), 1)) * 100 AS monthly_percent_change
+        , (cast(c.last_30_days_net_transfer_amount_usd as double) - coalesce(cast(p.previous_30_days_net_transfer_amount_usd as double), 0)) / coalesce(cast(p.previous_30_days_net_transfer_amount_usd as double), 1) AS monthly_percent_change
         , t.total_cross_chain_last_30_days_net_transfer_amount_usd
-        , (cast(c.last_30_days_net_transfer_amount_usd as double) / cast(t.total_cross_chain_last_30_days_net_transfer_amount_usd as double)) * 100 AS percent_total_last_30_days_net_transfer_amount_usd
+        , cast(c.last_30_days_net_transfer_amount_usd as double) / cast(t.total_cross_chain_last_30_days_net_transfer_amount_usd as double) AS percent_total_last_30_days_net_transfer_amount_usd
     from
         current_month as c
     left join previous_month as p