-
Notifications
You must be signed in to change notification settings - Fork 58
/
dbt_project.yml
240 lines (191 loc) · 9.03 KB
/
dbt_project.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
name: 'ra_data_warehouse'
version: '0.0.3'
require-dbt-version: ">=0.17.0"
config-version: 2
profile: 'ra_data_warehouse'
source-paths: ["models"]
analysis-paths: ["analysis"]
test-paths: ["tests"]
data-paths: ["data"]
macro-paths: ["macros"]
target-path: "target" # directory which will store compiled SQL files
clean-targets: # directories to be removed by `dbt clean`
- "target"
- "dbt_modules"
on-run-start:
- "{% do create_schema(api.Relation.create(database=target.project, schema=generate_prefixed_target_name().strip() + '_logs')) %}"
- "create table if not exists {{ generate_prefixed_target_name() }}_logs.audit_dbt_results (object string, status string, row_count numeric, execution_time numeric, load_ts timestamp)"
on-run-end:
- "insert into {{ generate_prefixed_target_name() }}_logs.audit_dbt_results (object, status, row_count, execution_time, load_ts) values {{ results_values(results) }}"
models:
ra_data_warehouse:
# source adapter general settings
sources:
materialized: view
schema: staging
# integration layer settings
integration:
materialized: view
schema: staging
# warehouse layer settings
warehouse:
materialized: table
# util module settings
utils:
materialized: table
schema: logs
# fivetran models
shopify:
+enabled: false
shopify_source:
+enabled: false
# seeds (reference files, loaded into tables) module settings
seeds:
ra_data_warehouse:
schema: seed
vars:
crm_warehouse_company_sources: ['hubspot_crm','harvest_projects','xero_accounting','stripe_payments','asana_projects','jira_projects']
crm_warehouse_contact_sources: ['hubspot_crm','harvest_projects','xero_accounting','mailchimp_email','asana_projects','jira_projects']
crm_warehouse_conversations_sources: ['hubspot_crm','intercom_messaging']
marketing_warehouse_ad_campaign_sources: ['facebook_ads','google_ads']
marketing_warehouse_ad_group_sources: ['facebook_ads','google_ads']
marketing_warehouse_ad_sources: ['facebook_ads','google_ads']
marketing_warehouse_email_event_sources: ['hubspot_email','mailchimp_email']
marketing_warehouse_email_list_sources: ['mailchimp_email']
product_warehouse_event_sources: ['segment_events_page','segment_events_track','snowplow_events_page','segment_events_order_checkout','appsflyer_events_track']
order_conversion_sources: ['custom']
subscriptions_warehouse_sources: []
marketing_warehouse_deal_sources: ['hubspot_crm']
stg_segment_events_etl: segment
stg_segment_events_segment_schema: segment_events
stg_segment_events_site: rittmananalytics.com
enable_event_type_mapping: true
stg_rudderstack_events_etl: rudderstack
stg_rudderstack_events_rudderstack_schema: rudderstack_ra_website
stg_rudderstack_events_site: rittmananalytics.com
enable_event_type_mapping: true
stg_mixpanel_events_id-prefix: mixpanel-
stg_mixpanel_events_stitch_schema: mixpanel_stitch
stg_mixpanel_events_fivetran_schema: fivetran_mixpanel
stg_mixpanel_events_etl: stitch
stg_mixpanel_events_site: rittmananalytics.com
stg_custom_conversions_schema: analytics_staging
stg_mailchimp_email_stitch_schema: stitch_mailchimp
stg_mailchimp_email_stitch_schema: stg_xero_accounting_stitch_schema
stg_xero_accounting_id-prefix: xero-
stg_xero_accounting_etl: stitch
stg_xero_accounting_stitch_schema: stitch_xero
stg_xero_accounting_fivetran_schema: fivetran_xero
stg_harvest_projects_stitch_schema: stitch_harvest
stg_harvest_projects_id-prefix: harvest-
stg_harvest_projects_etl: stitch
stg_segment_shopify_events_segment_schema: segment_shopify
stg_stripe_payments_stitch_schema: stitch_stripe
stg_stripe_payments_id-prefix: stripe-
stg_hubspot_crm_stitch_schema: stitch_hubspot
stg_hubspot_crm_fivetran_schema: fivetran_hubspot
stg_hubspot_crm_id-prefix: hubspot-
stg_hubspot_crm_etl: fivetran
stg_hubspot_email_stitch_database: ra-warehouse-dev
stg_hubspot_email_stitch_schema: stitch_hubspot
stg_hubspot_email_etl: stitch
stg_intercom_messaging_id-prefix: intercom-
stg_intercom_messaging_stitch_schema: stitch_intercom
stg_intercom_messaging_etl: stitch
stg_jira_projects_id-prefix: jira-
stg_jira_projects_etl: stitch
stg_jira_projects_stitch_schema: stitch_jira
stg_jira_projects_staff_email_domain: rittmananalytics.com
stg_jira_projects_jira_url: rittmananalytics.atlassian.net/jira
stg_intercom_messaging_id-prefix: intercom-
stg_intercom_messaging_etl: stitch
stg_git_repos_id-prefix: github-
stg_git_repos_etl: fivetran
stg_git_repos_fivetran_database: ra-warehouse-dev
stg_git_repos_fivetran_schema: fivetran_github
stg_asana_projects_id-prefix: asana-
stg_asana_projects_etl: stitch
stg_asana_projects_stitch_schema: stitch_asana
stg_asana_projects_staff_email_domain: rittmananalytics.com
stg_looker_usage_stats_id-prefix: looker-
stg_looker_usage_schema: fivetran_email
stg_looker_usage_stats_table: usage_stats
stg_gcp_billing_product_usage_id-prefix: gcp-billing-
stg_gcp_billing_product_usage_schema: gcp_billing_export
stg_bigquery_usage_id-prefix: bigquery-log-
stg_bigquery_usage_database: ra-warehouse-dev
stg_bigquery_usage_schema: bigquery_log_export
stg_heap_events_id-prefix: heap-
stg_heap_events_etl: heap
stg_heap_events_heap_schema: heap
stg_heap_events_site: rittmananalytics.com
enable_event_type_mapping: true
stg_snowplow_events_snowplow_schema: snowplow_events
stg_appsflyer_events_stitch_schema: stitch_appsflyer
stg_shopify_ecommerce_id-prefix: shopify-
stg_shopify_ecommerce_etl: fivetran
stg_shopify_ecommerce_fivetran_schema: shopify
stg_stripe_subscriptions_id-prefix: stripe-
stg_stripe_subscriptions_etl: stitch
stg_stripe_subscriptions_segment_schema: stripe
stg_stripe_subscriptions_stitch_schema: stripe
web_sessionization_trailing_window: 3
web_inactivity_cutoff: 30 * 60
attribution_create_account_event_type: contact_us_pressed
attribution_conversion_event_type: order_checkout
attribution_order_id_column: event_text
attribution_orders_event_table: order_checkout
attribution_lookback_days_window : 30
attribution_time_decay_days_window : 7
attribution_include_conversion_session: true
# enable_companies_merge_file: true
# enable_ip_geo_enrichment: false
facebook_ads_schema: fivetran_facebook_ads
facebook_ads_database: ra-warehouse-dev
google_ads_api_source: adwords
google_ads__url_passthrough_metrics: []
google_ads__criteria_passthrough_metrics: []
google_ads__ad_stats_passthrough_metrics: []
google_ads_schema: fivetran_google_ads
google_ads_database: ra-warehouse-dev
ad_reporting__facebook_ads_schema: staging
ad_reporting__facebook_ads_database: ra-warehouse-dev
ad_reporting__google_ads_schema: staging
ad_reporting__google_ads_database: ra-warehouse-dev
# shopify:
# shopify_customer: "{{ ref('stg_shopify__customer') }}"
# shopify_order: "{{ ref('stg_shopify__order') }}"
# shopify_order_line: "{{ ref('stg_shopify__order_line') }}"
# shopify_order_line_refund: "{{ ref('stg_shopify__order_line_refund') }}"
# shopify_product: "{{ ref('stg_shopify__product') }}"
# shopify_product_variant: "{{ ref('stg_shopify__product_variant') }}"
# shopify_transaction: "{{ ref('stg_shopify__transaction') }}"
# shopify_refund: "{{ ref('stg_shopify__refund') }}"
# shopify_order_adjustment: "{{ ref('stg_shopify__order_adjustment') }}"
# shopify_source:
# order_line_refund_source: "{{ source('shopify','order_line_refund') }}"
# order_line_source: "{{ source('shopify','order_line') }}"
# order_source: "{{ source('shopify','order') }}"
# product_source: "{{ source('shopify','product') }}"
# product_variant_source: "{{ source('shopify','product_variant') }}"
# transaction_source: "{{ source('shopify','transaction') }}"
# refund_source: "{{ source('shopify','refund') }}"
# order_adjustment_source: "{{ source('shopify','order_adjustment') }}"
# customer_pass_through_columns: []
# order_line_refund_pass_through_columns: []
# order_line_pass_through_columns: []
# order_pass_through_columns: []
# product_pass_through_columns: []
# product_variant_pass_through_columns: []
# transaction_pass_through_columns: []
stg_facebook_ads_source:
account_history: "{{ source('facebook_ads','account_history') }}"
ad_history: "{{ source('facebook_ads','ad_history') }}"
ad_set_history: "{{ source('facebook_ads','ad_set_history') }}"
basic_ad: "{{ source('facebook_ads','basic_ad') }}"
campaign_history: "{{ source('facebook_ads','campaign_history') }}"
creative_history: "{{ source('facebook_ads','creative_history') }}"
facebook_ads_schema: fivetran_facebook_ads
facebook_ads_database: ra-warehouse-dev
dbt_utils_dispatch_list: [spark_utils, fivetran_utils]
fivetran_utils_dispatch_list: [spark_utils]