expand LEAD default columns

CHANGELOG.md

@@ -1,3 +1,39 @@
+# dbt_marketo_source v0.10.1
+[PR #35](https://github.com/fivetran/dbt_marketo_source/pull/35) includes the following updates:
+
+## Feature Updates
+- Ensures that `stg_marketo__lead` has and documents the below columns, all [standard](https://developers.marketo.com/rest-api/lead-database/fields/list-of-standard-fields/) fields from Marketo. Previously, peristed all fields found in your `LEAD` source table but only _ensured_ that the `id`, `created_at`, `updated_at`, `email`, `first_name`, `last_name`, and `_fivetran_synced` fields were included. If your `LEAD` table contains the following fields, nothing will change for you. If any of the following default columns are missing from your `LEAD` table, `stg_marketo__lead` will create a NULL version with the proper data type:
+  - `phone`
+  - `main_phone`
+  - `mobile_phone`
+  - `company`
+  - `inferred_company`
+  - `address_lead`
+  - `address`
+  - `city`
+  - `state`
+  - `state_code`
+  - `country`
+  - `country_code`
+  - `postal_code`
+  - `billing_street`
+  - `billing_city`
+  - `billing_state`
+  - `billing_state_code`
+  - `billing_country`
+  - `billing_country_code`
+  - `billing_postal_code`
+  - `inferred_city`
+  - `inferred_state_region`
+  - `inferred_country`
+  - `inferred_postal_code`
+  - `inferred_phone_area_code`
+  - `anonymous_ip`
+  - `unsubscribed`
+  - `email_invalid`
+  - `do_not_call`
+> Note: the above fields will persist downstream into the [transform](https://github.com/fivetran/dbt_marketo/blob/main/models/marketo__leads.sql) `marketo__leads` model.
+
 # dbt_marketo_source v0.10.0
 ## 🚨 Breaking Changes 🚨:
 [PR #33](https://github.com/fivetran/dbt_marketo_source/pull/33) includes the following updates in connection with the Fivetran Marketo connector's [June 2023](https://fivetran.com/docs/applications/marketo/changelog#june2023) and [May 2023](https://fivetran.com/docs/applications/marketo/changelog#may2023) releases:

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'marketo_source'
-version: '0.10.0'
+version: '0.10.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'marketo_source_integration_tests'
-version: '0.10.0'
+version: '0.10.1'
 profile: 'integration_tests'
 config-version: 2
 vars:

macros/get_lead_columns.sql

@@ -7,9 +7,38 @@
     {"name": "first_name", "datatype": dbt.type_string()},
     {"name": "id", "datatype": dbt.type_int(), "alias": "lead_id"},
     {"name": "last_name", "datatype": dbt.type_string()},
-    {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"}
+    {"name": "updated_at", "datatype": dbt.type_timestamp(), "alias": "updated_timestamp"},
+    {"name": "phone", "datatype": dbt.type_string()},
+    {"name": "main_phone", "datatype": dbt.type_string()},
+    {"name": "mobile_phone", "datatype": dbt.type_string()},
+    {"name": "company", "datatype": dbt.type_string()},
+    {"name": "inferred_company", "datatype": dbt.type_string()},
+    {"name": "address_lead", "datatype": dbt.type_string()},
+    {"name": "address", "datatype": dbt.type_string()},
+    {"name": "city", "datatype": dbt.type_string()},
+    {"name": "state", "datatype": dbt.type_string()},
+    {"name": "state_code", "datatype": dbt.type_string()},
+    {"name": "country", "datatype": dbt.type_string()},
+    {"name": "country_code", "datatype": dbt.type_string()},
+    {"name": "postal_code", "datatype": dbt.type_string()},
+    {"name": "billing_street", "datatype": dbt.type_string()},
+    {"name": "billing_city", "datatype": dbt.type_string()},
+    {"name": "billing_state", "datatype": dbt.type_string()},
+    {"name": "billing_state_code", "datatype": dbt.type_string()},
+    {"name": "billing_country", "datatype": dbt.type_string()},
+    {"name": "billing_country_code", "datatype": dbt.type_string()},
+    {"name": "billing_postal_code", "datatype": dbt.type_string()},
+    {"name": "inferred_city", "datatype": dbt.type_string()},
+    {"name": "inferred_state_region", "datatype": dbt.type_string()},
+    {"name": "inferred_country", "datatype": dbt.type_string()},
+    {"name": "inferred_postal_code", "datatype": dbt.type_string()},
+    {"name": "inferred_phone_area_code", "datatype": dbt.type_string()},
+    {"name": "anonymous_ip", "datatype": dbt.type_string()},
+    {"name": "unsubscribed", "datatype": dbt.type_boolean(), "alias": "is_unsubscribed"},
+    {"name": "email_invalid", "datatype": dbt.type_boolean(), "alias": "is_email_invalid"},
+    {"name": "do_not_call", "datatype": dbt.type_boolean()}
 ] %}
 
 {{ return(columns) }}
 
-{% endmacro %}
+{% endmacro %}

models/src_marketo.yml

@@ -28,6 +28,93 @@ sources:
           - name: last_name
             description: The last name of the lead.
 
+          - name: phone
+            description: Lead’s Phone Number.
+
+          - name: main_phone
+            description: Primary phone number of the lead’s company
+
+          - name: mobile_phone
+            description: Lead’s mobile phone number
+
+          - name: company
+            description: Lead’s company name
+
+          - name: inferred_company
+            description: Company name inferred by reverse IP lookup of the lead’s first recorded web visit
+
+          - name: address_lead
+            description: Second address associated with the lead.
+
+          - name: address
+            description: Lead’s address. 
+
+          - name: city
+            description: Lead’s city
+
+          - name: state
+            description: Lead’s state
+
+          - name: state_code
+            description: Alpha-2 code of the Lead’s state
+
+          - name: country
+            description: Lead’s country
+
+          - name: country_code
+            description: Alpha-2 code of the Lead’s country
+
+          - name: postal_code
+            description: Lead’s postal code
+
+          - name: billing_street
+            description: Billing street address of the lead’s company
+
+          - name: billing_city
+            description: City of the lead’s billing address
+
+          - name: billing_state
+            description: State or province of the lead’s billing address
+
+          - name: billing_state_code
+            description: Alpha-2 code of the state or province of the lead’s billing address
+
+          - name: billing_country
+            description: Country of the lead’s billing address
+
+          - name: billing_country_code
+            description: Alpha-2 code of the country of the lead’s billing address
+
+          - name: billing_postal_code
+            description: Postal code of the lead’s billing address
+
+          - name: inferred_city
+            description: Lead’s city inferred by reverse IP lookup of lead’s first recorded web visit.
+
+          - name: inferred_state_region
+            description: Lead’s state region inferred by reverse IP lookup of lead’s first recorded web visit.
+
+          - name: inferred_country
+            description: Country inferred by reverse IP lookup of the lead’s first recorded web visit
+
+          - name: inferred_postal_code
+            description: Lead’s postal code inferred by reverse IP lookup of lead’s first recorded web visit.
+
+          - name: inferred_phone_area_code
+            description: Lead’s phone area code inferred by reverse IP lookup of lead’s first recorded web visit.
+
+          - name: anonymous_ip
+            description: IP address of the lead’s first recorded web visit
+
+          - name: unsubscribed
+            description: Lead’s email-unsubscribed status (boolean). Partially system managed. Will prevent receipt of non-operational emails if set to true.
+
+          - name: email_invalid
+            description: Email invalid status (boolean). All emails to the address will be blocked if set to true. Bounces indicating that the email is invalid will automatically set this field to true.
+
+          - name: do_not_call
+            description: Lead’s do-not-call preference (boolean)
+
       - name: lead_describe
         identifier: "{{ var('marketo_lead_describe_identifier', 'lead_describe')}}"
         description: Table with metadata about lead object columns

models/stg_marketo.yml

@@ -595,6 +595,93 @@ models:
       - name: last_name
         description: The last name of the lead.
 
+      - name: phone
+        description: Lead’s Phone Number.
+
+      - name: main_phone
+        description: Primary phone number of the lead’s company
+
+      - name: mobile_phone
+        description: Lead’s mobile phone number
+
+      - name: company
+        description: Lead’s company name
+
+      - name: inferred_company
+        description: Company name inferred by reverse IP lookup of the lead’s first recorded web visit
+
+      - name: address_lead
+        description: Second address associated with the lead.
+
+      - name: address
+        description: Lead’s address. 
+
+      - name: city
+        description: Lead’s city
+
+      - name: state
+        description: Lead’s state
+
+      - name: state_code
+        description: Alpha-2 code of the Lead’s state
+
+      - name: country
+        description: Lead’s country
+
+      - name: country_code
+        description: Alpha-2 code of the Lead’s country
+
+      - name: postal_code
+        description: Lead’s postal code
+
+      - name: billing_street
+        description: Billing street address of the lead’s company
+
+      - name: billing_city
+        description: City of the lead’s billing address
+
+      - name: billing_state
+        description: State or province of the lead’s billing address
+
+      - name: billing_state_code
+        description: Alpha-2 code of the state or province of the lead’s billing address
+
+      - name: billing_country
+        description: Country of the lead’s billing address
+
+      - name: billing_country_code
+        description: Alpha-2 code of the country of the lead’s billing address
+
+      - name: billing_postal_code
+        description: Postal code of the lead’s billing address
+
+      - name: inferred_city
+        description: Lead’s city inferred by reverse IP lookup of lead’s first recorded web visit.
+
+      - name: inferred_state_region
+        description: Lead’s state region inferred by reverse IP lookup of lead’s first recorded web visit.
+
+      - name: inferred_country
+        description: Country inferred by reverse IP lookup of the lead’s first recorded web visit
+
+      - name: inferred_postal_code
+        description: Lead’s postal code inferred by reverse IP lookup of lead’s first recorded web visit.
+
+      - name: inferred_phone_area_code
+        description: Lead’s phone area code inferred by reverse IP lookup of lead’s first recorded web visit.
+
+      - name: anonymous_ip
+        description: IP address of the lead’s first recorded web visit
+
+      - name: is_unsubscribed
+        description: Lead’s email-unsubscribed status (boolean). Partially system managed. Will prevent receipt of non-operational emails if set to true.
+
+      - name: is_email_invalid
+        description: Email invalid status (boolean). All emails to the address will be blocked if set to true. Bounces indicating that the email is invalid will automatically set this field to true.
+
+      - name: do_not_call
+        description: Lead’s do-not-call preference (boolean)
+
   - name: stg_marketo__program
     columns:
       - name: program_id

models/stg_marketo__lead.sql

@@ -13,7 +13,12 @@ with base as (
             )
         }}
         -- This will check if there are non-default columns to bring in
-        {% set default_cols = ['id', 'created_at', 'updated_at', 'email', 'first_name', 'last_name', '_fivetran_synced'] %}
+        {% set default_cols = ['id', 'created_at', 'updated_at', 'email', 'first_name', 'last_name', '_fivetran_synced',
+        'phone', 'main_phone', 'mobile_phone', 'company', 'inferred_company', 'address_lead', 'address', 'city', 'state',
+        'state_code', 'country', 'country_code', 'postal_code', 'billing_street', 'billing_city', 'billing_state', 
+        'billing_state_code', 'billing_country', 'billing_country_code', 'billing_postal_code', 'inferred_city', 'inferred_state_region', 
+        'inferred_country', 'inferred_postal_code', 'inferred_phone_area_code', 'anonymous_ip', 'unsubscribed', 'email_invalid', 'do_not_call'] %}
+
         {% set new_cols = dbt_utils.star(from=ref('stg_marketo__lead_tmp'), except=default_cols) %}
         {% if new_cols != '/* no columns returned from star() macro */' %}
             ,{{ new_cols }}