Feature/unstructured data

.buildkite/pipeline.yml

@@ -58,7 +58,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
 
-  - label: ":bricks: Run Tests - Databricks"
+  - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:

CHANGELOG.md

@@ -1,3 +1,14 @@
+# dbt_zendesk_source v0.12.0
+[PR #49](https://github.com/fivetran/dbt_zendesk_source/pull/49) includes the following updates:
+## Breaking changes
+- Added field `_fivetran_deleted` to the following models for use downstream:
+  - `stg_zendesk__ticket`
+  - `stg_zendesk__ticket_comment`
+  - `stg_zendesk__user`
+
+## Documentation
+- Updated documentation to include `_fivetran_deleted`.
+
 # dbt_zendesk_source v0.11.2
 
 [PR #49](https://github.com/fivetran/dbt_zendesk_source/pull/49) includes the following updates:

README.md

@@ -44,7 +44,7 @@ Include the following zendesk_source package version in your `packages.yml` file
 ```yaml
 packages:
   - package: fivetran/zendesk_source
-    version: [">=0.11.0", "<0.12.0"]
+    version: [">=0.12.0", "<0.13.0"]
 ```
 ## Step 3: Define database and schema variables
 By default, this package runs using your target database and the `zendesk` schema. If this is not where your Zendesk Support data is (for example, if your zendesk schema is named `zendesk_fivetran`), add the following configuration to your root `dbt_project.yml` file:

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'zendesk_source'
-version: '0.11.2'
+version: '0.12.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   zendesk_source:

integration_tests/dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'zendesk_source_integration_tests'
-version: '0.11.2'
+version: '0.12.0'
 
 profile: 'integration_tests'
 

macros/get_ticket_columns.sql

@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()},
     {"name": "allow_channelback", "datatype": "boolean"},
     {"name": "assignee_id", "datatype": dbt.type_int()},
     {"name": "brand_id", "datatype": dbt.type_int()},

macros/get_ticket_comment_columns.sql

@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_string()},
+    {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()},
     {"name": "body", "datatype": dbt.type_string()},
     {"name": "call_duration", "datatype": dbt.type_int()},
     {"name": "call_id", "datatype": dbt.type_int()},

macros/get_user_columns.sql

@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()},
     {"name": "active", "datatype": "boolean"},
     {"name": "alias", "datatype": dbt.type_string()},
     {"name": "authenticity_token", "datatype": dbt.type_int()},

models/src_zendesk.yml

@@ -70,6 +70,8 @@ sources:
             description: The address of the source the ticket was created from
           - name: via_source_to_name
             description: The name of the source the ticket was created from    
+          - name: _fivetran_deleted
+            description: Boolean created by Fivetran to indicate whether the record has been deleted.
 
       - name: brand
         identifier: "{{ var('zendesk_brand_identifier', 'brand')}}"
@@ -190,6 +192,8 @@ sources:
             description: Boolean field indicating if the comment is a twitter tweet
           - name: voice_comment
             description: Boolean field indicating if the comment is a voice comment
+          - name: _fivetran_deleted
+            description: Boolean created by Fivetran to indicate whether the record has been deleted.
 
       - name: user_tag
         identifier: "{{ var('zendesk_user_tag_identifier', 'user_tag') }}"
@@ -237,6 +241,8 @@ sources:
             description: Last time the user signed in to Zendesk Support or made an API request using an API token or basic authentication.
           - name: locale 
             description: The user's locale. A BCP-47 compliant tag for the locale. If both "locale" and "locale_id" are present on create or update, "locale_id" is ignored and only "locale" is used.
+          - name: _fivetran_deleted
+            description: Boolean created by Fivetran to indicate whether the record has been deleted.
 
       - name: schedule
         identifier: "{{ var('zendesk_schedule_identifier', 'schedule') }}"

models/stg_zendesk.yml

@@ -59,6 +59,8 @@ models:
         description: The address of the source the ticket was created from
       - name: source_to_name
         description: The name of the source the ticket was created from  
+      - name: _fivetran_deleted
+        description: Boolean created by Fivetran to indicate whether the record has been deleted.
 
   - name: stg_zendesk__brand
     description: >
@@ -155,6 +157,8 @@ models:
         description: Boolean field indicating if the comment is a twitter tweet
       - name: is_voice_comment
         description: Boolean field indicating if the comment is a voice comment
+      - name: _fivetran_deleted
+        description: Boolean created by Fivetran to indicate whether the record has been deleted.
 
   - name: stg_zendesk__user_tag
     description: Table containing all tags associated with a user. Only present if your account has user tagging enabled.
@@ -200,6 +204,8 @@ models:
         description: Last time the user signed in to Zendesk Support or made an API request using an API token or basic authentication.
       - name: locale 
         description: The user's locale. A BCP-47 compliant tag for the locale. If both "locale" and "locale_id" are present on create or update, "locale_id" is ignored and only "locale" is used.
+      - name: _fivetran_deleted
+        description: Boolean created by Fivetran to indicate whether the record has been deleted.
 
   - name: stg_zendesk__schedule
     description: The support schedules created with different business hours and holidays. 

models/stg_zendesk__ticket.sql

@@ -30,6 +30,7 @@ final as (
     select 
         id as ticket_id,
         _fivetran_synced,
+        _fivetran_deleted,
         assignee_id,
         brand_id,
         cast(created_at as {{ dbt.type_timestamp() }}) as created_at,

models/stg_zendesk__ticket_comment.sql

@@ -30,6 +30,7 @@ final as (
     select 
         id as ticket_comment_id,
         _fivetran_synced,
+        _fivetran_deleted,
         body,
         cast(created as {{ dbt.type_timestamp() }}) as created_at,
         public as is_public,

models/stg_zendesk__user.sql

@@ -31,6 +31,7 @@ final as (
         id as user_id,
         external_id,
         _fivetran_synced,
+        _fivetran_deleted,
         cast(last_login_at as {{ dbt.type_timestamp() }}) as last_login_at,
         cast(created_at as {{ dbt.type_timestamp() }}) as created_at,
         cast(updated_at as {{ dbt.type_timestamp() }}) as updated_at,