From c78a341634bfd815e09a60c9d118879bb7cd4a35 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Thu, 2 Nov 2023 09:20:26 -0500 Subject: [PATCH 1/8] documentation/basic-ad-update --- .github/workflows/add-2nd-reviewer.yml | 10 ++++++++++ .github/workflows/auto-release.yml | 13 +++++++++++++ CHANGELOG.md | 6 ++++++ README.md | 12 ++++-------- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- integration_tests/requirements.txt | 4 +++- 7 files changed, 38 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/add-2nd-reviewer.yml create mode 100644 .github/workflows/auto-release.yml diff --git a/.github/workflows/add-2nd-reviewer.yml b/.github/workflows/add-2nd-reviewer.yml new file mode 100644 index 0000000..392af8f --- /dev/null +++ b/.github/workflows/add-2nd-reviewer.yml @@ -0,0 +1,10 @@ +name: '2nd reviewer' +on: + pull_request_review: + types: [submitted] + +jobs: + auto_assign_reviewer: + if: github.event.review.state == 'approved' + uses: fivetran/dbt_package_automations/.github/workflows/add-2nd-reviewer.yml@feature/auto-releaser + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 0000000..d073ebf --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,13 @@ +name: 'auto release' +on: + pull_request: + types: + - closed + branches: + - main + +jobs: + call-workflow-passing-data: + if: github.event.pull_request.merged + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@feature/auto-releaser + secrets: inherit \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a13cb0..337a3a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# dbt_facebook_ads_source v0.7.1 + +[PR #]() includes the following udpates: +## Documentation Updates +- The prerequisite steps for generating the `basic_ad` pre-built report have been modified to reflect the current state of the Facebook Ads connector. + # dbt_facebook_ads_source v0.7.0 [PR #28](https://github.com/fivetran/dbt_facebook_ads_source/pull/28) includes the following updates: ## Feature update 🎉 diff --git a/README.md b/README.md index f862559..ea1d7ac 100644 --- a/README.md +++ b/README.md @@ -29,14 +29,10 @@ To use this dbt package, you must have the following: - At least one Fivetran Facebook Ads connector syncing data into your destination. - A **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination. -- You will need to configure your Facebook Ads connector to pull the `BASIC_AD` pre-built report. Follow the below steps in the Fivetran UI to do so: - 1. Navigate to the connector setup form (**Setup** -> **Edit connection details** for pre-existing connectors) - 2. Click **Add table** - 3. Select **Pre-built Report** - 4. Set the table name to `basic_ad` - 5. Select `BASIC_AD` as the corresponding pre-built report - 6. Select a daily aggregation period - 7. Click **Ok** and **Save & test**! +- You will need to configure your Facebook Ads connector to pull the `basic_ad` pre-built report. This pre-built report should be enabled in your connector by default. However, to confirm this pre-built report is actively syncing you may perform the following steps: + 1. Navigate to the connector schema tab. + 2. Search for `basic_ad` and confirm it is selected. + 3. If not selected, do so and sync. If already selected you are ready to run the models! ### Databricks Dispatch Configuration If you are using a Databricks destination with this package you will need to add the below (or a variation of the below) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively. diff --git a/dbt_project.yml b/dbt_project.yml index 6191663..608b5b5 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'facebook_ads_source' -version: '0.7.0' +version: '0.7.1' config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] models: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 12a8369..4f37f0d 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'facebook_ads_source_integration_tests_2' -version: '0.7.0' +version: '0.7.1' profile: 'integration_tests' config-version: 2 diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 87ae0f9..b7702d6 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,4 +4,6 @@ dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 \ No newline at end of file +dbt-databricks>=1.3.0,<2.0.0 + +oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437 \ No newline at end of file From 6f51f38452ba4931275a10c0e749bb544e2f99bd Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Thu, 2 Nov 2023 09:26:57 -0500 Subject: [PATCH 2/8] decisionlog update --- CHANGELOG.md | 3 ++- DECISIONLOG.md | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 DECISIONLOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 337a3a2..56a1778 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ # dbt_facebook_ads_source v0.7.1 -[PR #]() includes the following udpates: +[PR #31](https://github.com/fivetran/dbt_facebook_ads_source/pull/31) includes the following updates: ## Documentation Updates - The prerequisite steps for generating the `basic_ad` pre-built report have been modified to reflect the current state of the Facebook Ads connector. +- Adds the [DECISIONLOG](DECISIONLOG.md) to clarify why there exist differences among aggregations across different grains. # dbt_facebook_ads_source v0.7.0 [PR #28](https://github.com/fivetran/dbt_facebook_ads_source/pull/28) includes the following updates: diff --git a/DECISIONLOG.md b/DECISIONLOG.md new file mode 100644 index 0000000..f8c06af --- /dev/null +++ b/DECISIONLOG.md @@ -0,0 +1,4 @@ +## Why don't metrics add up across different grains (Ex. ad level vs campaign level)? +Not all ads are served at the ad level. In other words, there are some ads that are served only at the ad group, campaign, etc. levels. The implications are that since not ads are included in the ad-level report, their associated spend, for example, won't be included at that grain. Therefore your spend totals may differ across the ad grain and another grain. + +This is a reason why we have broken out the ad reporting packages into separate hierarchical end models (Ad, Ad Group, Campaign, and more). Because if we only used ad-level reports, we could be missing data. \ No newline at end of file From 4b9e223b7226e0f94069cb28560fd1c6df18931c Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Thu, 2 Nov 2023 09:32:40 -0500 Subject: [PATCH 3/8] auto releaser branch removal --- .github/workflows/add-2nd-reviewer.yml | 2 +- .github/workflows/auto-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/add-2nd-reviewer.yml b/.github/workflows/add-2nd-reviewer.yml index 392af8f..64ba3ee 100644 --- a/.github/workflows/add-2nd-reviewer.yml +++ b/.github/workflows/add-2nd-reviewer.yml @@ -6,5 +6,5 @@ on: jobs: auto_assign_reviewer: if: github.event.review.state == 'approved' - uses: fivetran/dbt_package_automations/.github/workflows/add-2nd-reviewer.yml@feature/auto-releaser + uses: fivetran/dbt_package_automations/.github/workflows/add-2nd-reviewer.yml secrets: inherit \ No newline at end of file diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index d073ebf..733c599 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -9,5 +9,5 @@ on: jobs: call-workflow-passing-data: if: github.event.pull_request.merged - uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@feature/auto-releaser + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml secrets: inherit \ No newline at end of file From c9d0d8f97d62d04c112f1badb0a94f465c9b3882 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Thu, 2 Nov 2023 10:47:32 -0500 Subject: [PATCH 4/8] main direct link --- .github/workflows/add-2nd-reviewer.yml | 2 +- .github/workflows/auto-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/add-2nd-reviewer.yml b/.github/workflows/add-2nd-reviewer.yml index 64ba3ee..3f056aa 100644 --- a/.github/workflows/add-2nd-reviewer.yml +++ b/.github/workflows/add-2nd-reviewer.yml @@ -6,5 +6,5 @@ on: jobs: auto_assign_reviewer: if: github.event.review.state == 'approved' - uses: fivetran/dbt_package_automations/.github/workflows/add-2nd-reviewer.yml + uses: fivetran/dbt_package_automations/.github/workflows/add-2nd-reviewer.yml@main secrets: inherit \ No newline at end of file diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 733c599..8ed5853 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -9,5 +9,5 @@ on: jobs: call-workflow-passing-data: if: github.event.pull_request.merged - uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main secrets: inherit \ No newline at end of file From ef56da75191dc7c91a8f38c45b9c3bdb56ec678a Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:54:54 -0600 Subject: [PATCH 5/8] Update requirements.txt --- integration_tests/requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index b7702d6..e615720 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,6 +4,4 @@ dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 - -oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437 \ No newline at end of file +dbt-databricks>=1.6.0,<2.0.0 From ad9d9f5b144cc55cfff1f5105fe1b16a368c3881 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:59:25 -0600 Subject: [PATCH 6/8] Update add-2nd-reviewer.yml --- .github/workflows/add-2nd-reviewer.yml | 84 +++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/.github/workflows/add-2nd-reviewer.yml b/.github/workflows/add-2nd-reviewer.yml index 3f056aa..d3c5487 100644 --- a/.github/workflows/add-2nd-reviewer.yml +++ b/.github/workflows/add-2nd-reviewer.yml @@ -5,6 +5,84 @@ on: jobs: auto_assign_reviewer: - if: github.event.review.state == 'approved' - uses: fivetran/dbt_package_automations/.github/workflows/add-2nd-reviewer.yml@main - secrets: inherit \ No newline at end of file + runs-on: ubuntu-latest + + steps: + - name: Check if PR needs second reviewer + id: check_needs_second_reviewer + run: | + # Get the PR Number + PR_NUMBER=$(jq -r ".pull_request.number" "$GITHUB_EVENT_PATH") + echo "PR Number: $PR_NUMBER" + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + + # Get list of current reviewers + REVIEWERS_COUNT=$(curl -s -H \ + -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER/requested_reviewers" | jq '.users | length') + echo "No. of Requested Reviewers: $REVIEWERS_COUNT" + + # Determine if two reviewers already exist + if [[ "$REVIEWERS_COUNT" -gt 1 ]]; then + NEEDS_SECOND_REVIEWER="false" + echo "2nd Reviewer already assigned." + exit 1 # This will cause the job to fail and skip subsequent steps. + + else + NEEDS_SECOND_REVIEWER="true" + fi + + echo "Adding 2nd Reviewer: $NEEDS_SECOND_REVIEWER" + echo "needs_second_reviewer=$NEEDS_SECOND_REVIEWER" >> $GITHUB_OUTPUT + + - name: Check Related Repository PR Reviewers + id: check_related_repository + run: | + # Get the current repository name + CURRENT_REPO_NAME="${{ github.repository }}" + + # Check if the current repository name contains 'source' + if [[ $CURRENT_REPO_NAME == *source* ]]; then + # If 'source' is in the name, remove it from the related repository name + RELATED_REPO=$(echo "$CURRENT_REPO_NAME" | sed 's/_source//') + else + # If 'source' is not in the name, add it to the related repository name + RELATED_REPO="${CURRENT_REPO_NAME}_source" + fi + + echo "Related Repository: $RELATED_REPO" + echo "related_repo=$RELATED_REPO" >> $GITHUB_OUTPUT + + # Calculate the start date for the past week (7 days ago). + START_DATE=$(date -I -d "7 days ago") + + # Make an API call to retrieve approved PRs from the related repository in the past week. + PR_INFO=$(curl -s \ + -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/$RELATED_REPO/pulls?state=open&sort=updated&direction=desc&since=$START_DATE" | \ + jq "map(select(.requested_reviewers | length >= 2)) | .[0]") + + if [[ -n "$PR_INFO" ]]; then + SECOND_REVIEWER=$(echo "$PR_INFO" | jq -r '.requested_reviewers[1].login') + else + SECOND_REVIEWER="dbt" + fi + + echo "2nd Reviewer set as: $SECOND_REVIEWER" + echo "second_reviewer=$SECOND_REVIEWER" >> $GITHUB_OUTPUT + + - name: Assign Second Reviewer + if: steps.check_needs_second_reviewer.outputs.needs_second_reviewer == 'true' + run: | + PR_NUMBER=${{ steps.check_needs_second_reviewer.outputs.pr_number }} + SECOND_REVIEWER={{ steps.check_needs_second_reviewer.outputs.second_reviewer }} + RESPONSE=$(curl -L \ + -X POST \ + -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER/requested_reviewers \ + -d "{\"team_reviewers\":[\"$SECOND_REVIEWER\"]}" + ) + echo "$RESPONSE" From 13dd8372f421f331ba5d6142fa1cdbd9b6f0e7e5 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 6 Nov 2023 18:02:25 -0600 Subject: [PATCH 7/8] Update add-2nd-reviewer.yml --- .github/workflows/add-2nd-reviewer.yml | 84 +------------------------- 1 file changed, 3 insertions(+), 81 deletions(-) diff --git a/.github/workflows/add-2nd-reviewer.yml b/.github/workflows/add-2nd-reviewer.yml index d3c5487..ea8ef13 100644 --- a/.github/workflows/add-2nd-reviewer.yml +++ b/.github/workflows/add-2nd-reviewer.yml @@ -5,84 +5,6 @@ on: jobs: auto_assign_reviewer: - runs-on: ubuntu-latest - - steps: - - name: Check if PR needs second reviewer - id: check_needs_second_reviewer - run: | - # Get the PR Number - PR_NUMBER=$(jq -r ".pull_request.number" "$GITHUB_EVENT_PATH") - echo "PR Number: $PR_NUMBER" - echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT - - # Get list of current reviewers - REVIEWERS_COUNT=$(curl -s -H \ - -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" \ - -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER/requested_reviewers" | jq '.users | length') - echo "No. of Requested Reviewers: $REVIEWERS_COUNT" - - # Determine if two reviewers already exist - if [[ "$REVIEWERS_COUNT" -gt 1 ]]; then - NEEDS_SECOND_REVIEWER="false" - echo "2nd Reviewer already assigned." - exit 1 # This will cause the job to fail and skip subsequent steps. - - else - NEEDS_SECOND_REVIEWER="true" - fi - - echo "Adding 2nd Reviewer: $NEEDS_SECOND_REVIEWER" - echo "needs_second_reviewer=$NEEDS_SECOND_REVIEWER" >> $GITHUB_OUTPUT - - - name: Check Related Repository PR Reviewers - id: check_related_repository - run: | - # Get the current repository name - CURRENT_REPO_NAME="${{ github.repository }}" - - # Check if the current repository name contains 'source' - if [[ $CURRENT_REPO_NAME == *source* ]]; then - # If 'source' is in the name, remove it from the related repository name - RELATED_REPO=$(echo "$CURRENT_REPO_NAME" | sed 's/_source//') - else - # If 'source' is not in the name, add it to the related repository name - RELATED_REPO="${CURRENT_REPO_NAME}_source" - fi - - echo "Related Repository: $RELATED_REPO" - echo "related_repo=$RELATED_REPO" >> $GITHUB_OUTPUT - - # Calculate the start date for the past week (7 days ago). - START_DATE=$(date -I -d "7 days ago") - - # Make an API call to retrieve approved PRs from the related repository in the past week. - PR_INFO=$(curl -s \ - -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" \ - -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/$RELATED_REPO/pulls?state=open&sort=updated&direction=desc&since=$START_DATE" | \ - jq "map(select(.requested_reviewers | length >= 2)) | .[0]") - - if [[ -n "$PR_INFO" ]]; then - SECOND_REVIEWER=$(echo "$PR_INFO" | jq -r '.requested_reviewers[1].login') - else - SECOND_REVIEWER="dbt" - fi - - echo "2nd Reviewer set as: $SECOND_REVIEWER" - echo "second_reviewer=$SECOND_REVIEWER" >> $GITHUB_OUTPUT - - - name: Assign Second Reviewer - if: steps.check_needs_second_reviewer.outputs.needs_second_reviewer == 'true' - run: | - PR_NUMBER=${{ steps.check_needs_second_reviewer.outputs.pr_number }} - SECOND_REVIEWER={{ steps.check_needs_second_reviewer.outputs.second_reviewer }} - RESPONSE=$(curl -L \ - -X POST \ - -H "Authorization: Bearer ${{ secrets.GH_TOKEN }}" \ - -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER/requested_reviewers \ - -d "{\"team_reviewers\":[\"$SECOND_REVIEWER\"]}" - ) - echo "$RESPONSE" + if: github.event.review.state == 'approved' + uses: fivetran/dbt_package_automations/.github/workflows/add-2nd-reviewer.yml@main + secrets: inherit From 0700e832d474aa9d6b3c4c12b4b4a06cba732bb3 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 7 Nov 2023 15:47:38 -0600 Subject: [PATCH 8/8] changelog readme link --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56a1778..28dd114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ [PR #31](https://github.com/fivetran/dbt_facebook_ads_source/pull/31) includes the following updates: ## Documentation Updates -- The prerequisite steps for generating the `basic_ad` pre-built report have been modified to reflect the current state of the Facebook Ads connector. +- The [prerequisite steps in the README](https://github.com/fivetran/dbt_facebook_ads_source#step-1-prerequisites) for generating the `basic_ad` pre-built report have been modified to reflect the current state of the Facebook Ads connector. - Adds the [DECISIONLOG](DECISIONLOG.md) to clarify why there exist differences among aggregations across different grains. # dbt_facebook_ads_source v0.7.0