From 1d8a1827548f8e6017a03e6faad559cac3ae1210 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 12 Apr 2024 11:43:16 -0500 Subject: [PATCH 1/5] debugging all the values --- .github/workflows/ci_dbt_core_testing.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/ci_dbt_core_testing.yml b/.github/workflows/ci_dbt_core_testing.yml index c5e04350..cb8ac9de 100644 --- a/.github/workflows/ci_dbt_core_testing.yml +++ b/.github/workflows/ci_dbt_core_testing.yml @@ -94,6 +94,7 @@ jobs: if [[ -z "${{ inputs.dbt-common-ref }}" ]]; then # these will be commits instead of branches if [[ "${{ github.event_name }}" == "merge_group" ]]; then + # TODO: what sha should this be?? REF=${{ github.event.merge_group.head_sha }} else REF=${{ github.event.pull_request.base.sha }} @@ -105,8 +106,28 @@ jobs: - name: "Final Refs" run: | + # variables set echo "dbt-core-ref=${{ steps.core-ref.outputs.ref }}" echo "dbt-common-ref=${{ steps.common-ref.outputs.ref }}" + # debugging values for dbt-common merge_group values + echo "merge queue values" + echo "github.event.merge_group.head_sha=${{ github.event.merge_group.head_sha }}" + echo "github.event.merge_group.head_ref=${{ github.event.merge_group.head_ref }}" + echo "github.event.merge_group.base_sha=${{ github.event.merge_group.base_sha }}" + echo "github.event.merge_group.base_ref=${{ github.event.merge_group.base_ref }}" + echo "github.event.merge_group.head_commit.id=${{ github.event.merge_group.head_commit.id }}" + echo "github.event.merge_commit_sha=${{ github.event.merge_commit_sha }}" + echo "GITHUB_SHA=$GITHUB_SHA" + echo "PR values" + echo "github.event.pull_request.base.sha=${{ github.event.pull_request.base.sha }}" + echo "github.event.pull_request.head.sha=${{ github.event.pull_request.head.sha }}" + echo "github.event.pull_request.merge_commit_sha=${{ github.event.pull_request.merge_commit_sha}}" + + # TODO: remove before final changes + - name: "Fail when it's a merge_queue event for debugging" + if: ${{ github.event_name == 'merge_group' }} + run: | + exit 1 dbt-core-unit-test: name: "dbt-core unit tests" From 322429af96137a022cd5e6b22d4fbcd65a8879ae Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 24 Apr 2024 09:21:18 -0500 Subject: [PATCH 2/5] update dependency before running tox --- .github/workflows/ci_dbt_core_testing.yml | 54 ++--------------------- 1 file changed, 4 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ci_dbt_core_testing.yml b/.github/workflows/ci_dbt_core_testing.yml index cb8ac9de..20eff6d0 100644 --- a/.github/workflows/ci_dbt_core_testing.yml +++ b/.github/workflows/ci_dbt_core_testing.yml @@ -94,7 +94,6 @@ jobs: if [[ -z "${{ inputs.dbt-common-ref }}" ]]; then # these will be commits instead of branches if [[ "${{ github.event_name }}" == "merge_group" ]]; then - # TODO: what sha should this be?? REF=${{ github.event.merge_group.head_sha }} else REF=${{ github.event.pull_request.base.sha }} @@ -104,31 +103,6 @@ jobs: fi echo "ref=$REF" >> $GITHUB_OUTPUT - - name: "Final Refs" - run: | - # variables set - echo "dbt-core-ref=${{ steps.core-ref.outputs.ref }}" - echo "dbt-common-ref=${{ steps.common-ref.outputs.ref }}" - # debugging values for dbt-common merge_group values - echo "merge queue values" - echo "github.event.merge_group.head_sha=${{ github.event.merge_group.head_sha }}" - echo "github.event.merge_group.head_ref=${{ github.event.merge_group.head_ref }}" - echo "github.event.merge_group.base_sha=${{ github.event.merge_group.base_sha }}" - echo "github.event.merge_group.base_ref=${{ github.event.merge_group.base_ref }}" - echo "github.event.merge_group.head_commit.id=${{ github.event.merge_group.head_commit.id }}" - echo "github.event.merge_commit_sha=${{ github.event.merge_commit_sha }}" - echo "GITHUB_SHA=$GITHUB_SHA" - echo "PR values" - echo "github.event.pull_request.base.sha=${{ github.event.pull_request.base.sha }}" - echo "github.event.pull_request.head.sha=${{ github.event.pull_request.head.sha }}" - echo "github.event.pull_request.merge_commit_sha=${{ github.event.pull_request.merge_commit_sha}}" - - # TODO: remove before final changes - - name: "Fail when it's a merge_queue event for debugging" - if: ${{ github.event_name == 'merge_group' }} - run: | - exit 1 - dbt-core-unit-test: name: "dbt-core unit tests" needs: [job-prep] @@ -153,19 +127,9 @@ jobs: python -m pip install --upgrade pip python -m pip --version - # tox takes care of installing the correct version of dbt-core dependencies but we need to - # install them first s that we can override the dbt-common branch - - name: "Manually install dbt-core dependencies" + - name: "Update dbt-common dependency to ${{ needs.job-prep.outputs.dbt-common-ref }}" run: | - python -m pip uninstall dbt-common -y - python -m pip install -r dev-requirements.txt -r editable-requirements.txt - - # Since the dbt-common dependency is pinned in dev-requirements.txt we need to force update it - # Since tox installs dependencies but doesn't force update, it won't get overridden in the next - # step since the requirements will already be met - - name: "Force update the version of dbt-common@${{ needs.job-prep.outputs.dbt-common-ref }}" - run: | - python -m pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ needs.job-prep.outputs.dbt-common-ref }} --force-reinstall + ./.github/scripts/update_dev_packages.sh ${{ needs.job-prep.outputs.dbt-common-ref }} - name: "Run unit tests" # Doing the check here instead of the top level because this is job a required check, the @@ -263,19 +227,9 @@ jobs: python -m pip install --upgrade pip python -m pip --version - # tox takes care of installing the correct version of dbt-core dependencies but we need to - # install them first s that we can override the dbt-common branch - - name: "Manually install dbt-core dependencies" - run: | - python -m pip install -r dev-requirements.txt -r editable-requirements.txt - - # Since the dbt-common dependency is pinned in dev-requirements.txt we need to force update it - # Since tox installs dependencies but doesn't force update, it won't get overridden in the next - # step since the requirements will already be met - - name: "Force update the version of dbt-common@${{ needs.job-prep.outputs.dbt-common-ref }}" + - name: "Update dbt-common dependency to ${{ needs.job-prep.outputs.dbt-common-ref }}" run: | - python -m pip uninstall dbt-common -y - python -m pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ needs.job-prep.outputs.dbt-common-ref }} --force-reinstall + ./.github/scripts/update_dev_packages.sh ${{ needs.job-prep.outputs.dbt-common-ref }} - name: "Run Functional tests" run: tox -- --ddtrace From e5ff1e45adf3099e0f1e978edd09183ec84fa927 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 24 Apr 2024 09:44:00 -0500 Subject: [PATCH 3/5] fix path, pass in repo --- .github/workflows/ci_dbt_core_testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_dbt_core_testing.yml b/.github/workflows/ci_dbt_core_testing.yml index 20eff6d0..308dcf20 100644 --- a/.github/workflows/ci_dbt_core_testing.yml +++ b/.github/workflows/ci_dbt_core_testing.yml @@ -129,7 +129,7 @@ jobs: - name: "Update dbt-common dependency to ${{ needs.job-prep.outputs.dbt-common-ref }}" run: | - ./.github/scripts/update_dev_packages.sh ${{ needs.job-prep.outputs.dbt-common-ref }} + ./scripts/update_dev_packages.sh ${{ github.repository.name}} ${{ needs.job-prep.outputs.dbt-common-ref }} - name: "Run unit tests" # Doing the check here instead of the top level because this is job a required check, the @@ -229,7 +229,7 @@ jobs: - name: "Update dbt-common dependency to ${{ needs.job-prep.outputs.dbt-common-ref }}" run: | - ./.github/scripts/update_dev_packages.sh ${{ needs.job-prep.outputs.dbt-common-ref }} + ./scripts/update_dev_packages.sh ${{ github.repository.name}} ${{ needs.job-prep.outputs.dbt-common-ref }} - name: "Run Functional tests" run: tox -- --ddtrace From 2664cf088b8b20ddd3363c64772ffed95ab9fa39 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 24 Apr 2024 09:52:22 -0500 Subject: [PATCH 4/5] put back final ref output --- .github/workflows/ci_dbt_core_testing.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci_dbt_core_testing.yml b/.github/workflows/ci_dbt_core_testing.yml index 308dcf20..fc54e947 100644 --- a/.github/workflows/ci_dbt_core_testing.yml +++ b/.github/workflows/ci_dbt_core_testing.yml @@ -103,6 +103,11 @@ jobs: fi echo "ref=$REF" >> $GITHUB_OUTPUT + - name: "Final Refs" + run: | + echo "dbt-core-ref=${{ steps.core-ref.outputs.ref }}" + echo "dbt-common-ref=${{ steps.common-ref.outputs.ref }}" + dbt-core-unit-test: name: "dbt-core unit tests" needs: [job-prep] From 4f9c9f79b133bd02044db96b06964a97ba0b9923 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 24 Apr 2024 13:59:45 -0500 Subject: [PATCH 5/5] install tox --- .github/workflows/ci_dbt_core_testing.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_dbt_core_testing.yml b/.github/workflows/ci_dbt_core_testing.yml index fc54e947..4beaceb7 100644 --- a/.github/workflows/ci_dbt_core_testing.yml +++ b/.github/workflows/ci_dbt_core_testing.yml @@ -127,10 +127,12 @@ jobs: with: python-version: "3.11" - - name: "Upgrade pip" + - name: "Upgrade pip, install tox" run: | python -m pip install --upgrade pip python -m pip --version + python -m pip install tox + tox --version - name: "Update dbt-common dependency to ${{ needs.job-prep.outputs.dbt-common-ref }}" run: | @@ -227,10 +229,12 @@ jobs: if: runner.os == 'Windows' uses: ./.github/actions/setup-postgres-windows - - name: "Upgrade pip" + - name: "Upgrade pip, install tox" run: | python -m pip install --upgrade pip python -m pip --version + python -m pip install tox + tox --version - name: "Update dbt-common dependency to ${{ needs.job-prep.outputs.dbt-common-ref }}" run: |