Skip to content

Commit e208d2a

Browse files
emmyoopemmyoop
authored
Merge branch 'main' into er/event-testing
2 parents e908c41 + 50555ee commit e208d2a

14 files changed

+467
-18
lines changed

‎.changes/unreleased/Features-20240131-152923.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
kind: Features
2+
body: Cache static objects between tests to accelerate integration runs.
3+
time: 2024-01-31T15:29:23.951857-05:00
4+
custom:
5+
Author: peterallenwebb
6+
Issue: "55"

‎.changes/unreleased/Features-20240201-101851.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
kind: Features
2+
body: Add dataclass utility `Mergeable`
3+
time: 2024-02-01T10:18:51.474231-08:00
4+
custom:
5+
Author: QMalcolm
6+
Issue: "58"

‎.changes/unreleased/Features-20240201-104437.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
kind: Features
2+
body: Added a new InvocationContext class for application context management.
3+
time: 2024-02-01T10:44:37.161298-05:00
4+
custom:
5+
Author: peterallenwebb
6+
Issue: "57"

‎.github/workflows/build.yml

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ on:
1919
merge_group:
2020
types: [checks_requested]
2121
workflow_dispatch:
22+
workflow_call:
2223

2324
permissions: read-all
2425

‎.github/workflows/ci_dbt_core_testing.yml

+263-13
Original file line numberDiff line numberDiff line change
@@ -2,52 +2,302 @@
22
# Runs all tests in dbt-core with this branch of dbt-common to ensure nothing is broken
33

44
# **why?**
5-
# Ensure dbt-common changes do nto break dbt-core
5+
# Ensure dbt-common changes do not break dbt-core
66

77
# **when?**
88
# This will run when trying to merge a PR into main.
99
# It can also be manually triggered.
1010

11-
## TODO: This is a stub. It does nothing right now.
12-
# It will be updated in the future as part of https://github.com/dbt-labs/dbt-common/issues/18
11+
# This workflow can be skipped by adding the "Skip Core Testing" label to the PR. This is
12+
# useful when making a change in both `dbt-core` and `dbt-common` where the changes are dependant
13+
# and cause the other repository to break.
1314

14-
name: Test Against dbt-core
15+
name: "dbt-core Tests"
16+
run-name: >-
17+
${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')
18+
&& format('dbt-core@{0} with dbt-common@{1}', inputs.dbt-core-ref, inputs.dbt-common-ref)
19+
|| 'dbt-core@main with dbt-common branch' }}
1520
1621
on:
1722
merge_group:
1823
types: [checks_requested]
24+
pull_request:
1925
workflow_dispatch:
26+
inputs:
27+
dbt-core-ref:
28+
description: "The branch of dbt-core to test against"
29+
default: "main"
30+
dbt-common-ref:
31+
description: "The branch of dbt-common to test against"
32+
default: "main"
33+
workflow_call:
34+
inputs:
35+
dbt-core-ref:
36+
description: "The branch of dbt-core to test against"
37+
type: string
38+
required: true
39+
default: "main"
40+
dbt-common-ref:
41+
description: "The branch of dbt-common to test against"
42+
type: string
43+
required: true
44+
default: "main"
2045

2146
permissions: read-all
2247

48+
# will cancel previous workflows triggered by the same event
49+
# and for the same ref for PRs/merges or same SHA otherwise
50+
# and for the same inputs on workflow_dispatch or workflow_call
51+
concurrency:
52+
group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(fromJson('["pull_request", "merge_group"]'), github.event_name) && github.event.pull_request.head.ref || github.sha }}-${{ contains(fromJson('["workflow_call", "workflow_dispatch"]'), github.event_name) && github.event.inputs.dbt-core-ref && github.event.inputs.dbt-common-ref || github.sha }}
53+
cancel-in-progress: true
54+
2355
defaults:
2456
run:
2557
shell: bash
2658

59+
# top-level adjustments can be made here
60+
env:
61+
# number of parallel processes to spawn for python integration testing
62+
PYTHON_INTEGRATION_TEST_WORKERS: 5
63+
2764
jobs:
28-
test-dbt-core:
29-
name: "This does nothing right now - always passes"
65+
job-prep:
66+
# This allow us to run the workflow on pull_requests as well so we can always run unit tests
67+
# and only run integration tests on merge for time purposes
68+
name: Setup Repo Refs
69+
runs-on: ubuntu-latest
70+
outputs:
71+
dbt-core-ref: ${{ steps.core-ref.outputs.ref }}
72+
dbt-common-ref: ${{ steps.common-ref.outputs.ref }}
73+
74+
steps:
75+
- name: "Input Refs"
76+
id: job-inputs
77+
run: |
78+
echo "inputs.dbt-core-ref=${{ inputs.dbt-core-ref }}"
79+
echo "inputs.dbt-common-ref=${{ inputs.dbt-common-ref }}"
80+
81+
- name: "Determine dbt-core ref"
82+
id: core-ref
83+
run: |
84+
if [[ -z "${{ inputs.dbt-core-ref }}" ]]; then
85+
REF="main"
86+
else
87+
REF=${{ inputs.dbt-core-ref }}
88+
fi
89+
echo "ref=$REF" >> $GITHUB_OUTPUT
90+
91+
- name: "Determine dbt-common ref"
92+
id: common-ref
93+
run: |
94+
if [[ -z "${{ inputs.dbt-common-ref }}" ]]; then
95+
# these will be commits instead of branches
96+
if [[ "${{ github.event_name }}" == "merge_group" ]]; then
97+
REF=${{ github.event.pull_request.merge_commit_sha }}
98+
else
99+
REF=${{ github.event.pull_request.base.sha }}
100+
fi
101+
else
102+
REF=${{ inputs.dbt-common-ref }}
103+
fi
104+
echo "ref=$REF" >> $GITHUB_OUTPUT
105+
106+
- name: "Final Refs"
107+
run: |
108+
echo "dbt-core-ref=${{ steps.core-ref.outputs.ref }}"
109+
echo "dbt-common-ref=${{ steps.common-ref.outputs.ref }}"
110+
111+
dbt-core-unit-test:
112+
name: "dbt-core unit tests"
113+
needs: [job-prep]
30114

31115
runs-on: ubuntu-latest
32116
timeout-minutes: 10
33117

34118
steps:
35-
- name: "Check out dbt-core"
119+
- name: "Check out dbt-core@${{ needs.job-prep.outputs.dbt-core-ref }}"
36120
uses: actions/checkout@v4
121+
with:
122+
repository: dbt-labs/dbt-core
123+
ref: ${{ needs.job-prep.outputs.dbt-core-ref }}
37124

38125
- name: "Set up Python 3.11"
39126
uses: actions/setup-python@v5
40127
with:
41128
python-version: "3.11"
42129

43-
- name: "Update the version of dbt-common"
130+
- name: "Upgrade pip"
131+
run: |
132+
python -m pip install --upgrade pip
133+
python -m pip --version
134+
135+
# tox takes care of installing the correct version of dbt-core dependencies but we need to
136+
# install them first s that we can override the dbt-common branch
137+
- name: "Manually install dbt-core dependencies"
138+
run: |
139+
python -m pip uninstall dbt-common -y
140+
python -m pip install -r dev-requirements.txt -r editable-requirements.txt
141+
142+
# Since the dbt-common dependency is pinned in dev-requirements.txt we need to force update it
143+
# Since tox installs dependencies but doesn't force update, it won't get overridden in the next
144+
# step since the requirements will already be met
145+
- name: "Force update the version of dbt-common@${{ needs.job-prep.outputs.dbt-common-ref }}"
146+
run: |
147+
python -m pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ needs.job-prep.outputs.dbt-common-ref }} --force-reinstall
148+
149+
- name: "Run unit tests"
150+
# Doing the check here instead of the top level because this is job a required check, the
151+
# label just means we should skip the tests
152+
if: ${{ !contains(github.event.label.name, 'Skip Core Testing')}}
153+
run: tox
154+
env:
155+
TOXENV: unit
156+
157+
- name: "Check installed versions"
158+
run: pip freeze
159+
160+
dbt-core-integration-metadata:
161+
name: "integration test metadata generation"
162+
runs-on: ubuntu-latest
163+
if: ${{ github.event_name != 'pull_request' && !contains(github.event.label.name, 'Skip Core Testing')}}
164+
outputs:
165+
split-groups: ${{ steps.generate-split-groups.outputs.split-groups }}
166+
include: ${{ steps.generate-include.outputs.include }}
167+
168+
steps:
169+
- name: "generate split-groups"
170+
id: generate-split-groups
171+
run: |
172+
MATRIX_JSON="["
173+
for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
174+
MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}")
175+
done
176+
MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}"
177+
MATRIX_JSON+="]"
178+
echo "split-groups=${MATRIX_JSON}"
179+
echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT
180+
181+
- name: "generate include"
182+
id: generate-include
183+
run: |
184+
INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-latest"' )
185+
INCLUDE_GROUPS="["
186+
for include in ${INCLUDE[@]}; do
187+
for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
188+
INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}")
189+
done
190+
done
191+
INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS | sed 's/,*$//g')
192+
INCLUDE_GROUPS+="]"
193+
echo "include=${INCLUDE_GROUPS}"
194+
echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT
195+
196+
dbt-core-integration-tests:
197+
name: "(${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}"
198+
if: ${{ github.event_name != 'pull_request' && !contains(github.event.label.name, 'Skip Core Testing')}}
199+
200+
runs-on: ${{ matrix.os }}
201+
timeout-minutes: 30
202+
needs: [job-prep, dbt-core-integration-metadata]
203+
strategy:
204+
fail-fast: false
205+
matrix:
206+
python-version: ["3.8", "3.9", "3.10", "3.11"]
207+
os: [ubuntu-20.04]
208+
split-group: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.split-groups) }}
209+
include: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.include) }}
210+
env:
211+
DBT_INVOCATION_ENV: github-actions
212+
DBT_TEST_USER_1: dbt_test_user_1
213+
DBT_TEST_USER_2: dbt_test_user_2
214+
DBT_TEST_USER_3: dbt_test_user_3
215+
216+
steps:
217+
- name: "Check out the repository@${{ needs.job-prep.outputs.dbt-core-ref }}"
218+
uses: actions/checkout@v4
219+
with:
220+
repository: dbt-labs/dbt-core
221+
ref: ${{ needs.job-prep.outputs.dbt-core-ref }}
222+
223+
- name: "Set up Python ${{ matrix.python-version }}"
224+
uses: actions/setup-python@v5
225+
with:
226+
python-version: ${{ matrix.python-version }}
227+
228+
- name: "Set up postgres (linux)"
229+
if: runner.os == 'Linux'
230+
uses: ./.github/actions/setup-postgres-linux
231+
232+
- name: "Set up postgres (macos)"
233+
if: runner.os == 'macOS'
234+
uses: ./.github/actions/setup-postgres-macos
235+
236+
- name: "Set up postgres (windows)"
237+
if: runner.os == 'Windows'
238+
uses: ./.github/actions/setup-postgres-windows
239+
240+
- name: "Upgrade pip"
241+
run: |
242+
python -m pip install --upgrade pip
243+
python -m pip --version
244+
245+
# tox takes care of installing the correct version of dbt-core dependencies but we need to
246+
# install them first s that we can override the dbt-common branch
247+
- name: "Manually install dbt-core dependencies"
248+
run: |
249+
python -m pip install -r dev-requirements.txt -r editable-requirements.txt
250+
251+
# Since the dbt-common dependency is pinned in dev-requirements.txt we need to force update it
252+
# Since tox installs dependencies but doesn't force update, it won't get overridden in the next
253+
# step since the requirements will already be met
254+
- name: "Force update the version of dbt-common@${{ needs.job-prep.outputs.dbt-common-ref }}"
255+
run: |
256+
python -m pip uninstall dbt-common -y
257+
python -m pip install pip install git+https://github.com/dbt-labs/dbt-common.git@${{ needs.job-prep.outputs.dbt-common-ref }} --force-reinstall
258+
259+
- name: "Run Functional tests"
260+
run: tox -- --ddtrace
261+
env:
262+
TOXENV: integration
263+
DBT_INVOCATION_ENV: github-actions
264+
DBT_TEST_USER_1: dbt_test_user_1
265+
DBT_TEST_USER_2: dbt_test_user_2
266+
DBT_TEST_USER_3: dbt_test_user_3
267+
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
268+
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
269+
DD_SITE: datadoghq.com
270+
DD_ENV: ci
271+
DD_SERVICE: dbt-core
272+
PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}
273+
274+
- name: "Check installed versions"
275+
run: pip freeze
276+
277+
integration-report:
278+
if: ${{ always() }}
279+
name: "dbt-core Integration Test Suite Report"
280+
runs-on: ubuntu-latest
281+
needs: [dbt-core-integration-tests]
282+
steps:
283+
- name: "Integration Tests Failed"
284+
if: ${{ contains(needs.dbt-core-integration-tests.result, 'failure') || contains(needs.dbt-core-integration-tests.result, 'cancelled') }}
285+
# when this is true the next step won't execute
286+
run: |
287+
echo "::notice title='Integration test suite failed'"
288+
exit 1
289+
290+
- name: "Integration Tests Passed"
291+
if: ${{ github.event_name != 'pull_request' }}
44292
run: |
45-
echo "Update the version of dbt-common ref'd in dbt-core"
293+
echo "::notice title='Integration test suite passed'"
46294
47-
- name: "Install dbt-core"
295+
- name: "Integration Tests Skipped on Pull Request"
296+
if: ${{ github.event_name == 'pull_request' && !contains(github.event.label.name, 'Skip Core Testing')}}
48297
run: |
49-
echo "Install dbt-core with updated dbt-common ref"
298+
echo "::notice title='Integration test suite skipped on Pull Requests - they will run on merge'"
50299
51-
- name: "Run Tests"
300+
- name: "Integration Tests Skipped by Label"
301+
if: ${{ contains(github.event.label.name, 'Skip Core Testing')}}
52302
run: |
53-
echo "Running tests in future versions."
303+
echo "::notice title='dbt-core test suite skipped due to Skip Core Testing label'"

‎.github/workflows/ci_tests.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ on:
1919
merge_group:
2020
types: [checks_requested]
2121
workflow_dispatch:
22+
workflow_call:
2223

2324
permissions: read-all
2425

@@ -64,7 +65,7 @@ jobs:
6465
6566
- name: Upload Unit Test Coverage to Codecov
6667
if: ${{ matrix.python-version == '3.11' }}
67-
uses: codecov/codecov-action@v3
68+
uses: codecov/codecov-action@v4
6869
with:
6970
token: ${{ secrets.CODECOV_TOKEN }}
7071
flags: unit

‎.github/workflows/scheduled_testing.yml

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# **what?**
2+
# The purpose of this workflow is to trigger CI to run for each
3+
# on a regular cadence. This will also test core + common to prevent breaks.
4+
# If the CI workflow fails, it will post to #dev-core-alerts to raise awareness.
5+
6+
# **why?**
7+
# Ensures dbt-common is always shippable and not broken.
8+
# Also, can catch any dependencies shifting beneath us that might
9+
# introduce breaking changes (could also impact Cloud).
10+
11+
# **when?**
12+
# Mainly on a schedule of 9:00, 13:00, 18:00 UTC everyday.
13+
# Manual trigger can also test on demand
14+
15+
name: Scheduled Testing [ placeholders]
16+
17+
on:
18+
# schedule:
19+
# - cron: '0 9,13,18 * * *' # 9:00, 13:00, 18:00 UTC
20+
workflow_dispatch: # for manual triggering
21+
22+
23+
# no special access is needed
24+
permissions: read-all
25+
26+
jobs:
27+
run_tests:
28+
# uses: dbt-labs/actions/.github/workflows/release-branch-tests.yml@main
29+
# with:
30+
# workflows_to_run: '["ci_tests.yml", "build.yml", "ci_dbt_core_testing.yml"]'
31+
# secrets: inherit
32+
runs-on: ubuntu-latest
33+
steps:
34+
35+
- name: Placeholder
36+
run: echo "This is a placeholder job"

0 commit comments

Comments
 (0)