From 705375ee27ac06baca4c4a008ac289774b02889d Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 5 Jan 2024 14:13:46 -0600 Subject: [PATCH 1/5] setup --- .github/workflows/main.yml | 164 +++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..a857b04e --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,164 @@ +# **what?** +# Runs code quality checks, unit tests, integration tests and +# verifies python build on all code commited to the repository. This workflow +# should not require any secrets since it runs for PRs from forked repos. By +# default, secrets are not passed to workflows running from a forked repos. + +# **why?** +# Ensure code for dbt meets a certain quality standard. + +# **when?** +# This will run for all PRs, when code is pushed to a release +# branch, and when manually triggered. + +name: Tests and Code Checks + +on: + push: + branches: + - "main" + - "*.latest" + - "releases/*" + pull_request: + workflow_dispatch: + +permissions: read-all + +# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +# top-level adjustments can be made here +env: + # number of parallel processes to spawn for python integration testing + PYTHON_INTEGRATION_TEST_WORKERS: 5 + +jobs: + code-quality: + name: code-quality + + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip --version + make dev + mypy --version + dbt --version + + - name: Run pre-commit hooks + run: pre-commit run --all-files --show-diff-on-failure + + unit: + name: unit test / python ${{ matrix.python-version }} + + runs-on: ubuntu-latest + timeout-minutes: 10 + + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + env: + TOXENV: "unit" + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip --version + python -m pip install tox + tox --version + + - name: Run tox + run: tox + + - name: Get current date + if: always() + id: date + run: | + CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts + echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT + + - name: Upload Unit Test Coverage to Codecov + if: ${{ matrix.python-version == '3.11' }} + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: unit + + build: + name: build packages + + runs-on: ubuntu-latest + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip install --upgrade setuptools wheel twine check-wheel-contents + python -m pip --version + + - name: Build distributions + run: ./scripts/build-dist.sh + + - name: Show distributions + run: ls -lh dist/ + + - name: Check distribution descriptions + run: | + twine check dist/* + + - name: Check wheel contents + run: | + check-wheel-contents dist/*.whl --ignore W007,W008 + + - name: Install wheel distributions + run: | + find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + + - name: Check wheel distributions + run: | + dbt --version + + - name: Install source distributions + # ignore dbt-1.0.0, which intentionally raises an error when installed from source + run: | + find ./dist/dbt-[a-z]*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + + - name: Check source distributions + run: | + dbt --version From f26075c4695690e321d084e396d7b17d02956901 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 5 Jan 2024 16:13:43 -0600 Subject: [PATCH 2/5] first pass at stubbing out workflows --- .github/workflows/build.yml | 50 ++++++++ .github/workflows/ci_code_quality.yml | 53 +++++++++ .github/workflows/ci_tests.yml | 73 ++++++++++++ .github/workflows/main.yml | 164 -------------------------- 4 files changed, 176 insertions(+), 164 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/ci_code_quality.yml create mode 100644 .github/workflows/ci_tests.yml delete mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..f74c64e3 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,50 @@ +# **what?** +# Verifies python build on all code commited to the repository. This workflow +# should not require any secrets since it runs for PRs from forked repos. By +# default, secrets are not passed to workflows running from a forked repos. + +# **why?** +# Ensure code for dbt meets a certain quality standard. + +# **when?** +# This will run for all PRs, when code is pushed to main, and when manually triggered. + +name: "Build" + +on: + push: + branches: + - "main" + pull_request: + workflow_dispatch: + +permissions: read-all + +# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + build: + # TODO: blocked on https://github.com/dbt-labs/dbt-adapter/issues/3 + name: build packages + + runs-on: ubuntu-latest + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: "Building" + run: | + echo "Building!" diff --git a/.github/workflows/ci_code_quality.yml b/.github/workflows/ci_code_quality.yml new file mode 100644 index 00000000..7b12e3d5 --- /dev/null +++ b/.github/workflows/ci_code_quality.yml @@ -0,0 +1,53 @@ +# **what?** +# Runs code quality checks on all code commited to the repository. This workflow +# should not require any secrets since it runs for PRs from forked repos. By +# default, secrets are not passed to workflows running from a forked repos. + +# **why?** +# Ensure code for dbt meets a certain quality standard. + +# **when?** +# This will run for all PRs, when code is pushed to main, and when manually triggered. + +name: "Check Code Quality" + +on: + push: + branches: + - "main" + pull_request: + workflow_dispatch: + +permissions: read-all + +# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + code-quality: + name: code-quality + + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Hatch + shell: bash + run: pip3 install hatch + + - name: Run Pre-commit Hooks + run: hatch run dev-env:pre-commit run --show-diff-on-failure --color=always --all-files diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml new file mode 100644 index 00000000..03cdffb8 --- /dev/null +++ b/.github/workflows/ci_tests.yml @@ -0,0 +1,73 @@ +# **what?** +# Runs unit tests on all code commited to the repository. This workflow +# should not require any secrets since it runs for PRs from forked repos. By +# default, secrets are not passed to workflows running from a forked repos. + +# **why?** +# Ensure code for dbt meets a certain quality standard. + +# **when?** +# This will run for all PRs, when code is pushed to main, and when manually triggered. + +name: Unit Tests + +on: + push: + branches: + - "main" + pull_request: + workflow_dispatch: + +permissions: read-all + +# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + unit: + name: "Run tests / python ${{ matrix.python-version }}" + + runs-on: ubuntu-latest + timeout-minutes: 10 + + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + steps: + - name: "Check out the repository" + uses: actions/checkout@v3 + + - name: "Set up Python ${{ matrix.python-version }}" + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: "Install Hatch" + shell: bash + run: pip3 install hatch + + - name: "Run Tests" + run: hatch run dev-env:pytest tests + + - name: "Get current date" + if: always() + id: date + run: | + CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts + echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT + + # TODO: what setup is needed here? + # - name: Upload Unit Test Coverage to Codecov + # if: ${{ matrix.python-version == '3.11' }} + # uses: codecov/codecov-action@v3 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} + # flags: unit diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index a857b04e..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,164 +0,0 @@ -# **what?** -# Runs code quality checks, unit tests, integration tests and -# verifies python build on all code commited to the repository. This workflow -# should not require any secrets since it runs for PRs from forked repos. By -# default, secrets are not passed to workflows running from a forked repos. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to a release -# branch, and when manually triggered. - -name: Tests and Code Checks - -on: - push: - branches: - - "main" - - "*.latest" - - "releases/*" - pull_request: - workflow_dispatch: - -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -# top-level adjustments can be made here -env: - # number of parallel processes to spawn for python integration testing - PYTHON_INTEGRATION_TEST_WORKERS: 5 - -jobs: - code-quality: - name: code-quality - - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip --version - make dev - mypy --version - dbt --version - - - name: Run pre-commit hooks - run: pre-commit run --all-files --show-diff-on-failure - - unit: - name: unit test / python ${{ matrix.python-version }} - - runs-on: ubuntu-latest - timeout-minutes: 10 - - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - - env: - TOXENV: "unit" - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip --version - python -m pip install tox - tox --version - - - name: Run tox - run: tox - - - name: Get current date - if: always() - id: date - run: | - CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts - echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT - - - name: Upload Unit Test Coverage to Codecov - if: ${{ matrix.python-version == '3.11' }} - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - flags: unit - - build: - name: build packages - - runs-on: ubuntu-latest - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade setuptools wheel twine check-wheel-contents - python -m pip --version - - - name: Build distributions - run: ./scripts/build-dist.sh - - - name: Show distributions - run: ls -lh dist/ - - - name: Check distribution descriptions - run: | - twine check dist/* - - - name: Check wheel contents - run: | - check-wheel-contents dist/*.whl --ignore W007,W008 - - - name: Install wheel distributions - run: | - find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check wheel distributions - run: | - dbt --version - - - name: Install source distributions - # ignore dbt-1.0.0, which intentionally raises an error when installed from source - run: | - find ./dist/dbt-[a-z]*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check source distributions - run: | - dbt --version From 014730c8788e1545c96a1057d7ebd8b0b5dd748c Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 5 Jan 2024 20:33:57 -0600 Subject: [PATCH 3/5] update build workflow --- .github/workflows/build.yml | 45 ++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f74c64e3..503f4dff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,8 +43,47 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.11' - - name: "Building" + - name: "Install build specific python dependencies" run: | - echo "Building!" + python -m pip install --user --upgrade pip + python -m pip install --upgrade wheel twine check-wheel-contents + python -m pip --version + + - name: "Install Hatch" + shell: bash + run: pip3 install hatch + + - name: "Build Python Package" + run: | + hatch build + + - name: "Show distributions" + run: ls -lh dist/ + + - name: "Check distribution descriptions" + run: | + twine check dist/* + + - name: "Check wheel contents" + run: | + check-wheel-contents dist/*.whl --ignore W007,W008 + + - name: "Install wheel distributions" + run: | + find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + + # TODO: how to validate here? we did dbt --version previously... + - name: "Check wheel distributions" + run: | + pip freeze + + - name: "Install source distributions" + run: | + find ./dist/dbt-[a-z]*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + + # TODO: how to validate here? we did dbt --version previously... + - name: "Check source distributions" + run: | + pip freeze From 6d91ff56fc25639b621366e4a6196584be899f12 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 5 Jan 2024 20:38:28 -0600 Subject: [PATCH 4/5] fix regex --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 503f4dff..a4efac7e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,7 +72,7 @@ jobs: - name: "Install wheel distributions" run: | - find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + find ./dist/dbt_common-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ # TODO: how to validate here? we did dbt --version previously... - name: "Check wheel distributions" @@ -81,7 +81,7 @@ jobs: - name: "Install source distributions" run: | - find ./dist/dbt-[a-z]*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + find ./dist/dbt_common-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ # TODO: how to validate here? we did dbt --version previously... - name: "Check source distributions" From c6a4f109bfaa1e2ddf5633d8317672e0d39a0952 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 8 Jan 2024 09:44:53 -0600 Subject: [PATCH 5/5] add grep to build --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4efac7e..c3b8366b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,16 +74,16 @@ jobs: run: | find ./dist/dbt_common-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - # TODO: how to validate here? we did dbt --version previously... + # TODO: how to validate here? we did dbt --version previously. this checks it's there, but not that it can do anything. maybe it's enough? - name: "Check wheel distributions" run: | - pip freeze + pip freeze | grep dbt-common - name: "Install source distributions" run: | find ./dist/dbt_common-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - # TODO: how to validate here? we did dbt --version previously... + # TODO: how to validate here? we did dbt --version previously. this checks it's there, but not that it can do anything. maybe it's enough? - name: "Check source distributions" run: | - pip freeze + pip freeze | grep dbt-common