Skip to content

Commit

Permalink
refactor: fix minor bugs, remove dagster-delta-polars, improve docs
Browse files Browse the repository at this point in the history
  • Loading branch information
ion-elgreco committed Feb 9, 2025
1 parent 52102d9 commit a90dfbb
Show file tree
Hide file tree
Showing 58 changed files with 3,542 additions and 1,067 deletions.
1 change: 0 additions & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@
libraries/dagster-delta/ @ion-elgreco
libraries/dagster-delta-polars/ @ion-elgreco @sverbruggen
.github/ @ion-elgreco
38 changes: 38 additions & 0 deletions .github/validate-release-version.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
"""Ensures that the files in `dist/` are prefixed with ${{ github.ref_name }}
Tag must adhere to naming convention of distributed files. For example, the tag
`dagster_delta-0.1.2` must match the prefix of the files in the `dist/` folder:
-rw-r--r--@ 2.0K Oct 23 14:06 dagster_delta-0.1.2-py3-none-any.whl
-rw-r--r--@ 1.6K Oct 23 14:06 dagster_delta-0.1.2.tar.gz
USAGE
$ python .github/validate-release-version.py libraries/dagster-delta/dist dagster_delta-0.1.3
"""

import sys
import os


if len(sys.argv) != 3:
print("Requires positional arguments: <path to dist> <github.ref_name>")
sys.exit(1)

dist_path = sys.argv[1]
github_ref_name = sys.argv[2]

if not os.path.exists(dist_path):
print("Release directory `dist/` must exist")
sys.exit(1)

for filename in os.listdir(dist_path):
if filename.startswith("."):
continue
if not filename.startswith(github_ref_name):
print(f"{filename} does not start with prefix {github_ref_name}")
sys.exit(1)


print(f"Success: all files in `dist/` are prefixed with {github_ref_name}")
147 changes: 0 additions & 147 deletions .github/workflows/CI.yml

This file was deleted.

12 changes: 12 additions & 0 deletions .github/workflows/quality-dagster-delta.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
name: quality-check-dagster-delta
on:
pull_request:
types: [opened, synchronize, reopened]
paths:
- 'libraries/dagster-delta/**'

jobs:
check:
uses: ./.github/workflows/template-quality-check.yml
with:
working_directory: ./libraries/dagster-delta
12 changes: 12 additions & 0 deletions .github/workflows/quality-dagster-unity-catalog-polars.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
name: quality-check-dagster-unity-catalog-polars
on:
pull_request:
types: [opened, synchronize, reopened]
paths:
- 'libraries/dagster-unity-catalog-polars/**'

jobs:
check:
uses: ./.github/workflows/template-quality-check.yml
with:
working_directory: ./libraries/dagster-unity-catalog-polars
15 changes: 15 additions & 0 deletions .github/workflows/release-dagster-delta-polars.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
name: build-and-release-dagster-delta-polars

on:
push:
tags:
- 'dagster_delta_polars-*.*.*'

jobs:
build-and-release-dagster-delta-polars:
uses: ./.github/workflows/template-release.yml
with:
library_name: dagster-delta-polars
working_directory: ./libraries/dagster-delta-polars
secrets:
pypi_token: ${{ secrets.PYPI_API_TOKEN_DDP }}
15 changes: 15 additions & 0 deletions .github/workflows/release-dagster-delta.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
name: build-and-release-dagster-delta

on:
push:
tags:
- 'dagster_delta-*.*.*'

jobs:
build-and-release-dagster-delta:
uses: ./.github/workflows/template-release.yml
with:
library_name: dagster-delta
working_directory: ./libraries/dagster-delta
secrets:
pypi_token: ${{ secrets.PYPI_API_TOKEN_DD }}
15 changes: 15 additions & 0 deletions .github/workflows/release-dagster-unity-catalog-polars.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
name: build-and-release-dagster-unity-catalog-polars

on:
push:
tags:
- 'dagster_unity_catalog_polars-*.*.*'

jobs:
build-and-release-dagster-unity-catalog-polars:
uses: ./.github/workflows/template-release.yml
with:
library_name: dagster-unity-catalog-polars
working_directory: ./libraries/dagster-unity-catalog-polars
secrets:
pypi_token: ${{ secrets.PYPI_API_TOKEN_DDUC }}
43 changes: 43 additions & 0 deletions .github/workflows/template-quality-check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: quality-check

on:
workflow_call:
inputs:
working_directory:
required: true
type: string

jobs:
check:
runs-on: ubuntu-latest
steps:

- name: Checkout
uses: actions/checkout@v4

- name: Install uv
uses: astral-sh/setup-uv@v3

- name: Install python
working-directory: ${{ inputs.working_directory }}
run: uv python install 3.9

- name: Sync dependencies
working-directory: ${{ inputs.working_directory }}
run: uv sync

- name: Ruff (lint)
working-directory: ${{ inputs.working_directory }}
run: uv run ruff check

- name: Ruff (formatting)
working-directory: ${{ inputs.working_directory }}
run: uv run ruff format --check .

- name: Pyright
working-directory: ${{ inputs.working_directory }}
run: uv run pyright

- name: Pytest
working-directory: ${{ inputs.working_directory }}
run: uv run pytest
50 changes: 50 additions & 0 deletions .github/workflows/template-release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# References
#
# https://docs.astral.sh/uv/guides/integration/github/
# https://docs.astral.sh/uv/guides/publish/#preparing-your-project-for-packaging
# https://docs.pypi.org/trusted-publishers/adding-a-publisher/
#

name: build-and-release

on:
workflow_call:
inputs:
library_name:
required: true
type: string
working_directory:
required: true
type: string
secrets:
pypi_token:
required: true
jobs:
build:
name: python
runs-on: ubuntu-latest
environment: production

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Install uv
uses: astral-sh/setup-uv@v3

- name: Install Python
working-directory: ${{ inputs.working_directory }}
run: uv python install

- name: Build
working-directory: ${{ inputs.working_directory }}
run: uv build

- name: Validate release version
run: python .github/validate-release-version.py ${{ inputs.working_directory }}/dist ${{ github.ref_name }}

- name: Publish
working-directory: ${{ inputs.working_directory }}
run: uv publish
env:
UV_PUBLISH_TOKEN: ${{ secrets.pypi_token }}
Loading

0 comments on commit a90dfbb

Please sign in to comment.