build: add more scaffolding

.github/workflows/ci.yaml

@@ -2,9 +2,35 @@
 name: CI
 on: push
 jobs:
-  pre-commit:
+  precommit:
+    name: Pre-commit 🔥
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v3
     - uses: pre-commit/[email protected]
+
+  run:
+    name: "tests & coverage 💊"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install hatch
+      run: |
+        python -m pip install hatch
+
+    - name: Lint
+      run: hatch run check
+
+    - name: Tests
+      run: hatch run +py=${{ matrix.python-version }} test:test
+

.github/workflows/coverage.yml

@@ -0,0 +1,76 @@
+name: Coverage
+
+permissions:
+  pull-requests: write
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  release:
+    types:
+      - published
+  schedule:
+    # At 12:00 on every day-of-month
+    - cron: "0 12 */1 * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  coverage:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Set up dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -qy \
+            pkg-config \
+            libdebuginfod-dev \
+            libunwind-dev \
+            liblz4-dev \
+            gdb \
+            lcov \
+            libdw-dev \
+            libelf-dev \
+            python3.10-dev \
+            python3.10-dbg
+      - name: Install Python dependencies
+        run: |
+          python3 -m pip install --upgrade pip cython pkgconfig
+          make test-install
+      - name: Disable ptrace security restrictions
+        run: |
+          echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+      - name: Add ./node_modules/.bin to PATH
+        run: |
+          export PATH="./node_modules/.bin:$PATH"
+      - name: Compute C++ coverage
+        run: |
+          make ccoverage
+      - name: Compute Python + Cython coverage
+        run: |
+          make pycoverage
+      - name: Upload C++ report to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: cppcoverage.lcov
+          flags: cpp
+      - name: Upload {P,C}ython report to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: pycoverage.lcov
+          flags: python_and_cython

.github/workflows/release.yml

@@ -0,0 +1,22 @@
+
+  towncrier-check:
+    name: Verify Towncrier entry added
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install Towncrier
+        run: |
+          python -m pip install towncrier
+      - name: Verify Towncrier entry added
+        if: github.event_name == 'pull_request'
+        env:
+          BASE_BRANCH: ${{ github.base_ref }}
+        run: |
+          # Fetch the pull request' base branch so towncrier will be able to
+          # compare the current branch with the base branch.
+          # Source: https://github.com/actions/checkout/#fetch-all-branches.
+          git fetch --no-tags origin +refs/heads/${BASE_BRANCH}:refs/remotes/origin/${BASE_BRANCH}
+          towncrier check --compare-with origin/${BASE_BRANCH}

README.md

@@ -19,3 +19,19 @@ pip install dagcellent
 ## License
 
 `dagcellent` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license.
+
+
+Changelog Generation
+--------------------
+
+We use ``towncrier`` for tracking changes and generating a changelog.
+When making a pull request, we require that you add a towncrier entry along with the code changes.
+You should create a file named ``<PR number>.<change type>`` in the ``doc/changes`` directory, where the PR number should be substituted for ``<PR number>``, and ``<change type>`` is either ``feature``, ``bugfix``, ``doc``, ``removal``, ``misc``, or ``deprecation``,
+depending on the type of change included in the PR.
+
+You can also create this file by installing ``towncrier`` and running 
+
+   towncrier create <PR number>.<change type>
+
+Running this will create a file in the ``doc/changes`` directory with a filename corresponding to the argument you passed to ``towncrier create``.
+In this file, you should add a short description of the changes that the PR introduces.

doc/changelog.rst

@@ -0,0 +1,6 @@
+Change Log
+**********
+
+.. towncrier release notes start
+Version 0.0.1 (July 13, 2024)
+++++++++++++++++++++++++++++++

pyproject.toml

@@ -105,8 +105,33 @@ package = "dagcellent"
 package_dir = "src"
 # Where you want your news files to come out. This can be .rst
 # or .md, towncrier's default template works with both.
-filename = "NEWS.rst"
+filename = "doc/changelog.rst"
+directory = "doc/changes"
 
+[[tool.towncrier.type]]
+directory = "deprecation"
+name = "Deprecations"
+showcontent = true
+[[tool.towncrier.type]]
+directory = "feature"
+name = "Features"
+showcontent = true
+[[tool.towncrier.type]]
+directory = "bugfix"
+name = "Bug Fixes"
+showcontent = true
+[[tool.towncrier.type]]
+directory = "removal"
+name = "Removals"
+showcontent = true
+[[tool.towncrier.type]]
+directory = "doc"
+name = "Documentation"
+showcontent = true
+[[tool.towncrier.type]]
+directory = "misc"
+name = "Miscellaneous"
+showcontent = true
 
 [tool.pyright]
 include = ["src"]
@@ -248,3 +273,5 @@ show_error_codes = true
 disable_error_code = [
     "annotation-unchecked",
 ]
+
+

src/dagcellent/dynamic_dag.py

@@ -1,14 +1,15 @@
 """Module to help creating Dynamic DAGs in Airflow."""
 from __future__ import annotations
 
-import logging
+import json
 from collections.abc import Callable, Iterable
 from pathlib import Path
+from typing import TypeVar
 
 import tomli
 from pydantic import BaseModel, ConfigDict
 
-_LOGGER = logging.getLogger(__name__)
+T = TypeVar("T", bound="BaseModel")
 
 
 class DagArguments(BaseModel):
@@ -49,23 +50,33 @@ def from_toml(cls, file: Path) -> Config:
 
         Args:
             file (Path): path to toml files.
-            test (bool, optional): testing mode. Defaults to False.
         """
         with open(file, "rb") as f:
             k = tomli.load(f)
         return cls(**k)
 
+    @classmethod
+    def from_json(cls, file: Path) -> Config:
+        """Load from JSON file to Config object.
+
+        Args:
+            file (Path): path to toml files.
+        """
+        with open(file) as f:
+            k = json.load(f)
+        return cls(**k)
+
 
 def parse_config_file(
-    resource_paths: Iterable[Path], parser: Callable[..., Config]
-) -> list[Config]:
+    resource_paths: Iterable[Path], parser: Callable[..., T]
+) -> list[T]:
     """Parse config files.
 
     Args:
         resource_paths (Iterable[Path]): file paths
         parser (Callable): parsing logic
     """
-    configs: list[Config] = []
+    configs: list[T] = []
     for _path in resource_paths:
         configs.append(parser(_path))
     return configs

tests/conftest.py

@@ -13,3 +13,13 @@ def toml_file(tmp_path_factory: pytest.TempPathFactory):
         f.writelines(TOML_TEST)
 
     return fn
+
+
+@pytest.fixture(scope="session")
+def json_file(tmp_path_factory: pytest.TempPathFactory):
+    JSON_TEST = '{"version":1, "description":"test"}'
+    fn = tmp_path_factory.mktemp("test") / "test.json"
+    with open(fn, "w+") as f:
+        f.writelines(JSON_TEST)
+
+    return fn

tests/test_dag_config.py

@@ -56,3 +56,25 @@ class MyConfig(Config):
     config = parse_config_file([toml_file], Config.from_toml)
     assert config[0].version == 1
     assert config[0].description == "test"
+
+
+def test_parse_json_config_file(json_file: pathlib.Path):
+    """Test the end API."""
+    # only one file passed
+    with pytest.raises(TypeError):
+        assert parse_config_file(pathlib.Path(), Config.from_json)  # type: ignore[arg-type]
+
+    with pytest.raises(FileNotFoundError):
+        assert parse_config_file([pathlib.Path("dummy.toml")], Config.from_json)
+
+    class MyConfig(Config):
+        """My config."""
+
+        test: str
+
+    with pytest.raises(ValidationError):
+        assert parse_config_file([json_file], MyConfig.from_json)
+
+    config = parse_config_file([json_file], Config.from_json)
+    assert config[0].version == 1
+    assert config[0].description == "test"