Merge pull request #2 from cern-sis/airflow-setup

global: init of the project

.github/workflows/test-and-build.yml

@@ -0,0 +1,63 @@
+name: Python application test with pytest
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  AIRFLOW_HOME: /home/runner/work/bi-dags/bi-dags
+  REGISTRY: registry.cern.ch
+  IMAGE: cern-sis/bi-dags
+
+jobs:
+  build_test:
+    name: Build and Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Python 3
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.11
+
+      - name: Build Image
+        id: build
+        uses: cern-sis/gh-workflows/.github/actions/docker-build@v6
+        with:
+          registry: ${{ env.REGISTRY }}
+          image: ${{ env.IMAGE }}
+          tags: type=ref,event=pr
+          cache: false
+          username: ${{ secrets.HARBOR_USERNAME }}
+          password: ${{ secrets.HARBOR_PASSWORD }}
+
+      - name: Run tests with pytest and generate report
+        run: >
+          docker run
+          --name biproject
+          --network=host
+          -v "$(pwd)"/tests:/opt/airflow/tests
+          --entrypoint pytest
+          $REGISTRY/$IMAGE@${{ steps.build.outputs.image-digest }}
+          tests
+          --cov=./
+          --cov-report=xml
+
+      - name: Copy test coverage file to host machine
+        run: docker cp biproject:/opt/airflow/coverage.xml .
+
+      - name: Upload Coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          verbose: true
+
+      - name: Deploy QA
+        if: ${{ github.event_name == 'push' }}
+        uses: cern-sis/gh-workflows/.github/actions/kubernetes-project-new-images@v6
+        with:
+          event-type: update
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE }}@${{ steps.build.outputs.image-digest }}
+          token: ${{ secrets.PAT_FIRE_EVENTS_ON_CERN_SIS_KUBERNETES }}

.gitignore

@@ -0,0 +1,18 @@
+# Local Dev
+*.log
+*.err
+*.pid
+*.out
+.DS_Store
+
+# Airflow
+logs
+airflow.db
+airflow.cfg
+standalone_admin_password.txt
+webserver_config.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: "22.8.0"
+    hooks:
+      - id: black
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v2.7.1"
+    hooks:
+      - id: prettier
+  - repo: https://github.com/pycqa/isort
+    rev: "5.12.0"
+    hooks:
+      - id: isort
+  - repo: https://github.com/pycqa/flake8
+    rev: "3.9.2"
+    hooks:
+      - id: flake8
+        args: ["--config=setup.cfg"]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-merge-conflict
+      - id: end-of-file-fixer
+      - id: trailing-whitespace

Dockerfile

@@ -0,0 +1,12 @@
+FROM apache/airflow:2.8.2-python3.8
+
+ENV PYTHONBUFFERED=0
+ENV AIRFLOW__LOGGING__LOGGING_LEVEL=INFO
+
+# install your pip packages
+COPY requirements.txt ./requirements.txt
+COPY requirements-test.txt ./requirements-test.txt
+
+COPY dags ./dags
+
+RUN pip install --no-cache-dir --user  -r requirements-test.txt -r requirements.txt

README.md

@@ -1 +1 @@
-# bi-dags
+# bi-dags

dags/example_dag/example_dag.py

@@ -0,0 +1,34 @@
+import logging
+
+import pendulum
+from airflow.decorators import dag, task
+
+
+def fetch(**kwargs):
+    return "Test String"
+
+
+def pull(test_string, **kwargs):
+    return test_string
+
+
+@dag(
+    start_date=pendulum.today("UTC").add(days=-1),
+    schedule="@hourly",
+    params={"from_date": None, "until_date": None, "per_page": None},
+)
+def test_dag():
+    @task()
+    def fetch_task(**kwargs):
+        return fetch()
+
+    @task()
+    def pull_task(test_string, **kwargs):
+        logging.info(f"The value from previous task is: {test_string}")
+        return pull
+
+    value = fetch_task()
+    pull_task(value)
+
+
+Test_dag = test_dag()

requirements-test.txt

@@ -0,0 +1,4 @@
+pre-commit==3.5.0
+pytest==7.4.4
+coverage==7.4.0
+pytest-cov==4.1.0

requirements.txt

@@ -0,0 +1,2 @@
+-c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.8.txt
+apache-airflow[celery, postgres]==2.8.1

tests/conftest.py

@@ -0,0 +1,3 @@
+import airflow
+
+print("Initilized environment with", airflow.__name__)

tests/unit/test_test_dag.py

@@ -0,0 +1,9 @@
+from example_dag.example_dag import fetch, pull
+
+
+def test_test_dag_fetch():
+    assert fetch() == "Test String"
+
+
+def test_test_dag_pull():
+    assert pull("Test String") == "Test String"