WIP Compine all (#55)

global: change the project structure

* It merges Airflow and Django into a mono repo,
   it has breaking changes.

.github/workflows/test.yml → .github/workflows/backoffice/test.yml

@@ -18,15 +18,15 @@ jobs:
     with:
       ref: ${{ inputs.ref }}
       image: cern-sis/inspire/backoffice
-      context: .
+      context: ./backoffice
       dockerfile: ./compose/production/django/Dockerfile
     secrets: inherit
 
   test:
     needs: build
     strategy:
       fail-fast: false
-    uses: ./.github/workflows/integration-tests.yml
+    uses: ./.github/workflows/backoffice/integration-tests.yml
     with:
       ref: ${{ inputs.ref }}
       image: "registry.cern.ch/cern-sis/inspire/backoffice@${{ needs.build.outputs.image-id }}"

.github/workflows/lint.yml

@@ -17,17 +17,5 @@ jobs:
         with:
           ref: ${{ inputs.ref }}
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.11"
-
-      - name: black
-        uses: psf/black@stable
-        with:
-          options: "--verbose"
-
-      - name: run flake8
-        uses: julianwachholz/flake8-action@v2
-        with:
-          checkName: "flake8"
+      - name: Pre-commit check
+        uses: pre-commit/[email protected]

.github/workflows/pull-request-main-backoffice.yml

@@ -0,0 +1,18 @@
+name: Pull request main
+
+on:
+  pull_request_target:
+    branches: [main]
+    paths:
+      - 'backoffice/**'
+
+jobs:
+  lint:
+    uses: ./.github/workflows/lint.yml
+    with:
+      ref: ${{ github.ref }}
+  test:
+    uses: ./.github/workflows/backoffice/test.yml
+    with:
+      ref: ${{ github.event.pull_request.head.sha }}
+    secrets: inherit

.github/workflows/pull-request-main.yml → .github/workflows/pull-request-main-workflows.yml

@@ -3,14 +3,15 @@ name: Pull request main
 on:
   pull_request_target:
     branches: [main]
-
+    paths:
+      - 'workflows/**'
 jobs:
   lint:
     uses: ./.github/workflows/lint.yml
     with:
       ref: ${{ github.ref }}
   test:
-    uses: ./.github/workflows/test.yml
+    uses: ./.github/workflows/workflows/test.yml
     with:
       ref: ${{ github.event.pull_request.head.sha }}
     secrets: inherit

.github/workflows/push-main-backoffice.yml

@@ -0,0 +1,34 @@
+name: Push main
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'backoffice/**'
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  lint:
+    uses: ./.github/workflows/lint.yml
+    with:
+      ref: ${{ github.ref }}
+  test:
+    needs: lint
+    uses: ./.github/workflows/backoffice/test.yml
+    with:
+      ref: ${{ github.ref }}
+    secrets: inherit
+  deploy:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: send event
+        uses: cern-sis/gh-workflows/.github/actions/[email protected]
+        with:
+          event-type: update
+          images: |
+            cern-sis/inspire/backoffice@${{ needs.test.outputs.image-id }}
+          token: ${{ secrets.PAT_FIRE_EVENTS_ON_CERN_SIS_KUBERNETES }}

.github/workflows/push-main.yml → .github/workflows/push-main-workflows.yml

@@ -3,6 +3,8 @@ name: Push main
 on:
   push:
     branches: [main]
+    paths:
+      - 'workflows/**'
 
 defaults:
   run:
@@ -14,7 +16,7 @@ jobs:
     with:
       ref: ${{ github.ref }}
   test:
-    uses: ./.github/workflows/test.yml
+    uses: ./.github/workflows/workflows/test.yml
     with:
       ref: ${{ github.ref }}
     secrets: inherit

.github/workflows/workflows/test.yml

@@ -0,0 +1,57 @@
+name: Tests
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: The reference to build
+        type: string
+        required: true
+    outputs:
+      image-id:
+        description: The ID of image that has been built
+        value: ${{ jobs.build.outputs.image-id }}
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build:
+    uses: ./.github/workflows/build.yml
+    with:
+      ref: ${{ inputs.ref }}
+      image: cern-sis/inspire/workflows
+    secrets: inherit
+  test:
+    needs: build
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_USER: airflow
+          POSTGRES_PASSWORD: airflow
+          POSTGRES_DB: airflow
+        ports:
+          - 5432:5432
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Test
+        run: >
+          docker run
+          --network=host
+          -v "$(pwd)"/tests:/opt/airflow/tests
+          -v "$(pwd)"/airflow.cfg:/opt/airflow/airflow.cfg
+          -v "$(pwd)"/requirements-test.txt:/opt/airflow/requirements-test.txt
+          -v "$(pwd)"/data:/opt/airflow/data
+          registry.cern.ch/cern-sis/inspire/workflows@${{ needs.build.outputs.image-id }}
+          bash -c "pip install -r requirements-test.txt && airflow db init && pytest /opt/airflow/tests"

.pre-commit-config.yaml

@@ -1,18 +1,18 @@
 repos:
-  - repo: https://github.com/psf/black
-    rev: "22.8.0"
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
     hooks:
-      - id: black
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v2.7.1"
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: fix-byte-order-marker
+      - id: mixed-line-ending
+      - id: name-tests-test
+        args: [ --pytest-test-first ]
+        exclude: '^(?!factories/)'
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.5.2
     hooks:
-      - id: prettier
-  - repo: https://github.com/pycqa/isort
-    rev: "5.12.0"
-    hooks:
-      - id: isort
-  - repo: https://github.com/pycqa/flake8
-    rev: "3.9.2"
-    hooks:
-      - id: flake8
-        args: ["--config=setup.cfg"]
+      - id: ruff
+        args: [ --fix, --unsafe-fixes ]
+      - id: ruff-format

Makefile

@@ -0,0 +1,28 @@
+# Makefile
+
+# Set the AIRFLOW_HOME environment variable
+export AIRFLOW_HOME=$(shell pwd)/workflows
+export AIRFLOW_PROJ_DIR=$(AIRFLOW_HOME)
+
+# Default target
+all: run
+
+run:
+	docker-compose up
+
+clean:
+	docker-compose down
+
+init: backoffice-init workflows-init
+
+backoffice-init:
+	docker-compose exec backoffice-webserver python manage.py create_groups
+	docker-compose exec backoffice-webserver python manage.py loaddata backoffice/users/fixtures/users.json
+	docker-compose exec backoffice-webserver python manage.py loaddata backoffice/users/fixtures/tokens.json
+	echo "Backoffice initialized"
+
+workflows-init:
+	docker-compose exec airflow-webserver /entrypoint airflow connections import ./scripts/connections/connections.json
+	docker-compose exec airflow-webserver /entrypoint airflow variables import ./scripts/variables/variables.json
+	echo "\033[31mCHANGE inspire_token in Admin->Variables\033[0m"
+	echo "Workflows initialized"

.envs/local/.django → backoffice/.envs/local/.django

@@ -10,6 +10,7 @@ REDIS_URL=redis://redis:6379/0
 # Celery
 CELERY_BROKER_URL=amqp://guest:guest@mq:5672/
 CELERY_RESULT_BACKEND=redis://redis:6379/2
+CELERY_TASK_ALWAYS_EAGER=True
 # Flower
 CELERY_FLOWER_USER=debug
 CELERY_FLOWER_PASSWORD=debug
@@ -19,5 +20,5 @@ OPENSEARCH_HOST=opensearch:9200
 OPENSEARCH_INDEX_PREFIX=backoffice-backend-local
 
 # Airflow
-AIRFLOW_BASE_URL=http://host.docker.internal:8082
-AIRFLOW_TOKEN=CHANGE_ME
+AIRFLOW_BASE_URL=http://host.docker.internal:8080
+AIRFLOW_TOKEN=YWlyZmxvdzphaXJmbG93

.envs/docker/.postgres → backoffice/.envs/local/.postgres

@@ -1,7 +1,7 @@
 # PostgreSQL
 # ------------------------------------------------------------------------------
-POSTGRES_HOST=postgres
+POSTGRES_HOST=postgres-backoffice
 POSTGRES_PORT=5432
 POSTGRES_DB=inspire
 POSTGRES_USER=inspire
-POSTGRES_PASSWORD=inspire
+POSTGRES_PASSWORD=inspire

backoffice/backoffice/__init__.py

@@ -0,0 +1,5 @@
+__version__ = "0.1.0"
+__version_info__ = tuple(
+    int(num) if num.isdigit() else num
+    for num in __version__.replace("-", ".", 1).split(".")
+)

backoffice/conftest.py → backoffice/backoffice/conftest.py

@@ -5,10 +5,10 @@
 
 
 @pytest.fixture(autouse=True)
-def media_storage(settings, tmpdir):
+def _media_storage(settings, tmpdir):
     settings.MEDIA_ROOT = tmpdir.strpath
 
 
-@pytest.fixture
+@pytest.fixture()
 def user(db) -> User:
     return UserFactory()

backoffice/contrib/sites/migrations/0001_initial.py → backoffice/backoffice/contrib/sites/migrations/0001_initial.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = []
 
     operations = [

backoffice/contrib/sites/migrations/0002_alter_domain_unique.py → backoffice/backoffice/contrib/sites/migrations/0002_alter_domain_unique.py

@@ -3,7 +3,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [("sites", "0001_initial")]
 
     operations = [

backoffice/contrib/sites/migrations/0003_set_site_domain_and_name.py → backoffice/backoffice/contrib/sites/migrations/0003_set_site_domain_and_name.py

@@ -3,6 +3,7 @@
 
 http://cookiecutter-django.readthedocs.io/en/latest/faq.html#why-is-there-a-django-contrib-sites-directory-in-cookiecutter-django
 """
+
 from django.conf import settings
 from django.db import migrations
 
@@ -57,7 +58,6 @@ def update_site_backward(apps, schema_editor):
 
 
 class Migration(migrations.Migration):
-
     dependencies = [("sites", "0002_alter_domain_unique")]
 
     operations = [migrations.RunPython(update_site_forward, update_site_backward)]

backoffice/contrib/sites/migrations/0004_alter_options_ordering_domain.py → backoffice/backoffice/contrib/sites/migrations/0004_alter_options_ordering_domain.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("sites", "0003_set_site_domain_and_name"),
     ]

backoffice/management/management/commands/create_groups.py → backoffice/backoffice/management/management/commands/create_groups.py

@@ -7,7 +7,8 @@
 
 class Command(BaseCommand):
     """
-    A management command that creates admin and curator groups and gives them all permissions to Workflow model.
+    A management command that creates admin and curator groups and gives
+    them all permissions to Workflow model.
     To run it, execute the following command:
     >> python manage.py create_groups
     """
@@ -22,6 +23,9 @@ def handle(self, *args, **options):
         curator_group.permissions.add(*permissions)
         self.stdout.write(
             self.style.SUCCESS(
-                "Successfully created admin and curator groups and gave them all permissions to Workflow model."
+                """
+                Successfully created admin and curator groups and
+                gave them all permissions to Workflow model.
+                """
             )
         )