diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index cebc3667..0be21437 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -51,6 +51,7 @@ jobs: -v "$(pwd)"/tests:/opt/airflow/tests -v "$(pwd)"/requirements-test.txt:/opt/airflow/requirements-test.txt -v "$(pwd)"/data:/opt/airflow/data + -v "$(pwd)"/scripts/connections/connections.json:/opt/airflow/connections.json -v "$(pwd)"/scripts/variables/variables.json:/opt/airflow/variables.json -e AIRFLOW__CORE__EXECUTOR=CeleryExecutor -e AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@127.0.0.1:5432/airflow @@ -61,4 +62,4 @@ jobs: -e AIRFLOW__CORE__LOAD_EXAMPLES="false" -e AIRFLOW__API__AUTH_BACKENDS="airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" registry.cern.ch/cern-sis/inspire/workflows@${{ needs.build.outputs.image-id }} - bash -c "pip install -r requirements-test.txt && airflow db init && airflow variables import /opt/airflow/variables.json && pytest /opt/airflow/tests" + bash -c "pip install -r requirements-test.txt && airflow db init && airflow connections import /opt/airflow/connections.json && airflow variables import /opt/airflow/variables.json && pytest /opt/airflow/tests" diff --git a/backoffice/backoffice/workflows/admin.py b/backoffice/backoffice/workflows/admin.py index 308def4b..8ece1435 100644 --- a/backoffice/backoffice/workflows/admin.py +++ b/backoffice/backoffice/workflows/admin.py @@ -3,7 +3,7 @@ from django_json_widget.widgets import JSONEditorWidget from backoffice.management.permissions import IsAdminOrCuratorUser -from backoffice.workflows.models import Workflow +from backoffice.workflows.models import Decision, Workflow class WorkflowsAdminSite(admin.AdminSite): @@ -30,8 +30,41 @@ def has_permission(self, request): ) +class ModelAdmin(admin.ModelAdmin): + def has_view_permission(self, request, obj=None): + """ + Returns True if the user has permission to view the Workflow model. + """ + permission_check = IsAdminOrCuratorUser() + return request.user.is_superuser or permission_check.has_permission( + request, self + ) + + def has_change_permission(self, request, obj=None): + """ + Returns True if the user has permission to change the Workflow model. + """ + permission_check = IsAdminOrCuratorUser() + return request.user.is_superuser or permission_check.has_permission( + request, self + ) + + def has_delete_permission(self, request, obj=None): + """ + Returns True if the user has permission to delete the Workflow model. + """ + permission_check = IsAdminOrCuratorUser() + return request.user.is_superuser or permission_check.has_permission( + request, self + ) + + formfield_overrides = { + JSONField: {"widget": JSONEditorWidget}, + } + + @admin.register(Workflow) -class WorkflowAdmin(admin.ModelAdmin): +class WorkflowAdmin(ModelAdmin): """ Admin class for Workflow model. Define get, update and delete permissions. """ @@ -56,33 +89,21 @@ class WorkflowAdmin(admin.ModelAdmin): "_updated_at", ] - formfield_overrides = { - JSONField: {"widget": JSONEditorWidget}, - } - def has_view_permission(self, request, obj=None): - """ - Returns True if the user has permission to view the Workflow model. - """ - permission_check = IsAdminOrCuratorUser() - return request.user.is_superuser or permission_check.has_permission( - request, self - ) +@admin.register(Decision) +class DecisionAdmin(admin.ModelAdmin): + """ + Admin class for Decision model. Define get, update and delete permissions. + """ - def has_change_permission(self, request, obj=None): - """ - Returns True if the user has permission to change the Workflow model. - """ - permission_check = IsAdminOrCuratorUser() - return request.user.is_superuser or permission_check.has_permission( - request, self - ) + ordering = ("-_updated_at",) + search_fields = ["id", "data"] + list_display = ("id", "action_value", "user", "workflow_id") + list_filter = [ + "action", + "user", + ] - def has_delete_permission(self, request, obj=None): - """ - Returns True if the user has permission to delete the Workflow model. - """ - permission_check = IsAdminOrCuratorUser() - return request.user.is_superuser or permission_check.has_permission( - request, self - ) + @admin.display(description="action") + def action_value(self, obj): + return obj.action diff --git a/backoffice/backoffice/workflows/api/serializers.py b/backoffice/backoffice/workflows/api/serializers.py index 8452342d..3ff27d99 100644 --- a/backoffice/backoffice/workflows/api/serializers.py +++ b/backoffice/backoffice/workflows/api/serializers.py @@ -6,7 +6,7 @@ from backoffice.workflows.constants import ResolutionDags, StatusChoices, WorkflowType from backoffice.workflows.documents import WorkflowDocument -from backoffice.workflows.models import Workflow, WorkflowTicket +from backoffice.workflows.models import Decision, Workflow, WorkflowTicket class WorkflowTicketSerializer(serializers.ModelSerializer): @@ -31,6 +31,12 @@ class Meta: fields = "__all__" +class DecisionSerializer(serializers.ModelSerializer): + class Meta: + model = Decision + fields = "__all__" + + class WorkflowDocumentSerializer(DocumentSerializer): class Meta: document = WorkflowDocument diff --git a/backoffice/backoffice/workflows/api/views.py b/backoffice/backoffice/workflows/api/views.py index 5ddd5881..80e292ee 100644 --- a/backoffice/backoffice/workflows/api/views.py +++ b/backoffice/backoffice/workflows/api/views.py @@ -25,6 +25,7 @@ from backoffice.workflows import airflow_utils from backoffice.workflows.api.serializers import ( AuthorResolutionSerializer, + DecisionSerializer, WorkflowAuthorSerializer, WorkflowDocumentSerializer, WorkflowSerializer, @@ -37,7 +38,7 @@ WorkflowType, ) from backoffice.workflows.documents import WorkflowDocument -from backoffice.workflows.models import Workflow, WorkflowTicket +from backoffice.workflows.models import Decision, Workflow, WorkflowTicket logger = logging.getLogger(__name__) @@ -100,6 +101,38 @@ def create(self, request, *args, **kwargs): ) +class DecisionViewSet(viewsets.ModelViewSet): + queryset = Decision.objects.all() + serializer_class = DecisionSerializer + + def get_queryset(self): + status = self.request.query_params.get("status") + if status: + return self.queryset.filter(status__status=status) + return self.queryset + + def create(self, request, *args, **kwargs): + workflow_id = request.data.get("workflow_id") + action = request.data.get("action") + + if not all([workflow_id, action]): + return Response( + {"error": "workflow_id and action are required."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + try: + decision = Decision.objects.create( + workflow_id=workflow_id, user=request.user, action=action + ) + serializer = DecisionSerializer(decision) + return Response(serializer.data, status=status.HTTP_201_CREATED) + except Exception as e: + return Response( + {"error": str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + class AuthorWorkflowViewSet(viewsets.ViewSet): serializer_class = WorkflowAuthorSerializer @@ -160,7 +193,7 @@ def resolve(self, request, pk=None): logger.info("Resolving data: %s", request.data) serializer = AuthorResolutionSerializer(data=request.data) if serializer.is_valid(raise_exception=True): - extra_data = {"create_ticket": serializer.validated_data["create_ticket"]} + extra_data = serializer.validated_data logger.info( "Trigger Airflow DAG: %s for %s", ResolutionDags[serializer.validated_data["value"]], diff --git a/backoffice/backoffice/workflows/constants.py b/backoffice/backoffice/workflows/constants.py index 273faeac..2af91708 100644 --- a/backoffice/backoffice/workflows/constants.py +++ b/backoffice/backoffice/workflows/constants.py @@ -33,6 +33,10 @@ class WorkflowType(models.TextChoices): class ResolutionDags(models.TextChoices): accept = "accept", "author_create_approved_dag" reject = "reject", "author_create_rejected_dag" + accept_curate = "accept_curate", "author_create_approved_dag" + + +DECISION_CHOICES = ResolutionDags.choices + [] class AuthorCreateDags(models.TextChoices): diff --git a/backoffice/backoffice/workflows/migrations/0009_decision.py b/backoffice/backoffice/workflows/migrations/0009_decision.py new file mode 100644 index 00000000..a33ca924 --- /dev/null +++ b/backoffice/backoffice/workflows/migrations/0009_decision.py @@ -0,0 +1,58 @@ +# Generated by Django 4.2.6 on 2024-08-15 12:25 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("workflows", "0008_alter_workflow_status_alter_workflow_workflow_type"), + ] + + operations = [ + migrations.CreateModel( + name="Decision", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "action", + models.CharField( + choices=[ + ("accept", "author_create_approved_dag"), + ("reject", "author_create_rejected_dag"), + ("accept_curate", "author_create_approved_dag"), + ], + max_length=30, + ), + ), + ("_created_at", models.DateTimeField(auto_now_add=True)), + ("_updated_at", models.DateTimeField(auto_now=True)), + ( + "user", + models.ForeignKey( + db_column="email", + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + to_field="email", + ), + ), + ( + "workflow", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="workflows.workflow", + ), + ), + ], + ), + ] diff --git a/backoffice/backoffice/workflows/models.py b/backoffice/backoffice/workflows/models.py index cd63f476..590c831d 100644 --- a/backoffice/backoffice/workflows/models.py +++ b/backoffice/backoffice/workflows/models.py @@ -2,7 +2,9 @@ from django.db import models +from backoffice.users.models import User from backoffice.workflows.constants import ( + DECISION_CHOICES, DEFAULT_STATUS_CHOICE, DEFAULT_TICKET_TYPE, DEFAULT_WORKFLOW_TYPE, @@ -43,3 +45,14 @@ class WorkflowTicket(models.Model): ticket_type = models.CharField( max_length=30, choices=TICKET_TYPES, default=DEFAULT_TICKET_TYPE ) + + +class Decision(models.Model): + user = models.ForeignKey( + User, to_field="email", db_column="email", on_delete=models.CASCADE + ) + workflow = models.ForeignKey(Workflow, on_delete=models.CASCADE) + action = models.CharField(max_length=30, choices=DECISION_CHOICES) + + _created_at = models.DateTimeField(auto_now_add=True) + _updated_at = models.DateTimeField(auto_now=True) diff --git a/backoffice/backoffice/workflows/tests/test_views.py b/backoffice/backoffice/workflows/tests/test_views.py index 9127ada4..a3978eaa 100644 --- a/backoffice/backoffice/workflows/tests/test_views.py +++ b/backoffice/backoffice/workflows/tests/test_views.py @@ -11,6 +11,7 @@ from django.test import TransactionTestCase from django.urls import reverse from django_opensearch_dsl.registries import registry +from rest_framework import status from rest_framework.test import APIClient from backoffice.workflows import airflow_utils @@ -23,6 +24,7 @@ User = get_user_model() Workflow = apps.get_model(app_label="workflows", model_name="Workflow") +Decision = apps.get_model(app_label="workflows", model_name="Decision") class BaseTransactionTestCase(TransactionTestCase): @@ -487,3 +489,26 @@ def test_ordering(self): if previous_date is not None: assert cur_date < previous_date previous_date = cur_date + + +class TestDecisionsViewSet(BaseTransactionTestCase): + endpoint = "/api/decisions" + reset_sequences = True + fixtures = ["backoffice/fixtures/groups.json"] + + def setUp(self): + super().setUp() + self.workflow = Workflow.objects.create( + data={}, status="running", core=True, is_update=False + ) + + def test_create_decision(self): + self.api_client.force_authenticate(user=self.curator) + data = { + "workflow_id": self.workflow.id, + "action": "accept", + } + + url = reverse("api:decisions-list") + response = self.api_client.post(url, format="json", data=data) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) diff --git a/backoffice/config/api_router.py b/backoffice/config/api_router.py index 179a686b..fce8e177 100644 --- a/backoffice/config/api_router.py +++ b/backoffice/config/api_router.py @@ -4,6 +4,7 @@ from backoffice.users.api.views import UserViewSet from backoffice.workflows.api.views import ( AuthorWorkflowViewSet, + DecisionViewSet, WorkflowTicketViewSet, WorkflowViewSet, ) @@ -20,5 +21,6 @@ ) router.register("workflows", WorkflowViewSet, basename="workflows") (router.register("workflow-ticket", WorkflowTicketViewSet, basename="workflow-ticket"),) +router.register("decisions", DecisionViewSet, basename="decisions") app_name = "api" urlpatterns = router.urls diff --git a/workflows/__init__.py b/workflows/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/workflows/dags/author/author_create/author_create_approved.py b/workflows/dags/author/author_create/author_create_approved.py index 083ae905..529fd4c0 100644 --- a/workflows/dags/author/author_create/author_create_approved.py +++ b/workflows/dags/author/author_create/author_create_approved.py @@ -4,6 +4,7 @@ from airflow.decorators import dag, task from airflow.models.param import Param from airflow.utils.trigger_rule import TriggerRule +from author.author_create.shared_tasks import create_decision_on_curation_choice from hooks.backoffice.workflow_management_hook import AUTHORS, WorkflowManagementHook from hooks.backoffice.workflow_ticket_management_hook import ( WorkflowTicketManagementHook, @@ -24,7 +25,6 @@ params={ "workflow_id": Param(type="string", default=""), "data": Param(type="object", default={}), - "create_ticket": Param(type="boolean", default=False), }, start_date=datetime.datetime(2024, 5, 5), schedule=None, @@ -66,10 +66,9 @@ def author_check_approval_branch(**context: dict) -> None: dag goes either to create_ticket_on_author_approval task or directly to create_author_on_inspire """ - if context["params"]["create_ticket"]: + if context["params"]["data"]["create_ticket"]: return "create_author_create_curation_ticket" - else: - return "empty_task" + return "close_author_create_user_ticket" @task def create_author_create_curation_ticket(**context: dict) -> None: @@ -138,11 +137,6 @@ def set_author_create_workflow_status_to_completed(**context: dict) -> None: collection=AUTHORS, ) - @task - def empty_task() -> None: - # Logic to combine the results of branches - pass - @task() def set_author_create_workflow_status_to_error(**context: dict) -> None: ti = context["ti"] @@ -165,24 +159,9 @@ def set_author_create_workflow_status_to_error(**context: dict) -> None: set_author_create_workflow_status_to_completed() ) set_workflow_status_to_error_task = set_author_create_workflow_status_to_error() - combine_ticket_and_no_ticket_task = empty_task() + create_decision_on_curation_choice_task = create_decision_on_curation_choice() # task dependencies - ticket_branch = create_author_create_curation_ticket_task - ( - ticket_branch - >> close_author_create_user_ticket_task - >> set_workflow_status_to_completed_task - ) - - no_ticket_branch = combine_ticket_and_no_ticket_task - ( - no_ticket_branch - >> close_author_create_user_ticket_task - >> set_workflow_status_to_completed_task - ) - - author_check_approval_branch_task >> [ticket_branch, no_ticket_branch] ( set_status_to_running_task >> create_author_on_inspire_task @@ -192,6 +171,16 @@ def set_author_create_workflow_status_to_error(**context: dict) -> None: author_check_approval_branch_task, set_workflow_status_to_error_task, ] + ( + [ + author_check_approval_branch_task + >> create_author_create_curation_ticket_task, + author_check_approval_branch_task, + ] + >> close_author_create_user_ticket_task + >> create_decision_on_curation_choice_task + >> set_workflow_status_to_completed_task + ) author_create_approved_dag() diff --git a/workflows/dags/author/author_create/author_create_rejected.py b/workflows/dags/author/author_create/author_create_rejected.py index e492038a..cdd7c3f8 100644 --- a/workflows/dags/author/author_create/author_create_rejected.py +++ b/workflows/dags/author/author_create/author_create_rejected.py @@ -2,6 +2,7 @@ from airflow.decorators import dag, task from airflow.models.param import Param +from author.author_create.shared_tasks import create_decision_on_curation_choice from hooks.backoffice.workflow_management_hook import AUTHORS, WorkflowManagementHook from hooks.backoffice.workflow_ticket_management_hook import ( WorkflowTicketManagementHook, @@ -67,9 +68,15 @@ def set_workflow_status_to_running(**context): set_status_to_running_task = set_workflow_status_to_running() close_ticket_task = close_author_create_user_ticket() set_status_completed_task = set_author_create_workflow_status_to_completed() + create_decision_on_curation_choice_task = create_decision_on_curation_choice() # task dependencies - set_status_to_running_task >> close_ticket_task >> set_status_completed_task + ( + set_status_to_running_task + >> close_ticket_task + >> create_decision_on_curation_choice_task + >> set_status_completed_task + ) author_create_rejected_dag() diff --git a/workflows/dags/process_until_breakpoint.py b/workflows/dags/author/author_create/process_until_breakpoint.py similarity index 100% rename from workflows/dags/process_until_breakpoint.py rename to workflows/dags/author/author_create/process_until_breakpoint.py diff --git a/workflows/dags/author/author_create/shared_tasks.py b/workflows/dags/author/author_create/shared_tasks.py new file mode 100644 index 00000000..5910e264 --- /dev/null +++ b/workflows/dags/author/author_create/shared_tasks.py @@ -0,0 +1,14 @@ +from airflow.decorators import task +from hooks.backoffice.base import BackofficeHook + + +@task() +def create_decision_on_curation_choice(**context): + print("wow") + print(context) + data = { + "action": context["params"]["data"]["value"], + "workflow_id": context["params"]["workflow_id"], + } + + return BackofficeHook().request(method="POST", data=data, endpoint="api/decisions/") diff --git a/workflows/plugins/hooks/backoffice/base.py b/workflows/plugins/hooks/backoffice/base.py index 7a266e30..e443d4b2 100644 --- a/workflows/plugins/hooks/backoffice/base.py +++ b/workflows/plugins/hooks/backoffice/base.py @@ -58,3 +58,11 @@ def run( prepped_request = session.prepare_request(req) self.log.info("Sending '%s' to url: %s", method, url) return self.run_and_check(session, prepped_request, extra_options) + + def request(self, method, data, endpoint): + return self.run_with_advanced_retry( + _retry_args=self.tenacity_retry_kwargs, + method=method, + data=data, + endpoint=endpoint, + ) diff --git a/workflows/requirements-test.txt b/workflows/requirements-test.txt index 0c69b775..de55b38d 100644 --- a/workflows/requirements-test.txt +++ b/workflows/requirements-test.txt @@ -1,3 +1,5 @@ pytest coverage pytest-cov +pytest-vcr==1.0.2 +vcrpy==6.0.1 diff --git a/workflows/tests/__init__.py b/workflows/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/workflows/tests/cassettes/TestAuthorCreate.test_create_decision_on_curation_choice.yaml b/workflows/tests/cassettes/TestAuthorCreate.test_create_decision_on_curation_choice.yaml new file mode 100644 index 00000000..d739c0bd --- /dev/null +++ b/workflows/tests/cassettes/TestAuthorCreate.test_create_decision_on_curation_choice.yaml @@ -0,0 +1,50 @@ +interactions: +- request: + body: '{"action": "create", "workflow_id": "ecaa62db-1326-43cf-8885-da96c544af42"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Authorization: + - Token 2e04111a61e8f5ba6ecec52af21bbb9e81732085 + Connection: + - keep-alive + Content-Length: + - '75' + Content-Type: + - application/json + User-Agent: + - python-requests/2.31.0 + method: POST + uri: http://host.docker.internal:8000/api/decisions/ + response: + body: + string: '{"id":1,"action":"create","_created_at":"2024-08-19T15:13:28.514638Z","_updated_at":"2024-08-19T15:13:28.514646Z","user":"admin@admin.com","workflow":"ecaa62db-1326-43cf-8885-da96c544af42"}' + headers: + Allow: + - GET, POST, HEAD, OPTIONS + Content-Language: + - en + Content-Length: + - '189' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Mon, 19 Aug 2024 15:13:28 GMT + Referrer-Policy: + - same-origin + Server: + - WSGIServer/0.2 CPython/3.11.6 + Vary: + - Accept, Accept-Language, Cookie, origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 201 + message: Created +version: 1 diff --git a/workflows/tests/test_author_create_tasks.py b/workflows/tests/test_author_create_tasks.py new file mode 100644 index 00000000..048c16e0 --- /dev/null +++ b/workflows/tests/test_author_create_tasks.py @@ -0,0 +1,20 @@ +import pytest +from dags.author.author_create.shared_tasks import ( + create_decision_on_curation_choice, +) + + +class TestAuthorCreate: + context = { + "params": { + "workflow_id": "ecaa62db-1326-43cf-8885-da96c544af42", + "data": { + "value": "create", + }, + } + } + + @pytest.mark.vcr() + def test_create_decision_on_curation_choice(self): + result = create_decision_on_curation_choice.function(**self.context) + assert result.status_code == 201