diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index cebc3667..4d1824b1 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -51,7 +51,7 @@ jobs: -v "$(pwd)"/tests:/opt/airflow/tests -v "$(pwd)"/requirements-test.txt:/opt/airflow/requirements-test.txt -v "$(pwd)"/data:/opt/airflow/data - -v "$(pwd)"/scripts/variables/variables.json:/opt/airflow/variables.json + -v "$(pwd)"/scripts:/opt/airflow/scripts -e AIRFLOW__CORE__EXECUTOR=CeleryExecutor -e AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@127.0.0.1:5432/airflow -e AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@127.0.0.1:5432/airflow @@ -61,4 +61,4 @@ jobs: -e AIRFLOW__CORE__LOAD_EXAMPLES="false" -e AIRFLOW__API__AUTH_BACKENDS="airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" registry.cern.ch/cern-sis/inspire/workflows@${{ needs.build.outputs.image-id }} - bash -c "pip install -r requirements-test.txt && airflow db init && airflow variables import /opt/airflow/variables.json && pytest /opt/airflow/tests" + bash -c "pip install -r requirements-test.txt && airflow db init && airflow connections import /opt/airflow/scripts/connections/connections.json && airflow variables import /opt/airflow/scripts/variables/variables.json && pytest /opt/airflow/tests" diff --git a/backoffice/backoffice/workflows/admin.py b/backoffice/backoffice/workflows/admin.py index 308def4b..3aa26827 100644 --- a/backoffice/backoffice/workflows/admin.py +++ b/backoffice/backoffice/workflows/admin.py @@ -3,7 +3,7 @@ from django_json_widget.widgets import JSONEditorWidget from backoffice.management.permissions import IsAdminOrCuratorUser -from backoffice.workflows.models import Workflow +from backoffice.workflows.models import Decision, Workflow class WorkflowsAdminSite(admin.AdminSite): @@ -30,8 +30,41 @@ def has_permission(self, request): ) +class BaseModelAdmin(admin.ModelAdmin): + def has_view_permission(self, request, obj=None): + """ + Returns True if the user has permission to view the Workflow model. + """ + permission_check = IsAdminOrCuratorUser() + return request.user.is_superuser or permission_check.has_permission( + request, self + ) + + def has_change_permission(self, request, obj=None): + """ + Returns True if the user has permission to change the Workflow model. + """ + permission_check = IsAdminOrCuratorUser() + return request.user.is_superuser or permission_check.has_permission( + request, self + ) + + def has_delete_permission(self, request, obj=None): + """ + Returns True if the user has permission to delete the Workflow model. + """ + permission_check = IsAdminOrCuratorUser() + return request.user.is_superuser or permission_check.has_permission( + request, self + ) + + formfield_overrides = { + JSONField: {"widget": JSONEditorWidget}, + } + + @admin.register(Workflow) -class WorkflowAdmin(admin.ModelAdmin): +class WorkflowAdmin(BaseModelAdmin): """ Admin class for Workflow model. Define get, update and delete permissions. """ @@ -56,33 +89,21 @@ class WorkflowAdmin(admin.ModelAdmin): "_updated_at", ] - formfield_overrides = { - JSONField: {"widget": JSONEditorWidget}, - } - def has_view_permission(self, request, obj=None): - """ - Returns True if the user has permission to view the Workflow model. - """ - permission_check = IsAdminOrCuratorUser() - return request.user.is_superuser or permission_check.has_permission( - request, self - ) +@admin.register(Decision) +class DecisionAdmin(BaseModelAdmin): + """ + Admin class for Decision model. Define get, update and delete permissions. + """ - def has_change_permission(self, request, obj=None): - """ - Returns True if the user has permission to change the Workflow model. - """ - permission_check = IsAdminOrCuratorUser() - return request.user.is_superuser or permission_check.has_permission( - request, self - ) + ordering = ("-_updated_at",) + search_fields = ["id", "data"] + list_display = ("id", "action_value", "user", "workflow_id") + list_filter = [ + "action", + "user", + ] - def has_delete_permission(self, request, obj=None): - """ - Returns True if the user has permission to delete the Workflow model. - """ - permission_check = IsAdminOrCuratorUser() - return request.user.is_superuser or permission_check.has_permission( - request, self - ) + @admin.display(description="action") + def action_value(self, obj): + return obj.action diff --git a/backoffice/backoffice/workflows/airflow_utils.py b/backoffice/backoffice/workflows/airflow_utils.py index 055ee001..6f570b81 100644 --- a/backoffice/backoffice/workflows/airflow_utils.py +++ b/backoffice/backoffice/workflows/airflow_utils.py @@ -156,6 +156,19 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None): :param params: parameters of new dag execution :returns: request response """ + delete_workflow_dag_runs(workflow_id, workflow_type) + + return trigger_airflow_dag( + WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params + ) + + +def delete_workflow_dag_runs(workflow_id, workflow_type): + """Deletes runs of a given workflow. + + :param workflow_id: workflow_id for dags that should be restarted + :param workflow_type: type of workflow the will be restarted + """ executed_dags_for_workflow = find_executed_dags(workflow_id, workflow_type) data = None @@ -166,6 +179,9 @@ def restart_workflow_dags(workflow_id, workflow_type, params=None): if data is None: data = dag_data["conf"].get("data") - return trigger_airflow_dag( + trigger_airflow_dag( WORKFLOW_DAGS[workflow_type].initialize, str(workflow_id), params or data + + return JsonResponse( + data={"message": f"Dag runs for worfklow {workflow_id} have been deleted"} ) diff --git a/backoffice/backoffice/workflows/api/serializers.py b/backoffice/backoffice/workflows/api/serializers.py index 8452342d..bf7bf9f5 100644 --- a/backoffice/backoffice/workflows/api/serializers.py +++ b/backoffice/backoffice/workflows/api/serializers.py @@ -6,11 +6,12 @@ from backoffice.workflows.constants import ResolutionDags, StatusChoices, WorkflowType from backoffice.workflows.documents import WorkflowDocument -from backoffice.workflows.models import Workflow, WorkflowTicket +from backoffice.workflows.models import Decision, Workflow, WorkflowTicket class WorkflowTicketSerializer(serializers.ModelSerializer): ticket_url = serializers.SerializerMethodField() + workflow_id = serializers.PrimaryKeyRelatedField(queryset=Workflow.objects.all()) class Meta: model = WorkflowTicket @@ -23,8 +24,17 @@ def get_ticket_url(self, obj): ) +class DecisionSerializer(serializers.ModelSerializer): + workflow = serializers.PrimaryKeyRelatedField(queryset=Workflow.objects.all()) + + class Meta: + model = Decision + fields = "__all__" + + class WorkflowSerializer(serializers.ModelSerializer): tickets = WorkflowTicketSerializer(many=True, read_only=True) + decisions = DecisionSerializer(many=True, read_only=True) class Meta: model = Workflow diff --git a/backoffice/backoffice/workflows/api/utils.py b/backoffice/backoffice/workflows/api/utils.py new file mode 100644 index 00000000..26868e98 --- /dev/null +++ b/backoffice/backoffice/workflows/api/utils.py @@ -0,0 +1,11 @@ +from backoffice.workflows.api.serializers import DecisionSerializer + + +def add_decision(workflow_id, user, action): + serializer_class = DecisionSerializer + data = {"workflow": workflow_id, "user": user, "action": action} + + serializer = serializer_class(data=data) + if serializer.is_valid(raise_exception=True): + serializer.save() + return serializer.data diff --git a/backoffice/backoffice/workflows/api/views.py b/backoffice/backoffice/workflows/api/views.py index 42df6630..0fe41ca5 100644 --- a/backoffice/backoffice/workflows/api/views.py +++ b/backoffice/backoffice/workflows/api/views.py @@ -23,6 +23,7 @@ from backoffice.utils.pagination import OSStandardResultsSetPagination from backoffice.workflows import airflow_utils +from backoffice.workflows.api import utils from backoffice.workflows.api.serializers import ( AuthorResolutionSerializer, WorkflowAuthorSerializer, @@ -37,7 +38,7 @@ WorkflowType, ) from backoffice.workflows.documents import WorkflowDocument -from backoffice.workflows.models import Workflow, WorkflowTicket +from backoffice.workflows.models import Decision, Workflow, WorkflowTicket logger = logging.getLogger(__name__) @@ -52,8 +53,14 @@ def get_queryset(self): return self.queryset.filter(status__status=status) return self.queryset + def perform_destroy(self, instance): + airflow_utils.delete_workflow_dag_runs(instance.id, instance.workflow_type) + super().perform_destroy(instance) + class WorkflowTicketViewSet(viewsets.ViewSet): + serializer_class = WorkflowTicketSerializer + def retrieve(self, request, *args, **kwargs): workflow_id = kwargs.get("pk") ticket_type = request.query_params.get("ticket_type") @@ -68,7 +75,7 @@ def retrieve(self, request, *args, **kwargs): workflow_ticket = WorkflowTicket.objects.get( workflow_id=workflow_id, ticket_type=ticket_type ) - serializer = WorkflowTicketSerializer(workflow_ticket) + serializer = self.serializer_class(workflow_ticket) return Response(serializer.data) except WorkflowTicket.DoesNotExist: return Response( @@ -77,27 +84,21 @@ def retrieve(self, request, *args, **kwargs): ) def create(self, request, *args, **kwargs): - workflow_id = request.data.get("workflow_id") - ticket_type = request.data.get("ticket_type") - ticket_id = request.data.get("ticket_id") - - if not all([workflow_id, ticket_type, ticket_id]): - return Response( - {"error": "Workflow_id, ticket_id and ticket_type are required."}, - status=status.HTTP_400_BAD_REQUEST, - ) + serializer = self.serializer_class(data=request.data) - try: - workflow = Workflow.objects.get(id=workflow_id) - workflow_ticket = WorkflowTicket.objects.create( - workflow_id=workflow, ticket_id=ticket_id, ticket_type=ticket_type - ) - serializer = WorkflowTicketSerializer(workflow_ticket) + if serializer.is_valid(raise_exception=True): + serializer.save() return Response(serializer.data, status=status.HTTP_201_CREATED) - except Exception as e: - return Response( - {"error": str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR - ) + + +class DecisionViewSet(viewsets.ModelViewSet): + queryset = Decision.objects.all() + + def create(self, request, *args, **kwargs): + data = utils.add_decision( + request.data["workflow_id"], request.user, request.data["action"] + ) + return Response(data, status=status.HTTP_201_CREATED) class AuthorWorkflowViewSet(viewsets.ViewSet): @@ -121,11 +122,12 @@ def create(self, request): WORKFLOW_DAGS[workflow.workflow_type].initialize, workflow.id, ) - return airflow_utils.trigger_airflow_dag( + airflow_utils.trigger_airflow_dag( WORKFLOW_DAGS[workflow.workflow_type].initialize, str(workflow.id), workflow.data, ) + return Response(serializer.data, status=status.HTTP_201_CREATED) @extend_schema( summary="Partially Updates Author", @@ -160,16 +162,22 @@ def resolve(self, request, pk=None): logger.info("Resolving data: %s", request.data) serializer = AuthorResolutionSerializer(data=request.data) if serializer.is_valid(raise_exception=True): - extra_data = {"create_ticket": serializer.validated_data["create_ticket"]} + extra_data = serializer.validated_data logger.info( "Trigger Airflow DAG: %s for %s", ResolutionDags[serializer.validated_data["value"]], pk, ) + utils.add_decision(pk, request.user, serializer.validated_data["value"]) - return airflow_utils.trigger_airflow_dag( + airflow_utils.trigger_airflow_dag( ResolutionDags[serializer.validated_data["value"]].label, pk, extra_data ) + workflow_serializer = self.serializer_class( + get_object_or_404(Workflow, pk=pk) + ) + + return Response(workflow_serializer.data) @extend_schema( summary="Restart an Author Workflow", @@ -270,9 +278,9 @@ def __init__(self, *args, **kwargs): "is_update": "is_update", } - ordering_fields = {"_updated_at": "_updated_at"} + ordering_fields = {"_updated_at": "_updated_at", "_score": "_score"} - ordering = ("-_updated_at",) + ordering = ("-_updated_at", "-_score") faceted_search_fields = { "status": { diff --git a/backoffice/backoffice/workflows/constants.py b/backoffice/backoffice/workflows/constants.py index 273faeac..6907deda 100644 --- a/backoffice/backoffice/workflows/constants.py +++ b/backoffice/backoffice/workflows/constants.py @@ -33,6 +33,10 @@ class WorkflowType(models.TextChoices): class ResolutionDags(models.TextChoices): accept = "accept", "author_create_approved_dag" reject = "reject", "author_create_rejected_dag" + accept_curate = "accept_curate", "author_create_approved_dag" + + +DECISION_CHOICES = ResolutionDags.choices class AuthorCreateDags(models.TextChoices): diff --git a/backoffice/backoffice/workflows/migrations/0009_decision.py b/backoffice/backoffice/workflows/migrations/0009_decision.py new file mode 100644 index 00000000..a33ca924 --- /dev/null +++ b/backoffice/backoffice/workflows/migrations/0009_decision.py @@ -0,0 +1,58 @@ +# Generated by Django 4.2.6 on 2024-08-15 12:25 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("workflows", "0008_alter_workflow_status_alter_workflow_workflow_type"), + ] + + operations = [ + migrations.CreateModel( + name="Decision", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "action", + models.CharField( + choices=[ + ("accept", "author_create_approved_dag"), + ("reject", "author_create_rejected_dag"), + ("accept_curate", "author_create_approved_dag"), + ], + max_length=30, + ), + ), + ("_created_at", models.DateTimeField(auto_now_add=True)), + ("_updated_at", models.DateTimeField(auto_now=True)), + ( + "user", + models.ForeignKey( + db_column="email", + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + to_field="email", + ), + ), + ( + "workflow", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="workflows.workflow", + ), + ), + ], + ), + ] diff --git a/backoffice/backoffice/workflows/migrations/0010_alter_decision_workflow_and_more.py b/backoffice/backoffice/workflows/migrations/0010_alter_decision_workflow_and_more.py new file mode 100644 index 00000000..a9cefea0 --- /dev/null +++ b/backoffice/backoffice/workflows/migrations/0010_alter_decision_workflow_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.6 on 2024-08-23 13:02 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("workflows", "0009_decision"), + ] + + operations = [ + migrations.AlterField( + model_name="decision", + name="workflow", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="decisions", + to="workflows.workflow", + ), + ), + migrations.AlterField( + model_name="workflowticket", + name="workflow_id", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="tickets", + to="workflows.workflow", + ), + ), + ] diff --git a/backoffice/backoffice/workflows/models.py b/backoffice/backoffice/workflows/models.py index cd63f476..d4fb3704 100644 --- a/backoffice/backoffice/workflows/models.py +++ b/backoffice/backoffice/workflows/models.py @@ -2,7 +2,9 @@ from django.db import models +from backoffice.users.models import User from backoffice.workflows.constants import ( + DECISION_CHOICES, DEFAULT_STATUS_CHOICE, DEFAULT_TICKET_TYPE, DEFAULT_WORKFLOW_TYPE, @@ -43,3 +45,19 @@ class WorkflowTicket(models.Model): ticket_type = models.CharField( max_length=30, choices=TICKET_TYPES, default=DEFAULT_TICKET_TYPE ) + + +class Decision(models.Model): + user = models.ForeignKey( + User, + to_field="email", + db_column="email", + on_delete=models.CASCADE, + ) + workflow = models.ForeignKey( + Workflow, related_name="decisions", on_delete=models.CASCADE + ) + action = models.CharField(max_length=30, choices=DECISION_CHOICES) + + _created_at = models.DateTimeField(auto_now_add=True) + _updated_at = models.DateTimeField(auto_now=True) diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag_runs.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag_runs.yaml new file mode 100644 index 00000000..f997984a --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag_runs.yaml @@ -0,0 +1,209 @@ +interactions: +- request: + body: '{"dag_run_id": "00000000-0000-0000-0000-000000000001", "conf": {"workflow_id": + "00000000-0000-0000-0000-000000000001"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T11:39:24.438383+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T11:39:24.438383+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T11:39:24.438383+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T11:39:24.438383+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T11:39:24.438383+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + method: DELETE + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + method: DELETE + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000001 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000001' not found\",\n \"status\": + 404,\n \"title\": \"Not Found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '293' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 11:39:24 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestWorkflowViewSet.test_delete.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestWorkflowViewSet.test_delete.yaml new file mode 100644 index 00000000..73499d6f --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestWorkflowViewSet.test_delete.yaml @@ -0,0 +1,372 @@ +interactions: +- request: + body: '{"dag_run_id": "00000000-0000-0000-0000-000000000002", "conf": {"workflow_id": + "00000000-0000-0000-0000-000000000002"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000002\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000002\",\n \"data_interval_end\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T13:16:38.257736+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T13:16:38.257736+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000002\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000002\",\n \"data_interval_end\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T13:16:38.257736+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T13:16:38.257736+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"00000000-0000-0000-0000-000000000002\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000002\",\n \"data_interval_end\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"data_interval_start\": \"2024-08-19T13:16:38.257736+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-08-19T13:16:38.257736+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-08-19T13:16:38.257736+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + method: DELETE + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_initialization_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '300' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_approved_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + method: GET + uri: http://airflow-webserver:8080/api/v1/dags/author_create_rejected_dag/dagRuns/00000000-0000-0000-0000-000000000002 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '00000000-0000-0000-0000-000000000002' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 19 Aug 2024 13:16:38 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +version: 1 diff --git a/backoffice/backoffice/workflows/tests/test_airflow_utils.py b/backoffice/backoffice/workflows/tests/test_airflow_utils.py index 2d508c0b..eb64b541 100644 --- a/backoffice/backoffice/workflows/tests/test_airflow_utils.py +++ b/backoffice/backoffice/workflows/tests/test_airflow_utils.py @@ -64,3 +64,10 @@ def test_restart_workflow_dags(self): self.workflow_id, self.workflow_type ) self.assertEqual(response.status_code, 200) + + @pytest.mark.vcr() + def test_delete_workflow_dag_runs(self): + response = airflow_utils.delete_workflow_dag_runs( + self.workflow_id, self.workflow_type + ) + self.assertEqual(response.status_code, 200) diff --git a/backoffice/backoffice/workflows/tests/test_utils.py b/backoffice/backoffice/workflows/tests/test_utils.py new file mode 100644 index 00000000..dc297c3e --- /dev/null +++ b/backoffice/backoffice/workflows/tests/test_utils.py @@ -0,0 +1,44 @@ +import uuid + +import pytest +from django.apps import apps +from django.contrib.auth import get_user_model +from django.test import TransactionTestCase +from rest_framework.exceptions import ValidationError + +from backoffice.workflows import constants +from backoffice.workflows.api import utils +from backoffice.workflows.constants import StatusChoices + +User = get_user_model() +Workflow = apps.get_model(app_label="workflows", model_name="Workflow") + + +class TestUtils(TransactionTestCase): + reset_sequences = True + fixtures = ["backoffice/fixtures/groups.json"] + + def setUp(self): + super().setUp() + self.workflow = Workflow.objects.create( + data={}, status=StatusChoices.APPROVAL, core=True, is_update=False + ) + self.user = User.objects.create_user( + email="testuser@test.com", password="12345" + ) + + def test_add_decision(self): + decision_data = utils.add_decision( + self.workflow.id, self.user, constants.ResolutionDags.accept + ) + + self.assertIsNotNone(decision_data) + + def test_add_decision_validation_errors(self): + with pytest.raises(ValidationError): + utils.add_decision(self.workflow.id, self.user, "wrong") + + with pytest.raises(ValidationError): + utils.add_decision( + uuid.UUID(int=0), self.user, constants.ResolutionDags.accept + ) diff --git a/backoffice/backoffice/workflows/tests/test_views.py b/backoffice/backoffice/workflows/tests/test_views.py index 1d5c8940..eef494ce 100644 --- a/backoffice/backoffice/workflows/tests/test_views.py +++ b/backoffice/backoffice/workflows/tests/test_views.py @@ -11,6 +11,7 @@ from django.test import TransactionTestCase from django.urls import reverse from django_opensearch_dsl.registries import registry +from rest_framework import status from rest_framework.test import APIClient from backoffice.workflows import airflow_utils @@ -18,11 +19,18 @@ WorkflowSerializer, WorkflowTicketSerializer, ) -from backoffice.workflows.constants import WORKFLOW_DAGS, StatusChoices, WorkflowType +from backoffice.workflows.constants import ( + WORKFLOW_DAGS, + AuthorCreateDags, + ResolutionDags, + StatusChoices, + WorkflowType, +) from backoffice.workflows.models import WorkflowTicket User = get_user_model() Workflow = apps.get_model(app_label="workflows", model_name="Workflow") +Decision = apps.get_model(app_label="workflows", model_name="Decision") class BaseTransactionTestCase(TransactionTestCase): @@ -55,7 +63,12 @@ class TestWorkflowViewSet(BaseTransactionTestCase): def setUp(self): super().setUp() self.workflow = Workflow.objects.create( - data={}, status=StatusChoices.APPROVAL, core=True, is_update=False + data={}, + status=StatusChoices.APPROVAL, + core=True, + is_update=False, + workflow_type=WorkflowType.AUTHOR_CREATE, + id=uuid.UUID(int=2), ) def test_list_curator(self): @@ -88,6 +101,36 @@ def test_tickets(self): assert "ticket_id" in workflow_data["tickets"][0] assert "ticket_type" in workflow_data["tickets"][0] + def test_decisions(self): + Decision.objects.create( + workflow=self.workflow, user=self.user, action=ResolutionDags.accept + ) + workflow_data = WorkflowSerializer(self.workflow).data + assert "decisions" in workflow_data + assert "action" in workflow_data["decisions"][0] + assert "user" in workflow_data["decisions"][0] + + @pytest.mark.vcr() + def test_delete(self): + self.api_client.force_authenticate(user=self.curator) + airflow_utils.trigger_airflow_dag( + AuthorCreateDags.initialize, str(self.workflow.id) + ) + assert airflow_utils.find_executed_dags( + self.workflow.id, self.workflow.workflow_type + ) + + url = reverse("api:workflows-detail", kwargs={"pk": self.workflow.id}) + response = self.api_client.delete(url) + + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + assert ( + airflow_utils.find_executed_dags( + self.workflow.id, self.workflow.workflow_type + ) + == {} + ) + class TestWorkflowSearchViewSet(BaseTransactionTestCase): endpoint = "/api/workflows/search/" @@ -172,6 +215,8 @@ def test_patch_admin(self): "test": "test", }, ) + self.assertEqual(response.json()["id"], str(self.workflow.id)) + self.assertIn("decisions", response.json()) def test_patch_anonymous(self): self.api_client.force_authenticate(user=self.user) @@ -242,8 +287,9 @@ def test_create_missing_params(self): ) assert response.status_code == 400 - assert response.data == { - "error": "Workflow_id, ticket_id and ticket_type are required." + assert response.json() == { + "workflow_id": ["This field is required."], + "ticket_id": ["This field is required."], } def test_create_happy_flow(self): @@ -317,12 +363,14 @@ def test_create_author(self): url = reverse("api:workflows-authors-list") response = self.api_client.post(url, format="json", data=data) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json(), data) @pytest.mark.vcr() def test_accept_author(self): self.api_client.force_authenticate(user=self.curator) - data = {"create_ticket": True, "value": "accept"} + action = "accept" + data = {"create_ticket": True, "value": action} response = self.api_client.post( reverse("api:workflows-authors-resolve", kwargs={"pk": self.workflow.id}), @@ -331,7 +379,11 @@ def test_accept_author(self): ) self.assertEqual(response.status_code, 200) - + self.assertEqual( + Decision.objects.filter(workflow=self.workflow.id)[0].action, action + ) + self.assertEqual(response.json()["id"], str(self.workflow.id)) + self.assertIn("decisions", response.json()) airflow_utils.delete_workflow_dag( WORKFLOW_DAGS[WorkflowType.AUTHOR_CREATE].approve, self.workflow.id ) @@ -339,7 +391,8 @@ def test_accept_author(self): @pytest.mark.vcr() def test_reject_author(self): self.api_client.force_authenticate(user=self.curator) - data = {"create_ticket": True, "value": "reject"} + action = "reject" + data = {"create_ticket": True, "value": action} response = self.api_client.post( reverse("api:workflows-authors-resolve", kwargs={"pk": self.workflow.id}), @@ -348,6 +401,11 @@ def test_reject_author(self): ) self.assertEqual(response.status_code, 200) + self.assertEqual( + Decision.objects.filter(workflow=self.workflow.id)[0].action, action + ) + self.assertEqual(response.json()["id"], str(self.workflow.id)) + self.assertIn("decisions", response.json()) airflow_utils.delete_workflow_dag( WORKFLOW_DAGS[WorkflowType.AUTHOR_CREATE].reject, self.workflow.id @@ -395,12 +453,15 @@ class TestWorkflowSearchFilterViewSet(BaseTransactionTestCase): reset_sequences = True fixtures = ["backoffice/fixtures/groups.json"] - def setUp(self): - super().setUp() - @classmethod def setUpClass(cls): super().setUpClass() + + index = registry.get_indices().pop() + with contextlib.suppress(opensearchpy.exceptions.NotFoundError): + index.delete() + index.create() + Workflow.objects.update_or_create( data={ "ids": [ @@ -476,7 +537,7 @@ def test_filter_workflow_type(self): for item in response.json()["results"]: assert item["workflow_type"] == WorkflowType.AUTHOR_CREATE - def test_ordering(self): + def test_ordering_updated_at(self): self.api_client.force_authenticate(user=self.admin) base_url = reverse("search:workflow-list") @@ -492,3 +553,45 @@ def test_ordering(self): if previous_date is not None: assert cur_date < previous_date previous_date = cur_date + + def test_ordering_score(self): + self.api_client.force_authenticate(user=self.admin) + + search_str = "search=Frank Castle^10 OR John^6" + + url = reverse("search:workflow-list") + f"?ordering=_score&{search_str}" + response = self.api_client.get(url) + self.assertEqual( + response.json()["results"][0]["data"]["name"]["preferred_name"], + "John Smith", + ) + + url = reverse("search:workflow-list") + f"?ordering=-_score&{search_str}" + response = self.api_client.get(url) + self.assertEqual( + response.json()["results"][0]["data"]["name"]["preferred_name"], + "Frank Castle", + ) + + +class TestDecisionsViewSet(BaseTransactionTestCase): + endpoint = "/api/decisions" + reset_sequences = True + fixtures = ["backoffice/fixtures/groups.json"] + + def setUp(self): + super().setUp() + self.workflow = Workflow.objects.create( + data={}, status="running", core=True, is_update=False + ) + + def test_create_decision(self): + self.api_client.force_authenticate(user=self.curator) + data = { + "workflow_id": self.workflow.id, + "action": "accept", + } + + url = reverse("api:decisions-list") + response = self.api_client.post(url, format="json", data=data) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) diff --git a/backoffice/config/api_router.py b/backoffice/config/api_router.py index 179a686b..fce8e177 100644 --- a/backoffice/config/api_router.py +++ b/backoffice/config/api_router.py @@ -4,6 +4,7 @@ from backoffice.users.api.views import UserViewSet from backoffice.workflows.api.views import ( AuthorWorkflowViewSet, + DecisionViewSet, WorkflowTicketViewSet, WorkflowViewSet, ) @@ -20,5 +21,6 @@ ) router.register("workflows", WorkflowViewSet, basename="workflows") (router.register("workflow-ticket", WorkflowTicketViewSet, basename="workflow-ticket"),) +router.register("decisions", DecisionViewSet, basename="decisions") app_name = "api" urlpatterns = router.urls diff --git a/docker-compose.yaml b/docker-compose.yaml index 96b5111d..a4a2e6b1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -57,7 +57,7 @@ x-airflow-common: &airflow-common AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres-airflow/airflow AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 AIRFLOW__CORE__FERNET_KEY: "" - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "true" + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "false" AIRFLOW__CORE__LOAD_EXAMPLES: "false" AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" AIRFLOW__WEBSERVER__RELOAD_ON_PLUGIN_CHANGE: "true" # used when modifying plugins @@ -104,7 +104,7 @@ services: - ./backoffice/.envs/local/.postgres ports: - "8000:8000" - command: sh -c 'poetry run python manage.py migrate && poetry run python manage.py opensearch index create --force && poetry run python manage.py runserver 0.0.0.0:8000' + command: sh -c 'poetry run python manage.py migrate && poetry run python manage.py opensearch index rebuild --force && poetry run python manage.py opensearch document index --force && poetry run python manage.py runserver 0.0.0.0:8000' postgres-airflow: image: postgres:13 diff --git a/workflows/dags/author/author_create/author_create_approved.py b/workflows/dags/author/author_create/author_create_approved.py index 083ae905..8927dd25 100644 --- a/workflows/dags/author/author_create/author_create_approved.py +++ b/workflows/dags/author/author_create/author_create_approved.py @@ -3,7 +3,7 @@ from airflow.decorators import dag, task from airflow.models.param import Param -from airflow.utils.trigger_rule import TriggerRule +from author.author_create.shared_tasks import close_author_create_user_ticket from hooks.backoffice.workflow_management_hook import AUTHORS, WorkflowManagementHook from hooks.backoffice.workflow_ticket_management_hook import ( WorkflowTicketManagementHook, @@ -119,16 +119,6 @@ def author_create_success_branch(**context: dict) -> str: else: return "set_author_create_workflow_status_to_error" - @task(trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS) - def close_author_create_user_ticket(**context: dict) -> None: - ticket_type = "author_create_user" - ticket_id = workflow_ticket_management_hook.get_ticket( - workflow_id=context["params"]["workflow_id"], ticket_type=ticket_type - )["ticket_id"] - endpoint = "api/tickets/resolve" - request_data = {"ticket_id": ticket_id} - inspire_http_hook.call_api(endpoint=endpoint, data=request_data, method="POST") - @task() def set_author_create_workflow_status_to_completed(**context: dict) -> None: status_name = "completed" diff --git a/workflows/dags/author/author_create/author_create_rejected.py b/workflows/dags/author/author_create/author_create_rejected.py index e492038a..9de52a1b 100644 --- a/workflows/dags/author/author_create/author_create_rejected.py +++ b/workflows/dags/author/author_create/author_create_rejected.py @@ -2,11 +2,8 @@ from airflow.decorators import dag, task from airflow.models.param import Param +from author.author_create.shared_tasks import close_author_create_user_ticket from hooks.backoffice.workflow_management_hook import AUTHORS, WorkflowManagementHook -from hooks.backoffice.workflow_ticket_management_hook import ( - WorkflowTicketManagementHook, -) -from hooks.inspirehep.inspire_http_hook import InspireHttpHook from include.utils.set_workflow_status import set_workflow_status_to_error @@ -31,19 +28,7 @@ def author_create_rejected_dag() -> None: 2. set_author_create_workflow_status_to_completed: Sets the status of the author creation workflow to 'completed'. """ - inspire_http_hook = InspireHttpHook() workflow_management_hook = WorkflowManagementHook() - workflow_ticket_management_hook = WorkflowTicketManagementHook() - - @task() - def close_author_create_user_ticket(**context: dict) -> None: - ticket_type = "author_create_user" - ticket_id = workflow_ticket_management_hook.get_ticket( - workflow_id=context["params"]["workflow_id"], ticket_type=ticket_type - )["ticket_id"] - endpoint = "/tickets/resolve" # TODO: the URL for resolving dag will change - request_data = {"ticket_id": ticket_id} - inspire_http_hook.call_api(endpoint=endpoint, data=request_data, method="POST") @task() def set_author_create_workflow_status_to_completed(**context: dict) -> None: diff --git a/workflows/dags/author/author_create/shared_tasks.py b/workflows/dags/author/author_create/shared_tasks.py new file mode 100644 index 00000000..b9b98803 --- /dev/null +++ b/workflows/dags/author/author_create/shared_tasks.py @@ -0,0 +1,17 @@ +from airflow.decorators import task +from airflow.utils.trigger_rule import TriggerRule +from hooks.backoffice.workflow_ticket_management_hook import ( + WorkflowTicketManagementHook, +) +from hooks.inspirehep.inspire_http_hook import InspireHttpHook + + +@task(trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS) +def close_author_create_user_ticket(**context: dict) -> None: + ticket_type = "author_create_user" + ticket_id = WorkflowTicketManagementHook().get_ticket( + workflow_id=context["params"]["workflow_id"], ticket_type=ticket_type + )["ticket_id"] + endpoint = "api/tickets/resolve" + request_data = {"ticket_id": ticket_id} + InspireHttpHook().call_api(endpoint=endpoint, data=request_data, method="POST") diff --git a/workflows/requirements-test.txt b/workflows/requirements-test.txt index 0c69b775..de55b38d 100644 --- a/workflows/requirements-test.txt +++ b/workflows/requirements-test.txt @@ -1,3 +1,5 @@ pytest coverage pytest-cov +pytest-vcr==1.0.2 +vcrpy==6.0.1 diff --git a/workflows/tests/__init__.py b/workflows/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/workflows/tests/cassettes/TestAuthorCreate.test_close_author_create_user_ticket.yaml b/workflows/tests/cassettes/TestAuthorCreate.test_close_author_create_user_ticket.yaml new file mode 100644 index 00000000..d7371e1c --- /dev/null +++ b/workflows/tests/cassettes/TestAuthorCreate.test_close_author_create_user_ticket.yaml @@ -0,0 +1,90 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Authorization: + - Token 2e04111a61e8f5ba6ecec52af21bbb9e81732085 + Connection: + - keep-alive + Content-Type: + - application/json + User-Agent: + - python-requests/2.31.0 + method: GET + uri: http://host.docker.internal:8000/api/workflow-ticket/f8301c06-8fa1-4124-845e-c270b910af5f/?ticket_type=author_create_user + response: + body: + string: '{"id":1,"ticket_url":"https://cerntraining.service-now.com/nav_to.do?uri=/u_request_fulfillment.do?sys_id=656f2d17878c929095f833340cbb3531","ticket_id":"656f2d17878c929095f833340cbb3531","ticket_type":"author_create_user","workflow_id":"f8301c06-8fa1-4124-845e-c270b910af5f"}' + headers: + Allow: + - GET, HEAD, OPTIONS + Content-Language: + - en + Content-Length: + - '275' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Wed, 21 Aug 2024 11:06:38 GMT + Referrer-Policy: + - same-origin + Server: + - WSGIServer/0.2 CPython/3.11.6 + Vary: + - Accept, Accept-Language, Cookie, origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 200 + message: OK +- request: + body: ticket_id=656f2d17878c929095f833340cbb3531 + headers: + Accept: + - application/vnd+inspire.record.raw+json + Accept-Encoding: + - gzip, deflate + Authorization: + - Bearer cZiS4W7K8sqyebkxQzpnSwuUKLr5Ne6qPfnoOAjP7M2IvHxQhKmwiCJpp2QC + Connection: + - keep-alive + Content-Length: + - '42' + Content-Type: + - application/x-www-form-urlencoded + User-Agent: + - python-requests/2.31.0 + method: POST + uri: https://inspirebeta.net/api/tickets/resolve + response: + body: + string: '{"message":"Ticket resolved"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset + content-length: + - '30' + content-type: + - application/json + date: + - Wed, 21 Aug 2024 11:06:39 GMT + server: + - gunicorn/19.10.0 + x-proxy-backend: + - inspire-qa_hep-web_http + status: + code: 200 + message: OK +version: 1 diff --git a/workflows/tests/test_author_create_tasks.py b/workflows/tests/test_author_create_tasks.py new file mode 100644 index 00000000..1958145a --- /dev/null +++ b/workflows/tests/test_author_create_tasks.py @@ -0,0 +1,17 @@ +import pytest +from dags.author.author_create.shared_tasks import ( + close_author_create_user_ticket, +) + + +class TestAuthorCreate: + context = { + "params": { + "workflow_id": "f8301c06-8fa1-4124-845e-c270b910af5f", + "data": {"value": "reject", "create_ticket": False}, + } + } + + @pytest.mark.vcr() + def test_close_author_create_user_ticket(self): + close_author_create_user_ticket.function(**self.context)