From becd89cde080504f60d008b20b0e4fca34a5be6e Mon Sep 17 00:00:00 2001 From: DonHaul Date: Mon, 18 Nov 2024 13:55:21 +0100 Subject: [PATCH] airflow: fixing snowticket workflow * ref: cern-sis/issues-inspire/issues/614 --- Makefile | 1 + .../backoffice/authors/airflow_utils.py | 18 ++- backoffice/backoffice/authors/api/views.py | 8 +- .../authors/fixtures/workflows.json | 30 ++++ ...AirflowUtils.test_trigger_airflow_dag.yaml | 27 ++-- .../authors/tests/test_airflow_utils.py | 13 +- .../backoffice/authors/tests/test_views.py | 1 - ...ExperimentSubmissionPageContainer.test.jsx | 38 +++++ .../author_create/author_create_approved.py | 99 ++++++++----- .../author_create/author_create_init.py | 51 +++++-- .../author_create/author_create_rejected.py | 16 ++- .../dags/author/author_create/shared_tasks.py | 17 --- .../author/author_update/author_update.py | 45 ++++-- .../hooks/inspirehep/inspire_http_hook.py | 63 ++++++++- workflows/plugins/include/utils/tickets.py | 9 ++ ....test_close_author_create_user_ticket.yaml | 90 ------------ ....test_close_author_create_user_ticket.yaml | 44 ++++++ ..._create_author_create_curation_ticket.yaml | 90 ++++++++++++ ...test_create_author_create_user_ticket.yaml | 132 ++++++++++++++++++ workflows/tests/test_author_create_tasks.py | 84 +++++++++-- 20 files changed, 668 insertions(+), 208 deletions(-) create mode 100644 backoffice/backoffice/authors/fixtures/workflows.json create mode 100644 ui/src/submissions/data/containers/__tests__/ExperimentSubmissionPageContainer.test.jsx delete mode 100644 workflows/dags/author/author_create/shared_tasks.py create mode 100644 workflows/plugins/include/utils/tickets.py delete mode 100644 workflows/tests/cassettes/TestAuthorCreate.test_close_author_create_user_ticket.yaml create mode 100644 workflows/tests/cassettes/TestAuthorCreateApproved.test_close_author_create_user_ticket.yaml create mode 100644 workflows/tests/cassettes/TestAuthorCreateApproved.test_create_author_create_curation_ticket.yaml create mode 100644 workflows/tests/cassettes/TestAuthorCreateInit.test_create_author_create_user_ticket.yaml diff --git a/Makefile b/Makefile index 25bc1095d6..3e3eeb1851 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ django-setup: docker compose exec backoffice-webserver python manage.py create_groups docker compose exec backoffice-webserver python manage.py loaddata backoffice/users/fixtures/users.json docker compose exec backoffice-webserver python manage.py loaddata backoffice/users/fixtures/tokens.json + docker compose exec backoffice-webserver python manage.py loaddata backoffice/authors/fixtures/workflows.json echo "\033[1;32memail: admin@admin.com / password: admin \033[0m" echo "Backoffice initialized" diff --git a/backoffice/backoffice/authors/airflow_utils.py b/backoffice/backoffice/authors/airflow_utils.py index 80a6404ab3..d389078ceb 100644 --- a/backoffice/backoffice/authors/airflow_utils.py +++ b/backoffice/backoffice/authors/airflow_utils.py @@ -5,19 +5,19 @@ from django.http import JsonResponse from requests.exceptions import RequestException from rest_framework import status +import json +from django.core.serializers.json import DjangoJSONEncoder from backoffice.authors.constants import WORKFLOW_DAGS AIRFLOW_BASE_URL = environ.get("AIRFLOW_BASE_URL") -AIRFLOW_HEADERS = { - "Authorization": f"Basic {environ.get('AIRFLOW_TOKEN')}", -} +AIRFLOW_HEADERS = {"Authorization": f"Basic {environ.get('AIRFLOW_TOKEN')}"} logger = logging.getLogger(__name__) -def trigger_airflow_dag(dag_id, workflow_id, extra_data=None): +def trigger_airflow_dag(dag_id, workflow_id, extra_data=None, workflow=None): """Triggers an airflow dag. :param dag_id: name of the dag to run @@ -27,8 +27,10 @@ def trigger_airflow_dag(dag_id, workflow_id, extra_data=None): data = {"dag_run_id": str(workflow_id), "conf": {"workflow_id": str(workflow_id)}} - if extra_data is not None: + if extra_data: data["conf"]["data"] = extra_data + if workflow: + data["conf"]["workflow"] = workflow url = f"{AIRFLOW_BASE_URL}/api/v1/dags/{dag_id}/dagRuns" @@ -39,7 +41,11 @@ def trigger_airflow_dag(dag_id, workflow_id, extra_data=None): data, url, ) - response = requests.post(url, json=data, headers=AIRFLOW_HEADERS) + response = requests.post( + url, + data=json.dumps(data, cls=DjangoJSONEncoder), + headers=AIRFLOW_HEADERS | {"Content-Type": "application/json"}, + ) response.raise_for_status() return JsonResponse(response.json()) except RequestException: diff --git a/backoffice/backoffice/authors/api/views.py b/backoffice/backoffice/authors/api/views.py index fe81f3f85f..89496ecdf7 100644 --- a/backoffice/backoffice/authors/api/views.py +++ b/backoffice/backoffice/authors/api/views.py @@ -140,7 +140,7 @@ def create(self, request): airflow_utils.trigger_airflow_dag( WORKFLOW_DAGS[workflow.workflow_type].initialize, str(workflow.id), - workflow.data, + workflow=serializer.data, ) return Response(serializer.data, status=status.HTTP_201_CREATED) @@ -177,7 +177,6 @@ def resolve(self, request, pk=None): logger.info("Resolving data: %s", request.data) serializer = self.resolution_serializer(data=request.data) if serializer.is_valid(raise_exception=True): - extra_data = serializer.validated_data logger.info( "Trigger Airflow DAG: %s for %s", AuthorResolutionDags[serializer.validated_data["value"]], @@ -185,10 +184,13 @@ def resolve(self, request, pk=None): ) utils.add_decision(pk, request.user, serializer.validated_data["value"]) + workflow = self.get_serializer(AuthorWorkflow.objects.get(pk=pk)).data + airflow_utils.trigger_airflow_dag( AuthorResolutionDags[serializer.validated_data["value"]].label, pk, - extra_data, + serializer.data, + workflow=workflow, ) workflow_serializer = self.serializer_class( get_object_or_404(AuthorWorkflow, pk=pk) diff --git a/backoffice/backoffice/authors/fixtures/workflows.json b/backoffice/backoffice/authors/fixtures/workflows.json new file mode 100644 index 0000000000..859e6a06b3 --- /dev/null +++ b/backoffice/backoffice/authors/fixtures/workflows.json @@ -0,0 +1,30 @@ +[ +{ + "model": "authors.authorworkflow", + "pk": "00000000-0000-0000-0000-000000001521", + "fields": { + "workflow_type": "AUTHOR_CREATE", + "data": { + "name": { + "value": "B, Third", + "preferred_name": "Third B" + }, + "status": "active", + "_collections": [ + "Authors" + ], + "acquisition_source": { + "email": "user@cern.ch", + "orcid": "0000-0000-0000-0000", + "method": "submitter", + "source": "submitter", + "datetime": "2024-11-18T11:34:19.809575", + "internal_uid": 50872 + } + }, + "status": "running", + "_created_at": "2024-11-25T13:49:53.009Z", + "_updated_at": "2024-11-25T13:49:54.756Z" + } +} +] diff --git a/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_trigger_airflow_dag.yaml b/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_trigger_airflow_dag.yaml index 1720604808..f2a60c79e6 100644 --- a/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_trigger_airflow_dag.yaml +++ b/backoffice/backoffice/authors/tests/cassettes/TestAirflowUtils.test_trigger_airflow_dag.yaml @@ -1,7 +1,8 @@ interactions: - request: body: '{"dag_run_id": "00000000-0000-0000-0000-000000000001", "conf": {"workflow_id": - "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}}}' + "00000000-0000-0000-0000-000000000001", "data": {"test": "test"}, "workflow": + {"id": "id"}}}' headers: Accept: - '*/*' @@ -10,7 +11,7 @@ interactions: Connection: - keep-alive Content-Length: - - '145' + - '171' Content-Type: - application/json method: POST @@ -18,25 +19,25 @@ interactions: response: body: string: "{\n \"conf\": {\n \"data\": {\n \"test\": \"test\"\n },\n - \ \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n },\n \"dag_id\": - \"author_create_initialization_dag\",\n \"dag_run_id\": \"00000000-0000-0000-0000-000000000001\",\n - \ \"data_interval_end\": \"2024-10-18T11:54:51.246355+00:00\",\n \"data_interval_start\": - \"2024-10-18T11:54:51.246355+00:00\",\n \"end_date\": null,\n \"execution_date\": - \"2024-10-18T11:54:51.246355+00:00\",\n \"external_trigger\": true,\n \"last_scheduling_decision\": - null,\n \"logical_date\": \"2024-10-18T11:54:51.246355+00:00\",\n \"note\": - null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n \"state\": - \"queued\"\n}\n" + \ \"workflow\": {\n \"id\": \"id\"\n },\n \"workflow_id\": \"00000000-0000-0000-0000-000000000001\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"00000000-0000-0000-0000-000000000001\",\n \"data_interval_end\": \"2024-11-20T14:25:30.752617+00:00\",\n + \ \"data_interval_start\": \"2024-11-20T14:25:30.752617+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-11-20T14:25:30.752617+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-11-20T14:25:30.752617+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" headers: Cache-Control: - no-store Connection: - close Content-Length: - - '621' + - '663' Content-Type: - application/json Date: - - Fri, 18 Oct 2024 11:54:51 GMT + - Wed, 20 Nov 2024 14:25:30 GMT Server: - gunicorn X-Robots-Tag: @@ -68,7 +69,7 @@ interactions: Content-Type: - application/json Date: - - Fri, 18 Oct 2024 11:54:51 GMT + - Wed, 20 Nov 2024 14:25:30 GMT Server: - gunicorn X-Robots-Tag: diff --git a/backoffice/backoffice/authors/tests/test_airflow_utils.py b/backoffice/backoffice/authors/tests/test_airflow_utils.py index 073330fa6a..caca9fe8e6 100644 --- a/backoffice/backoffice/authors/tests/test_airflow_utils.py +++ b/backoffice/backoffice/authors/tests/test_airflow_utils.py @@ -1,5 +1,5 @@ import uuid - +import json import pytest from backoffice.authors import airflow_utils from backoffice.authors.constants import WORKFLOW_DAGS, WorkflowType @@ -11,8 +11,14 @@ def setUp(self): self.workflow_id = uuid.UUID(int=1) self.workflow_type = WorkflowType.AUTHOR_CREATE self.dag_id = WORKFLOW_DAGS[self.workflow_type].initialize + self.extra_data = {"test": "test"} + self.workflow_serialized = {"id": "id"} + self.response = airflow_utils.trigger_airflow_dag( - self.dag_id, str(self.workflow_id), {"test": "test"} + self.dag_id, + str(self.workflow_id), + self.extra_data, + self.workflow_serialized, ) def tearDown(self): @@ -20,7 +26,10 @@ def tearDown(self): @pytest.mark.vcr def test_trigger_airflow_dag(self): + json_content = json.loads(self.response.content) self.assertEqual(self.response.status_code, 200) + self.assertEqual(json_content["conf"]["data"], self.extra_data) + self.assertEqual(json_content["conf"]["workflow"], self.workflow_serialized) @pytest.mark.vcr def test_restart_failed_tasks(self): diff --git a/backoffice/backoffice/authors/tests/test_views.py b/backoffice/backoffice/authors/tests/test_views.py index 0c19c686fc..24a0a8a5b2 100644 --- a/backoffice/backoffice/authors/tests/test_views.py +++ b/backoffice/backoffice/authors/tests/test_views.py @@ -305,7 +305,6 @@ def test_create_happy_flow(self): "ticket_id": "dc94caad1b4f71502d06117a3b4bcb25", "ticket_type": "author_create_user", } - # import ipdb; ipdb.set_trace() response = self.api_client.post(self.endpoint, format="json", data=data) assert response.status_code == 201 diff --git a/ui/src/submissions/data/containers/__tests__/ExperimentSubmissionPageContainer.test.jsx b/ui/src/submissions/data/containers/__tests__/ExperimentSubmissionPageContainer.test.jsx new file mode 100644 index 0000000000..fbc7fbb02c --- /dev/null +++ b/ui/src/submissions/data/containers/__tests__/ExperimentSubmissionPageContainer.test.jsx @@ -0,0 +1,38 @@ +import React from 'react'; +import { mount, shallow } from 'enzyme'; +import { Provider } from 'react-redux'; +import { MemoryRouter } from 'react-router-dom'; +import { fromJS } from 'immutable'; + +import ExperimentSubmissionPageContainer, { ExperimentSubmissionPage } from '../DataSubmissionPageContainer'; + +import { getStoreWithState } from '../../../../fixtures/store'; + +describe('ExperimentSubmissionSuccessPageContainer', () => { + it('passes props to ExperimentSubmissionSucessPage', () => { + const store = getStoreWithState({ + submissions: fromJS({ + submitError: null, + }), + }); + const wrapper = mount( + + + + + + ); + expect(wrapper.find(ExperimentSubmissionPage)).toHaveProp({ + error: null, + }); + }); + + describe('ExperimentSubmissionSucessPage', () => { + it('renders', () => { + const component = shallow( + {}} /> + ); + expect(component).toMatchSnapshot(); + }); + }); +}); diff --git a/workflows/dags/author/author_create/author_create_approved.py b/workflows/dags/author/author_create/author_create_approved.py index 4f31edbdbb..9ece060fdd 100644 --- a/workflows/dags/author/author_create/author_create_approved.py +++ b/workflows/dags/author/author_create/author_create_approved.py @@ -3,12 +3,15 @@ from airflow.decorators import dag, task from airflow.models.param import Param -from author.author_create.shared_tasks import close_author_create_user_ticket +from airflow.utils.trigger_rule import TriggerRule from hooks.backoffice.workflow_management_hook import AUTHORS, WorkflowManagementHook from hooks.backoffice.workflow_ticket_management_hook import ( AuthorWorkflowTicketManagementHook, ) -from hooks.inspirehep.inspire_http_hook import InspireHttpHook +from hooks.inspirehep.inspire_http_hook import ( + AUTHOR_CURATION_FUNCTIONAL_CATEGORY, + InspireHttpHook, +) from hooks.inspirehep.inspire_http_record_management_hook import ( InspireHTTPRecordManagementHook, ) @@ -16,6 +19,7 @@ get_wf_status_from_inspire_response, set_workflow_status_to_error, ) +from include.utils.tickets import get_ticket_by_type logger = logging.getLogger(__name__) @@ -30,6 +34,7 @@ schedule=None, catchup=False, on_failure_callback=set_workflow_status_to_error, # TODO: what if callback fails? Data in backoffice not up to date! + tags=["authors"], ) def author_create_approved_dag(): """Defines the DAG for the author creation workflow after curator's approval. @@ -60,29 +65,40 @@ def set_workflow_status_to_running(**context): @task.branch() def author_check_approval_branch(**context: dict) -> None: - """Branching for the workflow: based on create_ticket parameter + """Branching for the workflow: based on value parameter dag goes either to create_ticket_on_author_approval task or directly to create_author_on_inspire """ - if context["params"]["create_ticket"]: + if context["params"]["data"]["value"] == "accept_curate": return "create_author_create_curation_ticket" else: - return "empty_task" + return "close_author_create_user_ticket" @task def create_author_create_curation_ticket(**context: dict) -> None: - endpoint = "api/tickets/create" - request_data = { - "functional_category": "", - "workflow_id": context["params"]["workflow_id"], - "subject": "test", # TODO: update subject and description - "description": "test", - "caller_email": "", # leave empty - "template": "curation_needed_author", # TODO: check template - } - response = inspire_http_hook.call_api( - endpoint=endpoint, data=request_data, method="POST" + workflow_data = context["params"]["workflow"]["data"] + email = workflow_data["acquisition_source"]["email"] + + bai = f"[{workflow_data.get('bai')}]" if workflow_data.get("bai") else "" + + control_number = context["ti"].xcom_pull( + task_ids="create_author_on_inspire", key="control_number" + ) + + inspire_http_hook.get_conn() + + response = inspire_http_hook.create_ticket( + AUTHOR_CURATION_FUNCTIONAL_CATEGORY, + "curation_needed_author", + f"Curation needed for author" + f"{workflow_data.get('name').get('preferred_name')} {bai}", + email, + { + "email": email, + "record_url": f"{inspire_http_hook.base_url}/authors/{control_number}", + }, ) + workflow_ticket_management_hook.create_ticket_entry( workflow_id=context["params"]["workflow_id"], ticket_id=response.json()["ticket_id"], @@ -100,12 +116,14 @@ def create_author_on_inspire(**context: dict) -> str: status = get_wf_status_from_inspire_response(response) if response.ok: control_number = response.json()["metadata"]["control_number"] + context["ti"].xcom_push(key="control_number", value=control_number) logger.info(f"Created author with control number: {control_number}") workflow_data["data"]["control_number"] = control_number workflow_management_hook.partial_update_workflow( workflow_id=context["params"]["workflow_id"], workflow_partial_update_data={"data": workflow_data["data"]}, ) + logger.info(f"Workflow status: {status}") return status @task.branch() @@ -124,10 +142,26 @@ def set_author_create_workflow_status_to_completed(**context: dict) -> None: status_name=status_name, workflow_id=context["params"]["workflow_id"] ) - @task - def empty_task() -> None: - # Logic to combine the results of branches - pass + @task(trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS) + def close_author_create_user_ticket(**context: dict) -> None: + ticket_id = get_ticket_by_type( + context["params"]["workflow"], "author_create_user" + )["ticket_id"] + + workflow_data = context["params"]["workflow"]["data"] + email = workflow_data["acquisition_source"]["email"] + control_number = context["ti"].xcom_pull( + task_ids="create_author_on_inspire", key="control_number" + ) + + inspire_http_hook.get_conn() + + request_data = { + "user_name": workflow_data["acquisition_source"].get("given_names", email), + "author_name": workflow_data.get("name").get("preferred_name"), + "record_url": f"{inspire_http_hook.base_url}/authors/{control_number}", + } + inspire_http_hook.close_ticket(ticket_id, "user_accepted_author", request_data) @task() def set_author_create_workflow_status_to_error(**context: dict) -> None: @@ -150,24 +184,8 @@ def set_author_create_workflow_status_to_error(**context: dict) -> None: set_author_create_workflow_status_to_completed() ) set_workflow_status_to_error_task = set_author_create_workflow_status_to_error() - combine_ticket_and_no_ticket_task = empty_task() # task dependencies - ticket_branch = create_author_create_curation_ticket_task - ( - ticket_branch - >> close_author_create_user_ticket_task - >> set_workflow_status_to_completed_task - ) - - no_ticket_branch = combine_ticket_and_no_ticket_task - ( - no_ticket_branch - >> close_author_create_user_ticket_task - >> set_workflow_status_to_completed_task - ) - - author_check_approval_branch_task >> [ticket_branch, no_ticket_branch] ( set_status_to_running_task >> create_author_on_inspire_task @@ -177,6 +195,15 @@ def set_author_create_workflow_status_to_error(**context: dict) -> None: author_check_approval_branch_task, set_workflow_status_to_error_task, ] + ( + [ + author_check_approval_branch_task + >> create_author_create_curation_ticket_task, + author_check_approval_branch_task, + ] + >> close_author_create_user_ticket_task + >> set_workflow_status_to_completed_task + ) author_create_approved_dag() diff --git a/workflows/dags/author/author_create/author_create_init.py b/workflows/dags/author/author_create/author_create_init.py index 16c62bd33f..50eb846eaa 100644 --- a/workflows/dags/author/author_create/author_create_init.py +++ b/workflows/dags/author/author_create/author_create_init.py @@ -8,7 +8,10 @@ from hooks.backoffice.workflow_ticket_management_hook import ( AuthorWorkflowTicketManagementHook, ) -from hooks.inspirehep.inspire_http_hook import InspireHttpHook +from hooks.inspirehep.inspire_http_hook import ( + AUTHOR_SUBMIT_FUNCTIONAL_CATEGORY, + InspireHttpHook, +) from include.utils.set_workflow_status import set_workflow_status_to_error logger = logging.getLogger(__name__) @@ -24,6 +27,7 @@ catchup=False, # TODO: what if callback fails? Data in backoffice not up to date! on_failure_callback=set_workflow_status_to_error, + tags=["authors"], ) def author_create_initialization_dag(): """ @@ -59,24 +63,41 @@ def set_schema(**context): @task() def create_author_create_user_ticket(**context: dict) -> None: - endpoint = "/api/tickets/create" - request_data = { - "functional_category": "Author curation", - "template": "user_new_author", - "workflow_id": context["params"]["workflow_id"], - "subject": "test", # TODO: set the subject and description - "description": "test", - "caller_email": "", # leave empty - } - response = inspire_http_hook.call_api( - endpoint=endpoint, data=request_data, method="POST" + workflow_data = context["params"]["workflow"]["data"] + email = workflow_data["acquisition_source"]["email"] + + response = inspire_http_hook.create_ticket( + AUTHOR_SUBMIT_FUNCTIONAL_CATEGORY, + "curator_new_author", + f"Your suggestion to INSPIRE: author " + f"{workflow_data.get('name').get('preferred_name')}", + workflow_data["acquisition_source"]["email"], + { + "email": email, + "obj_url": inspire_http_hook.get_backoffice_url( + context["params"]["workflow_id"] + ), + }, + ) + + ticket_id = response.json()["ticket_id"] + + response = inspire_http_hook.reply_ticket( + ticket_id, + "user_new_author", + { + "user_name": workflow_data["acquisition_source"].get( + "given_names", email + ), + "author_name": workflow_data.get("name").get("preferred_name"), + }, + email, ) - logger.info(f"Ticket created. Response status code: {response.status_code}") - logger.info(response.json()) + workflow_ticket_management_hook.create_ticket_entry( workflow_id=context["params"]["workflow_id"], ticket_type="author_create_user", - ticket_id=response.json()["ticket_id"], + ticket_id=ticket_id, ) @task() diff --git a/workflows/dags/author/author_create/author_create_rejected.py b/workflows/dags/author/author_create/author_create_rejected.py index 5caae48629..230de5881f 100644 --- a/workflows/dags/author/author_create/author_create_rejected.py +++ b/workflows/dags/author/author_create/author_create_rejected.py @@ -1,10 +1,14 @@ import datetime +import logging from airflow.decorators import dag, task from airflow.models.param import Param -from author.author_create.shared_tasks import close_author_create_user_ticket from hooks.backoffice.workflow_management_hook import AUTHORS, WorkflowManagementHook +from hooks.inspirehep.inspire_http_hook import InspireHttpHook from include.utils.set_workflow_status import set_workflow_status_to_error +from include.utils.tickets import get_ticket_by_type + +logger = logging.getLogger(__name__) @dag( @@ -17,6 +21,7 @@ catchup=False, # TODO: what if callback fails? Data in backoffice not up to date! on_failure_callback=set_workflow_status_to_error, + tags=["authors"], ) def author_create_rejected_dag() -> None: """ @@ -28,6 +33,7 @@ def author_create_rejected_dag() -> None: 2. set_author_create_workflow_status_to_completed: Sets the status of the author creation workflow to 'completed'. """ + inspire_http_hook = InspireHttpHook() workflow_management_hook = WorkflowManagementHook(AUTHORS) @task() @@ -44,6 +50,14 @@ def set_workflow_status_to_running(**context): status_name=status_name, workflow_id=context["params"]["workflow_id"] ) + @task() + def close_author_create_user_ticket(**context: dict) -> None: + logger.info("Closing ticket for rejected author") + ticket_id = get_ticket_by_type( + context["params"]["workflow"], "author_create_user" + )["ticket_id"] + inspire_http_hook.close_ticket(ticket_id) + # task definitions set_status_to_running_task = set_workflow_status_to_running() close_ticket_task = close_author_create_user_ticket() diff --git a/workflows/dags/author/author_create/shared_tasks.py b/workflows/dags/author/author_create/shared_tasks.py deleted file mode 100644 index 9657dd81dc..0000000000 --- a/workflows/dags/author/author_create/shared_tasks.py +++ /dev/null @@ -1,17 +0,0 @@ -from airflow.decorators import task -from airflow.utils.trigger_rule import TriggerRule -from hooks.backoffice.workflow_ticket_management_hook import ( - AuthorWorkflowTicketManagementHook, -) -from hooks.inspirehep.inspire_http_hook import InspireHttpHook - - -@task(trigger_rule=TriggerRule.NONE_FAILED_MIN_ONE_SUCCESS) -def close_author_create_user_ticket(**context: dict) -> None: - ticket_type = "author_create_user" - ticket_id = AuthorWorkflowTicketManagementHook().get_ticket( - workflow_id=context["params"]["workflow_id"], ticket_type=ticket_type - )["ticket_id"] - endpoint = "api/tickets/resolve" - request_data = {"ticket_id": ticket_id} - InspireHttpHook().call_api(endpoint=endpoint, data=request_data, method="POST") diff --git a/workflows/dags/author/author_update/author_update.py b/workflows/dags/author/author_update/author_update.py index 66ccdac6ef..826b8c6791 100644 --- a/workflows/dags/author/author_update/author_update.py +++ b/workflows/dags/author/author_update/author_update.py @@ -6,7 +6,10 @@ from hooks.backoffice.workflow_ticket_management_hook import ( AuthorWorkflowTicketManagementHook, ) -from hooks.inspirehep.inspire_http_hook import InspireHttpHook +from hooks.inspirehep.inspire_http_hook import ( + AUTHOR_UPDATE_FUNCTIONAL_CATEGORY, + InspireHttpHook, +) from hooks.inspirehep.inspire_http_record_management_hook import ( InspireHTTPRecordManagementHook, ) @@ -25,6 +28,7 @@ }, catchup=False, on_failure_callback=set_workflow_status_to_error, # TODO: what if callback fails? Data in backoffice not up to date! + tags=["authors"], ) def author_update_dag(): """ @@ -51,24 +55,39 @@ def set_author_update_workflow_status_to_running(**context): @task() def create_ticket_on_author_update(**context): - endpoint = "/api/tickets/create" - request_data = { - "functional_category": "Author updates", - "template": "curator_update_author", - "workflow_id": context["params"]["workflow_id"], - "subject": "test", - "description": "test", - "caller_email": "", + workflow_data = context["params"]["workflow"]["data"] + email = workflow_data["acquisition_source"]["email"] + + subject = ( + f"Update to author {workflow_data.get('name').get('preferred_name')}" + f" on INSPIRE" + ) + recid = workflow_data["control_number"] + url = inspire_http_hook.get_url() + template_context = { + "url": f"{url}/authors/{recid}", + "bibedit_url": f"{url}/record/{recid}", + "url_author_form": f"{url}/submissions/authors/{recid}", } - response = inspire_http_hook.call_api( - endpoint=endpoint, data=request_data, method="POST" + + response = inspire_http_hook.create_ticket( + AUTHOR_UPDATE_FUNCTIONAL_CATEGORY, + "curator_update_author", + subject, + email, + template_context, ) + + ticket_id = response.json()["ticket_id"] + workflow_ticket_management_hook.create_ticket_entry( workflow_id=context["params"]["workflow_id"], - ticket_type="author_update_curation", - ticket_id=response.json()["ticket_id"], + ticket_type="author_update_user", + ticket_id=ticket_id, ) + return response.json() + @task() def update_author_on_inspire(**context): workflow_data = workflow_management_hook.get_workflow( diff --git a/workflows/plugins/hooks/inspirehep/inspire_http_hook.py b/workflows/plugins/hooks/inspirehep/inspire_http_hook.py index 15b93fd9f4..b5ad6753ba 100644 --- a/workflows/plugins/hooks/inspirehep/inspire_http_hook.py +++ b/workflows/plugins/hooks/inspirehep/inspire_http_hook.py @@ -9,6 +9,11 @@ logger = logging.getLogger() +AUTHOR_SUBMIT_FUNCTIONAL_CATEGORY = "Author submissions" +AUTHOR_CURATION_FUNCTIONAL_CATEGORY = "Author curation" +AUTHOR_UPDATE_FUNCTIONAL_CATEGORY = "Author updates" + + class InspireHttpHook(HttpHook): """ Hook to interact with Inspire API @@ -59,6 +64,62 @@ def call_api(self, method: str, endpoint: str, data: dict) -> Response: _retry_args=self.tenacity_retry_kwargs, endpoint=endpoint, headers=self.headers, - data=data, + json=data, method=method, ) + + def get_backoffice_url(self, workflow_id: str) -> str: + self.get_conn() + return f"{self.base_url}/backoffice/{workflow_id}" + + def get_url(self) -> str: + self.get_conn() + return self.base_url + + def create_ticket( + self, functional_category, template_name, subject, email, template_context + ): + # TODO add docstring + endpoint = "/api/tickets/create" + + request_data = { + "functional_category": functional_category, + "template": template_name, + "subject": subject, + "template_context": template_context, + "caller_email": email, + } + + return self.call_api(endpoint=endpoint, data=request_data, method="POST") + + def reply_ticket(self, ticket_id, template, template_context, email): + # TODO add docstring + endpoint = "/api/tickets/reply" + + request_data = { + "ticket_id": str(ticket_id), + "template": template, + "template_context": template_context, + "user_email": email, + } + logging.info(f"Replying to ticket {ticket_id}") + + return self.call_api(endpoint=endpoint, data=request_data, method="POST") + + def close_ticket(self, ticket_id, template=None, template_context=None): + # TODO add docstring + endpoint = "/api/tickets/resolve" + + request_data = {"ticket_id": str(ticket_id)} + if template is not None: + request_data.update( + { + "template": template, + "template_context": template_context, + } + ) + + logging.info(f"Closing ticket {ticket_id}") + print(request_data) + + return self.call_api(endpoint=endpoint, data=request_data, method="POST") diff --git a/workflows/plugins/include/utils/tickets.py b/workflows/plugins/include/utils/tickets.py new file mode 100644 index 0000000000..1f29bfb2e6 --- /dev/null +++ b/workflows/plugins/include/utils/tickets.py @@ -0,0 +1,9 @@ +import logging + +logger = logging.getLogger(__name__) + + +def get_ticket_by_type(workflow, ticket_type): + for ticket in workflow["tickets"]: + if ticket["ticket_type"] == ticket_type: + return ticket diff --git a/workflows/tests/cassettes/TestAuthorCreate.test_close_author_create_user_ticket.yaml b/workflows/tests/cassettes/TestAuthorCreate.test_close_author_create_user_ticket.yaml deleted file mode 100644 index f6ded44730..0000000000 --- a/workflows/tests/cassettes/TestAuthorCreate.test_close_author_create_user_ticket.yaml +++ /dev/null @@ -1,90 +0,0 @@ -interactions: -- request: - body: null - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Authorization: - - Token 2e04111a61e8f5ba6ecec52af21bbb9e81732085 - Connection: - - keep-alive - Content-Type: - - application/json - User-Agent: - - python-requests/2.31.0 - method: GET - uri: http://host.docker.internal:8001/api/workflows/authors/tickets/f8301c06-8fa1-4124-845e-c270b910af5f/?ticket_type=author_create_user - response: - body: - string: '{"id":1,"ticket_url":"https://cerntraining.service-now.com/nav_to.do?uri=/u_request_fulfillment.do?sys_id=656f2d17878c929095f833340cbb3531","ticket_id":"656f2d17878c929095f833340cbb3531","ticket_type":"author_create_user","workflow_id":"f8301c06-8fa1-4124-845e-c270b910af5f"}' - headers: - Allow: - - GET, HEAD, OPTIONS - Content-Language: - - en - Content-Length: - - '275' - Content-Type: - - application/json - Cross-Origin-Opener-Policy: - - same-origin - Date: - - Wed, 21 Aug 2024 11:06:38 GMT - Referrer-Policy: - - same-origin - Server: - - WSGIServer/0.2 CPython/3.11.6 - Vary: - - Accept, Accept-Language, Cookie, origin - X-Content-Type-Options: - - nosniff - X-Frame-Options: - - DENY - status: - code: 200 - message: OK -- request: - body: ticket_id=656f2d17878c929095f833340cbb3531 - headers: - Accept: - - application/vnd+inspire.record.raw+json - Accept-Encoding: - - gzip, deflate - Authorization: - - Bearer cZiS4W7K8sqyebkxQzpnSwuUKLr5Ne6qPfnoOAjP7M2IvHxQhKmwiCJpp2QC - Connection: - - keep-alive - Content-Length: - - '42' - Content-Type: - - application/x-www-form-urlencoded - User-Agent: - - python-requests/2.31.0 - method: POST - uri: https://inspirebeta.net/api/tickets/resolve - response: - body: - string: '{"message":"Ticket resolved"} - - ' - headers: - access-control-allow-origin: - - '*' - access-control-expose-headers: - - Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset - content-length: - - '30' - content-type: - - application/json - date: - - Wed, 21 Aug 2024 11:06:39 GMT - server: - - gunicorn/19.10.0 - x-proxy-backend: - - inspire-qa_hep-web_http - status: - code: 200 - message: OK -version: 1 diff --git a/workflows/tests/cassettes/TestAuthorCreateApproved.test_close_author_create_user_ticket.yaml b/workflows/tests/cassettes/TestAuthorCreateApproved.test_close_author_create_user_ticket.yaml new file mode 100644 index 0000000000..f45fb5001d --- /dev/null +++ b/workflows/tests/cassettes/TestAuthorCreateApproved.test_close_author_create_user_ticket.yaml @@ -0,0 +1,44 @@ +interactions: +- request: + body: '{"ticket_id": "4b451fa0870a561095f833340cbb3595", "template": "user_accepted_author", + "template_context": {"user_name": "micha.moshe.moskovic@cern.ch", "author_name": + "Third B", "record_url": "https://inspirebeta.net/authors/12345"}}' + headers: + Accept: + - application/vnd+inspire.record.raw+json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '233' + Content-Type: + - application/json + method: POST + uri: https://inspirebeta.net/api/tickets/resolve + response: + body: + string: '{"message":"Ticket resolved"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset + alt-svc: + - h3=":443";ma=60; + content-length: + - '30' + content-type: + - application/json + date: + - Wed, 20 Nov 2024 15:49:49 GMT + server: + - gunicorn/19.10.0 + x-proxy-backend: + - inspire-qa_hep-web_http + status: + code: 200 + message: OK +version: 1 diff --git a/workflows/tests/cassettes/TestAuthorCreateApproved.test_create_author_create_curation_ticket.yaml b/workflows/tests/cassettes/TestAuthorCreateApproved.test_create_author_create_curation_ticket.yaml new file mode 100644 index 0000000000..5aaa5e73aa --- /dev/null +++ b/workflows/tests/cassettes/TestAuthorCreateApproved.test_create_author_create_curation_ticket.yaml @@ -0,0 +1,90 @@ +interactions: +- request: + body: '{"functional_category": "Author curation", "template": "curation_needed_author", + "subject": "Curation needed for author Third B ", "template_context": {"email": + "micha.moshe.moskovic@cern.ch", "record_url": "https://inspirebeta.net/authors/12345"}, + "caller_email": "micha.moshe.moskovic@cern.ch"}' + headers: + Accept: + - application/vnd+inspire.record.raw+json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '296' + Content-Type: + - application/json + method: POST + uri: https://inspirebeta.net/api/tickets/create + response: + body: + string: '{"ticket_id":"94c0745e870e1250225886640cbb3509","ticket_url":"https://cerntraining.service-now.com/nav_to.do?uri=/u_request_fulfillment.do?sys_id=94c0745e870e1250225886640cbb3509"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset + alt-svc: + - h3=":443";ma=60; + content-length: + - '181' + content-type: + - application/json + date: + - Mon, 25 Nov 2024 14:03:46 GMT + server: + - gunicorn/19.10.0 + x-proxy-backend: + - inspire-qa_hep-web_http + status: + code: 200 + message: OK +- request: + body: '{"ticket_type": "author_create_curation", "ticket_id": "94c0745e870e1250225886640cbb3509", + "workflow": "00000000-0000-0000-0000-000000001521"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '142' + Content-Type: + - application/json + method: POST + uri: http://host.docker.internal:8001/api/workflows/authors/tickets/ + response: + body: + string: '{"id":2,"ticket_url":"https://cerntraining.service-now.com/nav_to.do?uri=/u_request_fulfillment.do?sys_id=94c0745e870e1250225886640cbb3509","workflow":"00000000-0000-0000-0000-000000001521","ticket_id":"94c0745e870e1250225886640cbb3509","ticket_type":"author_create_curation","_created_at":"2024-11-25T14:03:46.470233Z","_updated_at":"2024-11-25T14:03:46.470241Z"}' + headers: + Allow: + - GET, POST, HEAD, OPTIONS + Content-Language: + - en + Content-Length: + - '364' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Mon, 25 Nov 2024 14:03:46 GMT + Referrer-Policy: + - same-origin + Server: + - WSGIServer/0.2 CPython/3.11.6 + Vary: + - Accept, Accept-Language, Cookie, origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 201 + message: Created +version: 1 diff --git a/workflows/tests/cassettes/TestAuthorCreateInit.test_create_author_create_user_ticket.yaml b/workflows/tests/cassettes/TestAuthorCreateInit.test_create_author_create_user_ticket.yaml new file mode 100644 index 0000000000..d259eac9db --- /dev/null +++ b/workflows/tests/cassettes/TestAuthorCreateInit.test_create_author_create_user_ticket.yaml @@ -0,0 +1,132 @@ +interactions: +- request: + body: '{"functional_category": "Author submissions", "template": "curator_new_author", + "subject": "Your suggestion to INSPIRE: author Third B", "template_context": + {"email": "micha.moshe.moskovic@cern.ch", "obj_url": "https://inspirebeta.net/backoffice/66277811-fe66-4335-9aff-984583fb1228"}, + "caller_email": "micha.moshe.moskovic@cern.ch"}' + headers: + Accept: + - application/vnd+inspire.record.raw+json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '333' + Content-Type: + - application/json + method: POST + uri: https://inspirebeta.net/api/tickets/create + response: + body: + string: '{"ticket_id":"dfa417a0870a561095f833340cbb3599","ticket_url":"https://cerntraining.service-now.com/nav_to.do?uri=/u_request_fulfillment.do?sys_id=dfa417a0870a561095f833340cbb3599"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset + alt-svc: + - h3=":443";ma=60; + content-length: + - '181' + content-type: + - application/json + date: + - Wed, 20 Nov 2024 15:30:34 GMT + server: + - gunicorn/19.10.0 + x-proxy-backend: + - inspire-qa_hep-web_http + status: + code: 200 + message: OK +- request: + body: '{"ticket_id": "dfa417a0870a561095f833340cbb3599", "template": "user_new_author", + "template_context": {"user_name": "micha.moshe.moskovic@cern.ch", "author_name": + "Third B"}, "user_email": "micha.moshe.moskovic@cern.ch"}' + headers: + Accept: + - application/vnd+inspire.record.raw+json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '219' + Content-Type: + - application/json + method: POST + uri: https://inspirebeta.net/api/tickets/reply + response: + body: + string: '{"message":"Ticket was updated with the reply"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset + alt-svc: + - h3=":443";ma=60; + content-length: + - '48' + content-type: + - application/json + date: + - Wed, 20 Nov 2024 15:30:35 GMT + server: + - gunicorn/19.10.0 + x-proxy-backend: + - inspire-qa_hep-web_http + status: + code: 200 + message: OK +- request: + body: '{"ticket_type": "author_create_user", "ticket_id": "dfa417a0870a561095f833340cbb3599", + "workflow": "66277811-fe66-4335-9aff-984583fb1228"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '138' + Content-Type: + - application/json + method: POST + uri: http://host.docker.internal:8001/api/workflows/authors/tickets/ + response: + body: + string: '{"id":5,"ticket_url":"https://cerntraining.service-now.com/nav_to.do?uri=/u_request_fulfillment.do?sys_id=dfa417a0870a561095f833340cbb3599","workflow":"66277811-fe66-4335-9aff-984583fb1228","ticket_id":"dfa417a0870a561095f833340cbb3599","ticket_type":"author_create_user","_created_at":"2024-11-20T15:30:35.448536Z","_updated_at":"2024-11-20T15:30:35.448543Z"}' + headers: + Allow: + - GET, POST, HEAD, OPTIONS + Content-Language: + - en + Content-Length: + - '360' + Content-Type: + - application/json + Cross-Origin-Opener-Policy: + - same-origin + Date: + - Wed, 20 Nov 2024 15:30:35 GMT + Referrer-Policy: + - same-origin + Server: + - WSGIServer/0.2 CPython/3.11.6 + Vary: + - Accept, Accept-Language, Cookie, origin + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + status: + code: 201 + message: Created +version: 1 diff --git a/workflows/tests/test_author_create_tasks.py b/workflows/tests/test_author_create_tasks.py index 7a38d86bab..f1b749e04d 100644 --- a/workflows/tests/test_author_create_tasks.py +++ b/workflows/tests/test_author_create_tasks.py @@ -1,17 +1,81 @@ +from unittest.mock import Mock + import pytest -from dags.author.author_create.shared_tasks import ( - close_author_create_user_ticket, -) +from airflow.models import DagBag +dagbag = DagBag() -class TestAuthorCreate: - context = { - "params": { - "workflow_id": "f8301c06-8fa1-4124-845e-c270b910af5f", - "data": {"value": "reject", "create_ticket": False}, - } +base_context = { + "params": { + "workflow_id": "00000000-0000-0000-0000-000000001521", + "workflow": { + "_created_at": "2024-11-20T15:04:12.196460Z", + "_updated_at": "2024-11-20T15:04:14.693568Z", + "data": { + "$schema": "https://inspirehep.net/schemas/records/authors.json", + "_collections": ["Authors"], + "acquisition_source": { + "datetime": "2024-11-18T11:34:19.809575", + "email": "micha.moshe.moskovic@cern.ch", + "internal_uid": 50872, + "method": "submitter", + "orcid": "0000-0002-7638-5686", + "source": "submitter", + }, + "name": {"preferred_name": "Third B", "value": "B, Third"}, + "status": "active", + }, + "decisions": [ + { + "_created_at": "2024-11-20T15:07:26.145006Z", + "_updated_at": "2024-11-20T15:07:26.145015Z", + "action": "accept", + "id": 1, + "user": "admin@admin.com", + "workflow": "66277811-fe66-4335-9aff-984583fb1228", + } + ], + "id": "66277811-fe66-4335-9aff-984583fb1228", + "status": "running", + "tickets": [ + { + "id": 6, + "ticket_url": "https://cerntraining.service-now.com/nav_to.do?uri=/u_request_fulfillment.do?sys_id=4b451fa0870a561095f833340cbb3595", + "workflow": "a8604175-10d9-440b-88ed-56afa732bc7c", + "ticket_id": "4b451fa0870a561095f833340cbb3595", + "ticket_type": "author_create_user", + "_created_at": "2024-11-20T15:33:18.704138Z", + "_updated_at": "2024-11-20T15:33:18.704145Z", + } + ], + "workflow_type": "AUTHOR_CREATE", + }, } +} + + +class TestAuthorCreateInit: + dag = dagbag.get_dag("author_create_initialization_dag") + context = base_context + + @pytest.mark.vcr + def test_create_author_create_user_ticket(self): + task = self.dag.get_task("create_author_create_user_ticket") + task.execute(context=self.context) + + +class TestAuthorCreateApproved: + dag = dagbag.get_dag("author_create_approved_dag") + context = base_context + context["ti"] = Mock() + context["ti"].xcom_pull.return_value = "12345" @pytest.mark.vcr def test_close_author_create_user_ticket(self): - close_author_create_user_ticket.function(**self.context) + task = self.dag.get_task("close_author_create_user_ticket") + task.execute(context=self.context) + + @pytest.mark.vcr + def test_create_author_create_curation_ticket(self): + task = self.dag.get_task("create_author_create_curation_ticket") + task.execute(context=self.context)