From 69746004530e5ef9ed2d30399987be6044b71850 Mon Sep 17 00:00:00 2001 From: Jeremy Nelson Date: Fri, 22 Nov 2024 11:16:13 -0800 Subject: [PATCH 01/11] refactor: Sets schedule to None as DAG runs should only run with message --- ils_middleware/dags/cornell.py | 1 + ils_middleware/dags/stanford.py | 1 + 2 files changed, 2 insertions(+) diff --git a/ils_middleware/dags/cornell.py b/ils_middleware/dags/cornell.py index 4464e4f..ee514d5 100644 --- a/ils_middleware/dags/cornell.py +++ b/ils_middleware/dags/cornell.py @@ -47,6 +47,7 @@ def dag_failure_callback(ctx_dict) -> None: description="Cornell FOLIO DAG", start_date=datetime(2021, 8, 14), tags=["folio"], + schedule=None, catchup=False, ) as dag: diff --git a/ils_middleware/dags/stanford.py b/ils_middleware/dags/stanford.py index 43cc7b2..1975663 100644 --- a/ils_middleware/dags/stanford.py +++ b/ils_middleware/dags/stanford.py @@ -49,6 +49,7 @@ def dag_failure_callback(ctx_dict) -> None: start_date=datetime(2021, 8, 24), tags=["symphony", "folio"], catchup=False, + schedule=None, on_failure_callback=dag_failure_callback, ) as dag: From b5d624ed9410c13035e2dd059f0f99cdd1ee9969 Mon Sep 17 00:00:00 2001 From: Jeremy Nelson Date: Fri, 22 Nov 2024 12:55:27 -0800 Subject: [PATCH 02/11] refactor: Use UUID portion of Sinopia URL for XCOM key --- ils_middleware/tasks/amazon/sqs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ils_middleware/tasks/amazon/sqs.py b/ils_middleware/tasks/amazon/sqs.py index ff7f0d7..6b673a4 100644 --- a/ils_middleware/tasks/amazon/sqs.py +++ b/ils_middleware/tasks/amazon/sqs.py @@ -43,9 +43,10 @@ def parse_messages(**kwargs) -> str: resources_with_errors = [] resource_uri = message["resource"]["uri"] + resource_uuid = resource_uri.split("/")[-1] try: task_instance.xcom_push( - key=resource_uri, + key=resource_uuid, value={ "email": message["user"]["email"], "group": message["group"], From 8cc1f92e4a170fc55bd2a1de2ac44aa1ca09b4e2 Mon Sep 17 00:00:00 2001 From: Jeremy Nelson Date: Fri, 22 Nov 2024 12:55:55 -0800 Subject: [PATCH 03/11] refactor: Use Sinopia UUID for FOLIO XCOM keys --- ils_middleware/tasks/folio/build.py | 7 +++++-- ils_middleware/tasks/folio/graph.py | 5 +++-- ils_middleware/tasks/folio/map.py | 9 ++++++--- ils_middleware/tasks/folio/new.py | 9 +++++---- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/ils_middleware/tasks/folio/build.py b/ils_middleware/tasks/folio/build.py index 46b4808..860773a 100644 --- a/ils_middleware/tasks/folio/build.py +++ b/ils_middleware/tasks/folio/build.py @@ -294,10 +294,12 @@ def _inventory_record(**kwargs) -> dict: "source": "SINOPIA", "electronicAccess": [_electronic_access(**kwargs)], } + instance_uuid = instance_uri.split("/")[-1] + for folio_field in FOLIO_FIELDS: post_processing = transforms.get(folio_field, _default_transform) task_id = _task_ids(task_groups, folio_field) - raw_values = task_instance.xcom_pull(key=instance_uri, task_ids=task_id) + raw_values = task_instance.xcom_pull(key=instance_uuid, task_ids=task_id) if raw_values: record_field, values = post_processing( values=raw_values, @@ -335,5 +337,6 @@ def build_records(**kwargs): folio_client=folio_client, **kwargs, ) - task_instance.xcom_push(key=resource_uri, value=inventory_rec) + resource_uuid = resource_uri.split("/")[-1] + task_instance.xcom_push(key=resource_uuid, value=inventory_rec) return "build-complete" diff --git a/ils_middleware/tasks/folio/graph.py b/ils_middleware/tasks/folio/graph.py index ed8eedd..1a13b16 100644 --- a/ils_middleware/tasks/folio/graph.py +++ b/ils_middleware/tasks/folio/graph.py @@ -30,8 +30,9 @@ def construct_graph(**kwargs): resources = task_instance.xcom_pull(key="resources", task_ids="sqs-message-parse") for instance_uri in resources: + instance_uuid = instance_uri.split("/")[-1] resource = task_instance.xcom_pull( - key=instance_uri, task_ids="sqs-message-parse" + key=instance_uuid, task_ids="sqs-message-parse" ).get("resource") work_refs = resource.get("bfWorkRefs") if len(work_refs) < 1: @@ -41,7 +42,7 @@ def construct_graph(**kwargs): ] # Grabs the first Work URI, may need a way to specify a work graph = _build_graph(resource.get("data"), work_uri) task_instance.xcom_push( - key=instance_uri, + key=instance_uuid, value={"graph": graph.serialize(format="json-ld"), "work_uri": work_uri}, ) return "constructed_graphs" diff --git a/ils_middleware/tasks/folio/map.py b/ils_middleware/tasks/folio/map.py index 7c478f3..d19ff7d 100644 --- a/ils_middleware/tasks/folio/map.py +++ b/ils_middleware/tasks/folio/map.py @@ -85,10 +85,11 @@ def _build_and_query_graph(**kwargs) -> list: query_params = {} task_id = _task_id(task_groups) + instance_uuid = instance_uri.split("/")[-1] if uri_type.startswith("bf_work"): query_params[uri_type] = task_instance.xcom_pull( - key=instance_uri, task_ids=task_id + key=instance_uuid, task_ids=task_id ).get("work_uri") else: query_params[uri_type] = instance_uri @@ -97,7 +98,8 @@ def _build_and_query_graph(**kwargs) -> list: query = template.format(**query_params) graph = rdflib.Graph() - json_ld = task_instance.xcom_pull(key=instance_uri, task_ids=task_id).get("graph") + + json_ld = task_instance.xcom_pull(key=instance_uuid, task_ids=task_id).get("graph") graph.parse(data=json_ld, format="json-ld") logging.info(f"SPARQL:\n{query}") return [row for row in graph.query(query)] @@ -119,5 +121,6 @@ def map_to_folio(**kwargs): bf_class=bf_class, **kwargs, ) - task_instance.xcom_push(key=instance_uri, value=values) + instance_uuid = instance_uri.split("/")[-1] + task_instance.xcom_push(key=instance_uuid, value=values) return "mapping_complete" diff --git a/ils_middleware/tasks/folio/new.py b/ils_middleware/tasks/folio/new.py index 640a7b8..2a4d124 100644 --- a/ils_middleware/tasks/folio/new.py +++ b/ils_middleware/tasks/folio/new.py @@ -36,9 +36,9 @@ def _check_for_existance(records: list, folio_client: FolioClient) -> tuple: def _push_to_xcom(records: list, task_instance): for record in records: logger.debug(record) - task_instance.xcom_push( - key=record["electronicAccess"][0]["uri"], value=record["id"] - ) + uri = record["electronicAccess"][0]["uri"] + uuid = uri.split("/")[-1] + task_instance.xcom_push(key=uuid, value=record["id"]) def _post_to_okapi(**kwargs): @@ -95,8 +95,9 @@ def post_folio_records(**kwargs): inventory_records = [] for instance_uri in resources: + instance_uuid = instance_uri.split("/")[-1] inventory_records.append( - task_instance.xcom_pull(key=instance_uri, task_ids=task_id) + task_instance.xcom_pull(key=instance_uuid, task_ids=task_id) ) new_records, existing_records = _check_for_existance( From f01b0b6d9f75829bf6cb6f6d134f927ba8ff8fdb Mon Sep 17 00:00:00 2001 From: Jeremy Nelson Date: Fri, 22 Nov 2024 14:09:31 -0800 Subject: [PATCH 04/11] refactor: Adjusts tests for using Sinopia UUID for FOLIO --- tests/helpers/tasks.py | 14 +++++++------- tests/tasks/folio/test_build.py | 3 ++- tests/tasks/folio/test_graph.py | 4 ++-- tests/tasks/folio/test_map.py | 8 +++++--- tests/tasks/folio/test_new_inventory.py | 3 ++- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/helpers/tasks.py b/tests/helpers/tasks.py index 97229ac..8e1aa76 100644 --- a/tests/helpers/tasks.py +++ b/tests/helpers/tasks.py @@ -60,7 +60,7 @@ def test_mms_id(): mock_resources = { - "https://api.development.sinopia.io/resource/0000-1111-2222-3333": { + "0000-1111-2222-3333": { "user": "jpnelson", "group": "stanford", "editGroups": ["other", "pcc"], @@ -99,7 +99,7 @@ def test_mms_id(): "uri": "https://api.development.sinopia.io/resource/7b55e6f7-f91e-4c7a-bbcd-c074485ad18d", "timestamp": "2021-10-29T20:30:58.821Z", }, - "https://api.development.sinopia.io/resource/4444-5555-6666-7777": { + "4444-5555-6666-7777": { "user": "jpnelson", "group": "stanford", "editGroups": ["other", "pcc"], @@ -127,19 +127,19 @@ def test_mms_id(): "uri": "https://api.development.sinopia.io/resource/7b55e6f7-f91e-4c7a-bbcd-c074485ad18d", "timestamp": "2021-10-29T20:30:58.821Z", }, - "https://api.development.sinopia.io/resource/8888-9999-0000-1111": { + "8888-9999-0000-1111": { "user": "jpnelson", "group": "stanford", }, } mock_resource_attributes = { - "https://api.development.sinopia.io/resource/0000-1111-2222-3333": { + "0000-1111-2222-3333": { "email": "dscully@stanford.edu", "group": "stanford", "target": "ils", }, - "https://api.development.sinopia.io/resource/4444-5555-6666-7777": { + "4444-5555-6666-7777": { "email": "fmulder@stanford.edu", "group": "yale", "target": "ils", @@ -222,7 +222,7 @@ def marc_as_json(): } folio_ids = { - "https://api.development.sinopia.io/resource/0000-1111-2222-3333": { + "0000-1111-2222-3333": { "id": "98a0337a-ec22-53aa-8ffc-933a86d10159", "hrid": "in000789", "electronicAccess": [ @@ -231,7 +231,7 @@ def marc_as_json(): } ], }, - "https://api.development.sinopia.io/resource/4444-5555-6666-7777": { + "4444-5555-6666-7777": { "id": "147b1171-740e-513e-84d5-b63a9642792c", "hrid": "in000001234", "electronicAccess": [ diff --git a/tests/tasks/folio/test_build.py b/tests/tasks/folio/test_build.py index 4c96bcc..4b74b0a 100644 --- a/tests/tasks/folio/test_build.py +++ b/tests/tasks/folio/test_build.py @@ -142,7 +142,8 @@ def test_happypath_build_records( folio_url=okapi_uri, folio_connection_id="stanford_folio", ) - record = test_task_instance().xcom_pull(key=instance_uri) + instance_uuid = instance_uri.split("/")[-1] + record = test_task_instance().xcom_pull(key=instance_uuid) assert record["metadata"]["createdByUserId"].startswith( "faecc486-50f1-5082-a6d0-5e967e6f4786" diff --git a/tests/tasks/folio/test_graph.py b/tests/tasks/folio/test_graph.py index bc5ac60..8cf0fb3 100644 --- a/tests/tasks/folio/test_graph.py +++ b/tests/tasks/folio/test_graph.py @@ -77,7 +77,7 @@ def test_construct_graph(mock_requests, mock_task_instance): # noqa: F811 "graph" in test_task_instance() .xcom_pull( - key="https://api.development.sinopia.io/resource/0000-1111-2222-3333" + key="0000-1111-2222-3333" ) .keys() ) @@ -85,7 +85,7 @@ def test_construct_graph(mock_requests, mock_task_instance): # noqa: F811 assert ( test_task_instance() .xcom_pull( - key="https://api.development.sinopia.io/resource/0000-1111-2222-3333" + key="0000-1111-2222-3333" ) .get("work_uri") == "https://api.development.sinopia.io/resource/6497a461-42dc-42bf-b433-5e47c73f7e89" diff --git a/tests/tasks/folio/test_map.py b/tests/tasks/folio/test_map.py index 2804b1c..893eff6 100644 --- a/tests/tasks/folio/test_map.py +++ b/tests/tasks/folio/test_map.py @@ -22,7 +22,7 @@ def mock_task_instance(monkeypatch, test_graph: rdflib.Graph): def mock_xcom_pull(*args, **kwargs): key = kwargs.get("key") task_ids = kwargs.get("task_ids", "") - if key.startswith(instance_uri): + if key == "b0319047-acd0-4f30-bd8b-98e6c1bac6b0": if task_ids.startswith("bf-graph"): return { "graph": test_graph.serialize(format="json-ld"), @@ -53,7 +53,8 @@ def test_folio(mock_task_instance, test_graph: rdflib.Graph): # noqa: F811 task_groups_ids=[""], ) - title_list = test_task_instance().xcom_pull(key=instance_uri) + instance_uuid = instance_uri.split("/")[-1] + title_list = test_task_instance().xcom_pull(key=instance_uuid) # Main Title assert title_list[0].startswith("Scrivere di Islam") @@ -70,7 +71,8 @@ def test_folio_work(mock_task_instance, test_graph: rdflib.Graph): # noqa: F811 task_groups_ids=[""], ) - contributors_list = test_task_instance().xcom_pull(key=instance_uri) + instance_uuid = instance_uri.split("/")[-1] + contributors_list = test_task_instance().xcom_pull(key=instance_uuid) # First Contributor Name (tests for both options as SPARQL query is not # explictly defining an order diff --git a/tests/tasks/folio/test_new_inventory.py b/tests/tasks/folio/test_new_inventory.py index fdf4aa1..e92ed83 100644 --- a/tests/tasks/folio/test_new_inventory.py +++ b/tests/tasks/folio/test_new_inventory.py @@ -111,8 +111,9 @@ def test_happypath_post_folio_record( task_groups_ids=[""], ) + instance_uuid = instance_uri.split("/")[-1] assert ( - test_task_instance().xcom_pull(key=instance_uri) + test_task_instance().xcom_pull(key=instance_uuid) ) == "147b1171-740e-513e-84d5-b63a9642792c" From e04ca79ed5d908c179384b1df9dc92a6184cb651 Mon Sep 17 00:00:00 2001 From: Jeremy Nelson Date: Fri, 22 Nov 2024 14:24:58 -0800 Subject: [PATCH 05/11] refactor: Use UUID for Sinopia XCOM messages --- ils_middleware/tasks/sinopia/email.py | 6 ++++-- ils_middleware/tasks/sinopia/local_metadata.py | 8 +++++--- ils_middleware/tasks/sinopia/rdf2marc.py | 3 ++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ils_middleware/tasks/sinopia/email.py b/ils_middleware/tasks/sinopia/email.py index 53f4515..393c3c5 100644 --- a/ils_middleware/tasks/sinopia/email.py +++ b/ils_middleware/tasks/sinopia/email.py @@ -24,8 +24,9 @@ def send_update_success_emails(**kwargs) -> None: ses_hook = SesHook(aws_conn_id="aws_ses_connection") for resource_uri in resources: + resource_uuid = resource_uri.split("/")[-1] message = task_instance.xcom_pull( - key=resource_uri, task_ids="sqs-message-parse" + key=resource_uuid, task_ids="sqs-message-parse" ) email_attributes = _email_on_success_attributes(message) ses_hook.send_email(**email_attributes) @@ -51,8 +52,9 @@ def send_task_failure_notifications(**kwargs) -> None: key="conversion_failures", task_ids="process_symphony.rdf2marc" ) for resource_uri in failed_resources or []: + resource_uuid = resource_uri.split("/")[-1] message = task_instance.xcom_pull( - key=resource_uri, task_ids="sqs-message-parse" + key=resource_uuid, task_ids="sqs-message-parse" ) email_attributes = _email_on_failure_attributes(message) ses_hook.send_email(**email_attributes) diff --git a/ils_middleware/tasks/sinopia/local_metadata.py b/ils_middleware/tasks/sinopia/local_metadata.py index 51bb4aa..abf22a4 100644 --- a/ils_middleware/tasks/sinopia/local_metadata.py +++ b/ils_middleware/tasks/sinopia/local_metadata.py @@ -33,8 +33,9 @@ def _add_local_system_id( def _pull_identifiers(tasks, resource_uri, instance) -> typing.Optional[str]: + resource_uuid = resource_uri.split("/")[-1] for task_id in tasks: - value = instance.xcom_pull(key=resource_uri, task_ids=task_id) + value = instance.xcom_pull(key=resource_uuid, task_ids=task_id) if value: return value return None @@ -99,8 +100,9 @@ def new_local_admin_metadata(*args, **kwargs): sinopia_api_uri = Variable.get("sinopia_api_uri") for resource_uri in resources: + resource_uuid = resource_uri.split("/")[-1] message = task_instance.xcom_pull( - key=resource_uri, task_ids="sqs-message-parse" + key=resource_uuid, task_ids="sqs-message-parse" ) resource = message.get("resource") group = resource.get("group") @@ -158,4 +160,4 @@ def new_local_admin_metadata(*args, **kwargs): raise Exception(msg) logger.debug(f"Results of new_admin_result {new_admin_result.text}") - task_instance.xcom_push(key=resource_uri, value=admin_metadata_uri) + task_instance.xcom_push(key=resource_uuid, value=admin_metadata_uri) diff --git a/ils_middleware/tasks/sinopia/rdf2marc.py b/ils_middleware/tasks/sinopia/rdf2marc.py index b305389..54e343e 100644 --- a/ils_middleware/tasks/sinopia/rdf2marc.py +++ b/ils_middleware/tasks/sinopia/rdf2marc.py @@ -52,7 +52,8 @@ def Rdf2Marc(**kwargs): msg = f"RDF2MARC conversion failed for {instance_uri}, error: {payload.get('errorMessage')}" conversion_failures[instance_uri] = msg elif result["StatusCode"] == 200: - task_instance.xcom_push(key=instance_uri, value=marc_path) + instance_uuid = instance_uri.split("/")[-1] + task_instance.xcom_push(key=instance_uuid, value=marc_path) else: msg = f"RDF2MARC conversion failed for {instance_uri}: {result['FunctionError']}" conversion_failures[instance_uri] = msg From f7368ff1a3c66fa18c3c33060b0ab4ced5f33dda Mon Sep 17 00:00:00 2001 From: Jeremy Nelson Date: Fri, 22 Nov 2024 14:48:33 -0800 Subject: [PATCH 06/11] refactor: Fixes task test package for Sinopia tests --- tests/helpers/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/tasks.py b/tests/helpers/tasks.py index 8e1aa76..d37aa9e 100644 --- a/tests/helpers/tasks.py +++ b/tests/helpers/tasks.py @@ -144,7 +144,7 @@ def test_mms_id(): "group": "yale", "target": "ils", }, - "https://api.development.sinopia.io/resource/8888-9999-0000-1111": { + "8888-9999-0000-1111": { "email": "fmulder@stanford.edu", "group": "yale", "target": "ils", @@ -296,7 +296,7 @@ def mock_xcom_pull(*args, **kwargs): "email": mock_resource_attributes[key]["email"], "group": mock_resource_attributes[key]["group"], "target": mock_resource_attributes[key]["target"], - "resource_uri": key, + "resource_uri": f"https://api.development.sinopia.io/resource/{key}", "resource": mock_resources[key], } elif key == "overlay_resources": From 3f5cac19f4d6825197b7770c109b1efc5d80dc71 Mon Sep 17 00:00:00 2001 From: Jeremy Nelson Date: Fri, 22 Nov 2024 14:52:05 -0800 Subject: [PATCH 07/11] refactor: Use Sinopia UUID for Alma and Amazon tasks --- ils_middleware/tasks/alma/post_bfinstance.py | 6 ++++-- ils_middleware/tasks/alma/post_bfwork.py | 6 ++++-- ils_middleware/tasks/amazon/alma_instance_s3.py | 3 ++- ils_middleware/tasks/amazon/alma_work_s3.py | 3 ++- ils_middleware/tasks/amazon/s3.py | 7 ++++--- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ils_middleware/tasks/alma/post_bfinstance.py b/ils_middleware/tasks/alma/post_bfinstance.py index e1c6d8c..806edf3 100644 --- a/ils_middleware/tasks/alma/post_bfinstance.py +++ b/ils_middleware/tasks/alma/post_bfinstance.py @@ -67,7 +67,8 @@ def NewInstancetoAlma(**kwargs): if status == 200: xml_response = ET.fromstring(result) mms_id = xml_response.xpath("//mms_id/text()") - task_instance.xcom_push(key=instance_uri, value=mms_id) + instance_uuid = instance_uri.split("/")[-1] + task_instance.xcom_push(key=instance_uuid, value=mms_id) elif status == 400: # run xslt on the result in case the response is 400 and we need to update the record put_mms_id_str = parse_400(result) @@ -110,7 +111,8 @@ def putInstanceToAlma( put_update_status = put_update.status_code match put_update_status: case 200: - task_instance.xcom_push(key=instance_uri, value=put_mms_id_str) + instance_uuid = instance_uri.split("/")[-1] + task_instance.xcom_push(key=instance_uuid, value=put_mms_id_str) case 500: raise Exception(f"Internal server error from Alma API: {put_update_status}") case _: diff --git a/ils_middleware/tasks/alma/post_bfwork.py b/ils_middleware/tasks/alma/post_bfwork.py index 9652993..ff1cbfa 100644 --- a/ils_middleware/tasks/alma/post_bfwork.py +++ b/ils_middleware/tasks/alma/post_bfwork.py @@ -68,7 +68,8 @@ def NewWorktoAlma(**kwargs): if status == 200: xml_response = ET.fromstring(result) mms_id = xml_response.xpath("//mms_id/text()") - task_instance.xcom_push(key=instance_uri, value=mms_id) + instance_uuid = instance_uri.split("/")[-1] + task_instance.xcom_push(key=instance_uuid, value=mms_id) elif status == 400: # run xslt on the result in case the response is 400 and we need to update the record put_mms_id_str = parse_400(result) @@ -111,7 +112,8 @@ def putWorkToAlma( put_update_status = put_update.status_code match put_update_status: case 200: - task_instance.xcom_push(key=instance_uri, value=put_mms_id_str) + instance_uuid = instance_uri.split("/")[-1] + task_instance.xcom_push(key=instance_uuid, value=put_mms_id_str) case 500: raise Exception(f"Internal server error from Alma API: {put_update_status}") case _: diff --git a/ils_middleware/tasks/amazon/alma_instance_s3.py b/ils_middleware/tasks/amazon/alma_instance_s3.py index a7e8a1e..283b825 100644 --- a/ils_middleware/tasks/amazon/alma_instance_s3.py +++ b/ils_middleware/tasks/amazon/alma_instance_s3.py @@ -68,8 +68,9 @@ def send_instance_to_alma_s3(**kwargs): replace=True, ) # save to xcom + instance_uuid = instance_uri.split("/")[-1] task_instance.xcom_push( - key=instance_uri, value=instance_alma_xml.decode("utf-8") + key=instance_uuid, value=instance_alma_xml.decode("utf-8") ) logger.info(f"Saved BFInstance description for {instance_uri} to alma.") logger.info(f"bf_instance_alma_xml: {instance_alma_xml.decode('utf-8')}") diff --git a/ils_middleware/tasks/amazon/alma_work_s3.py b/ils_middleware/tasks/amazon/alma_work_s3.py index 6892bd7..42ed9cf 100644 --- a/ils_middleware/tasks/amazon/alma_work_s3.py +++ b/ils_middleware/tasks/amazon/alma_work_s3.py @@ -99,7 +99,8 @@ def load_to_s3(s3_hook, bfwork_alma_xml, instance_uri): def push_to_xcom(task_instance, instance_uri, bfwork_alma_xml): - task_instance.xcom_push(key=instance_uri, value=bfwork_alma_xml.decode("utf-8")) + instance_uuid = instance_uri.split("/")[-1] + task_instance.xcom_push(key=instance_uuid, value=bfwork_alma_xml.decode("utf-8")) def send_work_to_alma_s3(**kwargs): diff --git a/ils_middleware/tasks/amazon/s3.py b/ils_middleware/tasks/amazon/s3.py index 95a6517..e3597d1 100644 --- a/ils_middleware/tasks/amazon/s3.py +++ b/ils_middleware/tasks/amazon/s3.py @@ -28,7 +28,8 @@ def get_from_s3(**kwargs): key=f"marc/airflow/{instance_id}/record.mar", bucket_name=Variable.get("marc_s3_bucket"), ) - task_instance.xcom_push(key=instance_uri, value=temp_file) + instance_uuid = instance_uri.split("/")[-1] + task_instance.xcom_push(key=instance_uuid, value=temp_file) def send_to_s3(**kwargs): @@ -45,7 +46,7 @@ def send_to_s3(**kwargs): instance_path = urlparse(instance_uri).path instance_id = path.split(instance_path)[-1] temp_file = task_instance.xcom_pull( - key=instance_uri, task_ids="process_symphony.download_marc" + key=instance_id, task_ids="process_symphony.download_marc" ) marc_record = marc_record_from_temp_file(instance_id, temp_file) s3_hook.load_string( @@ -54,7 +55,7 @@ def send_to_s3(**kwargs): Variable.get("marc_s3_bucket"), replace=True, ) - task_instance.xcom_push(key=instance_uri, value=marc_record.as_json()) + task_instance.xcom_push(key=instance_id, value=marc_record.as_json()) def marc_record_from_temp_file(instance_id, temp_file): From a85f8005053d74f2903821f1e53be626b9fd7af3 Mon Sep 17 00:00:00 2001 From: Jeremy Nelson Date: Fri, 22 Nov 2024 16:11:33 -0800 Subject: [PATCH 08/11] refactor: Adjusts AWS tests to use UUID in XCOMs --- tests/tasks/amazon/test_alma_work_s3.py | 5 +++-- tests/tasks/amazon/test_s3.py | 8 ++++---- tests/tasks/amazon/test_sqs.py | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/tasks/amazon/test_alma_work_s3.py b/tests/tasks/amazon/test_alma_work_s3.py index 38d1c10..2e0ba89 100644 --- a/tests/tasks/amazon/test_alma_work_s3.py +++ b/tests/tasks/amazon/test_alma_work_s3.py @@ -177,8 +177,9 @@ def test_push_to_xcom(): instance_uri = "http://example.com/instance" bfwork_alma_xml = b"" push_to_xcom(mock_task_instance, instance_uri, bfwork_alma_xml) + instance_uuid = instance_uri.split("/")[-1] mock_task_instance.xcom_push.assert_called_once_with( - key=instance_uri, value=bfwork_alma_xml.decode("utf-8") + key=instance_uuid, value=bfwork_alma_xml.decode("utf-8") ) @@ -268,6 +269,6 @@ def normalize_xml(xml_str): assert expected_value == actual_value mock_task_instance.xcom_push.assert_called_once_with( - key=URIRef("https://example.com/resource/instance_uri"), + key="instance_uri", value=mock_task_instance.xcom_push.call_args[1]["value"], ) diff --git a/tests/tasks/amazon/test_s3.py b/tests/tasks/amazon/test_s3.py index 51a0f2d..39f58dc 100644 --- a/tests/tasks/amazon/test_s3.py +++ b/tests/tasks/amazon/test_s3.py @@ -38,13 +38,13 @@ def test_get_from_s3(mock_env_vars, mock_s3_hook, mock_task_instance): get_from_s3(task_instance=test_task_instance()) assert ( test_task_instance().xcom_pull( - key="https://api.development.sinopia.io/resource/0000-1111-2222-3333" + key="0000-1111-2222-3333" ) == "tests/fixtures/record.mar" ) assert ( test_task_instance().xcom_pull( - key="https://api.development.sinopia.io/resource/4444-5555-6666-7777" + key="4444-5555-6666-7777" ) == "tests/fixtures/record.mar" ) @@ -59,7 +59,7 @@ def test_send_to_s3(mock_env_vars, mock_s3_load_string, mock_task_instance): assert ( json.loads( test_task_instance().xcom_pull( - key="https://api.development.sinopia.io/resource/0000-1111-2222-3333" + key="0000-1111-2222-3333" ) ) == marc_json @@ -67,7 +67,7 @@ def test_send_to_s3(mock_env_vars, mock_s3_load_string, mock_task_instance): assert ( json.loads( test_task_instance().xcom_pull( - key="https://api.development.sinopia.io/resource/4444-5555-6666-7777" + key="4444-5555-6666-7777" ) ) == marc_json diff --git a/tests/tasks/amazon/test_sqs.py b/tests/tasks/amazon/test_sqs.py index 81c93cb..31a41bc 100644 --- a/tests/tasks/amazon/test_sqs.py +++ b/tests/tasks/amazon/test_sqs.py @@ -108,7 +108,8 @@ def test_parse_messages( assert result == "completed_parse" resource = mock_resources[0] - assert task_instance.xcom_pull(key=resource).get("resource") == mock_resource( + uuid = resource.split("/")[-1] + assert task_instance.xcom_pull(key=uuid).get("resource") == mock_resource( resource ) From 77c4d45693bd7a16aa40e723318e138a4b28a453 Mon Sep 17 00:00:00 2001 From: Jeremy Nelson Date: Fri, 22 Nov 2024 16:13:59 -0800 Subject: [PATCH 09/11] refactor: Adjusts Sinopia Tests for XCOM uuids --- tests/tasks/sinopia/test_local_metadata.py | 4 ++-- tests/tasks/sinopia/test_rdf2marc.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/tasks/sinopia/test_local_metadata.py b/tests/tasks/sinopia/test_local_metadata.py index dddc284..daef248 100644 --- a/tests/tasks/sinopia/test_local_metadata.py +++ b/tests/tasks/sinopia/test_local_metadata.py @@ -62,14 +62,14 @@ def test_new_local_admin_metadata( assert ( test_task_instance().xcom_pull( - key="https://api.development.sinopia.io/resource/0000-1111-2222-3333" + key="0000-1111-2222-3333" ) == "https://api.development.sinopia.io/resource/1a3cebda-34b9-4e15-bc79-f6a5f915ce76" ) assert ( test_task_instance().xcom_pull( - key="https://api.development.sinopia.io/resource/4444-5555-6666-7777" + key="4444-5555-6666-7777" ) == "https://api.development.sinopia.io/resource/1a3cebda-34b9-4e15-bc79-f6a5f915ce76" ) diff --git a/tests/tasks/sinopia/test_rdf2marc.py b/tests/tasks/sinopia/test_rdf2marc.py index f8516b7..7b98afb 100644 --- a/tests/tasks/sinopia/test_rdf2marc.py +++ b/tests/tasks/sinopia/test_rdf2marc.py @@ -31,13 +31,13 @@ def test_Rdf2Marc(mock_task_instance, mock_lambda): # noqa: F811 Rdf2Marc(task_instance=task_instance) assert ( task_instance.xcom_pull( - key="https://api.development.sinopia.io/resource/0000-1111-2222-3333" + key="0000-1111-2222-3333" ) == "airflow/0000-1111-2222-3333/record.mar" ) assert ( task_instance.xcom_pull( - key="https://api.development.sinopia.io/resource/4444-5555-6666-7777" + key="4444-5555-6666-7777" ) == "airflow/4444-5555-6666-7777/record.mar" ) From cf80fe172eb723c9aa6d32be2242d10b684dacc2 Mon Sep 17 00:00:00 2001 From: Jeremy Nelson Date: Fri, 22 Nov 2024 16:22:08 -0800 Subject: [PATCH 10/11] refactor: Symphony tasks use UUIDs in XCOMs --- ils_middleware/tasks/symphony/mod_json.py | 5 +++-- ils_middleware/tasks/symphony/new.py | 7 ++++--- ils_middleware/tasks/symphony/overlay.py | 7 +++++-- tests/tasks/symphony/test_mod_json.py | 2 +- tests/tasks/symphony/test_new.py | 2 +- tests/tasks/symphony/test_overlay.py | 2 +- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/ils_middleware/tasks/symphony/mod_json.py b/ils_middleware/tasks/symphony/mod_json.py index bac995b..2a3aad0 100644 --- a/ils_middleware/tasks/symphony/mod_json.py +++ b/ils_middleware/tasks/symphony/mod_json.py @@ -40,8 +40,9 @@ def to_symphony_json(**kwargs): resources = task_instance.xcom_pull(key="resources", task_ids="sqs-message-parse") for instance_uri in resources: + instance_uuid = instance_uri.split("/")[-1] marc_raw_json = task_instance.xcom_pull( - key=instance_uri, task_ids="process_symphony.marc_json_to_s3" + key=instance_uuid, task_ids="process_symphony.marc_json_to_s3" ) pymarc_json = json.loads(marc_raw_json) record = {"standard": "MARC21", "type": "BIB", "fields": []} @@ -49,4 +50,4 @@ def to_symphony_json(**kwargs): for field in pymarc_json["fields"]: record["fields"].append(_get_fields(field)) - task_instance.xcom_push(key=instance_uri, value=record) + task_instance.xcom_push(key=instance_uuid, value=record) diff --git a/ils_middleware/tasks/symphony/new.py b/ils_middleware/tasks/symphony/new.py index e33e842..33d1271 100644 --- a/ils_middleware/tasks/symphony/new.py +++ b/ils_middleware/tasks/symphony/new.py @@ -16,8 +16,9 @@ def NewMARCtoSymphony(**kwargs): ) for resource_uri in resources: + resource_uuid = resource_uri.split("/")[-1] marc_json = task_instance.xcom_pull( - key=resource_uri, task_ids="process_symphony.convert_to_symphony_json" + key=resource_uuid, task_ids="process_symphony.convert_to_symphony_json" ) payload = { @@ -54,9 +55,9 @@ def NewMARCtoSymphony(**kwargs): } ], } - + resource_uuid = resource_uri.split("/")[-1] task_instance.xcom_push( - key=resource_uri, + key=resource_uuid, value=SymphonyRequest( **kwargs, data=json.dumps(payload), diff --git a/ils_middleware/tasks/symphony/overlay.py b/ils_middleware/tasks/symphony/overlay.py index 442028c..5d5ee89 100644 --- a/ils_middleware/tasks/symphony/overlay.py +++ b/ils_middleware/tasks/symphony/overlay.py @@ -32,8 +32,9 @@ def overlay_marc_in_symphony(*args, **kwargs): else: catkey = resource["catkey"][0].get("SIRSI") + resource_uuid = resource_uri.split("/")[-1] marc_json = task_instance.xcom_pull( - key=resource_uri, task_ids="process_symphony.convert_to_symphony_json" + key=resource_uuid, task_ids="process_symphony.convert_to_symphony_json" ) payload = { @@ -43,8 +44,10 @@ def overlay_marc_in_symphony(*args, **kwargs): "bib": marc_json, } + resource_uuid = resource_uri.split("/")[-1] + task_instance.xcom_push( - key=resource_uri, + key=resource_uuid, value=SymphonyRequest( **kwargs, data=json.dumps(payload), diff --git a/tests/tasks/symphony/test_mod_json.py b/tests/tasks/symphony/test_mod_json.py index 9b4d362..d1c3289 100644 --- a/tests/tasks/symphony/test_mod_json.py +++ b/tests/tasks/symphony/test_mod_json.py @@ -10,7 +10,7 @@ def test_to_symphony_json(mock_task_instance): # noqa: F811 to_symphony_json(task_instance=task_instance) symphony_json = task_instance.xcom_pull( - key="https://api.development.sinopia.io/resource/0000-1111-2222-3333" + key="0000-1111-2222-3333" ) assert symphony_json["standard"].startswith("MARC21") assert symphony_json["leader"].startswith("01498nam a2200265 u 4500") diff --git a/tests/tasks/symphony/test_new.py b/tests/tasks/symphony/test_new.py index d12623f..b90914a 100644 --- a/tests/tasks/symphony/test_new.py +++ b/tests/tasks/symphony/test_new.py @@ -50,7 +50,7 @@ def test_NewMARCtoSymphony( ) assert ( task_instance.xcom_pull( - key="https://api.development.sinopia.io/resource/0000-1111-2222-3333" + key="0000-1111-2222-3333" ) == "45678" ) diff --git a/tests/tasks/symphony/test_overlay.py b/tests/tasks/symphony/test_overlay.py index 956121d..c185a93 100644 --- a/tests/tasks/symphony/test_overlay.py +++ b/tests/tasks/symphony/test_overlay.py @@ -56,7 +56,7 @@ def test_overlay_marc_in_symphony( marc_json=MARC_JSON, ) assert task_instance.xcom_pull( - key="https://api.development.sinopia.io/resource/0000-1111-2222-3333" + key="0000-1111-2222-3333" ).startswith(CATKEY) From 5fc464ee043678750b255a9ac267d8cc255ca8cd Mon Sep 17 00:00:00 2001 From: Jeremy Nelson Date: Fri, 22 Nov 2024 16:32:18 -0800 Subject: [PATCH 11/11] refactor: Applies black to tests --- tests/tasks/amazon/test_s3.py | 20 ++++---------------- tests/tasks/amazon/test_sqs.py | 4 +--- tests/tasks/folio/test_graph.py | 15 ++------------- tests/tasks/sinopia/test_local_metadata.py | 8 ++------ tests/tasks/sinopia/test_rdf2marc.py | 8 ++------ tests/tasks/symphony/test_mod_json.py | 4 +--- tests/tasks/symphony/test_new.py | 7 +------ tests/tasks/symphony/test_overlay.py | 4 +--- 8 files changed, 14 insertions(+), 56 deletions(-) diff --git a/tests/tasks/amazon/test_s3.py b/tests/tasks/amazon/test_s3.py index 39f58dc..318743a 100644 --- a/tests/tasks/amazon/test_s3.py +++ b/tests/tasks/amazon/test_s3.py @@ -37,15 +37,11 @@ def test_get_from_s3(mock_env_vars, mock_s3_hook, mock_task_instance): """Test downloading a file from S3 into a temp file""" get_from_s3(task_instance=test_task_instance()) assert ( - test_task_instance().xcom_pull( - key="0000-1111-2222-3333" - ) + test_task_instance().xcom_pull(key="0000-1111-2222-3333") == "tests/fixtures/record.mar" ) assert ( - test_task_instance().xcom_pull( - key="4444-5555-6666-7777" - ) + test_task_instance().xcom_pull(key="4444-5555-6666-7777") == "tests/fixtures/record.mar" ) @@ -57,19 +53,11 @@ def test_send_to_s3(mock_env_vars, mock_s3_load_string, mock_task_instance): mock_s3_load_string.call_count == 2 marc_json = marc_as_json() assert ( - json.loads( - test_task_instance().xcom_pull( - key="0000-1111-2222-3333" - ) - ) + json.loads(test_task_instance().xcom_pull(key="0000-1111-2222-3333")) == marc_json ) assert ( - json.loads( - test_task_instance().xcom_pull( - key="4444-5555-6666-7777" - ) - ) + json.loads(test_task_instance().xcom_pull(key="4444-5555-6666-7777")) == marc_json ) diff --git a/tests/tasks/amazon/test_sqs.py b/tests/tasks/amazon/test_sqs.py index 31a41bc..1a7ada1 100644 --- a/tests/tasks/amazon/test_sqs.py +++ b/tests/tasks/amazon/test_sqs.py @@ -109,9 +109,7 @@ def test_parse_messages( resource = mock_resources[0] uuid = resource.split("/")[-1] - assert task_instance.xcom_pull(key=uuid).get("resource") == mock_resource( - resource - ) + assert task_instance.xcom_pull(key=uuid).get("resource") == mock_resource(resource) @pytest.fixture diff --git a/tests/tasks/folio/test_graph.py b/tests/tasks/folio/test_graph.py index 8cf0fb3..f46cf18 100644 --- a/tests/tasks/folio/test_graph.py +++ b/tests/tasks/folio/test_graph.py @@ -73,21 +73,10 @@ def test_construct_graph(mock_requests, mock_task_instance): # noqa: F811 task_instance=test_task_instance(), ) - assert ( - "graph" - in test_task_instance() - .xcom_pull( - key="0000-1111-2222-3333" - ) - .keys() - ) + assert "graph" in test_task_instance().xcom_pull(key="0000-1111-2222-3333").keys() assert ( - test_task_instance() - .xcom_pull( - key="0000-1111-2222-3333" - ) - .get("work_uri") + test_task_instance().xcom_pull(key="0000-1111-2222-3333").get("work_uri") == "https://api.development.sinopia.io/resource/6497a461-42dc-42bf-b433-5e47c73f7e89" ) diff --git a/tests/tasks/sinopia/test_local_metadata.py b/tests/tasks/sinopia/test_local_metadata.py index daef248..c9fabf6 100644 --- a/tests/tasks/sinopia/test_local_metadata.py +++ b/tests/tasks/sinopia/test_local_metadata.py @@ -61,16 +61,12 @@ def test_new_local_admin_metadata( ) assert ( - test_task_instance().xcom_pull( - key="0000-1111-2222-3333" - ) + test_task_instance().xcom_pull(key="0000-1111-2222-3333") == "https://api.development.sinopia.io/resource/1a3cebda-34b9-4e15-bc79-f6a5f915ce76" ) assert ( - test_task_instance().xcom_pull( - key="4444-5555-6666-7777" - ) + test_task_instance().xcom_pull(key="4444-5555-6666-7777") == "https://api.development.sinopia.io/resource/1a3cebda-34b9-4e15-bc79-f6a5f915ce76" ) diff --git a/tests/tasks/sinopia/test_rdf2marc.py b/tests/tasks/sinopia/test_rdf2marc.py index 7b98afb..e4e8701 100644 --- a/tests/tasks/sinopia/test_rdf2marc.py +++ b/tests/tasks/sinopia/test_rdf2marc.py @@ -30,15 +30,11 @@ def mock_invoke_lambda(*args, **kwargs): def test_Rdf2Marc(mock_task_instance, mock_lambda): # noqa: F811 Rdf2Marc(task_instance=task_instance) assert ( - task_instance.xcom_pull( - key="0000-1111-2222-3333" - ) + task_instance.xcom_pull(key="0000-1111-2222-3333") == "airflow/0000-1111-2222-3333/record.mar" ) assert ( - task_instance.xcom_pull( - key="4444-5555-6666-7777" - ) + task_instance.xcom_pull(key="4444-5555-6666-7777") == "airflow/4444-5555-6666-7777/record.mar" ) diff --git a/tests/tasks/symphony/test_mod_json.py b/tests/tasks/symphony/test_mod_json.py index d1c3289..635911c 100644 --- a/tests/tasks/symphony/test_mod_json.py +++ b/tests/tasks/symphony/test_mod_json.py @@ -9,8 +9,6 @@ def test_to_symphony_json(mock_task_instance): # noqa: F811 to_symphony_json(task_instance=task_instance) - symphony_json = task_instance.xcom_pull( - key="0000-1111-2222-3333" - ) + symphony_json = task_instance.xcom_pull(key="0000-1111-2222-3333") assert symphony_json["standard"].startswith("MARC21") assert symphony_json["leader"].startswith("01498nam a2200265 u 4500") diff --git a/tests/tasks/symphony/test_new.py b/tests/tasks/symphony/test_new.py index b90914a..8cc295c 100644 --- a/tests/tasks/symphony/test_new.py +++ b/tests/tasks/symphony/test_new.py @@ -48,12 +48,7 @@ def test_NewMARCtoSymphony( library_key="GREEN", marc_json={"leader": "11222999 adf", "fields": [{"tag": "245"}]}, ) - assert ( - task_instance.xcom_pull( - key="0000-1111-2222-3333" - ) - == "45678" - ) + assert task_instance.xcom_pull(key="0000-1111-2222-3333") == "45678" @pytest.fixture diff --git a/tests/tasks/symphony/test_overlay.py b/tests/tasks/symphony/test_overlay.py index c185a93..4dee8b8 100644 --- a/tests/tasks/symphony/test_overlay.py +++ b/tests/tasks/symphony/test_overlay.py @@ -55,9 +55,7 @@ def test_overlay_marc_in_symphony( catkey=CATKEY, marc_json=MARC_JSON, ) - assert task_instance.xcom_pull( - key="0000-1111-2222-3333" - ).startswith(CATKEY) + assert task_instance.xcom_pull(key="0000-1111-2222-3333").startswith(CATKEY) def test_missing_catkey(