From 0159c4f02ab506f5c41e6cba084a6d6758da74e6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 28 May 2024 16:21:26 +0200 Subject: [PATCH 1/9] handling different event types Improve error logging and handling in ShotgridListener. Refactor code to enhance readability and maintainability. --- services/leecher/leecher/listener.py | 90 ++++++++++++++++++----- services/processor/processor/processor.py | 3 + 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/services/leecher/leecher/listener.py b/services/leecher/leecher/listener.py index a3ae344f..76ea2e5f 100644 --- a/services/leecher/leecher/listener.py +++ b/services/leecher/leecher/listener.py @@ -5,10 +5,13 @@ Shotgrid and converts them to Ayon events, and can be configured from the Ayon Addon settings page. """ +from pprint import pformat import sys import time import signal +import copy import socket +import traceback from typing import Any, Callable, Union from utils import get_logger @@ -135,19 +138,19 @@ def _build_shotgrid_filters(self): filters.append(["project", "in", sg_projects]) - sg_event_types = [] - - # TODO: Create a complex filter so skip event types "_Change" that - # we don't handle. - for entity_type in self.sg_enabled_entities: - for event_name in SG_EVENT_TYPES: - sg_event_types.append(event_name.format(entity_type)) - - if sg_event_types: + if sg_event_types := self._get_supported_event_types(): filters.append(["event_type", "in", sg_event_types]) return filters + def _get_supported_event_types(self) -> list[str]: + sg_event_types = [] + for entity_type in self.sg_enabled_entities: + sg_event_types.extend( + event_name.format(entity_type) for event_name in SG_EVENT_TYPES + ) + return sg_event_types + def _get_last_event_processed(self, sg_filters): """Find the Event ID for the last SG processed event. @@ -186,15 +189,27 @@ def start_listening(self): """ self.log.info("Start listening for Shotgrid Events...") - sg_filters = self._build_shotgrid_filters() - last_event_id = self._get_last_event_processed(sg_filters) + base_sg_filters = self._build_shotgrid_filters() + supported_event_types = self._get_supported_event_types() + last_event_id = self._get_last_event_processed(base_sg_filters) while True: - sg_filters = self._build_shotgrid_filters() - if not sg_filters: + if not base_sg_filters: + self.log.debug( + f"0 Leecher waiting {self.shotgrid_polling_frequency} " + "seconds..." + ) time.sleep(self.shotgrid_polling_frequency) + base_sg_filters = self._build_shotgrid_filters() continue + # try to get the last event processed in case we lost it due to + # a crash or restart + if not last_event_id: + last_event_id = self._get_last_event_processed(base_sg_filters) + + # and add the last event processed as a filter + sg_filters = copy.deepcopy(base_sg_filters) sg_filters.append(["id", "greater_than", last_event_id]) try: @@ -207,6 +222,10 @@ def start_listening(self): ) if not events: + self.log.debug( + f"1 Leecher waiting {self.shotgrid_polling_frequency} " + "seconds..." + ) time.sleep(self.shotgrid_polling_frequency) continue @@ -216,20 +235,55 @@ def start_listening(self): if not event: continue - # Filter out events we do not know how to handle if ( event["event_type"].endswith("_Change") - and event["attribute_name"].replace("sg_", "") not in list(self.custom_attribs_map.values()) + and event["attribute_name"].replace("sg_", "") in list(self.custom_attribs_map.values()) # noqa: E501 ): - last_event_id = event.get("id", None) + # events related to custom attributes changes + # check if event was caused by api user + if not self._is_api_user_event(event): + last_event_id = event.get("id", None) + else: + last_event_id = None + + elif event["event_type"] in supported_event_types: + # events related to changes in entities we track + # check if event was caused by api user + if not self._is_api_user_event(event): + last_event_id = event.get("id", None) + else: + last_event_id = None + + if not last_event_id: + self.log.info(f"Ignoring event: {pformat(event)}") continue last_event_id = self.func(event) - except Exception as err: - self.log.error(err, exc_info=True) + except Exception: + self.log.error(traceback.format_exc()) + self.log.debug( + f"2 Leecher waiting {self.shotgrid_polling_frequency} seconds..." + ) time.sleep(self.shotgrid_polling_frequency) + continue + + def _is_api_user_event(self, event: dict[str, Any]) -> bool: + """Check if the event was caused by an API user. + + Args: + event (dict): The Shotgrid Event data. + + Returns: + bool: True if the event was caused by an API user. + """ + # TODO: we have to create specific api user filtering + if ( + event.get("meta", {}).get("sudo_actual_user", {}).get("type") + == "ApiUser" + ): + return True def send_shotgrid_event_to_ayon(self, payload: dict[str, Any]) -> int: """Send the Shotgrid event as an Ayon event. diff --git a/services/processor/processor/processor.py b/services/processor/processor/processor.py index 027117ad..d52570b4 100644 --- a/services/processor/processor/processor.py +++ b/services/processor/processor/processor.py @@ -6,6 +6,7 @@ related events. """ import os +from pprint import pformat import time import types import socket @@ -184,6 +185,8 @@ def start_processing(self): description=f"Processing event with Handler {payload['action']}...", status="finished" ) + self.log.debug( + f"processing event {pformat(payload)}") handler.process_event( self, **payload, From d22ca53b6138be9b949d7637aebe83caa151583f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 28 May 2024 17:02:52 +0200 Subject: [PATCH 2/9] Refactor event handling and entity updates - Updated method parameters for event handling - Improved logging for event processing - Added validation for immutable entities --- .../ayon_shotgrid_hub/__init__.py | 19 ++++++++++--------- .../ayon_shotgrid_hub/update_from_shotgrid.py | 10 +++++++++- services/shotgrid_common/constants.py | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/__init__.py b/services/shotgrid_common/ayon_shotgrid_hub/__init__.py index c01f5588..e3c7013c 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/__init__.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/__init__.py @@ -331,7 +331,7 @@ def synchronize_projects(self, source="ayon"): "The `source` argument can only be `ayon` or `shotgrid`." ) - def react_to_shotgrid_event(self, sg_event): + def react_to_shotgrid_event(self, sg_event_meta): """React to events incoming from Shotgrid Whenever there's a `shotgrid.event` spawned by the `leecher` of a change @@ -341,17 +341,18 @@ def react_to_shotgrid_event(self, sg_event): this is to be expanded. Args: - sg_event (dict): The `meta` key of a ShotGrid Event, describing what - the change encompasses, i.e. a new shot, new asset, etc. + sg_event_meta (dict): The `meta` key of a ShotGrid Event, describing + what the change encompasses, i.e. a new shot, new asset, etc. """ if not self._ay_project: - self.log.info(f"Ignoring event, AYON project {self.project_name} not found.") + self.log.info( + f"Ignoring event, AYON project {self.project_name} not found.") return - match sg_event["type"]: + match sg_event_meta["type"]: case "new_entity" | "entity_revival": create_ay_entity_from_sg_event( - sg_event, + sg_event_meta, self._sg_project, self._sg, self._ay_project, @@ -362,7 +363,7 @@ def react_to_shotgrid_event(self, sg_event): case "attribute_change": update_ayon_entity_from_sg_event( - sg_event, + sg_event_meta, self._sg_project, self._sg, self._ay_project, @@ -373,7 +374,7 @@ def react_to_shotgrid_event(self, sg_event): case "entity_retirement": remove_ayon_entity_from_sg_event( - sg_event, + sg_event_meta, self._sg, self._ay_project, self.sg_project_code_field @@ -381,7 +382,7 @@ def react_to_shotgrid_event(self, sg_event): case _: raise ValueError( - f"Unable to process event {sg_event['type']}.") + f"Unable to process event {sg_event_meta['type']}.") def react_to_ayon_event(self, ayon_event): """React to events incoming from AYON diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py index 71c5eaf6..fc88dceb 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py @@ -37,7 +37,8 @@ CUST_FIELD_CODE_ID, # ShotGrid Field for the Ayon ID. SHOTGRID_ID_ATTRIB, # Ayon Entity Attribute. SHOTGRID_TYPE_ATTRIB, # Ayon Entity Attribute. - SHOTGRID_REMOVED_VALUE + SHOTGRID_REMOVED_VALUE, # Value for removed entities. + SG_RESTRICTED_ATTR_FIELDS, ) from utils import get_logger @@ -278,6 +279,13 @@ def update_ayon_entity_from_sg_event( if not ay_entity: raise ValueError("Unable to update a non existing entity.") + # make sure the entity is not immutable + if ( + ay_entity.immutable_for_hierarchy + and sg_event["attribute_name"] in SG_RESTRICTED_ATTR_FIELDS + ): + raise ValueError("Entity is immutable, aborting...") + # Ensure Ayon Entity has the correct ShotGrid ID ayon_entity_sg_id = str( ay_entity.attribs.get_attribute(SHOTGRID_ID_ATTRIB).value) diff --git a/services/shotgrid_common/constants.py b/services/shotgrid_common/constants.py index 86fa9a4c..54157b5f 100644 --- a/services/shotgrid_common/constants.py +++ b/services/shotgrid_common/constants.py @@ -133,7 +133,7 @@ CUST_FIELD_CODE_SYNC, ] -SG_EVENT_CHANGE_ATTR_FIELDS = [ +SG_RESTRICTED_ATTR_FIELDS = [ "code", "name" ] From ba0cb0b7874a1228cf07db7294b47c2ca709d79e Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 30 May 2024 17:00:25 +0200 Subject: [PATCH 3/9] Bump version to 0.4.2-dev.4 in multiple packages Update package versions to 0.4.2-dev.4 for shotgrid addon, leecher, processor, and transmitter in respective pyproject.toml files. --- client/ayon_shotgrid/version.py | 2 +- package.py | 2 +- services/leecher/pyproject.toml | 2 +- services/processor/pyproject.toml | 2 +- services/transmitter/pyproject.toml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/ayon_shotgrid/version.py b/client/ayon_shotgrid/version.py index 93a33177..53b10b7d 100644 --- a/client/ayon_shotgrid/version.py +++ b/client/ayon_shotgrid/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring shotgrid addon version.""" -__version__ = "0.4.2-dev.3" +__version__ = "0.4.2-dev.4" diff --git a/package.py b/package.py index 78bba7ca..eccc6981 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "shotgrid" title = "Shotgrid" -version = "0.4.2-dev.3" +version = "0.4.2-dev.4" client_dir = "ayon_shotgrid" services = { diff --git a/services/leecher/pyproject.toml b/services/leecher/pyproject.toml index 44abf4dd..545441e6 100644 --- a/services/leecher/pyproject.toml +++ b/services/leecher/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "shotgrid-leecher" -version = "0.4.2-dev.3" +version = "0.4.2-dev.4" description = "Shotgrid Integration for Ayon" authors = ["Oscar Domingo "] diff --git a/services/processor/pyproject.toml b/services/processor/pyproject.toml index a98e3361..401680b5 100644 --- a/services/processor/pyproject.toml +++ b/services/processor/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "shotgrid-processor" -version = "0.4.2-dev.3" +version = "0.4.2-dev.4" description = "Shotgrid Integration for Ayon" authors = ["Oscar Domingo "] diff --git a/services/transmitter/pyproject.toml b/services/transmitter/pyproject.toml index 55b648fa..4b169c1b 100644 --- a/services/transmitter/pyproject.toml +++ b/services/transmitter/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "shotgrid-transmitter" -version = "0.4.2-dev.3" +version = "0.4.2-dev.4" description = "Shotgrid Integration for Ayon" authors = ["Oscar Domingo "] From 8fb51c02a4eb9084e52001a82eb8d904f537fbb0 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 30 May 2024 17:10:26 +0200 Subject: [PATCH 4/9] Add validation for Shotgrid API Key presence Ensure Shotgrid API Key is set in the Addon System settings to avoid errors during processing. If key is missing, a ValueError will be raised with instructions on how to set it properly. --- services/leecher/leecher/listener.py | 8 ++++++++ services/processor/processor/processor.py | 8 ++++++++ services/transmitter/transmitter/transmitter.py | 7 +++++++ 3 files changed, 23 insertions(+) diff --git a/services/leecher/leecher/listener.py b/services/leecher/leecher/listener.py index 76ea2e5f..0b35c107 100644 --- a/services/leecher/leecher/listener.py +++ b/services/leecher/leecher/listener.py @@ -56,6 +56,14 @@ def __init__(self, func: Union[Callable, None] = None): # get server op related ShotGrid script api properties shotgrid_secret = ayon_api.get_secret( service_settings["script_key"]) + + if isinstance(shotgrid_secret, list): + raise ValueError( + "Shotgrid API Key not found. Make sure to set it in the " + "Addon System settings. " + "`ayon+settings://shotgrid/service_settings/script_key`" + ) + self.sg_api_key = shotgrid_secret.get("value") if not self.sg_api_key: raise ValueError( diff --git a/services/processor/processor/processor.py b/services/processor/processor/processor.py index d52570b4..e2338e1d 100644 --- a/services/processor/processor/processor.py +++ b/services/processor/processor/processor.py @@ -55,6 +55,14 @@ def __init__(self): # get server op related ShotGrid script api properties shotgrid_secret = ayon_api.get_secret( service_settings["script_key"]) + + if isinstance(shotgrid_secret, list): + raise ValueError( + "Shotgrid API Key not found. Make sure to set it in the " + "Addon System settings. " + "`ayon+settings://shotgrid/service_settings/script_key`" + ) + self.sg_api_key = shotgrid_secret.get("value") if not self.sg_api_key: raise ValueError( diff --git a/services/transmitter/transmitter/transmitter.py b/services/transmitter/transmitter/transmitter.py index 37c86168..784ca8af 100644 --- a/services/transmitter/transmitter/transmitter.py +++ b/services/transmitter/transmitter/transmitter.py @@ -44,6 +44,13 @@ def __init__(self): shotgrid_secret = ayon_api.get_secret( service_settings["script_key"]) + if isinstance(shotgrid_secret, list): + raise ValueError( + "Shotgrid API Key not found. Make sure to set it in the " + "Addon System settings. " + "`ayon+settings://shotgrid/service_settings/script_key`" + ) + self.sg_api_key = shotgrid_secret.get("value") if not self.sg_api_key: raise ValueError( From b9364d050bcc0f30773375a0c313d4813f11bd3c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 31 May 2024 14:05:34 +0200 Subject: [PATCH 5/9] improving info logging --- .../shotgrid_common/ayon_shotgrid_hub/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/__init__.py b/services/shotgrid_common/ayon_shotgrid_hub/__init__.py index e3c7013c..83ae0d17 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/__init__.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/__init__.py @@ -351,6 +351,11 @@ def react_to_shotgrid_event(self, sg_event_meta): match sg_event_meta["type"]: case "new_entity" | "entity_revival": + self.log.info( + f"Creating entity from SG event: {sg_event_meta['type']}" + f"| {sg_event_meta['entity_type']} " + f"| {sg_event_meta['entity_id']}" + ) create_ay_entity_from_sg_event( sg_event_meta, self._sg_project, @@ -362,6 +367,11 @@ def react_to_shotgrid_event(self, sg_event_meta): ) case "attribute_change": + self.log.info( + f"Updating entity from SG event: {sg_event_meta['type']} " + f"| {sg_event_meta['entity_type']} " + f"| {sg_event_meta['entity_id']}" + ) update_ayon_entity_from_sg_event( sg_event_meta, self._sg_project, @@ -373,6 +383,11 @@ def react_to_shotgrid_event(self, sg_event_meta): ) case "entity_retirement": + self.log.info( + f"Removing entity from SG event: {sg_event_meta['type']}" + f"| {sg_event_meta['entity_type']} " + f"| {sg_event_meta['entity_id']}" + ) remove_ayon_entity_from_sg_event( sg_event_meta, self._sg, From 7179ecb9944fe1421e2eaca04d70a3ef1eca2005 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 31 May 2024 15:53:36 +0200 Subject: [PATCH 6/9] Refactor service tools to use a unified service_main function. - Refactored service tools to utilize a common `service_main` function for processor, leecher, and transmitter services. This simplifies code duplication and improves maintainability. --- service_tools/main.py | 15 +++--- services/leecher/leecher/__init__.py | 4 +- services/leecher/leecher/__main__.py | 8 +-- services/leecher/leecher/listener.py | 10 +++- services/processor/processor/__init__.py | 7 ++- services/processor/processor/__main__.py | 7 ++- .../processor/handlers/shotgrid_event.py | 12 ++--- .../processor/handlers/sync_projects.py | 13 ++--- services/processor/processor/processor.py | 40 ++++++++++++++- .../ayon_shotgrid_hub/__init__.py | 49 ++----------------- services/transmitter/transmitter/__init__.py | 8 +-- services/transmitter/transmitter/__main__.py | 8 +-- .../transmitter/transmitter/transmitter.py | 40 +++++++++++++-- 13 files changed, 121 insertions(+), 100 deletions(-) diff --git a/service_tools/main.py b/service_tools/main.py index 4268ce99..692bedf6 100644 --- a/service_tools/main.py +++ b/service_tools/main.py @@ -92,22 +92,19 @@ def main(): sys.path.insert(0, path) if service_name == "processor": - from processor import ShotgridProcessor + from processor import service_main - shotgrid_processor = ShotgridProcessor() - sys.exit(shotgrid_processor.start_processing()) + service_main() elif service_name == "leecher": - from leecher import ShotgridListener + from leecher import service_main - shotgrid_listener = ShotgridListener() - sys.exit(shotgrid_listener.start_listening()) + service_main() else: - from transmitter import ShotgridTransmitter + from transmitter import service_main - shotgrid_transmitter = ShotgridTransmitter() - sys.exit(shotgrid_transmitter.start_processing()) + service_main() if __name__ == "__main__": diff --git a/services/leecher/leecher/__init__.py b/services/leecher/leecher/__init__.py index 911fa5ed..6ee67e79 100644 --- a/services/leecher/leecher/__init__.py +++ b/services/leecher/leecher/__init__.py @@ -1,7 +1,7 @@ -from .listener import ShotgridListener +from .listener import ShotgridListener, service_main __all__ = ( + "service_main", "ShotgridListener", ) - diff --git a/services/leecher/leecher/__main__.py b/services/leecher/leecher/__main__.py index ffe46e80..236ef70f 100644 --- a/services/leecher/leecher/__main__.py +++ b/services/leecher/leecher/__main__.py @@ -1,9 +1,5 @@ -import sys - -from .listener import ShotgridListener +from .listener import service_main if __name__ == "__main__": - shotgrid_listener = ShotgridListener() - sys.exit(shotgrid_listener.start_listening()) - + service_main() diff --git a/services/leecher/leecher/listener.py b/services/leecher/leecher/listener.py index 87feae2a..78d08d9f 100644 --- a/services/leecher/leecher/listener.py +++ b/services/leecher/leecher/listener.py @@ -36,9 +36,9 @@ def __init__(self): """ self.log.info("Initializing the Shotgrid Listener.") + self.settings = ayon_api.get_service_addon_settings() + try: - ayon_api.init_service() - self.settings = ayon_api.get_service_addon_settings() service_settings = self.settings["service_settings"] self.sg_url = self.settings["shotgrid_server"] @@ -331,3 +331,9 @@ def send_shotgrid_event_to_ayon( ) self.log.info("Dispatched Ayon event with payload:", payload) + + +def service_main(): + ayon_api.init_service() + shotgrid_listener = ShotgridListener() + sys.exit(shotgrid_listener.start_listening()) diff --git a/services/processor/processor/__init__.py b/services/processor/processor/__init__.py index 7aa281f9..1527b710 100644 --- a/services/processor/processor/__init__.py +++ b/services/processor/processor/__init__.py @@ -1,7 +1,10 @@ -from .processor import ShotgridProcessor +from .processor import ( + ShotgridProcessor, + service_main, +) __all__ = ( "ShotgridProcessor", + "service_main", ) - diff --git a/services/processor/processor/__main__.py b/services/processor/processor/__main__.py index 7f00d8aa..208ea16f 100644 --- a/services/processor/processor/__main__.py +++ b/services/processor/processor/__main__.py @@ -1,6 +1,5 @@ -import sys -from .processor import ShotgridProcessor +from .processor import service_main + if __name__ == "__main__": - shotgrid_processor = ShotgridProcessor() - sys.exit(shotgrid_processor.start_processing()) + service_main() diff --git a/services/processor/processor/handlers/shotgrid_event.py b/services/processor/processor/handlers/shotgrid_event.py index 42153241..036edb0e 100644 --- a/services/processor/processor/handlers/shotgrid_event.py +++ b/services/processor/processor/handlers/shotgrid_event.py @@ -9,7 +9,7 @@ def process_event( sg_processor, - **kwargs, + event, ): """React to Shotgrid Events. @@ -17,7 +17,7 @@ def process_event( function, where we attempt to replicate a change coming form Shotgrid, like creating a new Shot, renaming a Task, etc. """ - sg_payload = kwargs.get("sg_payload", {}) + sg_payload = event.get("sg_payload", {}) if not sg_payload: raise ValueError("The Event payload is empty!") @@ -25,11 +25,9 @@ def process_event( raise ValueError("The Event payload is missing the action to perform!") hub = AyonShotgridHub( - kwargs.get("project_name"), - kwargs.get("project_code"), - sg_processor.sg_url, - sg_processor.sg_api_key, - sg_processor.sg_script_name, + sg_processor.get_sg_connection(), + event.get("project_name"), + event.get("project_code"), sg_project_code_field=sg_processor.sg_project_code_field, custom_attribs_map=sg_processor.custom_attribs_map, custom_attribs_types=sg_processor.custom_attribs_types, diff --git a/services/processor/processor/handlers/sync_projects.py b/services/processor/processor/handlers/sync_projects.py index ee762fbb..aafb17ab 100644 --- a/services/processor/processor/handlers/sync_projects.py +++ b/services/processor/processor/handlers/sync_projects.py @@ -9,7 +9,7 @@ def process_event( sg_processor, - **kwargs, + event, ): """Synchronize a project between AYON and Shotgrid. @@ -18,11 +18,9 @@ def process_event( Shotgrid or AYON, and replicate it's structure in the other platform. """ hub = AyonShotgridHub( - kwargs.get("project_name"), - kwargs.get("project_code"), - sg_processor.sg_url, - sg_processor.sg_api_key, - sg_processor.sg_script_name, + sg_processor.get_sg_connection(), + event.get("project_name"), + event.get("project_code"), sg_project_code_field=sg_processor.sg_project_code_field, custom_attribs_map=sg_processor.custom_attribs_map, custom_attribs_types=sg_processor.custom_attribs_types, @@ -32,6 +30,5 @@ def process_event( # This will ensure that the project exists in both platforms. hub.create_project() sync_source = ( - "ayon" if kwargs.get("action") == "sync-from-ayon" else "shotgrid" - ) + "ayon" if event.get("action") == "sync-from-ayon" else "shotgrid") hub.synchronize_projects(source=sync_source) diff --git a/services/processor/processor/processor.py b/services/processor/processor/processor.py index 53b8fce9..f8e05bce 100644 --- a/services/processor/processor/processor.py +++ b/services/processor/processor/processor.py @@ -6,6 +6,7 @@ related events. """ import os +import sys from pprint import pformat import time import types @@ -14,12 +15,13 @@ import traceback import ayon_api +import shotgun_api3 from utils import get_logger class ShotgridProcessor: - + _sg: shotgun_api3.Shotgun = None log = get_logger(__file__) def __init__(self): @@ -145,6 +147,33 @@ def _get_handlers(self): return handlers_dict + def get_sg_connection(self): + """Ensure we can talk to AYON and Shotgrid. + + Start connections to the APIs and catch any possible error, we abort if + this steps fails for any reason. + """ + + if self._sg is None: + try: + self._sg = shotgun_api3.Shotgun( + self.sg_url, + script_name=self.sg_script_name, + api_key=self.sg_api_key + ) + except Exception as e: + self.log.error("Unable to create Shotgrid Session.") + raise e + + try: + self._sg.connect() + + except Exception as e: + self.log.error("Unable to connect to Shotgrid.") + raise e + + return self._sg + def start_processing(self): """Enroll AYON events of topic `shotgrid.event` @@ -205,7 +234,7 @@ def start_processing(self): f"processing event {pformat(payload)}") handler.process_event( self, - **payload, + payload, ) except Exception: @@ -242,3 +271,10 @@ def start_processing(self): except Exception: self.log.error(traceback.format_exc()) + + +def service_main(): + ayon_api.init_service() + + shotgrid_processor = ShotgridProcessor() + sys.exit(shotgrid_processor.start_processing()) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/__init__.py b/services/shotgrid_common/ayon_shotgrid_hub/__init__.py index 83ae0d17..680d6dab 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/__init__.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/__init__.py @@ -38,7 +38,6 @@ import ayon_api from ayon_api.entity_hub import EntityHub -import shotgun_api3 from utils import get_logger @@ -57,11 +56,9 @@ class AyonShotgridHub: entities and create entities/projects. Args: + sg_connection (shotgun_api3.Shotgun): The Shotgrid connection. project_name (str):The project name, cannot contain spaces. project_code (str): The project code (3 letter code). - sg_url (str): The URL of the Shotgrid instance. - sg_api_key (str): The API key of the Shotgrid instance. - sg_script_name (str): The Script Name of the Shotgrid instance. sg_project_code_field (str): The field in the Shotgrid Project entity that represents the project code. custom_attribs_map (dict): A dictionary mapping AYON attributes to @@ -77,11 +74,9 @@ class AyonShotgridHub: } def __init__(self, + sg_connection, project_name, project_code, - sg_url, - sg_api_key, - sg_script_name, sg_project_code_field=None, custom_attribs_map=None, custom_attribs_types=None, @@ -89,15 +84,7 @@ def __init__(self, ): self.settings = ayon_api.get_service_addon_settings() - self._sg = None - - if not all([sg_url, sg_api_key, sg_script_name]): - raise ValueError( - "AyonShotgridHub requires `sg_url`, `sg_api_key`" \ - "and `sg_script_name` as arguments." - ) - - self._initialize_apis(sg_url, sg_api_key, sg_script_name) + self._sg = sg_connection self._ay_project = None self._sg_project = None @@ -121,36 +108,6 @@ def __init__(self, self.project_name = project_name self.project_code = project_code - def _initialize_apis(self, sg_url=None, sg_api_key=None, sg_script_name=None): - """ Ensure we can talk to AYON and Shotgrid. - - Start connections to the APIs and catch any possible error, we abort if - this steps fails for any reason. - """ - try: - ayon_api.init_service() - except Exception as e: - self.log.error("Unable to connect to AYON.") - raise e - - if self._sg is None: - try: - self._sg = shotgun_api3.Shotgun( - sg_url, - script_name=sg_script_name, - api_key=sg_api_key - ) - except Exception as e: - self.log.error("Unable to create Shotgrid Session.") - raise e - - try: - self._sg.connect() - - except Exception as e: - self.log.error("Unable to connect to Shotgrid.") - raise e - def create_sg_attributes(self): """Create all AYON needed attributes in Shotgrid.""" create_ay_fields_in_sg_project( diff --git a/services/transmitter/transmitter/__init__.py b/services/transmitter/transmitter/__init__.py index 252f2c3d..5681db0d 100644 --- a/services/transmitter/transmitter/__init__.py +++ b/services/transmitter/transmitter/__init__.py @@ -1,8 +1,10 @@ -from .transmitter import ShotgridTransmitter - +from .transmitter import ( + ShotgridTransmitter, + service_main, +) __all__ = ( "ShotgridTransmitter", + "service_main", ) - diff --git a/services/transmitter/transmitter/__main__.py b/services/transmitter/transmitter/__main__.py index 5e33c6e3..0dc56a4d 100644 --- a/services/transmitter/transmitter/__main__.py +++ b/services/transmitter/transmitter/__main__.py @@ -1,9 +1,5 @@ -import sys - -from .transmitter import ShotgridTransmitter +from .transmitter import service_main if __name__ == "__main__": - shotgrid_transmitter = ShotgridTransmitter() - sys.exit(shotgrid_transmitter.start_processing()) - + service_main() diff --git a/services/transmitter/transmitter/transmitter.py b/services/transmitter/transmitter/transmitter.py index 6f138c6d..bdef3f7a 100644 --- a/services/transmitter/transmitter/transmitter.py +++ b/services/transmitter/transmitter/transmitter.py @@ -5,11 +5,13 @@ enroll the events of topic `entity.folder` and `entity.task` when any of the two are `created`, `renamed` or `deleted`. """ +import sys import time import socket import traceback import ayon_api +import shotgun_api3 from ayon_shotgrid_hub import AyonShotgridHub @@ -18,6 +20,7 @@ class ShotgridTransmitter: log = get_logger(__file__) + _sg: shotgun_api3.Shotgun = None def __init__(self): """ Ensure both Ayon and Shotgrid connections are available. @@ -92,6 +95,33 @@ def __init__(self): self.log.error("Unable to get Addon settings from the server.") raise e + def get_sg_connection(self): + """Ensure we can talk to AYON and Shotgrid. + + Start connections to the APIs and catch any possible error, we abort if + this steps fails for any reason. + """ + + if self._sg is None: + try: + self._sg = shotgun_api3.Shotgun( + self.sg_url, + script_name=self.sg_script_name, + api_key=self.sg_api_key, + ) + except Exception as e: + self.log.error("Unable to create Shotgrid Session.") + raise e + + try: + self._sg.connect() + + except Exception as e: + self.log.error("Unable to connect to Shotgrid.") + raise e + + return self._sg + def start_processing(self): """ Main loop querying AYON for `entity.*` events. @@ -183,11 +213,9 @@ def start_processing(self): project_code = ay_project.get("code") hub = AyonShotgridHub( + self.get_sg_connection(), project_name, project_code, - self.sg_url, - self.sg_api_key, - self.sg_script_name, sg_project_code_field=self.sg_project_code_field, custom_attribs_map=self.custom_attribs_map, custom_attribs_types=self.custom_attribs_types, @@ -214,3 +242,9 @@ def start_processing(self): "message": traceback.format_exc(), }, ) + + +def service_main(): + ayon_api.init_service() + shotgrid_transmitter = ShotgridTransmitter() + sys.exit(shotgrid_transmitter.start_processing()) From 1b9ecc5975d208fe88b432e9d8de4b72ef9564de Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 31 May 2024 15:54:51 +0200 Subject: [PATCH 7/9] Refactor initialization of Shotgrid Listener settings. - Refactored the way service addon settings are retrieved during initialization to ensure correct assignment and usage. --- services/leecher/leecher/listener.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/leecher/leecher/listener.py b/services/leecher/leecher/listener.py index 78d08d9f..178e5e54 100644 --- a/services/leecher/leecher/listener.py +++ b/services/leecher/leecher/listener.py @@ -36,9 +36,9 @@ def __init__(self): """ self.log.info("Initializing the Shotgrid Listener.") - self.settings = ayon_api.get_service_addon_settings() try: + self.settings = ayon_api.get_service_addon_settings() service_settings = self.settings["service_settings"] self.sg_url = self.settings["shotgrid_server"] From 4de98b18f5777cc21f61c8bcb018855a63738560 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 31 May 2024 15:55:44 +0200 Subject: [PATCH 8/9] Remove unnecessary whitespace and comments in ShotgridListener class. Details: - Removed extra blank line. - Deleted a redundant comment. --- services/leecher/leecher/listener.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/leecher/leecher/listener.py b/services/leecher/leecher/listener.py index 178e5e54..3874577b 100644 --- a/services/leecher/leecher/listener.py +++ b/services/leecher/leecher/listener.py @@ -36,7 +36,6 @@ def __init__(self): """ self.log.info("Initializing the Shotgrid Listener.") - try: self.settings = ayon_api.get_service_addon_settings() service_settings = self.settings["service_settings"] From 33f8640c998f5e6af6d3d3ca54ea079f10fb977b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 31 May 2024 15:57:00 +0200 Subject: [PATCH 9/9] Bump version to 0.4.2-dev.5 for shotgrid components. Update version numbers in multiple files for shotgrid components to 0.4.2-dev.5 to reflect the latest changes and improvements. --- client/ayon_shotgrid/version.py | 2 +- package.py | 2 +- services/leecher/pyproject.toml | 2 +- services/processor/pyproject.toml | 2 +- services/transmitter/pyproject.toml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/ayon_shotgrid/version.py b/client/ayon_shotgrid/version.py index 53b10b7d..2245dc55 100644 --- a/client/ayon_shotgrid/version.py +++ b/client/ayon_shotgrid/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring shotgrid addon version.""" -__version__ = "0.4.2-dev.4" +__version__ = "0.4.2-dev.5" diff --git a/package.py b/package.py index eccc6981..8f7b6367 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "shotgrid" title = "Shotgrid" -version = "0.4.2-dev.4" +version = "0.4.2-dev.5" client_dir = "ayon_shotgrid" services = { diff --git a/services/leecher/pyproject.toml b/services/leecher/pyproject.toml index 545441e6..3b9f5dea 100644 --- a/services/leecher/pyproject.toml +++ b/services/leecher/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "shotgrid-leecher" -version = "0.4.2-dev.4" +version = "0.4.2-dev.5" description = "Shotgrid Integration for Ayon" authors = ["Oscar Domingo "] diff --git a/services/processor/pyproject.toml b/services/processor/pyproject.toml index 401680b5..b4ad9559 100644 --- a/services/processor/pyproject.toml +++ b/services/processor/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "shotgrid-processor" -version = "0.4.2-dev.4" +version = "0.4.2-dev.5" description = "Shotgrid Integration for Ayon" authors = ["Oscar Domingo "] diff --git a/services/transmitter/pyproject.toml b/services/transmitter/pyproject.toml index 4b169c1b..4bb940a8 100644 --- a/services/transmitter/pyproject.toml +++ b/services/transmitter/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "shotgrid-transmitter" -version = "0.4.2-dev.4" +version = "0.4.2-dev.5" description = "Shotgrid Integration for Ayon" authors = ["Oscar Domingo "]