From d753d304e0a0d1c7d105e26698603a7d23899dae Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:58:51 +0100 Subject: [PATCH 1/2] store all server information at once --- client/ayon_deadline/addon.py | 148 +++++++++++++--------------------- client/ayon_deadline/lib.py | 36 +++++---- 2 files changed, 75 insertions(+), 109 deletions(-) diff --git a/client/ayon_deadline/addon.py b/client/ayon_deadline/addon.py index a9d91c2587..8696139e91 100644 --- a/client/ayon_deadline/addon.py +++ b/client/ayon_deadline/addon.py @@ -1,14 +1,16 @@ import os +from typing import Optional, List, Dict, Any, Tuple -from typing import Optional, List +import ayon_api from ayon_core.addon import AYONAddon, IPluginPaths from .lib import ( + DeadlineServerInfo, get_deadline_workers, get_deadline_groups, get_deadline_limit_groups, - get_deadline_pools + get_deadline_pools, ) from .version import __version__ @@ -35,10 +37,7 @@ def initialize(self, studio_settings): self.deadline_servers_info = deadline_servers_info - self._pools_by_server_name = {} - self._limit_groups_by_server_name = {} - self._groups_by_server_name = {} - self._machines_by_server_name = {} + self._server_info_by_name: Dict[str, DeadlineServerInfo] = {} def get_plugin_paths(self): """Deadline plugin paths.""" @@ -58,98 +57,61 @@ def get_publish_plugin_paths( paths.append(os.path.join(publish_dir, host_name)) return paths - def get_pools_by_server_name(self, server_name: str) -> List[str]: - """Returns dictionary of pools per DL server + def get_server_info_by_name(self, server_name: str) -> DeadlineServerInfo: + """Returns Deadline server info by name. Args: server_name (str): Deadline Server name from Project Settings. Returns: - Dict[str, List[str]]: {"default": ["pool1", "pool2"]} + DeadlineServerInfo: Deadline server info. """ - pools = self._pools_by_server_name.get(server_name) - if pools is None: - dl_server_info = self.deadline_servers_info[server_name] - - auth = (dl_server_info["default_username"], - dl_server_info["default_password"]) - pools = get_deadline_pools( - dl_server_info["value"], - auth + server_info = self._server_info_by_name.get(server_name) + if server_info is None: + server_url, auth, _ = self._get_deadline_con_info(server_name) + pools = get_deadline_pools(server_url, auth) + groups = get_deadline_groups(server_url, auth) + limit_groups = get_deadline_limit_groups(server_url, auth) + machines = get_deadline_workers(server_url, auth) + server_info = DeadlineServerInfo( + pools, groups, limit_groups, machines ) - self._pools_by_server_name[server_name] = pools - - return pools - - def get_groups_by_server_name(self, server_name: str) -> List[str]: - """Returns dictionary of groups per DL server - - Args: - server_name (str): Deadline Server name from Project Settings. - - Returns: - Dict[str, List[str]]: {"default": ["group1", "group2"]} - - """ - groups = self._groups_by_server_name.get(server_name) - if groups is None: - dl_server_info = self.deadline_servers_info[server_name] - - auth = (dl_server_info["default_username"], - dl_server_info["default_password"]) - groups = get_deadline_groups( - dl_server_info["value"], - auth - ) - self._groups_by_server_name[server_name] = groups - - return groups - - def get_limit_groups_by_server_name(self, server_name: str) -> List[str]: - """Returns dictionary of limit groups per DL server - - Args: - server_name (str): Deadline Server name from Project Settings. - - Returns: - Dict[str, List[str]]: {"default": ["limit1", "limit2"]} - - """ - limit_groups = self._limit_groups_by_server_name.get(server_name) - if limit_groups is None: - dl_server_info = self.deadline_servers_info[server_name] - - auth = (dl_server_info["default_username"], - dl_server_info["default_password"]) - limit_groups = get_deadline_limit_groups( - dl_server_info["value"], - auth - ) - self._limit_groups_by_server_name[server_name] = limit_groups - - return limit_groups - - def get_machines_by_server_name(self, server_name: str) -> List[str]: - """Returns dictionary of machines/workers per DL server - - Args: - server_name (str): Deadline Server name from Project Settings. - - Returns: - Dict[str, List[str]]: {"default": ["renderNode1", "PC1"]} - - """ - machines = self._machines_by_server_name.get(server_name) - if machines is None: - dl_server_info = self.deadline_servers_info[server_name] - - auth = (dl_server_info["default_username"], - dl_server_info["default_password"]) - machines = get_deadline_workers( - dl_server_info["value"], - auth - ) - self._machines_by_server_name[server_name] = machines - - return machines + self._server_info_by_name[server_name] = server_info + + return server_info + + def _get_deadline_con_info( + self, server_name: str + ) -> Tuple[str, Optional[Tuple[str, str]], bool]: + dl_server_info = self.deadline_servers_info[server_name] + auth = self._get_server_user_auth(dl_server_info) + return ( + dl_server_info["value"], + auth, + not dl_server_info["not_verify_ssl"] + ) + + def _get_server_user_auth( + self, server_info: Dict[str, Any] + ) -> Optional[Tuple[str, str]]: + server_name = server_info["name"] + + require_authentication = server_info["require_authentication"] + if require_authentication: + # TODO import 'get_addon_site_settings' when available + # in public 'ayon_api' + con = ayon_api.get_server_api_connection() + local_settings = con.get_addon_site_settings(self.name, self.version) + local_settings = local_settings["local_settings"] + for server_info in local_settings: + if server_name != server_info["server_name"]: + continue + + if server_info["username"] and server_info["password"]: + return server_info["username"], server_info["password"] + + default_username = server_info["default_username"] + default_password = server_info["default_password"] + if default_username and default_password: + return default_username, default_password diff --git a/client/ayon_deadline/lib.py b/client/ayon_deadline/lib.py index 6754747cba..296df6467e 100644 --- a/client/ayon_deadline/lib.py +++ b/client/ayon_deadline/lib.py @@ -25,6 +25,14 @@ JOB_ENV_DATA_KEY: str = "farmJobEnv" +@dataclass +class DeadlineServerInfo: + pools: List[str] + limit_groups: List[str] + groups: List[str] + machines: List[str] + + def get_ayon_render_job_envs() -> "dict[str, str]": """Get required env vars for valid render job submission.""" return { @@ -74,9 +82,8 @@ def get_deadline_pools( RuntimeError: If deadline webservice is unreachable. """ - endpoint = "{}/api/pools?NamesOnly=true".format(webservice_url) - return _get_deadline_info( - endpoint, auth, log, item_type="pools") + endpoint = f"{webservice_url}/api/pools?NamesOnly=true" + return _get_deadline_info(endpoint, auth, log, "pools") def get_deadline_groups( @@ -99,9 +106,8 @@ def get_deadline_groups( RuntimeError: If deadline webservice_url is unreachable. """ - endpoint = "{}/api/groups".format(webservice_url) - return _get_deadline_info( - endpoint, auth, log, item_type="groups") + endpoint = f"{webservice_url}/api/groups" + return _get_deadline_info(endpoint, auth, log, "groups") def get_deadline_limit_groups( @@ -124,9 +130,8 @@ def get_deadline_limit_groups( RuntimeError: If deadline webservice_url is unreachable. """ - endpoint = "{}/api/limitgroups?NamesOnly=true".format(webservice_url) - return _get_deadline_info( - endpoint, auth, log, item_type="limitgroups") + endpoint = f"{webservice_url}/api/limitgroups?NamesOnly=true" + return _get_deadline_info(endpoint, auth, log, "limitgroups") def get_deadline_workers( webservice_url: str, @@ -148,16 +153,15 @@ def get_deadline_workers( RuntimeError: If deadline webservice_url is unreachable. """ - endpoint = "{}/api/slaves?NamesOnly=true".format(webservice_url) - return _get_deadline_info( - endpoint, auth, log, item_type="workers") + endpoint = f"{webservice_url}/api/slaves?NamesOnly=true" + return _get_deadline_info(endpoint, auth, log, "workers") def _get_deadline_info( endpoint, - auth=None, - log=None, - item_type=None + auth, + log, + item_type ): from .abstract_submit_deadline import requests_get @@ -170,7 +174,7 @@ def _get_deadline_info( kwargs["auth"] = auth response = requests_get(endpoint, **kwargs) except requests.exceptions.ConnectionError as exc: - msg = 'Cannot connect to DL web service {}'.format(endpoint) + msg = f"Cannot connect to DL web service {endpoint}" log.error(msg) raise DeadlineWebserviceError(msg) from exc if not response.ok: From 28bbd78d32128cc3071aacc8377f6a60dde0588f Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:00:32 +0100 Subject: [PATCH 2/2] use server info on collect job info --- .../plugins/publish/global/collect_jobinfo.py | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/client/ayon_deadline/plugins/publish/global/collect_jobinfo.py b/client/ayon_deadline/plugins/publish/global/collect_jobinfo.py index 4b997c5de9..dad88d2e48 100644 --- a/client/ayon_deadline/plugins/publish/global/collect_jobinfo.py +++ b/client/ayon_deadline/plugins/publish/global/collect_jobinfo.py @@ -122,31 +122,24 @@ def apply_settings(cls, project_settings): limit_groups = [] machines = [] try: + server_info = deadline_addon.get_server_info_by_name( + deadline_server_name + ) pools = [ {"value": pool, "label": pool} - for pool in deadline_addon.get_pools_by_server_name( - deadline_server_name - ) + for pool in server_info.pools ] groups = [ {"value": group, "label": group} - for group in deadline_addon.get_groups_by_server_name( - deadline_server_name - ) + for group in server_info.groups ] limit_groups = [ {"value": limit_group, "label": limit_group} - for limit_group in ( - deadline_addon.get_limit_groups_by_server_name( - deadline_server_name - ) - ) + for limit_group in server_info.limit_groups ] machines = [ {"value": machine, "label": machine} - for machine in deadline_addon.get_machines_by_server_name( - deadline_server_name - ) + for machine in server_info.machines ] except DeadlineWebserviceError: cls.log.warning(f"Unable to connect to {deadline_server_name}")