From 640c24d9b70665d1ecec6cf293759c213eaf4271 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:33:39 +0200 Subject: [PATCH 001/136] add 'parents' to folder template data --- client/ayon_core/pipeline/template_data.py | 13 +++++++------ .../publish/collect_anatomy_instance_data.py | 5 ++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/client/ayon_core/pipeline/template_data.py b/client/ayon_core/pipeline/template_data.py index d5f06d6a59..2c5346f14b 100644 --- a/client/ayon_core/pipeline/template_data.py +++ b/client/ayon_core/pipeline/template_data.py @@ -87,14 +87,14 @@ def get_folder_template_data(folder_entity, project_name): """ path = folder_entity["path"] - hierarchy_parts = path.split("/") + parents = path.split("/") # Remove empty string from the beginning - hierarchy_parts.pop(0) + parents.pop(0) # Remove last part which is folder name - folder_name = hierarchy_parts.pop(-1) - hierarchy = "/".join(hierarchy_parts) - if hierarchy_parts: - parent_name = hierarchy_parts[-1] + folder_name = parents.pop(-1) + hierarchy = "/".join(parents) + if parents: + parent_name = parents[-1] else: parent_name = project_name @@ -103,6 +103,7 @@ def get_folder_template_data(folder_entity, project_name): "name": folder_name, "type": folder_entity["folderType"], "path": path, + "parents": parents, }, "asset": folder_name, "hierarchy": hierarchy, diff --git a/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py b/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py index b6636696c1..6eeca6ad29 100644 --- a/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py +++ b/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py @@ -407,8 +407,10 @@ def _fill_folder_data(self, instance, project_entity, anatomy_data): anatomy_data["hierarchy"] = hierarchy parent_name = project_entity["name"] + parents = [] if hierarchy: - parent_name = hierarchy.split("/")[-1] + parents = hierarchy.split("/") + parent_name = parents[-1] folder_name = instance.data["folderPath"].split("/")[-1] anatomy_data.update({ @@ -422,6 +424,7 @@ def _fill_folder_data(self, instance, project_entity, anatomy_data): # Using 'Shot' is current default behavior of editorial # (or 'newHierarchyIntegration') publishing. "type": "Shot", + "parents": parents, }, }) From c08d0baa88839020caa7e9372fb286f100d56fc3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:57:55 +0200 Subject: [PATCH 002/136] simplify split --- client/ayon_core/pipeline/template_data.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/template_data.py b/client/ayon_core/pipeline/template_data.py index 2c5346f14b..c7aa46fd62 100644 --- a/client/ayon_core/pipeline/template_data.py +++ b/client/ayon_core/pipeline/template_data.py @@ -87,9 +87,8 @@ def get_folder_template_data(folder_entity, project_name): """ path = folder_entity["path"] - parents = path.split("/") - # Remove empty string from the beginning - parents.pop(0) + # Remove empty string from the beginning and split by '/' + parents = path.lstrip("/").split("/") # Remove last part which is folder name folder_name = parents.pop(-1) hierarchy = "/".join(parents) From 282d1720ae958bbe8833ac64d6295e658a28b438 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 12 Sep 2024 16:39:09 +0200 Subject: [PATCH 003/136] Add staging directory functions and configurations - Added functions to handle custom staging directories - Updated imports and removed deprecated code - Created a new module for staging directory handling --- client/ayon_core/pipeline/__init__.py | 27 +-- .../ayon_core/pipeline/publish/constants.py | 1 - client/ayon_core/pipeline/publish/lib.py | 177 ++++++-------- client/ayon_core/pipeline/stagingdir.py | 220 ++++++++++++++++++ client/ayon_core/pipeline/tempdir.py | 94 ++++++-- .../publish/collect_custom_staging_dir.py | 76 ------ .../plugins/publish/extract_burnin.py | 11 +- .../publish/extract_color_transcode.py | 15 +- .../plugins/publish/extract_review.py | 7 +- 9 files changed, 399 insertions(+), 229 deletions(-) create mode 100644 client/ayon_core/pipeline/stagingdir.py delete mode 100644 client/ayon_core/plugins/publish/collect_custom_staging_dir.py diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index 8fd00ee6b6..d5c3140d37 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -8,6 +8,10 @@ from .anatomy import Anatomy +from .tempdir import get_temp_dir + +from .stagingdir import get_staging_dir + from .create import ( BaseCreator, Creator, @@ -116,10 +120,12 @@ "AYON_CONTAINER_ID", "AYON_INSTANCE_ID", "HOST_WORKFILE_EXTENSIONS", - # --- Anatomy --- "Anatomy", - + # --- Temp dir --- + "get_temp_dir", + # --- Staging dir --- + "get_staging_dir", # --- Create --- "BaseCreator", "Creator", @@ -127,42 +133,34 @@ "HiddenCreator", "CreatedInstance", "CreatorError", - "CreatorError", - # - legacy creation "LegacyCreator", "legacy_create", - "discover_creator_plugins", "discover_legacy_creator_plugins", "register_creator_plugin", "deregister_creator_plugin", "register_creator_plugin_path", "deregister_creator_plugin_path", - # --- Load --- "HeroVersionType", "IncompatibleLoaderError", "LoaderPlugin", "ProductLoaderPlugin", - "discover_loader_plugins", "register_loader_plugin", "deregister_loader_plugin_path", "register_loader_plugin_path", "deregister_loader_plugin", - "load_container", "remove_container", "update_container", "switch_container", - "loaders_from_representation", "get_representation_path", "get_representation_context", "get_repres_contexts", - # --- Publish --- "PublishValidationError", "PublishXmlValidationError", @@ -170,50 +168,41 @@ "AYONPyblishPluginMixin", "OpenPypePyblishPluginMixin", "OptionalPyblishPluginMixin", - # --- Actions --- "LauncherAction", "InventoryAction", - "discover_launcher_actions", "register_launcher_action", "register_launcher_action_path", - "discover_inventory_actions", "register_inventory_action", "register_inventory_action_path", "deregister_inventory_action", "deregister_inventory_action_path", - # --- Process context --- "install_ayon_plugins", "install_openpype_plugins", "install_host", "uninstall_host", "is_installed", - "register_root", "registered_root", - "register_host", "registered_host", "deregister_host", "get_process_id", - "get_global_context", "get_current_context", "get_current_host_name", "get_current_project_name", "get_current_folder_path", "get_current_task_name", - # Workfile templates "discover_workfile_build_plugins", "register_workfile_build_plugin", "deregister_workfile_build_plugin", "register_workfile_build_plugin_path", "deregister_workfile_build_plugin_path", - # Backwards compatible function names "install", "uninstall", diff --git a/client/ayon_core/pipeline/publish/constants.py b/client/ayon_core/pipeline/publish/constants.py index 38f5ffef3f..5240628365 100644 --- a/client/ayon_core/pipeline/publish/constants.py +++ b/client/ayon_core/pipeline/publish/constants.py @@ -8,4 +8,3 @@ DEFAULT_PUBLISH_TEMPLATE = "default" DEFAULT_HERO_PUBLISH_TEMPLATE = "default" -TRANSIENT_DIR_TEMPLATE = "default" diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 8b82622e4c..9cfcd3f71a 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -2,7 +2,6 @@ import sys import inspect import copy -import tempfile import xml.etree.ElementTree from typing import Optional, Union, List @@ -18,15 +17,11 @@ ) from ayon_core.settings import get_project_settings from ayon_core.addon import AddonsManager -from ayon_core.pipeline import ( - tempdir, - Anatomy -) +from ayon_core.pipeline import get_staging_dir from ayon_core.pipeline.plugin_discover import DiscoverResult from .constants import ( DEFAULT_PUBLISH_TEMPLATE, DEFAULT_HERO_PUBLISH_TEMPLATE, - TRANSIENT_DIR_TEMPLATE ) @@ -581,58 +576,6 @@ def context_plugin_should_run(plugin, context): return False -def get_instance_staging_dir(instance): - """Unified way how staging dir is stored and created on instances. - - First check if 'stagingDir' is already set in instance data. - In case there already is new tempdir will not be created. - - It also supports `AYON_TMPDIR`, so studio can define own temp - shared repository per project or even per more granular context. - Template formatting is supported also with optional keys. Folder is - created in case it doesn't exists. - - Available anatomy formatting keys: - - root[work | ] - - project[name | code] - - Note: - Staging dir does not have to be necessarily in tempdir so be careful - about its usage. - - Args: - instance (pyblish.lib.Instance): Instance for which we want to get - staging dir. - - Returns: - str: Path to staging dir of instance. - """ - staging_dir = instance.data.get('stagingDir') - if staging_dir: - return staging_dir - - anatomy = instance.context.data.get("anatomy") - - # get customized tempdir path from `AYON_TMPDIR` env var - custom_temp_dir = tempdir.create_custom_tempdir( - anatomy.project_name, anatomy) - - if custom_temp_dir: - staging_dir = os.path.normpath( - tempfile.mkdtemp( - prefix="pyblish_tmp_", - dir=custom_temp_dir - ) - ) - else: - staging_dir = os.path.normpath( - tempfile.mkdtemp(prefix="pyblish_tmp_") - ) - instance.data['stagingDir'] = staging_dir - - return staging_dir - - def get_publish_repre_path(instance, repre, only_published=False): """Get representation path that can be used for integration. @@ -685,6 +628,8 @@ def get_publish_repre_path(instance, repre, only_published=False): return None +# deprecated: backward compatibility only +# TODO: remove in the future def get_custom_staging_dir_info( project_name, host_name, @@ -694,67 +639,85 @@ def get_custom_staging_dir_info( product_name, project_settings=None, anatomy=None, - log=None + log=None, ): - """Checks profiles if context should use special custom dir as staging. + from ayon_core.pipeline.stagingdir import get_staging_dir_config + + tr_data = get_staging_dir_config( + host_name, + project_name, + task_type, + task_name, + product_type, + product_name, + project_settings=project_settings, + anatomy=anatomy, + log=log, + ) - Args: - project_name (str) - host_name (str) - product_type (str) - task_name (str) - task_type (str) - product_name (str) - project_settings(Dict[str, Any]): Prepared project settings. - anatomy (Dict[str, Any]) - log (Logger) (optional) + if not tr_data: + return None, None + + return tr_data["template"], tr_data["persistence"] + + +def get_instance_staging_dir(instance): + """Unified way how staging dir is stored and created on instances. + + First check if 'stagingDir' is already set in instance data. + In case there already is new tempdir will not be created. Returns: - (tuple) - Raises: - ValueError - if misconfigured template should be used + str: Path to staging dir """ - settings = project_settings or get_project_settings(project_name) - custom_staging_dir_profiles = (settings["core"] - ["tools"] - ["publish"] - ["custom_staging_dir_profiles"]) - if not custom_staging_dir_profiles: - return None, None + staging_dir = instance.data.get("stagingDir") - if not log: - log = Logger.get_logger("get_custom_staging_dir_info") + if staging_dir: + return staging_dir - filtering_criteria = { - "hosts": host_name, - "families": product_type, - "task_names": task_name, - "task_types": task_type, - "subsets": product_name - } - profile = filter_profiles(custom_staging_dir_profiles, - filtering_criteria, - logger=log) + anatomy_data = instance.data["anatomyData"] + formatting_data = copy.deepcopy(anatomy_data) - if not profile or not profile["active"]: - return None, None + product_type = instance.data["productType"] + product_name = instance.data["productName"] + + # context data based variables + project_entity = instance.context.data["projectEntity"] + folder_entity = instance.context.data["folderEntity"] + task_entity = instance.context.data["taskEntity"] + host_name = instance.context.data["hostName"] + project_settings = instance.context.data["project_settings"] + anatomy = instance.context.data["anatomy"] + current_file = instance.context.data.get("currentFile") + + # add current file as workfile name into formatting data + if current_file: + workfile = os.path.basename(current_file) + workfile_name, _ = os.path.splitext(workfile) + formatting_data["workfile_name"] = workfile_name + + dir_data = get_staging_dir( + host_name, + project_entity, + folder_entity, + task_entity, + product_type, + product_name, + anatomy, + project_settings=project_settings, + formatting_data=formatting_data, + ) - if not anatomy: - anatomy = Anatomy(project_name) + staging_dir_path = dir_data["stagingDir"] - template_name = profile["template_name"] or TRANSIENT_DIR_TEMPLATE + # TODO: not sure if this is necessary + # path might be already created by get_staging_dir + if not os.path.exists(staging_dir_path): + os.makedirs(staging_dir_path) - custom_staging_dir = anatomy.get_template_item( - "staging", template_name, "directory", default=None - ) - if custom_staging_dir is None: - raise ValueError(( - "Anatomy of project \"{}\" does not have set" - " \"{}\" template key!" - ).format(project_name, template_name)) - is_persistent = profile["custom_staging_dir_persistent"] + instance.data.update(dir_data) - return custom_staging_dir.template, is_persistent + return staging_dir_path def get_published_workfile_instance(context): diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py new file mode 100644 index 0000000000..e8fa1c4853 --- /dev/null +++ b/client/ayon_core/pipeline/stagingdir.py @@ -0,0 +1,220 @@ +from ayon_core.lib import Logger, filter_profiles, StringTemplate +from ayon_core.settings import get_project_settings +from .anatomy import Anatomy +from .tempdir import get_temp_dir +from ayon_core.pipeline.template_data import get_template_data + + +STAGING_DIR_TEMPLATES = "staging" + + +def get_staging_dir_config( + host_name, + project_name, + task_type, + task_name, + product_type, + product_name, + project_settings=None, + anatomy=None, + log=None, +): + """Get matching staging dir profile. + + Args: + host_name (str): Name of host. + project_name (str): Name of project. + task_type (str): Type of task. + task_name (str): Name of task. + product_type (str): Type of product. + product_name (str): Name of product. + project_settings(Dict[str, Any]): Prepared project settings. + anatomy (Dict[str, Any]) + log (Optional[logging.Logger]) + + Returns: + Dict or None: Data with directory template and is_persistent or None + Raises: + ValueError - if misconfigured template should be used + """ + settings = project_settings or get_project_settings(project_name) + + staging_dir_profiles = settings["core"]["tools"]["publish"][ + "custom_staging_dir_profiles" + ] + + if not staging_dir_profiles: + return None + + if not log: + log = Logger.get_logger("get_staging_dir_config") + + filtering_criteria = { + "hosts": host_name, + "task_types": task_type, + "task_names": task_name, + "product_types": product_type, + "product_names": product_name, + } + profile = filter_profiles( + staging_dir_profiles, filtering_criteria, logger=log) + + if not profile or not profile["active"]: + return None + + if not anatomy: + anatomy = Anatomy(project_name) + + # get template from template name + template_name = profile["template_name"] + _validate_template_name(project_name, template_name, anatomy) + + template = anatomy.templates[STAGING_DIR_TEMPLATES][template_name] + + if not template: + # template should always be found either from anatomy or from profile + raise ValueError( + "Staging dir profile is misconfigured! " + "No template was found for profile! " + "Check your project settings at: " + "'ayon+settings://core/tools/publish/custom_staging_dir_profiles'" + ) + + data_persistence = profile["custom_staging_dir_persistent"] + + return {"template": template, "persistence": data_persistence} + + +def _validate_template_name(project_name, template_name, anatomy): + """Check that staging dir section with appropriate template exist. + + Raises: + ValueError - if misconfigured template + """ + # TODO: only for backward compatibility of anatomy for older projects + if STAGING_DIR_TEMPLATES not in anatomy.templates: + raise ValueError( + ( + 'Anatomy of project "{}" does not have set' ' "{}" template section!' + ).format(project_name, template_name) + ) + + if template_name not in anatomy.templates[STAGING_DIR_TEMPLATES]: + raise ValueError( + ( + 'Anatomy of project "{}" does not have set' + ' "{}" template key at Staging Dir section!' + ).format(project_name, template_name) + ) + + +def get_staging_dir( + host_name, + project_entity, + folder_entity, + task_entity, + product_type, + product_name, + anatomy, + project_settings=None, + **kwargs +): + """Get staging dir data. + + If `force_temp` is set, staging dir will be created as tempdir. + If `always_get_some_dir` is set, staging dir will be created as tempdir if + no staging dir profile is found. + If `prefix` or `suffix` is not set, default values will be used. + + Arguments: + host_name (str): Name of host. + project_entity (Dict[str, Any]): Project entity. + folder_entity (Dict[str, Any]): Folder entity. + task_entity (Dict[str, Any]): Task entity. + product_type (str): Type of product. + product_name (str): Name of product. + anatomy (ayon_core.pipeline.Anatomy): Anatomy object. + project_settings (Optional[Dict[str, Any]]): Prepared project settings. + **kwargs: Arbitrary keyword arguments. See below. + + Keyword Arguments: + force_temp (bool): If True, staging dir will be created as tempdir. + always_return_path (bool): If True, staging dir will be created as + tempdir if no staging dir profile is found. + prefix (str): Prefix for staging dir. + suffix (str): Suffix for staging dir. + formatting_data (Dict[str, Any]): Data for formatting staging dir + template. + + Returns: + Dict[str, Any]: Staging dir data + """ + + log = kwargs.get("log") or Logger.get_logger("get_staging_dir") + always_return_path = kwargs.get("always_return_path") + + # make sure always_return_path is set to true by default + if always_return_path is None: + always_return_path = True + + if kwargs.get("force_temp"): + return get_temp_dir( + project_name=project_entity["name"], + anatomy=anatomy, + prefix=kwargs.get("prefix"), + suffix=kwargs.get("suffix"), + ) + + # making fewer queries to database + ctx_data = get_template_data( + project_entity, folder_entity, task_entity, host_name + ) + # add roots to ctx_data + ctx_data["root"] = anatomy.roots + + # add additional data + ctx_data.update({ + "product": { + "type": product_type, + "name": product_name + }, + "host": host_name, + }) + + # add additional data from kwargs + if kwargs.get("formatting_data"): + ctx_data.update(kwargs.get("formatting_data")) + + # get staging dir config + staging_dir_config = get_staging_dir_config( + host_name, + project_entity["name"], + task_entity["type"], + task_entity["name"], + product_type, + product_name, + project_settings=project_settings, + anatomy=anatomy, + log=log, + ) + + # if no preset matching and always_get_some_dir is set, return tempdir + if not staging_dir_config and always_return_path: + return { + "stagingDir": get_temp_dir( + project_name=project_name, + anatomy=anatomy, + prefix=kwargs.get("prefix"), + suffix=kwargs.get("suffix"), + ), + "stagingDir_persistent": False, + } + elif not staging_dir_config: + return None + + return { + "stagingDir": StringTemplate.format_template( + staging_dir_config["template"], ctx_data + ), + "stagingDir_persistent": staging_dir_config["persistence"], + } diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index 29d4659393..a6328135ee 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -3,11 +3,80 @@ """ import os +import tempfile +from pathlib import Path from ayon_core.lib import StringTemplate from ayon_core.pipeline import Anatomy -def create_custom_tempdir(project_name, anatomy=None): +def get_temp_dir( + project_name=None, anatomy=None, prefix=None, suffix=None, make_local=False +): + """Get temporary dir path. + + If `make_local` is set, tempdir will be created in local tempdir. + If `anatomy` is not set, default anatomy will be used. + If `prefix` or `suffix` is not set, default values will be used. + + It also supports `OPENPYPE_TMPDIR`, so studio can define own temp + shared repository per project or even per more granular context. + Template formatting is supported also with optional keys. Folder is + created in case it doesn't exists. + + Available anatomy formatting keys: + - root[work | ] + - project[name | code] + + Note: + Staging dir does not have to be necessarily in tempdir so be careful + about its usage. + + Args: + project_name (str)[optional]: Name of project. + anatomy (openpype.pipeline.Anatomy)[optional]: Anatomy object. + make_local (bool)[optional]: If True, temp dir will be created in + local tempdir. + suffix (str)[optional]: Suffix for tempdir. + prefix (str)[optional]: Prefix for tempdir. + + Returns: + str: Path to staging dir of instance. + """ + prefix = prefix or "ay_tmp_" + suffix = suffix or "" + + if make_local: + return _create_local_staging_dir(prefix, suffix) + + # make sure anatomy is set + if not anatomy: + anatomy = Anatomy(project_name) + + # get customized tempdir path from `OPENPYPE_TMPDIR` env var + custom_temp_dir = _create_custom_tempdir(anatomy.project_name, anatomy) + + return _create_local_staging_dir(prefix, suffix, custom_temp_dir) + + +def _create_local_staging_dir(prefix, suffix, dir=None): + """Create local staging dir + + Args: + prefix (str): prefix for tempdir + suffix (str): suffix for tempdir + + Returns: + str: path to tempdir + """ + # use pathlib for creating tempdir + staging_dir = Path(tempfile.mkdtemp( + prefix=prefix, suffix=suffix, dir=dir + )) + + return staging_dir.as_posix() + + +def _create_custom_tempdir(project_name, anatomy=None): """ Create custom tempdir Template path formatting is supporting: @@ -38,7 +107,7 @@ def create_custom_tempdir(project_name, anatomy=None): if anatomy is None: anatomy = Anatomy(project_name) # create base formate data - data = { + template_formatting_data = { "root": anatomy.roots, "project": { "name": anatomy.project_name, @@ -47,19 +116,14 @@ def create_custom_tempdir(project_name, anatomy=None): } # path is anatomy template custom_tempdir = StringTemplate.format_template( - env_tmpdir, data).normalized() + env_tmpdir, template_formatting_data) + + custom_tempdir_path = Path(custom_tempdir) else: # path is absolute - custom_tempdir = env_tmpdir - - # create the dir path if it doesn't exists - if not os.path.exists(custom_tempdir): - try: - # create it if it doesn't exists - os.makedirs(custom_tempdir) - except IOError as error: - raise IOError( - "Path couldn't be created: {}".format(error)) - - return custom_tempdir + custom_tempdir_path = Path(env_tmpdir) + + custom_tempdir_path.mkdir(parents=True, exist_ok=True) + + return custom_tempdir_path.as_posix() diff --git a/client/ayon_core/plugins/publish/collect_custom_staging_dir.py b/client/ayon_core/plugins/publish/collect_custom_staging_dir.py deleted file mode 100644 index 49c3a98dd2..0000000000 --- a/client/ayon_core/plugins/publish/collect_custom_staging_dir.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Requires: - anatomy - - -Provides: - instance.data -> stagingDir (folder path) - -> stagingDir_persistent (bool) -""" -import copy -import os.path - -import pyblish.api - -from ayon_core.pipeline.publish.lib import get_custom_staging_dir_info - - -class CollectCustomStagingDir(pyblish.api.InstancePlugin): - """Looks through profiles if stagingDir should be persistent and in special - location. - - Transient staging dir could be useful in specific use cases where is - desirable to have temporary renders in specific, persistent folders, could - be on disks optimized for speed for example. - - It is studio responsibility to clean up obsolete folders with data. - - Location of the folder is configured in `project_anatomy/templates/others`. - ('transient' key is expected, with 'folder' key) - - Which family/task type/product is applicable is configured in: - `project_settings/global/tools/publish/custom_staging_dir_profiles` - - """ - label = "Collect Custom Staging Directory" - order = pyblish.api.CollectorOrder + 0.4990 - - template_key = "transient" - - def process(self, instance): - product_type = instance.data["productType"] - product_name = instance.data["productName"] - host_name = instance.context.data["hostName"] - project_name = instance.context.data["projectName"] - project_settings = instance.context.data["project_settings"] - anatomy = instance.context.data["anatomy"] - task = instance.data["anatomyData"].get("task", {}) - - transient_tml, is_persistent = get_custom_staging_dir_info( - project_name, - host_name, - product_type, - product_name, - task.get("name"), - task.get("type"), - project_settings=project_settings, - anatomy=anatomy, - log=self.log) - - if transient_tml: - anatomy_data = copy.deepcopy(instance.data["anatomyData"]) - anatomy_data["root"] = anatomy.roots - scene_name = instance.context.data.get("currentFile") - if scene_name: - anatomy_data["scene_name"] = os.path.basename(scene_name) - transient_dir = transient_tml.format(**anatomy_data) - instance.data["stagingDir"] = transient_dir - - instance.data["stagingDir_persistent"] = is_persistent - result_str = "Adding '{}' as".format(transient_dir) - else: - result_str = "Not adding" - - self.log.debug("{} custom staging dir for instance with '{}'".format( - result_str, product_type - )) diff --git a/client/ayon_core/plugins/publish/extract_burnin.py b/client/ayon_core/plugins/publish/extract_burnin.py index 58a032a030..72578d9dc0 100644 --- a/client/ayon_core/plugins/publish/extract_burnin.py +++ b/client/ayon_core/plugins/publish/extract_burnin.py @@ -9,11 +9,13 @@ import pyblish.api from ayon_core import resources, AYON_CORE_ROOT -from ayon_core.pipeline import publish +from ayon_core.pipeline import ( + publish, + get_temp_dir +) from ayon_core.lib import ( run_ayon_launcher_process, - get_transcode_temp_directory, convert_input_paths_for_ffmpeg, should_convert_for_ffmpeg ) @@ -250,7 +252,10 @@ def main_process(self, instance): # - change staging dir of source representation # - must be set back after output definitions processing if do_convert: - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + ) repre["stagingDir"] = new_staging_dir convert_input_paths_for_ffmpeg( diff --git a/client/ayon_core/plugins/publish/extract_color_transcode.py b/client/ayon_core/plugins/publish/extract_color_transcode.py index a28a761e7e..ba173867f8 100644 --- a/client/ayon_core/plugins/publish/extract_color_transcode.py +++ b/client/ayon_core/plugins/publish/extract_color_transcode.py @@ -3,15 +3,15 @@ import clique import pyblish.api -from ayon_core.pipeline import publish +from ayon_core.pipeline import ( + publish, + get_temp_dir +) from ayon_core.lib import ( - is_oiio_supported, ) - from ayon_core.lib.transcoding import ( convert_colorspace, - get_transcode_temp_directory, ) from ayon_core.lib.profiles_filtering import filter_profiles @@ -104,7 +104,10 @@ def process(self, instance): new_repre = copy.deepcopy(repre) original_staging_dir = new_repre["stagingDir"] - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + ) new_repre["stagingDir"] = new_staging_dir if isinstance(new_repre["files"], list): @@ -254,7 +257,7 @@ def _translate_to_sequence(self, files_to_convert): (list) of [file.1001-1010#.exr] or [fileA.exr, fileB.exr] """ pattern = [clique.PATTERNS["frames"]] - collections, remainder = clique.assemble( + collections, _ = clique.assemble( files_to_convert, patterns=pattern, assume_padded_when_ambiguous=True) diff --git a/client/ayon_core/plugins/publish/extract_review.py b/client/ayon_core/plugins/publish/extract_review.py index 06b451bfbe..26cd2ef0b2 100644 --- a/client/ayon_core/plugins/publish/extract_review.py +++ b/client/ayon_core/plugins/publish/extract_review.py @@ -22,8 +22,8 @@ should_convert_for_ffmpeg, get_review_layer_name, convert_input_paths_for_ffmpeg, - get_transcode_temp_directory, ) +from ayon_core.pipeline import get_temp_dir from ayon_core.pipeline.publish import ( KnownPublishError, get_publish_instance_label, @@ -310,7 +310,10 @@ def main_process(self, instance): # - change staging dir of source representation # - must be set back after output definitions processing if do_convert: - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + ) repre["stagingDir"] = new_staging_dir convert_input_paths_for_ffmpeg( From 8b1674619ce7004069bed2fa10d8af39ffac3cb6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 12 Sep 2024 16:40:49 +0200 Subject: [PATCH 004/136] Add staging directory functionality and a new plugin for managing staging directories in the pipeline. - Added import statement for 'os' in creator_plugins.py - Implemented method 'apply_staging_dir' to apply staging directory with persistence to instance's transient data in creator_plugins.py - Updated comments and added TODOs related to staging directories in various files - Created a new plugin 'CollectManagedStagingDir' to manage staging directories in publish/lib.py --- .../pipeline/create/creator_plugins.py | 55 +++++++++++++++++++ client/ayon_core/pipeline/publish/lib.py | 2 +- .../publish/collect_managed_staging_dir.py | 43 +++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 client/ayon_core/plugins/publish/collect_managed_staging_dir.py diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 61c10ee736..1360a74519 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import os import copy import collections from typing import TYPE_CHECKING, Optional @@ -14,6 +15,7 @@ deregister_plugin, deregister_plugin_path ) +from ayon_core.pipeline import get_staging_dir from .constants import DEFAULT_VARIANT_VALUE from .product_name import get_product_name @@ -782,6 +784,59 @@ def get_pre_create_attr_defs(self): """ return self.pre_create_attr_defs + def apply_staging_dir(self, instance): + """Apply staging dir with persistence to instance's transient data. + + Method is called on instance creation and on instance update. + + Args: + instance (CreatedInstance): Instance for which should be staging + dir applied. + + Returns: + str: Path to staging dir. + """ + create_ctx = self.create_context + product_name = instance.get("productName") + product_type = instance.get("productType") + folder_path = instance.get("folderPath") + if not any([product_name, folder_path]): + return None + + version = instance.get("version") + if version is not None: + formatting_data = {"version": version} + + staging_dir_data = get_staging_dir( + create_ctx.host_name, + create_ctx.get_current_project_entity(), + create_ctx.get_current_folder_entity(), + create_ctx.get_current_task_entity(), + product_type, + product_name, + create_ctx.get_current_project_anatomy(), + create_ctx.get_current_project_settings(), + always_return_path=False, + log=self.log, + formatting_data=formatting_data, + ) + + if not staging_dir_data: + return None + + staging_dir_path = staging_dir_data["stagingDir"] + + # TODO: not sure if this is necessary + # path might be already created by get_staging_dir + if not os.path.exists(staging_dir_path): + os.makedirs(staging_dir_path) + + instance.transient_data.update(staging_dir_data) + + self.log.info(f"Applied staging dir to instance: {staging_dir_path}") + + return staging_dir_path + class HiddenCreator(BaseCreator): @abstractmethod diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 9cfcd3f71a..714794e8f8 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -628,7 +628,7 @@ def get_publish_repre_path(instance, repre, only_published=False): return None -# deprecated: backward compatibility only +# deprecated: backward compatibility only (2024-09-12) # TODO: remove in the future def get_custom_staging_dir_info( project_name, diff --git a/client/ayon_core/plugins/publish/collect_managed_staging_dir.py b/client/ayon_core/plugins/publish/collect_managed_staging_dir.py new file mode 100644 index 0000000000..ca6d5161c1 --- /dev/null +++ b/client/ayon_core/plugins/publish/collect_managed_staging_dir.py @@ -0,0 +1,43 @@ +""" +Requires: + anatomy + + +Provides: + instance.data -> stagingDir (folder path) + -> stagingDir_persistent (bool) +""" + +import pyblish.api + +from ayon_core.pipeline.publish import get_instance_staging_dir + + +class CollectManagedStagingDir(pyblish.api.InstancePlugin): + """Apply matching Staging Dir profile to a instance. + + Apply Staging dir via profiles could be useful in specific use cases + where is desirable to have temporary renders in specific, + persistent folders, could be on disks optimized for speed for example. + + It is studio's responsibility to clean up obsolete folders with data. + + Location of the folder is configured in: + `ayon+anatomy://_/templates/staging`. + + Which family/task type/subset is applicable is configured in: + `ayon+settings://core/tools/publish/custom_staging_dir_profiles` + """ + + label = "Collect Managed Staging Directory" + order = pyblish.api.CollectorOrder + 0.4990 + + def process(self, instance): + + staging_dir_path = get_instance_staging_dir(instance) + persistance = instance.data.get("stagingDir_persistent", False) + + self.log.info(( + f"Instance staging dir was set to `{staging_dir_path}` " + f"and persistence is set to `{persistance}`" + )) From 9e57f74b5c354a3d864eed71f545a5a7f93cc001 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 13 Sep 2024 16:20:46 +0200 Subject: [PATCH 005/136] Update variable names for clarity and consistency. - Renamed variables for better understanding and uniformity - Improved readability by using more descriptive names --- client/ayon_core/pipeline/publish/lib.py | 6 +++--- client/ayon_core/pipeline/stagingdir.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 714794e8f8..fb4db6ddf1 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -696,7 +696,7 @@ def get_instance_staging_dir(instance): workfile_name, _ = os.path.splitext(workfile) formatting_data["workfile_name"] = workfile_name - dir_data = get_staging_dir( + staging_dir_data = get_staging_dir( host_name, project_entity, folder_entity, @@ -708,14 +708,14 @@ def get_instance_staging_dir(instance): formatting_data=formatting_data, ) - staging_dir_path = dir_data["stagingDir"] + staging_dir_path = staging_dir_data["stagingDir"] # TODO: not sure if this is necessary # path might be already created by get_staging_dir if not os.path.exists(staging_dir_path): os.makedirs(staging_dir_path) - instance.data.update(dir_data) + instance.data.update(staging_dir_data) return staging_dir_path diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index e8fa1c4853..5ab9596528 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -202,7 +202,7 @@ def get_staging_dir( if not staging_dir_config and always_return_path: return { "stagingDir": get_temp_dir( - project_name=project_name, + project_name=project_entity["name"], anatomy=anatomy, prefix=kwargs.get("prefix"), suffix=kwargs.get("suffix"), From 2f6ca5a2385bd3073961cf49bca220e9d3fb88b9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 18 Oct 2024 18:35:27 +0200 Subject: [PATCH 006/136] Implemented explicit frames for simple files representations --- .../pipeline/farm/pyblish_functions.py | 79 ++++++++++++++++--- 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 98951b2766..5908644dca 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -7,7 +7,7 @@ import attr import ayon_api import clique -from ayon_core.lib import Logger +from ayon_core.lib import Logger, collect_frames from ayon_core.pipeline import get_current_project_name, get_representation_path from ayon_core.pipeline.create import get_product_name from ayon_core.pipeline.farm.patterning import match_aov_pattern @@ -295,11 +295,17 @@ def _add_review_families(families): return families -def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, - skip_integration_repre_list, - do_not_add_review, - context, - color_managed_plugin): +def prepare_representations( + skeleton_data, + exp_files, + anatomy, + aov_filter, + skip_integration_repre_list, + do_not_add_review, + context, + color_managed_plugin, + frames_to_render +): """Create representations for file sequences. This will return representations of expected files if they are not @@ -315,6 +321,8 @@ def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, skip_integration_repre_list (list): exclude specific extensions, do_not_add_review (bool): explicitly skip review color_managed_plugin (publish.ColormanagedPyblishPluginMixin) + frames_to_render (str): implicit or explicit range of frames to render + this value is sent to Deadline in JobInfo.Frames Returns: list of representations @@ -325,6 +333,8 @@ def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, log = Logger.get_logger("farm_publishing") + frames_to_render = _get_real_frames_to_render(frames_to_render) + # create representation for every collected sequence for collection in collections: ext = collection.tail.lstrip(".") @@ -361,18 +371,21 @@ def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, " This may cause issues on farm." ).format(staging)) - frame_start = int(skeleton_data.get("frameStartHandle")) + frame_start = int(frames_to_render[0]) + frame_end = int(frames_to_render[-1]) if skeleton_data.get("slate"): frame_start -= 1 + files = _get_real_files_to_rendered(collection, frames_to_render) + # explicitly disable review by user preview = preview and not do_not_add_review rep = { "name": ext, "ext": ext, - "files": [os.path.basename(f) for f in list(collection)], + "files": files, "frameStart": frame_start, - "frameEnd": int(skeleton_data.get("frameEndHandle")), + "frameEnd": frame_end, # If expectedFile are absolute, we need only filenames "stagingDir": staging, "fps": skeleton_data.get("fps"), @@ -413,10 +426,13 @@ def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, " This may cause issues on farm." ).format(staging)) + files = _get_real_files_to_rendered( + [os.path.basename(remainder)], frames_to_render) + rep = { "name": ext, "ext": ext, - "files": os.path.basename(remainder), + "files": files[0], "stagingDir": staging, } @@ -453,6 +469,49 @@ def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, return representations +def _get_real_frames_to_render(frames): + """Returns list of frames that should be rendered. + + Artists could want to selectively render only particular frames + """ + frames_to_render = [] + for frame in frames.split(","): + if "-" in frame: + splitted = frame.split("-") + frames_to_render.extend(range(int(splitted[0]), int(splitted[1]))) + else: + frames_to_render.append(frame) + return [str(frame_to_render) for frame_to_render in frames_to_render] + + +def _get_real_files_to_rendered(collection, frames_to_render): + """Use expected files based on real frames_to_render. + + Artists might explicitly set frames they want to render via Publisher UI. + This uses this value to filter out files + Args: + frames_to_render (list): of str '1001' + """ + files = [os.path.basename(f) for f in list(collection)] + file_name, extracted_frame = list(collect_frames(files).items())[0] + if extracted_frame: + found_frame_pattern_length = len(extracted_frame) + normalized_frames_to_render = set() + for frame_to_render in frames_to_render: + normalized_frames_to_render.add( + str(frame_to_render).zfill(found_frame_pattern_length) + ) + + filtered_files = [] + for file_name in files: + if any(frame in file_name + for frame in normalized_frames_to_render): + filtered_files.append(file_name) + + files = filtered_files + return files + + def create_instances_for_aov(instance, skeleton, aov_filter, skip_integration_repre_list, do_not_add_review): From 2b5ab5439aa7d33b5c70ca11ce476cadae81ba0f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 13:41:20 +0200 Subject: [PATCH 007/136] returning empty lines --- client/ayon_core/pipeline/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index d5c3140d37..505c847c36 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -120,12 +120,16 @@ "AYON_CONTAINER_ID", "AYON_INSTANCE_ID", "HOST_WORKFILE_EXTENSIONS", + # --- Anatomy --- "Anatomy", + # --- Temp dir --- "get_temp_dir", + # --- Staging dir --- "get_staging_dir", + # --- Create --- "BaseCreator", "Creator", @@ -134,6 +138,7 @@ "CreatedInstance", "CreatorError", "CreatorError", + # - legacy creation "LegacyCreator", "legacy_create", @@ -143,6 +148,7 @@ "deregister_creator_plugin", "register_creator_plugin_path", "deregister_creator_plugin_path", + # --- Load --- "HeroVersionType", "IncompatibleLoaderError", @@ -161,6 +167,7 @@ "get_representation_path", "get_representation_context", "get_repres_contexts", + # --- Publish --- "PublishValidationError", "PublishXmlValidationError", @@ -168,6 +175,7 @@ "AYONPyblishPluginMixin", "OpenPypePyblishPluginMixin", "OptionalPyblishPluginMixin", + # --- Actions --- "LauncherAction", "InventoryAction", @@ -179,6 +187,7 @@ "register_inventory_action_path", "deregister_inventory_action", "deregister_inventory_action_path", + # --- Process context --- "install_ayon_plugins", "install_openpype_plugins", @@ -197,12 +206,14 @@ "get_current_project_name", "get_current_folder_path", "get_current_task_name", + # Workfile templates "discover_workfile_build_plugins", "register_workfile_build_plugin", "deregister_workfile_build_plugin", "register_workfile_build_plugin_path", "deregister_workfile_build_plugin_path", + # Backwards compatible function names "install", "uninstall", From 2b765954a3451bc6cba358584b64fed9e8f633e6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 13:54:05 +0200 Subject: [PATCH 008/136] returning empty lines --- client/ayon_core/pipeline/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index 505c847c36..ea8b1617c6 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -137,11 +137,13 @@ "HiddenCreator", "CreatedInstance", "CreatorError", + "CreatorError", # - legacy creation "LegacyCreator", "legacy_create", + "discover_creator_plugins", "discover_legacy_creator_plugins", "register_creator_plugin", @@ -154,15 +156,18 @@ "IncompatibleLoaderError", "LoaderPlugin", "ProductLoaderPlugin", + "discover_loader_plugins", "register_loader_plugin", "deregister_loader_plugin_path", "register_loader_plugin_path", "deregister_loader_plugin", + "load_container", "remove_container", "update_container", "switch_container", + "loaders_from_representation", "get_representation_path", "get_representation_context", @@ -179,9 +184,11 @@ # --- Actions --- "LauncherAction", "InventoryAction", + "discover_launcher_actions", "register_launcher_action", "register_launcher_action_path", + "discover_inventory_actions", "register_inventory_action", "register_inventory_action_path", @@ -194,12 +201,15 @@ "install_host", "uninstall_host", "is_installed", + "register_root", "registered_root", + "register_host", "registered_host", "deregister_host", "get_process_id", + "get_global_context", "get_current_context", "get_current_host_name", From 396af0cf8610c2d2991c8a7842f164dcc49d98f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 13:52:43 +0200 Subject: [PATCH 009/136] Update client/ayon_core/pipeline/create/creator_plugins.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/create/creator_plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 1360a74519..32ac2bd61f 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -800,7 +800,7 @@ def apply_staging_dir(self, instance): product_name = instance.get("productName") product_type = instance.get("productType") folder_path = instance.get("folderPath") - if not any([product_name, folder_path]): + if not product_name or not folder_path: return None version = instance.get("version") From 9a860785bbce917cc2064db74a018ab012922d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 13:52:57 +0200 Subject: [PATCH 010/136] Update client/ayon_core/pipeline/create/creator_plugins.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/create/creator_plugins.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 32ac2bd61f..0de7707e38 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -828,8 +828,7 @@ def apply_staging_dir(self, instance): # TODO: not sure if this is necessary # path might be already created by get_staging_dir - if not os.path.exists(staging_dir_path): - os.makedirs(staging_dir_path) + os.makedirs(staging_dir_path, exist_ok=True) instance.transient_data.update(staging_dir_data) From bd03634ed1a3237e49f1d3307a96722d0960b2a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 13:53:38 +0200 Subject: [PATCH 011/136] Update client/ayon_core/pipeline/publish/lib.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/publish/lib.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index fb4db6ddf1..ba56c38c78 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -642,7 +642,15 @@ def get_custom_staging_dir_info( log=None, ): from ayon_core.pipeline.stagingdir import get_staging_dir_config - + warnings.warn( + ( + "Function 'get_custom_staging_dir_info' in" + " 'ayon_core.pipeline.publish' is deprecated. Please use" + " 'get_custom_staging_dir_info'" + " in 'ayon_core.pipeline.stagingdir'." + ), + DeprecationWarning, + ) tr_data = get_staging_dir_config( host_name, project_name, From fedf8e60c7b33f5873538773561269e686ee81b4 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 13:56:45 +0200 Subject: [PATCH 012/136] Add warnings module for future use Imported the 'warnings' module for potential future usage in the codebase. --- client/ayon_core/pipeline/publish/lib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index ba56c38c78..657af9570b 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -2,6 +2,7 @@ import sys import inspect import copy +import warnings import xml.etree.ElementTree from typing import Optional, Union, List From ea23f355f6dcb24f2dcb2806878afa27ddd11907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 14:00:36 +0200 Subject: [PATCH 013/136] Apply suggestions from code review Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/publish/lib.py | 3 +-- client/ayon_core/pipeline/stagingdir.py | 20 +++++++------------- client/ayon_core/pipeline/tempdir.py | 15 ++++++--------- 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 657af9570b..6a31da82b2 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -721,8 +721,7 @@ def get_instance_staging_dir(instance): # TODO: not sure if this is necessary # path might be already created by get_staging_dir - if not os.path.exists(staging_dir_path): - os.makedirs(staging_dir_path) + os.makedirs(staging_dir_path, exist_ok=True) instance.data.update(staging_dir_data) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index 5ab9596528..d0172c4848 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -1,9 +1,9 @@ from ayon_core.lib import Logger, filter_profiles, StringTemplate from ayon_core.settings import get_project_settings -from .anatomy import Anatomy -from .tempdir import get_temp_dir from ayon_core.pipeline.template_data import get_template_data +from .anatomy import Anatomy +from .tempdir import get_temp_dir STAGING_DIR_TEMPLATES = "staging" @@ -34,8 +34,10 @@ def get_staging_dir_config( Returns: Dict or None: Data with directory template and is_persistent or None + Raises: ValueError - if misconfigured template should be used + """ settings = project_settings or get_project_settings(project_name) @@ -91,14 +93,6 @@ def _validate_template_name(project_name, template_name, anatomy): Raises: ValueError - if misconfigured template """ - # TODO: only for backward compatibility of anatomy for older projects - if STAGING_DIR_TEMPLATES not in anatomy.templates: - raise ValueError( - ( - 'Anatomy of project "{}" does not have set' ' "{}" template section!' - ).format(project_name, template_name) - ) - if template_name not in anatomy.templates[STAGING_DIR_TEMPLATES]: raise ValueError( ( @@ -147,9 +141,9 @@ def get_staging_dir( template. Returns: - Dict[str, Any]: Staging dir data - """ + Optional[Dict[str, Any]]: Staging dir data + """ log = kwargs.get("log") or Logger.get_logger("get_staging_dir") always_return_path = kwargs.get("always_return_path") @@ -209,7 +203,7 @@ def get_staging_dir( ), "stagingDir_persistent": False, } - elif not staging_dir_config: + if not staging_dir_config: return None return { diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index a6328135ee..448e774e7c 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -23,24 +23,21 @@ def get_temp_dir( Template formatting is supported also with optional keys. Folder is created in case it doesn't exists. - Available anatomy formatting keys: - - root[work | ] - - project[name | code] - Note: Staging dir does not have to be necessarily in tempdir so be careful about its usage. Args: - project_name (str)[optional]: Name of project. - anatomy (openpype.pipeline.Anatomy)[optional]: Anatomy object. - make_local (bool)[optional]: If True, temp dir will be created in + project_name (str): Name of project. + anatomy (Optional[Anatomy]): Project Anatomy object. + suffix (Optional[str]): Suffix for tempdir. + prefix (Optional[str]): Prefix for tempdir. + make_local (Optional[bool]): If True, temp dir will be created in local tempdir. - suffix (str)[optional]: Suffix for tempdir. - prefix (str)[optional]: Prefix for tempdir. Returns: str: Path to staging dir of instance. + """ prefix = prefix or "ay_tmp_" suffix = suffix or "" From ea4ec677cac7a73f14225722f0dbc9c17328ff6a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 14:40:30 +0200 Subject: [PATCH 014/136] reviewer suggestions for changes - Renamed function `get_staging_dir` to `get_staging_dir_info` for clarity. - Updated variable names in multiple files to use `template_data` instead of `formatting_data`. - Adjusted function parameters in the staging directory module for consistency and added new optional parameters. - Improved logging by passing logger instances instead of creating new loggers within functions. --- client/ayon_core/pipeline/__init__.py | 4 +- .../pipeline/create/creator_plugins.py | 18 ++++----- client/ayon_core/pipeline/publish/lib.py | 17 ++++----- client/ayon_core/pipeline/stagingdir.py | 34 ++++++++++------- client/ayon_core/pipeline/tempdir.py | 37 ++++++++----------- 5 files changed, 54 insertions(+), 56 deletions(-) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index ea8b1617c6..4060501a92 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -10,7 +10,7 @@ from .tempdir import get_temp_dir -from .stagingdir import get_staging_dir +from .stagingdir import get_staging_dir_info from .create import ( BaseCreator, @@ -128,7 +128,7 @@ "get_temp_dir", # --- Staging dir --- - "get_staging_dir", + "get_staging_dir_info", # --- Create --- "BaseCreator", diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 0de7707e38..124395ae16 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -15,7 +15,7 @@ deregister_plugin, deregister_plugin_path ) -from ayon_core.pipeline import get_staging_dir +from ayon_core.pipeline import get_staging_dir_info from .constants import DEFAULT_VARIANT_VALUE from .product_name import get_product_name @@ -805,9 +805,9 @@ def apply_staging_dir(self, instance): version = instance.get("version") if version is not None: - formatting_data = {"version": version} + template_data = {"version": version} - staging_dir_data = get_staging_dir( + staging_dir_info = get_staging_dir_info( create_ctx.host_name, create_ctx.get_current_project_entity(), create_ctx.get_current_folder_entity(), @@ -817,20 +817,20 @@ def apply_staging_dir(self, instance): create_ctx.get_current_project_anatomy(), create_ctx.get_current_project_settings(), always_return_path=False, - log=self.log, - formatting_data=formatting_data, + logger=self.log, + template_data=template_data, ) - if not staging_dir_data: + if not staging_dir_info: return None - staging_dir_path = staging_dir_data["stagingDir"] + staging_dir_path = staging_dir_info["stagingDir"] # TODO: not sure if this is necessary - # path might be already created by get_staging_dir + # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) - instance.transient_data.update(staging_dir_data) + instance.transient_data.update(staging_dir_info) self.log.info(f"Applied staging dir to instance: {staging_dir_path}") diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 6a31da82b2..0f3a7c1d45 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -18,7 +18,7 @@ ) from ayon_core.settings import get_project_settings from ayon_core.addon import AddonsManager -from ayon_core.pipeline import get_staging_dir +from ayon_core.pipeline import get_staging_dir_info from ayon_core.pipeline.plugin_discover import DiscoverResult from .constants import ( DEFAULT_PUBLISH_TEMPLATE, @@ -685,7 +685,7 @@ def get_instance_staging_dir(instance): return staging_dir anatomy_data = instance.data["anatomyData"] - formatting_data = copy.deepcopy(anatomy_data) + template_data = copy.deepcopy(anatomy_data) product_type = instance.data["productType"] product_name = instance.data["productName"] @@ -703,9 +703,9 @@ def get_instance_staging_dir(instance): if current_file: workfile = os.path.basename(current_file) workfile_name, _ = os.path.splitext(workfile) - formatting_data["workfile_name"] = workfile_name + template_data["workfile_name"] = workfile_name - staging_dir_data = get_staging_dir( + staging_dir_info = get_staging_dir_info( host_name, project_entity, folder_entity, @@ -714,16 +714,15 @@ def get_instance_staging_dir(instance): product_name, anatomy, project_settings=project_settings, - formatting_data=formatting_data, + template_data=template_data, ) - staging_dir_path = staging_dir_data["stagingDir"] + staging_dir_path = staging_dir_info["stagingDir"] - # TODO: not sure if this is necessary - # path might be already created by get_staging_dir + # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) - instance.data.update(staging_dir_data) + instance.data.update(staging_dir_info) return staging_dir_path diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index d0172c4848..e9d425cf28 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -102,7 +102,7 @@ def _validate_template_name(project_name, template_name, anatomy): ) -def get_staging_dir( +def get_staging_dir_info( host_name, project_entity, folder_entity, @@ -111,9 +111,13 @@ def get_staging_dir( product_name, anatomy, project_settings=None, + template_data=None, + always_return_path=None, + force_tmp_dir=None, + logger=None, **kwargs ): - """Get staging dir data. + """Get staging dir info data. If `force_temp` is set, staging dir will be created as tempdir. If `always_get_some_dir` is set, staging dir will be created as tempdir if @@ -129,29 +133,31 @@ def get_staging_dir( product_name (str): Name of product. anatomy (ayon_core.pipeline.Anatomy): Anatomy object. project_settings (Optional[Dict[str, Any]]): Prepared project settings. + template_data (Optional[Dict[str, Any]]): Data for formatting staging + dir template. + always_return_path (Optional[bool]): If True, staging dir will be + created as tempdir if no staging dir profile is found. Input value + False will return None if no staging dir profile is found. + force_tmp_dir (Optional[bool]): If True, staging dir will be created as + tempdir. + logger (Optional[logging.Logger]): Logger instance. **kwargs: Arbitrary keyword arguments. See below. Keyword Arguments: - force_temp (bool): If True, staging dir will be created as tempdir. - always_return_path (bool): If True, staging dir will be created as - tempdir if no staging dir profile is found. prefix (str): Prefix for staging dir. suffix (str): Suffix for staging dir. - formatting_data (Dict[str, Any]): Data for formatting staging dir - template. Returns: - Optional[Dict[str, Any]]: Staging dir data + Optional[Dict[str, Any]]: Staging dir info data """ - log = kwargs.get("log") or Logger.get_logger("get_staging_dir") - always_return_path = kwargs.get("always_return_path") + log = logger or Logger.get_logger("get_staging_dir_info") # make sure always_return_path is set to true by default if always_return_path is None: always_return_path = True - if kwargs.get("force_temp"): + if force_tmp_dir: return get_temp_dir( project_name=project_entity["name"], anatomy=anatomy, @@ -175,9 +181,9 @@ def get_staging_dir( "host": host_name, }) - # add additional data from kwargs - if kwargs.get("formatting_data"): - ctx_data.update(kwargs.get("formatting_data")) + # add additional template formatting data + if template_data: + ctx_data.update(template_data) # get staging dir config staging_dir_config = get_staging_dir_config( diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index 448e774e7c..440ed882aa 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -10,29 +10,25 @@ def get_temp_dir( - project_name=None, anatomy=None, prefix=None, suffix=None, make_local=False + project_name, anatomy=None, prefix=None, suffix=None, use_local_temp=False ): """Get temporary dir path. - If `make_local` is set, tempdir will be created in local tempdir. + If `use_local_temp` is set, tempdir will be created in local tempdir. If `anatomy` is not set, default anatomy will be used. If `prefix` or `suffix` is not set, default values will be used. - It also supports `OPENPYPE_TMPDIR`, so studio can define own temp + It also supports `AYON_TMPDIR`, so studio can define own temp shared repository per project or even per more granular context. Template formatting is supported also with optional keys. Folder is created in case it doesn't exists. - Note: - Staging dir does not have to be necessarily in tempdir so be careful - about its usage. - Args: project_name (str): Name of project. anatomy (Optional[Anatomy]): Project Anatomy object. suffix (Optional[str]): Suffix for tempdir. prefix (Optional[str]): Prefix for tempdir. - make_local (Optional[bool]): If True, temp dir will be created in + use_local_temp (Optional[bool]): If True, temp dir will be created in local tempdir. Returns: @@ -42,7 +38,7 @@ def get_temp_dir( prefix = prefix or "ay_tmp_" suffix = suffix or "" - if make_local: + if use_local_temp: return _create_local_staging_dir(prefix, suffix) # make sure anatomy is set @@ -55,19 +51,20 @@ def get_temp_dir( return _create_local_staging_dir(prefix, suffix, custom_temp_dir) -def _create_local_staging_dir(prefix, suffix, dir=None): +def _create_local_staging_dir(prefix, suffix, dirpath=None): """Create local staging dir Args: prefix (str): prefix for tempdir suffix (str): suffix for tempdir + dirpath (Optional[str]): path to tempdir Returns: str: path to tempdir """ # use pathlib for creating tempdir staging_dir = Path(tempfile.mkdtemp( - prefix=prefix, suffix=suffix, dir=dir + prefix=prefix, suffix=suffix, dir=dirpath )) return staging_dir.as_posix() @@ -89,31 +86,27 @@ def _create_custom_tempdir(project_name, anatomy=None): Returns: str | None: formatted path or None """ - env_tmpdir = os.getenv("AYON_TMPDIR") + env_tmpdir = os.getenv( + "AYON_TMPDIR", + ) if not env_tmpdir: - env_tmpdir = os.getenv("OPENPYPE_TMPDIR") - if not env_tmpdir: - return - print( - "DEPRECATION WARNING: Used 'OPENPYPE_TMPDIR' environment" - " variable. Please use 'AYON_TMPDIR' instead." - ) + return None custom_tempdir = None if "{" in env_tmpdir: if anatomy is None: anatomy = Anatomy(project_name) # create base formate data - template_formatting_data = { + template_data = { "root": anatomy.roots, "project": { "name": anatomy.project_name, "code": anatomy.project_code, - } + }, } # path is anatomy template custom_tempdir = StringTemplate.format_template( - env_tmpdir, template_formatting_data) + env_tmpdir, template_data) custom_tempdir_path = Path(custom_tempdir) From 1eb09045832f29024b0b2dba00adc6d71ace132f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 14:48:47 +0200 Subject: [PATCH 015/136] Remove unnecessary root addition in get_staging_dir_info function. The code changes remove the unnecessary addition of roots to ctx_data in the get_staging_dir_info function. --- client/ayon_core/pipeline/stagingdir.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index e9d425cf28..818acef36a 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -169,8 +169,6 @@ def get_staging_dir_info( ctx_data = get_template_data( project_entity, folder_entity, task_entity, host_name ) - # add roots to ctx_data - ctx_data["root"] = anatomy.roots # add additional data ctx_data.update({ @@ -178,7 +176,7 @@ def get_staging_dir_info( "type": product_type, "name": product_name }, - "host": host_name, + "root": anatomy.roots }) # add additional template formatting data From fa9af2f8ded7e9c89519cfde2f01e04a3dd8b58a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 15:06:11 +0200 Subject: [PATCH 016/136] Refactor Creator class method to handle missing info better. - Updated return type in docstring - Added a comment for clarity - Removed unnecessary return statements --- client/ayon_core/pipeline/create/creator_plugins.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 801feedd3d..4cbf432efd 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -843,14 +843,16 @@ def apply_staging_dir(self, instance): dir applied. Returns: - str: Path to staging dir. + Optional[str]: Staging dir path or None if not applied. """ create_ctx = self.create_context product_name = instance.get("productName") product_type = instance.get("productType") folder_path = instance.get("folderPath") + + # this can only work if product name and folder path are available if not product_name or not folder_path: - return None + return version = instance.get("version") if version is not None: @@ -871,7 +873,7 @@ def apply_staging_dir(self, instance): ) if not staging_dir_info: - return None + return staging_dir_path = staging_dir_info["stagingDir"] From 8a074daa2bb226de239d6d8291069c8796398086 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 4 Nov 2024 16:56:41 +0100 Subject: [PATCH 017/136] Fix single frame render --- client/ayon_core/pipeline/farm/pyblish_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 5908644dca..c70967dfc1 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -478,7 +478,8 @@ def _get_real_frames_to_render(frames): for frame in frames.split(","): if "-" in frame: splitted = frame.split("-") - frames_to_render.extend(range(int(splitted[0]), int(splitted[1]))) + frames_to_render.extend( + range(int(splitted[0]), int(splitted[1])+1)) else: frames_to_render.append(frame) return [str(frame_to_render) for frame_to_render in frames_to_render] From 87bb613b751ea508ae54a9813bf6eb5852ca5b6b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 7 Nov 2024 17:38:00 +0100 Subject: [PATCH 018/136] Added optionality to new argument in method signature --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index c70967dfc1..e9f179c668 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -304,7 +304,7 @@ def prepare_representations( do_not_add_review, context, color_managed_plugin, - frames_to_render + frames_to_render=None ): """Create representations for file sequences. From aaadaffabe5aeb033f3b1f7e0fe3341c69356564 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 11 Nov 2024 16:57:41 +0800 Subject: [PATCH 019/136] refactoring the load container so that it can load the library project --- client/ayon_core/pipeline/load/utils.py | 4 +- .../ayon_core/tools/sceneinventory/control.py | 7 +- .../ayon_core/tools/sceneinventory/model.py | 2 +- .../tools/sceneinventory/models/containers.py | 133 +++++++++++------- client/ayon_core/tools/sceneinventory/view.py | 9 +- 5 files changed, 98 insertions(+), 57 deletions(-) diff --git a/client/ayon_core/pipeline/load/utils.py b/client/ayon_core/pipeline/load/utils.py index ee2c1af07f..6f69651a8f 100644 --- a/client/ayon_core/pipeline/load/utils.py +++ b/client/ayon_core/pipeline/load/utils.py @@ -465,7 +465,7 @@ def update_container(container, version=-1): from ayon_core.pipeline import get_current_project_name # Compute the different version from 'representation' - project_name = get_current_project_name() + project_name = container.get("project_name", get_current_project_name()) repre_id = container["representation"] if not _is_valid_representation_id(repre_id): raise ValueError( @@ -588,7 +588,7 @@ def switch_container(container, representation, loader_plugin=None): ) # Get the new representation to switch to - project_name = get_current_project_name() + project_name = container.get("project_name", get_current_project_name()) context = get_representation_context( project_name, representation["id"] diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index b890462506..8ce3a1bb7a 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -4,7 +4,7 @@ from ayon_core.host import HostBase from ayon_core.pipeline import ( registered_host, - get_current_context, + get_current_context ) from ayon_core.tools.common_models import HierarchyModel, ProjectsModel @@ -110,8 +110,9 @@ def get_representation_info_items(self, representation_ids): representation_ids ) - def get_version_items(self, product_ids): - return self._containers_model.get_version_items(product_ids) + def get_version_items(self, product_ids, representation_ids): + return self._containers_model.get_version_items( + product_ids, representation_ids) # Site Sync methods def is_sitesync_enabled(self): diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index b7f79986ac..9d1202a906 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -150,7 +150,7 @@ def refresh(self, selected=None): if repre_info.is_valid } version_items_by_product_id = self._controller.get_version_items( - product_ids + product_ids, repre_id ) # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 871455c96b..693a5948c9 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -6,6 +6,7 @@ from ayon_core.host import ILoadHost from ayon_core.tools.common_models.projects import StatusStates +from ayon_core.pipeline.context_tools import get_current_project_name # --- Implementation that should be in ayon-python-api --- @@ -93,13 +94,15 @@ def __init__( loader_name, namespace, object_name, - item_id + item_id, + project_name ): self.representation_id = representation_id self.loader_name = loader_name self.object_name = object_name self.namespace = namespace self.item_id = item_id + self.project_name = project_name @classmethod def from_container_data(cls, container): @@ -109,6 +112,8 @@ def from_container_data(cls, container): namespace=container["namespace"], object_name=container["objectName"], item_id=uuid.uuid4().hex, + project_name=container.get( + "project_name", get_current_project_name()) ) @@ -222,6 +227,9 @@ def get_container_items_by_id(self, item_ids): def get_representation_info_items(self, representation_ids): output = {} missing_repre_ids = set() + missing_repre_ids_by_project = {} + containers = self._controller.get_containers() + for repre_id in representation_ids: try: uuid.UUID(repre_id) @@ -229,54 +237,60 @@ def get_representation_info_items(self, representation_ids): output[repre_id] = RepresentationInfo.new_invalid() continue + project_name = self._find_project_name(containers, repre_id) + if project_name is None: + project_name = self._controller.get_current_project_name() + repre_info = self._repre_info_by_id.get(repre_id) if repre_info is None: missing_repre_ids.add(repre_id) + missing_repre_ids_by_project.update({project_name: repre_id}) else: output[repre_id] = repre_info if not missing_repre_ids: return output - project_name = self._controller.get_current_project_name() - repre_hierarchy_by_id = get_representations_hierarchy( - project_name, missing_repre_ids - ) - for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): - kwargs = { - "folder_id": None, - "folder_path": None, - "product_id": None, - "product_name": None, - "product_type": None, - "product_group": None, - "version_id": None, - "representation_name": None, - } - folder = repre_hierarchy.folder - product = repre_hierarchy.product - version = repre_hierarchy.version - repre = repre_hierarchy.representation - if folder: - kwargs["folder_id"] = folder["id"] - kwargs["folder_path"] = folder["path"] - if product: - group = product["attrib"]["productGroup"] - kwargs["product_id"] = product["id"] - kwargs["product_name"] = product["name"] - kwargs["product_type"] = product["productType"] - kwargs["product_group"] = group - if version: - kwargs["version_id"] = version["id"] - if repre: - kwargs["representation_name"] = repre["name"] - - repre_info = RepresentationInfo(**kwargs) - self._repre_info_by_id[repre_id] = repre_info - output[repre_id] = repre_info + for project_name, missing_ids in missing_repre_ids_by_project.items(): + repre_hierarchy_by_id = get_representations_hierarchy( + project_name, {missing_ids} + ) + for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): + kwargs = { + "folder_id": None, + "folder_path": None, + "product_id": None, + "product_name": None, + "product_type": None, + "product_group": None, + "version_id": None, + "representation_name": None, + } + folder = repre_hierarchy.folder + product = repre_hierarchy.product + version = repre_hierarchy.version + repre = repre_hierarchy.representation + if folder: + kwargs["folder_id"] = folder["id"] + kwargs["folder_path"] = folder["path"] + if product: + group = product["attrib"]["productGroup"] + kwargs["product_id"] = product["id"] + kwargs["product_name"] = product["name"] + kwargs["product_type"] = product["productType"] + kwargs["product_group"] = group + if version: + kwargs["version_id"] = version["id"] + if repre: + kwargs["representation_name"] = repre["name"] + + repre_info = RepresentationInfo(**kwargs) + self._repre_info_by_id[repre_id] = repre_info + output[repre_id] = repre_info return output - def get_version_items(self, product_ids): + def get_version_items(self, product_ids, representation_ids): + project_ids_by_project_names = {} if not product_ids: return {} @@ -293,20 +307,37 @@ def get_version_items(self, product_ids): def version_sorted(entity): return entity["version"] + containers = self.get_containers() + for repre_id in representation_ids: + project_name = self._find_project_name(containers, repre_id) + if project_name is None: + project_name = self._controller.get_current_project_name() + repre_hierarchy_by_id = get_representations_hierarchy( + project_name, {repre_id} + ) + product_ids_list = set() + for repre_hierarchy in repre_hierarchy_by_id.values(): + product = repre_hierarchy.product + product_id = product["id"] + if product_id not in missing_ids: + continue + product_ids_list.add(product_id) + project_ids_by_project_names.update({project_name: product_ids_list}) - project_name = self._controller.get_current_project_name() version_entities_by_product_id = { product_id: [] for product_id in missing_ids } - - version_entities = list(ayon_api.get_versions( - project_name, - product_ids=missing_ids, - fields={"id", "version", "productId", "status"} - )) - version_entities.sort(key=version_sorted) - for version_entity in version_entities: + version_entities_list = [] + for project_name, missing_product_ids in project_ids_by_project_names.items(): + version_entities = list(ayon_api.get_versions( + project_name, + product_ids=missing_product_ids, + fields={"id", "version", "productId", "status"} + )) + version_entities_list.extend(version_entities) + version_entities_list.sort(key=version_sorted) + for version_entity in version_entities_list: product_id = version_entity["productId"] version_entities_by_product_id[product_id].append( version_entity @@ -337,12 +368,18 @@ def version_sorted(entity): self._version_items_by_product_id[product_id] = ( version_items_by_id ) - return { product_id: dict(self._version_items_by_product_id[product_id]) for product_id in product_ids } + def _find_project_name(self, containers, representation_id): + # Function to find the project name by representation + for container in containers: + if container.get('representation') == representation_id: + return container.get('project_name', get_current_project_name()) + return None + def _update_cache(self): if self._items_cache is not None: return diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 22ba15fda8..5fc2113824 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -228,7 +228,10 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): return version_items_by_product_id = self._controller.get_version_items( - product_ids + product_ids, { + container_item.representation_id + for container_item in container_items_by_id.values() + } ) has_outdated = False has_loaded_hero_versions = False @@ -751,7 +754,7 @@ def _show_version_dialog(self, item_ids, active_repre_id): active_version_id = active_repre_info.version_id active_product_id = active_repre_info.product_id version_items_by_product_id = self._controller.get_version_items( - product_ids + product_ids, repre_ids ) version_items = list( version_items_by_product_id[active_product_id].values() @@ -943,7 +946,7 @@ def _on_switch_to_versioned(self, item_ids): if repre_info.is_valid } version_items_by_product_id = self._controller.get_version_items( - product_ids + product_ids, repre_ids ) update_containers = [] From dcb838e1454523b3cbee00f5fde54fdc0d36fb58 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 11 Nov 2024 19:17:00 +0800 Subject: [PATCH 020/136] resolve the project root during updating container --- client/ayon_core/pipeline/load/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/load/utils.py b/client/ayon_core/pipeline/load/utils.py index 6f69651a8f..a6c5f0ce1f 100644 --- a/client/ayon_core/pipeline/load/utils.py +++ b/client/ayon_core/pipeline/load/utils.py @@ -542,9 +542,6 @@ def update_container(container, version=-1): ) ) - path = get_representation_path(new_representation) - if not path or not os.path.exists(path): - raise ValueError("Path {} doesn't exist".format(path)) project_entity = ayon_api.get_project(project_name) context = { "project": project_entity, @@ -553,6 +550,9 @@ def update_container(container, version=-1): "version": new_version, "representation": new_representation, } + path = get_representation_path_from_context(context) + if not path or not os.path.exists(path): + raise ValueError("Path {} doesn't exist".format(path)) return Loader().update(container, context) From 735409f9acb945c3af3cf61c9f2d35a2ce51de1e Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 12 Nov 2024 20:13:49 +0800 Subject: [PATCH 021/136] do not get the container item from self.get_container --- .../ayon_core/tools/sceneinventory/model.py | 5 +- .../tools/sceneinventory/models/containers.py | 75 ++++++++----------- client/ayon_core/tools/sceneinventory/view.py | 22 +++--- 3 files changed, 48 insertions(+), 54 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 3857ea1700..687d130f04 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -130,6 +130,7 @@ def refresh(self, selected=None): self._clear_items() items_by_repre_id = {} + project_names = set() for container_item in container_items: # if ( # selected is not None @@ -137,8 +138,10 @@ def refresh(self, selected=None): # ): # continue repre_id = container_item.representation_id + project_name = container_item.project_name items = items_by_repre_id.setdefault(repre_id, []) items.append(container_item) + project_names.add(project_name) repre_id = set(items_by_repre_id.keys()) repre_info_by_id = self._controller.get_representation_info_items( @@ -150,7 +153,7 @@ def refresh(self, selected=None): if repre_info.is_valid } version_items_by_product_id = self._controller.get_version_items( - product_ids, repre_id + product_ids, project_names ) # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 8a4beed52c..f5618d9f35 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -6,7 +6,6 @@ from ayon_core.host import ILoadHost from ayon_core.tools.common_models.projects import StatusStates -from ayon_core.pipeline.context_tools import get_current_project_name # --- Implementation that should be in ayon-python-api --- @@ -112,8 +111,7 @@ def from_container_data(cls, container): namespace=container["namespace"], object_name=container["objectName"], item_id=uuid.uuid4().hex, - project_name=container.get( - "project_name", get_current_project_name()) + project_name=container.get("project_name", None) ) @@ -194,15 +192,21 @@ def __init__(self, controller): self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} + self._container_items_by_project = {} + self._project_name_by_repre_id = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} + self._product_id_by_project = {} def reset(self): self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} + self._container_items_by_project = {} + self._project_name_by_repre_id = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} + self._product_id_by_project = {} def get_containers(self): self._update_cache() @@ -226,10 +230,8 @@ def get_container_items_by_id(self, item_ids): def get_representation_info_items(self, representation_ids): output = {} - missing_repre_ids = set() missing_repre_ids_by_project = {} - containers = self._controller.get_containers() - + current_project_name = self._controller.get_current_project_name() for repre_id in representation_ids: try: uuid.UUID(repre_id) @@ -237,23 +239,23 @@ def get_representation_info_items(self, representation_ids): output[repre_id] = RepresentationInfo.new_invalid() continue - project_name = self._find_project_name(containers, repre_id) + project_name = self._project_name_by_repre_id.get(repre_id) if project_name is None: - project_name = self._controller.get_current_project_name() - + project_name = current_project_name repre_info = self._repre_info_by_id.get(repre_id) if repre_info is None: - missing_repre_ids.add(repre_id) - missing_repre_ids_by_project.update({project_name: repre_id}) + missing_repre_ids_by_project.setdefault( + project_name, set() + ).add(repre_id) else: output[repre_id] = repre_info - if not missing_repre_ids: + if not missing_repre_ids_by_project: return output for project_name, missing_ids in missing_repre_ids_by_project.items(): repre_hierarchy_by_id = get_representations_hierarchy( - project_name, {missing_ids} + project_name, missing_ids ) for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): kwargs = { @@ -286,19 +288,23 @@ def get_representation_info_items(self, representation_ids): repre_info = RepresentationInfo(**kwargs) self._repre_info_by_id[repre_id] = repre_info + self._product_id_by_project[project_name] = repre_info.product_id output[repre_id] = repre_info return output - def get_version_items(self, product_ids, representation_ids): - project_ids_by_project_names = {} + def get_version_items(self, product_ids, project_names): if not product_ids: return {} - missing_ids = { product_id for product_id in product_ids if product_id not in self._version_items_by_product_id } + + product_ids_by_project = { + project_name: self._product_id_by_project.get(project_name) + for project_name in project_names + } if missing_ids: status_items_by_name = { status_item.name: status_item @@ -307,34 +313,20 @@ def get_version_items(self, product_ids, representation_ids): def version_sorted(entity): return entity["version"] - containers = self.get_containers() - for repre_id in representation_ids: - project_name = self._find_project_name(containers, repre_id) - if project_name is None: - project_name = self._controller.get_current_project_name() - repre_hierarchy_by_id = get_representations_hierarchy( - project_name, {repre_id} - ) - product_ids_list = set() - for repre_hierarchy in repre_hierarchy_by_id.values(): - product = repre_hierarchy.product - product_id = product["id"] - if product_id not in missing_ids: - continue - product_ids_list.add(product_id) - project_ids_by_project_names.update({project_name: product_ids_list}) - + version_entities_list = [] version_entities_by_product_id = { product_id: [] for product_id in missing_ids } - version_entities_list = [] - for project_name, missing_product_ids in project_ids_by_project_names.items(): + for project_name, product_id in product_ids_by_project.items(): + if product_id not in missing_ids: + continue version_entities = list(ayon_api.get_versions( project_name, - product_ids=missing_product_ids, + product_ids={product_id}, fields={"id", "version", "productId", "status"} )) + version_entities_list.extend(version_entities) version_entities_list.sort(key=version_sorted) for version_entity in version_entities_list: @@ -342,7 +334,6 @@ def version_sorted(entity): version_entities_by_product_id[product_id].append( version_entity ) - for product_id, version_entities in ( version_entities_by_product_id.items() ): @@ -373,13 +364,6 @@ def version_sorted(entity): for product_id in product_ids } - def _find_project_name(self, containers, representation_id): - # Function to find the project name by representation - for container in containers: - if container.get('representation') == representation_id: - return container.get('project_name', get_current_project_name()) - return None - def _update_cache(self): if self._items_cache is not None: return @@ -395,6 +379,7 @@ def _update_cache(self): container_items = [] containers_by_id = {} container_items_by_id = {} + project_name_by_repre_id = {} invalid_ids_mapping = {} for container in containers: try: @@ -418,8 +403,10 @@ def _update_cache(self): containers_by_id[item.item_id] = container container_items_by_id[item.item_id] = item + project_name_by_repre_id[item.representation_id] = item.project_name container_items.append(item) self._containers_by_id = containers_by_id self._container_items_by_id = container_items_by_id + self._project_name_by_repre_id = project_name_by_repre_id self._items_cache = container_items diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 5fc2113824..c5a25fa6dc 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -208,6 +208,7 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): filtered_items = [] product_ids = set() version_ids = set() + project_names = set() for container_item in container_items_by_id.values(): repre_id = container_item.representation_id repre_info = repre_info_by_id.get(repre_id) @@ -215,6 +216,7 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): filtered_items.append(container_item) version_ids.add(repre_info.version_id) product_ids.add(repre_info.product_id) + project_names.add(container_item.project_name) # remove remove_icon = qtawesome.icon("fa.remove", color=DEFAULT_COLOR) @@ -228,11 +230,7 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): return version_items_by_product_id = self._controller.get_version_items( - product_ids, { - container_item.representation_id - for container_item in container_items_by_id.values() - } - ) + product_ids, project_names) has_outdated = False has_loaded_hero_versions = False has_available_hero_version = False @@ -742,6 +740,10 @@ def _show_version_dialog(self, item_ids, active_repre_id): container_item.representation_id for container_item in container_items_by_id.values() } + project_names = { + container_item.project_name + for container_item in container_items_by_id.values() + } repre_info_by_id = self._controller.get_representation_info_items( repre_ids ) @@ -754,8 +756,7 @@ def _show_version_dialog(self, item_ids, active_repre_id): active_version_id = active_repre_info.version_id active_product_id = active_repre_info.product_id version_items_by_product_id = self._controller.get_version_items( - product_ids, repre_ids - ) + product_ids, project_names) version_items = list( version_items_by_product_id[active_product_id].values() ) @@ -937,6 +938,10 @@ def _on_switch_to_versioned(self, item_ids): container_item.representation_id for container_item in containers_items_by_id.values() } + project_names = { + container_item.project_name + for container_item in containers_items_by_id.values() + } repre_info_by_id = self._controller.get_representation_info_items( repre_ids ) @@ -946,8 +951,7 @@ def _on_switch_to_versioned(self, item_ids): if repre_info.is_valid } version_items_by_product_id = self._controller.get_version_items( - product_ids, repre_ids - ) + product_ids, project_names) update_containers = [] update_versions = [] From 0bec953dec2d786afb09a420e86f83980ddb19a1 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 14 Nov 2024 22:46:43 +0800 Subject: [PATCH 022/136] query the product id per project --- .../ayon_core/tools/sceneinventory/control.py | 4 +- .../ayon_core/tools/sceneinventory/model.py | 21 +++-- .../tools/sceneinventory/models/containers.py | 30 +++---- client/ayon_core/tools/sceneinventory/view.py | 86 ++++++++++++------- 4 files changed, 85 insertions(+), 56 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 8ce3a1bb7a..640911df80 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -110,9 +110,9 @@ def get_representation_info_items(self, representation_ids): representation_ids ) - def get_version_items(self, product_ids, representation_ids): + def get_version_items(self, project_name, product_ids): return self._containers_model.get_version_items( - product_ids, representation_ids) + project_name, product_ids) # Site Sync methods def is_sitesync_enabled(self): diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 687d130f04..162a0d4b71 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -152,9 +152,20 @@ def refresh(self, selected=None): for repre_info in repre_info_by_id.values() if repre_info.is_valid } - version_items_by_product_id = self._controller.get_version_items( - product_ids, project_names - ) + + project_products = {project_name: set() for project_name in project_names} + for representation_id, items in items_by_repre_id.items(): + repre_info = repre_info_by_id.get(representation_id) + if repre_info and repre_info.is_valid: + product_id = repre_info.product_id + for item in items: + project_name = item.project_name + project_products[project_name].add(product_id) + version_items_by_product_id = {} + for project_name, product_ids in project_products.items(): + version_items_by_product_id.update(self._controller.get_version_items( + project_name, product_ids + )) # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( repre_id @@ -236,7 +247,6 @@ def refresh(self, selected=None): for container_item in container_items: object_name = container_item.object_name or "" unique_name = repre_name + object_name - item = QtGui.QStandardItem() item.setColumnCount(root_item.columnCount()) item.setData(container_item.namespace, QtCore.Qt.DisplayRole) @@ -251,7 +261,6 @@ def refresh(self, selected=None): item.setData(True, IS_CONTAINER_ITEM_ROLE) item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) container_model_items.append(item) - if not container_model_items: continue @@ -290,7 +299,7 @@ def refresh(self, selected=None): group_item.setData(active_site_icon, ACTIVE_SITE_ICON_ROLE) group_item.setData(remote_site_icon, REMOTE_SITE_ICON_ROLE) group_item.setData(False, IS_CONTAINER_ITEM_ROLE) - + print(group_item) if version_color is not None: group_item.setData(version_color, VERSION_COLOR_ROLE) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index f5618d9f35..4592b489e1 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -249,7 +249,6 @@ def get_representation_info_items(self, representation_ids): ).add(repre_id) else: output[repre_id] = repre_info - if not missing_repre_ids_by_project: return output @@ -292,7 +291,7 @@ def get_representation_info_items(self, representation_ids): output[repre_id] = repre_info return output - def get_version_items(self, product_ids, project_names): + def get_version_items(self, project_name, product_ids): if not product_ids: return {} missing_ids = { @@ -301,10 +300,7 @@ def get_version_items(self, product_ids, project_names): if product_id not in self._version_items_by_product_id } - product_ids_by_project = { - project_name: self._product_id_by_project.get(project_name) - for project_name in project_names - } + current_product_id = self._product_id_by_project.get(project_name) if missing_ids: status_items_by_name = { status_item.name: status_item @@ -313,24 +309,22 @@ def get_version_items(self, product_ids, project_names): def version_sorted(entity): return entity["version"] - version_entities_list = [] + if current_product_id not in missing_ids: + return version_entities_by_product_id = { product_id: [] for product_id in missing_ids } - for project_name, product_id in product_ids_by_project.items(): + version_entities = list(ayon_api.get_versions( + project_name, + product_ids={current_product_id}, + fields={"id", "version", "productId", "status"} + )) + version_entities.sort(key=version_sorted) + for version_entity in version_entities: + product_id = version_entity["productId"] if product_id not in missing_ids: continue - version_entities = list(ayon_api.get_versions( - project_name, - product_ids={product_id}, - fields={"id", "version", "productId", "status"} - )) - - version_entities_list.extend(version_entities) - version_entities_list.sort(key=version_sorted) - for version_entity in version_entities_list: - product_id = version_entity["productId"] version_entities_by_product_id[product_id].append( version_entity ) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index c5a25fa6dc..12a7ab2285 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -206,18 +206,20 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): # Exclude items that are "NOT FOUND" since setting versions, updating # and removal won't work for those items. filtered_items = [] - product_ids = set() + project_products = {} version_ids = set() - project_names = set() for container_item in container_items_by_id.values(): repre_id = container_item.representation_id + project_name = container_item.project_name repre_info = repre_info_by_id.get(repre_id) if repre_info and repre_info.is_valid: filtered_items.append(container_item) version_ids.add(repre_info.version_id) - product_ids.add(repre_info.product_id) - project_names.add(container_item.project_name) - + product_id = repre_info.product_id + if project_name not in project_products: + project_products[project_name] = set() + project_products[project_name].add(product_id) + print("p_products", project_products) # remove remove_icon = qtawesome.icon("fa.remove", color=DEFAULT_COLOR) remove_action = QtWidgets.QAction(remove_icon, "Remove items", menu) @@ -228,9 +230,12 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): # Keep remove action for invalid items menu.addAction(remove_action) return - - version_items_by_product_id = self._controller.get_version_items( - product_ids, project_names) + version_items_by_product_id = {} + for project_name, product_ids in project_products.items(): + version_items_by_product_id.update( + self._controller.get_version_items( + project_name, product_ids) + ) has_outdated = False has_loaded_hero_versions = False has_available_hero_version = False @@ -736,14 +741,11 @@ def _show_version_dialog(self, item_ids, active_repre_id): container_items_by_id = self._controller.get_container_items_by_id( item_ids ) + print(container_items_by_id, "container") repre_ids = { container_item.representation_id for container_item in container_items_by_id.values() } - project_names = { - container_item.project_name - for container_item in container_items_by_id.values() - } repre_info_by_id = self._controller.get_representation_info_items( repre_ids ) @@ -752,11 +754,25 @@ def _show_version_dialog(self, item_ids, active_repre_id): repre_info.product_id for repre_info in repre_info_by_id.values() } + project_products = {} + for container_item in container_items_by_id.values(): + repre_id = container_item.representation_id + project_name = container_item.project_name + repre_info = repre_info_by_id.get(repre_id) + if repre_info and repre_info.is_valid: + if project_name not in project_products: + project_products[project_name] = set() + product_id = repre_info.product_id + project_products[project_name].add(product_id) + print("proj_product", project_products) active_repre_info = repre_info_by_id[active_repre_id] active_version_id = active_repre_info.version_id active_product_id = active_repre_info.product_id - version_items_by_product_id = self._controller.get_version_items( - product_ids, project_names) + version_items_by_product_id = {} + for project_name, product_ids in project_products.items(): + version_items_by_product_id.update( + self._controller.get_version_items( + project_name, product_ids)) version_items = list( version_items_by_product_id[active_product_id].values() ) @@ -931,27 +947,37 @@ def update_all(self): self._update_containers_to_version(item_ids, version=-1) def _on_switch_to_versioned(self, item_ids): - containers_items_by_id = self._controller.get_container_items_by_id( - item_ids - ) + # Get container items by ID + containers_items_by_id = self._controller.get_container_items_by_id(item_ids) repre_ids = { container_item.representation_id for container_item in containers_items_by_id.values() } - project_names = { - container_item.project_name - for container_item in containers_items_by_id.values() - } - repre_info_by_id = self._controller.get_representation_info_items( - repre_ids - ) - product_ids = { - repre_info.product_id - for repre_info in repre_info_by_id.values() - if repre_info.is_valid + # Extract project names and their corresponding representation IDs + project_name_to_repre_ids = {} + for container_item in containers_items_by_id.values(): + project_name = container_item.project_name + repre_id = container_item.representation_id + if project_name not in project_name_to_repre_ids: + project_name_to_repre_ids[project_name] = set() + project_name_to_repre_ids[project_name].add(repre_id) + + # Get representation info items by ID + repre_info_by_id = self._controller.get_representation_info_items(repre_ids) + + # Create a dictionary to map project names to sets of product IDs + project_products = { + project_name: set() for project_name in project_name_to_repre_ids.keys() } - version_items_by_product_id = self._controller.get_version_items( - product_ids, project_names) + + print("project_products", project_products) + version_items_by_product_id = {} + for project_name, product_ids in project_name_to_repre_ids.items(): + version_items_by_product_id.update( + self._controller.get_version_items( + project_name, product_ids + ) + ) update_containers = [] update_versions = [] From 574ea3580da0ad9531c77e9297660c35e5e20f5b Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Fri, 15 Nov 2024 21:58:59 +0800 Subject: [PATCH 023/136] codes clean up & make sure it supports mulitple asset loading per project --- .../ayon_core/tools/sceneinventory/control.py | 4 +- .../ayon_core/tools/sceneinventory/model.py | 45 +++--- .../tools/sceneinventory/models/containers.py | 34 ++--- client/ayon_core/tools/sceneinventory/view.py | 132 +++++++++++------- 4 files changed, 124 insertions(+), 91 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 640911df80..310e41b117 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -105,9 +105,9 @@ def get_container_items(self): def get_container_items_by_id(self, item_ids): return self._containers_model.get_container_items_by_id(item_ids) - def get_representation_info_items(self, representation_ids): + def get_representation_info_items(self, project_name, representation_ids): return self._containers_model.get_representation_info_items( - representation_ids + project_name, representation_ids ) def get_version_items(self, project_name, product_ids): diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 162a0d4b71..a37e3a2b40 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -129,43 +129,54 @@ def refresh(self, selected=None): self._clear_items() - items_by_repre_id = {} - project_names = set() + items_by_repre_id = collections.defaultdict(list) + repre_ids_by_project = collections.defaultdict(set) for container_item in container_items: # if ( # selected is not None # and container_item.item_id not in selected # ): # continue + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) repre_id = container_item.representation_id - project_name = container_item.project_name - items = items_by_repre_id.setdefault(repre_id, []) - items.append(container_item) - project_names.add(project_name) + items_by_repre_id[repre_id].append(container_item) + repre_ids_by_project[project_name].add(repre_id) repre_id = set(items_by_repre_id.keys()) - repre_info_by_id = self._controller.get_representation_info_items( - repre_id - ) + repre_info_by_id = {} + for project_name, repre_ids in repre_ids_by_project.items(): + repre_info = self._controller.get_representation_info_items( + project_name, repre_ids + ) + repre_info_by_id.update(repre_info) product_ids = { repre_info.product_id for repre_info in repre_info_by_id.values() if repre_info.is_valid } - project_products = {project_name: set() for project_name in project_names} - for representation_id, items in items_by_repre_id.items(): + project_products = collections.defaultdict(set) + for container_item in container_items: + representation_id = container_item.representation_id + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) repre_info = repre_info_by_id.get(representation_id) if repre_info and repre_info.is_valid: product_id = repre_info.product_id - for item in items: - project_name = item.project_name - project_products[project_name].add(product_id) + project_products[project_name].add(product_id) + version_items_by_product_id = {} for project_name, product_ids in project_products.items(): - version_items_by_product_id.update(self._controller.get_version_items( + version_items = self._controller.get_version_items( project_name, product_ids - )) + ) + version_items_by_product_id.update(version_items) + # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( repre_id @@ -299,7 +310,7 @@ def refresh(self, selected=None): group_item.setData(active_site_icon, ACTIVE_SITE_ICON_ROLE) group_item.setData(remote_site_icon, REMOTE_SITE_ICON_ROLE) group_item.setData(False, IS_CONTAINER_ITEM_ROLE) - print(group_item) + if version_color is not None: group_item.setData(version_color, VERSION_COLOR_ROLE) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 4592b489e1..52b568af03 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -193,20 +193,18 @@ def __init__(self, controller): self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} - self._project_name_by_repre_id = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} - self._product_id_by_project = {} + self._product_ids_by_project = {} def reset(self): self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} - self._project_name_by_repre_id = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} - self._product_id_by_project = {} + self._product_ids_by_project = {} def get_containers(self): self._update_cache() @@ -228,20 +226,17 @@ def get_container_items_by_id(self, item_ids): for item_id in item_ids } - def get_representation_info_items(self, representation_ids): + def get_representation_info_items(self, project_name, representation_ids): output = {} missing_repre_ids_by_project = {} - current_project_name = self._controller.get_current_project_name() + if project_name is None: + project_name = self._controller.get_current_project_name() for repre_id in representation_ids: try: uuid.UUID(repre_id) except ValueError: output[repre_id] = RepresentationInfo.new_invalid() continue - - project_name = self._project_name_by_repre_id.get(repre_id) - if project_name is None: - project_name = current_project_name repre_info = self._repre_info_by_id.get(repre_id) if repre_info is None: missing_repre_ids_by_project.setdefault( @@ -256,6 +251,7 @@ def get_representation_info_items(self, representation_ids): repre_hierarchy_by_id = get_representations_hierarchy( project_name, missing_ids ) + self._product_ids_by_project[project_name] = set() for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): kwargs = { "folder_id": None, @@ -287,20 +283,22 @@ def get_representation_info_items(self, representation_ids): repre_info = RepresentationInfo(**kwargs) self._repre_info_by_id[repre_id] = repre_info - self._product_id_by_project[project_name] = repre_info.product_id + self._product_ids_by_project[project_name].add( + repre_info.product_id) output[repre_id] = repre_info return output def get_version_items(self, project_name, product_ids): if not product_ids: return {} + if project_name is None: + project_name = self._controller.get_current_project_name() missing_ids = { product_id for product_id in product_ids if product_id not in self._version_items_by_product_id } - - current_product_id = self._product_id_by_project.get(project_name) + current_product_ids = self._product_ids_by_project.get(project_name) if missing_ids: status_items_by_name = { status_item.name: status_item @@ -309,22 +307,19 @@ def get_version_items(self, project_name, product_ids): def version_sorted(entity): return entity["version"] - if current_product_id not in missing_ids: - return + current_missing_ids = current_product_ids.intersection(missing_ids) version_entities_by_product_id = { product_id: [] - for product_id in missing_ids + for product_id in current_missing_ids } version_entities = list(ayon_api.get_versions( project_name, - product_ids={current_product_id}, + product_ids=current_missing_ids, fields={"id", "version", "productId", "status"} )) version_entities.sort(key=version_sorted) for version_entity in version_entities: product_id = version_entity["productId"] - if product_id not in missing_ids: - continue version_entities_by_product_id[product_id].append( version_entity ) @@ -402,5 +397,4 @@ def _update_cache(self): self._containers_by_id = containers_by_id self._container_items_by_id = container_items_by_id - self._project_name_by_repre_id = project_name_by_repre_id self._items_cache = container_items diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 12a7ab2285..1e0b96570e 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -192,11 +192,20 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): container_item = container_items_by_id[item_id] active_repre_id = container_item.representation_id break + repre_ids_by_project = collections.defaultdict(set) + for container_item in container_items_by_id.values(): + repre_id = container_item.representation_id + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) + repre_ids_by_project[project_name].add(repre_id) + repre_info_by_id = {} + for project_name, repre_ids in repre_ids_by_project.items(): + repre_info = self._controller.get_representation_info_items( + project_name, repre_ids) + repre_info_by_id.update(repre_info) - repre_info_by_id = self._controller.get_representation_info_items({ - container_item.representation_id - for container_item in container_items_by_id.values() - }) valid_repre_ids = { repre_id for repre_id, repre_info in repre_info_by_id.items() @@ -206,20 +215,20 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): # Exclude items that are "NOT FOUND" since setting versions, updating # and removal won't work for those items. filtered_items = [] - project_products = {} + product_ids_by_project = collections.defaultdict(set) version_ids = set() for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = container_item.project_name + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) repre_info = repre_info_by_id.get(repre_id) if repre_info and repre_info.is_valid: filtered_items.append(container_item) version_ids.add(repre_info.version_id) product_id = repre_info.product_id - if project_name not in project_products: - project_products[project_name] = set() - project_products[project_name].add(product_id) - print("p_products", project_products) + product_ids_by_project[project_name].add(product_id) # remove remove_icon = qtawesome.icon("fa.remove", color=DEFAULT_COLOR) remove_action = QtWidgets.QAction(remove_icon, "Remove items", menu) @@ -231,11 +240,12 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): menu.addAction(remove_action) return version_items_by_product_id = {} - for project_name, product_ids in project_products.items(): - version_items_by_product_id.update( - self._controller.get_version_items( - project_name, product_ids) + for project_name, product_ids in product_ids_by_project.items(): + version_items = self._controller.get_version_items( + project_name, product_ids ) + version_items_by_product_id.update(version_items +) has_outdated = False has_loaded_hero_versions = False has_available_hero_version = False @@ -741,38 +751,47 @@ def _show_version_dialog(self, item_ids, active_repre_id): container_items_by_id = self._controller.get_container_items_by_id( item_ids ) - print(container_items_by_id, "container") - repre_ids = { - container_item.representation_id - for container_item in container_items_by_id.values() - } - repre_info_by_id = self._controller.get_representation_info_items( - repre_ids - ) + repre_ids_by_project = collections.defaultdict(set) + for container_item in container_items_by_id.values(): + repre_id = container_item.representation_id + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) + repre_ids_by_project[project_name].add(repre_id) + repre_info_by_id = {} + for project_name, repre_ids in repre_ids_by_project.items(): + repre_info = self._controller.get_representation_info_items( + project_name, repre_ids + ) + repre_info_by_id.update(repre_info) product_ids = { repre_info.product_id for repre_info in repre_info_by_id.values() } - project_products = {} + product_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = container_item.project_name + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) repre_info = repre_info_by_id.get(repre_id) - if repre_info and repre_info.is_valid: - if project_name not in project_products: - project_products[project_name] = set() - product_id = repre_info.product_id - project_products[project_name].add(product_id) - print("proj_product", project_products) + if not repre_info or not repre_info.is_valid: + continue + product_ids_by_project[project_name].add( + repre_info.product_id + ) active_repre_info = repre_info_by_id[active_repre_id] active_version_id = active_repre_info.version_id active_product_id = active_repre_info.product_id version_items_by_product_id = {} - for project_name, product_ids in project_products.items(): - version_items_by_product_id.update( - self._controller.get_version_items( - project_name, product_ids)) + for project_name, project_product_ids in product_ids_by_project.items(): + version_items = self._controller.get_version_items( + project_name, project_product_ids + ) + version_items_by_product_id.update(version_items) version_items = list( version_items_by_product_id[active_product_id].values() ) @@ -949,35 +968,44 @@ def update_all(self): def _on_switch_to_versioned(self, item_ids): # Get container items by ID containers_items_by_id = self._controller.get_container_items_by_id(item_ids) - repre_ids = { - container_item.representation_id - for container_item in containers_items_by_id.values() - } # Extract project names and their corresponding representation IDs - project_name_to_repre_ids = {} + repre_ids_by_project = collections.defaultdict(set) for container_item in containers_items_by_id.values(): project_name = container_item.project_name + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) repre_id = container_item.representation_id - if project_name not in project_name_to_repre_ids: - project_name_to_repre_ids[project_name] = set() - project_name_to_repre_ids[project_name].add(repre_id) + repre_ids_by_project[project_name].add(repre_id) # Get representation info items by ID - repre_info_by_id = self._controller.get_representation_info_items(repre_ids) + repre_info_by_id = {} + for project_name, repre_ids in repre_ids_by_project.items(): + repre_info = self._controller.get_representation_info_items( + project_name, repre_ids) + repre_info_by_id.update(repre_info) - # Create a dictionary to map project names to sets of product IDs - project_products = { - project_name: set() for project_name in project_name_to_repre_ids.keys() - } + product_ids_by_project = collections.defaultdict(set) + for container_item in containers_items_by_id.values(): + repre_id = container_item.representation_id + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) + repre_info = repre_info_by_id.get(repre_id) + if not repre_info or not repre_info.is_valid: + continue + product_ids_by_project[project_name].add( + repre_info.product_id + ) - print("project_products", project_products) version_items_by_product_id = {} - for project_name, product_ids in project_name_to_repre_ids.items(): - version_items_by_product_id.update( - self._controller.get_version_items( + for project_name, product_ids in product_ids_by_project.items(): + version_items = self._controller.get_version_items( project_name, product_ids - ) ) + version_items_by_product_id.update(version_items) update_containers = [] update_versions = [] From 8965a8859435473c0171053cc2908dd011c05ad8 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 18 Nov 2024 17:59:20 +0800 Subject: [PATCH 024/136] clean up code and add project name row into scene inventory --- .../ayon_core/tools/sceneinventory/model.py | 8 +- .../tools/sceneinventory/models/containers.py | 89 +++++++++---------- 2 files changed, 48 insertions(+), 49 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index a37e3a2b40..03627e60b9 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -36,6 +36,7 @@ # This value hold unique value of container that should be used to identify # containers inbetween refresh. ITEM_UNIQUE_NAME_ROLE = QtCore.Qt.UserRole + 24 +PROJECT_NAME_ROLE = QtCore.Qt.UserRole + 25 class InventoryModel(QtGui.QStandardItemModel): @@ -52,6 +53,7 @@ class InventoryModel(QtGui.QStandardItemModel): "Object name", "Active site", "Remote site", + "Project" ] name_col = column_labels.index("Name") version_col = column_labels.index("Version") @@ -63,6 +65,7 @@ class InventoryModel(QtGui.QStandardItemModel): object_name_col = column_labels.index("Object name") active_site_col = column_labels.index("Active site") remote_site_col = column_labels.index("Remote site") + project_col = column_labels.index("Project") display_role_by_column = { name_col: QtCore.Qt.DisplayRole, version_col: VERSION_LABEL_ROLE, @@ -72,6 +75,7 @@ class InventoryModel(QtGui.QStandardItemModel): product_group_col: PRODUCT_GROUP_NAME_ROLE, loader_col: LOADER_NAME_ROLE, object_name_col: OBJECT_NAME_ROLE, + project_col: PROJECT_NAME_ROLE, active_site_col: ACTIVE_SITE_PROGRESS_ROLE, remote_site_col: REMOTE_SITE_PROGRESS_ROLE, } @@ -85,7 +89,7 @@ class InventoryModel(QtGui.QStandardItemModel): foreground_role_by_column = { name_col: NAME_COLOR_ROLE, version_col: VERSION_COLOR_ROLE, - status_col: STATUS_COLOR_ROLE + status_col: STATUS_COLOR_ROLE, } width_by_column = { name_col: 250, @@ -95,6 +99,7 @@ class InventoryModel(QtGui.QStandardItemModel): product_type_col: 150, product_group_col: 120, loader_col: 150, + project_col: 150, } OUTDATED_COLOR = QtGui.QColor(235, 30, 30) @@ -269,6 +274,7 @@ def refresh(self, selected=None): item.setData(version_label, VERSION_LABEL_ROLE) item.setData(container_item.loader_name, LOADER_NAME_ROLE) item.setData(container_item.object_name, OBJECT_NAME_ROLE) + item.setData(container_item.project_name, PROJECT_NAME_ROLE) item.setData(True, IS_CONTAINER_ITEM_ROLE) item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) container_model_items.append(item) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 52b568af03..b8b9aa400a 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -228,9 +228,7 @@ def get_container_items_by_id(self, item_ids): def get_representation_info_items(self, project_name, representation_ids): output = {} - missing_repre_ids_by_project = {} - if project_name is None: - project_name = self._controller.get_current_project_name() + missing_repre_ids = set() for repre_id in representation_ids: try: uuid.UUID(repre_id) @@ -239,60 +237,55 @@ def get_representation_info_items(self, project_name, representation_ids): continue repre_info = self._repre_info_by_id.get(repre_id) if repre_info is None: - missing_repre_ids_by_project.setdefault( - project_name, set() - ).add(repre_id) + missing_repre_ids.add(repre_id) else: output[repre_id] = repre_info - if not missing_repre_ids_by_project: + if not missing_repre_ids: return output - for project_name, missing_ids in missing_repre_ids_by_project.items(): - repre_hierarchy_by_id = get_representations_hierarchy( - project_name, missing_ids - ) - self._product_ids_by_project[project_name] = set() - for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): - kwargs = { - "folder_id": None, - "folder_path": None, - "product_id": None, - "product_name": None, - "product_type": None, - "product_group": None, - "version_id": None, - "representation_name": None, - } - folder = repre_hierarchy.folder - product = repre_hierarchy.product - version = repre_hierarchy.version - repre = repre_hierarchy.representation - if folder: - kwargs["folder_id"] = folder["id"] - kwargs["folder_path"] = folder["path"] - if product: - group = product["attrib"]["productGroup"] - kwargs["product_id"] = product["id"] - kwargs["product_name"] = product["name"] - kwargs["product_type"] = product["productType"] - kwargs["product_group"] = group - if version: - kwargs["version_id"] = version["id"] - if repre: - kwargs["representation_name"] = repre["name"] - - repre_info = RepresentationInfo(**kwargs) - self._repre_info_by_id[repre_id] = repre_info - self._product_ids_by_project[project_name].add( - repre_info.product_id) - output[repre_id] = repre_info + repre_hierarchy_by_id = get_representations_hierarchy( + project_name, missing_repre_ids + ) + self._product_ids_by_project[project_name] = set() + for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): + kwargs = { + "folder_id": None, + "folder_path": None, + "product_id": None, + "product_name": None, + "product_type": None, + "product_group": None, + "version_id": None, + "representation_name": None, + } + folder = repre_hierarchy.folder + product = repre_hierarchy.product + version = repre_hierarchy.version + repre = repre_hierarchy.representation + if folder: + kwargs["folder_id"] = folder["id"] + kwargs["folder_path"] = folder["path"] + if product: + group = product["attrib"]["productGroup"] + kwargs["product_id"] = product["id"] + kwargs["product_name"] = product["name"] + kwargs["product_type"] = product["productType"] + kwargs["product_group"] = group + if version: + kwargs["version_id"] = version["id"] + if repre: + kwargs["representation_name"] = repre["name"] + + repre_info = RepresentationInfo(**kwargs) + self._repre_info_by_id[repre_id] = repre_info + self._product_ids_by_project[project_name].add( + repre_info.product_id) + output[repre_id] = repre_info return output def get_version_items(self, project_name, product_ids): if not product_ids: return {} - if project_name is None: - project_name = self._controller.get_current_project_name() missing_ids = { product_id for product_id in product_ids From 0866c002019834e6a4aeda22becb9e5c208f4515 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 18 Nov 2024 19:18:50 +0800 Subject: [PATCH 025/136] code tweaks - kuba's comment --- .../ayon_core/tools/sceneinventory/model.py | 12 ++------ .../tools/sceneinventory/models/containers.py | 5 +++- client/ayon_core/tools/sceneinventory/view.py | 29 ++++--------------- 3 files changed, 12 insertions(+), 34 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 03627e60b9..5f9e6bb77c 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -53,7 +53,7 @@ class InventoryModel(QtGui.QStandardItemModel): "Object name", "Active site", "Remote site", - "Project" + "Project", ] name_col = column_labels.index("Name") version_col = column_labels.index("Version") @@ -142,10 +142,7 @@ def refresh(self, selected=None): # and container_item.item_id not in selected # ): # continue - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_id = container_item.representation_id items_by_repre_id[repre_id].append(container_item) repre_ids_by_project[project_name].add(repre_id) @@ -166,10 +163,7 @@ def refresh(self, selected=None): project_products = collections.defaultdict(set) for container_item in container_items: representation_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_info = repre_info_by_id.get(representation_id) if repre_info and repre_info.is_valid: product_id = repre_info.product_id diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index b8b9aa400a..c12a05fd99 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -5,6 +5,7 @@ from ayon_api.graphql import GraphQlQuery from ayon_core.host import ILoadHost +from ayon_core.pipeline import get_current_project_name from ayon_core.tools.common_models.projects import StatusStates @@ -111,7 +112,9 @@ def from_container_data(cls, container): namespace=container["namespace"], object_name=container["objectName"], item_id=uuid.uuid4().hex, - project_name=container.get("project_name", None) + project_name=container.get( + "project_name", get_current_project_name() + ) ) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 1e0b96570e..a049fd1e0b 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -195,10 +195,7 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): repre_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) repre_info_by_id = {} for project_name, repre_ids in repre_ids_by_project.items(): @@ -219,10 +216,7 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): version_ids = set() for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_info = repre_info_by_id.get(repre_id) if repre_info and repre_info.is_valid: filtered_items.append(container_item) @@ -754,10 +748,7 @@ def _show_version_dialog(self, item_ids, active_repre_id): repre_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) repre_info_by_id = {} for project_name, repre_ids in repre_ids_by_project.items(): @@ -773,10 +764,7 @@ def _show_version_dialog(self, item_ids, active_repre_id): product_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_info = repre_info_by_id.get(repre_id) if not repre_info or not repre_info.is_valid: continue @@ -972,10 +960,6 @@ def _on_switch_to_versioned(self, item_ids): repre_ids_by_project = collections.defaultdict(set) for container_item in containers_items_by_id.values(): project_name = container_item.project_name - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) repre_id = container_item.representation_id repre_ids_by_project[project_name].add(repre_id) @@ -989,10 +973,7 @@ def _on_switch_to_versioned(self, item_ids): product_ids_by_project = collections.defaultdict(set) for container_item in containers_items_by_id.values(): repre_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_info = repre_info_by_id.get(repre_id) if not repre_info or not repre_info.is_valid: continue From 004e9626ee5c9e3899bf72fee0af4c88e2b75b8a Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 19 Nov 2024 00:19:48 +0800 Subject: [PATCH 026/136] big roy comment - refactoring the dict per repre_id per project --- client/ayon_core/tools/sceneinventory/model.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 5f9e6bb77c..7630e9ee45 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -135,7 +135,8 @@ def refresh(self, selected=None): self._clear_items() items_by_repre_id = collections.defaultdict(list) - repre_ids_by_project = collections.defaultdict(set) + item_by_repre_id_by_project_id = collections.defaultdict( + lambda: collections.defaultdict(set)) for container_item in container_items: # if ( # selected is not None @@ -145,20 +146,17 @@ def refresh(self, selected=None): project_name = container_item.project_name repre_id = container_item.representation_id items_by_repre_id[repre_id].append(container_item) - repre_ids_by_project[project_name].add(repre_id) + item_by_repre_id_by_project_id[project_name][repre_id].add(container_item) - repre_id = set(items_by_repre_id.keys()) repre_info_by_id = {} - for project_name, repre_ids in repre_ids_by_project.items(): + repre_id = set() + for project_name, repre_ids in item_by_repre_id_by_project_id.items(): + repre_ids = set(items_by_repre_id.keys()) repre_info = self._controller.get_representation_info_items( project_name, repre_ids ) repre_info_by_id.update(repre_info) - product_ids = { - repre_info.product_id - for repre_info in repre_info_by_id.values() - if repre_info.is_valid - } + repre_id.update(repre_ids) project_products = collections.defaultdict(set) for container_item in container_items: From f003d8af1d654460ab802a7b1a604f092a6d163d Mon Sep 17 00:00:00 2001 From: Robin De Lillo Date: Mon, 18 Nov 2024 15:29:40 -0500 Subject: [PATCH 027/136] Apply suggestions from code review Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/publish/lib.py | 28 +++++++++--------------- client/ayon_core/pipeline/stagingdir.py | 15 ++++++++----- client/ayon_core/pipeline/tempdir.py | 4 +--- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 67a65aec09..3fad15f1a2 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -687,33 +687,25 @@ def get_instance_staging_dir(instance): anatomy_data = instance.data["anatomyData"] template_data = copy.deepcopy(anatomy_data) - product_type = instance.data["productType"] - product_name = instance.data["productName"] - # context data based variables - project_entity = instance.context.data["projectEntity"] - folder_entity = instance.context.data["folderEntity"] - task_entity = instance.context.data["taskEntity"] - host_name = instance.context.data["hostName"] - project_settings = instance.context.data["project_settings"] - anatomy = instance.context.data["anatomy"] - current_file = instance.context.data.get("currentFile") + context = instance.context # add current file as workfile name into formatting data + current_file = context.data.get("currentFile") if current_file: workfile = os.path.basename(current_file) workfile_name, _ = os.path.splitext(workfile) template_data["workfile_name"] = workfile_name staging_dir_info = get_staging_dir_info( - host_name, - project_entity, - folder_entity, - task_entity, - product_type, - product_name, - anatomy, - project_settings=project_settings, + context.data["hostName"], + context.data["projectEntity"], + instance.data.get("folderEntity"), + instance.data.get("taskEntity"), + instance.data["productType"], + instance.data["productName"], + anatomy=context.data["anatomy"], + project_settings=context.data["project_settings"], template_data=template_data, ) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index 818acef36a..1c658ac817 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -9,12 +9,12 @@ def get_staging_dir_config( - host_name, project_name, task_type, task_name, product_type, product_name, + host_name, project_settings=None, anatomy=None, log=None, @@ -24,8 +24,8 @@ def get_staging_dir_config( Args: host_name (str): Name of host. project_name (str): Name of project. - task_type (str): Type of task. - task_name (str): Name of task. + task_type (Optional[str]): Type of task. + task_name (Optional[str]): Name of task. product_type (str): Type of product. product_name (str): Name of product. project_settings(Dict[str, Any]): Prepared project settings. @@ -103,13 +103,13 @@ def _validate_template_name(project_name, template_name, anatomy): def get_staging_dir_info( - host_name, project_entity, folder_entity, task_entity, product_type, product_name, - anatomy, + host_name, + anatomy=None, project_settings=None, template_data=None, always_return_path=None, @@ -157,6 +157,11 @@ def get_staging_dir_info( if always_return_path is None: always_return_path = True + if anatomy is None: + anatomy = Anatomy( + project_entity["name"], project_entity=project_entity + ) + if force_tmp_dir: return get_temp_dir( project_name=project_entity["name"], diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index 8a9334ecc2..b5f4a31ee7 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -86,9 +86,7 @@ def _create_custom_tempdir(project_name, anatomy=None): Returns: str | None: formatted path or None """ - env_tmpdir = os.getenv( - "AYON_TMPDIR", - ) + env_tmpdir = os.getenv("AYON_TMPDIR") if not env_tmpdir: return From e96133b34892bf0529f0bcc81e56822d5177ffbc Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 18 Nov 2024 16:34:31 -0500 Subject: [PATCH 028/136] Address feedback from PR. --- .../pipeline/create/creator_plugins.py | 2 +- client/ayon_core/pipeline/publish/lib.py | 1 + client/ayon_core/pipeline/stagingdir.py | 54 +++++++++---------- client/ayon_core/pipeline/tempdir.py | 11 ++-- 4 files changed, 31 insertions(+), 37 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 4cbf432efd..667f70c27d 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -873,7 +873,7 @@ def apply_staging_dir(self, instance): ) if not staging_dir_info: - return + return None staging_dir_path = staging_dir_info["stagingDir"] diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 3fad15f1a2..8d56deec04 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -707,6 +707,7 @@ def get_instance_staging_dir(instance): anatomy=context.data["anatomy"], project_settings=context.data["project_settings"], template_data=template_data, + always_return_path=True, ) staging_dir_path = staging_dir_info["stagingDir"] diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index 1c658ac817..07ef122337 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -77,7 +77,7 @@ def get_staging_dir_config( # template should always be found either from anatomy or from profile raise ValueError( "Staging dir profile is misconfigured! " - "No template was found for profile! " + f"No template was found for profile: {profile}! " "Check your project settings at: " "'ayon+settings://core/tools/publish/custom_staging_dir_profiles'" ) @@ -112,10 +112,11 @@ def get_staging_dir_info( anatomy=None, project_settings=None, template_data=None, - always_return_path=None, - force_tmp_dir=None, + always_return_path=True, + force_tmp_dir=False, logger=None, - **kwargs + prefix=None, + suffix=None, ): """Get staging dir info data. @@ -141,11 +142,8 @@ def get_staging_dir_info( force_tmp_dir (Optional[bool]): If True, staging dir will be created as tempdir. logger (Optional[logging.Logger]): Logger instance. - **kwargs: Arbitrary keyword arguments. See below. - - Keyword Arguments: - prefix (str): Prefix for staging dir. - suffix (str): Suffix for staging dir. + prefix (Optional[str]) Optional prefix for staging dir name. + suffix (Optional[str]): Optional suffix for staging dir name. Returns: Optional[Dict[str, Any]]: Staging dir info data @@ -153,10 +151,6 @@ def get_staging_dir_info( """ log = logger or Logger.get_logger("get_staging_dir_info") - # make sure always_return_path is set to true by default - if always_return_path is None: - always_return_path = True - if anatomy is None: anatomy = Anatomy( project_entity["name"], project_entity=project_entity @@ -166,11 +160,11 @@ def get_staging_dir_info( return get_temp_dir( project_name=project_entity["name"], anatomy=anatomy, - prefix=kwargs.get("prefix"), - suffix=kwargs.get("suffix"), + prefix=prefix, + suffix=suffix, ) - # making fewer queries to database + # making few queries to database ctx_data = get_template_data( project_entity, folder_entity, task_entity, host_name ) @@ -192,8 +186,8 @@ def get_staging_dir_info( staging_dir_config = get_staging_dir_config( host_name, project_entity["name"], - task_entity["type"], - task_entity["name"], + task_entity.get("type"), + task_entity.get("name"), product_type, product_name, project_settings=project_settings, @@ -201,19 +195,19 @@ def get_staging_dir_info( log=log, ) - # if no preset matching and always_get_some_dir is set, return tempdir - if not staging_dir_config and always_return_path: - return { - "stagingDir": get_temp_dir( - project_name=project_entity["name"], - anatomy=anatomy, - prefix=kwargs.get("prefix"), - suffix=kwargs.get("suffix"), - ), - "stagingDir_persistent": False, - } if not staging_dir_config: - return None + if always_return_path: # no config found but force an output + return { + "stagingDir": get_temp_dir( + project_name=project_entity["name"], + anatomy=anatomy, + prefix=kwargs.get("prefix"), + suffix=kwargs.get("suffix"), + ), + "stagingDir_persistent": False, + } + else: + return None return { "stagingDir": StringTemplate.format_template( diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index b5f4a31ee7..af2ff44a8f 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -5,6 +5,7 @@ import os import tempfile from pathlib import Path + from ayon_core.lib import StringTemplate from ayon_core.pipeline import Anatomy @@ -48,7 +49,7 @@ def get_temp_dir( # get customized tempdir path from `OPENPYPE_TMPDIR` env var custom_temp_dir = _create_custom_tempdir(anatomy.project_name, anatomy) - return _create_local_staging_dir(prefix, suffix, custom_temp_dir) + return _create_local_staging_dir(prefix, suffix, dirpath=custom_temp_dir) def _create_local_staging_dir(prefix, suffix, dirpath=None): @@ -70,7 +71,7 @@ def _create_local_staging_dir(prefix, suffix, dirpath=None): return staging_dir.as_posix() -def _create_custom_tempdir(project_name, anatomy=None): +def _create_custom_tempdir(project_name, anatomy): """ Create custom tempdir Template path formatting is supporting: @@ -81,19 +82,17 @@ def _create_custom_tempdir(project_name, anatomy=None): Args: project_name (str): project name - anatomy (ayon_core.pipeline.Anatomy)[optional]: Anatomy object + anatomy (ayon_core.pipeline.Anatomy): Anatomy object Returns: str | None: formatted path or None """ env_tmpdir = os.getenv("AYON_TMPDIR") if not env_tmpdir: - return + return None custom_tempdir = None if "{" in env_tmpdir: - if anatomy is None: - anatomy = Anatomy(project_name) # create base formate data template_data = { "root": anatomy.roots, From 7375538f22d84b8a4493fc84f77cfd4d12cd8fbc Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 18 Nov 2024 16:43:31 -0500 Subject: [PATCH 029/136] Fix lint. --- client/ayon_core/pipeline/stagingdir.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index 07ef122337..2dd5c2f3eb 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -201,8 +201,8 @@ def get_staging_dir_info( "stagingDir": get_temp_dir( project_name=project_entity["name"], anatomy=anatomy, - prefix=kwargs.get("prefix"), - suffix=kwargs.get("suffix"), + prefix=prefix, + suffix=suffix, ), "stagingDir_persistent": False, } From 3078ba2a239c9780bf784d07849d42c2e02ea0c5 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 19 Nov 2024 15:45:27 +0800 Subject: [PATCH 030/136] kuba's comment - add current project name as argument in the get_container_data and make sure the scene inventory won't show Entity N/A as name and version --- client/ayon_core/tools/sceneinventory/control.py | 2 +- client/ayon_core/tools/sceneinventory/model.py | 16 +++++++++------- .../tools/sceneinventory/models/containers.py | 8 ++++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 310e41b117..8c02881b82 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -4,7 +4,7 @@ from ayon_core.host import HostBase from ayon_core.pipeline import ( registered_host, - get_current_context + get_current_context, ) from ayon_core.tools.common_models import HierarchyModel, ProjectsModel diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 7630e9ee45..5f9e6bb77c 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -135,8 +135,7 @@ def refresh(self, selected=None): self._clear_items() items_by_repre_id = collections.defaultdict(list) - item_by_repre_id_by_project_id = collections.defaultdict( - lambda: collections.defaultdict(set)) + repre_ids_by_project = collections.defaultdict(set) for container_item in container_items: # if ( # selected is not None @@ -146,17 +145,20 @@ def refresh(self, selected=None): project_name = container_item.project_name repre_id = container_item.representation_id items_by_repre_id[repre_id].append(container_item) - item_by_repre_id_by_project_id[project_name][repre_id].add(container_item) + repre_ids_by_project[project_name].add(repre_id) + repre_id = set(items_by_repre_id.keys()) repre_info_by_id = {} - repre_id = set() - for project_name, repre_ids in item_by_repre_id_by_project_id.items(): - repre_ids = set(items_by_repre_id.keys()) + for project_name, repre_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( project_name, repre_ids ) repre_info_by_id.update(repre_info) - repre_id.update(repre_ids) + product_ids = { + repre_info.product_id + for repre_info in repre_info_by_id.values() + if repre_info.is_valid + } project_products = collections.defaultdict(set) for container_item in container_items: diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index c12a05fd99..e135cb0031 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -5,7 +5,6 @@ from ayon_api.graphql import GraphQlQuery from ayon_core.host import ILoadHost -from ayon_core.pipeline import get_current_project_name from ayon_core.tools.common_models.projects import StatusStates @@ -105,7 +104,7 @@ def __init__( self.project_name = project_name @classmethod - def from_container_data(cls, container): + def from_container_data(cls, current_project_name, container): return cls( representation_id=container["representation"], loader_name=container["loader"], @@ -113,7 +112,7 @@ def from_container_data(cls, container): object_name=container["objectName"], item_id=uuid.uuid4().hex, project_name=container.get( - "project_name", get_current_project_name() + "project_name", current_project_name ) ) @@ -368,7 +367,8 @@ def _update_cache(self): invalid_ids_mapping = {} for container in containers: try: - item = ContainerItem.from_container_data(container) + current_project_name = self._controller.get_current_project_name() + item = ContainerItem.from_container_data(current_project_name, container) repre_id = item.representation_id try: uuid.UUID(repre_id) From b8ba7f47b0dee929fd1259e49fc791fa13ec3ed3 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 19 Nov 2024 16:49:08 -0500 Subject: [PATCH 031/136] Fix staging data computation. --- client/ayon_core/pipeline/create/creator_plugins.py | 8 +++++--- client/ayon_core/pipeline/publish/lib.py | 4 ++-- client/ayon_core/pipeline/stagingdir.py | 6 +++--- .../plugins/publish/collect_managed_staging_dir.py | 4 ++++ client/ayon_core/plugins/publish/extract_burnin.py | 2 +- .../ayon_core/plugins/publish/extract_color_transcode.py | 2 +- client/ayon_core/plugins/publish/extract_review.py | 2 +- 7 files changed, 17 insertions(+), 11 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 667f70c27d..93e1f6f5cb 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -857,16 +857,18 @@ def apply_staging_dir(self, instance): version = instance.get("version") if version is not None: template_data = {"version": version} + else: + template_data = {} staging_dir_info = get_staging_dir_info( - create_ctx.host_name, create_ctx.get_current_project_entity(), create_ctx.get_current_folder_entity(), create_ctx.get_current_task_entity(), product_type, product_name, - create_ctx.get_current_project_anatomy(), - create_ctx.get_current_project_settings(), + create_ctx.host_name, + anatomy=create_ctx.get_current_project_anatomy(), + project_settings=create_ctx.get_current_project_settings(), always_return_path=False, logger=self.log, template_data=template_data, diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 8d56deec04..4c36f473d1 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -653,12 +653,12 @@ def get_custom_staging_dir_info( DeprecationWarning, ) tr_data = get_staging_dir_config( - host_name, project_name, task_type, task_name, product_type, product_name, + host_name, project_settings=project_settings, anatomy=anatomy, log=log, @@ -698,12 +698,12 @@ def get_instance_staging_dir(instance): template_data["workfile_name"] = workfile_name staging_dir_info = get_staging_dir_info( - context.data["hostName"], context.data["projectEntity"], instance.data.get("folderEntity"), instance.data.get("taskEntity"), instance.data["productType"], instance.data["productName"], + context.data["hostName"], anatomy=context.data["anatomy"], project_settings=context.data["project_settings"], template_data=template_data, diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index 2dd5c2f3eb..c7cc95ff55 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -184,12 +184,12 @@ def get_staging_dir_info( # get staging dir config staging_dir_config = get_staging_dir_config( - host_name, project_entity["name"], - task_entity.get("type"), + task_entity.get("taskType"), task_entity.get("name"), product_type, product_name, + host_name, project_settings=project_settings, anatomy=anatomy, log=log, @@ -211,7 +211,7 @@ def get_staging_dir_info( return { "stagingDir": StringTemplate.format_template( - staging_dir_config["template"], ctx_data + staging_dir_config["template"]["directory"], ctx_data ), "stagingDir_persistent": staging_dir_config["persistence"], } diff --git a/client/ayon_core/plugins/publish/collect_managed_staging_dir.py b/client/ayon_core/plugins/publish/collect_managed_staging_dir.py index ca6d5161c1..1034b9a716 100644 --- a/client/ayon_core/plugins/publish/collect_managed_staging_dir.py +++ b/client/ayon_core/plugins/publish/collect_managed_staging_dir.py @@ -33,7 +33,11 @@ class CollectManagedStagingDir(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder + 0.4990 def process(self, instance): + """ Collect the staging data and stores it to the instance. + Args: + instance (object): The instance to inspect. + """ staging_dir_path = get_instance_staging_dir(instance) persistance = instance.data.get("stagingDir_persistent", False) diff --git a/client/ayon_core/plugins/publish/extract_burnin.py b/client/ayon_core/plugins/publish/extract_burnin.py index 3eb4254a5e..8e8764fc33 100644 --- a/client/ayon_core/plugins/publish/extract_burnin.py +++ b/client/ayon_core/plugins/publish/extract_burnin.py @@ -254,7 +254,7 @@ def main_process(self, instance): if do_convert: new_staging_dir = get_temp_dir( project_name=instance.context.data["projectName"], - make_local=True, + use_local_temp=True, ) repre["stagingDir"] = new_staging_dir diff --git a/client/ayon_core/plugins/publish/extract_color_transcode.py b/client/ayon_core/plugins/publish/extract_color_transcode.py index 4f0053c426..3c11a016ec 100644 --- a/client/ayon_core/plugins/publish/extract_color_transcode.py +++ b/client/ayon_core/plugins/publish/extract_color_transcode.py @@ -106,7 +106,7 @@ def process(self, instance): original_staging_dir = new_repre["stagingDir"] new_staging_dir = get_temp_dir( project_name=instance.context.data["projectName"], - make_local=True, + use_local_temp=True, ) new_repre["stagingDir"] = new_staging_dir diff --git a/client/ayon_core/plugins/publish/extract_review.py b/client/ayon_core/plugins/publish/extract_review.py index 26cd2ef0b2..7c38b0453b 100644 --- a/client/ayon_core/plugins/publish/extract_review.py +++ b/client/ayon_core/plugins/publish/extract_review.py @@ -312,7 +312,7 @@ def main_process(self, instance): if do_convert: new_staging_dir = get_temp_dir( project_name=instance.context.data["projectName"], - make_local=True, + use_local_temp=True, ) repre["stagingDir"] = new_staging_dir From d9c1a299b97ff8a4c5c6612a2644d41e7d564a40 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 21 Nov 2024 17:23:36 +0800 Subject: [PATCH 032/136] loading the asset per repre_id and per project --- .../ayon_core/tools/sceneinventory/model.py | 250 +++++++++--------- .../tools/sceneinventory/models/containers.py | 22 +- 2 files changed, 129 insertions(+), 143 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 5f9e6bb77c..29818e387f 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -131,50 +131,39 @@ def refresh(self, selected=None): """Refresh the model""" # for debugging or testing, injecting items from outside container_items = self._controller.get_container_items() - self._clear_items() - - items_by_repre_id = collections.defaultdict(list) - repre_ids_by_project = collections.defaultdict(set) - for container_item in container_items: + repre_id = set() + repre_id_by_project_id = collections.defaultdict(set) + version_items_by_product_id = collections.defaultdict(dict) + repre_info_by_id_by_project = collections.defaultdict(list) + item_by_repre_id_by_project_id = collections.defaultdict( + lambda: collections.defaultdict(set)) + for project_name, container_item in container_items.items(): # if ( # selected is not None # and container_item.item_id not in selected # ): # continue - project_name = container_item.project_name - repre_id = container_item.representation_id - items_by_repre_id[repre_id].append(container_item) - repre_ids_by_project[project_name].add(repre_id) - - repre_id = set(items_by_repre_id.keys()) - repre_info_by_id = {} - for project_name, repre_ids in repre_ids_by_project.items(): + for item in container_item.values(): + representation_id = item.representation_id + if item.project_name != project_name: + continue + repre_id.add(representation_id) + item_by_repre_id_by_project_id[project_name][representation_id].add(item) repre_info = self._controller.get_representation_info_items( - project_name, repre_ids + project_name, repre_id ) - repre_info_by_id.update(repre_info) - product_ids = { - repre_info.product_id - for repre_info in repre_info_by_id.values() - if repre_info.is_valid - } + repre_info_by_id_by_project[project_name] = repre_info - project_products = collections.defaultdict(set) - for container_item in container_items: - representation_id = container_item.representation_id - project_name = container_item.project_name - repre_info = repre_info_by_id.get(representation_id) - if repre_info and repre_info.is_valid: - product_id = repre_info.product_id - project_products[project_name].add(product_id) - - version_items_by_product_id = {} - for project_name, product_ids in project_products.items(): + product_ids = { + repre_info.product_id + for repre_info in repre_info.values() + if repre_info.is_valid + } version_items = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id.update(version_items) + version_items_by_product_id[project_name] = version_items # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( @@ -216,112 +205,113 @@ def refresh(self, selected=None): root_item = self.invisibleRootItem() group_items = [] - for repre_id, container_items in items_by_repre_id.items(): - repre_info = repre_info_by_id[repre_id] - version_color = None - if not repre_info.is_valid: - version_label = "N/A" - group_name = "< Entity N/A >" - item_icon = invalid_item_icon - is_latest = False - is_hero = False - status_name = None - - else: - group_name = "{}_{}: ({})".format( - repre_info.folder_path.rsplit("/")[-1], - repre_info.product_name, - repre_info.representation_name + for project_name, items_by_repre_id in item_by_repre_id_by_project_id.items(): + for repre_id, container_items in items_by_repre_id.items(): + repre_info = repre_info_by_id_by_project[project_name][repre_id] + version_color = None + if not repre_info.is_valid: + version_label = "N/A" + group_name = "< Entity N/A >" + item_icon = invalid_item_icon + is_latest = False + is_hero = False + status_name = None + + else: + group_name = "{}_{}: ({})".format( + repre_info.folder_path.rsplit("/")[-1], + repre_info.product_name, + repre_info.representation_name + ) + item_icon = valid_item_icon + + version_items = ( + version_items_by_product_id[project_name][repre_info.product_id] + ) + version_item = version_items[repre_info.version_id] + version_label = format_version(version_item.version) + is_hero = version_item.version < 0 + is_latest = version_item.is_latest + if not version_item.is_latest: + version_color = self.OUTDATED_COLOR + status_name = version_item.status + + status_color, status_short, status_icon = self._get_status_data( + status_name ) - item_icon = valid_item_icon - version_items = ( - version_items_by_product_id[repre_info.product_id] + repre_name = ( + repre_info.representation_name or "" + ) + container_model_items = [] + for container_item in container_items: + object_name = container_item.object_name or "" + unique_name = repre_name + object_name + item = QtGui.QStandardItem() + item.setColumnCount(root_item.columnCount()) + item.setData(container_item.namespace, QtCore.Qt.DisplayRole) + item.setData(self.GRAYOUT_COLOR, NAME_COLOR_ROLE) + item.setData(self.GRAYOUT_COLOR, VERSION_COLOR_ROLE) + item.setData(item_icon, QtCore.Qt.DecorationRole) + item.setData(repre_info.product_id, PRODUCT_ID_ROLE) + item.setData(container_item.item_id, ITEM_ID_ROLE) + item.setData(version_label, VERSION_LABEL_ROLE) + item.setData(container_item.loader_name, LOADER_NAME_ROLE) + item.setData(container_item.object_name, OBJECT_NAME_ROLE) + item.setData(container_item.project_name, PROJECT_NAME_ROLE) + item.setData(True, IS_CONTAINER_ITEM_ROLE) + item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) + container_model_items.append(item) + if not container_model_items: + continue + + progress = progress_by_id[repre_id] + active_site_progress = "{}%".format( + max(progress["active_site"], 0) * 100 + ) + remote_site_progress = "{}%".format( + max(progress["remote_site"], 0) * 100 ) - version_item = version_items[repre_info.version_id] - version_label = format_version(version_item.version) - is_hero = version_item.version < 0 - is_latest = version_item.is_latest - if not version_item.is_latest: - version_color = self.OUTDATED_COLOR - status_name = version_item.status - - status_color, status_short, status_icon = self._get_status_data( - status_name - ) - - repre_name = ( - repre_info.representation_name or "" - ) - container_model_items = [] - for container_item in container_items: - object_name = container_item.object_name or "" - unique_name = repre_name + object_name - item = QtGui.QStandardItem() - item.setColumnCount(root_item.columnCount()) - item.setData(container_item.namespace, QtCore.Qt.DisplayRole) - item.setData(self.GRAYOUT_COLOR, NAME_COLOR_ROLE) - item.setData(self.GRAYOUT_COLOR, VERSION_COLOR_ROLE) - item.setData(item_icon, QtCore.Qt.DecorationRole) - item.setData(repre_info.product_id, PRODUCT_ID_ROLE) - item.setData(container_item.item_id, ITEM_ID_ROLE) - item.setData(version_label, VERSION_LABEL_ROLE) - item.setData(container_item.loader_name, LOADER_NAME_ROLE) - item.setData(container_item.object_name, OBJECT_NAME_ROLE) - item.setData(container_item.project_name, PROJECT_NAME_ROLE) - item.setData(True, IS_CONTAINER_ITEM_ROLE) - item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) - container_model_items.append(item) - if not container_model_items: - continue - - progress = progress_by_id[repre_id] - active_site_progress = "{}%".format( - max(progress["active_site"], 0) * 100 - ) - remote_site_progress = "{}%".format( - max(progress["remote_site"], 0) * 100 - ) - - group_item = QtGui.QStandardItem() - group_item.setColumnCount(root_item.columnCount()) - group_item.setData(group_name, QtCore.Qt.DisplayRole) - group_item.setData(group_name, ITEM_UNIQUE_NAME_ROLE) - group_item.setData(group_item_icon, QtCore.Qt.DecorationRole) - group_item.setData(group_item_font, QtCore.Qt.FontRole) - group_item.setData(repre_info.product_id, PRODUCT_ID_ROLE) - group_item.setData(repre_info.product_type, PRODUCT_TYPE_ROLE) - group_item.setData(product_type_icon, PRODUCT_TYPE_ICON_ROLE) - group_item.setData(is_latest, VERSION_IS_LATEST_ROLE) - group_item.setData(is_hero, VERSION_IS_HERO_ROLE) - group_item.setData(version_label, VERSION_LABEL_ROLE) - group_item.setData(len(container_items), COUNT_ROLE) - group_item.setData(status_name, STATUS_NAME_ROLE) - group_item.setData(status_short, STATUS_SHORT_ROLE) - group_item.setData(status_color, STATUS_COLOR_ROLE) - group_item.setData(status_icon, STATUS_ICON_ROLE) - - group_item.setData( - active_site_progress, ACTIVE_SITE_PROGRESS_ROLE - ) - group_item.setData( - remote_site_progress, REMOTE_SITE_PROGRESS_ROLE - ) - group_item.setData(active_site_icon, ACTIVE_SITE_ICON_ROLE) - group_item.setData(remote_site_icon, REMOTE_SITE_ICON_ROLE) - group_item.setData(False, IS_CONTAINER_ITEM_ROLE) - if version_color is not None: - group_item.setData(version_color, VERSION_COLOR_ROLE) + group_item = QtGui.QStandardItem() + group_item.setColumnCount(root_item.columnCount()) + group_item.setData(group_name, QtCore.Qt.DisplayRole) + group_item.setData(group_name, ITEM_UNIQUE_NAME_ROLE) + group_item.setData(group_item_icon, QtCore.Qt.DecorationRole) + group_item.setData(group_item_font, QtCore.Qt.FontRole) + group_item.setData(repre_info.product_id, PRODUCT_ID_ROLE) + group_item.setData(repre_info.product_type, PRODUCT_TYPE_ROLE) + group_item.setData(product_type_icon, PRODUCT_TYPE_ICON_ROLE) + group_item.setData(is_latest, VERSION_IS_LATEST_ROLE) + group_item.setData(is_hero, VERSION_IS_HERO_ROLE) + group_item.setData(version_label, VERSION_LABEL_ROLE) + group_item.setData(len(container_items), COUNT_ROLE) + group_item.setData(status_name, STATUS_NAME_ROLE) + group_item.setData(status_short, STATUS_SHORT_ROLE) + group_item.setData(status_color, STATUS_COLOR_ROLE) + group_item.setData(status_icon, STATUS_ICON_ROLE) - if repre_info.product_group: group_item.setData( - repre_info.product_group, PRODUCT_GROUP_NAME_ROLE + active_site_progress, ACTIVE_SITE_PROGRESS_ROLE ) - group_item.setData(group_icon, PRODUCT_GROUP_ICON_ROLE) + group_item.setData( + remote_site_progress, REMOTE_SITE_PROGRESS_ROLE + ) + group_item.setData(active_site_icon, ACTIVE_SITE_ICON_ROLE) + group_item.setData(remote_site_icon, REMOTE_SITE_ICON_ROLE) + group_item.setData(False, IS_CONTAINER_ITEM_ROLE) + + if version_color is not None: + group_item.setData(version_color, VERSION_COLOR_ROLE) + + if repre_info.product_group: + group_item.setData( + repre_info.product_group, PRODUCT_GROUP_NAME_ROLE + ) + group_item.setData(group_icon, PRODUCT_GROUP_ICON_ROLE) - group_item.appendRows(container_model_items) - group_items.append(group_item) + group_item.appendRows(container_model_items) + group_items.append(group_item) if group_items: root_item.appendRows(group_items) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index e135cb0031..ca67fb59f9 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -191,7 +191,7 @@ def from_entity(cls, version_entity, is_latest, is_last_approved): class ContainersModel: def __init__(self, controller): self._controller = controller - self._items_cache = None + self._project_cache = None self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} @@ -200,7 +200,7 @@ def __init__(self, controller): self._product_ids_by_project = {} def reset(self): - self._items_cache = None + self._project_cache = None self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} @@ -220,7 +220,7 @@ def get_containers_by_item_ids(self, item_ids): def get_container_items(self): self._update_cache() - return list(self._items_cache) + return self._project_cache def get_container_items_by_id(self, item_ids): return { @@ -248,7 +248,6 @@ def get_representation_info_items(self, project_name, representation_ids): repre_hierarchy_by_id = get_representations_hierarchy( project_name, missing_repre_ids ) - self._product_ids_by_project[project_name] = set() for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): kwargs = { "folder_id": None, @@ -280,8 +279,6 @@ def get_representation_info_items(self, project_name, representation_ids): repre_info = RepresentationInfo(**kwargs) self._repre_info_by_id[repre_id] = repre_info - self._product_ids_by_project[project_name].add( - repre_info.product_id) output[repre_id] = repre_info return output @@ -293,7 +290,6 @@ def get_version_items(self, project_name, product_ids): for product_id in product_ids if product_id not in self._version_items_by_product_id } - current_product_ids = self._product_ids_by_project.get(project_name) if missing_ids: status_items_by_name = { status_item.name: status_item @@ -302,14 +298,13 @@ def get_version_items(self, project_name, product_ids): def version_sorted(entity): return entity["version"] - current_missing_ids = current_product_ids.intersection(missing_ids) version_entities_by_product_id = { product_id: [] - for product_id in current_missing_ids + for product_id in missing_ids } version_entities = list(ayon_api.get_versions( project_name, - product_ids=current_missing_ids, + product_ids=missing_ids, fields={"id", "version", "productId", "status"} )) version_entities.sort(key=version_sorted) @@ -349,7 +344,7 @@ def version_sorted(entity): } def _update_cache(self): - if self._items_cache is not None: + if self._project_cache is not None: return host = self._controller.get_host() @@ -363,7 +358,7 @@ def _update_cache(self): container_items = [] containers_by_id = {} container_items_by_id = {} - project_name_by_repre_id = {} + project_cache = collections.defaultdict(dict) invalid_ids_mapping = {} for container in containers: try: @@ -388,9 +383,10 @@ def _update_cache(self): containers_by_id[item.item_id] = container container_items_by_id[item.item_id] = item - project_name_by_repre_id[item.representation_id] = item.project_name + project_cache[item.project_name] = container_items_by_id container_items.append(item) self._containers_by_id = containers_by_id self._container_items_by_id = container_items_by_id self._items_cache = container_items + self._project_cache = project_cache From 7a224914ea7dc59bcfa158cd6ead2404c4074a9e Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 21 Nov 2024 17:43:17 +0800 Subject: [PATCH 033/136] remove unused variable --- client/ayon_core/tools/sceneinventory/model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 29818e387f..640a8017ab 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -133,7 +133,6 @@ def refresh(self, selected=None): container_items = self._controller.get_container_items() self._clear_items() repre_id = set() - repre_id_by_project_id = collections.defaultdict(set) version_items_by_product_id = collections.defaultdict(dict) repre_info_by_id_by_project = collections.defaultdict(list) item_by_repre_id_by_project_id = collections.defaultdict( From c56cd07e67b22b4e34cf6c246229150799fa5ab0 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:47:37 +0100 Subject: [PATCH 034/136] Provided backward compatibility for prepare_representations --- client/ayon_core/pipeline/farm/pyblish_functions.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e9f179c668..e236ec6c3d 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -333,7 +333,13 @@ def prepare_representations( log = Logger.get_logger("farm_publishing") - frames_to_render = _get_real_frames_to_render(frames_to_render) + if frames_to_render is not None: + frames_to_render = _get_real_frames_to_render(frames_to_render) + else: + # Backwards compatibility for older logic + frame_start = int(skeleton_data.get("frameStartHandle")) + frame_end = int(skeleton_data.get("frameEndHandle")) + frames_to_render = list(range(frame_start, frame_end + 1)) # create representation for every collected sequence for collection in collections: From c2716872d43d20f1e7de051375244893cd192d38 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:49:24 +0100 Subject: [PATCH 035/136] Do not convert to str unnecessary Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e9f179c668..aa69633a22 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -482,7 +482,8 @@ def _get_real_frames_to_render(frames): range(int(splitted[0]), int(splitted[1])+1)) else: frames_to_render.append(frame) - return [str(frame_to_render) for frame_to_render in frames_to_render] + frames_to_render.sort() + return frames_to_render def _get_real_files_to_rendered(collection, frames_to_render): From de88260ddac22419b3a87606aaae08bfc9ec4e09 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:50:23 +0100 Subject: [PATCH 036/136] frames_to_render are now list of integers Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index aa69633a22..876a5b504f 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -481,7 +481,7 @@ def _get_real_frames_to_render(frames): frames_to_render.extend( range(int(splitted[0]), int(splitted[1])+1)) else: - frames_to_render.append(frame) + frames_to_render.append(int(frame)) frames_to_render.sort() return frames_to_render From 630d2d49130a9cea142aea999203fc00106a269d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:50:36 +0100 Subject: [PATCH 037/136] frames_to_render are now list of integers Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 876a5b504f..6740950d78 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -371,8 +371,8 @@ def prepare_representations( " This may cause issues on farm." ).format(staging)) - frame_start = int(frames_to_render[0]) - frame_end = int(frames_to_render[-1]) + frame_start = frames_to_render[0] + frame_end = frames_to_render[-1] if skeleton_data.get("slate"): frame_start -= 1 From 5f3175258725853011ab4b9eb2c58c1fc6959eda Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:50:59 +0100 Subject: [PATCH 038/136] Used comprehension Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 6740950d78..09df371371 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -498,11 +498,10 @@ def _get_real_files_to_rendered(collection, frames_to_render): file_name, extracted_frame = list(collect_frames(files).items())[0] if extracted_frame: found_frame_pattern_length = len(extracted_frame) - normalized_frames_to_render = set() - for frame_to_render in frames_to_render: - normalized_frames_to_render.add( - str(frame_to_render).zfill(found_frame_pattern_length) - ) + normalized_frames_to_render = { + str(frame_to_render).zfill(found_frame_pattern_length) + for frame_to_render in frames_to_render + } filtered_files = [] for file_name in files: From fd20885ac26366b996c2b14d102cbefdc3a75341 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:51:12 +0100 Subject: [PATCH 039/136] Used comprehension Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../ayon_core/pipeline/farm/pyblish_functions.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 09df371371..4ba088f92c 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -503,13 +503,14 @@ def _get_real_files_to_rendered(collection, frames_to_render): for frame_to_render in frames_to_render } - filtered_files = [] - for file_name in files: - if any(frame in file_name - for frame in normalized_frames_to_render): - filtered_files.append(file_name) - - files = filtered_files + files = [ + filename + for filename in files + if any( + frame in filename + for frame in normalized_frames_to_render + ) + ] return files From 80ab628d44b083f168e1a7e822f4f1145d40145b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:52:51 +0100 Subject: [PATCH 040/136] Changed condition to bail early --- .../pipeline/farm/pyblish_functions.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e236ec6c3d..37a018e116 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -501,21 +501,23 @@ def _get_real_files_to_rendered(collection, frames_to_render): """ files = [os.path.basename(f) for f in list(collection)] file_name, extracted_frame = list(collect_frames(files).items())[0] - if extracted_frame: - found_frame_pattern_length = len(extracted_frame) - normalized_frames_to_render = set() - for frame_to_render in frames_to_render: - normalized_frames_to_render.add( - str(frame_to_render).zfill(found_frame_pattern_length) - ) + if not extracted_frame: + return files + + found_frame_pattern_length = len(extracted_frame) + normalized_frames_to_render = set() + for frame_to_render in frames_to_render: + normalized_frames_to_render.add( + str(frame_to_render).zfill(found_frame_pattern_length) + ) - filtered_files = [] - for file_name in files: - if any(frame in file_name - for frame in normalized_frames_to_render): - filtered_files.append(file_name) + filtered_files = [] + for file_name in files: + if any(frame in file_name + for frame in normalized_frames_to_render): + filtered_files.append(file_name) - files = filtered_files + files = filtered_files return files From 63592f9e2bb799350338e5f29e39cdc59bd28077 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 14:32:54 +0100 Subject: [PATCH 041/136] Used comprehension Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e1d83a175e..e3470f4c41 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -507,11 +507,10 @@ def _get_real_files_to_rendered(collection, frames_to_render): return files found_frame_pattern_length = len(extracted_frame) - normalized_frames_to_render = set() - for frame_to_render in frames_to_render: - normalized_frames_to_render.add( - str(frame_to_render).zfill(found_frame_pattern_length) - ) + normalized_frames_to_render = { + str(frame_to_render).zfill(found_frame_pattern_length) + for frame_to_render in frames_to_render + } filtered_files = [] for file_name in files: From 112c4bdc0eec32f7d140964a911d88c5926a27e7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 14:33:08 +0100 Subject: [PATCH 042/136] Used comprehension Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../ayon_core/pipeline/farm/pyblish_functions.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e3470f4c41..16364a17ee 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -512,14 +512,14 @@ def _get_real_files_to_rendered(collection, frames_to_render): for frame_to_render in frames_to_render } - filtered_files = [] - for file_name in files: - if any(frame in file_name - for frame in normalized_frames_to_render): - filtered_files.append(file_name) - - files = filtered_files - return files + return [ + file_name + for file_name in files + if any( + frame in file_name + for frame in normalized_frames_to_render + ) + ] def create_instances_for_aov(instance, skeleton, aov_filter, From ddf0a2b00a6cad3677924ec25ef6341daeb92001 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:00:43 +0100 Subject: [PATCH 043/136] force to use older opencolorio than 2.4.0 --- client/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/pyproject.toml b/client/pyproject.toml index a0be9605b6..20b51ff219 100644 --- a/client/pyproject.toml +++ b/client/pyproject.toml @@ -15,6 +15,6 @@ qtawesome = "0.7.3" aiohttp-middlewares = "^2.0.0" Click = "^8" OpenTimelineIO = "0.16.0" -opencolorio = "^2.3.2" +opencolorio = "<2.4.0" Pillow = "9.5.0" websocket-client = ">=0.40.0,<2" From 5b35547072b89f25c4c53d41612fa7d5b27b1d6f Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:10:57 +0100 Subject: [PATCH 044/136] fix syntax of version requirement --- client/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/pyproject.toml b/client/pyproject.toml index 20b51ff219..edf7f57317 100644 --- a/client/pyproject.toml +++ b/client/pyproject.toml @@ -15,6 +15,6 @@ qtawesome = "0.7.3" aiohttp-middlewares = "^2.0.0" Click = "^8" OpenTimelineIO = "0.16.0" -opencolorio = "<2.4.0" +opencolorio = "^2.3.2,<2.4.0" Pillow = "9.5.0" websocket-client = ">=0.40.0,<2" From a70135bb5156fc3883047561edcda702f81f8731 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 21 Nov 2024 22:35:25 +0800 Subject: [PATCH 045/136] implement switch and set version by repre_id/product_id per project --- .../ayon_core/tools/sceneinventory/model.py | 20 +- .../tools/sceneinventory/models/containers.py | 11 +- client/ayon_core/tools/sceneinventory/view.py | 184 +++++++++--------- 3 files changed, 110 insertions(+), 105 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 640a8017ab..75af957cfa 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -133,24 +133,26 @@ def refresh(self, selected=None): container_items = self._controller.get_container_items() self._clear_items() repre_id = set() + repre_ids_by_project = collections.defaultdict(set) version_items_by_product_id = collections.defaultdict(dict) - repre_info_by_id_by_project = collections.defaultdict(list) + repre_info_by_id_by_project = collections.defaultdict(dict) item_by_repre_id_by_project_id = collections.defaultdict( lambda: collections.defaultdict(set)) - for project_name, container_item in container_items.items(): + for container_item in container_items: # if ( # selected is not None # and container_item.item_id not in selected # ): # continue - for item in container_item.values(): - representation_id = item.representation_id - if item.project_name != project_name: - continue - repre_id.add(representation_id) - item_by_repre_id_by_project_id[project_name][representation_id].add(item) + project_name = container_item.project_name + representation_id = container_item.representation_id + repre_id.add(representation_id) + repre_ids_by_project[project_name].add(representation_id) + item_by_repre_id_by_project_id[project_name][representation_id].add(container_item) + + for project_name, representation_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( - project_name, repre_id + project_name, representation_ids ) repre_info_by_id_by_project[project_name] = repre_info diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index ca67fb59f9..dc41bdc8fa 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -191,7 +191,7 @@ def from_entity(cls, version_entity, is_latest, is_last_approved): class ContainersModel: def __init__(self, controller): self._controller = controller - self._project_cache = None + self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} @@ -200,7 +200,7 @@ def __init__(self, controller): self._product_ids_by_project = {} def reset(self): - self._project_cache = None + self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} @@ -220,7 +220,7 @@ def get_containers_by_item_ids(self, item_ids): def get_container_items(self): self._update_cache() - return self._project_cache + return list(self._items_cache) def get_container_items_by_id(self, item_ids): return { @@ -344,7 +344,7 @@ def version_sorted(entity): } def _update_cache(self): - if self._project_cache is not None: + if self._items_cache is not None: return host = self._controller.get_host() @@ -358,7 +358,6 @@ def _update_cache(self): container_items = [] containers_by_id = {} container_items_by_id = {} - project_cache = collections.defaultdict(dict) invalid_ids_mapping = {} for container in containers: try: @@ -383,10 +382,8 @@ def _update_cache(self): containers_by_id[item.item_id] = container container_items_by_id[item.item_id] = item - project_cache[item.project_name] = container_items_by_id container_items.append(item) self._containers_by_id = containers_by_id self._container_items_by_id = container_items_by_id self._items_cache = container_items - self._project_cache = project_cache diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index a049fd1e0b..24e0195e31 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -197,26 +197,29 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): repre_id = container_item.representation_id project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) - repre_info_by_id = {} + + repre_info_by_project = collections.defaultdict(dict) for project_name, repre_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( project_name, repre_ids) - repre_info_by_id.update(repre_info) - - valid_repre_ids = { - repre_id - for repre_id, repre_info in repre_info_by_id.items() - if repre_info.is_valid - } - + repre_info_by_project[project_name].update(repre_info) # Exclude items that are "NOT FOUND" since setting versions, updating # and removal won't work for those items. filtered_items = [] - product_ids_by_project = collections.defaultdict(set) version_ids = set() + valid_repre_ids = set() + product_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): - repre_id = container_item.representation_id project_name = container_item.project_name + repre_info_by_id = repre_info_by_project.get(project_name) + repre_id = container_item.representation_id + all_valid_repre_ids = { + repre_id + for repre_id, repre_info in repre_info_by_id.items() + if repre_info.is_valid + } + valid_repre_ids.update(all_valid_repre_ids) + repre_info = repre_info_by_id.get(repre_id) if repre_info and repre_info.is_valid: filtered_items.append(container_item) @@ -233,47 +236,47 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): # Keep remove action for invalid items menu.addAction(remove_action) return - version_items_by_product_id = {} + version_items_by_product_id_by_project = collections.defaultdict(dict) for project_name, product_ids in product_ids_by_project.items(): version_items = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id.update(version_items -) + version_items_by_product_id_by_project[project_name] = version_items has_outdated = False has_loaded_hero_versions = False has_available_hero_version = False has_outdated_approved = False last_version_by_product_id = {} - for product_id, version_items_by_id in ( - version_items_by_product_id.items() - ): - _has_outdated_approved = False - _last_approved_version_item = None - for version_item in version_items_by_id.values(): - if version_item.is_hero: - has_available_hero_version = True + for version_items_by_product_id in version_items_by_product_id_by_project.values(): + for product_id, version_items_by_id in ( + version_items_by_product_id.items() + ): + _has_outdated_approved = False + _last_approved_version_item = None + for version_item in version_items_by_id.values(): + if version_item.is_hero: + has_available_hero_version = True - elif version_item.is_last_approved: - _last_approved_version_item = version_item - _has_outdated_approved = True + elif version_item.is_last_approved: + _last_approved_version_item = version_item + _has_outdated_approved = True - if version_item.version_id not in version_ids: - continue + if version_item.version_id not in version_ids: + continue - if version_item.is_hero: - has_loaded_hero_versions = True - elif not version_item.is_latest: - has_outdated = True + if version_item.is_hero: + has_loaded_hero_versions = True + elif not version_item.is_latest: + has_outdated = True - if ( - _has_outdated_approved - and _last_approved_version_item is not None - ): - last_version_by_product_id[product_id] = ( - _last_approved_version_item - ) - has_outdated_approved = True + if ( + _has_outdated_approved + and _last_approved_version_item is not None + ): + last_version_by_product_id[product_id] = ( + _last_approved_version_item + ) + has_outdated_approved = True switch_to_versioned = None if has_loaded_hero_versions: @@ -294,8 +297,9 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): approved_version_by_item_id = {} if has_outdated_approved: for container_item in container_items_by_id.values(): + project_name = container_item.project_name repre_id = container_item.representation_id - repre_info = repre_info_by_id.get(repre_id) + repre_info = repre_info_by_project[project_name][repre_id] if not repre_info or not repre_info.is_valid: continue version_item = last_version_by_product_id.get( @@ -750,54 +754,53 @@ def _show_version_dialog(self, item_ids, active_repre_id): repre_id = container_item.representation_id project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) - repre_info_by_id = {} + + versions = set() + repre_info_by_project = collections.defaultdict(dict) + version_items_by_product_id_by_project = collections.defaultdict(dict) for project_name, repre_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( project_name, repre_ids ) - repre_info_by_id.update(repre_info) + repre_info_by_project[project_name].update(repre_info) - product_ids = { - repre_info.product_id - for repre_info in repre_info_by_id.values() - } - product_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): - repre_id = container_item.representation_id project_name = container_item.project_name + repre_info_by_id = repre_info_by_project.get(project_name) + repre_id = container_item.representation_id repre_info = repre_info_by_id.get(repre_id) - if not repre_info or not repre_info.is_valid: - continue - product_ids_by_project[project_name].add( + product_ids = { repre_info.product_id - ) - active_repre_info = repre_info_by_id[active_repre_id] - active_version_id = active_repre_info.version_id - active_product_id = active_repre_info.product_id - version_items_by_product_id = {} - for project_name, project_product_ids in product_ids_by_project.items(): + for repre_info in repre_info_by_id.values() + if repre_info.is_valid + } version_items = self._controller.get_version_items( - project_name, project_product_ids + project_name, product_ids ) - version_items_by_product_id.update(version_items) - version_items = list( - version_items_by_product_id[active_product_id].values() - ) - versions = {version_item.version for version_item in version_items} + version_items_by_product_id_by_project[project_name] = version_items + active_repre_info = repre_info_by_id[active_repre_id] + active_version_id = active_repre_info.version_id + active_product_id = active_repre_info.product_id + version_items = list( + version_items_by_product_id_by_project[project_name][active_product_id].values() + ) + all_versions = {version_item.version for version_item in version_items} + versions.update(all_versions) product_ids_by_version = collections.defaultdict(set) - for version_items_by_id in version_items_by_product_id.values(): - for version_item in version_items_by_id.values(): - version = version_item.version - _prod_version = version - if _prod_version < 0: - _prod_version = -1 - product_ids_by_version[_prod_version].add( - version_item.product_id - ) - if version in versions: - continue - versions.add(version) - version_items.append(version_item) + for version_items_by_product_id in version_items_by_product_id_by_project.values(): + for version_items_by_id in version_items_by_product_id.values(): + for version_item in version_items_by_id.values(): + version = version_item.version + _prod_version = version + if _prod_version < 0: + _prod_version = -1 + product_ids_by_version[_prod_version].add( + version_item.product_id + ) + if version in versions: + continue + versions.add(version) + version_items.append(version_item) def version_sorter(item): hero_value = 0 @@ -862,8 +865,9 @@ def version_sorter(item): filtered_item_ids = set() for container_item in container_items_by_id.values(): + project_name = container_item.project_name repre_id = container_item.representation_id - repre_info = repre_info_by_id[repre_id] + repre_info = repre_info_by_project[project_name][repre_id] if repre_info.product_id in product_ids: filtered_item_ids.add(container_item.item_id) @@ -964,37 +968,39 @@ def _on_switch_to_versioned(self, item_ids): repre_ids_by_project[project_name].add(repre_id) # Get representation info items by ID - repre_info_by_id = {} + repre_info_by_project = collections.defaultdict(dict) for project_name, repre_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( project_name, repre_ids) - repre_info_by_id.update(repre_info) + repre_info_by_project[project_name].update(repre_info) product_ids_by_project = collections.defaultdict(set) + version_items_by_product_id_by_project = collections.defaultdict(dict) for container_item in containers_items_by_id.values(): - repre_id = container_item.representation_id project_name = container_item.project_name + repre_info_by_id = repre_info_by_project.get(project_name) + repre_id = container_item.representation_id repre_info = repre_info_by_id.get(repre_id) - if not repre_info or not repre_info.is_valid: - continue - product_ids_by_project[project_name].add( + product_ids = { repre_info.product_id - ) - - version_items_by_product_id = {} - for project_name, product_ids in product_ids_by_project.items(): + for repre_info in repre_info_by_id.values() + if repre_info.is_valid + } version_items = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id.update(version_items) + version_items_by_product_id_by_project[project_name] = version_items update_containers = [] update_versions = [] for item_id, container_item in containers_items_by_id.items(): repre_id = container_item.representation_id + project_name = container_item.project_name repre_info = repre_info_by_id[repre_id] product_id = repre_info.product_id - version_items_id = version_items_by_product_id[product_id] + version_items_id = ( + version_items_by_product_id_by_project[project_name][product_id] + ) version_item = version_items_id.get(repre_info.version_id, {}) if not version_item or not version_item.is_hero: continue From 77e5317ee3bae6b2aa792a9525cc3433e24ddeca Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 21 Nov 2024 22:37:55 +0800 Subject: [PATCH 046/136] remove unused variable --- client/ayon_core/tools/sceneinventory/view.py | 1 - 1 file changed, 1 deletion(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 24e0195e31..025bff6e9f 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -974,7 +974,6 @@ def _on_switch_to_versioned(self, item_ids): project_name, repre_ids) repre_info_by_project[project_name].update(repre_info) - product_ids_by_project = collections.defaultdict(set) version_items_by_product_id_by_project = collections.defaultdict(dict) for container_item in containers_items_by_id.values(): project_name = container_item.project_name From 26251bb9b4ebae496ab7aab0a7a3e0a1a95c1611 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:29:31 +0100 Subject: [PATCH 047/136] simplified parsing of template --- client/ayon_core/lib/path_templates.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index dc88ec956b..1a99ae459d 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -1,6 +1,7 @@ import os import re import numbers +from string import Formatter KEY_PATTERN = re.compile(r"(\{.*?[^{0]*\})") KEY_PADDING_PATTERN = re.compile(r"([^:]+)\S+[><]\S+") @@ -48,16 +49,16 @@ def __init__(self, template): self._template = template parts = [] - last_end_idx = 0 - for item in KEY_PATTERN.finditer(template): - start, end = item.span() - if start > last_end_idx: - parts.append(template[last_end_idx:start]) - parts.append(FormattingPart(template[start:end])) - last_end_idx = end - - if last_end_idx < len(template): - parts.append(template[last_end_idx:len(template)]) + formatter = Formatter() + + for item in formatter.parse(template): + literal_text, field_name, format_spec, conversion = item + if literal_text: + parts.append(literal_text) + if field_name: + parts.append( + FormattingPart(field_name, format_spec, conversion) + ) new_parts = [] for part in parts: From e4875cc5096a5381861b45f45c28eb6c2a68215e Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:37:34 +0100 Subject: [PATCH 048/136] fill FormattingPart init --- client/ayon_core/lib/path_templates.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index 1a99ae459d..3871b97849 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -437,8 +437,21 @@ class FormattingPart: Args: template(str): String containing the formatting key. """ - def __init__(self, template): - self._template = template + def __init__(self, field_name, format_spec, conversion): + format_spec_v = "" + if format_spec: + format_spec_v = f":{format_spec}" + conversion_v = "" + if conversion: + conversion_v = f"!{conversion}" + + self._field_name = field_name + self._format_spec = format_spec_v + self._conversion = conversion_v + + template_base = f"{field_name}{format_spec_v}{conversion_v}" + self._template_base = template_base + self._template = f"{{{template_base}}}" @property def template(self): From d15148f001f17f4488119c402a54005be34e0d38 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:38:49 +0100 Subject: [PATCH 049/136] support list in StringTemplate --- client/ayon_core/lib/path_templates.py | 124 +++++++++++++++++-------- 1 file changed, 83 insertions(+), 41 deletions(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index 3871b97849..9b545f2851 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -1,10 +1,10 @@ import os import re +import copy import numbers +from typing import List from string import Formatter -KEY_PATTERN = re.compile(r"(\{.*?[^{0]*\})") -KEY_PADDING_PATTERN = re.compile(r"([^:]+)\S+[><]\S+") SUB_DICT_PATTERN = re.compile(r"([^\[\]]+)") OPTIONAL_PATTERN = re.compile(r"(<.*?[^{0]*>)[^0-9]*?") @@ -369,11 +369,10 @@ def used_values(self): @staticmethod def split_keys_to_subdicts(values): output = {} + formatter = Formatter() for key, value in values.items(): - key_padding = list(KEY_PADDING_PATTERN.findall(key)) - if key_padding: - key = key_padding[0] - key_subdict = list(SUB_DICT_PATTERN.findall(key)) + _, field_name, _, _ = next(formatter.parse(f"{{{key}}}")) + key_subdict = list(SUB_DICT_PATTERN.findall(field_name)) data = output last_key = key_subdict.pop(-1) for subkey in key_subdict: @@ -502,6 +501,16 @@ def validate_key_is_matched(key): return False return not queue + @staticmethod + def keys_to_template_base(keys: List[str]): + if not keys: + return None + # Create copy of keys + keys = list(keys) + template_base = keys.pop(0) + joined_keys = "".join([f"[{key}]" for key in keys]) + return f"{template_base}{joined_keys}" + def format(self, data, result): """Format the formattings string. @@ -509,7 +518,7 @@ def format(self, data, result): data(dict): Data that should be used for formatting. result(TemplatePartResult): Object where result is stored. """ - key = self.template[1:-1] + key = self._template_base if key in result.realy_used_values: result.add_output(result.realy_used_values[key]) return result @@ -521,17 +530,38 @@ def format(self, data, result): return result # check if key expects subdictionary keys (e.g. project[name]) - existence_check = key - key_padding = list(KEY_PADDING_PATTERN.findall(existence_check)) - if key_padding: - existence_check = key_padding[0] - key_subdict = list(SUB_DICT_PATTERN.findall(existence_check)) + key_subdict = list(SUB_DICT_PATTERN.findall(self._field_name)) value = data missing_key = False invalid_type = False used_keys = [] + keys_to_value = None + used_value = None + for sub_key in key_subdict: + if isinstance(value, list): + if not sub_key.lstrip("-").isdigit(): + invalid_type = True + break + sub_key = int(sub_key) + if sub_key < 0: + sub_key = len(value) + sub_key + + invalid = 0 > sub_key < len(data) + if invalid: + used_keys.append(sub_key) + missing_key = True + break + + used_keys.append(sub_key) + if keys_to_value is None: + keys_to_value = list(used_keys) + keys_to_value.pop(-1) + used_value = copy.deepcopy(value) + value = value[sub_key] + continue + if ( value is None or (hasattr(value, "items") and sub_key not in value) @@ -547,45 +577,57 @@ def format(self, data, result): used_keys.append(sub_key) value = value.get(sub_key) - if missing_key or invalid_type: - if len(used_keys) == 0: - invalid_key = key_subdict[0] - else: - invalid_key = used_keys[0] - for idx, sub_key in enumerate(used_keys): - if idx == 0: - continue - invalid_key += "[{0}]".format(sub_key) + field_name = key_subdict[0] + if used_keys: + field_name = self.keys_to_template_base(used_keys) + if missing_key or invalid_type: if missing_key: - result.add_missing_key(invalid_key) + result.add_missing_key(field_name) elif invalid_type: - result.add_invalid_type(invalid_key, value) + result.add_invalid_type(field_name, value) result.add_output(self.template) return result - if self.validate_value_type(value): - fill_data = {} - first_value = True - for used_key in reversed(used_keys): - if first_value: - first_value = False - fill_data[used_key] = value - else: - _fill_data = {used_key: fill_data} - fill_data = _fill_data - - formatted_value = self.template.format(**fill_data) - result.add_realy_used_value(key, formatted_value) - result.add_used_value(existence_check, formatted_value) - result.add_output(formatted_value) + if not self.validate_value_type(value): + result.add_invalid_type(key, value) + result.add_output(self.template) return result - result.add_invalid_type(key, value) - result.add_output(self.template) - + fill_data = root_fill_data = {} + parent_fill_data = None + parent_key = None + fill_value = data + value_filled = False + for used_key in used_keys: + if isinstance(fill_value, list): + parent_fill_data[parent_key] = fill_value + value_filled = True + break + fill_value = fill_value[used_key] + parent_fill_data = fill_data + fill_data = parent_fill_data.setdefault(used_key, {}) + parent_key = used_key + + if not value_filled: + parent_fill_data[used_keys[-1]] = value + + template = f"{{{field_name}{self._format_spec}{self._conversion}}}" + formatted_value = template.format(**root_fill_data) + used_key = key + if keys_to_value is not None: + used_key = self.keys_to_template_base(keys_to_value) + + if used_value is None: + if isinstance(value, numbers.Number): + used_value = value + else: + used_value = formatted_value + result.add_realy_used_value(self._field_name, used_value) + result.add_used_value(used_key, used_value) + result.add_output(formatted_value) return result From e625901aebe0bab4a97e11ca4725c95ac29a86dc Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Fri, 22 Nov 2024 16:05:17 +0800 Subject: [PATCH 050/136] big roy's comment - code tweak --- client/ayon_core/tools/sceneinventory/model.py | 6 +++--- .../tools/sceneinventory/models/containers.py | 2 +- client/ayon_core/tools/sceneinventory/view.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 75af957cfa..849d8b8d17 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -132,7 +132,7 @@ def refresh(self, selected=None): # for debugging or testing, injecting items from outside container_items = self._controller.get_container_items() self._clear_items() - repre_id = set() + repre_ids = set() repre_ids_by_project = collections.defaultdict(set) version_items_by_product_id = collections.defaultdict(dict) repre_info_by_id_by_project = collections.defaultdict(dict) @@ -146,7 +146,7 @@ def refresh(self, selected=None): # continue project_name = container_item.project_name representation_id = container_item.representation_id - repre_id.add(representation_id) + repre_ids.add(representation_id) repre_ids_by_project[project_name].add(representation_id) item_by_repre_id_by_project_id[project_name][representation_id].add(container_item) @@ -168,7 +168,7 @@ def refresh(self, selected=None): # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( - repre_id + repre_ids ) sites_info = self._controller.get_sites_information() site_icons = { diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index dc41bdc8fa..aea94b97ef 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -359,9 +359,9 @@ def _update_cache(self): containers_by_id = {} container_items_by_id = {} invalid_ids_mapping = {} + current_project_name = self._controller.get_current_project_name() for container in containers: try: - current_project_name = self._controller.get_current_project_name() item = ContainerItem.from_container_data(current_project_name, container) repre_id = item.representation_id try: diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 025bff6e9f..d4381f55cd 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -755,7 +755,7 @@ def _show_version_dialog(self, item_ids, active_repre_id): project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) - versions = set() + version_ids = set() repre_info_by_project = collections.defaultdict(dict) version_items_by_product_id_by_project = collections.defaultdict(dict) for project_name, repre_ids in repre_ids_by_project.items(): @@ -784,8 +784,8 @@ def _show_version_dialog(self, item_ids, active_repre_id): version_items = list( version_items_by_product_id_by_project[project_name][active_product_id].values() ) - all_versions = {version_item.version for version_item in version_items} - versions.update(all_versions) + version_ids.update(version_item.version for version_item in version_items) + product_ids_by_version = collections.defaultdict(set) for version_items_by_product_id in version_items_by_product_id_by_project.values(): for version_items_by_id in version_items_by_product_id.values(): @@ -797,9 +797,9 @@ def _show_version_dialog(self, item_ids, active_repre_id): product_ids_by_version[_prod_version].add( version_item.product_id ) - if version in versions: + if version in version_ids: continue - versions.add(version) + version_ids.add(version) version_items.append(version_item) def version_sorter(item): From 842033ddc65e30e6f5d877be5dc57624d7e8c1b3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:55:39 +0100 Subject: [PATCH 051/136] use 'folderPath' to calculate 'hierarchy' --- .../plugins/publish/collect_anatomy_instance_data.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py b/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py index a0bd57d7dc..abd64ec03d 100644 --- a/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py +++ b/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py @@ -413,14 +413,16 @@ def _fill_folder_data(self, instance, project_entity, anatomy_data): # Backwards compatible (Deprecated since 24/06/06) or instance.data.get("newAssetPublishing") ): - hierarchy = instance.data["hierarchy"] - anatomy_data["hierarchy"] = hierarchy + folder_path = instance.data["folderPath"] + parents = folder_path.lstrip("/").split("/") + folder_name = parents.pop(-1) parent_name = project_entity["name"] - if hierarchy: - parent_name = hierarchy.split("/")[-1] + hierarchy = "" + if parents: + parent_name = parents[-1] + hierarchy = "/".join(parents) - folder_name = instance.data["folderPath"].split("/")[-1] anatomy_data.update({ "asset": folder_name, "hierarchy": hierarchy, From 7edc759842024453cf60702b1bf3b5c431c96fae Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:57:49 +0100 Subject: [PATCH 052/136] remove unused variables --- client/ayon_core/tools/sceneinventory/models/containers.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index aea94b97ef..ad78061468 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -194,19 +194,15 @@ def __init__(self, controller): self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} - self._container_items_by_project = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} - self._product_ids_by_project = {} def reset(self): self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} - self._container_items_by_project = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} - self._product_ids_by_project = {} def get_containers(self): self._update_cache() From ef26dc2dc2cb50957710e25c2e7cc9aacad8d7da Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:58:02 +0100 Subject: [PATCH 053/136] added empty lines for readability --- client/ayon_core/tools/sceneinventory/models/containers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index ad78061468..b1cbb38587 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -294,10 +294,12 @@ def get_version_items(self, project_name, product_ids): def version_sorted(entity): return entity["version"] + version_entities_by_product_id = { product_id: [] for product_id in missing_ids } + version_entities = list(ayon_api.get_versions( project_name, product_ids=missing_ids, @@ -309,6 +311,7 @@ def version_sorted(entity): version_entities_by_product_id[product_id].append( version_entity ) + for product_id, version_entities in ( version_entities_by_product_id.items() ): @@ -334,6 +337,7 @@ def version_sorted(entity): self._version_items_by_product_id[product_id] = ( version_items_by_id ) + return { product_id: dict(self._version_items_by_product_id[product_id]) for product_id in product_ids From 7935ed3284fca091eed45eb13acfd3ea456e0145 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:58:23 +0100 Subject: [PATCH 054/136] site sync expects project name --- client/ayon_core/tools/sceneinventory/control.py | 14 ++++++++++---- .../tools/sceneinventory/models/sitesync.py | 10 +++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 8c02881b82..12dfc72e77 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -124,14 +124,20 @@ def get_sites_information(self): def get_site_provider_icons(self): return self._sitesync_model.get_site_provider_icons() - def get_representations_site_progress(self, representation_ids): + def get_representations_site_progress( + self, project_name, representation_ids + ): return self._sitesync_model.get_representations_site_progress( - representation_ids + project_name, representation_ids ) - def resync_representations(self, representation_ids, site_type): + def resync_representations( + self, project_name, representation_ids, site_type + ): return self._sitesync_model.resync_representations( - representation_ids, site_type + project_name, + representation_ids, + site_type ) # Switch dialog methods diff --git a/client/ayon_core/tools/sceneinventory/models/sitesync.py b/client/ayon_core/tools/sceneinventory/models/sitesync.py index 1a1f08bf02..1738ec2c15 100644 --- a/client/ayon_core/tools/sceneinventory/models/sitesync.py +++ b/client/ayon_core/tools/sceneinventory/models/sitesync.py @@ -54,7 +54,9 @@ def get_sites_information(self): "remote_site_provider": self._get_remote_site_provider() } - def get_representations_site_progress(self, representation_ids): + def get_representations_site_progress( + self, project_name, representation_ids + ): """Get progress of representations sync.""" representation_ids = set(representation_ids) @@ -68,7 +70,6 @@ def get_representations_site_progress(self, representation_ids): if not self.is_sitesync_enabled(): return output - project_name = self._controller.get_current_project_name() sitesync_addon = self._get_sitesync_addon() repre_entities = ayon_api.get_representations( project_name, representation_ids @@ -86,10 +87,13 @@ def get_representations_site_progress(self, representation_ids): return output - def resync_representations(self, representation_ids, site_type): + def resync_representations( + self, project_name, representation_ids, site_type + ): """ Args: + project_name (str): Project name. representation_ids (Iterable[str]): Representation ids. site_type (Literal[active_site, remote_site]): Site type. """ From b25715ee2bfc801486da97259890eb6425e1e35d Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:59:47 +0100 Subject: [PATCH 055/136] use project name in site sync calls --- .../ayon_core/tools/sceneinventory/model.py | 13 +++++---- .../tools/sceneinventory/models/sitesync.py | 2 +- client/ayon_core/tools/sceneinventory/view.py | 29 +++++++++++++------ 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 849d8b8d17..cf9814a8e1 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -132,7 +132,6 @@ def refresh(self, selected=None): # for debugging or testing, injecting items from outside container_items = self._controller.get_container_items() self._clear_items() - repre_ids = set() repre_ids_by_project = collections.defaultdict(set) version_items_by_product_id = collections.defaultdict(dict) repre_info_by_id_by_project = collections.defaultdict(dict) @@ -146,7 +145,6 @@ def refresh(self, selected=None): # continue project_name = container_item.project_name representation_id = container_item.representation_id - repre_ids.add(representation_id) repre_ids_by_project[project_name].add(representation_id) item_by_repre_id_by_project_id[project_name][representation_id].add(container_item) @@ -167,9 +165,13 @@ def refresh(self, selected=None): version_items_by_product_id[project_name] = version_items # SiteSync addon information - progress_by_id = self._controller.get_representations_site_progress( - repre_ids - ) + progress_by_project = {} + for project_name, repre_ids in repre_ids_by_project.items(): + progress_by_id = self._controller.get_representations_site_progress( + project_name, repre_ids + ) + progress_by_project[project_name] = progress_by_id + sites_info = self._controller.get_sites_information() site_icons = { provider: get_qt_icon(icon_def) @@ -207,6 +209,7 @@ def refresh(self, selected=None): root_item = self.invisibleRootItem() group_items = [] for project_name, items_by_repre_id in item_by_repre_id_by_project_id.items(): + progress_by_id = progress_by_project[project_name] for repre_id, container_items in items_by_repre_id.items(): repre_info = repre_info_by_id_by_project[project_name][repre_id] version_color = None diff --git a/client/ayon_core/tools/sceneinventory/models/sitesync.py b/client/ayon_core/tools/sceneinventory/models/sitesync.py index 1738ec2c15..c8e1ac2cd3 100644 --- a/client/ayon_core/tools/sceneinventory/models/sitesync.py +++ b/client/ayon_core/tools/sceneinventory/models/sitesync.py @@ -103,7 +103,7 @@ def resync_representations( active_site = self._get_active_site() remote_site = self._get_remote_site() progress = self.get_representations_site_progress( - representation_ids + project_name, representation_ids ) for repre_id in representation_ids: repre_progress = progress.get(repre_id) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index d4381f55cd..9bd2d65cd0 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -413,12 +413,13 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): self._handle_sitesync(menu, valid_repre_ids) - def _handle_sitesync(self, menu, repre_ids): + def _handle_sitesync(self, menu, repre_ids_by_project_name): """Adds actions for download/upload when SyncServer is enabled Args: menu (OptionMenu) - repre_ids (list) of object_ids + repre_ids_by_project_name (Dict[str, Set[str]]): Representation + ids by project name. Returns: (OptionMenu) @@ -427,7 +428,7 @@ def _handle_sitesync(self, menu, repre_ids): if not self._controller.is_sitesync_enabled(): return - if not repre_ids: + if not repre_ids_by_project_name: return menu.addSeparator() @@ -439,7 +440,10 @@ def _handle_sitesync(self, menu, repre_ids): menu ) download_active_action.triggered.connect( - lambda: self._add_sites(repre_ids, "active_site")) + lambda: self._add_sites( + repre_ids_by_project_name, "active_site" + ) + ) upload_icon = qtawesome.icon("fa.upload", color=DEFAULT_COLOR) upload_remote_action = QtWidgets.QAction( @@ -448,23 +452,30 @@ def _handle_sitesync(self, menu, repre_ids): menu ) upload_remote_action.triggered.connect( - lambda: self._add_sites(repre_ids, "remote_site")) + lambda: self._add_sites( + repre_ids_by_project_name, "remote_site" + ) + ) menu.addAction(download_active_action) menu.addAction(upload_remote_action) - def _add_sites(self, repre_ids, site_type): + def _add_sites(self, repre_ids_by_project_name, site_type): """(Re)sync all 'repre_ids' to specific site. It checks if opposite site has fully available content to limit accidents. (ReSync active when no remote >> losing active content) Args: - repre_ids (list) + repre_ids_by_project_name (Dict[str, Set[str]]): Representation + ids by project name. site_type (Literal[active_site, remote_site]): Site type. - """ - self._controller.resync_representations(repre_ids, site_type) + """ + for project_name, repre_ids in repre_ids_by_project_name.items(): + self._controller.resync_representations( + project_name, repre_ids, site_type + ) self.data_changed.emit() From a30698eb4b12381ec7c01f7ca1c98ae0e12fab3c Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:02:49 +0100 Subject: [PATCH 056/136] refactor view codebase --- client/ayon_core/tools/sceneinventory/view.py | 105 ++++++++++-------- 1 file changed, 60 insertions(+), 45 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 9bd2d65cd0..b3322ffc60 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -198,34 +198,41 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) - repre_info_by_project = collections.defaultdict(dict) + repre_info_by_project = {} + repre_ids_by_project_name = {} + version_ids_by_project = {} + product_ids_by_project = {} for project_name, repre_ids in repre_ids_by_project.items(): - repre_info = self._controller.get_representation_info_items( - project_name, repre_ids) - repre_info_by_project[project_name].update(repre_info) + repres_info = self._controller.get_representation_info_items( + project_name, repre_ids + ) + + repre_info_by_project[project_name] = repres_info + repre_ids = set() + version_ids = set() + product_ids = set() + for repre_id, repre_info in repres_info.items(): + if not repre_info.is_valid: + continue + repre_ids.add(repre_id) + version_ids.add(repre_info.version_id) + product_ids.add(repre_info.product_id) + + repre_ids_by_project_name[project_name] = repre_ids + version_ids_by_project[project_name] = version_ids + product_ids_by_project[project_name] = product_ids + # Exclude items that are "NOT FOUND" since setting versions, updating # and removal won't work for those items. filtered_items = [] - version_ids = set() - valid_repre_ids = set() - product_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): project_name = container_item.project_name - repre_info_by_id = repre_info_by_project.get(project_name) repre_id = container_item.representation_id - all_valid_repre_ids = { - repre_id - for repre_id, repre_info in repre_info_by_id.items() - if repre_info.is_valid - } - valid_repre_ids.update(all_valid_repre_ids) - + repre_info_by_id = repre_info_by_project.get(project_name, {}) repre_info = repre_info_by_id.get(repre_id) if repre_info and repre_info.is_valid: filtered_items.append(container_item) - version_ids.add(repre_info.version_id) - product_id = repre_info.product_id - product_ids_by_project[project_name].add(product_id) + # remove remove_icon = qtawesome.icon("fa.remove", color=DEFAULT_COLOR) remove_action = QtWidgets.QAction(remove_icon, "Remove items", menu) @@ -236,18 +243,23 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): # Keep remove action for invalid items menu.addAction(remove_action) return - version_items_by_product_id_by_project = collections.defaultdict(dict) - for project_name, product_ids in product_ids_by_project.items(): - version_items = self._controller.get_version_items( + + version_items_by_project = { + project_name: self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id_by_project[project_name] = version_items + for project_name, product_ids in product_ids_by_project.items() + } + has_outdated = False has_loaded_hero_versions = False has_available_hero_version = False has_outdated_approved = False last_version_by_product_id = {} - for version_items_by_product_id in version_items_by_product_id_by_project.values(): + for project_name, version_items_by_product_id in ( + version_items_by_project.items() + ): + version_ids = version_ids_by_project[project_name] for product_id, version_items_by_id in ( version_items_by_product_id.items() ): @@ -411,7 +423,7 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index): menu.addAction(remove_action) - self._handle_sitesync(menu, valid_repre_ids) + self._handle_sitesync(menu, repre_ids_by_project_name) def _handle_sitesync(self, menu, repre_ids_by_project_name): """Adds actions for download/upload when SyncServer is enabled @@ -979,44 +991,47 @@ def _on_switch_to_versioned(self, item_ids): repre_ids_by_project[project_name].add(repre_id) # Get representation info items by ID - repre_info_by_project = collections.defaultdict(dict) + repres_info_by_project = {} + version_items_by_project = {} for project_name, repre_ids in repre_ids_by_project.items(): - repre_info = self._controller.get_representation_info_items( - project_name, repre_ids) - repre_info_by_project[project_name].update(repre_info) + repre_info_by_id = self._controller.get_representation_info_items( + project_name, repre_ids + ) + repres_info_by_project[project_name] = repre_info_by_id - version_items_by_product_id_by_project = collections.defaultdict(dict) - for container_item in containers_items_by_id.values(): - project_name = container_item.project_name - repre_info_by_id = repre_info_by_project.get(project_name) - repre_id = container_item.representation_id - repre_info = repre_info_by_id.get(repre_id) product_ids = { repre_info.product_id for repre_info in repre_info_by_id.values() if repre_info.is_valid } - version_items = self._controller.get_version_items( + version_items_by_product_id = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id_by_project[project_name] = version_items + version_items_by_project[project_name] = ( + version_items_by_product_id + ) update_containers = [] update_versions = [] - for item_id, container_item in containers_items_by_id.items(): - repre_id = container_item.representation_id + for container_item in containers_items_by_id.values(): project_name = container_item.project_name + repre_id = container_item.representation_id + + repre_info_by_id = repres_info_by_project[project_name] repre_info = repre_info_by_id[repre_id] - product_id = repre_info.product_id - version_items_id = ( - version_items_by_product_id_by_project[project_name][product_id] + + version_items_by_product_id = ( + version_items_by_project[project_name] ) - version_item = version_items_id.get(repre_info.version_id, {}) + product_id = repre_info.product_id + version_items_by_id = version_items_by_product_id[product_id] + version_item = version_items_by_id.get(repre_info.version_id, {}) if not version_item or not version_item.is_hero: continue + version = abs(version_item.version) version_found = False - for version_item in version_items_id.values(): + for version_item in version_items_by_id.values(): if version_item.is_hero: continue if version_item.version == version: @@ -1029,8 +1044,8 @@ def _on_switch_to_versioned(self, item_ids): update_containers.append(container_item.item_id) update_versions.append(version) - # Specify version per item to update to - self._update_containers(update_containers, update_versions) + # Specify version per item to update to + self._update_containers(update_containers, update_versions) def _update_containers(self, item_ids, versions): """Helper to update items to given version (or version per item) From 562f2edacee460709963213c1aeae155416c097f Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:22:27 +0100 Subject: [PATCH 057/136] site sync is fully project specific --- .../ayon_core/tools/sceneinventory/control.py | 4 +- .../ayon_core/tools/sceneinventory/model.py | 52 ++++--- .../tools/sceneinventory/models/sitesync.py | 129 +++++++++--------- 3 files changed, 102 insertions(+), 83 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 12dfc72e77..4f23b8e942 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -118,8 +118,8 @@ def get_version_items(self, project_name, product_ids): def is_sitesync_enabled(self): return self._sitesync_model.is_sitesync_enabled() - def get_sites_information(self): - return self._sitesync_model.get_sites_information() + def get_sites_information(self, project_name): + return self._sitesync_model.get_sites_information(project_name) def get_site_provider_icons(self): return self._sitesync_model.get_site_provider_icons() diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index cf9814a8e1..95272470aa 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -133,10 +133,10 @@ def refresh(self, selected=None): container_items = self._controller.get_container_items() self._clear_items() repre_ids_by_project = collections.defaultdict(set) - version_items_by_product_id = collections.defaultdict(dict) + version_items_by_project = collections.defaultdict(dict) repre_info_by_id_by_project = collections.defaultdict(dict) - item_by_repre_id_by_project_id = collections.defaultdict( - lambda: collections.defaultdict(set)) + item_by_repre_id_by_project = collections.defaultdict( + lambda: collections.defaultdict(list)) for container_item in container_items: # if ( # selected is not None @@ -146,7 +146,11 @@ def refresh(self, selected=None): project_name = container_item.project_name representation_id = container_item.representation_id repre_ids_by_project[project_name].add(representation_id) - item_by_repre_id_by_project_id[project_name][representation_id].add(container_item) + ( + item_by_repre_id_by_project + [project_name] + [representation_id] + ).append(container_item) for project_name, representation_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( @@ -162,17 +166,20 @@ def refresh(self, selected=None): version_items = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id[project_name] = version_items + version_items_by_project[project_name] = version_items # SiteSync addon information - progress_by_project = {} - for project_name, repre_ids in repre_ids_by_project.items(): - progress_by_id = self._controller.get_representations_site_progress( + progress_by_project = { + project_name: self._controller.get_representations_site_progress( project_name, repre_ids ) - progress_by_project[project_name] = progress_by_id + for project_name, repre_ids in repre_ids_by_project.items() + } - sites_info = self._controller.get_sites_information() + sites_info_by_project_name = { + project_name: self._controller.get_sites_information(project_name) + for project_name in repre_ids_by_project.keys() + } site_icons = { provider: get_qt_icon(icon_def) for provider, icon_def in ( @@ -203,15 +210,26 @@ def refresh(self, selected=None): group_item_font = QtGui.QFont() group_item_font.setBold(True) - active_site_icon = site_icons.get(sites_info["active_site_provider"]) - remote_site_icon = site_icons.get(sites_info["remote_site_provider"]) - root_item = self.invisibleRootItem() group_items = [] - for project_name, items_by_repre_id in item_by_repre_id_by_project_id.items(): + for project_name, items_by_repre_id in ( + item_by_repre_id_by_project.items() + ): + sites_info = sites_info_by_project_name[project_name] + active_site_icon = site_icons.get( + sites_info["active_site_provider"] + ) + remote_site_icon = site_icons.get( + sites_info["remote_site_provider"] + ) + progress_by_id = progress_by_project[project_name] + repre_info_by_id = repre_info_by_id_by_project[project_name] + version_items_by_product_id = ( + version_items_by_project[project_name] + ) for repre_id, container_items in items_by_repre_id.items(): - repre_info = repre_info_by_id_by_project[project_name][repre_id] + repre_info = repre_info_by_id[repre_id] version_color = None if not repre_info.is_valid: version_label = "N/A" @@ -230,7 +248,7 @@ def refresh(self, selected=None): item_icon = valid_item_icon version_items = ( - version_items_by_product_id[project_name][repre_info.product_id] + version_items_by_product_id[repre_info.product_id] ) version_item = version_items[repre_info.version_id] version_label = format_version(version_item.version) @@ -266,8 +284,6 @@ def refresh(self, selected=None): item.setData(True, IS_CONTAINER_ITEM_ROLE) item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) container_model_items.append(item) - if not container_model_items: - continue progress = progress_by_id[repre_id] active_site_progress = "{}%".format( diff --git a/client/ayon_core/tools/sceneinventory/models/sitesync.py b/client/ayon_core/tools/sceneinventory/models/sitesync.py index c8e1ac2cd3..546d2b15c0 100644 --- a/client/ayon_core/tools/sceneinventory/models/sitesync.py +++ b/client/ayon_core/tools/sceneinventory/models/sitesync.py @@ -11,18 +11,18 @@ def __init__(self, controller): self._sitesync_addon = NOT_SET self._sitesync_enabled = None - self._active_site = NOT_SET - self._remote_site = NOT_SET - self._active_site_provider = NOT_SET - self._remote_site_provider = NOT_SET + self._active_site = {} + self._remote_site = {} + self._active_site_provider = {} + self._remote_site_provider = {} def reset(self): self._sitesync_addon = NOT_SET self._sitesync_enabled = None - self._active_site = NOT_SET - self._remote_site = NOT_SET - self._active_site_provider = NOT_SET - self._remote_site_provider = NOT_SET + self._active_site = {} + self._remote_site = {} + self._active_site_provider = {} + self._remote_site_provider = {} def is_sitesync_enabled(self): """Site sync is enabled. @@ -46,12 +46,16 @@ def get_site_provider_icons(self): sitesync_addon = self._get_sitesync_addon() return sitesync_addon.get_site_icons() - def get_sites_information(self): + def get_sites_information(self, project_name): return { - "active_site": self._get_active_site(), - "active_site_provider": self._get_active_site_provider(), - "remote_site": self._get_remote_site(), - "remote_site_provider": self._get_remote_site_provider() + "active_site": self._get_active_site(project_name), + "remote_site": self._get_remote_site(project_name), + "active_site_provider": self._get_active_site_provider( + project_name + ), + "remote_site_provider": self._get_remote_site_provider( + project_name + ) } def get_representations_site_progress( @@ -74,8 +78,8 @@ def get_representations_site_progress( repre_entities = ayon_api.get_representations( project_name, representation_ids ) - active_site = self._get_active_site() - remote_site = self._get_remote_site() + active_site = self._get_active_site(project_name) + remote_site = self._get_remote_site(project_name) for repre_entity in repre_entities: repre_output = output[repre_entity["id"]] @@ -97,11 +101,9 @@ def resync_representations( representation_ids (Iterable[str]): Representation ids. site_type (Literal[active_site, remote_site]): Site type. """ - - project_name = self._controller.get_current_project_name() sitesync_addon = self._get_sitesync_addon() - active_site = self._get_active_site() - remote_site = self._get_remote_site() + active_site = self._get_active_site(project_name) + remote_site = self._get_remote_site(project_name) progress = self.get_representations_site_progress( project_name, representation_ids ) @@ -136,48 +138,49 @@ def _cache_sitesync_addon(self): self._sitesync_addon = sitesync_addon self._sitesync_enabled = sync_enabled - def _get_active_site(self): - if self._active_site is NOT_SET: - self._cache_sites() - return self._active_site - - def _get_remote_site(self): - if self._remote_site is NOT_SET: - self._cache_sites() - return self._remote_site - - def _get_active_site_provider(self): - if self._active_site_provider is NOT_SET: - self._cache_sites() - return self._active_site_provider - - def _get_remote_site_provider(self): - if self._remote_site_provider is NOT_SET: - self._cache_sites() - return self._remote_site_provider - - def _cache_sites(self): - active_site = None - remote_site = None - active_site_provider = None - remote_site_provider = None - if self.is_sitesync_enabled(): - sitesync_addon = self._get_sitesync_addon() - project_name = self._controller.get_current_project_name() - active_site = sitesync_addon.get_active_site(project_name) - remote_site = sitesync_addon.get_remote_site(project_name) - active_site_provider = "studio" - remote_site_provider = "studio" - if active_site != "studio": - active_site_provider = sitesync_addon.get_provider_for_site( - project_name, active_site - ) - if remote_site != "studio": - remote_site_provider = sitesync_addon.get_provider_for_site( - project_name, remote_site - ) + def _get_active_site(self, project_name): + if project_name not in self._active_site: + self._cache_sites(project_name) + return self._active_site[project_name] + + def _get_remote_site(self, project_name): + if project_name not in self._remote_site: + self._cache_sites(project_name) + return self._remote_site[project_name] + + def _get_active_site_provider(self, project_name): + if project_name not in self._active_site_provider: + self._cache_sites(project_name) + return self._active_site_provider[project_name] + + def _get_remote_site_provider(self, project_name): + if project_name not in self._remote_site_provider: + self._cache_sites(project_name) + return self._remote_site_provider[project_name] + + def _cache_sites(self, project_name): + self._active_site[project_name] = None + self._remote_site[project_name] = None + self._active_site_provider[project_name] = None + self._remote_site_provider[project_name] = None + if not self.is_sitesync_enabled(): + return + + sitesync_addon = self._get_sitesync_addon() + active_site = sitesync_addon.get_active_site(project_name) + remote_site = sitesync_addon.get_remote_site(project_name) + active_site_provider = "studio" + remote_site_provider = "studio" + if active_site != "studio": + active_site_provider = sitesync_addon.get_provider_for_site( + project_name, active_site + ) + if remote_site != "studio": + remote_site_provider = sitesync_addon.get_provider_for_site( + project_name, remote_site + ) - self._active_site = active_site - self._remote_site = remote_site - self._active_site_provider = active_site_provider - self._remote_site_provider = remote_site_provider + self._active_site[project_name] = active_site + self._remote_site[project_name] = remote_site + self._active_site_provider[project_name] = active_site_provider + self._remote_site_provider[project_name] = remote_site_provider From eea31b676d91c7ae746f7f1d2417737273c6cb05 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:25:46 +0100 Subject: [PATCH 058/136] don't slow down project name getter --- client/ayon_core/pipeline/load/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/load/utils.py b/client/ayon_core/pipeline/load/utils.py index a6c5f0ce1f..de8e1676e7 100644 --- a/client/ayon_core/pipeline/load/utils.py +++ b/client/ayon_core/pipeline/load/utils.py @@ -465,7 +465,9 @@ def update_container(container, version=-1): from ayon_core.pipeline import get_current_project_name # Compute the different version from 'representation' - project_name = container.get("project_name", get_current_project_name()) + project_name = container.get("project_name") + if project_name is None: + project_name = get_current_project_name() repre_id = container["representation"] if not _is_valid_representation_id(repre_id): raise ValueError( @@ -588,7 +590,9 @@ def switch_container(container, representation, loader_plugin=None): ) # Get the new representation to switch to - project_name = container.get("project_name", get_current_project_name()) + project_name = container.get("project_name") + if project_name is None: + project_name = get_current_project_name() context = get_representation_context( project_name, representation["id"] From e21c7a157e5009e521d804fb0621bf55d750cba2 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:46:56 +0100 Subject: [PATCH 059/136] use project name to get correct status icons --- .../ayon_core/tools/sceneinventory/control.py | 5 +- .../ayon_core/tools/sceneinventory/model.py | 48 ++++++++++++------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 4f23b8e942..60d9bc77a9 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -86,8 +86,9 @@ def get_current_folder_id(self): self._current_folder_set = True return self._current_folder_id - def get_project_status_items(self): - project_name = self.get_current_project_name() + def get_project_status_items(self, project_name=None): + if project_name is None: + project_name = self.get_current_project_name() return self._projects_model.get_project_status_items( project_name, None ) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 95272470aa..79af0e5cf5 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -121,8 +121,8 @@ def __init__(self, controller, parent=None): self._default_icon_color = get_default_entity_icon_color() - self._last_project_statuses = {} - self._last_status_icons_by_name = {} + self._last_project_statuses = collections.defaultdict(dict) + self._last_status_icons_by_name = collections.defaultdict(dict) def outdated(self, item): return item.get("isOutdated", True) @@ -131,7 +131,10 @@ def refresh(self, selected=None): """Refresh the model""" # for debugging or testing, injecting items from outside container_items = self._controller.get_container_items() + self._clear_items() + + project_names = set() repre_ids_by_project = collections.defaultdict(set) version_items_by_project = collections.defaultdict(dict) repre_info_by_id_by_project = collections.defaultdict(dict) @@ -145,6 +148,7 @@ def refresh(self, selected=None): # continue project_name = container_item.project_name representation_id = container_item.representation_id + project_names.add(project_name) repre_ids_by_project[project_name].add(representation_id) ( item_by_repre_id_by_project @@ -178,7 +182,7 @@ def refresh(self, selected=None): sites_info_by_project_name = { project_name: self._controller.get_sites_information(project_name) - for project_name in repre_ids_by_project.keys() + for project_name in project_names } site_icons = { provider: get_qt_icon(icon_def) @@ -186,11 +190,17 @@ def refresh(self, selected=None): self._controller.get_site_provider_icons().items() ) } - self._last_project_statuses = { - status_item.name: status_item - for status_item in self._controller.get_project_status_items() - } - self._last_status_icons_by_name = {} + last_project_statuses = collections.defaultdict(dict) + for project_name in project_names: + status_items_by_name = { + status_item.name: status_item + for status_item in self._controller.get_project_status_items( + project_name + ) + } + last_project_statuses[project_name] = status_items_by_name + self._last_project_statuses = last_project_statuses + self._last_status_icons_by_name = collections.defaultdict(dict) group_item_icon = qtawesome.icon( "fa.folder", color=self._default_icon_color @@ -258,9 +268,9 @@ def refresh(self, selected=None): version_color = self.OUTDATED_COLOR status_name = version_item.status - status_color, status_short, status_icon = self._get_status_data( - status_name - ) + ( + status_color, status_short, status_icon + ) = self._get_status_data(project_name, status_name) repre_name = ( repre_info.representation_name or "" @@ -392,17 +402,21 @@ def _clear_items(self): root_item = self.invisibleRootItem() root_item.removeRows(0, root_item.rowCount()) - def _get_status_data(self, status_name): - status_item = self._last_project_statuses.get(status_name) - status_icon = self._get_status_icon(status_name, status_item) + def _get_status_data(self, project_name, status_name): + status_item = self._last_project_statuses[project_name].get( + status_name + ) + status_icon = self._get_status_icon( + project_name, status_name, status_item + ) status_color = status_short = None if status_item is not None: status_color = status_item.color status_short = status_item.short return status_color, status_short, status_icon - def _get_status_icon(self, status_name, status_item): - icon = self._last_status_icons_by_name.get(status_name) + def _get_status_icon(self, project_name, status_name, status_item): + icon = self._last_status_icons_by_name[project_name].get(status_name) if icon is not None: return icon @@ -415,7 +429,7 @@ def _get_status_icon(self, status_name, status_item): }) if icon is None: icon = QtGui.QIcon() - self._last_status_icons_by_name[status_name] = icon + self._last_status_icons_by_name[project_name][status_name] = icon return icon From 87907b550b1410a6b6912f611287123e310e7f33 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:05:29 +0100 Subject: [PATCH 060/136] fix switch version --- client/ayon_core/tools/sceneinventory/view.py | 92 +++++++++++-------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index b3322ffc60..9112db2ef3 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -772,60 +772,71 @@ def _show_version_dialog(self, item_ids, active_repre_id): container_items_by_id = self._controller.get_container_items_by_id( item_ids ) + project_names = set() repre_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): repre_id = container_item.representation_id project_name = container_item.project_name + project_names.add(project_name) repre_ids_by_project[project_name].add(repre_id) - version_ids = set() - repre_info_by_project = collections.defaultdict(dict) - version_items_by_product_id_by_project = collections.defaultdict(dict) + active_project_name = None + active_repre_info = None + repre_info_by_project = {} + version_items_by_project = {} for project_name, repre_ids in repre_ids_by_project.items(): - repre_info = self._controller.get_representation_info_items( + repres_info = self._controller.get_representation_info_items( project_name, repre_ids ) - repre_info_by_project[project_name].update(repre_info) + if active_repre_info is None: + active_project_name = project_name + active_repre_info = repres_info.get(active_repre_id) - for container_item in container_items_by_id.values(): - project_name = container_item.project_name - repre_info_by_id = repre_info_by_project.get(project_name) - repre_id = container_item.representation_id - repre_info = repre_info_by_id.get(repre_id) product_ids = { repre_info.product_id - for repre_info in repre_info_by_id.values() + for repre_info in repres_info.values() if repre_info.is_valid } - version_items = self._controller.get_version_items( + version_items_by_product_id = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id_by_project[project_name] = version_items - active_repre_info = repre_info_by_id[active_repre_id] - active_version_id = active_repre_info.version_id - active_product_id = active_repre_info.product_id - version_items = list( - version_items_by_product_id_by_project[project_name][active_product_id].values() - ) - version_ids.update(version_item.version for version_item in version_items) - product_ids_by_version = collections.defaultdict(set) - for version_items_by_product_id in version_items_by_product_id_by_project.values(): - for version_items_by_id in version_items_by_product_id.values(): - for version_item in version_items_by_id.values(): - version = version_item.version - _prod_version = version - if _prod_version < 0: - _prod_version = -1 - product_ids_by_version[_prod_version].add( - version_item.product_id - ) - if version in version_ids: - continue - version_ids.add(version) - version_items.append(version_item) + repre_info_by_project[project_name] = repres_info + version_items_by_project[project_name] = version_items_by_product_id + + active_version_id = active_repre_info.version_id + active_product_id = active_repre_info.product_id + + versions = set() + product_ids = set() + version_items = [] + product_ids_by_version_by_project = {} + for project_name, version_items_by_product_id in ( + version_items_by_project.items() + ): + product_ids_by_version = collections.defaultdict(set) + product_ids_by_version_by_project[project_name] = ( + product_ids_by_version + ) + versions |= { + version_item.version + for version_item in version_items_by_product_id.values() + } + for version_item in version_items_by_product_id.values(): + version = version_item.version + _prod_version = version + if _prod_version < 0: + _prod_version = -1 + product_ids_by_version[_prod_version].add( + version_item.product_id + ) + product_ids.add(version_item.product_id) + if version in versions: + continue + versions.add(version) + version_items.append((project_name, version_item)) - def version_sorter(item): + def version_sorter(_, item): hero_value = 0 i_version = item.version if i_version < 0: @@ -844,7 +855,8 @@ def version_sorter(item): version_options = [] active_version_idx = 0 - for idx, version_item in enumerate(version_items): + for idx, item in enumerate(version_items): + project_name, version_item = item version = version_item.version label = format_version(version) if version_item.version_id == active_version_id: @@ -884,11 +896,13 @@ def version_sorter(item): product_version = -1 version = HeroVersionType(version) - product_ids = product_ids_by_version[product_version] - filtered_item_ids = set() for container_item in container_items_by_id.values(): project_name = container_item.project_name + product_ids_by_version = ( + product_ids_by_version_by_project[project_name] + ) + product_ids = product_ids_by_version[product_version] repre_id = container_item.representation_id repre_info = repre_info_by_project[project_name][repre_id] if repre_info.product_id in product_ids: From b28f4b0ff1ea9662c169f508302659f459352075 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:10:05 +0100 Subject: [PATCH 061/136] comment out unused variables --- client/ayon_core/tools/sceneinventory/view.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 9112db2ef3..5892e4f983 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -780,7 +780,7 @@ def _show_version_dialog(self, item_ids, active_repre_id): project_names.add(project_name) repre_ids_by_project[project_name].add(repre_id) - active_project_name = None + # active_project_name = None active_repre_info = None repre_info_by_project = {} version_items_by_project = {} @@ -789,7 +789,7 @@ def _show_version_dialog(self, item_ids, active_repre_id): project_name, repre_ids ) if active_repre_info is None: - active_project_name = project_name + # active_project_name = project_name active_repre_info = repres_info.get(active_repre_id) product_ids = { @@ -805,7 +805,7 @@ def _show_version_dialog(self, item_ids, active_repre_id): version_items_by_project[project_name] = version_items_by_product_id active_version_id = active_repre_info.version_id - active_product_id = active_repre_info.product_id + # active_product_id = active_repre_info.product_id versions = set() product_ids = set() From 207b1961a441b4d13df7eac7e80faf39b4468ace Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:35:21 +0100 Subject: [PATCH 062/136] support all errors in ruff linter --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f2d09d925d..d09fabf8b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ target-version = "py39" [tool.ruff.lint] # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. -select = ["E4", "E7", "E9", "F", "W"] +select = ["E", "F", "W"] ignore = [] # Allow fix for all enabled rules (when `--fix`) is provided. From d30d5b541994a2243d41252a44459d39d0696123 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:37:05 +0100 Subject: [PATCH 063/136] fix line lengths --- client/ayon_core/addon/base.py | 4 +-- client/ayon_core/cli.py | 3 +- client/ayon_core/lib/attribute_definitions.py | 4 ++- client/ayon_core/pipeline/create/context.py | 14 ++++++--- .../ayon_core/pipeline/create/product_name.py | 6 +++- client/ayon_core/pipeline/editorial.py | 13 ++++++-- .../pipeline/farm/pyblish_functions.py | 20 +++++++++--- .../plugins/publish/collect_hierarchy.py | 3 +- .../plugins/publish/extract_otio_review.py | 31 ++++++++++++------- .../publish/validate_unique_subsets.py | 10 +++--- client/ayon_core/scripts/otio_burnin.py | 17 ++++++---- client/ayon_core/settings/lib.py | 3 +- client/ayon_core/tools/creator/widgets.py | 4 ++- .../tools/launcher/models/actions.py | 6 ++-- .../tools/loader/ui/_multicombobox.py | 6 +++- .../tools/loader/ui/products_model.py | 6 ++-- .../publisher/widgets/overview_widget.py | 4 ++- .../publisher/widgets/product_context.py | 6 ++-- .../tools/publisher/widgets/tasks_model.py | 4 +-- client/ayon_core/tools/publisher/window.py | 6 +++- .../tools/sceneinventory/models/containers.py | 4 +-- client/ayon_core/tools/utils/lib.py | 7 +++-- server/settings/publish_plugins.py | 21 +++++++++---- server/settings/tools.py | 8 +++-- .../editorial/test_extract_otio_review.py | 31 +++++++++++-------- 25 files changed, 159 insertions(+), 82 deletions(-) diff --git a/client/ayon_core/addon/base.py b/client/ayon_core/addon/base.py index 982626ad9d..364a84cb7b 100644 --- a/client/ayon_core/addon/base.py +++ b/client/ayon_core/addon/base.py @@ -535,8 +535,8 @@ def ensure_is_process_ready( Implementation of this method is optional. Note: - The logic can be similar to logic in tray, but tray does not require - to be logged in. + The logic can be similar to logic in tray, but tray does not + require to be logged in. Args: process_context (ProcessContext): Context of child diff --git a/client/ayon_core/cli.py b/client/ayon_core/cli.py index b80b243db2..6b4a1f824f 100644 --- a/client/ayon_core/cli.py +++ b/client/ayon_core/cli.py @@ -146,7 +146,8 @@ def publish_report_viewer(): @main_cli.command() @click.argument("output_path") @click.option("--project", help="Define project context") -@click.option("--folder", help="Define folder in project (project must be set)") +@click.option( + "--folder", help="Define folder in project (project must be set)") @click.option( "--strict", is_flag=True, diff --git a/client/ayon_core/lib/attribute_definitions.py b/client/ayon_core/lib/attribute_definitions.py index e1381944f6..e8327a45b6 100644 --- a/client/ayon_core/lib/attribute_definitions.py +++ b/client/ayon_core/lib/attribute_definitions.py @@ -616,7 +616,9 @@ def serialize(self): return data @staticmethod - def prepare_enum_items(items: "EnumItemsInputType") -> List["EnumItemDict"]: + def prepare_enum_items( + items: "EnumItemsInputType" + ) -> List["EnumItemDict"]: """Convert items to unified structure. Output is a list where each item is dictionary with 'value' diff --git a/client/ayon_core/pipeline/create/context.py b/client/ayon_core/pipeline/create/context.py index 6bfd64b822..e29971415d 100644 --- a/client/ayon_core/pipeline/create/context.py +++ b/client/ayon_core/pipeline/create/context.py @@ -1283,12 +1283,16 @@ def bulk_value_changes(self, sender=None): @contextmanager def bulk_pre_create_attr_defs_change(self, sender=None): - with self._bulk_context("pre_create_attrs_change", sender) as bulk_info: + with self._bulk_context( + "pre_create_attrs_change", sender + ) as bulk_info: yield bulk_info @contextmanager def bulk_create_attr_defs_change(self, sender=None): - with self._bulk_context("create_attrs_change", sender) as bulk_info: + with self._bulk_context( + "create_attrs_change", sender + ) as bulk_info: yield bulk_info @contextmanager @@ -1946,9 +1950,9 @@ def remove_instances(self, instances, sender=None): creator are just removed from context. Args: - instances (List[CreatedInstance]): Instances that should be removed. - Remove logic is done using creator, which may require to - do other cleanup than just remove instance from context. + instances (List[CreatedInstance]): Instances that should be + removed. Remove logic is done using creator, which may require + to do other cleanup than just remove instance from context. sender (Optional[str]): Sender of the event. """ diff --git a/client/ayon_core/pipeline/create/product_name.py b/client/ayon_core/pipeline/create/product_name.py index eaeef6500e..0daec8a7ad 100644 --- a/client/ayon_core/pipeline/create/product_name.py +++ b/client/ayon_core/pipeline/create/product_name.py @@ -1,5 +1,9 @@ import ayon_api -from ayon_core.lib import StringTemplate, filter_profiles, prepare_template_data +from ayon_core.lib import ( + StringTemplate, + filter_profiles, + prepare_template_data, +) from ayon_core.settings import get_project_settings from .constants import DEFAULT_PRODUCT_TEMPLATE diff --git a/client/ayon_core/pipeline/editorial.py b/client/ayon_core/pipeline/editorial.py index a49a981d2a..2928ef5f63 100644 --- a/client/ayon_core/pipeline/editorial.py +++ b/client/ayon_core/pipeline/editorial.py @@ -222,6 +222,9 @@ def remap_range_on_file_sequence(otio_clip, in_out_range): source_range = otio_clip.source_range available_range_rate = available_range.start_time.rate media_in = available_range.start_time.value + available_range_start_frame = ( + available_range.start_time.to_frames() + ) # Temporary. # Some AYON custom OTIO exporter were implemented with relative @@ -230,7 +233,7 @@ def remap_range_on_file_sequence(otio_clip, in_out_range): # while we are updating those. if ( is_clip_from_media_sequence(otio_clip) - and otio_clip.available_range().start_time.to_frames() == media_ref.start_frame + and available_range_start_frame == media_ref.start_frame and source_range.start_time.to_frames() < media_ref.start_frame ): media_in = 0 @@ -303,8 +306,12 @@ def get_media_range_with_retimes(otio_clip, handle_start, handle_end): rounded_av_rate = round(available_range_rate, 2) rounded_src_rate = round(source_range.start_time.rate, 2) if rounded_av_rate != rounded_src_rate: - conformed_src_in = source_range.start_time.rescaled_to(available_range_rate) - conformed_src_duration = source_range.duration.rescaled_to(available_range_rate) + conformed_src_in = source_range.start_time.rescaled_to( + available_range_rate + ) + conformed_src_duration = source_range.duration.rescaled_to( + available_range_rate + ) conformed_source_range = otio.opentime.TimeRange( start_time=conformed_src_in, duration=conformed_src_duration diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 16364a17ee..559561c827 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -8,7 +8,10 @@ import ayon_api import clique from ayon_core.lib import Logger, collect_frames -from ayon_core.pipeline import get_current_project_name, get_representation_path +from ayon_core.pipeline import ( + get_current_project_name, + get_representation_path, +) from ayon_core.pipeline.create import get_product_name from ayon_core.pipeline.farm.patterning import match_aov_pattern from ayon_core.pipeline.publish import KnownPublishError @@ -771,9 +774,14 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, project_settings = instance.context.data.get("project_settings") - use_legacy_product_name = True try: - use_legacy_product_name = project_settings["core"]["tools"]["creator"]["use_legacy_product_names_for_renders"] # noqa: E501 + use_legacy_product_name = ( + project_settings + ["core"] + ["tools"] + ["creator"] + ["use_legacy_product_names_for_renders"] + ) except KeyError: warnings.warn( ("use_legacy_for_renders not found in project settings. " @@ -789,7 +797,9 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, dynamic_data=dynamic_data) else: - product_name, group_name = get_product_name_and_group_from_template( + ( + product_name, group_name + ) = get_product_name_and_group_from_template( task_entity=instance.data["taskEntity"], project_name=instance.context.data["projectName"], host_name=instance.context.data["hostName"], @@ -932,7 +942,7 @@ def _collect_expected_files_for_aov(files): # but we really expect only one collection. # Nothing else make sense. if len(cols) != 1: - raise ValueError("Only one image sequence type is expected.") # noqa: E501 + raise ValueError("Only one image sequence type is expected.") return list(cols[0]) diff --git a/client/ayon_core/plugins/publish/collect_hierarchy.py b/client/ayon_core/plugins/publish/collect_hierarchy.py index 00f5c06c0b..266c2e1458 100644 --- a/client/ayon_core/plugins/publish/collect_hierarchy.py +++ b/client/ayon_core/plugins/publish/collect_hierarchy.py @@ -43,7 +43,8 @@ def process(self, context): shot_data = { "entity_type": "folder", - # WARNING unless overwritten, default folder type is hardcoded to shot + # WARNING unless overwritten, default folder type is hardcoded + # to shot "folder_type": instance.data.get("folder_type") or "Shot", "tasks": instance.data.get("tasks") or {}, "comments": instance.data.get("comments", []), diff --git a/client/ayon_core/plugins/publish/extract_otio_review.py b/client/ayon_core/plugins/publish/extract_otio_review.py index b222c6efc3..fb9b269258 100644 --- a/client/ayon_core/plugins/publish/extract_otio_review.py +++ b/client/ayon_core/plugins/publish/extract_otio_review.py @@ -129,26 +129,33 @@ def process(self, instance): res_data[key] = value break - self.to_width, self.to_height = res_data["width"], res_data["height"] - self.log.debug("> self.to_width x self.to_height: {} x {}".format( - self.to_width, self.to_height - )) + self.to_width, self.to_height = ( + res_data["width"], res_data["height"] + ) + self.log.debug( + "> self.to_width x self.to_height:" + f" {self.to_width} x {self.to_height}" + ) available_range = r_otio_cl.available_range() + available_range_start_frame = ( + available_range.start_time.to_frames() + ) processing_range = None self.actual_fps = available_range.duration.rate start = src_range.start_time.rescaled_to(self.actual_fps) duration = src_range.duration.rescaled_to(self.actual_fps) + src_frame_start = src_range.start_time.to_frames() # Temporary. - # Some AYON custom OTIO exporter were implemented with relative - # source range for image sequence. Following code maintain - # backward-compatibility by adjusting available range + # Some AYON custom OTIO exporter were implemented with + # relative source range for image sequence. Following code + # maintain backward-compatibility by adjusting available range # while we are updating those. if ( is_clip_from_media_sequence(r_otio_cl) - and available_range.start_time.to_frames() == media_ref.start_frame - and src_range.start_time.to_frames() < media_ref.start_frame + and available_range_start_frame == media_ref.start_frame + and src_frame_start < media_ref.start_frame ): available_range = otio.opentime.TimeRange( otio.opentime.RationalTime(0, rate=self.actual_fps), @@ -246,7 +253,8 @@ def process(self, instance): # Extraction via FFmpeg. else: path = media_ref.target_url - # Set extract range from 0 (FFmpeg ignores embedded timecode). + # Set extract range from 0 (FFmpeg ignores + # embedded timecode). extract_range = otio.opentime.TimeRange( otio.opentime.RationalTime( ( @@ -414,7 +422,8 @@ def _render_segment(self, sequence=None, to defined image sequence format. Args: - sequence (list): input dir path string, collection object, fps in list + sequence (list): input dir path string, collection object, + fps in list. video (list)[optional]: video_path string, otio_range in list gap (int)[optional]: gap duration end_offset (int)[optional]: offset gap frame start in frames diff --git a/client/ayon_core/plugins/publish/validate_unique_subsets.py b/client/ayon_core/plugins/publish/validate_unique_subsets.py index 4badeb8112..4067dd75a5 100644 --- a/client/ayon_core/plugins/publish/validate_unique_subsets.py +++ b/client/ayon_core/plugins/publish/validate_unique_subsets.py @@ -11,8 +11,8 @@ class ValidateProductUniqueness(pyblish.api.ContextPlugin): """Validate all product names are unique. This only validates whether the instances currently set to publish from - the workfile overlap one another for the folder + product they are publishing - to. + the workfile overlap one another for the folder + product they are + publishing to. This does not perform any check against existing publishes in the database since it is allowed to publish into existing products resulting in @@ -72,8 +72,10 @@ def process(self, context): # All is ok return - msg = ("Instance product names {} are not unique. ".format(non_unique) + - "Please remove or rename duplicates.") + msg = ( + f"Instance product names {non_unique} are not unique." + " Please remove or rename duplicates." + ) formatting_data = { "non_unique": ",".join(non_unique) } diff --git a/client/ayon_core/scripts/otio_burnin.py b/client/ayon_core/scripts/otio_burnin.py index 6b132b9a6a..cb72606222 100644 --- a/client/ayon_core/scripts/otio_burnin.py +++ b/client/ayon_core/scripts/otio_burnin.py @@ -79,7 +79,8 @@ class ModifiedBurnins(ffmpeg_burnins.Burnins): - Datatypes explanation: string format must be supported by FFmpeg. Examples: "#000000", "0x000000", "black" - must be accesible by ffmpeg = name of registered Font in system or path to font file. + must be accesible by ffmpeg = name of registered Font in system + or path to font file. Examples: "Arial", "C:/Windows/Fonts/arial.ttf" - Possible keys: @@ -87,17 +88,21 @@ class ModifiedBurnins(ffmpeg_burnins.Burnins): "bg_opacity" - Opacity of background (box around text) - "bg_color" - Background color - "bg_padding" - Background padding in pixels - - "x_offset" - offsets burnin vertically by entered pixels from border - - "y_offset" - offsets burnin horizontally by entered pixels from border - + "x_offset" - offsets burnin vertically by entered pixels + from border - + "y_offset" - offsets burnin horizontally by entered pixels + from border - - x_offset & y_offset should be set at least to same value as bg_padding!! "font" - Font Family for text - "font_size" - Font size in pixels - "font_color" - Color of text - "frame_offset" - Default start frame - - - required IF start frame is not set when using frames or timecode burnins + - required IF start frame is not set when using frames + or timecode burnins - On initializing class can be set General options through "options_init" arg. - General can be overridden when adding burnin + On initializing class can be set General options through + "options_init" arg. + General options can be overridden when adding burnin. ''' TOP_CENTERED = ffmpeg_burnins.TOP_CENTERED diff --git a/client/ayon_core/settings/lib.py b/client/ayon_core/settings/lib.py index 3126bafd57..aa56fa8326 100644 --- a/client/ayon_core/settings/lib.py +++ b/client/ayon_core/settings/lib.py @@ -190,6 +190,7 @@ def get_current_project_settings(): project_name = os.environ.get("AYON_PROJECT_NAME") if not project_name: raise ValueError( - "Missing context project in environemt variable `AYON_PROJECT_NAME`." + "Missing context project in environment" + " variable `AYON_PROJECT_NAME`." ) return get_project_settings(project_name) diff --git a/client/ayon_core/tools/creator/widgets.py b/client/ayon_core/tools/creator/widgets.py index 96ce899881..bbc6848e6c 100644 --- a/client/ayon_core/tools/creator/widgets.py +++ b/client/ayon_core/tools/creator/widgets.py @@ -217,7 +217,9 @@ def __init__(self, parent=None): product_type_label = QtWidgets.QLabel(self) product_type_label.setObjectName("CreatorProductTypeLabel") - product_type_label.setAlignment(QtCore.Qt.AlignBottom | QtCore.Qt.AlignLeft) + product_type_label.setAlignment( + QtCore.Qt.AlignBottom | QtCore.Qt.AlignLeft + ) help_label = QtWidgets.QLabel(self) help_label.setAlignment(QtCore.Qt.AlignTop | QtCore.Qt.AlignLeft) diff --git a/client/ayon_core/tools/launcher/models/actions.py b/client/ayon_core/tools/launcher/models/actions.py index 7158c05431..8bd30daffa 100644 --- a/client/ayon_core/tools/launcher/models/actions.py +++ b/client/ayon_core/tools/launcher/models/actions.py @@ -21,9 +21,9 @@ class ApplicationAction(LauncherAction): Application action based on 'ApplicationManager' system. - Handling of applications in launcher is not ideal and should be completely - redone from scratch. This is just a temporary solution to keep backwards - compatibility with AYON launcher. + Handling of applications in launcher is not ideal and should be + completely redone from scratch. This is just a temporary solution + to keep backwards compatibility with AYON launcher. Todos: Move handling of errors to frontend. diff --git a/client/ayon_core/tools/loader/ui/_multicombobox.py b/client/ayon_core/tools/loader/ui/_multicombobox.py index c026952418..9efe57ef0f 100644 --- a/client/ayon_core/tools/loader/ui/_multicombobox.py +++ b/client/ayon_core/tools/loader/ui/_multicombobox.py @@ -517,7 +517,11 @@ def _paint_items(self, painter, indexes, content_rect): def setItemCheckState(self, index, state): self.setItemData(index, state, QtCore.Qt.CheckStateRole) - def set_value(self, values: Optional[Iterable[Any]], role: Optional[int] = None): + def set_value( + self, + values: Optional[Iterable[Any]], + role: Optional[int] = None, + ): if role is None: role = self._value_role diff --git a/client/ayon_core/tools/loader/ui/products_model.py b/client/ayon_core/tools/loader/ui/products_model.py index bc24d4d7f7..3571788134 100644 --- a/client/ayon_core/tools/loader/ui/products_model.py +++ b/client/ayon_core/tools/loader/ui/products_model.py @@ -499,8 +499,10 @@ def refresh(self, project_name, folder_ids): version_item.version_id for version_item in last_version_by_product_id.values() } - repre_count_by_version_id = self._controller.get_versions_representation_count( - project_name, version_ids + repre_count_by_version_id = ( + self._controller.get_versions_representation_count( + project_name, version_ids + ) ) sync_availability_by_version_id = ( self._controller.get_version_sync_availability( diff --git a/client/ayon_core/tools/publisher/widgets/overview_widget.py b/client/ayon_core/tools/publisher/widgets/overview_widget.py index a09ee80ed5..c6c3b774f0 100644 --- a/client/ayon_core/tools/publisher/widgets/overview_widget.py +++ b/client/ayon_core/tools/publisher/widgets/overview_widget.py @@ -339,7 +339,9 @@ def _on_change_anim_finished(self): self._change_visibility_for_state() self._product_content_layout.addWidget(self._create_widget, 7) self._product_content_layout.addWidget(self._product_views_widget, 3) - self._product_content_layout.addWidget(self._product_attributes_wrap, 7) + self._product_content_layout.addWidget( + self._product_attributes_wrap, 7 + ) def _change_visibility_for_state(self): self._create_widget.setVisible( diff --git a/client/ayon_core/tools/publisher/widgets/product_context.py b/client/ayon_core/tools/publisher/widgets/product_context.py index 04c9ca7e56..30b318982b 100644 --- a/client/ayon_core/tools/publisher/widgets/product_context.py +++ b/client/ayon_core/tools/publisher/widgets/product_context.py @@ -214,8 +214,8 @@ class TasksCombobox(QtWidgets.QComboBox): Combobox gives ability to select only from intersection of task names for folder paths in selected instances. - If folder paths in selected instances does not have same tasks then combobox - will be empty. + If folder paths in selected instances does not have same tasks + then combobox will be empty. """ value_changed = QtCore.Signal() @@ -604,7 +604,7 @@ def set_value(self, variants=None): class GlobalAttrsWidget(QtWidgets.QWidget): - """Global attributes mainly to define context and product name of instances. + """Global attributes to define context and product name of instances. product name is or may be affected on context. Gives abiity to modify context and product name of instance. This change is not autopromoted but diff --git a/client/ayon_core/tools/publisher/widgets/tasks_model.py b/client/ayon_core/tools/publisher/widgets/tasks_model.py index 16a4111f59..8bfa81116a 100644 --- a/client/ayon_core/tools/publisher/widgets/tasks_model.py +++ b/client/ayon_core/tools/publisher/widgets/tasks_model.py @@ -22,8 +22,8 @@ class TasksModel(QtGui.QStandardItemModel): tasks with same names then model is empty too. Args: - controller (AbstractPublisherFrontend): Controller which handles creation and - publishing. + controller (AbstractPublisherFrontend): Controller which handles + creation and publishing. """ def __init__( diff --git a/client/ayon_core/tools/publisher/window.py b/client/ayon_core/tools/publisher/window.py index a912495d4e..ed5b909a55 100644 --- a/client/ayon_core/tools/publisher/window.py +++ b/client/ayon_core/tools/publisher/window.py @@ -998,7 +998,11 @@ def _on_creator_error(self, event): new_item["label"] = new_item.pop("creator_label") new_item["identifier"] = new_item.pop("creator_identifier") new_failed_info.append(new_item) - self.add_error_message_dialog(event["title"], new_failed_info, "Creator:") + self.add_error_message_dialog( + event["title"], + new_failed_info, + "Creator:" + ) def _on_convertor_error(self, event): new_failed_info = [] diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 4f3ddf1ded..4280445b60 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -366,8 +366,8 @@ def _update_cache(self): try: uuid.UUID(repre_id) except (ValueError, TypeError, AttributeError): - # Fake not existing representation id so container is shown in UI - # but as invalid + # Fake not existing representation id so container + # is shown in UI but as invalid item.representation_id = invalid_ids_mapping.setdefault( repre_id, uuid.uuid4().hex ) diff --git a/client/ayon_core/tools/utils/lib.py b/client/ayon_core/tools/utils/lib.py index 200e281664..4b303c0143 100644 --- a/client/ayon_core/tools/utils/lib.py +++ b/client/ayon_core/tools/utils/lib.py @@ -556,9 +556,10 @@ def get_qta_icon_by_name_and_color(cls, icon_name, icon_color): log.info("Didn't find icon \"{}\"".format(icon_name)) elif used_variant != icon_name: - log.debug("Icon \"{}\" was not found \"{}\" is used instead".format( - icon_name, used_variant - )) + log.debug( + f"Icon \"{icon_name}\" was not found" + f" \"{used_variant}\" is used instead" + ) cls._qtawesome_cache[full_icon_name] = icon return icon diff --git a/server/settings/publish_plugins.py b/server/settings/publish_plugins.py index 16b1f37187..8893b00e23 100644 --- a/server/settings/publish_plugins.py +++ b/server/settings/publish_plugins.py @@ -358,7 +358,10 @@ class ExtractOIIOTranscodeOutputModel(BaseSettingsModel): custom_tags: list[str] = SettingsField( default_factory=list, title="Custom Tags", - description="Additional custom tags that will be added to the created representation." + description=( + "Additional custom tags that will be added" + " to the created representation." + ) ) @@ -892,9 +895,11 @@ class PublishPuginsModel(BaseSettingsModel): default_factory=CollectFramesFixDefModel, title="Collect Frames to Fix", ) - CollectUSDLayerContributions: CollectUSDLayerContributionsModel = SettingsField( - default_factory=CollectUSDLayerContributionsModel, - title="Collect USD Layer Contributions", + CollectUSDLayerContributions: CollectUSDLayerContributionsModel = ( + SettingsField( + default_factory=CollectUSDLayerContributionsModel, + title="Collect USD Layer Contributions", + ) ) ValidateEditorialAssetName: ValidateBaseModel = SettingsField( default_factory=ValidateBaseModel, @@ -1214,7 +1219,9 @@ class PublishPuginsModel(BaseSettingsModel): "TOP_RIGHT": "{anatomy[version]}", "BOTTOM_LEFT": "{username}", "BOTTOM_CENTERED": "{folder[name]}", - "BOTTOM_RIGHT": "{frame_start}-{current_frame}-{frame_end}", + "BOTTOM_RIGHT": ( + "{frame_start}-{current_frame}-{frame_end}" + ), "filter": { "families": [], "tags": [] @@ -1240,7 +1247,9 @@ class PublishPuginsModel(BaseSettingsModel): "TOP_RIGHT": "{anatomy[version]}", "BOTTOM_LEFT": "{username}", "BOTTOM_CENTERED": "{folder[name]}", - "BOTTOM_RIGHT": "{frame_start}-{current_frame}-{frame_end}", + "BOTTOM_RIGHT": ( + "{frame_start}-{current_frame}-{frame_end}" + ), "filter": { "families": [], "tags": [] diff --git a/server/settings/tools.py b/server/settings/tools.py index a2785c1edf..96851be1da 100644 --- a/server/settings/tools.py +++ b/server/settings/tools.py @@ -83,8 +83,8 @@ class CreatorToolModel(BaseSettingsModel): filter_creator_profiles: list[FilterCreatorProfile] = SettingsField( default_factory=list, title="Filter creator profiles", - description="Allowed list of creator labels that will be only shown if " - "profile matches context." + description="Allowed list of creator labels that will be only shown" + " if profile matches context." ) @validator("product_types_smart_select") @@ -426,7 +426,9 @@ class GlobalToolsModel(BaseSettingsModel): ], "task_types": [], "tasks": [], - "template": "{product[type]}{Task[name]}_{Renderlayer}_{Renderpass}" + "template": ( + "{product[type]}{Task[name]}_{Renderlayer}_{Renderpass}" + ) }, { "product_types": [ diff --git a/tests/client/ayon_core/pipeline/editorial/test_extract_otio_review.py b/tests/client/ayon_core/pipeline/editorial/test_extract_otio_review.py index ea31e1a260..8b1c9da30e 100644 --- a/tests/client/ayon_core/pipeline/editorial/test_extract_otio_review.py +++ b/tests/client/ayon_core/pipeline/editorial/test_extract_otio_review.py @@ -130,19 +130,20 @@ def test_image_sequence_and_handles_out_of_range(): expected = [ # 5 head black frames generated from gap (991-995) - "/path/to/ffmpeg -t 0.2 -r 25.0 -f lavfi -i color=c=black:s=1280x720 -tune " - "stillimage -start_number 991 C:/result/output.%03d.jpg", + "/path/to/ffmpeg -t 0.2 -r 25.0 -f lavfi -i color=c=black:s=1280x720" + " -tune stillimage -start_number 991 C:/result/output.%03d.jpg", # 9 tail back frames generated from gap (1097-1105) - "/path/to/ffmpeg -t 0.36 -r 25.0 -f lavfi -i color=c=black:s=1280x720 -tune " - "stillimage -start_number 1097 C:/result/output.%03d.jpg", + "/path/to/ffmpeg -t 0.36 -r 25.0 -f lavfi -i color=c=black:s=1280x720" + " -tune stillimage -start_number 1097 C:/result/output.%03d.jpg", # Report from source tiff (996-1096) # 996-1000 = additional 5 head frames # 1001-1095 = source range conformed to 25fps # 1096-1096 = additional 1 tail frames "/path/to/ffmpeg -start_number 1000 -framerate 25.0 -i " - f"C:\\tif_seq{os.sep}output.%04d.tif -start_number 996 C:/result/output.%03d.jpg" + f"C:\\tif_seq{os.sep}output.%04d.tif -start_number 996" + f" C:/result/output.%03d.jpg" ] assert calls == expected @@ -179,13 +180,13 @@ def test_short_movie_head_gap_handles(): expected = [ # 10 head black frames generated from gap (991-1000) - "/path/to/ffmpeg -t 0.4 -r 25.0 -f lavfi -i color=c=black:s=1280x720 -tune " - "stillimage -start_number 991 C:/result/output.%03d.jpg", + "/path/to/ffmpeg -t 0.4 -r 25.0 -f lavfi -i color=c=black:s=1280x720" + " -tune stillimage -start_number 991 C:/result/output.%03d.jpg", # source range + 10 tail frames # duration = 50fr (source) + 10fr (tail handle) = 60 fr = 2.4s - "/path/to/ffmpeg -ss 0.0 -t 2.4 -i C:\\data\\movie.mp4 -start_number 1001 " - "C:/result/output.%03d.jpg" + "/path/to/ffmpeg -ss 0.0 -t 2.4 -i C:\\data\\movie.mp4" + " -start_number 1001 C:/result/output.%03d.jpg" ] assert calls == expected @@ -208,7 +209,8 @@ def test_short_movie_tail_gap_handles(): # 10 head frames + source range # duration = 10fr (head handle) + 66fr (source) = 76fr = 3.16s "/path/to/ffmpeg -ss 1.0416666666666667 -t 3.1666666666666665 -i " - "C:\\data\\qt_no_tc_24fps.mov -start_number 991 C:/result/output.%03d.jpg" + "C:\\data\\qt_no_tc_24fps.mov -start_number 991" + " C:/result/output.%03d.jpg" ] assert calls == expected @@ -234,10 +236,12 @@ def test_multiple_review_clips_no_gap(): expected = [ # 10 head black frames generated from gap (991-1000) - '/path/to/ffmpeg -t 0.4 -r 25.0 -f lavfi -i color=c=black:s=1280x720 -tune ' + '/path/to/ffmpeg -t 0.4 -r 25.0 -f lavfi' + ' -i color=c=black:s=1280x720 -tune ' 'stillimage -start_number 991 C:/result/output.%03d.jpg', - # Alternance 25fps tiff sequence and 24fps exr sequence for 100 frames each + # Alternance 25fps tiff sequence and 24fps exr sequence + # for 100 frames each '/path/to/ffmpeg -start_number 1000 -framerate 25.0 -i ' f'C:\\no_tc{os.sep}output.%04d.tif ' '-start_number 1001 C:/result/output.%03d.jpg', @@ -315,7 +319,8 @@ def test_multiple_review_clips_with_gap(): expected = [ # Gap on review track (12 frames) - '/path/to/ffmpeg -t 0.5 -r 24.0 -f lavfi -i color=c=black:s=1280x720 -tune ' + '/path/to/ffmpeg -t 0.5 -r 24.0 -f lavfi' + ' -i color=c=black:s=1280x720 -tune ' 'stillimage -start_number 991 C:/result/output.%03d.jpg', '/path/to/ffmpeg -start_number 1000 -framerate 24.0 -i ' From 3de2755de52567694e4dd1a248cbbfa69499e1b6 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:37:18 +0100 Subject: [PATCH 064/136] remove unrelated information from docstring --- client/ayon_core/lib/local_settings.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/client/ayon_core/lib/local_settings.py b/client/ayon_core/lib/local_settings.py index 690781151c..08030ae87e 100644 --- a/client/ayon_core/lib/local_settings.py +++ b/client/ayon_core/lib/local_settings.py @@ -276,12 +276,7 @@ def delete_item(self, name): @abstractmethod def _delete_item(self, name): # type: (str) -> None - """Delete item from settings. - - Note: - see :meth:`ayon_core.lib.user_settings.ARegistrySettings.delete_item` - - """ + """Delete item from settings.""" pass def __delitem__(self, name): @@ -433,12 +428,7 @@ def delete_item_from_section(self, section, name): config.write(cfg) def _delete_item(self, name): - """Delete item from default section. - - Note: - See :meth:`~ayon_core.lib.IniSettingsRegistry.delete_item_from_section` - - """ + """Delete item from default section.""" self.delete_item_from_section("MAIN", name) From ef6d7b5a6ce863c0a0231fb8da5ea939b6d29717 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:37:27 +0100 Subject: [PATCH 065/136] put noqa at correct place --- client/ayon_core/pipeline/entity_uri.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/entity_uri.py b/client/ayon_core/pipeline/entity_uri.py index 1dee9a1423..1362389ee9 100644 --- a/client/ayon_core/pipeline/entity_uri.py +++ b/client/ayon_core/pipeline/entity_uri.py @@ -18,13 +18,13 @@ def parse_ayon_entity_uri(uri: str) -> Optional[dict]: Example: >>> parse_ayon_entity_uri( - >>> "ayon://test/char/villain?product=modelMain&version=2&representation=usd" # noqa: E501 + >>> "ayon://test/char/villain?product=modelMain&version=2&representation=usd" >>> ) {'project': 'test', 'folderPath': '/char/villain', 'product': 'modelMain', 'version': 1, 'representation': 'usd'} >>> parse_ayon_entity_uri( - >>> "ayon+entity://project/folder?product=renderMain&version=3&representation=exr" # noqa: E501 + >>> "ayon+entity://project/folder?product=renderMain&version=3&representation=exr" >>> ) {'project': 'project', 'folderPath': '/folder', 'product': 'renderMain', 'version': 3, @@ -34,7 +34,7 @@ def parse_ayon_entity_uri(uri: str) -> Optional[dict]: dict[str, Union[str, int]]: The individual key with their values as found in the ayon entity URI. - """ + """ # noqa: E501 if not (uri.startswith("ayon+entity://") or uri.startswith("ayon://")): return {} From 9cd354efb299c43e27864119ea1779116407ee23 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:09:45 +0100 Subject: [PATCH 066/136] added constant to define store key for env variables --- client/ayon_core/pipeline/publish/__init__.py | 2 ++ client/ayon_core/pipeline/publish/constants.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/client/ayon_core/pipeline/publish/__init__.py b/client/ayon_core/pipeline/publish/__init__.py index ac71239acf..5363e0b378 100644 --- a/client/ayon_core/pipeline/publish/__init__.py +++ b/client/ayon_core/pipeline/publish/__init__.py @@ -3,6 +3,7 @@ ValidateContentsOrder, ValidateSceneOrder, ValidateMeshOrder, + FARM_JOB_ENV_DATA_KEY, ) from .publish_plugins import ( @@ -59,6 +60,7 @@ "ValidateContentsOrder", "ValidateSceneOrder", "ValidateMeshOrder", + "FARM_JOB_ENV_DATA_KEY", "AbstractMetaInstancePlugin", "AbstractMetaContextPlugin", diff --git a/client/ayon_core/pipeline/publish/constants.py b/client/ayon_core/pipeline/publish/constants.py index 38f5ffef3f..f2f4e851a9 100644 --- a/client/ayon_core/pipeline/publish/constants.py +++ b/client/ayon_core/pipeline/publish/constants.py @@ -9,3 +9,5 @@ DEFAULT_PUBLISH_TEMPLATE = "default" DEFAULT_HERO_PUBLISH_TEMPLATE = "default" TRANSIENT_DIR_TEMPLATE = "default" + +FARM_JOB_ENV_DATA_KEY: str = "farmJobEnv" From fae4eed3ed97b306b93bd2e98f79bbee335b8604 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:10:59 +0100 Subject: [PATCH 067/136] added new plugin collecting environment variables to context --- .../publish/collect_farm_env_variables.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 client/ayon_core/plugins/publish/collect_farm_env_variables.py diff --git a/client/ayon_core/plugins/publish/collect_farm_env_variables.py b/client/ayon_core/plugins/publish/collect_farm_env_variables.py new file mode 100644 index 0000000000..935b4d5c9f --- /dev/null +++ b/client/ayon_core/plugins/publish/collect_farm_env_variables.py @@ -0,0 +1,48 @@ +import os + +import pyblish.api + +from ayon_core.pipeline.publish import FARM_JOB_ENV_DATA_KEY + + +class CollectCoreJobEnvVars(pyblish.api.ContextPlugin): + """Collect set of environment variables to submit with deadline jobs""" + order = pyblish.api.CollectorOrder - 0.45 + label = "AYON core Farm Environment Variables" + targets = ["local"] + + ENV_KEYS = [ + # AYON + "AYON_BUNDLE_NAME", + "AYON_DEFAULT_SETTINGS_VARIANT", + "AYON_PROJECT_NAME", + "AYON_FOLDER_PATH", + "AYON_TASK_NAME", + "AYON_APP_NAME", + "AYON_WORKDIR", + "AYON_APP_NAME", + "AYON_LOG_NO_COLORS", + "AYON_IN_TESTS", + "IS_TEST", # backwards compatibility + ] + + def process(self, context): + env = context.data.setdefault(FARM_JOB_ENV_DATA_KEY, {}) + for key in [ + # AYON + "AYON_BUNDLE_NAME", + "AYON_DEFAULT_SETTINGS_VARIANT", + "AYON_PROJECT_NAME", + "AYON_FOLDER_PATH", + "AYON_TASK_NAME", + "AYON_WORKDIR", + "AYON_LOG_NO_COLORS", + "AYON_IN_TESTS", + # backwards compatibility + "IS_TEST", + ]: + value = os.getenv(key) + if value: + self.log.debug(f"Setting job env: {key}: {value}") + env[key] = value + From 85be6b2e422b327d6c506018192f3e10a8fee998 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:53:38 +0100 Subject: [PATCH 068/136] remove unnecessary attribute --- .../plugins/publish/collect_farm_env_variables.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/client/ayon_core/plugins/publish/collect_farm_env_variables.py b/client/ayon_core/plugins/publish/collect_farm_env_variables.py index 935b4d5c9f..0201973643 100644 --- a/client/ayon_core/plugins/publish/collect_farm_env_variables.py +++ b/client/ayon_core/plugins/publish/collect_farm_env_variables.py @@ -11,21 +11,6 @@ class CollectCoreJobEnvVars(pyblish.api.ContextPlugin): label = "AYON core Farm Environment Variables" targets = ["local"] - ENV_KEYS = [ - # AYON - "AYON_BUNDLE_NAME", - "AYON_DEFAULT_SETTINGS_VARIANT", - "AYON_PROJECT_NAME", - "AYON_FOLDER_PATH", - "AYON_TASK_NAME", - "AYON_APP_NAME", - "AYON_WORKDIR", - "AYON_APP_NAME", - "AYON_LOG_NO_COLORS", - "AYON_IN_TESTS", - "IS_TEST", # backwards compatibility - ] - def process(self, context): env = context.data.setdefault(FARM_JOB_ENV_DATA_KEY, {}) for key in [ From 07c246ba74ceb628a05ccd11bfea5e20bf393cfe Mon Sep 17 00:00:00 2001 From: ynbot Date: Mon, 25 Nov 2024 13:57:32 +0000 Subject: [PATCH 069/136] [Automated] Update assign_pr_to_project caller workflow --- .github/workflows/assign_pr_to_project.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/assign_pr_to_project.yml diff --git a/.github/workflows/assign_pr_to_project.yml b/.github/workflows/assign_pr_to_project.yml new file mode 100644 index 0000000000..86707fc9da --- /dev/null +++ b/.github/workflows/assign_pr_to_project.yml @@ -0,0 +1,15 @@ +name: 🔸Auto assign pr +on: + pull_request: + types: + - opened + +jobs: + auto-assign-pr: + uses: ynput/ops-repo-automation/.github/workflows/pr_to_project.yml@develop + with: + repo: "${{ github.repository }}" + project_id: 16 + pull_request_number: ${{ github.event.pull_request.number }} + secrets: + token: ${{ secrets.YNPUT_BOT_TOKEN }} From 333363f024119022437e5b520b2faa7bcf82e392 Mon Sep 17 00:00:00 2001 From: ynbot Date: Mon, 25 Nov 2024 14:07:54 +0000 Subject: [PATCH 070/136] [Automated] Update validate_pr_labels caller workflow --- .github/workflows/validate_pr_labels.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/validate_pr_labels.yml diff --git a/.github/workflows/validate_pr_labels.yml b/.github/workflows/validate_pr_labels.yml new file mode 100644 index 0000000000..00e5742afe --- /dev/null +++ b/.github/workflows/validate_pr_labels.yml @@ -0,0 +1,18 @@ +name: 🔎 Validate PR Labels +on: + pull_request: + types: + - opened + - edited + - labeled + - unlabeled + +jobs: + validate-type-label: + uses: ynput/ops-repo-automation/.github/workflows/validate_pr_labels.yml@develop + with: + repo: "${{ github.repository }}" + pull_request_number: ${{ github.event.pull_request.number }} + query_prefix: "type: " + secrets: + token: ${{ secrets.YNPUT_BOT_TOKEN }} From 463ad79a062b1fe3e23b3351189ef1200de678fb Mon Sep 17 00:00:00 2001 From: Robin De Lillo Date: Mon, 25 Nov 2024 10:14:38 -0500 Subject: [PATCH 071/136] Apply suggestions from code review Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/stagingdir.py | 48 +++++++++++++------------ 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index c7cc95ff55..4395f1a5d5 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -1,7 +1,7 @@ from ayon_core.lib import Logger, filter_profiles, StringTemplate from ayon_core.settings import get_project_settings -from ayon_core.pipeline.template_data import get_template_data +from .template_data import get_template_data from .anatomy import Anatomy from .tempdir import get_temp_dir @@ -71,7 +71,7 @@ def get_staging_dir_config( template_name = profile["template_name"] _validate_template_name(project_name, template_name, anatomy) - template = anatomy.templates[STAGING_DIR_TEMPLATES][template_name] + template = anatomy.get_template_item("staging", template_name) if not template: # template should always be found either from anatomy or from profile @@ -93,7 +93,7 @@ def _validate_template_name(project_name, template_name, anatomy): Raises: ValueError - if misconfigured template """ - if template_name not in anatomy.templates[STAGING_DIR_TEMPLATES]: + if template_name not in anatomy.templates["staging"]: raise ValueError( ( 'Anatomy of project "{}" does not have set' @@ -195,23 +195,25 @@ def get_staging_dir_info( log=log, ) - if not staging_dir_config: - if always_return_path: # no config found but force an output - return { - "stagingDir": get_temp_dir( - project_name=project_entity["name"], - anatomy=anatomy, - prefix=prefix, - suffix=suffix, - ), - "stagingDir_persistent": False, - } - else: - return None - - return { - "stagingDir": StringTemplate.format_template( - staging_dir_config["template"]["directory"], ctx_data - ), - "stagingDir_persistent": staging_dir_config["persistence"], - } + if staging_dir_config: + return { + "stagingDir": StringTemplate.format_template( + staging_dir_config["template"]["directory"], + ctx_data + ), + "stagingDir_persistent": staging_dir_config["persistence"], + } + + # no config found but force an output + if always_return_path: + return { + "stagingDir": get_temp_dir( + project_name=project_entity["name"], + anatomy=anatomy, + prefix=prefix, + suffix=suffix, + ), + "stagingDir_persistent": False, + } + + return None From 0a13574509b517e3d3dd796e7f73ee4d42ce10a4 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 25 Nov 2024 10:20:12 -0500 Subject: [PATCH 072/136] Rename stagingdir to staging_dir. --- client/ayon_core/pipeline/__init__.py | 2 +- client/ayon_core/pipeline/publish/lib.py | 2 +- client/ayon_core/pipeline/{stagingdir.py => staging_dir.py} | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) rename client/ayon_core/pipeline/{stagingdir.py => staging_dir.py} (99%) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index c58e385d79..41bcd0dbd1 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -9,7 +9,7 @@ from .tempdir import get_temp_dir -from .stagingdir import get_staging_dir_info +from .staging_dir import get_staging_dir_info from .create import ( BaseCreator, diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 4c36f473d1..c0dfe8c910 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -642,7 +642,7 @@ def get_custom_staging_dir_info( anatomy=None, log=None, ): - from ayon_core.pipeline.stagingdir import get_staging_dir_config + from ayon_core.pipeline.staging_dir import get_staging_dir_config warnings.warn( ( "Function 'get_custom_staging_dir_info' in" diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/staging_dir.py similarity index 99% rename from client/ayon_core/pipeline/stagingdir.py rename to client/ayon_core/pipeline/staging_dir.py index 4395f1a5d5..0e993ecae1 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -5,8 +5,6 @@ from .anatomy import Anatomy from .tempdir import get_temp_dir -STAGING_DIR_TEMPLATES = "staging" - def get_staging_dir_config( project_name, From 2d6911513feab68c4aaf3bba66051bf0babbf196 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 25 Nov 2024 10:23:43 -0500 Subject: [PATCH 073/136] Fix lint. --- client/ayon_core/pipeline/staging_dir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index 0e993ecae1..e46426057d 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -203,7 +203,7 @@ def get_staging_dir_info( } # no config found but force an output - if always_return_path: + if always_return_path: return { "stagingDir": get_temp_dir( project_name=project_entity["name"], From 2066fe61a124e0639735cf533ac33894287271cf Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 25 Nov 2024 10:55:04 -0500 Subject: [PATCH 074/136] Fix anatomy template. --- client/ayon_core/pipeline/staging_dir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index e46426057d..86aaf3002f 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -196,7 +196,7 @@ def get_staging_dir_info( if staging_dir_config: return { "stagingDir": StringTemplate.format_template( - staging_dir_config["template"]["directory"], + str(staging_dir_config["template"]["directory"]), ctx_data ), "stagingDir_persistent": staging_dir_config["persistence"], From a60796eb73f631ed14f8c0c4bcd193b05c40a5c3 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 25 Nov 2024 14:27:13 -0500 Subject: [PATCH 075/136] Adjust missing taskEntity. --- client/ayon_core/pipeline/staging_dir.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index 86aaf3002f..c8e3251e7b 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -147,6 +147,7 @@ def get_staging_dir_info( Optional[Dict[str, Any]]: Staging dir info data """ + task_entity = task_entity or {} log = logger or Logger.get_logger("get_staging_dir_info") if anatomy is None: From 1c7ab66246365903fc8aef14be184bf56cc8731c Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 25 Nov 2024 16:07:17 -0500 Subject: [PATCH 076/136] Fix audio extraction from OTIO timeline. --- .../publish/extract_otio_audio_tracks.py | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py b/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py index 98723beffa..88eb2da059 100644 --- a/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py +++ b/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py @@ -71,20 +71,17 @@ def add_audio_to_instances(self, audio_file, instances): name = inst.data["folderPath"] recycling_file = [f for f in created_files if name in f] - - # frameranges - timeline_in_h = inst.data["clipInH"] - timeline_out_h = inst.data["clipOutH"] - fps = inst.data["fps"] - - # create duration - duration = (timeline_out_h - timeline_in_h) + 1 + audio_clip = inst.data["otioClip"] + audio_range = audio_clip.range_in_parent() + duration = audio_range.duration.to_frames() # ffmpeg generate new file only if doesn't exists already if not recycling_file: - # convert to seconds - start_sec = float(timeline_in_h / fps) - duration_sec = float(duration / fps) + parent_track = audio_clip.parent() + parent_track_start = parent_track.range_in_parent().start_time + relative_start_time = audio_range.start_time - parent_track_start + start_sec = relative_start_time.to_seconds() + duration_sec = audio_range.duration.to_seconds() # temp audio file audio_fpath = self.create_temp_file(name) @@ -163,9 +160,7 @@ def get_audio_track_items(self, otio_timeline): output = [] # go trough all audio tracks - for otio_track in otio_timeline.tracks: - if "Audio" not in otio_track.kind: - continue + for otio_track in otio_timeline.audio_tracks(): self.log.debug("_" * 50) playhead = 0 for otio_clip in otio_track: @@ -173,19 +168,22 @@ def get_audio_track_items(self, otio_timeline): if isinstance(otio_clip, otio.schema.Gap): playhead += otio_clip.source_range.duration.value elif isinstance(otio_clip, otio.schema.Clip): - start = otio_clip.source_range.start_time.value - duration = otio_clip.source_range.duration.value - fps = otio_clip.source_range.start_time.rate + media_av_start = otio_clip.available_range().start_time + clip_start = otio_clip.source_range.start_time + fps = clip_start.rate + conformed_av_start = media_av_start.rescaled_to(fps) + start = clip_start - conformed_av_start # ffmpeg ignores embedded tc + duration = otio_clip.source_range.duration media_path = otio_clip.media_reference.target_url input = { "mediaPath": media_path, "delayFrame": playhead, - "startFrame": start, - "durationFrame": duration, + "startFrame": start.to_frames(), + "durationFrame": duration.to_frames(), "delayMilSec": int(float(playhead / fps) * 1000), - "startSec": float(start / fps), - "durationSec": float(duration / fps), - "fps": fps + "startSec": start.to_seconds(), + "durationSec": duration.to_seconds(), + "fps": float(fps) } if input not in output: output.append(input) From 3f8430dceac2132629a510e298a56f165a2998a0 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 25 Nov 2024 16:13:28 -0500 Subject: [PATCH 077/136] Fix lint. --- client/ayon_core/plugins/publish/extract_otio_audio_tracks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py b/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py index 88eb2da059..d80d745111 100644 --- a/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py +++ b/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py @@ -172,7 +172,8 @@ def get_audio_track_items(self, otio_timeline): clip_start = otio_clip.source_range.start_time fps = clip_start.rate conformed_av_start = media_av_start.rescaled_to(fps) - start = clip_start - conformed_av_start # ffmpeg ignores embedded tc + # ffmpeg ignores embedded tc + start = clip_start - conformed_av_start duration = otio_clip.source_range.duration media_path = otio_clip.media_reference.target_url input = { From d891a0088fdb90bfbddb5bcc332dd3691596a1e1 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 25 Nov 2024 16:14:53 -0500 Subject: [PATCH 078/136] Fix lint. --- client/ayon_core/plugins/publish/extract_otio_audio_tracks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py b/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py index d80d745111..3d22894a75 100644 --- a/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py +++ b/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py @@ -79,7 +79,8 @@ def add_audio_to_instances(self, audio_file, instances): if not recycling_file: parent_track = audio_clip.parent() parent_track_start = parent_track.range_in_parent().start_time - relative_start_time = audio_range.start_time - parent_track_start + relative_start_time = ( + audio_range.start_time - parent_track_start) start_sec = relative_start_time.to_seconds() duration_sec = audio_range.duration.to_seconds() From 0c80fe0ad6d48e854ba0bed5fdeba61e4bcf116f Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 25 Nov 2024 23:26:35 +0100 Subject: [PATCH 079/136] The `_representation_conversion` method converts in-place - it does not return anything --- client/ayon_core/pipeline/delivery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/delivery.py b/client/ayon_core/pipeline/delivery.py index 366c261e08..55c840f3a5 100644 --- a/client/ayon_core/pipeline/delivery.py +++ b/client/ayon_core/pipeline/delivery.py @@ -387,7 +387,7 @@ def get_representations_delivery_template_data( # convert representation entity. Fixed in 'ayon_api' 1.0.10. if isinstance(template_data, str): con = ayon_api.get_server_api_connection() - repre_entity = con._representation_conversion(repre_entity) + con._representation_conversion(repre_entity) template_data = repre_entity["context"] template_data.update(copy.deepcopy(general_template_data)) From cd5f89afe572637dc111337ac343655b785dc25f Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 26 Nov 2024 09:58:05 +0100 Subject: [PATCH 080/136] remove OpenPype env key --- client/ayon_core/plugins/publish/collect_farm_env_variables.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/ayon_core/plugins/publish/collect_farm_env_variables.py b/client/ayon_core/plugins/publish/collect_farm_env_variables.py index 0201973643..7b4618527b 100644 --- a/client/ayon_core/plugins/publish/collect_farm_env_variables.py +++ b/client/ayon_core/plugins/publish/collect_farm_env_variables.py @@ -23,8 +23,6 @@ def process(self, context): "AYON_WORKDIR", "AYON_LOG_NO_COLORS", "AYON_IN_TESTS", - # backwards compatibility - "IS_TEST", ]: value = os.getenv(key) if value: From c7940b4fd0f175892139a424ceb9922c5325e820 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:00:25 +0100 Subject: [PATCH 081/136] added comment to workdir env --- .../ayon_core/plugins/publish/collect_farm_env_variables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/plugins/publish/collect_farm_env_variables.py b/client/ayon_core/plugins/publish/collect_farm_env_variables.py index 7b4618527b..a7d9bce08d 100644 --- a/client/ayon_core/plugins/publish/collect_farm_env_variables.py +++ b/client/ayon_core/plugins/publish/collect_farm_env_variables.py @@ -14,15 +14,15 @@ class CollectCoreJobEnvVars(pyblish.api.ContextPlugin): def process(self, context): env = context.data.setdefault(FARM_JOB_ENV_DATA_KEY, {}) for key in [ - # AYON "AYON_BUNDLE_NAME", "AYON_DEFAULT_SETTINGS_VARIANT", "AYON_PROJECT_NAME", "AYON_FOLDER_PATH", "AYON_TASK_NAME", - "AYON_WORKDIR", "AYON_LOG_NO_COLORS", "AYON_IN_TESTS", + # NOTE Not sure why workdir is needed? + "AYON_WORKDIR", ]: value = os.getenv(key) if value: From a5842c4fdfc01088ad21bc6a3743145878a64ac8 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:22:59 +0100 Subject: [PATCH 082/136] added missing env keys for farm --- .../ayon_core/plugins/publish/collect_farm_env_variables.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/ayon_core/plugins/publish/collect_farm_env_variables.py b/client/ayon_core/plugins/publish/collect_farm_env_variables.py index a7d9bce08d..2e28b1b164 100644 --- a/client/ayon_core/plugins/publish/collect_farm_env_variables.py +++ b/client/ayon_core/plugins/publish/collect_farm_env_variables.py @@ -13,9 +13,14 @@ class CollectCoreJobEnvVars(pyblish.api.ContextPlugin): def process(self, context): env = context.data.setdefault(FARM_JOB_ENV_DATA_KEY, {}) + + # Disable colored logs on farm + env["AYON_LOG_NO_COLORS"] = "1" + for key in [ "AYON_BUNDLE_NAME", "AYON_DEFAULT_SETTINGS_VARIANT", + "AYON_USERNAME", "AYON_PROJECT_NAME", "AYON_FOLDER_PATH", "AYON_TASK_NAME", From 73420cd8a0c8c3f60bcad65fba29096c7c3de7df Mon Sep 17 00:00:00 2001 From: ynbot Date: Tue, 26 Nov 2024 11:46:45 +0000 Subject: [PATCH 083/136] [Automated] Update assign_pr_to_project caller workflow --- .github/workflows/assign_pr_to_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/assign_pr_to_project.yml b/.github/workflows/assign_pr_to_project.yml index 86707fc9da..4bb3d1742c 100644 --- a/.github/workflows/assign_pr_to_project.yml +++ b/.github/workflows/assign_pr_to_project.yml @@ -6,7 +6,7 @@ on: jobs: auto-assign-pr: - uses: ynput/ops-repo-automation/.github/workflows/pr_to_project.yml@develop + uses: ynput/ops-repo-automation/.github/workflows/pr_to_project.yml@main with: repo: "${{ github.repository }}" project_id: 16 From d663d68890fc9c77f4a8e22414c79cbaf5c1e2b5 Mon Sep 17 00:00:00 2001 From: ynbot Date: Tue, 26 Nov 2024 11:52:16 +0000 Subject: [PATCH 084/136] [Automated] Update validate_pr_labels caller workflow --- .github/workflows/validate_pr_labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_pr_labels.yml b/.github/workflows/validate_pr_labels.yml index 00e5742afe..f25e263c98 100644 --- a/.github/workflows/validate_pr_labels.yml +++ b/.github/workflows/validate_pr_labels.yml @@ -9,7 +9,7 @@ on: jobs: validate-type-label: - uses: ynput/ops-repo-automation/.github/workflows/validate_pr_labels.yml@develop + uses: ynput/ops-repo-automation/.github/workflows/validate_pr_labels.yml@main with: repo: "${{ github.repository }}" pull_request_number: ${{ github.event.pull_request.number }} From 47fa5e56027d7ac182b78b64391cea64f61fa032 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 26 Nov 2024 20:21:21 +0800 Subject: [PATCH 085/136] check on the active product id before adding version_items --- client/ayon_core/tools/sceneinventory/view.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 5892e4f983..93c889d037 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -805,7 +805,7 @@ def _show_version_dialog(self, item_ids, active_repre_id): version_items_by_project[project_name] = version_items_by_product_id active_version_id = active_repre_info.version_id - # active_product_id = active_repre_info.product_id + active_product_id = active_repre_info.product_id versions = set() product_ids = set() @@ -820,21 +820,23 @@ def _show_version_dialog(self, item_ids, active_repre_id): ) versions |= { version_item.version - for version_item in version_items_by_product_id.values() + for version_item in + version_items_by_product_id[active_product_id].values() } - for version_item in version_items_by_product_id.values(): - version = version_item.version - _prod_version = version - if _prod_version < 0: - _prod_version = -1 - product_ids_by_version[_prod_version].add( - version_item.product_id - ) - product_ids.add(version_item.product_id) - if version in versions: - continue - versions.add(version) - version_items.append((project_name, version_item)) + for version_item_by_id in version_items_by_product_id.values(): + for version_item in version_item_by_id.values(): + version = version_item.version + _prod_version = version + if _prod_version < 0: + _prod_version = -1 + product_ids_by_version[_prod_version].add( + version_item.product_id + ) + product_ids.add(version_item.product_id) + if version in versions: + continue + versions.add(version) + version_items.append((project_name, version_item)) def version_sorter(_, item): hero_value = 0 From fafcbe8992e5d22efacdfb72b53bca39c16b5126 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 26 Nov 2024 20:24:57 +0800 Subject: [PATCH 086/136] check on the active product id before adding version_items --- client/ayon_core/tools/sceneinventory/view.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 93c889d037..741587c064 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -805,7 +805,7 @@ def _show_version_dialog(self, item_ids, active_repre_id): version_items_by_project[project_name] = version_items_by_product_id active_version_id = active_repre_info.version_id - active_product_id = active_repre_info.product_id + # active_product_id = active_repre_info.product_id versions = set() product_ids = set() @@ -818,11 +818,6 @@ def _show_version_dialog(self, item_ids, active_repre_id): product_ids_by_version_by_project[project_name] = ( product_ids_by_version ) - versions |= { - version_item.version - for version_item in - version_items_by_product_id[active_product_id].values() - } for version_item_by_id in version_items_by_product_id.values(): for version_item in version_item_by_id.values(): version = version_item.version @@ -838,8 +833,9 @@ def _show_version_dialog(self, item_ids, active_repre_id): versions.add(version) version_items.append((project_name, version_item)) - def version_sorter(_, item): + def version_sorter(items): hero_value = 0 + item = items[-1] i_version = item.version if i_version < 0: hero_value = 1 From 10e66c4c39fb6505823255e1e0b040e1c4dacb69 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 26 Nov 2024 20:30:28 +0800 Subject: [PATCH 087/136] comsetic fix --- client/ayon_core/tools/sceneinventory/model.py | 9 ++++++--- .../ayon_core/tools/sceneinventory/models/containers.py | 3 ++- client/ayon_core/tools/sceneinventory/view.py | 7 +++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 79af0e5cf5..235b125eab 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -273,7 +273,8 @@ def refresh(self, selected=None): ) = self._get_status_data(project_name, status_name) repre_name = ( - repre_info.representation_name or "" + repre_info.representation_name or + "" ) container_model_items = [] for container_item in container_items: @@ -281,7 +282,8 @@ def refresh(self, selected=None): unique_name = repre_name + object_name item = QtGui.QStandardItem() item.setColumnCount(root_item.columnCount()) - item.setData(container_item.namespace, QtCore.Qt.DisplayRole) + item.setData(container_item.namespace, + QtCore.Qt.DisplayRole) item.setData(self.GRAYOUT_COLOR, NAME_COLOR_ROLE) item.setData(self.GRAYOUT_COLOR, VERSION_COLOR_ROLE) item.setData(item_icon, QtCore.Qt.DecorationRole) @@ -290,7 +292,8 @@ def refresh(self, selected=None): item.setData(version_label, VERSION_LABEL_ROLE) item.setData(container_item.loader_name, LOADER_NAME_ROLE) item.setData(container_item.object_name, OBJECT_NAME_ROLE) - item.setData(container_item.project_name, PROJECT_NAME_ROLE) + item.setData(container_item.project_name, + PROJECT_NAME_ROLE) item.setData(True, IS_CONTAINER_ITEM_ROLE) item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) container_model_items.append(item) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index b1cbb38587..ec1ed39e87 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -362,7 +362,8 @@ def _update_cache(self): current_project_name = self._controller.get_current_project_name() for container in containers: try: - item = ContainerItem.from_container_data(current_project_name, container) + item = ContainerItem.from_container_data( + current_project_name, container) repre_id = item.representation_id try: uuid.UUID(repre_id) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 741587c064..ba23e115c0 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -802,7 +802,9 @@ def _show_version_dialog(self, item_ids, active_repre_id): ) repre_info_by_project[project_name] = repres_info - version_items_by_project[project_name] = version_items_by_product_id + version_items_by_project[project_name] = ( + version_items_by_product_id + ) active_version_id = active_repre_info.version_id # active_product_id = active_repre_info.product_id @@ -994,7 +996,8 @@ def update_all(self): def _on_switch_to_versioned(self, item_ids): # Get container items by ID - containers_items_by_id = self._controller.get_container_items_by_id(item_ids) + containers_items_by_id = self._controller.get_container_items_by_id( + item_ids) # Extract project names and their corresponding representation IDs repre_ids_by_project = collections.defaultdict(set) for container_item in containers_items_by_id.values(): From 899b50ec93a5480d63e6c219119eeff5be4e1683 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 26 Nov 2024 09:08:59 -0500 Subject: [PATCH 088/136] Adjust for missing reference. --- .../plugins/publish/extract_otio_audio_tracks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py b/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py index 3d22894a75..472694d334 100644 --- a/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py +++ b/client/ayon_core/plugins/publish/extract_otio_audio_tracks.py @@ -166,9 +166,8 @@ def get_audio_track_items(self, otio_timeline): playhead = 0 for otio_clip in otio_track: self.log.debug(otio_clip) - if isinstance(otio_clip, otio.schema.Gap): - playhead += otio_clip.source_range.duration.value - elif isinstance(otio_clip, otio.schema.Clip): + if (isinstance(otio_clip, otio.schema.Clip) and + not otio_clip.media_reference.is_missing_reference): media_av_start = otio_clip.available_range().start_time clip_start = otio_clip.source_range.start_time fps = clip_start.rate @@ -190,7 +189,8 @@ def get_audio_track_items(self, otio_timeline): if input not in output: output.append(input) self.log.debug("__ input: {}".format(input)) - playhead += otio_clip.source_range.duration.value + + playhead += otio_clip.source_range.duration.value return output From 7ccf04ed586b69200d3fba744ef6ab470d86577d Mon Sep 17 00:00:00 2001 From: ynbot Date: Tue, 26 Nov 2024 15:17:38 +0000 Subject: [PATCH 089/136] [Automated] Update assign_pr_to_project caller workflow --- .github/workflows/assign_pr_to_project.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/assign_pr_to_project.yml b/.github/workflows/assign_pr_to_project.yml index 4bb3d1742c..92d2ff2916 100644 --- a/.github/workflows/assign_pr_to_project.yml +++ b/.github/workflows/assign_pr_to_project.yml @@ -1,5 +1,16 @@ name: 🔸Auto assign pr on: + workflow_dispatch: + inputs: + pr_number: + type: number + description: "Run workflow for this PR number" + required: true + project_id: + type: number + description: "Github Project Number" + required: true + default: 16 pull_request: types: - opened @@ -9,7 +20,7 @@ jobs: uses: ynput/ops-repo-automation/.github/workflows/pr_to_project.yml@main with: repo: "${{ github.repository }}" - project_id: 16 - pull_request_number: ${{ github.event.pull_request.number }} + project_id: ${{ inputs.project_id || 16 }} + pull_request_number: ${{ github.event.pull_request.number || inputs.pr_number }} secrets: token: ${{ secrets.YNPUT_BOT_TOKEN }} From 5d7aeaf0a707efac9347213c56a8d27cd741c88a Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:50:48 +0100 Subject: [PATCH 090/136] better variable name --- client/ayon_core/tools/sceneinventory/view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index ba23e115c0..43c6c8e2d0 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -820,8 +820,8 @@ def _show_version_dialog(self, item_ids, active_repre_id): product_ids_by_version_by_project[project_name] = ( product_ids_by_version ) - for version_item_by_id in version_items_by_product_id.values(): - for version_item in version_item_by_id.values(): + for version_items_by_id in version_items_by_product_id.values(): + for version_item in version_items_by_id.values(): version = version_item.version _prod_version = version if _prod_version < 0: From 1776df18e2375e66a0520fddd9d8ef919d6bcdb3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:50:59 +0100 Subject: [PATCH 091/136] don't store project name to version items --- client/ayon_core/tools/sceneinventory/view.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 43c6c8e2d0..a95a51ae37 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -833,11 +833,10 @@ def _show_version_dialog(self, item_ids, active_repre_id): if version in versions: continue versions.add(version) - version_items.append((project_name, version_item)) + version_items.append(version_item) - def version_sorter(items): + def version_sorter(item): hero_value = 0 - item = items[-1] i_version = item.version if i_version < 0: hero_value = 1 @@ -855,8 +854,7 @@ def version_sorter(items): version_options = [] active_version_idx = 0 - for idx, item in enumerate(version_items): - project_name, version_item = item + for idx, version_item in enumerate(version_items): version = version_item.version label = format_version(version) if version_item.version_id == active_version_id: From 5c115ce166b70391048b83597d06635783da1118 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:04:11 +0100 Subject: [PATCH 092/136] switch dialog can work per project --- .../sceneinventory/switch_dialog/dialog.py | 18 ++++++++------ client/ayon_core/tools/sceneinventory/view.py | 24 ++++++++++++++----- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/switch_dialog/dialog.py b/client/ayon_core/tools/sceneinventory/switch_dialog/dialog.py index 4977ad13c6..a6d88ed44a 100644 --- a/client/ayon_core/tools/sceneinventory/switch_dialog/dialog.py +++ b/client/ayon_core/tools/sceneinventory/switch_dialog/dialog.py @@ -46,8 +46,13 @@ class SwitchAssetDialog(QtWidgets.QDialog): switched = QtCore.Signal() - def __init__(self, controller, parent=None, items=None): - super(SwitchAssetDialog, self).__init__(parent) + def __init__(self, controller, project_name, items, parent=None): + super().__init__(parent) + + current_project_name = controller.get_current_project_name() + folder_id = None + if current_project_name == project_name: + folder_id = controller.get_current_folder_id() self.setWindowTitle("Switch selected items ...") @@ -147,11 +152,10 @@ def __init__(self, controller, parent=None, items=None): self._init_repre_name = None self._fill_check = False + self._project_name = project_name + self._folder_id = folder_id - self._project_name = controller.get_current_project_name() - self._folder_id = controller.get_current_folder_id() - - self._current_folder_btn.setEnabled(self._folder_id is not None) + self._current_folder_btn.setEnabled(folder_id is not None) self._controller = controller @@ -159,7 +163,7 @@ def __init__(self, controller, parent=None, items=None): self._prepare_content_data() def showEvent(self, event): - super(SwitchAssetDialog, self).showEvent(event) + super().showEvent(event) self._show_timer.start() def refresh(self, init_refresh=False): diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index a95a51ae37..918de6f7a4 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -912,14 +912,26 @@ def version_sorter(item): def _show_switch_dialog(self, item_ids): """Display Switch dialog""" - containers_by_id = self._controller.get_containers_by_item_ids( + container_items_by_id = self._controller.get_container_items_by_id( item_ids ) - dialog = SwitchAssetDialog( - self._controller, self, list(containers_by_id.values()) - ) - dialog.switched.connect(self.data_changed.emit) - dialog.show() + container_ids_by_project_name = collections.defaultdict(set) + for container_id, container_item in container_items_by_id.values(): + project_name = container_item.project_name + container_ids_by_project_name[project_name].add(container_id) + + for project_name, container_ids in container_ids_by_project_name.items(): + containers_by_id = self._controller.get_containers_by_item_ids( + container_ids + ) + dialog = SwitchAssetDialog( + self._controller, + project_name, + list(containers_by_id.values()), + self + ) + dialog.switched.connect(self.data_changed.emit) + dialog.show() def _show_remove_warning_dialog(self, item_ids): """Prompt a dialog to inform the user the action will remove items""" From 2eb97b972e0adbaef92d3a381b127d52aefa7b0d Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:12:04 +0100 Subject: [PATCH 093/136] show project name on group instead of items --- client/ayon_core/tools/sceneinventory/model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 235b125eab..885553acaf 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -292,8 +292,6 @@ def refresh(self, selected=None): item.setData(version_label, VERSION_LABEL_ROLE) item.setData(container_item.loader_name, LOADER_NAME_ROLE) item.setData(container_item.object_name, OBJECT_NAME_ROLE) - item.setData(container_item.project_name, - PROJECT_NAME_ROLE) item.setData(True, IS_CONTAINER_ITEM_ROLE) item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) container_model_items.append(item) @@ -323,6 +321,7 @@ def refresh(self, selected=None): group_item.setData(status_short, STATUS_SHORT_ROLE) group_item.setData(status_color, STATUS_COLOR_ROLE) group_item.setData(status_icon, STATUS_ICON_ROLE) + group_item.setData(project_name, PROJECT_NAME_ROLE) group_item.setData( active_site_progress, ACTIVE_SITE_PROGRESS_ROLE From 1ab7a652a3e38fe995105a3658be0ee48cc25a82 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:13:55 +0100 Subject: [PATCH 094/136] fix formatting --- client/ayon_core/tools/sceneinventory/view.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 918de6f7a4..fd67c43ac7 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -920,7 +920,9 @@ def _show_switch_dialog(self, item_ids): project_name = container_item.project_name container_ids_by_project_name[project_name].add(container_id) - for project_name, container_ids in container_ids_by_project_name.items(): + for project_name, container_ids in ( + container_ids_by_project_name.items() + ): containers_by_id = self._controller.get_containers_by_item_ids( container_ids ) From 2842c904d161b05d5e6d8b19473a41a67b6d8646 Mon Sep 17 00:00:00 2001 From: ynbot Date: Fri, 29 Nov 2024 08:17:04 +0000 Subject: [PATCH 095/136] [Automated] Update assign_pr_to_project caller workflow --- .github/workflows/assign_pr_to_project.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/assign_pr_to_project.yml b/.github/workflows/assign_pr_to_project.yml index 92d2ff2916..14e1a02075 100644 --- a/.github/workflows/assign_pr_to_project.yml +++ b/.github/workflows/assign_pr_to_project.yml @@ -17,10 +17,11 @@ on: jobs: auto-assign-pr: + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} uses: ynput/ops-repo-automation/.github/workflows/pr_to_project.yml@main with: repo: "${{ github.repository }}" - project_id: ${{ inputs.project_id || 16 }} - pull_request_number: ${{ github.event.pull_request.number || inputs.pr_number }} + project_id: "${{ inputs.project_id }}" + pull_request_number: "${{ github.event.pull_request.number || inputs.pr_number }}" secrets: token: ${{ secrets.YNPUT_BOT_TOKEN }} From 630f7f6c1e7c53cb69a92a4ed1b42820806e2bb4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 29 Nov 2024 10:59:14 +0100 Subject: [PATCH 096/136] fill values for farm with correct values --- .../publish/collect_farm_env_variables.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/client/ayon_core/plugins/publish/collect_farm_env_variables.py b/client/ayon_core/plugins/publish/collect_farm_env_variables.py index 2e28b1b164..cb52e5c32e 100644 --- a/client/ayon_core/plugins/publish/collect_farm_env_variables.py +++ b/client/ayon_core/plugins/publish/collect_farm_env_variables.py @@ -2,6 +2,7 @@ import pyblish.api +from ayon_core.lib import get_ayon_username from ayon_core.pipeline.publish import FARM_JOB_ENV_DATA_KEY @@ -15,16 +16,22 @@ def process(self, context): env = context.data.setdefault(FARM_JOB_ENV_DATA_KEY, {}) # Disable colored logs on farm - env["AYON_LOG_NO_COLORS"] = "1" + for key, value in ( + ("AYON_LOG_NO_COLORS", "1"), + ("AYON_PROJECT_NAME", context.data["projectName"]), + ("AYON_FOLDER_PATH", context.data.get("folderPath")), + ("AYON_TASK_NAME", context.data.get("task")), + # NOTE we should use 'context.data["user"]' but that has higher + # order. + ("AYON_USERNAME", get_ayon_username()), + ): + if value: + self.log.debug(f"Setting job env: {key}: {value}") + env[key] = value for key in [ "AYON_BUNDLE_NAME", "AYON_DEFAULT_SETTINGS_VARIANT", - "AYON_USERNAME", - "AYON_PROJECT_NAME", - "AYON_FOLDER_PATH", - "AYON_TASK_NAME", - "AYON_LOG_NO_COLORS", "AYON_IN_TESTS", # NOTE Not sure why workdir is needed? "AYON_WORKDIR", From fcbf8ddd91f5ad2e39ba807b24533638d81e985d Mon Sep 17 00:00:00 2001 From: Ynbot Date: Fri, 29 Nov 2024 10:25:25 +0000 Subject: [PATCH 097/136] [Automated] Add generated package files from main --- client/ayon_core/version.py | 2 +- package.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/version.py b/client/ayon_core/version.py index ab8c9424fa..a7373cd291 100644 --- a/client/ayon_core/version.py +++ b/client/ayon_core/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'core' version.""" -__version__ = "1.0.9+dev" +__version__ = "1.0.10" diff --git a/package.py b/package.py index b90db4cde4..b14c38bdd5 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "core" title = "Core" -version = "1.0.9+dev" +version = "1.0.10" client_dir = "ayon_core" diff --git a/pyproject.toml b/pyproject.toml index d09fabf8b2..31f00a0fc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.poetry] name = "ayon-core" -version = "1.0.9+dev" +version = "1.0.10" description = "" authors = ["Ynput Team "] readme = "README.md" From 457f234266f0a55486b946a626923abe341df5db Mon Sep 17 00:00:00 2001 From: Ynbot Date: Fri, 29 Nov 2024 10:26:08 +0000 Subject: [PATCH 098/136] [Automated] Update version in package.py for develop --- client/ayon_core/version.py | 2 +- package.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/version.py b/client/ayon_core/version.py index a7373cd291..b2ece45120 100644 --- a/client/ayon_core/version.py +++ b/client/ayon_core/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'core' version.""" -__version__ = "1.0.10" +__version__ = "1.0.10+dev" diff --git a/package.py b/package.py index b14c38bdd5..58ae5c08d9 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "core" title = "Core" -version = "1.0.10" +version = "1.0.10+dev" client_dir = "ayon_core" diff --git a/pyproject.toml b/pyproject.toml index 31f00a0fc2..d7cf9fa6ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.poetry] name = "ayon-core" -version = "1.0.10" +version = "1.0.10+dev" description = "" authors = ["Ynput Team "] readme = "README.md" From fdc351f4d05457516191ddf305482c8128296f69 Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Fri, 29 Nov 2024 14:33:46 +0200 Subject: [PATCH 099/136] fix a typo --- client/ayon_core/tools/attribute_defs/files_widget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/tools/attribute_defs/files_widget.py b/client/ayon_core/tools/attribute_defs/files_widget.py index 46399c5fce..aa500720ed 100644 --- a/client/ayon_core/tools/attribute_defs/files_widget.py +++ b/client/ayon_core/tools/attribute_defs/files_widget.py @@ -254,7 +254,7 @@ def _on_about_to_be_removed(self, parent_index, start, end): """Make sure that removed items are removed from items mapping. Connected with '_on_insert'. When user drag item and drop it to same - view the item is actually removed and creted again but it happens in + view the item is actually removed and created again but it happens in inner calls of Qt. """ From f40ee8f54793dd8125006935d7ea9d4fc2048fef Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Fri, 29 Nov 2024 14:34:24 +0200 Subject: [PATCH 100/136] add missing argument in `context_menu_requested` signal --- client/ayon_core/tools/attribute_defs/files_widget.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/tools/attribute_defs/files_widget.py b/client/ayon_core/tools/attribute_defs/files_widget.py index aa500720ed..6199d0c202 100644 --- a/client/ayon_core/tools/attribute_defs/files_widget.py +++ b/client/ayon_core/tools/attribute_defs/files_widget.py @@ -522,7 +522,7 @@ def lessThan(self, left, right): class ItemWidget(QtWidgets.QWidget): - context_menu_requested = QtCore.Signal(QtCore.QPoint) + context_menu_requested = QtCore.Signal(QtCore.QPoint, bool) def __init__( self, item_id, label, pixmap_icon, is_sequence, multivalue, parent=None @@ -589,7 +589,7 @@ def resizeEvent(self, event): def _on_actions_clicked(self): pos = self._split_btn.rect().bottomLeft() point = self._split_btn.mapToGlobal(pos) - self.context_menu_requested.emit(point) + self.context_menu_requested.emit(point, False) class InViewButton(IconButton): From 230ae53e4e2d1fbf2ce1c2765b657a6d64365d69 Mon Sep 17 00:00:00 2001 From: ynbot Date: Sat, 30 Nov 2024 14:31:16 +0000 Subject: [PATCH 101/136] [Automated] Update assign_pr_to_project caller workflow --- .github/workflows/assign_pr_to_project.yml | 35 +++++++++++++++++----- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/.github/workflows/assign_pr_to_project.yml b/.github/workflows/assign_pr_to_project.yml index 14e1a02075..e61d281c2a 100644 --- a/.github/workflows/assign_pr_to_project.yml +++ b/.github/workflows/assign_pr_to_project.yml @@ -3,25 +3,46 @@ on: workflow_dispatch: inputs: pr_number: - type: number + type: string description: "Run workflow for this PR number" required: true project_id: - type: number + type: string description: "Github Project Number" required: true - default: 16 + default: "16" pull_request: types: - opened +env: + GH_TOKEN: ${{ github.token }} + jobs: + get-pr-repo: + runs-on: ubuntu-latest + outputs: + pr_repo_name: ${{ steps.get-repo-name.outputs.repo_name || github.event.pull_request.head.repo.full_name }} + + # INFO `github.event.pull_request.head.repo.full_name` is not available on manual triggered (dispatched) runs + steps: + - name: Get PR repo name + if: ${{ github.event_name == 'workflow_dispatch' }} + id: get-repo-name + run: | + repo_name=$(gh pr view ${{ inputs.pr_number }} --json headRepository,headRepositoryOwner --repo ${{ github.repository }} | jq -r '.headRepositoryOwner.login + "/" + .headRepository.name') + echo "repo_name=$repo_name" >> $GITHUB_OUTPUT + auto-assign-pr: - if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + needs: + - get-pr-repo + if: ${{ needs.get-pr-repo.outputs.pr_repo_name == github.repository }} uses: ynput/ops-repo-automation/.github/workflows/pr_to_project.yml@main with: repo: "${{ github.repository }}" - project_id: "${{ inputs.project_id }}" - pull_request_number: "${{ github.event.pull_request.number || inputs.pr_number }}" + project_id: ${{ inputs.project_id != '' && fromJSON(inputs.project_id) || 16 }} + pull_request_number: ${{ github.event.pull_request.number || fromJSON(inputs.pr_number) }} secrets: - token: ${{ secrets.YNPUT_BOT_TOKEN }} + # INFO fallback to default `github.token` is required for PRs from forks + # INFO organization secrets won't be available to forks + token: ${{ secrets.YNPUT_BOT_TOKEN || github.token}} From f5a67f099d291f29789230931bc29f9b9184b4e7 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 2 Dec 2024 14:58:42 -0500 Subject: [PATCH 102/136] Append {version} regex to staging dir. --- .../pipeline/create/creator_plugins.py | 17 +++++++++++--- client/ayon_core/pipeline/publish/lib.py | 8 ++++--- client/ayon_core/pipeline/staging_dir.py | 23 +++++++++++-------- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 93e1f6f5cb..37f3e5b943 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -7,7 +7,7 @@ from abc import ABC, abstractmethod from ayon_core.settings import get_project_settings -from ayon_core.lib import Logger +from ayon_core.lib import Logger, get_version_from_path from ayon_core.pipeline.plugin_discover import ( discover, register_plugin, @@ -860,6 +860,14 @@ def apply_staging_dir(self, instance): else: template_data = {} + # TODO: confirm feature + anatomy_data_settings = self.project_settings["core"]["publish"]["CollectAnatomyInstanceData"] + follow_workfile_version = anatomy_data_settings["follow_workfile_version"] + if follow_workfile_version: + current_workfile = self.create_context.get_current_workfile_path() + workfile_version = get_version_from_path(current_workfile) + template_data = {"version": int(workfile_version)} + staging_dir_info = get_staging_dir_info( create_ctx.get_current_project_entity(), create_ctx.get_current_folder_entity(), @@ -877,12 +885,15 @@ def apply_staging_dir(self, instance): if not staging_dir_info: return None - staging_dir_path = staging_dir_info["stagingDir"] + staging_dir_path = staging_dir_info.dir # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) - instance.transient_data.update(staging_dir_info) + instance.transient_data.update({ + "stagingDir": staging_dir_path, + "stagingDir_persistent": staging_dir_info.persistent, + }) self.log.info(f"Applied staging dir to instance: {staging_dir_path}") diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index c0dfe8c910..b86e439b72 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -710,12 +710,14 @@ def get_instance_staging_dir(instance): always_return_path=True, ) - staging_dir_path = staging_dir_info["stagingDir"] + staging_dir_path = staging_dir_info.dir # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) - - instance.data.update(staging_dir_info) + instance.data.update({ + "stagingDir": staging_dir_path, + "stagingDir_persistent": staging_dir_info.persistent, + }) return staging_dir_path diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index c8e3251e7b..fa216712b2 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -1,3 +1,5 @@ +from collections import namedtuple + from ayon_core.lib import Logger, filter_profiles, StringTemplate from ayon_core.settings import get_project_settings @@ -6,6 +8,9 @@ from .tempdir import get_temp_dir +StagingDir = namedtuple("StagingDir", ["dir", "persistent"]) + + def get_staging_dir_config( project_name, task_type, @@ -144,7 +149,7 @@ def get_staging_dir_info( suffix (Optional[str]): Optional suffix for staging dir name. Returns: - Optional[Dict[str, Any]]: Staging dir info data + Optional[StagingDir]: Staging dir info data """ task_entity = task_entity or {} @@ -195,24 +200,24 @@ def get_staging_dir_info( ) if staging_dir_config: - return { - "stagingDir": StringTemplate.format_template( + return StagingDir( + StringTemplate.format_template( str(staging_dir_config["template"]["directory"]), ctx_data ), - "stagingDir_persistent": staging_dir_config["persistence"], - } + staging_dir_config["persistence"], + ) # no config found but force an output if always_return_path: - return { - "stagingDir": get_temp_dir( + return StagingDir( + get_temp_dir( project_name=project_entity["name"], anatomy=anatomy, prefix=prefix, suffix=suffix, ), - "stagingDir_persistent": False, - } + False, + ) return None From fa014fa93cdd311ea7086d5cc3216deb14c3c14d Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 2 Dec 2024 15:02:08 -0500 Subject: [PATCH 103/136] Fix lint. --- client/ayon_core/pipeline/create/creator_plugins.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 37f3e5b943..87f67a3e80 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -861,8 +861,9 @@ def apply_staging_dir(self, instance): template_data = {} # TODO: confirm feature - anatomy_data_settings = self.project_settings["core"]["publish"]["CollectAnatomyInstanceData"] - follow_workfile_version = anatomy_data_settings["follow_workfile_version"] + publish_settings = self.project_settings["core"]["publish"] + anatomy_settings = publish_settings["CollectAnatomyInstanceData"] + follow_workfile_version = anatomy_settings["follow_workfile_version"] if follow_workfile_version: current_workfile = self.create_context.get_current_workfile_path() workfile_version = get_version_from_path(current_workfile) From 9a0e490233151c116ea0ec2e88ad93fde1d7adda Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 3 Dec 2024 16:52:19 +0800 Subject: [PATCH 104/136] use items() for key, value --- client/ayon_core/tools/sceneinventory/view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index fd67c43ac7..bb95e37d4e 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -916,7 +916,7 @@ def _show_switch_dialog(self, item_ids): item_ids ) container_ids_by_project_name = collections.defaultdict(set) - for container_id, container_item in container_items_by_id.values(): + for container_id, container_item in container_items_by_id.items(): project_name = container_item.project_name container_ids_by_project_name[project_name].add(container_id) From 648c2c52fe0ac20e1f8bcdb74c19aa6089db0fff Mon Sep 17 00:00:00 2001 From: Robin De Lillo Date: Tue, 3 Dec 2024 10:50:09 -0500 Subject: [PATCH 105/136] Apply suggestions from code review Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../pipeline/create/creator_plugins.py | 11 +++-- client/ayon_core/pipeline/staging_dir.py | 43 ++++++++++--------- client/ayon_core/pipeline/tempdir.py | 3 +- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 87f67a3e80..780cb71fca 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -862,8 +862,11 @@ def apply_staging_dir(self, instance): # TODO: confirm feature publish_settings = self.project_settings["core"]["publish"] - anatomy_settings = publish_settings["CollectAnatomyInstanceData"] - follow_workfile_version = anatomy_settings["follow_workfile_version"] + follow_workfile_version = ( + publish_settings + ["CollectAnatomyInstanceData"] + ["follow_workfile_version"] + ) if follow_workfile_version: current_workfile = self.create_context.get_current_workfile_path() workfile_version = get_version_from_path(current_workfile) @@ -871,8 +874,8 @@ def apply_staging_dir(self, instance): staging_dir_info = get_staging_dir_info( create_ctx.get_current_project_entity(), - create_ctx.get_current_folder_entity(), - create_ctx.get_current_task_entity(), + create_ctx.get_folder_entity(folder_path), + create_ctx.get_task_entity(folder_path, instance.get("task")), product_type, product_name, create_ctx.host_name, diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index fa216712b2..3c1c7c1ab2 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -8,7 +8,10 @@ from .tempdir import get_temp_dir -StagingDir = namedtuple("StagingDir", ["dir", "persistent"]) +@dataclass +class StagingDir: + directory: str + persistent: bool def get_staging_dir_config( @@ -78,10 +81,9 @@ def get_staging_dir_config( if not template: # template should always be found either from anatomy or from profile - raise ValueError( - "Staging dir profile is misconfigured! " - f"No template was found for profile: {profile}! " - "Check your project settings at: " + raise KeyError( + f"Staging template '{template_name}' was not found." + "Check project anatomy or settings at: " "'ayon+settings://core/tools/publish/custom_staging_dir_profiles'" ) @@ -98,10 +100,8 @@ def _validate_template_name(project_name, template_name, anatomy): """ if template_name not in anatomy.templates["staging"]: raise ValueError( - ( - 'Anatomy of project "{}" does not have set' - ' "{}" template key at Staging Dir section!' - ).format(project_name, template_name) + f'Anatomy of project "{project_name}" does not have set' + f' "{template_name}" template key at Staging Dir category!' ) @@ -131,14 +131,14 @@ def get_staging_dir_info( Arguments: host_name (str): Name of host. project_entity (Dict[str, Any]): Project entity. - folder_entity (Dict[str, Any]): Folder entity. - task_entity (Dict[str, Any]): Task entity. + folder_entity (Optional[Dict[str, Any]]): Folder entity. + task_entity (Optional[Dict[str, Any]]): Task entity. product_type (str): Type of product. product_name (str): Name of product. - anatomy (ayon_core.pipeline.Anatomy): Anatomy object. + anatomy (Optional[Anatomy]): Anatomy object. project_settings (Optional[Dict[str, Any]]): Prepared project settings. - template_data (Optional[Dict[str, Any]]): Data for formatting staging - dir template. + template_data (Optional[Dict[str, Any]]): Additional data for + formatting staging dir template. always_return_path (Optional[bool]): If True, staging dir will be created as tempdir if no staging dir profile is found. Input value False will return None if no staging dir profile is found. @@ -152,7 +152,6 @@ def get_staging_dir_info( Optional[StagingDir]: Staging dir info data """ - task_entity = task_entity or {} log = logger or Logger.get_logger("get_staging_dir_info") if anatomy is None: @@ -185,12 +184,16 @@ def get_staging_dir_info( # add additional template formatting data if template_data: ctx_data.update(template_data) + task_name = task_type = None + if task_entity: + task_name = task_entity["name"] + task_type = task_entity["taskType"] # get staging dir config staging_dir_config = get_staging_dir_config( project_entity["name"], - task_entity.get("taskType"), - task_entity.get("name"), + task_type, + task_name , product_type, product_name, host_name, @@ -200,11 +203,9 @@ def get_staging_dir_info( ) if staging_dir_config: + dir_template = staging_dir_config["template"]["directory"] return StagingDir( - StringTemplate.format_template( - str(staging_dir_config["template"]["directory"]), - ctx_data - ), + dir_template.format_strict(ctx_data), staging_dir_config["persistence"], ) diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index af2ff44a8f..52995d3f6a 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -36,7 +36,8 @@ def get_temp_dir( str: Path to staging dir of instance. """ - prefix = prefix or "ay_tmp_" + if prefix is None: + prefix = "ay_tmp_" suffix = suffix or "" if use_local_temp: From 0672f5c8bb2125ebaafed227f48111e1b1396aeb Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 3 Dec 2024 11:38:29 -0500 Subject: [PATCH 106/136] Address feedback from PR. --- .../pipeline/create/creator_plugins.py | 57 ++++++++++++------- client/ayon_core/pipeline/publish/lib.py | 2 +- client/ayon_core/pipeline/staging_dir.py | 9 +-- 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 780cb71fca..6ccafe1bc7 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -833,17 +833,15 @@ def get_pre_create_attr_defs(self): """ return self.pre_create_attr_defs - def apply_staging_dir(self, instance): - """Apply staging dir with persistence to instance's transient data. - - Method is called on instance creation and on instance update. + def get_staging_dir(self, instance): + """Return the staging dir and persistence from instance. Args: instance (CreatedInstance): Instance for which should be staging - dir applied. + dir gathered. Returns: - Optional[str]: Staging dir path or None if not applied. + Optional[namedtuple]: Staging dir path and persistence or None """ create_ctx = self.create_context product_name = instance.get("productName") @@ -852,25 +850,32 @@ def apply_staging_dir(self, instance): # this can only work if product name and folder path are available if not product_name or not folder_path: - return - - version = instance.get("version") - if version is not None: - template_data = {"version": version} - else: - template_data = {} + return None - # TODO: confirm feature publish_settings = self.project_settings["core"]["publish"] follow_workfile_version = ( publish_settings ["CollectAnatomyInstanceData"] ["follow_workfile_version"] ) - if follow_workfile_version: + + # Gather version number provided from the instance. + version = instance.get("version") + + # If follow workfile, gather version from workfile path. + if version is None and follow_workfile_version: current_workfile = self.create_context.get_current_workfile_path() workfile_version = get_version_from_path(current_workfile) - template_data = {"version": int(workfile_version)} + version = int(workfile_version) + + # Fill-up version with next version available. + elif version is None: + versions = self.get_next_versions_for_instances( + [instance] + ) + version, = tuple(versions.values()) + + template_data = {"version": version} staging_dir_info = get_staging_dir_info( create_ctx.get_current_project_entity(), @@ -886,12 +891,26 @@ def apply_staging_dir(self, instance): template_data=template_data, ) - if not staging_dir_info: - return None + return staging_dir_info or None + + def apply_staging_dir(self, instance): + """Apply staging dir with persistence to instance's transient data. + + Method is called on instance creation and on instance update. - staging_dir_path = staging_dir_info.dir + Args: + instance (CreatedInstance): Instance for which should be staging + dir applied. + + Returns: + Optional[str]: Staging dir path or None if not applied. + """ + staging_dir_info = self.get_staging_dir(instance) + if staging_dir_info is None: + return None # path might be already created by get_staging_dir_info + staging_dir_path = staging_dir_info.directory os.makedirs(staging_dir_path, exist_ok=True) instance.transient_data.update({ diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index b86e439b72..2ba40d7687 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -710,7 +710,7 @@ def get_instance_staging_dir(instance): always_return_path=True, ) - staging_dir_path = staging_dir_info.dir + staging_dir_path = staging_dir_info.directory # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index 3c1c7c1ab2..0317e55720 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -1,6 +1,6 @@ -from collections import namedtuple +from dataclasses import dataclass -from ayon_core.lib import Logger, filter_profiles, StringTemplate +from ayon_core.lib import Logger, filter_profiles from ayon_core.settings import get_project_settings from .template_data import get_template_data @@ -42,7 +42,7 @@ def get_staging_dir_config( Dict or None: Data with directory template and is_persistent or None Raises: - ValueError - if misconfigured template should be used + KeyError - if misconfigured template should be used """ settings = project_settings or get_project_settings(project_name) @@ -129,12 +129,12 @@ def get_staging_dir_info( If `prefix` or `suffix` is not set, default values will be used. Arguments: - host_name (str): Name of host. project_entity (Dict[str, Any]): Project entity. folder_entity (Optional[Dict[str, Any]]): Folder entity. task_entity (Optional[Dict[str, Any]]): Task entity. product_type (str): Type of product. product_name (str): Name of product. + host_name (str): Name of host. anatomy (Optional[Anatomy]): Anatomy object. project_settings (Optional[Dict[str, Any]]): Prepared project settings. template_data (Optional[Dict[str, Any]]): Additional data for @@ -184,6 +184,7 @@ def get_staging_dir_info( # add additional template formatting data if template_data: ctx_data.update(template_data) + task_name = task_type = None if task_entity: task_name = task_entity["name"] From ed7752a4df7fdfc2c1a27de98ccf890c9a5395ea Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 3 Dec 2024 16:56:53 -0500 Subject: [PATCH 107/136] Fix extract_otio_review --- client/ayon_core/plugins/publish/extract_otio_review.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/ayon_core/plugins/publish/extract_otio_review.py b/client/ayon_core/plugins/publish/extract_otio_review.py index fb9b269258..c8d2086865 100644 --- a/client/ayon_core/plugins/publish/extract_otio_review.py +++ b/client/ayon_core/plugins/publish/extract_otio_review.py @@ -78,6 +78,7 @@ def process(self, instance): if otio_review_clips is None: self.log.info(f"Instance `{instance}` has no otioReviewClips") + return # add plugin wide attributes self.representation_files = [] From 9bcc9b40191d0ecdbd435e5a58e714f6dfbd86fe Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:08:04 +0100 Subject: [PATCH 108/136] fix 'realy' typo to 'really' --- client/ayon_core/lib/path_templates.py | 35 +++++++++++++++++++------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index 9b545f2851..bc4ed648b7 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -292,7 +292,7 @@ def __init__(self, optional=False): # Used values stored by key with all modifirs # - value is already formatted string # Example: {"version:0>3": "001"} - self._realy_used_values = {} + self._really_used_values: Dict[str, Any] = {} # Concatenated string output after formatting self._output = "" # Is this result from optional part @@ -314,7 +314,7 @@ def add_output(self, other): if other.optional and not other.solved: return self._used_values.update(other.used_values) - self._realy_used_values.update(other.realy_used_values) + self._really_used_values.update(other.really_used_values) else: raise TypeError("Cannot add data from \"{}\" to \"{}\"".format( @@ -359,8 +359,17 @@ def invalid_optional_types(self): return self._invalid_optional_types @property - def realy_used_values(self): - return self._realy_used_values + def really_used_values(self) -> Dict[str, Any]: + return self._really_used_values + + @property + def realy_used_values(self) -> Dict[str, Any]: + warnings.warn( + "Property 'realy_used_values' is deprecated." + " Use 'really_used_values' instead.", + DeprecationWarning + ) + return self._really_used_values @property def used_values(self): @@ -391,8 +400,16 @@ def get_clean_used_values(self): return self.split_keys_to_subdicts(new_used_values) - def add_realy_used_value(self, key, value): - self._realy_used_values[key] = value + def add_really_used_value(self, key: str, value: Any): + self._really_used_values[key] = value + + def add_realy_used_value(self, key: str, value: Any): + warnings.warn( + "Method 'add_realy_used_value' is deprecated." + " Use 'add_really_used_value' instead.", + DeprecationWarning + ) + self.add_really_used_value(key, value) def add_used_value(self, key, value): self._used_values[key] = value @@ -519,8 +536,8 @@ def format(self, data, result): result(TemplatePartResult): Object where result is stored. """ key = self._template_base - if key in result.realy_used_values: - result.add_output(result.realy_used_values[key]) + if key in result.really_used_values: + result.add_output(result.really_used_values[key]) return result # ensure key is properly formed [({})] properly closed. @@ -625,7 +642,7 @@ def format(self, data, result): used_value = value else: used_value = formatted_value - result.add_realy_used_value(self._field_name, used_value) + result.add_really_used_value(self._field_name, used_value) result.add_used_value(used_key, used_value) result.add_output(formatted_value) return result From a80bbfbd5764b33bf51e44b0e92152e6311b58e0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:13:33 +0100 Subject: [PATCH 109/136] added basic typehints --- client/ayon_core/lib/path_templates.py | 174 +++++++++++++++---------- 1 file changed, 103 insertions(+), 71 deletions(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index bc4ed648b7..3e66344ff4 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -2,8 +2,13 @@ import re import copy import numbers -from typing import List +import warnings from string import Formatter +import typing +from typing import List, Dict, Any, Set, Optional + +if typing.TYPE_CHECKING: + from typing import Union SUB_DICT_PATTERN = re.compile(r"([^\[\]]+)") OPTIONAL_PATTERN = re.compile(r"(<.*?[^{0]*>)[^0-9]*?") @@ -19,9 +24,7 @@ class TemplateUnsolved(Exception): def __init__(self, template, missing_keys, invalid_types): invalid_type_items = [] for _key, _type in invalid_types.items(): - invalid_type_items.append( - "\"{0}\" {1}".format(_key, str(_type)) - ) + invalid_type_items.append(f"\"{_key}\" {str(_type)}") invalid_types_msg = "" if invalid_type_items: @@ -34,20 +37,21 @@ def __init__(self, template, missing_keys, invalid_types): missing_keys_msg = self.missing_keys_msg.format( ", ".join(missing_keys) ) - super(TemplateUnsolved, self).__init__( + super().__init__( self.msg.format(template, missing_keys_msg, invalid_types_msg) ) class StringTemplate: """String that can be formatted.""" - def __init__(self, template): + def __init__(self, template: str): if not isinstance(template, str): - raise TypeError("<{}> argument must be a string, not {}.".format( - self.__class__.__name__, str(type(template)) - )) + raise TypeError( + f"<{self.__class__.__name__}> argument must be a string," + f" not {str(type(template))}." + ) - self._template = template + self._template: str = template parts = [] formatter = Formatter() @@ -78,15 +82,17 @@ def __init__(self, template): if substr: new_parts.append(substr) - self._parts = self.find_optional_parts(new_parts) + self._parts: List["Union[str, OptionalPart, FormattingPart]"] = ( + self.find_optional_parts(new_parts) + ) - def __str__(self): + def __str__(self) -> str: return self.template - def __repr__(self): - return "<{}> {}".format(self.__class__.__name__, self.template) + def __repr__(self) -> str: + return f"<{self.__class__.__name__}> {self.template}" - def __contains__(self, other): + def __contains__(self, other: str) -> bool: return other in self.template def replace(self, *args, **kwargs): @@ -94,10 +100,10 @@ def replace(self, *args, **kwargs): return self @property - def template(self): + def template(self) -> str: return self._template - def format(self, data): + def format(self, data: Dict[str, Any]) -> "TemplateResult": """ Figure out with whole formatting. Separate advanced keys (*Like '{project[name]}') from string which must @@ -109,6 +115,7 @@ def format(self, data): Returns: TemplateResult: Filled or partially filled template containing all data needed or missing for filling template. + """ result = TemplatePartResult() for part in self._parts: @@ -136,23 +143,29 @@ def format(self, data): invalid_types ) - def format_strict(self, *args, **kwargs): - result = self.format(*args, **kwargs) + def format_strict(self, data: Dict[str, Any]) -> "TemplateResult": + result = self.format(data) result.validate() return result @classmethod - def format_template(cls, template, data): + def format_template( + cls, template: str, data: Dict[str, Any] + ) -> "TemplateResult": objected_template = cls(template) return objected_template.format(data) @classmethod - def format_strict_template(cls, template, data): + def format_strict_template( + cls, template: str, data: Dict[str, Any] + ) -> "TemplateResult": objected_template = cls(template) return objected_template.format_strict(data) @staticmethod - def find_optional_parts(parts): + def find_optional_parts( + parts: List["Union[str, FormattingPart]"] + ) -> List["Union[str, OptionalPart, FormattingPart]"]: new_parts = [] tmp_parts = {} counted_symb = -1 @@ -217,11 +230,11 @@ class TemplateResult(str): of number. """ - used_values = None - solved = None - template = None - missing_keys = None - invalid_types = None + used_values: Dict[str, Any] = None + solved: bool = None + template: str = None + missing_keys: List[str] = None + invalid_types: Dict[str, Any] = None def __new__( cls, filled_template, template, solved, @@ -249,7 +262,7 @@ def validate(self): self.invalid_types ) - def copy(self): + def copy(self) -> "TemplateResult": cls = self.__class__ return cls( str(self), @@ -260,7 +273,7 @@ def copy(self): self.invalid_types ) - def normalized(self): + def normalized(self) -> "TemplateResult": """Convert to normalized path.""" cls = self.__class__ @@ -276,27 +289,28 @@ def normalized(self): class TemplatePartResult: """Result to store result of template parts.""" - def __init__(self, optional=False): + def __init__(self, optional: bool = False): # Missing keys or invalid value types of required keys - self._missing_keys = set() - self._invalid_types = {} + self._missing_keys: Set[str] = set() + self._invalid_types: Dict[str, Any] = {} # Missing keys or invalid value types of optional keys - self._missing_optional_keys = set() - self._invalid_optional_types = {} + self._missing_optional_keys: Set[str] = set() + self._invalid_optional_types: Dict[str, Any] = {} # Used values stored by key with origin type # - key without any padding or key modifiers # - value from filling data # Example: {"version": 1} - self._used_values = {} + self._used_values: Dict[str, Any] = {} # Used values stored by key with all modifirs # - value is already formatted string # Example: {"version:0>3": "001"} self._really_used_values: Dict[str, Any] = {} # Concatenated string output after formatting - self._output = "" + self._output: str = "" # Is this result from optional part - self._optional = True + # TODO find out why we don't use 'optional' from args + self._optional: bool = True def add_output(self, other): if isinstance(other, str): @@ -322,7 +336,7 @@ def add_output(self, other): ) @property - def solved(self): + def solved(self) -> bool: if self.optional: if ( len(self.missing_optional_keys) > 0 @@ -335,27 +349,27 @@ def solved(self): ) @property - def optional(self): + def optional(self) -> bool: return self._optional @property - def output(self): + def output(self) -> str: return self._output @property - def missing_keys(self): + def missing_keys(self) -> Set[str]: return self._missing_keys @property - def missing_optional_keys(self): + def missing_optional_keys(self) -> Set[str]: return self._missing_optional_keys @property - def invalid_types(self): + def invalid_types(self) -> Dict[str, Any]: return self._invalid_types @property - def invalid_optional_types(self): + def invalid_optional_types(self) -> Dict[str, Any]: return self._invalid_optional_types @property @@ -372,11 +386,11 @@ def realy_used_values(self) -> Dict[str, Any]: return self._really_used_values @property - def used_values(self): + def used_values(self) -> Dict[str, Any]: return self._used_values @staticmethod - def split_keys_to_subdicts(values): + def split_keys_to_subdicts(values: Dict[str, Any]) -> Dict[str, Any]: output = {} formatter = Formatter() for key, value in values.items(): @@ -391,7 +405,7 @@ def split_keys_to_subdicts(values): data[last_key] = value return output - def get_clean_used_values(self): + def get_clean_used_values(self) -> Dict[str, Any]: new_used_values = {} for key, value in self.used_values.items(): if isinstance(value, FormatObject): @@ -411,16 +425,16 @@ def add_realy_used_value(self, key: str, value: Any): ) self.add_really_used_value(key, value) - def add_used_value(self, key, value): + def add_used_value(self, key: str, value: Any): self._used_values[key] = value - def add_missing_key(self, key): + def add_missing_key(self, key: str): if self._optional: self._missing_optional_keys.add(key) else: self._missing_keys.add(key) - def add_invalid_type(self, key, value): + def add_invalid_type(self, key: str, value: Any): if self._optional: self._invalid_optional_types[key] = type(value) else: @@ -438,10 +452,10 @@ def __init__(self): def __format__(self, *args, **kwargs): return self.value.__format__(*args, **kwargs) - def __str__(self): + def __str__(self) -> str: return str(self.value) - def __repr__(self): + def __repr__(self) -> str: return self.__str__() @@ -451,9 +465,17 @@ class FormattingPart: Containt only single key to format e.g. "{project[name]}". Args: - template(str): String containing the formatting key. + field_name (str): Name of key. + format_spec (str): Format specification. + conversion (Union[str, None]): Conversion type. + """ - def __init__(self, field_name, format_spec, conversion): + def __init__( + self, + field_name: str, + format_spec: str, + conversion: "Union[str, None]", + ): format_spec_v = "" if format_spec: format_spec_v = f":{format_spec}" @@ -461,26 +483,26 @@ def __init__(self, field_name, format_spec, conversion): if conversion: conversion_v = f"!{conversion}" - self._field_name = field_name - self._format_spec = format_spec_v - self._conversion = conversion_v + self._field_name: str = field_name + self._format_spec: str = format_spec_v + self._conversion: str = conversion_v template_base = f"{field_name}{format_spec_v}{conversion_v}" - self._template_base = template_base - self._template = f"{{{template_base}}}" + self._template_base: str = template_base + self._template: str = f"{{{template_base}}}" @property - def template(self): + def template(self) -> str: return self._template - def __repr__(self): + def __repr__(self) -> str: return "".format(self._template) - def __str__(self): + def __str__(self) -> str: return self._template @staticmethod - def validate_value_type(value): + def validate_value_type(value: Any) -> bool: """Check if value can be used for formatting of single key.""" if isinstance(value, (numbers.Number, FormatObject)): return True @@ -491,7 +513,7 @@ def validate_value_type(value): return False @staticmethod - def validate_key_is_matched(key): + def validate_key_is_matched(key: str) -> bool: """Validate that opening has closing at correct place. Future-proof, only square brackets are currently used in keys. @@ -528,12 +550,15 @@ def keys_to_template_base(keys: List[str]): joined_keys = "".join([f"[{key}]" for key in keys]) return f"{template_base}{joined_keys}" - def format(self, data, result): + def format( + self, data: Dict[str, Any], result: TemplatePartResult + ) -> TemplatePartResult: """Format the formattings string. Args: data(dict): Data that should be used for formatting. result(TemplatePartResult): Object where result is stored. + """ key = self._template_base if key in result.really_used_values: @@ -658,20 +683,27 @@ class OptionalPart: 'FormattingPart'. """ - def __init__(self, parts): - self._parts = parts + def __init__( + self, + parts: List["Union[str, OptionalPart, FormattingPart]"] + ): + self._parts: List["Union[str, OptionalPart, FormattingPart]"] = parts @property - def parts(self): + def parts(self) -> List["Union[str, OptionalPart, FormattingPart]"]: return self._parts - def __str__(self): + def __str__(self) -> str: return "<{}>".format("".join([str(p) for p in self._parts])) - def __repr__(self): + def __repr__(self) -> str: return "".format("".join([str(p) for p in self._parts])) - def format(self, data, result): + def format( + self, + data: Dict[str, Any], + result: TemplatePartResult, + ) -> TemplatePartResult: new_result = TemplatePartResult(True) for part in self._parts: if isinstance(part, str): From 04daa9306c3a0f5d70d24fa75fe936a415c6532f Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:14:59 +0100 Subject: [PATCH 110/136] remove unused import --- client/ayon_core/lib/path_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index 3e66344ff4..e3cae78a87 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -5,7 +5,7 @@ import warnings from string import Formatter import typing -from typing import List, Dict, Any, Set, Optional +from typing import List, Dict, Any, Set if typing.TYPE_CHECKING: from typing import Union From cc23f407afb8d78816dd8c124b0236cf9b8dd2ad Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Wed, 4 Dec 2024 11:23:29 -0500 Subject: [PATCH 111/136] Address feedback from PR. --- client/ayon_core/pipeline/staging_dir.py | 11 ++++------- client/ayon_core/pipeline/tempdir.py | 6 ++---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index 0317e55720..ea22d99389 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -173,13 +173,10 @@ def get_staging_dir_info( ) # add additional data - ctx_data.update({ - "product": { - "type": product_type, - "name": product_name - }, - "root": anatomy.roots - }) + ctx_data["product"] = { + "type": product_type, + "name": product_name + } # add additional template formatting data if template_data: diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index 52995d3f6a..fe057b7fc7 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -65,11 +65,9 @@ def _create_local_staging_dir(prefix, suffix, dirpath=None): str: path to tempdir """ # use pathlib for creating tempdir - staging_dir = Path(tempfile.mkdtemp( + return tempfile.mkdtemp( prefix=prefix, suffix=suffix, dir=dirpath - )) - - return staging_dir.as_posix() + ) def _create_custom_tempdir(project_name, anatomy): From 9f590cd2cec1656c19c407bc77a19dfe728f3fc1 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Wed, 4 Dec 2024 16:37:25 -0500 Subject: [PATCH 112/136] Implement review representations in OTIO subset resources. --- .../publish/collect_otio_subset_resources.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py index c142036b83..f7b1c9d9b2 100644 --- a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py +++ b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py @@ -149,6 +149,7 @@ def process(self, instance): self.log.info( "frame_start-frame_end: {}-{}".format(frame_start, frame_end)) + review_repre = None if is_sequence: # file sequence way @@ -177,6 +178,11 @@ def process(self, instance): repre = self._create_representation( frame_start, frame_end, collection=collection) + if "review" in instance.data["families"]: + review_repre = self._create_representation( + frame_start, frame_end, collection=collection, + delete=True, review=True) + else: _trim = False dirname, filename = os.path.split(media_ref.target_url) @@ -191,17 +197,26 @@ def process(self, instance): repre = self._create_representation( frame_start, frame_end, file=filename, trim=_trim) + if "review" in instance.data["families"]: + review_repre = self._create_representation( + frame_start, frame_end, + file=filename, delete=True, review=True) + instance.data["originalDirname"] = self.staging_dir + # add representation to instance data if repre: colorspace = instance.data.get("colorspace") # add colorspace data to representation self.set_representation_colorspace( repre, instance.context, colorspace) - # add representation to instance data instance.data["representations"].append(repre) + # add review representation to instance data + if review_repre: + instance.data["representations"].append(review_repre) + self.log.debug(instance.data) def _create_representation(self, start, end, **kwargs): @@ -221,7 +236,8 @@ def _create_representation(self, start, end, **kwargs): representation_data = { "frameStart": start, "frameEnd": end, - "stagingDir": self.staging_dir + "stagingDir": self.staging_dir, + "tags": [], } if kwargs.get("collection"): @@ -247,8 +263,10 @@ def _create_representation(self, start, end, **kwargs): "frameEnd": end, }) - if kwargs.get("trim") is True: - representation_data["tags"] = ["trim"] + for tag_name in ("trim", "delete", "review"): + if kwargs.get(tag_name) is True: + representation_data["tags"].append(tag_name) + return representation_data def get_template_name(self, instance): From 156d3e6a1cd1e9807486ba8c5278f382b3c15058 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Wed, 4 Dec 2024 16:45:20 -0500 Subject: [PATCH 113/136] Fix lint. --- .../ayon_core/plugins/publish/collect_otio_subset_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py index f7b1c9d9b2..cc1ef3edef 100644 --- a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py +++ b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py @@ -215,7 +215,7 @@ def process(self, instance): # add review representation to instance data if review_repre: - instance.data["representations"].append(review_repre) + instance.data["representations"].append(review_repre) self.log.debug(instance.data) From 023e0722f8935b84238292f283a90916e920bedc Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:43:18 +0100 Subject: [PATCH 114/136] capture all possible errors that can happen during UUID conversion --- client/ayon_core/tools/loader/control.py | 2 +- client/ayon_core/tools/sceneinventory/models/containers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/tools/loader/control.py b/client/ayon_core/tools/loader/control.py index 2da77337fb..412e6677f0 100644 --- a/client/ayon_core/tools/loader/control.py +++ b/client/ayon_core/tools/loader/control.py @@ -382,7 +382,7 @@ def get_loaded_product_ids(self): try: uuid.UUID(repre_id) repre_ids.add(repre_id) - except ValueError: + except (ValueError, TypeError, AttributeError): pass product_ids = self._products_model.get_product_ids_by_repre_ids( diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 9059485dff..572a96976b 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -230,7 +230,7 @@ def get_representation_info_items(self, project_name, representation_ids): for repre_id in representation_ids: try: uuid.UUID(repre_id) - except ValueError: + except (ValueError, TypeError, AttributeError): output[repre_id] = RepresentationInfo.new_invalid() continue repre_info = self._repre_info_by_id.get(repre_id) From 2292ecbac11da62427c2007665f587123503cc66 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:45:19 +0100 Subject: [PATCH 115/136] log about invalid representation id --- client/ayon_core/tools/sceneinventory/models/containers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 572a96976b..08b86f6456 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -4,6 +4,7 @@ import ayon_api from ayon_api.graphql import GraphQlQuery +from ayon_core.lib import Logger from ayon_core.host import ILoadHost from ayon_core.tools.common_models.projects import StatusStates @@ -196,6 +197,7 @@ def __init__(self, controller): self._container_items_by_id = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} + self._log = Logger.get_logger("ContainersModel") def reset(self): self._items_cache = None @@ -368,6 +370,10 @@ def _update_cache(self): try: uuid.UUID(repre_id) except (ValueError, TypeError, AttributeError): + self._log.warning( + "Container contains invalid representation id." + f"\n{container}" + ) # Fake not existing representation id so container # is shown in UI but as invalid item.representation_id = invalid_ids_mapping.setdefault( From 373df562543b1fed3c8d00d0b425cd6cbddf61aa Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:45:35 +0100 Subject: [PATCH 116/136] fix calling of missing method --- client/ayon_core/tools/sceneinventory/models/containers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 08b86f6456..f25ef2b94c 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -380,10 +380,10 @@ def _update_cache(self): repre_id, uuid.uuid4().hex ) - except Exception as e: + except Exception: # skip item if required data are missing - self._controller.log_error( - f"Failed to create item: {e}" + self._log.warning( + f"Failed to create container item", exc_info=True ) continue From b6d3ddc1c8f288e98d68f528334c8c61394f3ecd Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:54:47 +0100 Subject: [PATCH 117/136] more safeguard for invalid containers --- client/ayon_core/tools/loader/control.py | 14 +++++++------- .../tools/sceneinventory/models/containers.py | 17 +++++++++++------ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/client/ayon_core/tools/loader/control.py b/client/ayon_core/tools/loader/control.py index 412e6677f0..4ce220f282 100644 --- a/client/ayon_core/tools/loader/control.py +++ b/client/ayon_core/tools/loader/control.py @@ -372,14 +372,14 @@ def get_loaded_product_ids(self): repre_ids = set() for container in containers: - repre_id = container.get("representation") - # Ignore invalid representation ids. - # - invalid representation ids may be available if e.g. is - # opened scene from OpenPype whe 'ObjectId' was used instead - # of 'uuid'. - # NOTE: Server call would crash if there is any invalid id. - # That would cause crash we won't get any information. try: + repre_id = container.get("representation") + # Ignore invalid representation ids. + # - invalid representation ids may be available if e.g. is + # opened scene from OpenPype whe 'ObjectId' was used instead + # of 'uuid'. + # NOTE: Server call would crash if there is any invalid id. + # That would cause crash we won't get any information. uuid.UUID(repre_id) repre_ids.add(repre_id) except (ValueError, TypeError, AttributeError): diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index f25ef2b94c..c761121d4d 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -350,12 +350,14 @@ def _update_cache(self): return host = self._controller.get_host() - if isinstance(host, ILoadHost): - containers = list(host.get_containers()) - elif hasattr(host, "ls"): - containers = list(host.ls()) - else: - containers = [] + containers = [] + try: + if isinstance(host, ILoadHost): + containers = list(host.get_containers()) + elif hasattr(host, "ls"): + containers = list(host.ls()) + except Exception: + self._log.error("Failed to get containers", exc_info=True) container_items = [] containers_by_id = {} @@ -363,6 +365,9 @@ def _update_cache(self): invalid_ids_mapping = {} current_project_name = self._controller.get_current_project_name() for container in containers: + if not container: + continue + try: item = ContainerItem.from_container_data( current_project_name, container) From 3c697b92f57fa597364da39b7e73a03ad963e563 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:57:22 +0100 Subject: [PATCH 118/136] remove unnecessary f-string --- client/ayon_core/tools/sceneinventory/models/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index c761121d4d..f841f87c8e 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -388,7 +388,7 @@ def _update_cache(self): except Exception: # skip item if required data are missing self._log.warning( - f"Failed to create container item", exc_info=True + "Failed to create container item", exc_info=True ) continue From a26e9207d2957402c3de2a23fc0633d85e094675 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:10:33 +0100 Subject: [PATCH 119/136] fix long line --- client/ayon_core/tools/loader/control.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/tools/loader/control.py b/client/ayon_core/tools/loader/control.py index 4ce220f282..16cf7c31c7 100644 --- a/client/ayon_core/tools/loader/control.py +++ b/client/ayon_core/tools/loader/control.py @@ -376,8 +376,8 @@ def get_loaded_product_ids(self): repre_id = container.get("representation") # Ignore invalid representation ids. # - invalid representation ids may be available if e.g. is - # opened scene from OpenPype whe 'ObjectId' was used instead - # of 'uuid'. + # opened scene from OpenPype whe 'ObjectId' was used + # instead of 'uuid'. # NOTE: Server call would crash if there is any invalid id. # That would cause crash we won't get any information. uuid.UUID(repre_id) From 72862554b4df26b304efb609200b901748c01c4f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 16:13:46 +0100 Subject: [PATCH 120/136] Fix missing slate frame frames_to_render should contain also additional slate frame --- client/ayon_core/pipeline/farm/pyblish_functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 559561c827..c399855044 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -384,6 +384,7 @@ def prepare_representations( frame_end = frames_to_render[-1] if skeleton_data.get("slate"): frame_start -= 1 + frames_to_render.insert(0, frame_start) files = _get_real_files_to_rendered(collection, frames_to_render) From 49c5b875a8a275adb6264474df31a2fc4be3584a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 16:19:58 +0100 Subject: [PATCH 121/136] Refactor removed filtering by frame Remainder has now real way to find frame pattern from single file. Doesn't make sense to filter on single file. --- client/ayon_core/pipeline/farm/pyblish_functions.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 559561c827..147763391b 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -435,13 +435,10 @@ def prepare_representations( " This may cause issues on farm." ).format(staging)) - files = _get_real_files_to_rendered( - [os.path.basename(remainder)], frames_to_render) - rep = { "name": ext, "ext": ext, - "files": files[0], + "files": os.path.basename(remainder), "stagingDir": staging, } From 2a20a9d169b2ed49eada6679166bd61a5db8889e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 16:20:57 +0100 Subject: [PATCH 122/136] Refactor filtering based on frame_to_render Previous implementation was naive and could be dangerous. Updated docstrings. Renamed. --- .../pipeline/farm/pyblish_functions.py | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 147763391b..722dc4b5c6 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -385,8 +385,7 @@ def prepare_representations( if skeleton_data.get("slate"): frame_start -= 1 - files = _get_real_files_to_rendered(collection, frames_to_render) - + files = _get_real_files_to_render(collection, frames_to_render) # explicitly disable review by user preview = preview and not do_not_add_review rep = { @@ -492,31 +491,47 @@ def _get_real_frames_to_render(frames): return frames_to_render -def _get_real_files_to_rendered(collection, frames_to_render): - """Use expected files based on real frames_to_render. +def _get_real_files_to_render(collection, frames_to_render): + """Filter files with frames that should be really rendered. + + 'expected_files' are collected from DCC based on timeline setting. This is + being calculated differently in each DCC. Filtering here is on single place + + But artists might explicitly set frames they want to render in Publisher UI + This range would override and filter previously prepared expected files + from DCC. - Artists might explicitly set frames they want to render via Publisher UI. - This uses this value to filter out files Args: - frames_to_render (list): of str '1001' - """ - files = [os.path.basename(f) for f in list(collection)] - file_name, extracted_frame = list(collect_frames(files).items())[0] + collection (clique.Collection): absolute paths + frames_to_render (list[int]): of int 1001 + Returns: + (list[str]) - if not extracted_frame: - return files + Example: + -------- - found_frame_pattern_length = len(extracted_frame) + expectedFiles = [ + "foo_v01.0001.exr", + "foo_v01.0002.exr", + ] + frames_to_render = '0001' + >> + ["foo_v01.0001.exr"] - only explicitly requested frame returned + """ + found_frame_pattern_length = collection.padding normalized_frames_to_render = { str(frame_to_render).zfill(found_frame_pattern_length) for frame_to_render in frames_to_render } + head_name = os.path.basename(collection.head) + + file_names = [os.path.basename(f) for f in collection] return [ file_name - for file_name in files + for file_name in file_names if any( - frame in file_name + f"{head_name}{frame}{collection.tail}" == file_name for frame in normalized_frames_to_render ) ] From b832c850c33cba13bb63cf6b7b58feaba5297510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 5 Dec 2024 16:57:47 +0100 Subject: [PATCH 123/136] Update client/ayon_core/plugins/publish/collect_otio_subset_resources.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../plugins/publish/collect_otio_subset_resources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py index cc1ef3edef..2d8e91fe09 100644 --- a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py +++ b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py @@ -199,8 +199,8 @@ def process(self, instance): if "review" in instance.data["families"]: review_repre = self._create_representation( - frame_start, frame_end, - file=filename, delete=True, review=True) + frame_start, frame_end, + file=filename, delete=True, review=True) instance.data["originalDirname"] = self.staging_dir From 81c71a757fae3ea0120b397017192336becb4a5a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 17:15:40 +0100 Subject: [PATCH 124/136] Prepare normalized expected file names Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 722dc4b5c6..a7780ba97c 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -525,7 +525,10 @@ def _get_real_files_to_render(collection, frames_to_render): } head_name = os.path.basename(collection.head) - + normalized_filenames = { + f"{head_name}{frame}{collection.tail}" + for frame in normalized_frames_to_render + } file_names = [os.path.basename(f) for f in collection] return [ file_name From c886be22bdf2494e952c39d4f38959e35c732150 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 17:19:19 +0100 Subject: [PATCH 125/136] Refactor logic to use set --- client/ayon_core/pipeline/farm/pyblish_functions.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index a7780ba97c..1f6e17972a 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -533,10 +533,7 @@ def _get_real_files_to_render(collection, frames_to_render): return [ file_name for file_name in file_names - if any( - f"{head_name}{frame}{collection.tail}" == file_name - for frame in normalized_frames_to_render - ) + if file_name in normalized_filenames ] From 06f6b519f0d45aa5e9dfadf3f184106d98166eb7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 12:42:07 +0100 Subject: [PATCH 126/136] Refactor logic to be even simpler --- .../pipeline/farm/pyblish_functions.py | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 1f6e17972a..902aa41af4 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -518,23 +518,14 @@ def _get_real_files_to_render(collection, frames_to_render): >> ["foo_v01.0001.exr"] - only explicitly requested frame returned """ - found_frame_pattern_length = collection.padding - normalized_frames_to_render = { - str(frame_to_render).zfill(found_frame_pattern_length) - for frame_to_render in frames_to_render - } - - head_name = os.path.basename(collection.head) - normalized_filenames = { - f"{head_name}{frame}{collection.tail}" - for frame in normalized_frames_to_render - } - file_names = [os.path.basename(f) for f in collection] - return [ - file_name - for file_name in file_names - if file_name in normalized_filenames - ] + included_frames = set(collection.indexes).intersection(frames_to_render) + real_collection = clique.Collection( + collection.head, + collection.tail, + collection.padding, + indexes=included_frames + ) + return list(real_collection) def create_instances_for_aov(instance, skeleton, aov_filter, From 6836a7f79b546e7dc500d2c8ba911469e3035012 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 13:51:09 +0100 Subject: [PATCH 127/136] Fix docstring --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 902aa41af4..425b616adc 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -514,7 +514,7 @@ def _get_real_files_to_render(collection, frames_to_render): "foo_v01.0001.exr", "foo_v01.0002.exr", ] - frames_to_render = '0001' + frames_to_render = 1 >> ["foo_v01.0001.exr"] - only explicitly requested frame returned """ From d65865563f7aa34ede47b17e2dfc671a03f84255 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 14:53:17 +0100 Subject: [PATCH 128/136] Fix expected files must be only file names --- client/ayon_core/pipeline/farm/pyblish_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 425b616adc..71068cb093 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -525,7 +525,8 @@ def _get_real_files_to_render(collection, frames_to_render): collection.padding, indexes=included_frames ) - return list(real_collection) + real_full_paths = list(real_collection) + return [os.path.basename(file_url) for file_url in real_full_paths] def create_instances_for_aov(instance, skeleton, aov_filter, From 4bf03f0a51ed5b73034b537f55d6c853fb572b3a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 15:39:08 +0100 Subject: [PATCH 129/136] Remove unused import 'collect_frames' from pyblish_functions --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 616e25596c..e48d99602e 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -7,7 +7,7 @@ import attr import ayon_api import clique -from ayon_core.lib import Logger, collect_frames +from ayon_core.lib import Logger from ayon_core.pipeline import ( get_current_project_name, get_representation_path, From 55eb8e497fc30f3de22229fea904b223136cf33f Mon Sep 17 00:00:00 2001 From: Ynbot Date: Mon, 9 Dec 2024 15:13:06 +0000 Subject: [PATCH 130/136] [Automated] Add generated package files from main --- client/ayon_core/version.py | 2 +- package.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/version.py b/client/ayon_core/version.py index b2ece45120..0d3b533f57 100644 --- a/client/ayon_core/version.py +++ b/client/ayon_core/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'core' version.""" -__version__ = "1.0.10+dev" +__version__ = "1.0.11" diff --git a/package.py b/package.py index 58ae5c08d9..464fbb007b 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "core" title = "Core" -version = "1.0.10+dev" +version = "1.0.11" client_dir = "ayon_core" diff --git a/pyproject.toml b/pyproject.toml index d7cf9fa6ed..ab452816ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.poetry] name = "ayon-core" -version = "1.0.10+dev" +version = "1.0.11" description = "" authors = ["Ynput Team "] readme = "README.md" From a84a3cad33be7e6fc29d6b5539f1e33f556374e4 Mon Sep 17 00:00:00 2001 From: Ynbot Date: Mon, 9 Dec 2024 15:13:44 +0000 Subject: [PATCH 131/136] [Automated] Update version in package.py for develop --- client/ayon_core/version.py | 2 +- package.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/version.py b/client/ayon_core/version.py index 0d3b533f57..a4ae75914c 100644 --- a/client/ayon_core/version.py +++ b/client/ayon_core/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'core' version.""" -__version__ = "1.0.11" +__version__ = "1.0.11+dev" diff --git a/package.py b/package.py index 464fbb007b..b8d88fc2ad 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "core" title = "Core" -version = "1.0.11" +version = "1.0.11+dev" client_dir = "ayon_core" diff --git a/pyproject.toml b/pyproject.toml index ab452816ad..bdfaf797e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.poetry] name = "ayon-core" -version = "1.0.11" +version = "1.0.11+dev" description = "" authors = ["Ynput Team "] readme = "README.md" From 55dacd5cec1eb58681bce5bf6d784c6b35ddc401 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:08:37 +0100 Subject: [PATCH 132/136] use 'taskType' instead of 'type' --- .../pipeline/workfile/path_resolving.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/path_resolving.py b/client/ayon_core/pipeline/workfile/path_resolving.py index 47d6f4ddfa..dee27ae4db 100644 --- a/client/ayon_core/pipeline/workfile/path_resolving.py +++ b/client/ayon_core/pipeline/workfile/path_resolving.py @@ -34,15 +34,23 @@ def get_workfile_template_key_from_context( host_name (str): Host name. project_settings (Dict[str, Any]): Project settings for passed 'project_name'. Not required at all but makes function faster. - """ + Returns: + str: Workfile template name. + + """ folder_entity = ayon_api.get_folder_by_path( - project_name, folder_path, fields={"id"} + project_name, + folder_path, + fields={"id"}, ) task_entity = ayon_api.get_task_by_name( - project_name, folder_entity["id"], task_name + project_name, + folder_entity["id"], + task_name, + fields={"taskType"}, ) - task_type = task_entity.get("type") + task_type = task_entity.get("taskType") return get_workfile_template_key( project_name, task_type, host_name, project_settings From 55a4c42c8377ab67777062cf046191b2e83c91ff Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:08:45 +0100 Subject: [PATCH 133/136] added typehints --- .../ayon_core/pipeline/workfile/path_resolving.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/path_resolving.py b/client/ayon_core/pipeline/workfile/path_resolving.py index dee27ae4db..61c6e5b876 100644 --- a/client/ayon_core/pipeline/workfile/path_resolving.py +++ b/client/ayon_core/pipeline/workfile/path_resolving.py @@ -2,6 +2,7 @@ import re import copy import platform +from typing import Optional, Dict, Any import ayon_api @@ -16,12 +17,12 @@ def get_workfile_template_key_from_context( - project_name, - folder_path, - task_name, - host_name, - project_settings=None -): + project_name: str, + folder_path: str, + task_name: str, + host_name: str, + project_settings: Optional[Dict[str, Any]] = None, +) -> str: """Helper function to get template key for workfile template. Do the same as `get_workfile_template_key` but returns value for "session From 6d7415360e1b3177affd5b734a7e0b332015efc7 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:18:08 +0100 Subject: [PATCH 134/136] fix item menu request --- client/ayon_core/tools/attribute_defs/files_widget.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/tools/attribute_defs/files_widget.py b/client/ayon_core/tools/attribute_defs/files_widget.py index 6199d0c202..42e805d72e 100644 --- a/client/ayon_core/tools/attribute_defs/files_widget.py +++ b/client/ayon_core/tools/attribute_defs/files_widget.py @@ -522,7 +522,7 @@ def lessThan(self, left, right): class ItemWidget(QtWidgets.QWidget): - context_menu_requested = QtCore.Signal(QtCore.QPoint, bool) + context_menu_requested = QtCore.Signal(QtCore.QPoint) def __init__( self, item_id, label, pixmap_icon, is_sequence, multivalue, parent=None @@ -841,7 +841,7 @@ def _on_rows_inserted(self, parent_index, start_row, end_row): self._multivalue ) widget.context_menu_requested.connect( - self._on_context_menu_requested + self._on_item_context_menu_request ) self._files_view.setIndexWidget(index, widget) self._files_proxy_model.setData( @@ -923,6 +923,9 @@ def _on_context_menu_requested(self, pos, valid_index): if menu.actions(): menu.popup(pos) + def _on_item_context_menu_request(self, pos): + self._on_context_menu_requested(pos, True) + def dragEnterEvent(self, event): if self._multivalue: return From 38b6aeadbac8978f9416c41a3a8d1be9f9d02b42 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:18:36 +0100 Subject: [PATCH 135/136] don't pass boolean to signal --- client/ayon_core/tools/attribute_defs/files_widget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/tools/attribute_defs/files_widget.py b/client/ayon_core/tools/attribute_defs/files_widget.py index 42e805d72e..652a33e29a 100644 --- a/client/ayon_core/tools/attribute_defs/files_widget.py +++ b/client/ayon_core/tools/attribute_defs/files_widget.py @@ -589,7 +589,7 @@ def resizeEvent(self, event): def _on_actions_clicked(self): pos = self._split_btn.rect().bottomLeft() point = self._split_btn.mapToGlobal(pos) - self.context_menu_requested.emit(point, False) + self.context_menu_requested.emit(point) class InViewButton(IconButton): From 9ea0c79f42da8c2b80ef87b13ee2d1e579ca7a7d Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 10 Dec 2024 15:18:48 +0100 Subject: [PATCH 136/136] use unused variable --- client/ayon_core/tools/attribute_defs/files_widget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/tools/attribute_defs/files_widget.py b/client/ayon_core/tools/attribute_defs/files_widget.py index 652a33e29a..8a40b3ff38 100644 --- a/client/ayon_core/tools/attribute_defs/files_widget.py +++ b/client/ayon_core/tools/attribute_defs/files_widget.py @@ -859,7 +859,7 @@ def _on_rows_removed(self, parent_index, start_row, end_row): for row in range(self._files_proxy_model.rowCount()): index = self._files_proxy_model.index(row, 0) item_id = index.data(ITEM_ID_ROLE) - available_item_ids.add(index.data(ITEM_ID_ROLE)) + available_item_ids.add(item_id) widget_ids = set(self._widgets_by_id.keys()) for item_id in available_item_ids: