Unreal: Deadline support with Perforce

client/ayon_core/hosts/unreal/api/plugin.py

@@ -21,59 +21,102 @@
     UILabelDef
 )
 from ayon_core.pipeline import (
+    AutoCreator,
     Creator,
     LoaderPlugin,
     CreatorError,
     CreatedInstance
 )
 
 
-@six.add_metaclass(ABCMeta)
-class UnrealBaseCreator(Creator):
-    """Base class for Unreal creator plugins."""
+class UnrealCreateLogic():
+    """Universal class for logic that Unreal creators could inherit from."""
     root = "/Game/Ayon/AyonPublishInstances"
     suffix = "_INS"
 
+
     @staticmethod
-    def cache_instance_data(shared_data):
+    def get_cached_instances(shared_data):
         """Cache instances for Creators to shared data.
 
-        Create `unreal_cached_instances` key when needed in shared data and
+        Create `unreal_cached_subsets` key when needed in shared data and
         fill it with all collected instances from the scene under its
         respective creator identifiers.
 
         If legacy instances are detected in the scene, create
-        `unreal_cached_legacy_instances` there and fill it with
-        all legacy products under family as a key.
+        `unreal_cached_legacy_subsets` there and fill it with
+        all legacy subsets under product_type as a key.
 
         Args:
             Dict[str, Any]: Shared data.
 
+        Return:
+            Dict[str, Any]: Shared data dictionary.
+
         """
-        if "unreal_cached_instances" in shared_data:
-            return
-
-        unreal_cached_instances = collections.defaultdict(list)
-        unreal_cached_legacy_instances = collections.defaultdict(list)
-        for instance in ls_inst():
-            creator_id = instance.get("creator_identifier")
-            if creator_id:
-                unreal_cached_instances[creator_id].append(instance)
-            else:
-                family = instance.get("family")
-                unreal_cached_legacy_instances[family].append(instance)
+        if shared_data.get("unreal_cached_subsets") is None:
+            unreal_cached_subsets = collections.defaultdict(list)
+            unreal_cached_legacy_subsets = collections.defaultdict(list)
+            for instance in ls_inst():
+                creator_id = instance.get("creator_identifier")
+                if creator_id:
+                    unreal_cached_subsets[creator_id].append(instance)
+                else:
+                    product_type = instance.get("product_type")
+                    unreal_cached_legacy_subsets[product_type].append(instance)
+
+            shared_data["unreal_cached_subsets"] = unreal_cached_subsets
+            shared_data["unreal_cached_legacy_subsets"] = (
+                unreal_cached_legacy_subsets
+            )
+        return shared_data
 
-        shared_data["unreal_cached_instances"] = unreal_cached_instances
-        shared_data["unreal_cached_legacy_instances"] = (
-            unreal_cached_legacy_instances
-        )
+    def _default_collect_instances(self):
+        # cache instances if missing
+        self.get_cached_instances(self.collection_shared_data)
+        for instance in self.collection_shared_data[
+                "unreal_cached_subsets"].get(self.identifier, []):
+            # Unreal saves metadata as string, so we need to convert it back
+            instance['creator_attributes'] = ast.literal_eval(
+                instance.get('creator_attributes', '{}'))
+            instance['publish_attributes'] = ast.literal_eval(
+                instance.get('publish_attributes', '{}'))
+            created_instance = CreatedInstance.from_existing(instance, self)
+            self._add_instance_to_context(created_instance)
 
-    def create(self, product_name, instance_data, pre_create_data):
+    def _default_update_instances(self, update_list):
+        for created_inst, changes in update_list:
+            instance_node = created_inst.get("instance_path", "")
+
+            if not instance_node:
+                unreal.log_warning(
+                    f"Instance node not found for {created_inst}")
+                continue
+
+            new_values = {
+                key: changes[key].new_value
+                for key in changes.changed_keys
+            }
+            imprint(
+                instance_node,
+                new_values
+            )
+
+    def _default_remove_instances(self, instances):
+        for instance in instances:
+            instance_node = instance.data.get("instance_path", "")
+            if instance_node:
+                unreal.EditorAssetLibrary.delete_asset(instance_node)
+
+            self._remove_instance_from_context(instance)
+
+
+    def create_unreal(self, product_name, instance_data, pre_create_data):
         try:
             instance_name = f"{product_name}{self.suffix}"
             pub_instance = create_publish_instance(instance_name, self.root)
 
-            instance_data["productName"] = product_name
+            instance_data["product_name"] = product_name
             instance_data["instance_path"] = f"{self.root}/{instance_name}"
 
             instance = CreatedInstance(
@@ -92,7 +135,8 @@ def create(self, product_name, instance_data, pre_create_data):
                 obj = ar.get_asset_by_object_path(member).get_asset()
                 assets.add(obj)
 
-            imprint(f"{self.root}/{instance_name}", instance.data_to_store())
+            imprint(f"{self.root}/{instance_name}",
+                    instance.data_to_store())
 
             return instance
 
@@ -102,47 +146,36 @@ def create(self, product_name, instance_data, pre_create_data):
                 CreatorError(f"Creator error: {er}"),
                 sys.exc_info()[2])
 
+
+class UnrealBaseAutoCreator(AutoCreator, UnrealCreateLogic):
+    """Base class for Unreal auto creator plugins."""
+
     def collect_instances(self):
-        # cache instances if missing
-        self.cache_instance_data(self.collection_shared_data)
-        for instance in self.collection_shared_data[
-                "unreal_cached_instances"].get(self.identifier, []):
-            # Unreal saves metadata as string, so we need to convert it back
-            instance['creator_attributes'] = ast.literal_eval(
-                instance.get('creator_attributes', '{}'))
-            instance['publish_attributes'] = ast.literal_eval(
-                instance.get('publish_attributes', '{}'))
-            created_instance = CreatedInstance.from_existing(instance, self)
-            self._add_instance_to_context(created_instance)
+        return self._default_collect_instances()
 
     def update_instances(self, update_list):
-        for created_inst, changes in update_list:
-            instance_node = created_inst.get("instance_path", "")
+        return self._default_update_instances(update_list)
 
-            if not instance_node:
-                unreal.log_warning(
-                    f"Instance node not found for {created_inst}")
-                continue
+    def remove_instances(self, instances):
+        return self._default_remove_instances(instances)
 
-            new_values = {
-                key: changes[key].new_value
-                for key in changes.changed_keys
-            }
-            imprint(
-                instance_node,
-                new_values
-            )
 
-    def remove_instances(self, instances):
-        for instance in instances:
-            instance_node = instance.data.get("instance_path", "")
-            if instance_node:
-                unreal.EditorAssetLibrary.delete_asset(instance_node)
+class UnrealBaseCreator(UnrealCreateLogic, Creator):
+    """Base class for Unreal creator plugins."""
 
-            self._remove_instance_from_context(instance)
+    def create(self, subset_name, instance_data, pre_create_data):
+        self.create_unreal(subset_name, instance_data, pre_create_data)
+
+    def collect_instances(self):
+        return self._default_collect_instances()
+
+    def update_instances(self, update_list):
+        return self._default_update_instances(update_list)
+
+    def remove_instances(self, instances):
+        return self._default_remove_instances(instances)
 
 
-@six.add_metaclass(ABCMeta)
 class UnrealAssetCreator(UnrealBaseCreator):
     """Base class for Unreal creator plugins based on assets."""
 

client/ayon_core/hosts/unreal/api/rendering.py

@@ -11,6 +11,13 @@
 queue = None
 executor = None
 
+SUPPORTED_EXTENSION_MAP = {
+    "png": unreal.MoviePipelineImageSequenceOutput_PNG,
+    "exr": unreal.MoviePipelineImageSequenceOutput_EXR,
+    "jpg": unreal.MoviePipelineImageSequenceOutput_JPG,
+    "bmp": unreal.MoviePipelineImageSequenceOutput_BMP,
+}
+
 
 def _queue_finish_callback(exec, success):
     unreal.log("Render completed. Success: " + str(success))
@@ -30,6 +37,66 @@ def _job_finish_callback(job, success):
     unreal.log("Individual job completed.")
 
 
+def get_render_config(project_name, project_settings=None):
+    """Returns Unreal asset from render config.
+
+    Expects configured location of render config set in Settings. This path
+    must contain stored render config in Unreal project
+    Args:
+        project_name (str):
+        project_settings (dict): Settings from get_project_settings
+    Returns
+        (str, uasset): path and UAsset
+    Raises:
+        RuntimeError if no path to config is set
+    """
+    if not project_settings:
+        project_settings = get_project_settings(project_name)
+
+    ar = unreal.AssetRegistryHelpers.get_asset_registry()
+    config_path = project_settings["unreal"]["render_config_path"]
+
+    if not config_path:
+        raise RuntimeError("Please provide location for stored render "
+            "config in `ayon+settings://unreal/render_config_path`")
+
+    unreal.log(f"Configured config path {config_path}")
+    if not unreal.EditorAssetLibrary.does_asset_exist(config_path):
+        raise RuntimeError(f"No config found at {config_path}")
+
+    unreal.log("Found saved render configuration")
+    config = ar.get_asset_by_object_path(config_path).get_asset()
+
+    return config_path, config
+
+
+def set_output_extension_from_settings(render_format, config):
+    """Forces output extension from Settings if available.
+
+    Clear all other extensions if there is value in Settings.
+    Args:
+        render_format (str): "png"|"jpg"|"exr"|"bmp"
+        config (unreal.MoviePipelineMasterConfig)
+    Returns
+        (unreal.MoviePipelineMasterConfig)
+    """
+    if not render_format:
+        return config
+
+    cls_from_map = SUPPORTED_EXTENSION_MAP.get(render_format.lower())
+    if not cls_from_map:
+        return config
+
+    for ext, cls in SUPPORTED_EXTENSION_MAP.items():
+        current_sett = config.find_setting_by_class(cls)
+        if current_sett and ext == render_format:
+            return config
+        config.remove_setting(current_sett)
+
+    config.find_or_add_setting_by_class(cls_from_map)
+    return config
+
+
 def start_rendering():
     """
     Start the rendering process.
@@ -60,14 +127,14 @@ def start_rendering():
             inst_data.append(data)
 
     try:
-        project = os.environ.get("AYON_PROJECT_NAME")
-        anatomy = Anatomy(project)
+        project_name = os.environ.get("AYON_PROJECT_NAME")
+        anatomy = Anatomy(project_name)
         root = anatomy.roots['renders']
     except Exception as e:
         raise Exception(
             "Could not find render root in anatomy settings.") from e
 
-    render_dir = f"{root}/{project}"
+    render_dir = f"{root}/{project_name}"
 
     # subsystem = unreal.get_editor_subsystem(
     #     unreal.MoviePipelineQueueSubsystem)
@@ -77,12 +144,8 @@ def start_rendering():
 
     ar = unreal.AssetRegistryHelpers.get_asset_registry()
 
-    data = get_project_settings(project)
-    config = None
-    config_path = str(data.get("unreal").get("render_config_path"))
-    if config_path and unreal.EditorAssetLibrary.does_asset_exist(config_path):
-        unreal.log("Found saved render configuration")
-        config = ar.get_asset_by_object_path(config_path).get_asset()
+    project_settings = get_project_settings(project_name)
+    _, config = get_render_config(project_name, project_settings)
 
     for i in inst_data:
         sequence = ar.get_asset_by_object_path(i["sequence"]).get_asset()
@@ -127,6 +190,7 @@ def start_rendering():
             if config:
                 job.get_configuration().copy_from(config)
 
+            job_config = job.get_configuration()
             # User data could be used to pass data to the job, that can be
             # read in the job's OnJobFinished callback. We could,
             # for instance, pass the AyonPublishInstance's path to the job.
@@ -135,7 +199,7 @@ def start_rendering():
             output_dir = render_setting.get('output')
             shot_name = render_setting.get('sequence').get_name()
 
-            settings = job.get_configuration().find_or_add_setting_by_class(
+            settings = job_config.find_or_add_setting_by_class(
                 unreal.MoviePipelineOutputSetting)
             settings.output_resolution = unreal.IntPoint(1920, 1080)
             settings.custom_start_frame = render_setting.get("frame_range")[0]
@@ -144,30 +208,22 @@ def start_rendering():
             settings.file_name_format = f"{shot_name}" + ".{frame_number}"
             settings.output_directory.path = f"{render_dir}/{output_dir}"
 
-            job.get_configuration().find_or_add_setting_by_class(
+            job_config.find_or_add_setting_by_class(
                 unreal.MoviePipelineDeferredPassBase)
 
-            render_format = data.get("unreal").get("render_format", "png")
-
-            if render_format == "png":
-                job.get_configuration().find_or_add_setting_by_class(
-                    unreal.MoviePipelineImageSequenceOutput_PNG)
-            elif render_format == "exr":
-                job.get_configuration().find_or_add_setting_by_class(
-                    unreal.MoviePipelineImageSequenceOutput_EXR)
-            elif render_format == "jpg":
-                job.get_configuration().find_or_add_setting_by_class(
-                    unreal.MoviePipelineImageSequenceOutput_JPG)
-            elif render_format == "bmp":
-                job.get_configuration().find_or_add_setting_by_class(
-                    unreal.MoviePipelineImageSequenceOutput_BMP)
+            render_format = project_settings.get("unreal").get("render_format",
+                                                               "png")
+
+            set_output_extension_from_settings(render_format,
+                                               job_config)
 
     # If there are jobs in the queue, start the rendering process.
     if queue.get_jobs():
         global executor
         executor = unreal.MoviePipelinePIEExecutor()
 
-        preroll_frames = data.get("unreal").get("preroll_frames", 0)
+        preroll_frames = project_settings.get("unreal").get("preroll_frames",
+                                                            0)
 
         settings = unreal.MoviePipelinePIEExecutorSettings()
         settings.set_editor_property(

client/ayon_core/hosts/unreal/hooks/pre_workfile_preparation.py

@@ -185,15 +185,19 @@ def execute(self):
             unreal_project_name = f"P{unreal_project_name}"
             unreal_project_filename = f'{unreal_project_name}.uproject'
 
-        project_path = Path(os.path.join(workdir, unreal_project_name))
+        last_workfile_path = self.data.get("last_workfile_path")
+        if last_workfile_path and os.path.exists(last_workfile_path):
+            project_path = Path(os.path.dirname(last_workfile_path))
+            unreal_project_filename = Path(os.path.basename(last_workfile_path))
+        else:
+            project_path = Path(os.path.join(workdir, unreal_project_name))
+            project_path.mkdir(parents=True, exist_ok=True)
 
         self.log.info((
             f"{self.signature} requested UE version: "
             f"[ {engine_version} ]"
         ))
 
-        project_path.mkdir(parents=True, exist_ok=True)
-
         # engine_path points to the specific Unreal Engine root
         # so, we are going up from the executable itself 3 levels.
         engine_path: Path = Path(executable).parents[3]