Merge pull request #45 from ynput/enhancement/AY-1014_Houdini-Overwri…

…te-the-last-publish_houdini-twin

Overwrite the last publish

client/ayon_houdini/api/lib.py

@@ -149,11 +149,26 @@ def validate_fps():
     return True
 
 
-def render_rop(ropnode):
+def render_rop(ropnode, frame_range=None):
     """Render ROP node utility for Publishing.
 
     This renders a ROP node with the settings we want during Publishing.
+
+    Args:
+        ropnode (hou.RopNode): Node to render
+        frame_range (tuple): Copied from Houdini's help..
+            Sequence of 2 or 3 values, overrides the frame range and frame
+            increment to render. The first two values specify the start and
+            end frames, and the third value (if given) specifies the frame
+            increment. If no frame increment is given and the ROP node
+            doesn't specify a frame increment, then a value of 1 will be
+            used. If no frame range is given, and the ROP node doesn't
+            specify a frame range, then the current frame will be rendered.
     """
+
+    if frame_range is None:
+        frame_range = ()
+
     # Print verbose when in batch mode without UI
     verbose = not hou.isUIAvailable()
 
@@ -164,7 +179,8 @@ def render_rop(ropnode):
                        output_progress=verbose,
                        # Render only this node
                        # (do not render any of its dependencies)
-                       ignore_inputs=True)
+                       ignore_inputs=True,
+                       frame_range=frame_range)
     except hou.Error as exc:
         # The hou.Error is not inherited from a Python Exception class,
         # so we explicitly capture the houdini error, otherwise pyblish

client/ayon_houdini/api/plugin.py

@@ -7,6 +7,7 @@
 import six
 import hou
 
+import clique
 import pyblish.api
 from ayon_core.pipeline import (
     CreatorError,
@@ -19,7 +20,7 @@
 )
 from ayon_core.lib import BoolDef
 
-from .lib import imprint, read, lsattr, add_self_publish_button
+from .lib import imprint, read, lsattr, add_self_publish_button, render_rop
 from .usd import get_ayon_entity_uri_from_representation_context
 
 
@@ -354,3 +355,34 @@ class HoudiniExtractorPlugin(publish.Extractor):
 
     hosts = ["houdini"]
     settings_category = SETTINGS_CATEGORY
+
+    def render_rop(self, instance: pyblish.api.Instance):
+        """Render the ROP node of the instance.
+
+        If `instance.data["frames_to_fix"]` is set and is not empty it will
+        be interpreted as a set of frames that will be rendered instead of the
+        full rop nodes frame range.
+
+        Only `instance.data["instance_node"]` is required.
+        """
+        # Log the start of the render
+        rop_node = hou.node(instance.data["instance_node"])
+        self.log.debug(f"Rendering {rop_node.path()}")
+
+        frames_to_fix = clique.parse(instance.data.get("frames_to_fix", ""),
+                                     "{ranges}")
+        if len(set(frames_to_fix)) < 2:
+            render_rop(rop_node)
+            return
+
+        # Render only frames to fix
+        for frame_range in frames_to_fix.separate():
+            frame_range = list(frame_range)
+            first_frame = int(frame_range[0])
+            last_frame = int(frame_range[-1])
+            self.log.debug(
+                f"Rendering frames to fix [{first_frame}, {last_frame}]"
+            )
+            # for step to be 1 since clique doesn't support steps.
+            frame_range = (first_frame, last_frame, 1)
+            render_rop(rop_node, frame_range=frame_range)

client/ayon_houdini/plugins/publish/collect_frames_fix.py

@@ -0,0 +1,143 @@
+import pyblish.api
+import ayon_api
+
+from ayon_core.lib.attribute_definitions import (
+    TextDef,
+    BoolDef
+)
+from ayon_core.pipeline.publish import AYONPyblishPluginMixin
+
+from ayon_houdini.api import plugin
+
+
+class CollectFramesFixDefHou(
+    plugin.HoudiniInstancePlugin,
+    AYONPyblishPluginMixin
+):
+    """Provides text field to insert frame(s) to be re-rendered.
+
+    Published files of last version of an instance product are collected into
+    `instance.data["last_version_published_files"]`. All these but frames
+    mentioned in text field will be reused for new version.
+    """
+    order = pyblish.api.CollectorOrder + 0.495
+    label = "Collect Frames to Fix"
+    targets = ["local"]
+    families = ["*"]
+
+    rewrite_version_enable = False
+
+    def process(self, instance):
+        attribute_values = self.get_attr_values_from_data(instance.data)
+        frames_to_fix: str = attribute_values.get("frames_to_fix", "")
+        rewrite_version: bool = (
+            self.rewrite_version_enable
+            and attribute_values.get("rewrite_version", False)
+        )
+        if not frames_to_fix:
+            if rewrite_version:
+                self.log.warning(
+                    "Rewrite version is enabled but no frames to fix are "
+                    "specified. Rewriting last version will be skipped.")
+            return
+
+        self.log.info(f"Frames to fix: {frames_to_fix}")
+        instance.data["frames_to_fix"] = frames_to_fix
+
+        # Skip instances that are set to not be integrated so we ignore
+        # the original `render` instance from which local AOV instances are
+        # spawned off.
+        if not instance.data.get("integrate", True):
+            self.log.debug("Skipping collecting frames to fix data for "
+                           "instance because instance is set to not integrate")
+            return
+
+        product_name: str = instance.data["productName"]
+        folder_entity: dict = instance.data["folderEntity"]
+        project_entity: dict = instance.data["projectEntity"]
+        project_name: str = project_entity["name"]
+
+        product_entity = ayon_api.get_product_by_name(
+            project_name,
+            product_name,
+            folder_id=folder_entity["id"])
+        if not product_entity:
+            self.log.warning(
+                f"No existing product found for '{product_name}'. "
+                "Re-render not possible."
+            )
+            return
+
+        product_type = product_entity["productType"]
+        instance_product_type = instance.data["productType"]
+        if product_type != instance_product_type:
+            self.log.error(
+                f"Existing product '{product_name}' product type "
+                f"'{product_type}' is not the same as instance product type "
+                f"'{instance_product_type}'. Re-render may have unintended "
+                f"side effects.")
+
+        version_entity = ayon_api.get_last_version_by_product_id(
+            project_name,
+            product_id=product_entity["id"],
+        )
+        if not version_entity:
+            self.log.warning(
+                f"No last version found for product '{product_name}', "
+                "re-render not possible."
+            )
+            return
+
+        representations = ayon_api.get_representations(
+            project_name, version_ids={version_entity["id"]}
+        )
+
+        # Get all published files for the representation
+        published_files: "list[str]" = []
+        for repre in representations:
+            for file_info in repre.get("files"):
+                published_files.append(file_info["path"])
+
+        instance.data["last_version_published_files"] = published_files
+        self.log.debug(f"last_version_published_files: {published_files}")
+
+        if rewrite_version:
+            instance.data["version"] = version_entity["version"]
+            # limits triggering version validator
+            instance.data.pop("latestVersion")
+
+    @classmethod
+    def get_attribute_defs(cls):
+        attributes = [
+            TextDef("frames_to_fix", label="Frames to fix",
+                    placeholder="5,10-15",
+                    regex="[0-9,-]+",
+                    tooltip=(
+                        "When specified, only these frames will be rendered.\n"
+                        "The remainder of the frame range for the instance "
+                        "will be copied from the previous published version.\n"
+                        "This allows re-rendering only certain frames or "
+                        "extending the frame range of the previous version.\n"
+                        "The frames to fix must be inside the instance's "
+                        "frame range.\n"
+                        "Example: 5,10-15"
+                    ))
+        ]
+
+        if cls.rewrite_version_enable:
+            attributes.append(
+                BoolDef(
+                    "rewrite_version",
+                    label="Rewrite latest version",
+                    default=False,
+                    tooltip=(
+                        "When enabled the new version will be published into"
+                        "the previous version and apply only the 'fixed "
+                        "frames'.\n"
+                        "**Note:** This does nothing if no Frames to Fix are "
+                        "specified."
+                    )
+                )
+            )
+
+        return attributes

client/ayon_houdini/plugins/publish/collect_local_render_instances.py

@@ -153,7 +153,11 @@ def process(self, instance):
                 "productGroup": product_group,
                 "families": ["render.local.hou", "review"],
                 "instance_node": instance.data["instance_node"],
+                # The following three items are necessary for
+                # `ExtractLastPublished`
                 "publish_attributes": instance.data["publish_attributes"],
+                "stagingDir": staging_dir,
+                "frames": aov_filenames,
                 "representations": [representation]
             })
 

client/ayon_houdini/plugins/publish/extract_last_published.py

@@ -0,0 +1,124 @@
+import os
+import shutil
+
+import clique
+import pyblish.api
+
+from ayon_core.lib import collect_frames
+from ayon_houdini.api import plugin
+
+
+class ExtractLastPublished(plugin.HoudiniExtractorPlugin):
+    """Extractor copying files from last published to staging directory.
+
+    It works only if instance data includes "last_version_published_files"
+    and there are frames to fix.
+
+    The files from last published are based on files which will be
+    extended/fixed for specific frames.
+
+    NOTE: 
+        This plugin is closely taken from ayon-nuke.
+        It contains some Houdini addon specific logic as various addons may
+          have unique methods for managing `staging_dir`, `expectedFiles`
+          and `frames`.
+    TODO:
+        It's preferable to to generalize this plugin for broader use and
+          integrate it into ayon-core.
+    """
+
+    order = pyblish.api.ExtractorOrder - 0.1
+    label = "Extract Last Published"
+    targets = ["local"]  # Same target as `CollectFramesFixDef`
+    families = ["*"]
+
+    def process(self, instance):
+        frames_to_fix = instance.data.get("frames_to_fix")
+        if not frames_to_fix:
+            self.log.debug("Skipping, No frames to fix.")
+            return
+
+        if not instance.data.get("integrate", True):
+            self.log.debug("Skipping collecting frames to fix data for "
+                           "instance because instance is set to not integrate")
+            return
+
+        last_published = instance.data.get("last_version_published_files")
+        if not last_published:
+            self.log.debug("Skipping, No last publish found.")
+            return
+
+        last_published_and_frames = collect_frames(last_published)
+        if not all(last_published_and_frames.values()):
+            self.log.debug("Skipping, No file sequence found in the "
+                           "last version published files.")
+            return
+
+        staging_dir, expected_filenames = self.get_expected_files_and_staging_dir(instance)
+
+        os.makedirs(staging_dir, exist_ok=True)
+
+        expected_and_frames = collect_frames(expected_filenames)
+        frames_and_expected = {v: k for k, v in expected_and_frames.items()}
+        frames_to_fix = clique.parse(frames_to_fix, "{ranges}")
+
+        anatomy = instance.context.data["anatomy"]
+
+        # TODO: This currently copies ALL frames from the last version instead
+        #  of only those within the frame range we're currently looking to
+        #  publish. It should instead, iterate over all expected frames for
+        #  current instance, exclude all "to fix" frames and copy the
+        #  other existing ones.
+        for file_path, frame in last_published_and_frames.items():
+            if frame is None:
+                continue
+            file_path = anatomy.fill_root(file_path)
+            if not os.path.exists(file_path):
+                continue
+            target_file_name = frames_and_expected.get(frame)
+            if not target_file_name:
+                continue
+
+            out_path = os.path.join(staging_dir, target_file_name)
+
+            # Copy only the frames that we won't render.
+            if frame and frame not in frames_to_fix:
+                self.log.debug(f"Copying '{file_path}' -> '{out_path}'")
+                shutil.copy(file_path, out_path)
+
+    def get_expected_files_and_staging_dir(self, instance):
+        """Get expected file names or frames.
+
+        This method includes Houdini specific code.
+
+        Args:
+            instance (pyblish.api.Instance): The instance to publish.
+
+        Returns:
+            tuple[str, list[str]]: A 2-tuple of staging dir and the list of
+                expected frames for the current publish instance.
+        """
+        expected_filenames = []
+        staging_dir = instance.data.get("stagingDir")
+        expected_files = instance.data.get("expectedFiles", [])
+
+        # 'expectedFiles' are preferred over 'frames'
+        if expected_files:
+            # Products with expected files
+            # This can be Render products or submitted cache to farm.
+            for expected in expected_files:
+                # expected.values() is a list of lists
+                expected_filenames.extend(sum(expected.values(), []))
+        else:
+            # Products with frames or single file.
+            frames = instance.data.get("frames", "")
+            if isinstance(frames, str):
+                # single file.
+                expected_filenames.append("{}/{}".format(staging_dir, frames))
+            else:
+                # list of frame.
+                expected_filenames.extend(
+                    ["{}/{}".format(staging_dir, f) for f in frames]
+                )
+
+        return staging_dir, expected_filenames

client/ayon_houdini/plugins/publish/extract_render.py

@@ -4,7 +4,6 @@
 import pyblish.api
 
 from ayon_houdini.api import plugin
-from ayon_houdini.api.lib import render_rop
 
 
 class ExtractRender(plugin.HoudiniExtractorPlugin):
@@ -58,8 +57,11 @@ def process(self, instance):
             return
 
         if creator_attribute.get("render_target") == "local":
-            ropnode = hou.node(instance.data.get("instance_node"))
-            render_rop(ropnode)
+            # FIXME Render the entire frame range if any of the AOVs does not have a
+            # previously rendered version. This situation breaks the publishing.
+            # because There will be missing frames as ROP nodes typically cannot render different
+            #  frame ranges for each AOV; they always use the same frame range for all AOVs.
+            self.render_rop(instance)
 
         # `ExpectedFiles` is a list that includes one dict.
         expected_files = instance.data["expectedFiles"][0]