Merge pull request #880 from ynput/enhancement/878-publisher-allow-co…

…ntext-promise

Create: Allow context promise for editorial workflow

client/ayon_core/pipeline/create/context.py

@@ -6,7 +6,8 @@
 import collections
 import inspect
 from contextlib import contextmanager
-from typing import Optional
+import typing
+from typing import Optional, Iterable, Dict
 
 import pyblish.logic
 import pyblish.api
@@ -31,13 +32,15 @@
     HostMissRequiredMethod,
 )
 from .changes import TrackChangesItem
-from .structures import PublishAttributes, ConvertorItem
+from .structures import PublishAttributes, ConvertorItem, InstanceContextInfo
 from .creator_plugins import (
     Creator,
     AutoCreator,
     discover_creator_plugins,
     discover_convertor_plugins,
 )
+if typing.TYPE_CHECKING:
+    from .structures import CreatedInstance
 
 # Import of functions and classes that were moved to different file
 # TODO Should be removed in future release - Added 24/08/28, 0.4.3-dev.1
@@ -183,6 +186,10 @@ def __init__(
         # Shared data across creators during collection phase
         self._collection_shared_data = None
 
+        # Context validation cache
+        self._folder_id_by_folder_path = {}
+        self._task_names_by_folder_path = {}
+
         self.thumbnail_paths_by_instance_id = {}
 
         # Trigger reset if was enabled
@@ -202,17 +209,19 @@ def publish_attributes(self):
         """Access to global publish attributes."""
         return self._publish_attributes
 
-    def get_instance_by_id(self, instance_id):
+    def get_instance_by_id(
+        self, instance_id: str
+    ) -> Optional["CreatedInstance"]:
         """Receive instance by id.
 
         Args:
             instance_id (str): Instance id.
 
         Returns:
-            Union[CreatedInstance, None]: Instance or None if instance with
+            Optional[CreatedInstance]: Instance or None if instance with
                 given id is not available.
-        """
 
+        """
         return self._instances_by_id.get(instance_id)
 
     def get_sorted_creators(self, identifiers=None):
@@ -224,8 +233,8 @@ def get_sorted_creators(self, identifiers=None):
 
         Returns:
             List[BaseCreator]: Sorted creator plugins by 'order' value.
-        """
 
+        """
         if identifiers is not None:
             identifiers = set(identifiers)
             creators = [
@@ -491,6 +500,8 @@ def reset_preparation(self):
 
         # Give ability to store shared data for collection phase
         self._collection_shared_data = {}
+        self._folder_id_by_folder_path = {}
+        self._task_names_by_folder_path = {}
 
     def reset_finalization(self):
         """Cleanup of attributes after reset."""
@@ -715,7 +726,7 @@ def context_data_changes(self):
             self._original_context_data, self.context_data_to_store()
         )
 
-    def creator_adds_instance(self, instance):
+    def creator_adds_instance(self, instance: "CreatedInstance"):
         """Creator adds new instance to context.
 
         Instances should be added only from creators.
@@ -942,7 +953,7 @@ def create_with_unified_error(self, identifier, *args, **kwargs):
     def _remove_instance(self, instance):
         self._instances_by_id.pop(instance.id, None)
 
-    def creator_removed_instance(self, instance):
+    def creator_removed_instance(self, instance: "CreatedInstance"):
         """When creator removes instance context should be acknowledged.
 
         If creator removes instance conext should know about it to avoid
@@ -990,7 +1001,7 @@ def bulk_instances_collection(self):
                 [],
                 self._bulk_instances_to_process
             )
-            self.validate_instances_context(instances_to_validate)
+            self.get_instances_context_info(instances_to_validate)
 
     def reset_instances(self):
         """Reload instances"""
@@ -1079,26 +1090,70 @@ def execute_autocreators(self):
         if failed_info:
             raise CreatorsCreateFailed(failed_info)
 
-    def validate_instances_context(self, instances=None):
-        """Validate 'folder' and 'task' instance context."""
+    def get_instances_context_info(
+        self, instances: Optional[Iterable["CreatedInstance"]] = None
+    ) -> Dict[str, InstanceContextInfo]:
+        """Validate 'folder' and 'task' instance context.
+
+        Args:
+            instances (Optional[Iterable[CreatedInstance]]): Instances to
+                validate. If not provided all instances are validated.
+
+        Returns:
+            Dict[str, InstanceContextInfo]: Validation results by instance id.
+
+        """
         # Use all instances from context if 'instances' are not passed
         if instances is None:
-            instances = tuple(self._instances_by_id.values())
+            instances = self._instances_by_id.values()
+        instances = tuple(instances)
+        info_by_instance_id = {
+            instance.id: InstanceContextInfo(
+                instance.get("folderPath"),
+                instance.get("task"),
+                False,
+                False,
+            )
+            for instance in instances
+        }
 
         # Skip if instances are empty
-        if not instances:
-            return
+        if not info_by_instance_id:
+            return info_by_instance_id
 
         project_name = self.project_name
 
-        task_names_by_folder_path = {}
+        to_validate = []
+        task_names_by_folder_path = collections.defaultdict(set)
         for instance in instances:
-            folder_path = instance.get("folderPath")
-            task_name = instance.get("task")
-            if folder_path:
-                task_names_by_folder_path[folder_path] = set()
-                if task_name:
-                    task_names_by_folder_path[folder_path].add(task_name)
+            context_info = info_by_instance_id[instance.id]
+            if instance.has_promised_context:
+                context_info.folder_is_valid = True
+                context_info.task_is_valid = True
+                continue
+            # TODO allow context promise
+            folder_path = context_info.folder_path
+            if not folder_path:
+                continue
+
+            if folder_path in self._folder_id_by_folder_path:
+                folder_id = self._folder_id_by_folder_path[folder_path]
+                if folder_id is None:
+                    continue
+                context_info.folder_is_valid = True
+
+            task_name = context_info.task_name
+            if task_name is not None:
+                tasks_cache = self._task_names_by_folder_path.get(folder_path)
+                if tasks_cache is not None:
+                    context_info.task_is_valid = task_name in tasks_cache
+                    continue
+
+            to_validate.append(instance)
+            task_names_by_folder_path[folder_path].add(task_name)
+
+        if not to_validate:
+            return info_by_instance_id
 
         # Backwards compatibility for cases where folder name is set instead
         #   of folder path
@@ -1120,7 +1175,9 @@ def validate_instances_context(self, instances=None):
                 fields={"id", "path"}
             ):
                 folder_id = folder_entity["id"]
-                folder_paths_by_id[folder_id] = folder_entity["path"]
+                folder_path = folder_entity["path"]
+                folder_paths_by_id[folder_id] = folder_path
+                self._folder_id_by_folder_path[folder_path] = folder_id
 
         folder_entities_by_name = collections.defaultdict(list)
         if folder_names:
@@ -1131,8 +1188,10 @@ def validate_instances_context(self, instances=None):
             ):
                 folder_id = folder_entity["id"]
                 folder_name = folder_entity["name"]
-                folder_paths_by_id[folder_id] = folder_entity["path"]
+                folder_path = folder_entity["path"]
+                folder_paths_by_id[folder_id] = folder_path
                 folder_entities_by_name[folder_name].append(folder_entity)
+                self._folder_id_by_folder_path[folder_path] = folder_id
 
         tasks_entities = ayon_api.get_tasks(
             project_name,
@@ -1145,28 +1204,28 @@ def validate_instances_context(self, instances=None):
             folder_id = task_entity["folderId"]
             folder_path = folder_paths_by_id[folder_id]
             task_names_by_folder_path[folder_path].add(task_entity["name"])
+        self._task_names_by_folder_path.update(task_names_by_folder_path)
 
-        for instance in instances:
-            if not instance.has_valid_folder or not instance.has_valid_task:
-                continue
-
+        for instance in to_validate:
             folder_path = instance["folderPath"]
+            task_name = instance.get("task")
             if folder_path and "/" not in folder_path:
                 folder_entities = folder_entities_by_name.get(folder_path)
                 if len(folder_entities) == 1:
                     folder_path = folder_entities[0]["path"]
                     instance["folderPath"] = folder_path
 
             if folder_path not in task_names_by_folder_path:
-                instance.set_folder_invalid(True)
                 continue
+            context_info = info_by_instance_id[instance.id]
+            context_info.folder_is_valid = True
 
-            task_name = instance["task"]
-            if not task_name:
-                continue
-
-            if task_name not in task_names_by_folder_path[folder_path]:
-                instance.set_task_invalid(True)
+            if (
+                not task_name
+                or task_name in task_names_by_folder_path[folder_path]
+            ):
+                context_info.task_is_valid = True
+        return info_by_instance_id
 
     def save_changes(self):
         """Save changes. Update all changed values."""

client/ayon_core/pipeline/create/structures.py

@@ -1,6 +1,7 @@
 import copy
 import collections
 from uuid import uuid4
+from typing import Optional
 
 from ayon_core.lib.attribute_definitions import (
     UnknownDef,
@@ -396,6 +397,24 @@ def deserialize_attributes(self, data):
                 )
 
 
+class InstanceContextInfo:
+    def __init__(
+        self,
+        folder_path: Optional[str],
+        task_name: Optional[str],
+        folder_is_valid: bool,
+        task_is_valid: bool,
+    ):
+        self.folder_path: Optional[str] = folder_path
+        self.task_name: Optional[str] = task_name
+        self.folder_is_valid: bool = folder_is_valid
+        self.task_is_valid: bool = task_is_valid
+
+    @property
+    def is_valid(self) -> bool:
+        return self.folder_is_valid and self.task_is_valid
+
+
 class CreatedInstance:
     """Instance entity with data that will be stored to workfile.
 
@@ -528,9 +547,6 @@ def __init__(
         if not self._data.get("instance_id"):
             self._data["instance_id"] = str(uuid4())
 
-        self._folder_is_valid = self.has_set_folder
-        self._task_is_valid = self.has_set_task
-
     def __str__(self):
         return (
             "<CreatedInstance {product[name]}"
@@ -699,6 +715,17 @@ def creator_attribute_defs(self):
     def publish_attributes(self):
         return self._data["publish_attributes"]
 
+    @property
+    def has_promised_context(self) -> bool:
+        """Get context data that are promised to be set by creator.
+
+        Returns:
+            bool: Has context that won't bo validated. Artist can't change
+                value when set to True.
+
+        """
+        return self._data.get("has_promised_context", False)
+
     def data_to_store(self):
         """Collect data that contain json parsable types.
 
@@ -826,46 +853,3 @@ def deserialize_on_remote(cls, serialized_data):
         obj.publish_attributes.deserialize_attributes(publish_attributes)
 
         return obj
-
-    # Context validation related methods/properties
-    @property
-    def has_set_folder(self):
-        """Folder path is set in data."""
-
-        return "folderPath" in self._data
-
-    @property
-    def has_set_task(self):
-        """Task name is set in data."""
-
-        return "task" in self._data
-
-    @property
-    def has_valid_context(self):
-        """Context data are valid for publishing."""
-
-        return self.has_valid_folder and self.has_valid_task
-
-    @property
-    def has_valid_folder(self):
-        """Folder set in context exists in project."""
-
-        if not self.has_set_folder:
-            return False
-        return self._folder_is_valid
-
-    @property
-    def has_valid_task(self):
-        """Task set in context exists in project."""
-
-        if not self.has_set_task:
-            return False
-        return self._task_is_valid
-
-    def set_folder_invalid(self, invalid):
-        # TODO replace with `set_folder_path`
-        self._folder_is_valid = not invalid
-
-    def set_task_invalid(self, invalid):
-        # TODO replace with `set_task_name`
-        self._task_is_valid = not invalid

client/ayon_core/tools/publisher/abstract.py

@@ -322,6 +322,12 @@ def get_instances_by_id(
     ) -> Dict[str, Union[CreatedInstance, None]]:
         pass
 
+    @abstractmethod
+    def get_instances_context_info(
+        self, instance_ids: Optional[Iterable[str]] = None
+    ):
+        pass
+
     @abstractmethod
     def get_existing_product_names(self, folder_path: str) -> List[str]:
         pass

client/ayon_core/tools/publisher/control.py

@@ -190,6 +190,9 @@ def get_instances(self):
     def get_instances_by_id(self, instance_ids=None):
         return self._create_model.get_instances_by_id(instance_ids)
 
+    def get_instances_context_info(self, instance_ids=None):
+        return self._create_model.get_instances_context_info(instance_ids)
+
     def get_convertor_items(self):
         return self._create_model.get_convertor_items()
 

client/ayon_core/tools/publisher/models/create.py

@@ -306,6 +306,14 @@ def get_instances_by_id(
             for instance_id in instance_ids
         }
 
+    def get_instances_context_info(
+        self, instance_ids: Optional[Iterable[str]] = None
+    ):
+        instances = self.get_instances_by_id(instance_ids).values()
+        return self._create_context.get_instances_context_info(
+            instances
+        )
+
     def get_convertor_items(self) -> Dict[str, ConvertorItem]:
         return self._create_context.convertor_items_by_id