From f5d39378d3b8c2a0be0d522390f4c43e950b8ded Mon Sep 17 00:00:00 2001 From: Ynbot Date: Fri, 8 Nov 2024 18:32:06 +0000 Subject: [PATCH 01/13] [Automated] Add generated package files to main --- client/ayon_houdini/version.py | 2 +- package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_houdini/version.py b/client/ayon_houdini/version.py index 5aed7e652d..e5072a8619 100644 --- a/client/ayon_houdini/version.py +++ b/client/ayon_houdini/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'houdini' version.""" -__version__ = "0.3.16+dev" +__version__ = "0.3.17" diff --git a/package.py b/package.py index e7fd2904c7..1fe1a0f70a 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "houdini" title = "Houdini" -version = "0.3.16+dev" +version = "0.3.17" app_host_name = "houdini" client_dir = "ayon_houdini" From 197b6b052b44e10dadaf1ba624e1cc07173e58e6 Mon Sep 17 00:00:00 2001 From: Ynbot Date: Fri, 8 Nov 2024 18:32:41 +0000 Subject: [PATCH 02/13] [Automated] Update version in package.py for develop --- client/ayon_houdini/version.py | 2 +- package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_houdini/version.py b/client/ayon_houdini/version.py index e5072a8619..bc4015c5c2 100644 --- a/client/ayon_houdini/version.py +++ b/client/ayon_houdini/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'houdini' version.""" -__version__ = "0.3.17" +__version__ = "0.3.17+dev" diff --git a/package.py b/package.py index 1fe1a0f70a..1e6da3ad1f 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "houdini" title = "Houdini" -version = "0.3.17" +version = "0.3.17+dev" app_host_name = "houdini" client_dir = "ayon_houdini" From ef270611a456e3ea058522f72881e7099d0cb96c Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 8 Nov 2024 23:11:44 +0100 Subject: [PATCH 03/13] Validate render product paths are unique between instances --- .../validate_render_product_paths_unique.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py diff --git a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py new file mode 100644 index 0000000000..90b1724297 --- /dev/null +++ b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py @@ -0,0 +1,109 @@ +import inspect +from collections import defaultdict +from typing import List + +import pyblish.api +import clique +import hou + +from ayon_core.pipeline import ( + OptionalPyblishPluginMixin, + PublishValidationError +) + +from ayon_houdini.api import plugin +from ayon_houdini.api.action import SelectInvalidAction + + +class ValidateRenderProductPathsUnique(plugin.HoudiniContextPlugin, + OptionalPyblishPluginMixin): + """Validate that render product paths are unique. + + This allows to catch before rendering whether multiple render ROPs would + end up writing to the same filepaths. This can be a problem when rendering + because each render job would overwrite the files of the other at + rendertime. + + """ + order = pyblish.api.ValidatorOrder + families = ["usdrender"] + hosts = ["houdini"] + label = "Unique Render Product Paths" + actions = [SelectInvalidAction] + optional = True + + def process(self, context): + if not self.is_active(context.data): + return + + invalid = self.get_invalid(context) + if not invalid: + return + + node_paths = [node.path() for node in invalid] + node_paths.sort() + invalid_list = "\n".join(f"- {path}" for path in node_paths) + raise PublishValidationError( + "Multiple instances render to the same path. " + "Please make sure each ROP renders to a unique output path:\n" + f"{invalid_list}", + title=self.label, + description=self.get_description() + ) + + @classmethod + def get_invalid(cls, context) -> "List[hou.Node]": + # Get instances matching this plugin families + instances = pyblish.api.instances_by_plugin(list(context), cls) + if len(instances) < 2: + return [] + + # Get expected rendered filepaths + paths_to_instance_id = defaultdict(list) + for instance in instances: + expected_files = instance.data.get("expectedFiles", []) + files_by_aov = expected_files[0] + for aov_name, aov_filepaths in files_by_aov.items(): + for aov_filepath in aov_filepaths: + paths_to_instance_id[aov_filepath].append(instance.id) + + # Get invalid instances by instance.id + invalid_instance_ids = set() + invalid_paths = [] + for path, path_instance_ids in paths_to_instance_id.items(): + if len(path_instance_ids) > 1: + for path_instance_d in path_instance_ids: + invalid_instance_ids.add(path_instance_d) + invalid_paths.append(path) + + if not invalid_instance_ids: + return [] + + # Log invalid sequences as single collection + collections, remainder = clique.assemble(invalid_paths) + for collection in collections: + cls.log.warning(f"Multiple instances output to path: {collection}") + for path in remainder: + cls.log.warning(f"Multiple instances output to path: {path}") + + # Get the invalid instances so we could also add a select action. + invalid = [] + for instance in [ + instance for instance in instances + if instance.id in invalid_instance_ids + ]: + node = hou.node(instance.data["instance_node"]) + invalid.append(node) + + return invalid + + def get_description(self): + return inspect.cleandoc( + """### Output paths overwrite each other + + Multiple instances output to the same path. This can cause each + render to overwrite the other providing unexpected results. + + Update the output paths to be unique across all instances. + """ + ) From ee538901e3f9aa8547deef992865569494ca8961 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 8 Nov 2024 23:12:07 +0100 Subject: [PATCH 04/13] Support `ContextPlugin` for select invalid action --- client/ayon_houdini/api/action.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/client/ayon_houdini/api/action.py b/client/ayon_houdini/api/action.py index a14296950b..105d6ccf2f 100644 --- a/client/ayon_houdini/api/action.py +++ b/client/ayon_houdini/api/action.py @@ -17,20 +17,26 @@ class SelectInvalidAction(pyblish.api.Action): def process(self, context, plugin): - errored_instances = get_errored_instances_from_context(context, - plugin=plugin) - # Get the invalid nodes for the plug-ins self.log.info("Finding invalid nodes..") invalid = list() - for instance in errored_instances: - invalid_nodes = plugin.get_invalid(instance) - if invalid_nodes: - if isinstance(invalid_nodes, (list, tuple)): - invalid.extend(invalid_nodes) - else: - self.log.warning("Plug-in returned to be invalid, " - "but has no selectable nodes.") + if issubclass(plugin, pyblish.api.ContextPlugin): + invalid = plugin.get_invalid(context) + else: + errored_instances = get_errored_instances_from_context( + context, plugin=plugin + ) + for instance in errored_instances: + invalid_nodes = plugin.get_invalid(instance) + if invalid_nodes: + if isinstance(invalid_nodes, (list, tuple)): + invalid.extend(invalid_nodes) + else: + self.log.warning("Plug-in returned to be invalid, " + "but has no selectable nodes.") + + errored_instances = get_errored_instances_from_context(context, + plugin=plugin) hou.clearAllSelected() if invalid: From aa2b0d26518d62352bbf230c3e5db7613d6a75c7 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 8 Nov 2024 23:17:22 +0100 Subject: [PATCH 05/13] Remove redundant line --- client/ayon_houdini/api/action.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/ayon_houdini/api/action.py b/client/ayon_houdini/api/action.py index 105d6ccf2f..a230e9f6ed 100644 --- a/client/ayon_houdini/api/action.py +++ b/client/ayon_houdini/api/action.py @@ -35,9 +35,6 @@ def process(self, context, plugin): self.log.warning("Plug-in returned to be invalid, " "but has no selectable nodes.") - errored_instances = get_errored_instances_from_context(context, - plugin=plugin) - hou.clearAllSelected() if invalid: self.log.info("Selecting invalid nodes: {}".format( From edc3539c738d64ec9d383fd95d41f7c83109b27f Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 9 Nov 2024 00:33:44 +0100 Subject: [PATCH 06/13] Validate more families + re-use logic from `CollectFilesForCleaningUp` --- .../validate_render_product_paths_unique.py | 52 ++++++++++++++++--- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py index 90b1724297..8f5fa16514 100644 --- a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py +++ b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py @@ -1,6 +1,6 @@ import inspect from collections import defaultdict -from typing import List +from typing import Dict, List import pyblish.api import clique @@ -15,6 +15,39 @@ from ayon_houdini.api.action import SelectInvalidAction +def get_instance_expected_files(instance: pyblish.api.Instance) -> List[str]: + """Get the expected source render files for the instance.""" + # Prefer 'expectedFiles' over 'frames' because it usually contains more + # output files than just a single file or single sequence of files. + expected_files: List[Dict[str, List[str]]] = ( + instance.data.get("expectedFiles", []) + ) + filepaths: List[str] = [] + if expected_files: + # Products with expected files + # This can be Render products or submitted cache to farm. + for expected in expected_files: + # expected.values() is a list of lists + filepaths.extend(expected.values()) + else: + # Products with frames or single file. + staging_dir = instance.data.get("stagingDir") + frames = instance.data.get("frames") + if frames is None or not staging_dir: + return [] + + if isinstance(frames, str): + # single file. + filepaths.append(f"{staging_dir}/{frames}") + else: + # list of frame. + filepaths.extend( + [f"{staging_dir}/{frame}" for frame in frames] + ) + + return filepaths + + class ValidateRenderProductPathsUnique(plugin.HoudiniContextPlugin, OptionalPyblishPluginMixin): """Validate that render product paths are unique. @@ -26,7 +59,15 @@ class ValidateRenderProductPathsUnique(plugin.HoudiniContextPlugin, """ order = pyblish.api.ValidatorOrder - families = ["usdrender"] + families = [ + # Render products + "usdrender", "karma_rop", "redshift_rop", "arnold_rop", "mantra_rop", + + # Product families from collect frames plug-in + "camera", "vdbcache", "imagesequence", "ass", "redshiftproxy", + "review", "pointcache", "fbx", "model" + ] + hosts = ["houdini"] label = "Unique Render Product Paths" actions = [SelectInvalidAction] @@ -61,11 +102,8 @@ def get_invalid(cls, context) -> "List[hou.Node]": # Get expected rendered filepaths paths_to_instance_id = defaultdict(list) for instance in instances: - expected_files = instance.data.get("expectedFiles", []) - files_by_aov = expected_files[0] - for aov_name, aov_filepaths in files_by_aov.items(): - for aov_filepath in aov_filepaths: - paths_to_instance_id[aov_filepath].append(instance.id) + for filepath in get_instance_expected_files(instance): + paths_to_instance_id[filepath].append(instance.id) # Get invalid instances by instance.id invalid_instance_ids = set() From cfd382a6fae2aa63c311e12e7a8b6f84c8413108 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 9 Nov 2024 11:32:42 +0100 Subject: [PATCH 07/13] Fix expected files iteration --- .../plugins/publish/validate_render_product_paths_unique.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py index 8f5fa16514..682e86d67d 100644 --- a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py +++ b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py @@ -27,8 +27,8 @@ def get_instance_expected_files(instance: pyblish.api.Instance) -> List[str]: # Products with expected files # This can be Render products or submitted cache to farm. for expected in expected_files: - # expected.values() is a list of lists - filepaths.extend(expected.values()) + for sequence_files in expected.values(): + filepaths.extend(sequence_files) else: # Products with frames or single file. staging_dir = instance.data.get("stagingDir") @@ -103,6 +103,7 @@ def get_invalid(cls, context) -> "List[hou.Node]": paths_to_instance_id = defaultdict(list) for instance in instances: for filepath in get_instance_expected_files(instance): + cls.log.info(filepath) paths_to_instance_id[filepath].append(instance.id) # Get invalid instances by instance.id From 594a97a130fbf859b8936a4a7fd34def6eac5e6b Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 9 Nov 2024 11:32:51 +0100 Subject: [PATCH 08/13] Cosmetics --- .../plugins/publish/validate_render_product_paths_unique.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py index 682e86d67d..9c20734ce7 100644 --- a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py +++ b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py @@ -40,10 +40,8 @@ def get_instance_expected_files(instance: pyblish.api.Instance) -> List[str]: # single file. filepaths.append(f"{staging_dir}/{frames}") else: - # list of frame. - filepaths.extend( - [f"{staging_dir}/{frame}" for frame in frames] - ) + # list of frames + filepaths.extend(f"{staging_dir}/{frame}" for frame in frames) return filepaths From 1049a04d6e66241506eab79135bb95a99418e0d6 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 9 Nov 2024 11:35:19 +0100 Subject: [PATCH 09/13] Remove log --- .../plugins/publish/validate_render_product_paths_unique.py | 1 - 1 file changed, 1 deletion(-) diff --git a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py index 9c20734ce7..cb032cf679 100644 --- a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py +++ b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py @@ -101,7 +101,6 @@ def get_invalid(cls, context) -> "List[hou.Node]": paths_to_instance_id = defaultdict(list) for instance in instances: for filepath in get_instance_expected_files(instance): - cls.log.info(filepath) paths_to_instance_id[filepath].append(instance.id) # Get invalid instances by instance.id From 641d1776c0f7af11a63acdb05a7b82d1bd561d38 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 9 Nov 2024 11:35:38 +0100 Subject: [PATCH 10/13] Skip local render instances that are split into runtime instances --- .../plugins/publish/validate_render_product_paths_unique.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py index cb032cf679..06a686434b 100644 --- a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py +++ b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py @@ -100,6 +100,12 @@ def get_invalid(cls, context) -> "List[hou.Node]": # Get expected rendered filepaths paths_to_instance_id = defaultdict(list) for instance in instances: + # Skip the original instance when local rendering and those have + # created additional runtime instances per AOV. This avoids + # validating similar instances multiple times. + if not instance.data.get("integrate", True): + continue + for filepath in get_instance_expected_files(instance): paths_to_instance_id[filepath].append(instance.id) From 63c750fbf8393b1e423f918f4b0bcec5cd93a0a7 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 11 Nov 2024 00:02:57 +0100 Subject: [PATCH 11/13] Allow `lopimportcam` for review --- client/ayon_houdini/plugins/publish/validate_scene_review.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_houdini/plugins/publish/validate_scene_review.py b/client/ayon_houdini/plugins/publish/validate_scene_review.py index f45cd1c97d..08240043c6 100644 --- a/client/ayon_houdini/plugins/publish/validate_scene_review.py +++ b/client/ayon_houdini/plugins/publish/validate_scene_review.py @@ -63,7 +63,7 @@ def get_invalid_camera_path(self, rop_node): if not camera_node: return "Camera path does not exist: '{}'".format(path) type_name = camera_node.type().name() - if type_name != "cam": + if type_name not in {"cam", "lopimportcam"}: return "Camera path is not a camera: '{}' (type: {})".format( path, type_name ) From 52850be1bb8f9db87d65a42eee31c12bd91dfe18 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 11 Nov 2024 17:04:49 +0100 Subject: [PATCH 12/13] Add note about multiple paths conflicting from just one instance --- .../plugins/publish/validate_render_product_paths_unique.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py index 06a686434b..33214abf17 100644 --- a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py +++ b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py @@ -147,5 +147,10 @@ def get_description(self): render to overwrite the other providing unexpected results. Update the output paths to be unique across all instances. + + It may be the case that a single instance outputs multiple files + that overwrite each other, like separate AOV outputs from one ROP. + In that case it may be necessary to update the individual AOV + output paths, instead of outputs between separate instances. """ ) From cdc3a4f11ae5bd0ae35dbe570ef00fda2b906859 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 11 Nov 2024 17:05:46 +0100 Subject: [PATCH 13/13] Do not skip if only one instance, because the single instance may still overlap output paths of its own --- .../plugins/publish/validate_render_product_paths_unique.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py index 33214abf17..b76be502fb 100644 --- a/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py +++ b/client/ayon_houdini/plugins/publish/validate_render_product_paths_unique.py @@ -94,7 +94,7 @@ def process(self, context): def get_invalid(cls, context) -> "List[hou.Node]": # Get instances matching this plugin families instances = pyblish.api.instances_by_plugin(list(context), cls) - if len(instances) < 2: + if not instances: return [] # Get expected rendered filepaths