Merge branch 'develop' into change_bin_brdige_client_version

client/ayon_usd/__init__.py

@@ -1,13 +1,15 @@
 """USD Addon for AYON - client part."""
 
 from .addon import USDAddon
+from .ayon_bin_client.ayon_bin_distro.util.zip import extract_zip_file
 from .utils import (
-    get_download_dir
+    get_download_dir,
+    get_usd_pinning_envs,
 )
-from .ayon_bin_client.ayon_bin_distro.util.zip import extract_zip_file
 
 __all__ = (
     "USDAddon",
     "extract_zip_file",
     "get_download_dir",
+    "get_usd_pinning_envs",
 )

client/ayon_usd/addon.py

@@ -4,20 +4,22 @@
 import os
 from datetime import datetime, timezone
 
-from ayon_core.addon import AYONAddon, ITrayAddon
-
 from ayon_core import style
+from ayon_core.addon import AYONAddon, IPluginPaths, ITrayAddon
+
 from ayon_core.settings import get_studio_settings
 
 from . import config, utils
-from .utils import ADDON_DATA_JSON_PATH, DOWNLOAD_DIR, USD_ADDON_ROOT_DIR
+from .utils import ADDON_DATA_JSON_PATH, DOWNLOAD_DIR
 from .version import __version__
 
 from .ayon_bin_client.ayon_bin_distro.work_handler import worker
 from .ayon_bin_client.ayon_bin_distro.util import zip
 
+USD_ADDON_DIR = os.path.dirname(os.path.abspath(__file__))
+
 
-class USDAddon(AYONAddon, ITrayAddon):
+class USDAddon(AYONAddon, ITrayAddon, IPluginPaths):
     """Addon to add USD Support to AYON.
 
     Addon can also skip distribution of binaries from server and can
@@ -142,4 +144,11 @@ def tray_menu(self, tray_menu):
 
     def get_launch_hook_paths(self):
         """Get paths to launch hooks."""
-        return [os.path.join(USD_ADDON_ROOT_DIR, "hooks")]
+        return [os.path.join(USD_ADDON_DIR, "hooks")]
+
+    def get_plugin_paths(self):
+        return {
+            "publish": [
+                os.path.join(USD_ADDON_DIR, "plugins", "publish")
+            ]
+        }

client/ayon_usd/hooks/usd_pinning_root.py

@@ -0,0 +1,29 @@
+"""Pre-launch hook to set USD pinning related environment variable."""
+from typing import ClassVar
+
+from ayon_applications import LaunchTypes, PreLaunchHook
+
+
+class UsdPinningRoot(PreLaunchHook):
+    """Pre-launch hook to set USD_ROOT environment variable."""
+
+    app_groups: ClassVar = {"maya", "houdini", "blender", "unreal"}
+    # this should be set to farm_render, but this issue
+    # https://github.com/ynput/ayon-applications/issues/2
+    # stands in the way
+    launch_types: ClassVar = {LaunchTypes.farm_publish}
+
+    def execute(self) -> None:
+        """Set environments necessary for pinning."""
+        if self.launch_context.env.get("PINNING_FILE_PATH"):
+            return
+        anatomy = self.launch_context.anatomy
+        self.launch_context.env["PINNING_FILE_PATH"] = anatomy.fill_root(
+            self.launch_context.env.get("PINNING_FILE_PATH"),
+        )
+
+        roots = anatomy.roots()
+        self.launch_context.env[
+            "PROJECT_ROOTS"
+        ] = ",".join(f"{key}={value}"
+                     for key, value in roots.items())

client/ayon_usd/plugins/publish/extract_skeleton_pinning_json.py

@@ -0,0 +1,83 @@
+"""Extract Skeleton Pinning JSON file.
+
+This extractor creates a simple placeholder JSON file that is filled by
+Integrator plugin (Integrate Pinning File). This way, publishing process
+is much more simple and doesn't require any hacks.
+
+Side effects:
+    - Creates a JSON file with skeleton pinning data that doesn't contain
+      any real data, it's just a placeholder. If, for whatever reason, the
+      publishing process is interrupted, the placeholder file will be
+      still there even if the real data is not present.
+
+    - Adds a timestamp to the JSON file. This timestamp can be later used
+      to check if the processed data is up-to-date.
+
+"""
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import ClassVar
+
+import pyblish.api
+from ayon_core.pipeline import OptionalPyblishPluginMixin, KnownPublishError
+
+
+class ExtractSkeletonPinningJSON(pyblish.api.InstancePlugin,
+                                 OptionalPyblishPluginMixin):
+    """Extract Skeleton Pinning JSON file.
+
+    Extracted JSON file doesn't contain any data, it's just a placeholder
+    that is filled by Integrator plugin (Integrate Pinning File).
+    """
+
+    label = "Extract Skeleton Pinning JSON"
+    order = pyblish.api.ExtractorOrder + 0.49
+    families: ClassVar = ["usd", "usdrender"]
+
+    @staticmethod
+    def _has_usd_representation(representations: list) -> bool:
+        return any(
+            representation.get("name") == "usd"
+            for representation in representations
+        )
+
+    def process(self, instance: pyblish.api.Instance) -> None:
+        """Process the plugin."""
+        if not self.is_active(instance.data):
+            return
+
+        # we need to handle usdrender differently as usd for rendering will
+        # be produced much later on the farm.
+        if "usdrender" not in instance.data.get("families", []):
+            if not self._has_usd_representation(instance.data["representations"]):
+                self.log.info("No USD representation found, skipping.")
+                return
+
+        try:
+            staging_dir = Path(instance.data["stagingDir"])
+        except KeyError:
+            self.log.debug("No staging directory on instance found.")
+            try:
+                staging_dir = Path(instance.data["ifdFile"]).parent
+            except KeyError as e:
+                self.log.error("No staging directory found.")
+                raise KnownPublishError("Cannot determine staging directory.") from e
+
+        pin_file = f"{staging_dir.stem}_pin.json"
+        pin_file_path = staging_dir.joinpath(pin_file)
+        pin_representation = {
+            "name": "usd_pinning",
+            "ext": "json",
+            "files": pin_file_path.name,
+            "stagingDir": staging_dir.as_posix(),
+        }
+        current_timestamp = datetime.now().timestamp()
+        skeleton_pinning_data = {
+            "timestamp": current_timestamp,
+        }
+        Path.mkdir(staging_dir, parents=True, exist_ok=True)
+        with open(pin_file_path, "w") as f:
+            json.dump(skeleton_pinning_data, f, indent=4)
+
+        instance.data["representations"].append(pin_representation)

client/ayon_usd/plugins/publish/integrate_pinning_file.py

@@ -0,0 +1,134 @@
+"""Extract pinning file from USD file as a json file.
+
+This is WIP and will be refactored in the future.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import ayon_api
+import pyblish.api
+from ayon_core.pipeline import get_current_host_name, get_current_project_name
+from ayon_core.pipeline.publish import KnownPublishError
+from ayon_usd.standalone.usd.pinning import generate_pinning_file
+
+
+class IntegrateUsdPinningFile(pyblish.api.InstancePlugin):
+    """Extract pinning file from USD file as a json file."""
+
+    order = pyblish.api.IntegratorOrder + 0.01
+    label = "Integrate data into USD pinning file"
+    families = ["usd", "usdrender"]
+
+    def process(self, instance: pyblish.api.Instance) -> None:
+        """Process the plugin."""
+
+        anatomy = instance.context.data["anatomy"]
+        usd_pinning_path = None
+
+        # get pinning json file
+        if "usdrender" in instance.data.get("families", []):
+            self.log.debug(
+                "Extracting USD pinning file for usdrender family.")
+            usd_file_path = self.save_usd(instance)
+            for rep in instance.data["representations"]:
+                if rep["name"] == "usd_pinning":
+                    usd_pinning_path = Path(rep["stagingDir"]) / rep["files"]
+                    break
+        else:
+            if instance.data.get("versionEntity") is None:
+                err_msg = "Instance was not integrated to AYON yet."
+                raise KnownPublishError(err_msg)
+
+            ayon_api.get_representation_by_name(
+                get_current_project_name(),
+                representation_name="usd_pinning",
+                version_id=instance.data["versionEntity"]["id"])
+
+            published_repres = instance.data.get("published_representations")
+            usd_file_rootless_path = None
+            usd_pinning_rootless_file_path = None
+
+            for repre_info in published_repres.values():
+                rep = repre_info["representation"]
+
+                if rep["name"] == "usd":
+                    usd_file_rootless_path = rep["attrib"]["path"]
+                    continue
+                if rep["name"] == "usd_pinning":
+                    usd_pinning_rootless_file_path = rep["attrib"]["path"]
+                    continue
+
+                if usd_file_rootless_path and usd_pinning_rootless_file_path:
+                    break
+
+            if not usd_file_rootless_path or not usd_pinning_rootless_file_path:
+                self.log.info("No USD or USD pinning file found, skipping.")
+                return
+
+            # get the full path of the usd file
+            usd_file_path = Path(
+                anatomy.fill_root(usd_file_rootless_path))
+            usd_pinning_path = Path(
+                anatomy.fill_root(usd_pinning_rootless_file_path))
+
+        if not usd_pinning_path:
+            self.log.error("No USD pinning file found.")
+            return
+
+        generate_pinning_file(
+            usd_file_path.as_posix(),
+            ayon_api.get_project_roots_by_site_id(get_current_project_name()),
+            usd_pinning_path.as_posix())
+
+        # clean temporary usd file
+        if "usdrender" in instance.data.get("families", []):
+            self.log.debug(f"Removing temporary USD file: {usd_file_path}")
+            usd_file_path.unlink()
+
+    def save_usd(self, instance: pyblish.api.Instance) -> Path:
+        """Save USD file to disk.
+
+        Args:
+            instance (pyblish.api.Instance): Instance object.
+
+        Returns:
+            str: The rootless path to the saved USD file.
+
+        """
+        if get_current_host_name() == "houdini":
+            return self._save_usd_from_houdini(instance)
+        raise NotImplementedError(
+            f"Unsupported host {get_current_host_name()}")
+
+    def _save_usd_from_houdini(self, instance: pyblish.api.Instance) -> Path:
+        """Save USD file from Houdini.
+
+        This is called only from running host so we can safely assume
+        that Houdini Addon is available.
+
+        Args:
+            instance (pyblish.api.Instance): Instance object.
+
+        Returns:
+            str: The rootless path to the saved USD file.
+
+        """
+        import hou  # noqa: F401
+        from ayon_houdini.api import maintained_selection  # noqa: F401
+
+        ropnode = hou.node(instance.data.get("instance_node"))
+        filename = ropnode.parm("lopoutput").eval()
+        directory = ropnode.parm("savetodirectory_directory").eval()
+        filepath = Path(directory) / filename
+
+        # create temp usdrop node
+        with maintained_selection():
+            temp_usd_node = hou.node("/out").createNode("usd")
+            temp_usd_node.parm("lopoutput").set(
+                filepath.as_posix())
+            temp_usd_node.parm("loppath").set(ropnode.parm("loppath").eval())
+            temp_usd_node.render()
+            temp_usd_node.destroy()
+
+        return filepath

client/ayon_usd/standalone/usd/pinning/_pinning_file_generation_funcs.py

@@ -343,13 +343,13 @@ def generate_pinning_file(
     # Assume that the environment sets up the correct default AyonUsdResolver
     resolver = Ar.GetResolver()
     pinning_data = get_asset_dependencies(entry_usd, resolver)
-
-    # for windows we need to make the drive letter lower case.
-    if sys.platform.startswith('win'):
-        for key, val in pinning_data.items():
-            pinning_data.pop(key)
-            pinning_data[_normalize_path(key)] = _normalize_path(val)
 
+    # on Windows, we need to make the drive letter lower case.
+    if sys.platform.startswith('win'):
+        pinning_data = {
+            _normalize_path(key): _normalize_path(val)
+            for key, val in pinning_data.items()
+        }
 
     rootless_pinning_data = remove_root_from_dependency_info(
         pinning_data, root_info

client/ayon_usd/utils.py

@@ -217,3 +217,64 @@ def _append(_env: dict, key: str, path: str):
         "PYTHONPATH": python_path,
         ld_path_key: ld_library_path
     }
+
+
+def get_usd_pinning_envs(instance) -> dict:
+    """Get USD pinning file path from published representations.
+
+    This sets the rootless path to the USD pinning file and enables
+    the static global cache. It is not setting ``PROJECT_ROOTS`` because
+    they are platform dependent and on farm they need to be set on
+    task level.
+
+    Args:
+        instance (pyblish.api.Instance): Published representations.
+
+    Returns:
+        dict: environment variables needed for USD pinning.
+
+    """
+    usd_pinning_rootless_file_path = None
+    if not instance.data.get("published_representations"):
+        usd_pinning_file_path = get_pinning_file_path(instance)
+        anatomy = instance.context.data["anatomy"]
+        usd_pinning_rootless_file_path = anatomy.find_root_template_from_path(
+            usd_pinning_file_path,
+        )[1]
+    else:
+
+        for repre_info in instance.data.get(
+                "published_representations").values():
+            rep = repre_info["representation"]
+
+            if rep["name"] == "usd_pinning":
+                usd_pinning_rootless_file_path = rep["attrib"]["path"]
+                break
+
+    if not usd_pinning_rootless_file_path:
+        return {}
+
+    return {
+        "PINNING_FILE_PATH": usd_pinning_rootless_file_path,
+        "ENABLE_STATIC_GLOBAL_CACHE": "1",
+    }
+
+def get_pinning_file_path(instance) -> pathlib.Path:
+    """Get the path to the pinning file.
+    
+    Args:
+        instance (pyblish.api.Instance): Instance to get the pinning
+            file path for.
+        
+    Returns:
+        pathlib.Path | None: Path to the pinning file.
+    
+    """
+    return next(
+        (
+            pathlib.Path(rep["stagingDir"]) / rep["files"]
+            for rep in instance.data["representations"]
+            if rep["name"] == "usd_pinning"
+        ),
+        None,
+    )