diff --git a/Docs/Ayon_Docs/Admin_Docs.md b/Docs/Ayon_Docs/Admin_Docs.md index da10c72..3d2e156 100644 --- a/Docs/Ayon_Docs/Admin_Docs.md +++ b/Docs/Ayon_Docs/Admin_Docs.md @@ -5,8 +5,8 @@ > LakeFs keys (LakeFs is the data server we use to distribute Bin data) you can > get those Keys on our Discord server just ask one of the Ynput staff for them. > the settings are the following: -> `ayon+settings://ayon_usd/LakeFs_Settings/access_key_id` and -> `ayon+settings://ayon_usd/LakeFs_Settings/secret_access_key` +> `ayon+settings://ayon_usd/lakefs/access_key_id` and +> `ayon+settings://ayon_usd/lakefs/secret_access_key` USD is a modern, open-source, scene description and file format developed by Pixar Animation Studios. It's an Extensive and extendable C++Lib that is used @@ -42,22 +42,22 @@ need to touch them tho. #### LakeFs Config -**LakeFs Settings:** `ayon+settings://ayon_usd/LakeFs_Settings`\ +**LakeFs Settings:** `ayon+settings://ayon_usd/lakefs`\ LakeFs is the backend of our bin distribution system the addon will use the specified server to download the resolvers and AyonUsdLibs from LakeFs. **LakeFs Server Uri:** -`ayon+settings://ayon_usd/LakeFs_Settings/ayon_usd_lake_fs_server_repo`\ +`ayon+settings://ayon_usd/lakefs/server_repo`\ this is the Uri used to host the LakeFs server. the Ynput server can be found at `https://lake.ayon.cloud` **LakeFs Repository Uri:** -`ayon+settings://ayon_usd/LakeFs_Settings/ayon_usd_lake_fs_server_repo`\ +`ayon+settings://ayon_usd/lakefs/server_repo`\ this is a LakeFs internal link that also specifies the branch your downloading from.\ this can be great if you want to pin your pipeline to a specific release. -**Asset Resolvers:** `ayon+settings://ayon_usd/LakeFs_Settings/asset_resolvers`\ +**Asset Resolvers:** `ayon+settings://ayon_usd/lakefs/asset_resolvers`\ allows you to associate a specific Application name with a specific resolver.\ we always set up all the resolvers we compile but if you have special App_Names in your Applications then you might want to add an App Alias.\ @@ -66,32 +66,32 @@ it as an alias for the Hou19.5 entry because they share the same resolver. #### Usd Resolver Config -`ayon+settings://ayon_usd/Ayon_UsdResolver_Settings` +`ayon+settings://ayon_usd/ayon_usd_resolver` -**Log Lvl** `ayon+settings://ayon_usd/Ayon_UsdResolver_Settings/ayon_log_lvl` +**Log Lvl** `ayon+settings://ayon_usd/ayon_usd_resolver/ayon_log_lvl` control the log lvl of the AyonUsdResolver. It is advised to have this at Warn or Critical as Logging will impact the performance. **File Logger Enabled** -`ayon+settings://ayon_usd/Ayon_UsdResolver_Settings/ayon_file_logger_enabled` +`ayon+settings://ayon_usd/ayon_usd_resolver/ayon_file_logger_enabled` AyonUsdResolver includes a file logger if needed. **Logging Keys** -`ayon+settings://ayon_usd/Ayon_UsdResolver_Settings/ayon_logger_logging_keys` +`ayon+settings://ayon_usd/ayon_usd_resolver/ayon_logger_logging_keys` AyonUsdResolver Logger has a few predefined logging keys that can be enabled for Debugging. it is advised to only do this with Developer bundles as it can expose AYON Server data. it will also generate quite a big output. **File Logger Path** -`ayon+settings://ayon_usd/Ayon_UsdResolver_Settings/file_logger_file_path` The +`ayon+settings://ayon_usd/ayon_usd_resolver/file_logger_file_path` The Ayon File Logger needs an output path this needs to be a relative or absolute path to a folder. #### UsdLib Config: -`ayon+settings://ayon_usd/Usd_Settings` +`ayon+settings://ayon_usd/usd` -**Tf_Debug** `ayon+settings://ayon_usd/Usd_Settings/usd_tf_debug`\ +**Tf_Debug** `ayon+settings://ayon_usd/usd/usd_tf_debug`\ this allows you to set the UsdTfDebug env variable to get extra debug info from the UsdLib.\ [Usd Survival Guide (Luca Sheller)](https://lucascheller.github.io/VFX-UsdSurvivalGuide/core/profiling/debug.html)\ diff --git a/client/ayon_usd/__init__.py b/client/ayon_usd/__init__.py index ea9f668..2162b21 100644 --- a/client/ayon_usd/__init__.py +++ b/client/ayon_usd/__init__.py @@ -2,14 +2,12 @@ from .addon import USDAddon from .utils import ( - get_download_dir, - get_downloaded_usd_root, + get_download_dir ) from .ayon_bin_client.ayon_bin_distro.util.zip import extract_zip_file __all__ = ( "USDAddon", - "get_downloaded_usd_root", "extract_zip_file", "get_download_dir", ) diff --git a/client/ayon_usd/addon.py b/client/ayon_usd/addon.py index 43af200..bd5254b 100644 --- a/client/ayon_usd/addon.py +++ b/client/ayon_usd/addon.py @@ -7,8 +7,11 @@ from ayon_core.addon import AYONAddon, ITrayAddon from ayon_core import style +from ayon_core.settings import get_studio_settings from . import config, utils +from .utils import ADDON_DATA_JSON_PATH, DOWNLOAD_DIR, USD_ADDON_ROOT_DIR +from .version import __version__ from .ayon_bin_client.ayon_bin_distro.gui import progress_ui from .ayon_bin_client.ayon_bin_distro.work_handler import worker @@ -24,8 +27,8 @@ class USDAddon(AYONAddon, ITrayAddon): Cares about supplying USD Framework. """ - name = config.ADDON_NAME - version = config.ADDON_VERSION + name = "ayon_usd" + version = __version__ _download_window = None def tray_init(self): @@ -36,7 +39,11 @@ def initialize(self, module_settings): """Initialize USD Addon.""" if not module_settings["ayon_usd"]["allow_addon_start"]: raise SystemError( - "The experimental AyonUsd addon is currently activated, but you haven't yet acknowledged the user agreement indicating your understanding that this feature is experimental. Please go to the Studio settings and check the agreement checkbox." + "The experimental AyonUsd addon is currently activated, " + "but you haven't yet acknowledged the user agreement " + "indicating your understanding that this feature is " + "experimental. Please go to the Studio Settings and " + "check the agreement checkbox." ) self.enabled = True self._download_window = None @@ -46,64 +53,69 @@ def tray_start(self): Download USD if needed. """ - super(USDAddon, self).tray_start() - - if not os.path.exists(config.DOWNLOAD_DIR): - os.makedirs(config.DOWNLOAD_DIR, exist_ok=True) - if os.path.exists(str(config.DOWNLOAD_DIR) + ".zip"): - os.remove(str(config.DOWNLOAD_DIR) + ".zip") - if not os.path.exists(config.ADDON_DATA_JSON_PATH): - with open(config.ADDON_DATA_JSON_PATH, "w+") as data_json: - init_data = {} - init_data["ayon_usd_addon_first_init_utc"] = str( - datetime.now().astimezone(timezone.utc) - ) - json.dump( - init_data, - data_json, - ) - - if not utils.is_usd_lib_download_needed(): + + os.makedirs(DOWNLOAD_DIR, exist_ok=True) + + if not os.path.exists(ADDON_DATA_JSON_PATH): + now = datetime.now().astimezone(timezone.utc) + with open(ADDON_DATA_JSON_PATH, "w+") as json_file: + init_data = { + "ayon_usd_addon_first_init_utc": str(now) + } + json.dump(init_data, json_file) + + settings = get_studio_settings() + if not utils.is_usd_lib_download_needed(settings): print("usd is already downloaded") return - lake_fs_usd_lib_path = f"{config.get_addon_settings_value(config.get_addon_settings(),config.ADDON_SETTINGS_LAKE_FS_REPO_URI)}{config.get_usd_lib_conf_from_lakefs()}" + lake_fs_usd_lib_path = config.get_lakefs_usdlib_path(settings) + # Get modified time on LakeFS + lake_fs = config.get_global_lake_instance(settings) usd_lib_lake_fs_time_cest = ( - config.get_global_lake_instance() + lake_fs .get_element_info(lake_fs_usd_lib_path) .get("Modified Time") ) if not usd_lib_lake_fs_time_cest: - raise ValueError("could not find UsdLib time stamp on LakeFs server") + raise ValueError( + "Unable to find UsdLib date modified timestamp on " + f"LakeFs server: {lake_fs_usd_lib_path}" + ) - with open(config.ADDON_DATA_JSON_PATH, "r+") as data_json: - addon_data_json = json.load(data_json) + with open(ADDON_DATA_JSON_PATH, "r+") as json_file: + addon_data_json = json.load(json_file) addon_data_json["usd_lib_lake_fs_time_cest"] = usd_lib_lake_fs_time_cest - data_json.seek(0) + json_file.seek(0) json.dump( addon_data_json, - data_json, + json_file, ) - data_json.truncate() + json_file.truncate() controller = worker.Controller() usd_download_work_item = controller.construct_work_item( - func=config.get_global_lake_instance().clone_element, + func=lake_fs.clone_element, kwargs={ "lake_fs_object_uir": lake_fs_usd_lib_path, - "dist_path": config.DOWNLOAD_DIR, + "dist_path": DOWNLOAD_DIR, }, progress_title="Download UsdLib", ) + usd_zip_path = os.path.join( + DOWNLOAD_DIR, + os.path.basename(config.get_lakefs_usdlib_path(settings)) + ) + usd_lib_path = os.path.splitext(usd_zip_path)[0] controller.construct_work_item( func=zip.extract_zip_file, kwargs={ - "zip_file_path": config.USD_ZIP_PATH, - "dest_dir": config.USD_LIB_PATH, + "zip_file_path": usd_zip_path, + "dest_dir": usd_lib_path, }, progress_title="Unzip UsdLib", dependency_id=[usd_download_work_item.get_uuid()], @@ -114,7 +126,7 @@ def tray_start(self): close_on_finish=True, auto_close_timeout=1, delet_progress_bar_on_finish=False, - title=f"{config.ADDON_NAME}-Addon [UsdLib Download]", + title="ayon_usd-Addon [UsdLib Download]", ) download_ui.setStyleSheet(style.load_stylesheet()) download_ui.start() @@ -130,4 +142,4 @@ def tray_menu(self, tray_menu): def get_launch_hook_paths(self): """Get paths to launch hooks.""" - return [os.path.join(config.USD_ADDON_DIR, "hooks")] + return [os.path.join(USD_ADDON_ROOT_DIR, "hooks")] diff --git a/client/ayon_usd/config.py b/client/ayon_usd/config.py index b34dce2..0ae5bd8 100644 --- a/client/ayon_usd/config.py +++ b/client/ayon_usd/config.py @@ -1,175 +1,62 @@ -"""USD Addon utility functions.""" - -import functools -import os import platform -import json -import hashlib -from pathlib import Path -import ayon_api -from ayon_usd import version -from ayon_usd.ayon_bin_client.ayon_bin_distro.lakectlpy import wrapper - -CURRENT_DIR: Path = Path(os.path.dirname(os.path.abspath(__file__))) -DOWNLOAD_DIR: Path = CURRENT_DIR / "downloads" -NOT_SET = type("NOT_SET", (), {"__bool__": lambda: False})() -ADDON_NAME: str = version.name -ADDON_VERSION: str = version.__version__ -AYON_BUNDLE_NAME = os.environ["AYON_BUNDLE_NAME"] -USD_ADDON_DIR = os.path.dirname(os.path.abspath(__file__)) - -ADDON_DATA_JSON_PATH = os.path.join(DOWNLOAD_DIR, "ayon_usd_addon_info.json") - - -# Addon Settings -# LakeFs -ADDON_SETTINGS_LAKE_FS_URI = ("lakefs_settings", "ayon_usd_lake_fs_server_uri") -ADDON_SETTINGS_LAKE_FS_REPO_URI = ("lakefs_settings", "ayon_usd_lake_fs_server_repo") -ADDON_SETTINGS_LAKE_FS_KEY_ID = ("lakefs_settings", "access_key_id") -ADDON_SETTINGS_LAKE_FS_KEY = ("lakefs_settings", "secret_access_key") -# Resolver def -ADDON_SETTINGS_ASSET_RESOLVERS = ("lakefs_settings", "asset_resolvers") -ADDON_SETTINGS_ASSET_RESOLVERS_OVERWRITES = ("lakefs_settings", "lake_fs_overrides") -# Usd settings -ADDON_SETTINGS_USD_TF_DEBUG = ("usd_settings", "usd_tf_debug") -# Resolver Settings -ADDON_SETTINGS_USD_RESOLVER_LOG_LVL = ("ayon_usd_resolver_settings", "ayon_log_lvl") - -ADDON_SETTINGS_USD_RESOLVER_LOG_FILLE_LOOGER_ENABLED = ( - "ayon_usd_resolver_settings", - "ayon_file_logger_enabled", -) - -ADDON_SETTINGS_USD_RESOLVER_LOG_FILLE_LOOGER_FILE_PATH = ( - "ayon_usd_resolver_settings", - "file_logger_file_path", -) - -ADDON_SETTINGS_USD_RESOLVER_LOG_LOGGIN_KEYS = ( - "ayon_usd_resolver_settings", - "ayon_logger_logging_keys", -) - - -def get_addon_settings_value(settings: dict, key_path: tuple): - try: - selected_element = settings - for key in key_path: - selected_element = selected_element[key] - - return selected_element - except (KeyError, TypeError) as e: - raise KeyError(f"Error accessing settings with key path {key_path}: {e}") - - -class SingletonFuncCache: - _instance = None - _cache = {} - def __new__(cls, *args, **kwargs): - if cls._instance is None: - cls._instance = super().__new__(cls, *args, **kwargs) - return cls._instance - - @classmethod - def func_io_cache(cls, func): - @functools.wraps(func) - def cache_func(*args, **kwargs): - - cache_key = tuple((func.__name__, cls._hash_args_kwargs(args, kwargs))) - - if cache_key in cls._cache.keys(): - return cls._cache[cache_key] - result = func(*args, **kwargs) - cls._cache[cache_key] = result - - return result - - return cache_func - - @staticmethod - def _hash_args_kwargs(args, kwargs): - """Generate a hashable key from *args and **kwargs.""" - args_hash = SingletonFuncCache._make_hashable(args) - kwargs_hash = SingletonFuncCache._make_hashable(kwargs) - return args_hash + kwargs_hash - - @staticmethod - def _make_hashable(obj): - """Converts an object to a hashable representation.""" - - if isinstance(obj, (int, float, str, bool, type(None))): - return hashlib.sha256(str(obj).encode()).hexdigest() - - if isinstance(obj, dict) or hasattr(obj, "__dict__"): - return hashlib.sha256(json.dumps(obj, sort_keys=True).encode()).hexdigest() +from ayon_usd.ayon_bin_client.ayon_bin_distro.lakectlpy import wrapper +from ayon_core.settings import get_studio_settings - try: - return hashlib.sha256(json.dumps(obj).encode()).hexdigest() - except TypeError: - return hashlib.sha256(str(id(obj)).encode()).hexdigest() - def debug(self): - return self._cache +class _LocalCache: + lake_instance = None -def print_cache(): - print(SingletonFuncCache().debug()) +def get_global_lake_instance(settings=None): + """Create lakefs connection. + Warning: + This returns singleton object which uses connection information + available on first call. -def get_addon_settings(): + Args: + settings (Optional[Dict[str, Any]]): Prepared studio or + project settings. - return ayon_api.get_addon_settings( - addon_name=ADDON_NAME, - addon_version=ADDON_VERSION, - variant=AYON_BUNDLE_NAME, - ) + Returns: + wrapper.LakeCtl: Connection object. + """ + if _LocalCache.lake_instance is not None: + return _LocalCache.lake_instance -@SingletonFuncCache.func_io_cache -def get_global_lake_instance(): - addon_settings = ( - get_addon_settings() - ) # the function is cached, but this reduces the call stack + if not settings: + settings = get_studio_settings() + lakefs = settings["ayon_usd"]["lakefs"] return wrapper.LakeCtl( - server_url=str( - get_addon_settings_value(addon_settings, ADDON_SETTINGS_LAKE_FS_URI) - ), - access_key_id=str( - get_addon_settings_value(addon_settings, ADDON_SETTINGS_LAKE_FS_KEY_ID) - ), - secret_access_key=str( - get_addon_settings_value(addon_settings, ADDON_SETTINGS_LAKE_FS_KEY) - ), + server_url=lakefs["server_uri"], + access_key_id=lakefs["access_key_id"], + secret_access_key=lakefs["secret_access_key"], ) -@SingletonFuncCache.func_io_cache -def _get_lake_fs_repo_items() -> list: - lake_repo_uri = str( - get_addon_settings_value(get_addon_settings(), ADDON_SETTINGS_LAKE_FS_REPO_URI) - ) - if not lake_repo_uri: +def _get_lakefs_repo_items(lake_fs_repo: str) -> list: + """Return all repo object names in the LakeFS repository""" + if not lake_fs_repo: return [] - return get_global_lake_instance().list_repo_objects(lake_repo_uri) + return get_global_lake_instance().list_repo_objects(lake_fs_repo) -@SingletonFuncCache.func_io_cache -def get_usd_lib_conf_from_lakefs() -> str: - usd_zip_lake_path = "" - for item in _get_lake_fs_repo_items(): - if "AyonUsdBin/usd" in item and platform.system().lower() in item: - usd_zip_lake_path = item - return usd_zip_lake_path +def get_lakefs_usdlib_name(lake_fs_repo: str) -> str: + """Return AyonUsdBin/usd LakeFS repo object name for current platform.""" + platform_name = platform.system().lower() + for item in _get_lakefs_repo_items(lake_fs_repo): + if "AyonUsdBin/usd" in item and platform_name in item: + return item + raise RuntimeError( + "No AyonUsdBin/usd item found for current platform " + f"'{platform_name}' on LakeFS server: {lake_fs_repo}") -USD_ZIP_PATH = Path( - os.path.join( - DOWNLOAD_DIR, - os.path.basename( - f"{get_addon_settings_value(get_addon_settings(), ADDON_SETTINGS_LAKE_FS_REPO_URI)}{get_usd_lib_conf_from_lakefs()}" - ), - ) -) -USD_LIB_PATH = Path(str(USD_ZIP_PATH).replace(USD_ZIP_PATH.suffix, "")) +def get_lakefs_usdlib_path(settings: dict) -> str: + """Return AyonUsdBin/usd LakeFS full url for current platform. """ + lake_fs_repo = settings["ayon_usd"]["lakefs"]["server_repo"] + usd_lib_conf = get_lakefs_usdlib_name(lake_fs_repo) + return f"{lake_fs_repo}{usd_lib_conf}" diff --git a/client/ayon_usd/hooks/__init__.py b/client/ayon_usd/hooks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/client/ayon_usd/hooks/pre_resolver_init.py b/client/ayon_usd/hooks/pre_resolver_init.py index d0bb3a0..742e106 100644 --- a/client/ayon_usd/hooks/pre_resolver_init.py +++ b/client/ayon_usd/hooks/pre_resolver_init.py @@ -4,6 +4,7 @@ import os from ayon_applications import LaunchTypes, PreLaunchHook from ayon_usd import config, utils +from ayon_usd.addon import ADDON_DATA_JSON_PATH class InitializeAssetResolver(PreLaunchHook): @@ -16,74 +17,69 @@ class InitializeAssetResolver(PreLaunchHook): launch_types = {LaunchTypes.local} def _setup_resolver(self, local_resolver, settings): - self.log.info(f"Initializing USD asset resolver for application: {self.app_name}") - env_var_dict = utils.get_resolver_setup_info( - local_resolver, settings, self.app_name, self.log + self.log.info( + f"Initializing USD asset resolver for application: {self.app_name}" ) - for key in env_var_dict: - value = env_var_dict[key] - self.launch_context.env[key] = value + updated_env = utils.get_resolver_setup_info( + local_resolver, settings, env=self.launch_context.env + ) + self.launch_context.env.update(updated_env) def execute(self): """Pre-launch hook entry method.""" - - self.log.debug(self.app_group) - settings = self.data["project_settings"][config.ADDON_NAME] - - resolver_lake_fs_path = utils.get_resolver_to_download(settings, self.app_name) + project_settings = self.data["project_settings"] + resolver_lake_fs_path = utils.get_resolver_to_download( + project_settings, self.app_name) if not resolver_lake_fs_path: raise RuntimeError( - f"no Resolver could be found but AYON-Usd addon is activated {self.app_name}" + "No USD Resolver could be found but " + f"AYON-Usd addon is activated {self.app_name}" ) - with open(config.ADDON_DATA_JSON_PATH, "r") as data_json: - addon_data_json = json.load(data_json) - - try: - key = str(self.app_name).replace("/", "_") - local_resolver_data = addon_data_json[f"resolver_data_{key}"] - - except KeyError: - local_resolver_data = None - + self.log.info(f"Using resolver from LakeFS: {resolver_lake_fs_path}") + lake_fs = config.get_global_lake_instance() lake_fs_resolver_time_stamp = ( - config.get_global_lake_instance().get_element_info(resolver_lake_fs_path)[ + lake_fs.get_element_info(resolver_lake_fs_path).get( "Modified Time" - ] + ) + ) + if not lake_fs_resolver_time_stamp: + raise ValueError( + "Could not find resolver timestamp on LakeFs server " + f"for application: {self.app_name}" + ) + + # Check for existing local resolver that matches the lakefs timestamp + with open(ADDON_DATA_JSON_PATH, "r") as data_json: + addon_data_json = json.load(data_json) + + key = str(self.app_name).replace("/", "_") + local_resolver_key = f"resolver_data_{key}" + local_resolver_timestamp, local_resolver = ( + addon_data_json.get(local_resolver_key, [None, None]) ) if ( - local_resolver_data - and lake_fs_resolver_time_stamp == local_resolver_data[0] - and os.path.exists(local_resolver_data[1]) + local_resolver + and lake_fs_resolver_time_stamp == local_resolver_timestamp + and os.path.exists(local_resolver) ): - - self._setup_resolver(local_resolver_data[1], settings) + self._setup_resolver(local_resolver, project_settings) return - local_resolver = utils.download_and_extract_resolver( + # If no existing match, download the resolver + local_resolver = utils.lakefs_download_and_extract( resolver_lake_fs_path, str(utils.get_download_dir()) ) - if not local_resolver: return - key = str(self.app_name).replace("/", "_") - resolver_time_stamp = ( - config.get_global_lake_instance() - .get_element_info(resolver_lake_fs_path) - .get("Modified Time") - ) - if not resolver_time_stamp: - raise ValueError( - f"could not find resolver time stamp on LakeFs server for {self.app_name}" - ) - addon_data_json[f"resolver_data_{key}"] = [ - resolver_time_stamp, + addon_data_json[local_resolver_key] = [ + lake_fs_resolver_time_stamp, local_resolver, ] - with open(config.ADDON_DATA_JSON_PATH, "w") as addon_json: + with open(ADDON_DATA_JSON_PATH, "w") as addon_json: json.dump(addon_data_json, addon_json) - self._setup_resolver(local_resolver, settings) + self._setup_resolver(local_resolver, project_settings) diff --git a/client/ayon_usd/utils.py b/client/ayon_usd/utils.py index 58713a1..13c2c2a 100644 --- a/client/ayon_usd/utils.py +++ b/client/ayon_usd/utils.py @@ -6,20 +6,13 @@ import pathlib import sys -import ayon_api from ayon_usd.ayon_bin_client.ayon_bin_distro.work_handler import worker from ayon_usd.ayon_bin_client.ayon_bin_distro.util import zip from ayon_usd import config - -def get_addon_settings() -> dict: - """Get addon settings. - - Return: - dict: Addon settings. - - """ - return ayon_api.get_addon_settings(config.ADDON_NAME, config.ADDON_VERSION) +USD_ADDON_ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +DOWNLOAD_DIR = os.path.join(USD_ADDON_ROOT_DIR, "downloads") +ADDON_DATA_JSON_PATH = os.path.join(DOWNLOAD_DIR, "ayon_usd_addon_info.json") def get_download_dir(create_if_missing=True): @@ -32,59 +25,69 @@ def get_download_dir(create_if_missing=True): str: Path to download dir. """ - if create_if_missing and not os.path.exists(config.DOWNLOAD_DIR): - os.makedirs(config.DOWNLOAD_DIR, exist_ok=True) - return config.DOWNLOAD_DIR + if create_if_missing and not os.path.exists(DOWNLOAD_DIR): + os.makedirs(DOWNLOAD_DIR, exist_ok=True) + return DOWNLOAD_DIR -@config.SingletonFuncCache.func_io_cache -def get_downloaded_usd_root() -> str: +def get_downloaded_usd_root(lake_fs_repo_uri) -> str: """Get downloaded USDLib os local root path.""" - target_usd_lib = config.get_usd_lib_conf_from_lakefs() - usd_lib_local_path = os.path.join( - config.DOWNLOAD_DIR, - os.path.basename(target_usd_lib).replace( - f".{target_usd_lib.split('.')[-1]}", "" - ), - ) - return usd_lib_local_path - + target_usd_lib = config.get_lakefs_usdlib_name(lake_fs_repo_uri) + filename_no_ext = os.path.splitext(os.path.basename(target_usd_lib))[0] + return os.path.join(DOWNLOAD_DIR, filename_no_ext) -def is_usd_lib_download_needed() -> bool: - # TODO redocument - usd_lib_dir = os.path.abspath(get_downloaded_usd_root()) - if os.path.exists(usd_lib_dir): +def is_usd_lib_download_needed(settings: dict) -> bool: + """Return whether a USD libraries need (re-)download from the Lake FS + repository. - ctl = config.get_global_lake_instance() - lake_fs_usd_lib_path = f"{config.get_addon_settings_value(config.get_addon_settings(),config.ADDON_SETTINGS_LAKE_FS_REPO_URI)}{config.get_usd_lib_conf_from_lakefs()}" + This will be the case if it's the first time syncing, the timestamp on the + server is newer or the local files have been removed. - with open(config.ADDON_DATA_JSON_PATH, "r") as data_json: - addon_data_json = json.load(data_json) - try: - usd_lib_lake_fs_time_stamp_local = addon_data_json[ - "usd_lib_lake_fs_time_cest" - ] - except KeyError: - return True + Arguments: + settings (dict): Studio or Project settings. - if ( - usd_lib_lake_fs_time_stamp_local - == ctl.get_element_info(lake_fs_usd_lib_path)["Modified Time"] - ): - return False + Returns: + bool: When true, a new download is required. - return True + """ + lake_fs_repo = settings["ayon_usd"]["lakefs"]["server_repo"] + usd_lib_dir = os.path.abspath(get_downloaded_usd_root(lake_fs_repo)) + if not os.path.exists(usd_lib_dir): + return True + + with open(ADDON_DATA_JSON_PATH, "r") as data_json: + addon_data_json = json.load(data_json) + try: + usd_lib_lake_fs_time_stamp_local = addon_data_json[ + "usd_lib_lake_fs_time_cest" + ] + except KeyError: + return True + + lake_fs_usd_lib_path = config.get_lakefs_usdlib_path(settings) + lake_fs = config.get_global_lake_instance(settings) + lake_fs_timestamp = lake_fs.get_element_info( + lake_fs_usd_lib_path).get("Modified Time") + if ( + not lake_fs_timestamp + or usd_lib_lake_fs_time_stamp_local != lake_fs_timestamp + ): + return True + return False -def download_and_extract_resolver(resolver_lake_fs_path: str, download_dir: str) -> str: - """downloads an individual object based on the lake_fs_path and extracts the zip into the specific download_dir +def lakefs_download_and_extract(resolver_lake_fs_path: str, + download_dir: str) -> str: + """Download individual object based on the lake_fs_path and extracts + the zip into the specific download_dir. Args - resolver_lake_fs_path (): - download_dir (): + resolver_lake_fs_path (str): Lake FS Path for the resolver + download_dir (str): Directory to download and unzip to. Returns: + str: Result from the ZIP file extraction. """ controller = worker.Controller() @@ -107,7 +110,6 @@ def download_and_extract_resolver(resolver_lake_fs_path: str, download_dir: str) return str(extract_zip_item.func_return) -@config.SingletonFuncCache.func_io_cache def get_resolver_to_download(settings, app_name: str) -> str: """ Gets LakeFs path that can be used with copy element to download @@ -117,10 +119,8 @@ def get_resolver_to_download(settings, app_name: str) -> str: Returns: str: LakeFs object path to be used with lake_fs_py wrapper """ - resolver_overwrite_list = config.get_addon_settings_value( - settings, config.ADDON_SETTINGS_ASSET_RESOLVERS_OVERWRITES - ) - + lakefs = settings["ayon_usd"]["lakefs"] + resolver_overwrite_list = lakefs["lake_fs_overrides"] if resolver_overwrite_list: resolver_overwrite = next( ( @@ -134,9 +134,7 @@ def get_resolver_to_download(settings, app_name: str) -> str: if resolver_overwrite: return resolver_overwrite["lake_fs_path"] - resolver_list = config.get_addon_settings_value( - settings, config.ADDON_SETTINGS_ASSET_RESOLVERS - ) + resolver_list = lakefs["asset_resolvers"] if not resolver_list: return "" @@ -152,78 +150,70 @@ def get_resolver_to_download(settings, app_name: str) -> str: if not resolver: return "" - lake_base_path = config.get_addon_settings_value( - settings, config.ADDON_SETTINGS_LAKE_FS_REPO_URI - ) - resolver_lake_path = lake_base_path + resolver["lake_fs_path"] + lake_fs_repo_uri = lakefs["server_repo"] + resolver_lake_path = lake_fs_repo_uri + resolver["lake_fs_path"] return resolver_lake_path -@config.SingletonFuncCache.func_io_cache -def get_resolver_setup_info(resolver_dir, settings, app_name: str, logger=None) -> dict: - pxr_plugin_paths = [] - ld_path = [] - python_path = [] +def get_resolver_setup_info( + resolver_dir, + settings, + env=None) -> dict: + """Get the environment variables to load AYON USD setup. - if val := os.getenv("PXR_PLUGINPATH_NAME"): - pxr_plugin_paths.extend(val.split(os.pathsep)) - if val := os.getenv("LD_LIBRARY_PATH"): - ld_path.extend(val.split(os.pathsep)) - if val := os.getenv("PYTHONPATH"): - python_path.extend(val.split(os.pathsep)) + Arguments: + resolver_dir (str): Directory of the resolver. + settings (dict[str, Any]): Studio settings. + env (dict[str, str]): Source environment to build on. - resolver_plugin_info_path = os.path.join( - resolver_dir, "ayonUsdResolver", "resources", "plugInfo.json" - ) - resolver_ld_path = os.path.join(resolver_dir, "ayonUsdResolver", "lib") - resolver_python_path = os.path.join( - resolver_dir, "ayonUsdResolver", "lib", "python" - ) + Returns: + dict[str, str]: The environment needed to load AYON USD correctly. + """ + + resolver_root = pathlib.Path(resolver_dir) / "ayonUsdResolver" + resolver_plugin_info_path = resolver_root / "resources" / "plugInfo.json" + resolver_ld_path = resolver_root / "lib" + resolver_python_path = resolver_root / "lib" / "python" if ( not os.path.exists(resolver_python_path) or not os.path.exists(resolver_ld_path) - or not os.path.exists(resolver_python_path) ): raise RuntimeError( - f"Cant start Resolver missing path resolver_python_path: {resolver_python_path}, resolver_ld_path: {resolver_ld_path}, resolver_python_path: {resolver_python_path}" + f"Cant start Resolver missing path " + f"resolver_python_path: {resolver_python_path}, " + f"resolver_ld_path: {resolver_ld_path}" ) - pxr_plugin_paths.append(pathlib.Path(resolver_plugin_info_path).as_posix()) - ld_path.append(pathlib.Path(resolver_ld_path).as_posix()) - python_path.append(pathlib.Path(resolver_python_path).as_posix()) - - if logger: - logger.info(f"Asset resolver {app_name} initiated.") - resolver_setup_info_dict = {} - resolver_setup_info_dict["PXR_PLUGINPATH_NAME"] = os.pathsep.join(pxr_plugin_paths) - resolver_setup_info_dict["PYTHONPATH"] = os.pathsep.join(python_path) - if platform.system().lower() == "windows": - resolver_setup_info_dict["PATH"] = os.pathsep.join(ld_path) - else: - resolver_setup_info_dict["LD_LIBRARY_PATH"] = os.pathsep.join(ld_path) - resolver_setup_info_dict["TF_DEBUG"] = config.get_addon_settings_value( - settings, config.ADDON_SETTINGS_USD_TF_DEBUG - ) + def _append(_env: dict, key: str, path: str): + """Add path to key in env""" + current: str = _env.get(key) + if current: + return os.pathsep.join([current, path]) + return path - resolver_setup_info_dict["AYONLOGGERLOGLVL"] = config.get_addon_settings_value( - settings, config.ADDON_SETTINGS_USD_RESOLVER_LOG_LVL - ) + ld_path_key = "LD_LIBRARY_PATH" + if platform.system().lower() == "windows": + ld_path_key = "PATH" - resolver_setup_info_dict["AYONLOGGERSFILELOGGING"] = ( - config.get_addon_settings_value( - settings, config.ADDON_SETTINGS_USD_RESOLVER_LOG_FILLE_LOOGER_ENABLED - ) + pxr_pluginpath_name = _append( + env, "PXR_PLUGINPATH_NAME", resolver_plugin_info_path.as_posix() ) - - resolver_setup_info_dict["AYONLOGGERSFILEPOS"] = config.get_addon_settings_value( - settings, config.ADDON_SETTINGS_USD_RESOLVER_LOG_FILLE_LOOGER_FILE_PATH + ld_library_path = _append( + env, ld_path_key, resolver_ld_path.as_posix() ) - - resolver_setup_info_dict["AYON_LOGGIN_LOGGIN_KEYS"] = ( - config.get_addon_settings_value( - settings, config.ADDON_SETTINGS_USD_RESOLVER_LOG_LOGGIN_KEYS - ) + python_path = _append( + env, "PYTHONPATH", resolver_python_path.as_posix() ) - return resolver_setup_info_dict + resolver_settings = settings["ayon_usd"]["ayon_usd_resolver"] + return { + "TF_DEBUG": settings["ayon_usd"]["usd"]["usd_tf_debug"], + "AYONLOGGERLOGLVL": resolver_settings["ayon_log_lvl"], + "AYONLOGGERSFILELOGGING": resolver_settings["ayon_file_logger_enabled"], # noqa + "AYONLOGGERSFILEPOS": resolver_settings["file_logger_file_path"], + "AYON_LOGGIN_LOGGIN_KEYS": resolver_settings["ayon_logger_logging_keys"], # noqa + "PXR_PLUGINPATH_NAME": pxr_pluginpath_name, + "PYTHONPATH": python_path, + ld_path_key: ld_library_path + } diff --git a/client/ayon_usd/version.py b/client/ayon_usd/version.py index dfbb833..064a319 100644 --- a/client/ayon_usd/version.py +++ b/client/ayon_usd/version.py @@ -1,4 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'ayon_usd' version.""" -name = "ayon_usd" __version__ = "1.0.4-dev.37" diff --git a/create_package.py b/create_package.py index da23c9c..2866ec0 100644 --- a/create_package.py +++ b/create_package.py @@ -52,7 +52,6 @@ VERSION_PY_CONTENT = f'''# -*- coding: utf-8 -*- """Package declaring AYON addon '{ADDON_NAME}' version.""" -name = "{ADDON_NAME}" __version__ = "{ADDON_VERSION}" ''' diff --git a/server/settings/main.py b/server/settings/main.py index 1139478..1c84bbc 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -53,7 +53,8 @@ class AppPlatformPathModel(BaseSettingsModel): app_alias_list: list[str] = SettingsField( title="Application Alias", - description="Allows an admin to define a list of App Names that use the same resolver as the parent application", + description="Define a list of App Names that use the same " + "resolver as the parent application", default_factory=list, ) @@ -65,8 +66,13 @@ class AppPlatformPathModel(BaseSettingsModel): ) lake_fs_path: str = SettingsField( title="LakeFs Object Path", - description="The LakeFs internal path to the resolver zip, e.g: `AyonUsdResolverBin/Hou/ayon-usd-resolver_hou19.5_linux_py37.zip`\n" - "This information can be found on LakeFs server Object Information.", + description=( + "The LakeFs internal path to the resolver zip, e.g: " + "`AyonUsdResolverBin/Hou/ayon-usd-resolver_hou19.5_linux_py37.zip`" + "\n" + "This information can be found on LakeFs server Object " + "Information." + ), ) @@ -85,7 +91,10 @@ class AppPlatformURIModel(BaseSettingsModel): ) uri: str = SettingsField( title="LakeFs Object Uri", - description="Path to USD Asset Resolver plugin zip file on the LakeFs server, e.g: `lakefs://ayon-usd/V001/AyonUsdResolverBin/Hou/ayon-usd-resolver_hou19.5_linux_py37.zip`", + description=( + "Path to USD Asset Resolver plugin zip file on the LakeFs server, " + "e.g: `lakefs://ayon-usd/V001/AyonUsdResolverBin/Hou/ayon-usd-resolver_hou19.5_linux_py37.zip`" # noqa + ), ) @@ -94,12 +103,12 @@ class LakeFsSettings(BaseSettingsModel): _layout = "collapsed" - ayon_usd_lake_fs_server_uri: str = SettingsField( + server_uri: str = SettingsField( "https://lake.ayon.cloud", title="LakeFs Server Uri", description="The url to your LakeFs server.", ) - ayon_usd_lake_fs_server_repo: str = SettingsField( + server_repo: str = SettingsField( "lakefs://ayon-usd/main/", title="LakeFs Repository Uri", description="The url to your LakeFs Repository Path", @@ -202,7 +211,10 @@ class LakeFsSettings(BaseSettingsModel): ) lake_fs_overrides: list[AppPlatformURIModel] = SettingsField( title="Resolver Application overwrites", - description="Allows an admin to define a specific Resolver Zip for a specific Application", + description=( + "Allows to define a specific Resolver Zip " + "for a specific Application" + ), default_factory=list, ) @@ -216,13 +228,13 @@ class AyonResolverSettings(BaseSettingsModel): "WARN", title="AyonResolver Log Lvl", enum_resolver=log_lvl_enum, - description="Allows you to set the Verbosity of the AyonUsdResolver logger", + description="Set verbosity of the AyonUsdResolver logger", ) ayon_file_logger_enabled: str = SettingsField( "OFF", title="AyonResolver File Logger Enabled ", enum_resolver=file_logger_enum, - description="Allows you to enable or disable the AyonUsdResolver file logger, default is Off", + description="Enable or disable AyonUsdResolver file logger", ) ayon_logger_logging_keys: str = SettingsField( "", @@ -233,7 +245,11 @@ class AyonResolverSettings(BaseSettingsModel): file_logger_file_path: str = SettingsField( "", title="AyonResolver File logger file path", - description="Allows you to set a custom location where the file logger will export to. This can be a relative or absolute path. This is only used if `ayon_file_logger_enabled` is enabled.", + description=( + "Allows you to set a custom location where the file logger will " + "export to. This can be a relative or absolute path. This is only " + "used if `ayon_file_logger_enabled` is enabled." + ), ) @@ -252,17 +268,19 @@ class USDSettings(BaseSettingsModel): """USD settings.""" allow_addon_start: bool = SettingsField( - False, title="I Understand and Accept that this is an experimental feature" + False, title=( + "I understand and accept that this is an experimental feature" + ) ) - lakefs_settings: LakeFsSettings = SettingsField( + lakefs: LakeFsSettings = SettingsField( default_factory=LakeFsSettings, title="LakeFs Config" ) - ayon_usd_resolver_settings: AyonResolverSettings = SettingsField( + ayon_usd_resolver: AyonResolverSettings = SettingsField( default_factory=AyonResolverSettings, title="UsdResolver Config" ) - usd_settings: UsdSettings = SettingsField( + usd: UsdSettings = SettingsField( default_factory=UsdSettings, title="UsdLib Config" )