From eab7a04083d69d6af7a59f8dcafe45eeb3d693bb Mon Sep 17 00:00:00 2001 From: Bruno Coimbra Date: Fri, 12 Jan 2024 10:14:37 -0600 Subject: [PATCH] Add Parameter generators --- creation/lib/cvWParams.py | 1 + lib/credentials.py | 114 +++++++++++++++++++++++++++++++++----- lib/generators.py | 13 +++-- 3 files changed, 109 insertions(+), 19 deletions(-) diff --git a/creation/lib/cvWParams.py b/creation/lib/cvWParams.py index b65040049..9e9c7f4a3 100644 --- a/creation/lib/cvWParams.py +++ b/creation/lib/cvWParams.py @@ -346,6 +346,7 @@ def init_defaults(self): parameter_defaults = cWParams.CommentedOrderedDict() parameter_defaults["name"] = (None, "string", "parameter name", None) parameter_defaults["value"] = (None, "string", "parameter value", None) + parameter_defaults["type"] = ("static", "string", "parameter type", None) security_defaults = cWParams.CommentedOrderedDict() security_defaults["proxy_selection_plugin"] = ( diff --git a/lib/credentials.py b/lib/credentials.py index dbb39985a..d9a162ef8 100644 --- a/lib/credentials.py +++ b/lib/credentials.py @@ -17,7 +17,6 @@ import enum import gzip import os -import re import shutil import sys import tempfile @@ -27,13 +26,13 @@ from hashlib import md5 from importlib import import_module from io import BytesIO -from typing import Any, Generic, Iterable, List, Mapping, Optional, Set, Type, TypeVar, Union +from typing import Generic, Iterable, List, Mapping, Optional, Set, Type, TypeVar, Union import jwt import M2Crypto.EVP import M2Crypto.X509 -from glideinwms.lib import condorMonitor, logSupport, pubCrypto, symCrypto +from glideinwms.lib import logSupport, pubCrypto, symCrypto from glideinwms.lib.generators import Generator, load_generator from glideinwms.lib.util import hash_nc, is_str_safe @@ -135,6 +134,16 @@ def from_string(cls, string: str) -> "ParameterName": raise CredentialError(f"Unknown Parameter name: {string}") +class ParameterType(enum.Enum): + GENERATOR = "generator" + STATIC = "static" + + @classmethod + def from_string(cls, string: str) -> "ParameterType": + string = string.lower() + return ParameterType(string) + + class TrustDomain(enum.Enum): GRID = "grid" @@ -267,12 +276,12 @@ def __init__( raise CredentialError("CredentialPair requires a Credential subclass as second base class") credential_class = self.__class__.__bases__[1] - super(credential_class, self).__init__(string, path) # type: ignore + super(credential_class, self).__init__(string, path) # pylint: disable=bad-super-call # type: ignore self.private_credential = credential_class(private_string, private_path) def renew(self) -> None: try: - self.__renew__() # type: ignore + self.__renew__() # pylint: disable=no-member # type: ignore self.private_credential.__renew__() except NotImplementedError: pass @@ -287,29 +296,64 @@ def __setitem__(self, __k, __v): class Parameter: - def __init__(self, name: ParameterName, value): + param_type = ParameterType.STATIC + + def __init__(self, name: ParameterName, value: str): if not isinstance(name, ParameterName): raise TypeError("Name must be a ParameterName") - self.name = name - self.value = value + self._name = name + self._value = value + + @property + def name(self) -> ParameterName: + return self._name + + @property + def value(self): + return self._value def __repr__(self) -> str: - return f"{self.__class__.__name__}(name={self.name.value!r}, value={self.value!r})" + return f"{self.__class__.__name__}(name={self._name.value!r}, value={self._value!r}, param_type={self.param_type.value!r})" def __str__(self) -> str: return f"{self.name.value}={self.value}" +class ParameterGenerator(Parameter): + param_type = ParameterType.GENERATOR + + def __init__(self, name: ParameterName, value: str): + try: + self._generator = load_generator(value) + except ImportError as err: + raise TypeError(f"Could not load generator: {value}") from err + + super().__init__(name, value) + + @property + def value(self): + return self._generator.generate() + + class ParameterDict(dict): def __setitem__(self, __k, __v): + if isinstance(__k, str): + __k = ParameterName.from_string(__k) if not isinstance(__k, ParameterName): raise TypeError("Key must be a ParameterType") super().__setitem__(__k, __v) + def __getitem__(self, __k): + if isinstance(__k, str): + __k = ParameterName.from_string(__k) + if not isinstance(__k, ParameterName): + raise TypeError("Key must be a ParameterType") + return super().__getitem__(__k).value + def add(self, parameter: Parameter): if not isinstance(parameter, Parameter): raise TypeError("Parameter must be a Parameter") - self[parameter.name] = parameter.value + self[parameter.name] = parameter class CredentialGenerator(Credential[Credential]): @@ -564,8 +608,8 @@ def add_credential(self, credential, id=None, purpose=None, trust_domain=None, s rbCredential = RequestCredential(credential, purpose, trust_domain, security_class) self.credentials[id or rbCredential.id] = rbCredential - def add_parameter(self, name: str, value: str): - self.parameters.add(Parameter(ParameterName.from_string(name), value)) + def add_parameter(self, parameter: Parameter): + self.parameters.add(parameter) def load_from_element(self, element_descript): for path in element_descript.merged_data["Proxies"]: @@ -574,8 +618,11 @@ def load_from_element(self, element_descript): trust_domain = element_descript.merged_data["ProxyTrustDomains"].get(path, "None") security_class = element_descript.merged_data["ProxySecurityClasses"].get(path, id) self.add_credential(credential, trust_domain=trust_domain, security_class=security_class) - for name, parameter in element_descript.merged_data["Parameters"].items(): - self.add_parameter(name, parameter["value"]) + for name, data in element_descript.merged_data["Parameters"].items(): + parameter = create_parameter( + ParameterName.from_string(name), data["value"], ParameterType.from_string(data["type"]) + ) + self.add_parameter(parameter) class SubmitBundle: @@ -752,7 +799,7 @@ def create_credential_pair( credential_class = credential_of_type(cred_type) if issubclass(credential_class, CredentialPair): return credential_class(string, path, private_string, private_path) - except CredentialError as err: + except CredentialError: pass except Exception as err: raise CredentialError( @@ -763,6 +810,43 @@ def create_credential_pair( ) +def parameter_of_type(param_type: ParameterType) -> Type[Parameter]: + """Returns the parameter subclass for the given type. + + Args: + param_type (ParameterType): parameter type + + Raises: + CredentialError: if the parameter type is unknown + + Returns: + Parameter: parameter subclass + """ + + class_dict = {} + for i in Parameter.__subclasses__(): + class_dict[i.param_type] = i + class_dict[ParameterType.STATIC] = Parameter + try: + return class_dict[param_type] + except KeyError as err: + raise CredentialError(f"Unknown Parameter type: {param_type}") from err + + +def create_parameter(name: ParameterName, value: str, param_type: Optional[ParameterType] = None) -> Parameter: + parameter_types = [param_type] if param_type else ParameterType + for param_type in parameter_types: + try: + parameter_class = parameter_of_type(param_type) + if issubclass(parameter_class, Parameter): + return parameter_class(name, value) + except TypeError: + pass # Parameter type incompatible with input + except Exception as err: + raise CredentialError(f'Unexpected error loading parameter: name="{name}", value="{value}"') from err + raise CredentialError(f'Could not load parameter: name="{name}", value="{value}"') + + def get_scitoken(elementDescript, trust_domain): """Look for a local SciToken specified for the trust domain. diff --git a/lib/generators.py b/lib/generators.py index 5633e6060..feae274b6 100644 --- a/lib/generators.py +++ b/lib/generators.py @@ -14,12 +14,15 @@ # import inspect +import os +import re import sys from abc import ABC, abstractmethod -from importlib import import_module from typing import Generic, List, Mapping, Optional, TypeVar +from glideinwms.lib.util import import_module + sys.path.append("/etc/gwms-frontend/plugin.d") _loaded_generators = {} @@ -51,15 +54,17 @@ def load_generator(module: str) -> Generator: Generator: generator object """ + module_name = re.sub(r"\.py[co]?$", "", os.path.basename(module)) # Extract module name from path + try: - if not module in _loaded_generators: + if not module_name in _loaded_generators: imported_module = import_module(module) - if module not in _loaded_generators: + if module_name not in _loaded_generators: del imported_module raise ImportError(f"Module {module} does not export a generator") except ImportError as e: raise ImportError(f"Failed to import module {module}") from e - return _loaded_generators[module] + return _loaded_generators[module_name] def export_generator(generator: Generator):