diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 301ce32f7d4..dd820c54c55 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -31,6 +31,10 @@ jobs: ###### Python 3 - TEST_NAME: "Python 3 client" ARGS: CLIENT_USE_PYTHON3=Yes + - TEST_NAME: "Python 3 server" + ARGS: SERVER_USE_PYTHON3=Yes + - TEST_NAME: "Python 3 server and client" + ARGS: CLIENT_USE_PYTHON3=Yes SERVER_USE_PYTHON3=Yes MYSQL_VER=8.0 steps: - uses: actions/checkout@v2 diff --git a/setup.cfg b/setup.cfg index 85b8407aa0e..7273919156d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,6 +51,11 @@ install_requires = six sqlalchemy subprocess32 +zip_safe = False +include_package_data = True + +[options.package_data] +* = ConfigTemplate.cfg, *.sql [options.packages.find] where=src diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py index bff05030703..1f54aa1f413 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py @@ -12,7 +12,9 @@ import imp import six +from DIRAC.Core.Utilities.Decorators import deprecated from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton +from DIRAC.Core.Utilities.Extensions import extensionsByPriority @six.add_metaclass(DIRACSingleton) @@ -25,33 +27,36 @@ def __init__(self): def __load(self): if self.__orderedExtNames: return - for extName in self.getCSExtensions() + ['']: + for extName in extensionsByPriority(): try: - if not extName.endswith("DIRAC"): - extension = '%sDIRAC' % extName - res = imp.find_module(extension) + res = imp.find_module(extName) if res[0]: res[0].close() - self.__orderedExtNames.append(extension) - self.__modules[extension] = res + self.__orderedExtNames.append(extName) + self.__modules[extName] = res except ImportError: pass def getCSExtensions(self): if not self.__csExt: - from DIRAC.ConfigurationSystem.Client.Config import gConfig - exts = gConfig.getValue('/DIRAC/Extensions', []) - for iP in range(len(exts)): - ext = exts[iP] + if six.PY3: + exts = extensionsByPriority() + else: + from DIRAC.ConfigurationSystem.Client.Config import gConfig + exts = gConfig.getValue('/DIRAC/Extensions', []) + + self.__csExt = [] + for ext in exts: if ext.endswith("DIRAC"): ext = ext[:-5] - exts[iP] = ext - self.__csExt = exts + # If the extension is now "" (i.e. vanilla DIRAC), don't include it + if ext: + self.__csExt.append(ext) return self.__csExt + @deprecated("Use DIRAC.Core.Utilities.Extensions.extensionsByPriority instead") def getInstalledExtensions(self): - self.__load() - return list(self.__orderedExtNames) + return extensionsByPriority() def getExtensionPath(self, extName): self.__load() @@ -83,11 +88,12 @@ def getCSExtensions(): return Extensions().getCSExtensions() +@deprecated("Use DIRAC.Core.Utilities.Extensions.extensionsByPriority instead") def getInstalledExtensions(): """ Return list of extensions registered in the CS and available in local installation """ - return Extensions().getInstalledExtensions() + return extensionsByPriority() def skipCACheck(): diff --git a/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py b/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py index 267e18acb86..34e1f98b10b 100755 --- a/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +++ b/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py @@ -404,7 +404,7 @@ def enableCS(self): """ Force the connection the Configuration Server - (And incidentaly reinitialize the ObjectLoader and logger) + (And incidentally reinitialize the ObjectLoader and logger) """ res = gRefresher.enable() diff --git a/src/DIRAC/Core/Base/private/ModuleLoader.py b/src/DIRAC/Core/Base/private/ModuleLoader.py index 34e87e43007..3f89d8b6b5a 100644 --- a/src/DIRAC/Core/Base/private/ModuleLoader.py +++ b/src/DIRAC/Core/Base/private/ModuleLoader.py @@ -4,17 +4,14 @@ from __future__ import division from __future__ import print_function -import six import os -import imp from DIRAC.Core.Utilities import List from DIRAC import gConfig, S_ERROR, S_OK, gLogger -from DIRAC.ConfigurationSystem.Client.Helpers import getInstalledExtensions from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.Core.Utilities.Extensions import extensionsByPriority, recurseImport class ModuleLoader(object): - def __init__(self, importLocation, sectionFinder, superClass, csSuffix=False, moduleSuffix=False): self.__modules = {} self.__loadedModules = {} @@ -43,7 +40,7 @@ def loadModules(self, modulesList, hideExceptions=False): for modName in modulesList: gLogger.verbose("Checking %s" % modName) # if it's a executor modName name just load it and be done with it - if modName.find("/") > -1: + if "/" in modName: gLogger.verbose("Module %s seems to be a valid name. Try to load it!" % modName) result = self.loadModule(modName, hideExceptions=hideExceptions) if not result['OK']: @@ -64,13 +61,12 @@ def loadModules(self, modulesList, hideExceptions=False): return result # Look what is installed parentModule = None - for rootModule in getInstalledExtensions(): - if system.find("System") != len(system) - 6: - parentImport = "%s.%sSystem.%s" % (rootModule, system, self.__csSuffix) - else: - parentImport = "%s.%s.%s" % (rootModule, system, self.__csSuffix) + for rootModule in extensionsByPriority(): + if not system.endswith("System"): + system += "System" + parentImport = "%s.%s.%s" % (rootModule, system, self.__csSuffix) # HERE! - result = self.__recurseImport(parentImport) + result = recurseImport(parentImport) if not result['OK']: return result parentModule = result['Value'] @@ -81,7 +77,7 @@ def loadModules(self, modulesList, hideExceptions=False): parentPath = parentModule.__path__[0] gLogger.notice("Found modules path at %s" % parentImport) for entry in os.listdir(parentPath): - if entry[-3:] != ".py" or entry == "__init__.py": + if entry == "__init__.py" or not entry.endswith(".py"): continue if not os.path.isfile(os.path.join(parentPath, entry)): continue @@ -111,9 +107,9 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): if loadGroup: gLogger.info("Found load group %s. Will load %s" % (modName, ", ".join(loadGroup))) for loadModName in loadGroup: - if loadModName.find("/") == -1: + if "/" not in loadModName: loadModName = "%s/%s" % (modList[0], loadModName) - result = self.loadModule(loadModName, hideExceptions=hideExceptions, parentModule=False) + result = self.loadModule(loadModName, hideExceptions=hideExceptions) if not result['OK']: return result return S_OK() @@ -123,7 +119,7 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): loadName = modName gLogger.info("Loading %s" % (modName)) else: - if loadName.find("/") == -1: + if "/" not in loadName: loadName = "%s/%s" % (modList[0], loadName) gLogger.info("Loading %s (%s)" % (modName, loadName)) # If already loaded, skip @@ -143,10 +139,10 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): gLogger.info("Trying to %s from CS defined path %s" % (loadName, handlerPath)) gLogger.verbose("Found handler for %s: %s" % (loadName, handlerPath)) handlerPath = handlerPath.replace("/", ".") - if handlerPath.find(".py", len(handlerPath) - 3) > -1: + if handlerPath.endswith(".py"): handlerPath = handlerPath[:-3] className = List.fromChar(handlerPath, ".")[-1] - result = self.__recurseImport(handlerPath) + result = recurseImport(handlerPath) if not result['OK']: return S_ERROR("Cannot load user defined handler %s: %s" % (handlerPath, result['Message'])) gLogger.verbose("Loaded %s" % handlerPath) @@ -156,17 +152,17 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): modImport = module if self.__modSuffix: modImport = "%s%s" % (modImport, self.__modSuffix) - result = self.__recurseImport(modImport, parentModule, hideExceptions=hideExceptions) + result = recurseImport(modImport, parentModule, hideExceptions=hideExceptions) else: # Check to see if the module exists in any of the root modules gLogger.info("Trying to autodiscover %s" % loadName) - rootModulesToLook = getInstalledExtensions() + rootModulesToLook = extensionsByPriority() for rootModule in rootModulesToLook: importString = '%s.%sSystem.%s.%s' % (rootModule, system, self.__importLocation, module) if self.__modSuffix: importString = "%s%s" % (importString, self.__modSuffix) gLogger.verbose("Trying to load %s" % importString) - result = self.__recurseImport(importString, hideExceptions=hideExceptions) + result = recurseImport(importString, hideExceptions=hideExceptions) # Error while loading if not result['OK']: return result @@ -203,29 +199,3 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): gLogger.notice("Loaded module %s" % modName) return S_OK() - - def __recurseImport(self, modName, parentModule=None, hideExceptions=False): - gLogger.debug("importing recursively %s, parentModule=%s, hideExceptions=%s" % (modName, - parentModule, - hideExceptions)) - if isinstance(modName, six.string_types): - modName = List.fromChar(modName, ".") - try: - if parentModule: - impData = imp.find_module(modName[0], parentModule.__path__) - else: - impData = imp.find_module(modName[0]) - impModule = imp.load_module(modName[0], *impData) - if impData[0]: - impData[0].close() - except ImportError as excp: - strExcp = str(excp) - if strExcp.find("No module named") == 0 and strExcp.find(modName[0]) == len(strExcp) - len(modName[0]): - return S_OK() - errMsg = "Can't load %s" % ".".join(modName) - if not hideExceptions: - gLogger.exception(errMsg) - return S_ERROR(errMsg) - if len(modName) == 1: - return S_OK(impModule) - return self.__recurseImport(modName[1:], impModule) diff --git a/src/DIRAC/Core/DISET/private/MessageFactory.py b/src/DIRAC/Core/DISET/private/MessageFactory.py index 807b94f4db4..5c211f601d9 100644 --- a/src/DIRAC/Core/DISET/private/MessageFactory.py +++ b/src/DIRAC/Core/DISET/private/MessageFactory.py @@ -7,7 +7,7 @@ from DIRAC import S_OK, S_ERROR from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.Core.Utilities import List -from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals +from DIRAC.Core.Utilities.Extensions import extensionsByPriority class MessageFactory(object): @@ -256,10 +256,9 @@ def loadObjects(path, reFilter=None, parentClass=None): reFilter = re.compile(r".*[a-z1-9]\.py$") pathList = List.fromChar(path, "/") - parentModuleList = ["%sDIRAC" % ext for ext in CSGlobals.getCSExtensions()] + ['DIRAC'] objectsToLoad = {} # Find which object files match - for parentModule in parentModuleList: + for parentModule in extensionsByPriority(): objDir = os.path.join(DIRAC.rootPath, parentModule, *pathList) if not os.path.isdir(objDir): continue diff --git a/src/DIRAC/Core/Utilities/DErrno.py b/src/DIRAC/Core/Utilities/DErrno.py index 1dcfc1353f3..ea0cdf23cc4 100644 --- a/src/DIRAC/Core/Utilities/DErrno.py +++ b/src/DIRAC/Core/Utilities/DErrno.py @@ -43,9 +43,11 @@ import six import os -import imp +import importlib import sys +from DIRAC.Core.Utilities.Extensions import extensionsByPriority + # To avoid conflict, the error numbers should be greater than 1000 # We decided to group the by range of 100 per system @@ -344,50 +346,22 @@ def includeExtensionErrors(): Should be called only at the initialization of DIRAC, so by the parseCommandLine, dirac-agent.py, dirac-service.py, dirac-executor.py """ - - def __recurseImport(modName, parentModule=None, fullName=False): - """ Internal function to load modules - """ - if isinstance(modName, six.string_types): - modName = modName.split(".") - if not fullName: - fullName = ".".join(modName) + for extension in reversed(extensionsByPriority()): + if extension == "DIRAC": + continue try: - if parentModule: - impData = imp.find_module(modName[0], parentModule.__path__) - else: - impData = imp.find_module(modName[0]) - impModule = imp.load_module(modName[0], *impData) - if impData[0]: - impData[0].close() + ext_derrno = importlib.import_module('%s.Core.Utilities.DErrno' % extension) except ImportError: - return None - if len(modName) == 1: - return impModule - return __recurseImport(modName[1:], impModule, fullName=fullName) - - from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals - allExtensions = CSGlobals.getCSExtensions() - - for extension in allExtensions: - ext_derrno = None - try: - - ext_derrno = __recurseImport('%sDIRAC.Core.Utilities.DErrno' % extension) - - if ext_derrno: - # The next 3 dictionary MUST be present for consistency - - # Global name of errors - sys.modules[__name__].__dict__.update(ext_derrno.extra_dErrName) - # Dictionary with the error codes - sys.modules[__name__].dErrorCode.update(ext_derrno.extra_dErrorCode) - # Error description string - sys.modules[__name__].dStrError.update(ext_derrno.extra_dStrError) - - # extra_compatErrorString is optional - for err in getattr(ext_derrno, 'extra_compatErrorString', []): - sys.modules[__name__].compatErrorString.setdefault(err, []).extend(ext_derrno.extra_compatErrorString[err]) - - except Exception: pass + else: + # The next 3 dictionary MUST be present for consistency + # Global name of errors + sys.modules[__name__].__dict__.update(ext_derrno.extra_dErrName) + # Dictionary with the error codes + sys.modules[__name__].dErrorCode.update(ext_derrno.extra_dErrorCode) + # Error description string + sys.modules[__name__].dStrError.update(ext_derrno.extra_dStrError) + + # extra_compatErrorString is optional + for err in getattr(ext_derrno, 'extra_compatErrorString', []): + sys.modules[__name__].compatErrorString.setdefault(err, []).extend(ext_derrno.extra_compatErrorString[err]) diff --git a/src/DIRAC/Core/Utilities/DIRACScript.py b/src/DIRAC/Core/Utilities/DIRACScript.py index 3bd4244f3d5..35db01aa4c2 100644 --- a/src/DIRAC/Core/Utilities/DIRACScript.py +++ b/src/DIRAC/Core/Utilities/DIRACScript.py @@ -75,47 +75,3 @@ def __call__(self, func=None): ) return entrypoint.load()._func() - - -def _entrypointToExtension(entrypoint): - """"Get the extension name from an EntryPoint object""" - # In Python 3.9 this can be "entrypoint.module" - module = entrypoint.pattern.match(entrypoint.value).groupdict()["module"] - extensionName = module.split(".")[0] - return extensionName - - -def _extensionsByPriority(): - """Discover extensions using the setuptools metadata - - TODO: This should move into a function which can also be called to fill the CS - """ - # This is only available in Python 3.8+ so it has to be here for now - from importlib import metadata # pylint: disable=no-name-in-module - - priorities = defaultdict(list) - for entrypoint in metadata.entry_points()['dirac']: - extensionName = _entrypointToExtension(entrypoint) - extension_metadata = entrypoint.load()() - priorities[extension_metadata["priority"]].append(extensionName) - - extensions = [] - for priority, extensionNames in sorted(priorities.items()): - if len(extensionNames) != 1: - print( - "WARNING: Found multiple extensions with priority", - "{} ({})".format(priority, extensionNames), - ) - # If multiple are passed, sort the extensions so things are deterministic at least - extensions.extend(sorted(extensionNames)) - return extensions - - -def _getExtensionMetadata(extensionName): - """Get the metadata for a given extension name""" - # This is only available in Python 3.8+ so it has to be here for now - from importlib import metadata # pylint: disable=no-name-in-module - - for entrypoint in metadata.entry_points()['dirac']: - if extensionName == _entrypointToExtension(entrypoint): - return entrypoint.load()() diff --git a/src/DIRAC/Core/Utilities/ObjectLoader.py b/src/DIRAC/Core/Utilities/ObjectLoader.py index e1e4621c964..0ca54b2a852 100644 --- a/src/DIRAC/Core/Utilities/ObjectLoader.py +++ b/src/DIRAC/Core/Utilities/ObjectLoader.py @@ -8,14 +8,13 @@ import six import re -import imp import pkgutil import collections from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities import DErrno -from DIRAC.Core.Utilities import List, DIRACSingleton -from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals +from DIRAC.Core.Utilities import DIRACSingleton +from DIRAC.Core.Utilities.Extensions import extensionsByPriority, recurseImport @six.add_metaclass(DIRACSingleton.DIRACSingleton) @@ -41,7 +40,6 @@ def __init__(self, baseModules=False): def _init(self, baseModules): """ Actually performs the initialization """ - if not baseModules: baseModules = ['DIRAC'] self.__rootModules = baseModules @@ -70,54 +68,20 @@ def __rootImport(self, modName, hideExceptions=False): if rootModule: impName = "%s.%s" % (rootModule, impName) gLogger.debug("Trying to load %s" % impName) - result = self.__recurseImport(impName, hideExceptions=hideExceptions) - # Error. Something cannot be imported. Return error + result = recurseImport(impName, hideExceptions=hideExceptions) if not result['OK']: return result - # Huge success! if result['Value']: return S_OK((impName, result['Value'])) - # Nothing found, continue - # Return nothing found return S_OK() - def __recurseImport(self, modName, parentModule=None, hideExceptions=False, fullName=False): - """ Internal function to load modules - """ - if isinstance(modName, six.string_types): - modName = List.fromChar(modName, ".") - if not fullName: - fullName = ".".join(modName) - if fullName in self.__objs: - return S_OK(self.__objs[fullName]) - try: - if parentModule: - impData = imp.find_module(modName[0], parentModule.__path__) - else: - impData = imp.find_module(modName[0]) - impModule = imp.load_module(modName[0], *impData) - if impData[0]: - impData[0].close() - except Exception as excp: - if "No module named" in str(excp) and modName[0] in str(excp): - return S_OK(None) - errMsg = "Can't load %s in %s" % (".".join(modName), parentModule.__path__[0]) - if not hideExceptions: - gLogger.exception(errMsg) - return S_ERROR(DErrno.EIMPERR, errMsg) - if len(modName) == 1: - self.__objs[fullName] = impModule - return S_OK(impModule) - return self.__recurseImport(modName[1:], impModule, - hideExceptions=hideExceptions, fullName=fullName) - def __generateRootModules(self, baseModules): """ Iterate over all the possible root modules """ self.__rootModules = baseModules - for rootModule in reversed(CSGlobals.getCSExtensions()): - if rootModule[-5:] != "DIRAC" and rootModule not in self.__rootModules: - self.__rootModules.append("%sDIRAC" % rootModule) + for rootModule in reversed(extensionsByPriority()): + if rootModule not in self.__rootModules: + self.__rootModules.append(rootModule) self.__rootModules.append("") # Reversing the order because we want first to look in the extension(s) @@ -136,18 +100,16 @@ def loadModule(self, importString, hideExceptions=False): def loadObject(self, importString, objName=False, hideExceptions=False): """ Load an object from inside a module """ + if not objName: + objName = importString.split(".")[-1] + result = self.loadModule(importString, hideExceptions=hideExceptions) if not result['OK']: return result modObj = result['Value'] - modFile = modObj.__file__ - - if not objName: - objName = List.fromChar(importString, ".")[-1] - try: result = S_OK(getattr(modObj, objName)) - result['ModuleFile'] = modFile + result['ModuleFile'] = modObj.__file__ return result except AttributeError: return S_ERROR(DErrno.EIMPERR, "%s does not contain a %s object" % (importString, objName)) @@ -163,31 +125,23 @@ def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False, :param continueOnError: if True, continue loading further module even if one fails """ - - if 'OrderedDict' in dir(collections): - modules = collections.OrderedDict() - else: - modules = {} - + modules = collections.OrderedDict() if isinstance(reFilter, six.string_types): reFilter = re.compile(reFilter) for rootModule in self.__rootModules: + impPath = modulePath if rootModule: - impPath = "%s.%s" % (rootModule, modulePath) - else: - impPath = modulePath + impPath = "%s.%s" % (rootModule, impPath) gLogger.debug("Trying to load %s" % impPath) - result = self.__recurseImport(impPath) + result = recurseImport(impPath) if not result['OK']: return result if not result['Value']: continue - parentModule = result['Value'] - fsPath = parentModule.__path__[0] - gLogger.verbose("Loaded module %s at %s" % (impPath, fsPath)) + gLogger.verbose("Loaded module %s at %s" % (impPath, parentModule.__path__)) for _modLoader, modName, isPkg in pkgutil.walk_packages(parentModule.__path__): if reFilter and not reFilter.match(modName): @@ -204,7 +158,7 @@ def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False, if modKeyName in modules: continue fullName = "%s.%s" % (impPath, modName) - result = self.__recurseImport(modName, parentModule=parentModule, fullName=fullName) + result = recurseImport(fullName) if not result['OK']: if continueOnError: gLogger.error("Error loading module but continueOnError is true", "module %s error %s" % (fullName, result)) @@ -212,18 +166,15 @@ def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False, return result if not result['Value']: continue - modObj = result['Value'] - try: - modClass = getattr(modObj, modName) - except AttributeError: + modClass = getattr(result['Value'], modName, None) + if not modClass: gLogger.warn("%s does not contain a %s object" % (fullName, modName)) continue if parentClass and not issubclass(modClass, parentClass): continue - # Huge success! modules[modKeyName] = modClass return S_OK(modules) diff --git a/src/DIRAC/Core/Utilities/Plotting/ObjectLoader.py b/src/DIRAC/Core/Utilities/Plotting/ObjectLoader.py index 24c64e990e0..d190d88cd03 100644 --- a/src/DIRAC/Core/Utilities/Plotting/ObjectLoader.py +++ b/src/DIRAC/Core/Utilities/Plotting/ObjectLoader.py @@ -6,10 +6,11 @@ from __future__ import print_function import re import os +import six import DIRAC from DIRAC import gLogger from DIRAC.Core.Utilities import List -from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals +from DIRAC.Core.Utilities.Extensions import extensionsByPriority def loadObjects(path, reFilter=None, parentClass=None): @@ -23,11 +24,13 @@ def loadObjects(path, reFilter=None, parentClass=None): reFilter = re.compile(r".*[a-z1-9]\.py$") pathList = List.fromChar(path, "/") - parentModuleList = ["%sDIRAC" % ext for ext in CSGlobals.getCSExtensions()] + ['DIRAC'] objectsToLoad = {} # Find which object files match - for parentModule in parentModuleList: - objDir = os.path.join(DIRAC.rootPath, parentModule, *pathList) + for parentModule in extensionsByPriority(): + if six.PY3: + objDir = os.path.join(os.path.dirname(os.path.dirname(DIRAC.__file__)), parentModule, *pathList) + else: + objDir = os.path.join(DIRAC.rootPath, parentModule, *pathList) if not os.path.isdir(objDir): continue for objFile in os.listdir(objDir): diff --git a/src/DIRAC/Core/Utilities/Version.py b/src/DIRAC/Core/Utilities/Version.py index 12fc51ffb7a..1ea925e9403 100644 --- a/src/DIRAC/Core/Utilities/Version.py +++ b/src/DIRAC/Core/Utilities/Version.py @@ -3,50 +3,34 @@ from __future__ import print_function __RCSID__ = "$Id$" -import DIRAC +import importlib from DIRAC import S_OK -from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions +from DIRAC.Core.Utilities.Extensions import extensionsByPriority def getCurrentVersion(): """ Get a string corresponding to the current version of the DIRAC package and all the installed extension packages """ - - version = 'DIRAC ' + DIRAC.version - - for ext in getCSExtensions(): + for ext in extensionsByPriority(): try: - import imp - module = imp.find_module("%sDIRAC" % ext) - extModule = imp.load_module("%sDIRAC" % ext, *module) - version = extModule.version - except ImportError: + return S_OK(importlib.import_module(ext).version) + except (ImportError, AttributeError): pass - except AttributeError: - pass - - return S_OK(version) def getVersion(): """ Get a dictionary corresponding to the current version of the DIRAC package and all the installed extension packages """ - vDict = {'Extensions': {}} - vDict['DIRAC'] = DIRAC.version - - for ext in getCSExtensions(): + for ext in extensionsByPriority(): try: - import imp - module = imp.find_module("%sDIRAC" % ext) - extModule = imp.load_module("%sDIRAC" % ext, *module) - vDict['Extensions'][ext] = extModule.version - except ImportError: + version = importlib.import_module(ext).version + except (ImportError, AttributeError): pass - except AttributeError: - pass - + if ext.endswith("DIRAC") and ext != "DIRAC": + ext = ext[:len("DIRAC")] + vDict['Extensions'][ext] = version return S_OK(vDict) diff --git a/src/DIRAC/Core/Utilities/test/Test_ObjectLoader.py b/src/DIRAC/Core/Utilities/test/Test_ObjectLoader.py index 292f76d63a4..abd40502b3b 100644 --- a/src/DIRAC/Core/Utilities/test/Test_ObjectLoader.py +++ b/src/DIRAC/Core/Utilities/test/Test_ObjectLoader.py @@ -1,36 +1,20 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import unittest from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.DISET.RequestHandler import RequestHandler -class ObjectLoaderMainSuccessScenario(unittest.TestCase): +def _check(result): + assert result["OK"], result['Message'] + return result['Value'] - def setUp(self): - self.ol = ObjectLoader() - def __check(self, result): - if not result['OK']: - self.fail(result['Message']) - return result['Value'] - - def test_load(self): - self.__check(self.ol.loadObject("Core.Utilities.List", 'fromChar')) - self.__check(self.ol.loadObject("Core.Utilities.ObjectLoader", "ObjectLoader")) - dataFilter = self.__check(self.ol.getObjects("WorkloadManagementSystem.Service", ".*Handler")) - dataClass = self.__check(self.ol.getObjects("WorkloadManagementSystem.Service", parentClass=RequestHandler)) - self.assertEqual(sorted(dataFilter), sorted(dataClass)) - -############################################################################# -# Test Suite run -############################################################################# - - -if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase(ObjectLoaderMainSuccessScenario) - testResult = unittest.TextTestRunner(verbosity=2).run(suite) - -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF# +def test_load(): + _check(ObjectLoader().loadObject("Core.Utilities.List", 'fromChar')) + _check(ObjectLoader().loadObject("Core.Utilities.ObjectLoader", "ObjectLoader")) + assert _check(ObjectLoader().loadObject("Core.Utilities.ObjectLoader")) is ObjectLoader + dataFilter = _check(ObjectLoader().getObjects("WorkloadManagementSystem.Service", ".*Handler")) + dataClass = _check(ObjectLoader().getObjects("WorkloadManagementSystem.Service", parentClass=RequestHandler)) + assert sorted(dataFilter) == sorted(dataClass) diff --git a/src/DIRAC/Core/scripts/dirac_configure.py b/src/DIRAC/Core/scripts/dirac_configure.py index 6aa5194c8fd..7a3dd96ee74 100755 --- a/src/DIRAC/Core/scripts/dirac_configure.py +++ b/src/DIRAC/Core/scripts/dirac_configure.py @@ -245,11 +245,11 @@ def runConfigurationWizard(params): """Interactively configure DIRAC using metadata from installed extensions""" import subprocess from prompt_toolkit import prompt, print_formatted_text, HTML - from DIRAC.Core.Utilities.DIRACScript import _extensionsByPriority, _getExtensionMetadata + from DIRAC.Core.Utilities.Extensions import extensionsByPriority, getExtensionMetadata - extensions = _extensionsByPriority() + extensions = extensionsByPriority() - extensionMetadata = _getExtensionMetadata(extensions[-1]) + extensionMetadata = getExtensionMetadata(extensions[-1]) defaultSetup = extensionMetadata.get("default_setup", "") setups = extensionMetadata.get("setups", {}) diff --git a/src/DIRAC/Core/scripts/dirac_install_db.py b/src/DIRAC/Core/scripts/dirac_install_db.py index 519a2763b10..8c08f6227c8 100755 --- a/src/DIRAC/Core/scripts/dirac_install_db.py +++ b/src/DIRAC/Core/scripts/dirac_install_db.py @@ -37,14 +37,14 @@ def main(): result = gComponentInstaller.installDatabase(db) if not result['OK']: print("ERROR: failed to correctly install %s" % db, result['Message']) - else: - extension, system = result['Value'] - gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True) - - if db != 'InstalledComponentsDB': - result = MonitoringUtilities.monitorInstallation('DB', system, db) - if not result['OK']: - print("ERROR: failed to register installation in database: %s" % result['Message']) + continue + extension, system = result['Value'] + gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True) + + if db != 'InstalledComponentsDB': + result = MonitoringUtilities.monitorInstallation('DB', system, db) + if not result['OK']: + print("ERROR: failed to register installation in database: %s" % result['Message']) if __name__ == "__main__": diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py index a4bde2680f1..f534530e879 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py @@ -586,7 +586,7 @@ def _getExistingMetadata(self, lfns, connection=False): return res successful = res['Value']['Successful'] failed = res['Value']['Failed'] - for lfn, error in res['Value']['Failed'].items(): + for lfn, error in list(failed.items()): if error == 'No such file or directory': failed.pop(lfn) return S_OK((successful, failed)) @@ -713,7 +713,7 @@ def setFileStatus(self, lfns, connection=False): return res failed = res['Value']['Failed'] successful = {} - for lfn in res['Value']['Successful'].keys(): + for lfn in res['Value']['Successful']: status = lfns[lfn] if isinstance(status, six.string_types): if status not in self.db.validFileStatus: @@ -870,7 +870,7 @@ def exists(self, lfns, connection=False): # either {lfn : guid} # or P lfn : {PFN : .., GUID : ..} } if isinstance(lfns, dict): - val = lfns.values() + val = list(lfns.values()) # We have values, take the first to identify the type if val: @@ -881,7 +881,7 @@ def exists(self, lfns, connection=False): guidList = [lfns[lfn]['GUID'] for lfn in lfns] elif isinstance(val, six.string_types): # We hope that it is the GUID which is given - guidList = lfns.values() + guidList = list(lfns.values()) if guidList: # A dict { guid: lfn to which it is supposed to be associated } @@ -918,7 +918,7 @@ def getFileSize(self, lfns, connection=False): return res totalSize = 0 - for lfn in res['Value']['Successful'].keys(): + for lfn in res['Value']['Successful']: size = res['Value']['Successful'][lfn]['Size'] res['Value']['Successful'][lfn] = size totalSize += size @@ -1202,7 +1202,7 @@ def changeFileGroup(self, lfns): return res failed = res['Value']['Failed'] successful = {} - for lfn in res['Value']['Successful'].keys(): + for lfn in res['Value']['Successful']: group = lfns[lfn] if isinstance(group, six.string_types): groupRes = self.db.ugManager.findGroup(group) @@ -1232,7 +1232,7 @@ def changeFileOwner(self, lfns): return res failed = res['Value']['Failed'] successful = {} - for lfn in res['Value']['Successful'].keys(): + for lfn in res['Value']['Successful']: owner = lfns[lfn] if isinstance(owner, six.string_types): userRes = self.db.ugManager.findUser(owner) @@ -1262,7 +1262,7 @@ def changeFileMode(self, lfns): return res failed = res['Value']['Failed'] successful = {} - for lfn in res['Value']['Successful'].keys(): + for lfn in res['Value']['Successful']: mode = lfns[lfn] currentMode = res['Value']['Successful'][lfn]['Mode'] if int(currentMode) == int(mode): diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py index cbe5343a264..37c47ea1452 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py @@ -18,13 +18,10 @@ class FileCatalogDB(DB): def __init__(self, databaseLocation='DataManagement/FileCatalogDB'): - """ Standard Constructor - """ - # The database location can be specified in System/Database form or in just the Database name # in the DataManagement system db = databaseLocation - if db.find('/') == -1: + if "/" not in db: db = 'DataManagement/' + db super(FileCatalogDB, self).__init__('FileCatalogDB', db) @@ -37,10 +34,8 @@ def __init__(self, databaseLocation='DataManagement/FileCatalogDB'): self.dmeta = None self.fmeta = None self.datasetManager = None - self.objectLoader = None def setConfig(self, databaseConfig): - self.directories = {} # In memory storage of the various parameters self.users = {} @@ -63,9 +58,6 @@ def setConfig(self, databaseConfig): self.visibleFileStatus = databaseConfig['VisibleFileStatus'] self.visibleReplicaStatus = databaseConfig['VisibleReplicaStatus'] - # Obtain the plugins to be used for DB interaction - self.objectLoader = ObjectLoader() - # Load the configured components for compAttribute, componentType in [("ugManager", "UserGroupManager"), ("seManager", "SEManager"), @@ -87,7 +79,7 @@ def __loadCatalogComponent(self, componentType, componentName): """ Create an object of a given catalog component """ componentModule = 'DataManagementSystem.DB.FileCatalogComponents.%s.%s' % (componentType, componentName) - result = self.objectLoader.loadObject(componentModule, componentName) + result = ObjectLoader().loadObject(componentModule) if not result['OK']: gLogger.error('Failed to load catalog component', '%s: %s' % (componentName, result['Message'])) return result @@ -245,7 +237,7 @@ def exists(self, lfns, credDict): successful = res['Value']['Successful'] notExist = [] - for lfn in res['Value']['Successful'].keys(): + for lfn in list(res['Value']['Successful']): if not successful[lfn]: notExist.append(lfn) successful.pop(lfn) diff --git a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py index 7d59f0df1db..e1436379f51 100644 --- a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py +++ b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py @@ -60,17 +60,21 @@ __RCSID__ = "$Id$" -import os +import getpass +import glob +import importlib +import inspect import io +import os +import pkgutil import re -import glob +import shutil import stat import time -import subprocess32 as subprocess -import shutil -import inspect -import importlib +from collections import defaultdict +import importlib_resources +import subprocess32 as subprocess from diraccfg import CFG import DIRAC @@ -96,11 +100,50 @@ from DIRAC.Core.Base.AgentModule import AgentModule from DIRAC.Core.Base.ExecutorModule import ExecutorModule from DIRAC.Core.DISET.RequestHandler import RequestHandler +from DIRAC.Core.Utilities.Decorators import deprecated from DIRAC.Core.Utilities.PrettyPrint import printTable +from DIRAC.Core.Utilities.Extensions import ( + extensionsByPriority, findDatabases, findModules, findAgents, findServices, + findExecutors, findSystems, +) __RCSID__ = "$Id$" +def _safeFloat(value): + try: + return float(value) + except ValueError: + return -1 + + +def _safeInt(value): + try: + return int(value) + except ValueError: + return -1 + + +def _makeComponentDict(component, setupDict, installedDict, compType, system, runitDict): + componentDict = { + 'Setup': component in setupDict[compType][system], + 'Installed': component in installedDict[compType][system], + 'RunitStatus': 'Unknown', + 'Timeup': 0, + 'PID': 0, + } + compDir = system + '_' + component + if compDir in runitDict: + componentDict['RunitStatus'] = runitDict[compDir]['RunitStatus'] + componentDict['Timeup'] = runitDict[compDir]['Timeup'] + componentDict['PID'] = _safeInt(runitDict[compDir].get('PID', -1)) + componentDict['CPU'] = _safeFloat(runitDict[compDir].get('CPU', -1)) + componentDict['MEM'] = _safeFloat(runitDict[compDir].get('MEM', -1)) + componentDict['RSS'] = _safeFloat(runitDict[compDir].get('RSS', -1)) + componentDict['VSZ'] = _safeFloat(runitDict[compDir].get('VSZ', -1)) + return componentDict + + class ComponentInstaller(object): def __init__(self): @@ -143,10 +186,18 @@ def __init__(self): self.loadDiracCfg() + def resultIndexes(self, componentTypes): + resultIndexes = {} + for cType in componentTypes: + result = self._getSectionName(cType) + if not result['OK']: + return result + resultIndexes[cType] = result['Value'] + return S_OK(resultIndexes) + def loadDiracCfg(self): """ Read again defaults from dirac.cfg """ - from DIRAC.Core.Utilities.Network import getFQDN self.localCfg = CFG() @@ -272,18 +323,13 @@ def getInfo(self): if not result['OK']: return result rDict = result['Value'] - if self.setup: - rDict['Setup'] = self.setup - else: - rDict['Setup'] = 'Unknown' + rDict['Setup'] = self.setup or 'Unknown' return S_OK(rDict) + @deprecated("Use DIRAC.Core.Utilities.Extensions.extensionsByPriority instead") def getExtensions(self): - """ - Get the list of installed extensions - """ - initList = glob.glob(os.path.join(rootPath, '*DIRAC', '__init__.py')) - extensions = [os.path.basename(os.path.dirname(k)) for k in initList] + """Get the list of installed extensions""" + extensions = extensionsByPriority() try: extensions.remove('DIRAC') except Exception: @@ -292,17 +338,13 @@ def getExtensions(self): if self.exitOnError: DIRAC.exit(-1) return S_ERROR(error) - return S_OK(extensions) def _addCfgToDiracCfg(self, cfg): """ Merge cfg into existing dirac.cfg file """ - if str(self.localCfg): - newCfg = self.localCfg.mergeWith(cfg) - else: - newCfg = cfg + newCfg = self.localCfg.mergeWith(cfg) if str(self.localCfg) else cfg result = newCfg.writeToFile(self.cfgFile) if not result: return result @@ -313,7 +355,6 @@ def _addCfgToCS(self, cfg): """ Merge cfg into central CS """ - gLogger.debug("Adding CFG to CS:") gLogger.debug(cfg) @@ -343,10 +384,7 @@ def _addCfgToLocalCS(self, cfg): csFile = os.path.join(rootPath, 'etc', '%s.cfg' % csName) if os.path.exists(csFile): csCfg.loadFromFile(csFile) - if str(csCfg): - newCfg = csCfg.mergeWith(cfg) - else: - newCfg = cfg + newCfg = csCfg.mergeWith(cfg) if str(csCfg) else cfg return newCfg.writeToFile(csFile) def _removeOptionFromCS(self, path): @@ -394,11 +432,7 @@ def _getCentralCfg(self, installCfg): if vo: centralCfg['DIRAC'].addKey('VirtualOrganization', vo, '') # pylint: disable=no-member - for section in ['Systems', - 'Resources', - 'Resources/Sites', - 'Operations', - 'Registry']: + for section in ['Systems', 'Resources', 'Resources/Sites', 'Operations', 'Registry']: if installCfg.isSection(section): centralCfg.createNewSection(section, contents=installCfg[section]) @@ -582,10 +616,7 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): cType = installation['Component']['Type'] # Is the component a rename of another module? - if installation['Instance'] == installation['Component']['DIRACModule']: - isRenamed = False - else: - isRenamed = True + isRenamed = installation['Instance'] != installation['Component']['DIRACModule'] result = self.monitoringClient.getInstallations( {'UnInstallationTime': None}, @@ -617,8 +648,8 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): if not result['OK']: # It is maybe in the FailoverURLs ? result = self._removeOptionFromCS(cfgPath('Systems', system, compInstance, 'FailoverURLs', component)) - if not result['OK']: - return result + if not result['OK']: + return result if removeMain: result = self._removeSectionFromCS(cfgPath('Systems', system, @@ -639,14 +670,10 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): if not result['OK']: # it is maybe in the FailoverURLs ? result = self._removeOptionFromCS( - cfgPath( - 'Systems', - system, - compInstance, - 'FailoverURLs', - installation['Component']['Module'])) - if not result['OK']: - return result + cfgPath('Systems', system, compInstance, 'FailoverURLs', installation['Component']['Module']) + ) + if not result['OK']: + return result return S_OK('Successfully removed entries from CS') return S_OK('Instances of this component still exist. It won\'t be completely removed') @@ -779,22 +806,19 @@ def getComponentCfg(self, componentType, system, component, compInstance, extens return result sectionName = result['Value'] - componentModule = component - if "Module" in specialOptions and specialOptions['Module']: - componentModule = specialOptions['Module'] - + componentModule = specialOptions.get('Module', component) compCfg = CFG() - if addDefaultOptions: - extensionsDIRAC = [x + 'DIRAC' for x in extensions] + extensions - for ext in extensionsDIRAC + ['DIRAC']: - cfgTemplatePath = os.path.join(rootPath, ext, '%sSystem' % system, 'ConfigTemplate.cfg') - if os.path.exists(cfgTemplatePath): - gLogger.notice('Loading configuration template', cfgTemplatePath) - # Look up the component in this template - loadCfg = CFG() - loadCfg.loadFromFile(cfgTemplatePath) - compCfg = loadCfg.mergeWith(compCfg) + for ext in extensions: + cfgTemplateModule = "%s.%sSystem" % (ext, system) + try: + cfgTemplate = importlib_resources.read_text(cfgTemplateModule, "ConfigTemplate.cfg") + except (ImportError, OSError): + continue + gLogger.notice('Loading configuration template from', cfgTemplateModule) + loadCfg = CFG() + loadCfg.loadFromBuffer(cfgTemplate) + compCfg = loadCfg.mergeWith(compCfg) compPath = cfgPath(sectionName, componentModule) if not compCfg.isSection(compPath): @@ -971,19 +995,10 @@ def printOverallStatus(self, rDict): return S_OK() + @deprecated("Use DIRAC.Core.Utilities.Extensions.findSystems instead") def getAvailableSystems(self, extensions): - """ - Get the list of all systems (in all given extensions) locally available - """ - systems = [] - - for extension in extensions: - extensionPath = os.path.join(DIRAC.rootPath, extension, '*System') - for system in [os.path.basename(k).split('System')[0] for k in glob.glob(extensionPath)]: - if system not in systems: - systems.append(system) - - return systems + """Get the list of all systems (in all given extensions) locally available""" + return list(findSystems(extensions)) def getSoftwareComponents(self, extensions): """ @@ -991,90 +1006,47 @@ def getSoftwareComponents(self, extensions): is installed on the system """ # The Gateway does not need a handler - services = {'Framework': ['Gateway']} - agents = {} - executors = {} - remainders = {} - - resultDict = {} + services = defaultdict(list, {"Framework": ["Gateway"]}) + agents = defaultdict(list) + executors = defaultdict(list) + remainders = defaultdict(lambda: defaultdict(list)) - remainingTypes = [cType for cType in self.componentTypes if cType not in ['service', 'agent', 'executor']] - resultIndexes = {} # Components other than services, agents and executors - for cType in remainingTypes: - result = self._getSectionName(cType) - if not result['OK']: - return result - resultIndexes[cType] = result['Value'] - resultDict[resultIndexes[cType]] = {} - remainders[cType] = {} - - for extension in ['DIRAC'] + [x + 'DIRAC' for x in extensions]: - if not os.path.exists(os.path.join(rootPath, extension)): - # Not all the extensions are necessarily installed in this self.instance - continue - systemList = os.listdir(os.path.join(rootPath, extension)) - for sys in systemList: - system = sys.replace('System', '') - try: - agentDir = os.path.join(rootPath, extension, sys, 'Agent') - agentList = os.listdir(agentDir) - for agent in agentList: - if os.path.splitext(agent)[1] == ".py": - agentFile = os.path.join(agentDir, agent) - with io.open(agentFile, 'rt') as afile: - body = afile.read() - if body.find('AgentModule') != -1 or body.find('OptimizerModule') != -1: - if system not in agents: - agents[system] = [] - agents[system].append(agent.replace('.py', '')) - except OSError: - pass - try: - serviceDir = os.path.join(rootPath, extension, sys, 'Service') - serviceList = os.listdir(serviceDir) - for service in serviceList: - if service.find('Handler') != -1 and os.path.splitext(service)[1] == '.py': - if system not in services: - services[system] = [] - if system == 'Configuration' and service == 'ConfigurationHandler.py': - service = 'ServerHandler.py' - services[system].append(service.replace('.py', '').replace('Handler', '')) - except OSError: - pass - try: - executorDir = os.path.join(rootPath, extension, sys, 'Executor') - executorList = os.listdir(executorDir) - for executor in executorList: - if os.path.splitext(executor)[1] == ".py": - executorFile = os.path.join(executorDir, executor) - with io.open(executorFile, 'rt') as afile: - body = afile.read() - if body.find('OptimizerExecutor') != -1: - if system not in executors: - executors[system] = [] - executors[system].append(executor.replace('.py', '')) - except OSError: - pass + remainingTypes = set(self.componentTypes) - {'service', 'agent', 'executor'} + result = self.resultIndexes(remainingTypes) + if not result["OK"]: + return result + resultIndexes = result["Value"] - # Rest of component types - for cType in remainingTypes: - try: - remainDir = os.path.join(rootPath, extension, sys, cType.title()) - remainList = os.listdir(remainDir) - for remainder in remainList: - if os.path.splitext(remainder)[1] == ".py": - if system not in remainders[cType]: - remainders[cType][system] = [] - remainders[cType][system].append(remainder.replace('.py', '')) - except OSError: - pass - - resultDict['Services'] = services - resultDict['Agents'] = agents - resultDict['Executors'] = executors - for cType in remainingTypes: - resultDict[resultIndexes[cType]] = remainders[cType] + for extension in extensions: + for system, agent in findAgents(extension): + loader = pkgutil.get_loader(".".join([extension, system, "Agent", agent])) + with io.open(loader.get_filename(), "rt") as fp: + body = fp.read() + if "AgentModule" in body or "OptimizerModule" in body: + agents[system.replace("System", "")].append(agent) + + for system, service in findServices(extension): + if system == "Configuration" and service == "ConfigurationHandler": + service = "ServerHandler" + services[system.replace("System", "")].append(service.replace("Handler", "")) + + for system, executor in findExecutors(extension): + loader = pkgutil.get_loader(".".join([extension, system, "Executor", executor])) + with io.open(loader.get_filename(), "rt") as fp: + body = fp.read() + if "OptimizerExecutor" in body: + executors[system.replace("System", "")].append(executor) + + # Rest of component types + for cType in remainingTypes: + for system, remainder in findModules(extension, cType.title()): + remainders[cType][system.replace("System", "")].append(remainder) + + resultDict = {resultIndexes[cType]: dict(remainders[cType]) for cType in remainingTypes} + resultDict["Services"] = dict(services) + resultDict["Agents"] = dict(agents) + resultDict["Executors"] = dict(executors) return S_OK(resultDict) def getInstalledComponents(self): @@ -1082,70 +1054,62 @@ def getInstalledComponents(self): Get the list of all the components ( services and agents ) installed on the system in the runit directory """ + result = self.resultIndexes(self.componentTypes) + if not result["OK"]: + return result + resultIndexes = result["Value"] - resultDict = {} - resultIndexes = {} - for cType in self.componentTypes: - result = self._getSectionName(cType) - if not result['OK']: - return result - resultIndexes[cType] = result['Value'] - resultDict[resultIndexes[cType]] = {} - - systemList = os.listdir(self.runitDir) - for system in systemList: + resultDict = defaultdict(lambda: defaultdict(list)) + for system in os.listdir(self.runitDir): systemDir = os.path.join(self.runitDir, system) - components = os.listdir(systemDir) - for component in components: + for component in os.listdir(systemDir): + runFile = os.path.join(systemDir, component, 'run') try: - runFile = os.path.join(systemDir, component, 'run') with io.open(runFile, 'rt') as rFile: body = rFile.read() - - for cType in self.componentTypes: - if body.find('dirac-%s' % (cType)) != -1: - if system not in resultDict[resultIndexes[cType]]: - resultDict[resultIndexes[cType]][system] = [] - resultDict[resultIndexes[cType]][system].append(component) except IOError: pass + else: + for cType in self.componentTypes: + if 'dirac-%s' % (cType) in body: + resultDict[cType][system].append(component) - return S_OK(resultDict) + return S_OK({ + resultIndexes[cType]: dict(resultDict[cType]) + for cType in self.componentTypes + }) def getSetupComponents(self): """ Get the list of all the components ( services and agents ) set up for running with runsvdir in startup directory """ - - resultDict = {} - resultIndexes = {} - for cType in self.componentTypes: - result = self._getSectionName(cType) - if not result['OK']: - return result - resultIndexes[cType] = result['Value'] - resultDict[resultIndexes[cType]] = {} - if not os.path.isdir(self.startDir): return S_ERROR('Startup Directory does not exit: %s' % self.startDir) - componentList = os.listdir(self.startDir) - for component in componentList: + + result = self.resultIndexes(self.componentTypes) + if not result["OK"]: + return result + resultIndexes = result["Value"] + + resultDict = defaultdict(lambda: defaultdict(list)) + for component in os.listdir(self.startDir): + runFile = os.path.join(self.startDir, component, 'run') try: - runFile = os.path.join(self.startDir, component, 'run') with io.open(runFile, 'rt') as rfile: body = rfile.read() - - for cType in self.componentTypes: - if body.find('dirac-%s' % (cType)) != -1: - system, compT = component.split('_', 1) - if system not in resultDict[resultIndexes[cType]]: - resultDict[resultIndexes[cType]][system] = [] - resultDict[resultIndexes[cType]][system].append(compT) except IOError: pass + else: + for cType in self.componentTypes: + if 'dirac-%s' % (cType) in body: + system, compT = component.split('_', 1) + resultDict[cType][system].append(compT) - return S_OK(resultDict) + return S_OK({ + resultIndexes[cType]: dict(resultDict[cType]) + for cType in self.componentTypes + }) def getStartupComponentStatus(self, componentTupleList): """ @@ -1248,7 +1212,6 @@ def getOverallStatus(self, extensions): Get the list of all the components ( services and agents ) set up for running with runsvdir in startup directory """ - result = self.getSoftwareComponents(extensions) if not result['OK']: return result @@ -1270,119 +1233,33 @@ def getOverallStatus(self, extensions): runitDict = result['Value'] # Collect the info now - resultDict = {} - resultIndexes = {} - for cType in self.componentTypes: - result = self._getSectionName(cType) - if not result['OK']: - return result - resultIndexes[cType] = result['Value'] - resultDict[resultIndexes[cType]] = {} + result = self.resultIndexes(self.componentTypes) + if not result["OK"]: + return result + resultIndexes = result["Value"] - for compType in resultIndexes.values(): + resultDict = defaultdict(lambda: defaultdict(list)) + for cType in resultIndexes.values(): if 'Services' in softDict: - for system in softDict[compType]: - resultDict[compType][system] = {} - for component in softDict[compType][system]: - if system == 'Configuration' and component == 'Configuration': + for system in softDict[cType]: + for component in softDict[cType][system]: + if system == component == 'Configuration': # Fix to avoid missing CS due to different between Service name and Handler name component = 'Server' - resultDict[compType][system][component] = {} - resultDict[compType][system][component]['Setup'] = False - resultDict[compType][system][component]['Installed'] = False - resultDict[compType][system][component]['RunitStatus'] = 'Unknown' - resultDict[compType][system][component]['Timeup'] = 0 - resultDict[compType][system][component]['PID'] = 0 - # TODO: why do we need a try here? - try: - if component in setupDict[compType][system]: - resultDict[compType][system][component]['Setup'] = True - except Exception: - pass - try: - if component in installedDict[compType][system]: - resultDict[compType][system][component]['Installed'] = True - except Exception: - pass - try: - compDir = system + '_' + component - if compDir in runitDict: - resultDict[compType][system][component]['RunitStatus'] = runitDict[compDir]['RunitStatus'] - resultDict[compType][system][component]['Timeup'] = runitDict[compDir]['Timeup'] - try: - resultDict[compType][system][component]['PID'] = int(runitDict[compDir]['PID']) - except ValueError: - resultDict[compType][system][component]['PID'] = -1 - try: - resultDict[compType][system][component]['CPU'] = float(runitDict[compDir]['CPU']) - except ValueError: - resultDict[compType][system][component]['CPU'] = -1 - try: - resultDict[compType][system][component]['MEM'] = float(runitDict[compDir]['MEM']) - except ValueError: - resultDict[compType][system][component]['MEM'] = -1 - try: - resultDict[compType][system][component]['RSS'] = float(runitDict[compDir]['RSS']) - except ValueError: - resultDict[compType][system][component]['RSS'] = -1 - try: - resultDict[compType][system][component]['VSZ'] = float(runitDict[compDir]['VSZ']) - except ValueError: - resultDict[compType][system][component]['VSZ'] = -1 - except Exception: - # print str(x) - pass - + resultDict[cType][system][component] = _makeComponentDict( + component, setupDict, installedDict, cType, system, runitDict + ) # Installed components can be not the same as in the software list if 'Services' in installedDict: - for system in installedDict[compType]: - for component in installedDict[compType][system]: - if compType in resultDict: - if system in resultDict[compType]: - if component in resultDict[compType][system]: - continue - resultDict[compType][system][component] = {} - resultDict[compType][system][component]['Setup'] = False - resultDict[compType][system][component]['Installed'] = True - resultDict[compType][system][component]['RunitStatus'] = 'Unknown' - resultDict[compType][system][component]['Timeup'] = 0 - resultDict[compType][system][component]['PID'] = 0 - # TODO: why do we need a try here? - try: - if component in setupDict[compType][system]: - resultDict[compType][system][component]['Setup'] = True - except Exception: - pass - try: - compDir = system + '_' + component - if compDir in runitDict: - resultDict[compType][system][component]['RunitStatus'] = runitDict[compDir]['RunitStatus'] - resultDict[compType][system][component]['Timeup'] = runitDict[compDir]['Timeup'] - try: - resultDict[compType][system][component]['PID'] = int(runitDict[compDir]['PID']) - except ValueError: - resultDict[compType][system][component]['PID'] = -1 - try: - resultDict[compType][system][component]['CPU'] = float(runitDict[compDir]['CPU']) - except ValueError: - resultDict[compType][system][component]['CPU'] = -1 - try: - resultDict[compType][system][component]['MEM'] = float(runitDict[compDir]['MEM']) - except ValueError: - resultDict[compType][system][component]['MEM'] = -1 - try: - resultDict[compType][system][component]['RSS'] = float(runitDict[compDir]['RSS']) - except ValueError: - resultDict[compType][system][component]['RSS'] = -1 - try: - resultDict[compType][system][component]['VSZ'] = float(runitDict[compDir]['VSZ']) - except ValueError: - resultDict[compType][system][component]['VSZ'] = -1 - except Exception: - # print str(x) - pass + for system in installedDict[cType]: + for component in installedDict[cType][system]: + if component in resultDict.get(cType, {}).get(system, {}): + continue + resultDict[cType][system][component] = _makeComponentDict( + component, setupDict, installedDict, cType, system, runitDict + ) - return S_OK(resultDict) + return S_OK({k: dict(v) for k, v in resultDict.items()}) def checkComponentModule(self, componentType, system, module): """ @@ -1471,11 +1348,7 @@ def getLogTail(self, system, component, length=100): else: with io.open(logFileName, 'rt') as logFile: lines = [line.strip() for line in logFile.readlines()] - - if len(lines) < length: - retDict[compName] = '\n'.join(lines) - else: - retDict[compName] = '\n'.join(lines[-length:]) + retDict[compName] = '\n'.join(lines[-length:]) return S_OK(retDict) @@ -1523,12 +1396,8 @@ def setupSite(self, scriptCfg, cfg=None): setupAddConfiguration = self.localCfg.getOption(cfgInstallPath('AddConfiguration'), True) for serviceTuple in setupServices: - error = '' if len(serviceTuple) != 2: error = 'Wrong service specification: system/service' - # elif serviceTuple[0] not in setupSystems: - # error = 'System %s not available' % serviceTuple[0] - if error: if self.exitOnError: gLogger.error(error) DIRAC.exit(-1) @@ -1538,12 +1407,8 @@ def setupSite(self, scriptCfg, cfg=None): setupSystems.append(serviceSysInstance) for agentTuple in setupAgents: - error = '' if len(agentTuple) != 2: error = 'Wrong agent specification: system/agent' - # elif agentTuple[0] not in setupSystems: - # error = 'System %s not available' % agentTuple[0] - if error: if self.exitOnError: gLogger.error(error) DIRAC.exit(-1) @@ -1553,10 +1418,8 @@ def setupSite(self, scriptCfg, cfg=None): setupSystems.append(agentSysInstance) for executorTuple in setupExecutors: - error = '' if len(executorTuple) != 2: error = 'Wrong executor specification: system/executor' - if error: if self.exitOnError: gLogger.error(error) DIRAC.exit(-1) @@ -1566,10 +1429,7 @@ def setupSite(self, scriptCfg, cfg=None): setupSystems.append(executorSysInstance) # And to find out the available extensions - result = self.getExtensions() - if not result['OK']: - return result - extensions = [k.replace('DIRAC', '') for k in result['Value']] + extensions = extensionsByPriority() # Make sure the necessary directories are there if self.basePath != self.instancePath: @@ -1602,9 +1462,7 @@ def setupSite(self, scriptCfg, cfg=None): # it is pointless to look for more detailed command. # Nobody uses runsvdir.... so if it is there, it is us. - cmdFound = any(['runsvdir' in process for process in processList]) - - if not cmdFound: + if all('runsvdir' not in process for process in processList): gLogger.notice('Starting runsvdir ...') with io.open(os.devnull, 'w') as devnull: subprocess.Popen(['nohup', 'runsvdir', self.startDir, 'log: DIRAC runsv'], @@ -1614,6 +1472,7 @@ def setupSite(self, scriptCfg, cfg=None): # This server hosts the Master of the CS from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData gLogger.notice('Installing Master Configuration Server') + cfg = self.__getCfg(cfgPath('DIRAC', 'Setups', self.setup), 'Configuration', self.instance) self._addCfgToDiracCfg(cfg) cfg = self.__getCfg(cfgPath('DIRAC', 'Configuration'), 'Master', 'yes') @@ -1639,19 +1498,15 @@ def setupSite(self, scriptCfg, cfg=None): if not result['OK']: if self.exitOnError: DIRAC.exit(-1) - else: - return result + return result compCfg = result['Value'] cfg = cfg.mergeWith(compCfg) gConfigurationData.mergeWithLocal(cfg) - self.addDefaultOptionsToComponentCfg('service', 'Configuration', 'Server', []) - if installCfg: - centralCfg = self._getCentralCfg(installCfg) - else: - centralCfg = self._getCentralCfg(self.localCfg) + self.addDefaultOptionsToComponentCfg('service', 'Configuration', 'Server', ["DIRAC"]) + centralCfg = self._getCentralCfg(installCfg or self.localCfg) self._addCfgToLocalCS(centralCfg) - self.setupComponent('service', 'Configuration', 'Server', [], checkModule=False) + self.setupComponent('service', 'Configuration', 'Server', ["DIRAC"], checkModule=False) self.runsvctrlComponent('Configuration', 'Server', 't') while ['Configuration', 'Server'] in setupServices: @@ -1729,7 +1584,7 @@ def setupSite(self, scriptCfg, cfg=None): DIRAC.exit(-1) return result installedDatabases = result['Value'] - result = self.getAvailableDatabases(CSGlobals.getCSExtensions()) + result = self.getAvailableDatabases() gLogger.debug("Available databases", result) if not result['OK']: return result @@ -1752,21 +1607,19 @@ def setupSite(self, scriptCfg, cfg=None): if not result['OK']: gLogger.error('Database %s CS registration failed: %s' % (dbName, result['Message'])) - if self.mysqlPassword: - if not self._addMySQLToDiracCfg(): - error = 'Failed to add MySQL user/password to local configuration' - if self.exitOnError: - gLogger.error(error) - DIRAC.exit(-1) - return S_ERROR(error) + if self.mysqlPassword and not self._addMySQLToDiracCfg(): + error = 'Failed to add MySQL user/password to local configuration' + if self.exitOnError: + gLogger.error(error) + DIRAC.exit(-1) + return S_ERROR(error) - if self.noSQLHost: - if not self._addNoSQLToDiracCfg(): - error = 'Failed to add NoSQL connection details to local configuration' - if self.exitOnError: - gLogger.error(error) - DIRAC.exit(-1) - return S_ERROR(error) + if self.noSQLHost and not self._addNoSQLToDiracCfg(): + error = 'Failed to add NoSQL connection details to local configuration' + if self.exitOnError: + gLogger.error(error) + DIRAC.exit(-1) + return S_ERROR(error) # 3.- Then installed requested services for system, service in setupServices: @@ -1854,18 +1707,18 @@ def installComponent(self, componentType, system, component, extensions, compone # Any "Load" or "Module" option in the configuration defining what modules the given "component" # needs to load will be taken care of by self.checkComponentModule. if checkModule: - cModule = componentModule - if not cModule: - cModule = component + cModule = componentModule or component result = self.checkComponentModule(componentType, system, cModule) - if not result['OK']: - if not self.checkComponentSoftware(componentType, system, cModule, extensions)[ - 'OK'] and componentType != 'executor': - error = 'Software for %s %s/%s is not installed' % (componentType, system, component) - if self.exitOnError: - gLogger.error(error) - DIRAC.exit(-1) - return S_ERROR(error) + if ( + not result['OK'] + and not self.checkComponentSoftware(componentType, system, cModule, extensions)['OK'] + and componentType != 'executor' + ): + error = 'Software for %s %s/%s is not installed' % (componentType, system, component) + if self.exitOnError: + gLogger.error(error) + DIRAC.exit(-1) + return S_ERROR(error) gLogger.notice('Installing %s %s/%s' % (componentType, system, component)) @@ -1917,8 +1770,7 @@ def installComponent(self, componentType, system, component, extensions, compone [[ "%(componentType)s" = "agent" ]] && renice 20 -p $$ #%(bashVariables)s # -exec dirac-%(componentType)s \ - %(system)s/%(component)s --cfg %(componentCfg)s < /dev/null +exec dirac-%(componentType)s %(system)s/%(component)s --cfg %(componentCfg)s < /dev/null """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), 'bashVariables': bashVars, 'componentType': componentType.replace("-", "_"), @@ -2019,11 +1871,9 @@ def uninstallComponent(self, system, component, removeLogs): """ Remove startup and runit directories """ - result = self.runsvctrlComponent(system, component, 'd') - if not result['OK']: - pass + self.runsvctrlComponent(system, component, 'd') - result = self.unsetupComponent(system, component) + self.unsetupComponent(system, component) if removeLogs: for runitCompDir in glob.glob(os.path.join(self.runitDir, system, component)): @@ -2074,9 +1924,7 @@ def installPortal(self): """ Install runit directories for the Web Portal """ - # Check that the software for the Web Portal is installed - error = '' webDir = os.path.join(self.linkedRootPath, 'WebAppDIRAC') if not os.path.exists(webDir): error = 'WebApp extension not installed at %s' % webDir @@ -2128,15 +1976,14 @@ def getMySQLPasswords(self): """ Get MySQL passwords from local configuration or prompt """ - import getpass if not self.mysqlRootPwd: self.mysqlRootPwd = getpass.getpass('MySQL root password: ') if not self.mysqlPassword: # Take it if it is already defined self.mysqlPassword = self.localCfg.getOption('/Systems/Databases/Password', '') - if not self.mysqlPassword: - self.mysqlPassword = getpass.getpass('MySQL Dirac password: ') + if not self.mysqlPassword: + self.mysqlPassword = getpass.getpass('MySQL Dirac password: ') return S_OK() @@ -2158,24 +2005,21 @@ def getMySQLStatus(self): result = self.execCommand(0, ['mysqladmin', 'status']) if not result['OK']: return result - output = result['Value'][1] - _d1, uptime, nthreads, nquestions, nslow, nopens, nflash, nopen, nqpersec = output.split(':') - resDict = {} - resDict['UpTime'] = uptime.strip().split()[0] - resDict['NumberOfThreads'] = nthreads.strip().split()[0] - resDict['NumberOfQuestions'] = nquestions.strip().split()[0] - resDict['NumberOfSlowQueries'] = nslow.strip().split()[0] - resDict['NumberOfOpens'] = nopens.strip().split()[0] - resDict['OpenTables'] = nopen.strip().split()[0] - resDict['FlushTables'] = nflash.strip().split()[0] - resDict['QueriesPerSecond'] = nqpersec.strip().split()[0] - return S_OK(resDict) - def getAvailableDatabases(self, extensions=[]): - """ Find all databases defined - """ + keys = [ + None, "UpTime", "NumberOfThreads", "NumberOfQuestions", + "NumberOfSlowQueries", "NumberOfOpens", "FlushTables", "OpenTables", + "QueriesPerSecond" + ] + return S_OK({ + key: output.strip().split()[0] + for key, output in zip(keys, result['Value'][1].split(":")) if key + }) + + def getAvailableDatabases(self, extensions=None): + """Find all databases defined""" if not extensions: - extensions = CSGlobals.getCSExtensions() + extensions = extensionsByPriority() res = self.getAvailableSQLDatabases(extensions) gLogger.debug("Available SQL databases", res) @@ -2202,17 +2046,15 @@ def getAvailableSQLDatabases(self, extensions): :return: dict of MySQL DBs """ dbDict = {} - for extension in extensions + ['']: - databases = glob.glob(os.path.join(rootPath, - ('%sDIRAC' % extension).replace('DIRACDIRAC', 'DIRAC'), - '*', 'DB', '*.sql')) - for dbPath in databases: - dbName = os.path.basename(dbPath).replace('.sql', '') + for extension in extensions: + databases = findDatabases(extensions) + for systemName, dbSql in databases: + dbName = dbSql.replace('.sql', '') dbDict[dbName] = {} dbDict[dbName]['Type'] = 'MySQL' - dbDict[dbName]['Extension'] = extension - dbDict[dbName]['System'] = dbPath.split('/')[-3].replace('System', '') - + # TODO: Does this need to be replaced + dbDict[dbName]['Extension'] = extension.replace("DIRAC", "") + dbDict[dbName]['System'] = systemName.replace('System', '') return S_OK(dbDict) def getAvailableESDatabases(self, extensions): @@ -2234,40 +2076,26 @@ def getAvailableESDatabases(self, extensions): :return: dict of ES DBs """ dbDict = {} - for extension in extensions + ['']: - - # Find *DB.py definitions - pyDBs = glob.glob(os.path.join(rootPath, - ('%sDIRAC' % extension).replace('DIRACDIRAC', 'DIRAC'), - '*', 'DB', '*DB.py')) - pyDBs = [x.replace('.py', '') for x in pyDBs if '__init__' not in x] - - # Find sql files - sqlDBs = glob.glob(os.path.join(rootPath, - ('%sDIRAC' % extension).replace('DIRACDIRAC', 'DIRAC'), - '*', 'DB', '*.sql')) - sqlDBs = [x.replace('.sql', '') for x in sqlDBs] - - # Find *DB.py files that do not have a sql part - possible = set(pyDBs) - set(sqlDBs) - databases = [] - for p in possible: - # Introspect all possible ones + for extension in extensions: + sqlDatabases = findDatabases(extension) + for systemName, dbName in findModules(extension, "DB", "*DB"): + if (systemName, dbName + ".sql") in sqlDatabases: + continue + + # Introspect all possible ones for a ElasticDB attribute try: - p_mod = p.replace(rootPath, '').lstrip('/').replace('/', '.') - mdb_mod = importlib.import_module(p_mod, p_mod.split('.')[-1]) - cl = getattr(mdb_mod, p_mod.split('.')[-1]) - if 'ElasticDB' in str(inspect.getmro(cl)): - databases.append(p) + module = importlib.import_module(".".join([extension, systemName, "DB", dbName])) + dbClass = getattr(module, dbName) except (AttributeError, ImportError): - pass + continue + if 'ElasticDB' not in str(inspect.getmro(dbClass)): + continue - for dbPath in databases: - dbName = os.path.basename(dbPath) dbDict[dbName] = {} dbDict[dbName]['Type'] = 'ES' - dbDict[dbName]['Extension'] = extension - dbDict[dbName]['System'] = dbPath.split('/')[-3].replace('System', '') + # TODO: Does this need to be replaced + dbDict[dbName]['Extension'] = extension.replace("DIRAC", "") + dbDict[dbName]['System'] = systemName return S_OK(dbDict) @@ -2289,7 +2117,6 @@ def installDatabase(self, dbName): """ Install requested DB in MySQL server """ - if not self.mysqlRootPwd: rootPwdPath = cfgInstallPath('Database', 'RootPwd') return S_ERROR('Missing %s in %s' % (rootPwdPath, self.cfgFile)) @@ -2302,27 +2129,22 @@ def installDatabase(self, dbName): gLogger.notice('Installing', dbName) - dbFile = glob.glob(os.path.join(rootPath, 'DIRAC', '*', 'DB', '%s.sql' % dbName)) # is there by chance an extension of it? - for extension in CSGlobals.getCSExtensions(): - dbFileInExtension = glob.glob(os.path.join(rootPath, - '%sDIRAC' % extension, - '*', - 'DB', - '%s.sql' % dbName)) - if dbFileInExtension: - dbFile = dbFileInExtension + for extension in extensionsByPriority(): + databases = {k: v for v, k in findDatabases(extension)} + filename = dbName + ".sql" + if filename in databases: break - - if not dbFile: + else: error = 'Database %s not found' % dbName gLogger.error(error) if self.exitOnError: DIRAC.exit(-1) return S_ERROR(error) - - dbFile = dbFile[0] - gLogger.debug("Installing %s" % dbFile) + systemName = databases[filename] + moduleName = ".".join([extension, systemName, "DB"]) + gLogger.debug("Installing %s from %s" % (filename, moduleName)) + dbSql = importlib_resources.read_text(moduleName, filename) # just check result = self.execMySQL('SHOW STATUS') @@ -2362,7 +2184,7 @@ def installDatabase(self, dbName): # first getting the lines to be executed, and then execute them try: - cmdLines = self._createMySQLCMDLines(dbFile) + cmdLines = self._createMySQLCMDLines(dbSql) # We need to run one SQL cmd at once, mysql is much happier that way. # Create a string of commands, ignoring comment lines @@ -2387,13 +2209,13 @@ def installDatabase(self, dbName): DIRAC.exit(-1) return S_ERROR(error) - return S_OK(dbFile.split('/')[-4:-2]) + return S_OK([extension, systemName]) def uninstallDatabase(self, gConfig_o, dbName): """ Remove a database from DIRAC """ - result = self.getAvailableDatabases(CSGlobals.getCSExtensions()) + result = self.getAvailableDatabases() if not result['OK']: return result @@ -2405,24 +2227,21 @@ def uninstallDatabase(self, gConfig_o, dbName): return S_OK('DB successfully uninstalled') - def _createMySQLCMDLines(self, dbFile): - """ Creates a list of MYSQL commands to be executed, inspecting the dbFile(s) - """ + def _createMySQLCMDLines(self, dbSql): + """Creates a list of MYSQL commands to be executed, inspecting the SQL + :param str dbSql: The SQL to parse + :returns: list of str corresponding to executable SQL statements + """ cmdLines = [] - - with io.open(dbFile, 'rt') as fd: - dbLines = fd.readlines() - - for line in dbLines: + for line in dbSql.split("\n"): # Should we first source an SQL file (is this sql file an extension)? if line.lower().startswith('source'): sourcedDBbFileName = line.split(' ')[1].replace('\n', '') gLogger.info("Found file to source: %s" % sourcedDBbFileName) - sourcedDBbFile = os.path.join(rootPath, sourcedDBbFileName) - with io.open(sourcedDBbFile, 'rt') as fdSourced: - dbLinesSourced = fdSourced.readlines() - for lineSourced in dbLinesSourced: + module, filename = sourcedDBbFileName.rsplit("/", 1) + dbSourced = importlib_resources.read_text(module.replace("/", "."), filename) + for lineSourced in dbSourced.split("\n"): if lineSourced.strip(): cmdLines.append(lineSourced.strip()) @@ -2488,7 +2307,7 @@ def execCommand(self, timeout, cmd): """ Execute command tuple and handle Error cases """ - gLogger.debug("executing command %s with timeout %d" % (cmd, timeout)) + gLogger.debug("Executing command %s with timeout %d" % (cmd, timeout)) result = systemCall(timeout, cmd) if not result['OK']: if timeout and result['Message'].find('Timeout') == 0: @@ -2566,7 +2385,6 @@ def addTornadoOptionsToCS(self, gConfig_o): """ Add the section with the component options to the CS """ - if gConfig_o: gConfig_o.forceRefresh() @@ -2589,7 +2407,6 @@ def setupTornadoService(self, system, component, extensions, """ Install and create link in startup """ - # Create the startup entry now # Force the system and component to be 'Tornado' but preserve the interface and the code # just to allow for easier refactoring maybe later diff --git a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py index 4f73f1fdb9a..2e545e4bb8d 100644 --- a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py +++ b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py @@ -21,7 +21,7 @@ from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities from DIRAC.MonitoringSystem.Client.MonitoringClient import MonitoringClient from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions +from DIRAC.Core.Utilities.Extensions import extensionsByPriority from DIRAC.Core.Utilities import List from DIRAC.Core.Utilities.PromptUser import promptUser from DIRAC.Core.Utilities.PrettyPrint import printTable @@ -713,19 +713,19 @@ def do_install(self, args): # Install Module section if not yet there if module: result = gComponentInstaller.addDefaultOptionsToCS(gConfig, option, system, module, - getCSExtensions(), hostSetup) + extensionsByPriority(), hostSetup) # in case of Error we must stop, this can happen when the module name is wrong... if not result['OK']: self._errMsg(result['Message']) return # Add component section with specific parameters only result = gComponentInstaller.addDefaultOptionsToCS(gConfig, option, system, component, - getCSExtensions(), hostSetup, specialOptions, + extensionsByPriority(), hostSetup, specialOptions, addDefaultOptions=True) else: # Install component section result = gComponentInstaller.addDefaultOptionsToCS(gConfig, option, system, component, - getCSExtensions(), hostSetup, specialOptions) + extensionsByPriority(), hostSetup, specialOptions) if not result['OK']: self._errMsg(result['Message']) diff --git a/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py b/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py index 2ceec745eef..3ec724fe31f 100644 --- a/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py +++ b/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py @@ -30,7 +30,7 @@ class ProxyManagerHandler(RequestHandler): def initializeHandler(cls, serviceInfoDict): useMyProxy = cls.srv_getCSOption("UseMyProxy", False) try: - result = ObjectLoader().loadObject('FrameworkSystem.DB.ProxyDB', 'ProxyDB') + result = ObjectLoader().loadObject('FrameworkSystem.DB.ProxyDB') if not result['OK']: gLogger.error('Failed to load ProxyDB class: %s' % result['Message']) return result diff --git a/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py b/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py index 74857a9fd8e..815b513bf8b 100644 --- a/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py +++ b/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py @@ -32,6 +32,7 @@ from DIRAC import S_OK, S_ERROR, gConfig, rootPath, gLogger from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.Core.Utilities import Os +from DIRAC.Core.Utilities.Extensions import extensionsByPriority from DIRAC.Core.Utilities.File import mkLink from DIRAC.Core.Utilities.Time import dateTime, fromString, hour, day from DIRAC.Core.Utilities.Subprocess import shellCall, systemCall @@ -101,7 +102,7 @@ def export_getSoftwareComponents(self): """ Get the list of all the components ( services and agents ) for which the software is installed on the system """ - return gComponentInstaller.getSoftwareComponents(getCSExtensions()) + return gComponentInstaller.getSoftwareComponents(extensionsByPriority()) types_getInstalledComponents = [] @@ -125,7 +126,7 @@ def export_getOverallStatus(self): """ Get the complete status information for the components in the given list """ - result = gComponentInstaller.getOverallStatus(getCSExtensions()) + result = gComponentInstaller.getOverallStatus(extensionsByPriority()) if not result['OK']: return result statusDict = result['Value'] @@ -152,7 +153,9 @@ def export_getStartupComponentStatus(self, componentTupleList): def export_installComponent(self, componentType, system, component, componentModule=''): """ Install runit directory for the specified component """ - return gComponentInstaller.installComponent(componentType, system, component, getCSExtensions(), componentModule) + return gComponentInstaller.installComponent( + componentType, system, component, extensionsByPriority(), componentModule + ) types_setupComponent = [six.string_types, six.string_types, six.string_types] @@ -160,7 +163,9 @@ def export_setupComponent(self, componentType, system, component, componentModul """ Setup the specified component for running with the runsvdir daemon It implies installComponent """ - result = gComponentInstaller.setupComponent(componentType, system, component, getCSExtensions(), componentModule) + result = gComponentInstaller.setupComponent( + componentType, system, component, extensionsByPriority(), componentModule + ) gConfig.forceRefresh() return result @@ -169,7 +174,7 @@ def export_setupComponent(self, componentType, system, component, componentModul def export_addDefaultOptionsToComponentCfg(self, componentType, system, component): """ Add default component options local component cfg """ - return gComponentInstaller.addDefaultOptionsToComponentCfg(componentType, system, component, getCSExtensions()) + return gComponentInstaller.addDefaultOptionsToComponentCfg(componentType, system, component, extensionsByPriority()) types_unsetupComponent = [six.string_types, six.string_types] @@ -238,7 +243,7 @@ def export_getDatabases(self, mysqlPassword=None): def export_getAvailableDatabases(self): """ Get the list of databases which software is installed in the system """ - return gComponentInstaller.getAvailableDatabases(getCSExtensions()) + return gComponentInstaller.getAvailableDatabases() types_installDatabase = [six.string_types] @@ -271,7 +276,7 @@ def export_addDefaultOptionsToCS(self, componentType, system, component, overwri """ Add default component options to the global CS or to the local options """ return gComponentInstaller.addDefaultOptionsToCS(gConfig, componentType, system, component, - getCSExtensions(), + extensionsByPriority(), overwrite=overwrite) ####################################################################################### @@ -623,24 +628,15 @@ def export_getUsedPorts(self): def export_getComponentDocumentation(self, cType, system, module): if cType == 'service': module = '%sHandler' % module - - result = gComponentInstaller.getExtensions() - extensions = result['Value'] # Look for the component in extensions - for extension in extensions: + for extension in extensionsByPriority(): + moduleName = ([extension, system + "System", cType.capitalize(), module]) try: - importedModule = importlib.import_module('%s.%sSystem.%s.%s' % (extension, system, - cType.capitalize(), module)) + importedModule = importlib.import_module(moduleName) return S_OK(importedModule.__doc__) except Exception: pass - - # If not in an extension, try in base DIRAC - try: - importedModule = importlib.import_module('DIRAC.%sSystem.%s.%s' % (system, cType.capitalize(), module)) - return S_OK(importedModule.__doc__) - except Exception: - return S_ERROR('No documentation was found') + return S_ERROR('No documentation was found') @staticmethod def __storeHostInfo(): diff --git a/src/DIRAC/FrameworkSystem/private/monitoring/RRDManager.py b/src/DIRAC/FrameworkSystem/private/monitoring/RRDManager.py index 80495851858..27dc89a6f59 100755 --- a/src/DIRAC/FrameworkSystem/private/monitoring/RRDManager.py +++ b/src/DIRAC/FrameworkSystem/private/monitoring/RRDManager.py @@ -126,7 +126,7 @@ def create(self, type, rrdFile, bucketLength): # 1m res for 1 month # cmd += " RRA:%s:0.9:1:43200" % cf # 1m red for 1 year - cmd += " RRA:%s:0.999:1:%s" % (cf, 31536000 / bucketLength) + cmd += " RRA:%s:0.999:1:%s" % (cf, int(31536000 / bucketLength)) return self.__exec(cmd, rrdFilePath) def __getLastUpdateTime(self, rrdFile): diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py index 6d9e88d584d..deda9bcfc54 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py @@ -18,8 +18,8 @@ from DIRAC import gConfig, gLogger, S_OK from DIRAC.Core.Base import Script from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.Core.Utilities.Extensions import extensionsByPriority from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities -from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions __RCSID__ = "$Id$" @@ -70,67 +70,62 @@ def main(): if len(args) != 2: Script.showHelp(exitCode=1) - cType = None system = args[0] component = args[1] - compOrMod = module if module else component + compOrMod = module or component - result = gComponentInstaller.getSoftwareComponents(getCSExtensions()) + result = gComponentInstaller.getSoftwareComponents(extensionsByPriority()) if not result['OK']: gLogger.error(result['Message']) DIRACexit(1) - else: - availableComponents = result['Value'] + availableComponents = result['Value'] for compType in availableComponents: if system in availableComponents[compType] and compOrMod in availableComponents[compType][system]: cType = compType[:-1].lower() break - - if not cType: + else: gLogger.error('Component %s/%s is not available for installation' % (system, component)) DIRACexit(1) if module: result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, module, - getCSExtensions(), + extensionsByPriority(), overwrite=overwrite) result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, component, - getCSExtensions(), + extensionsByPriority(), specialOptions=specialOptions, overwrite=overwrite, addDefaultOptions=False) else: result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, component, - getCSExtensions(), + extensionsByPriority(), specialOptions=specialOptions, overwrite=overwrite) if not result['OK']: gLogger.error(result['Message']) DIRACexit(1) - else: - result = gComponentInstaller.installComponent(cType, system, component, getCSExtensions(), module) + result = gComponentInstaller.installComponent(cType, system, component, extensionsByPriority(), module) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) + result = gComponentInstaller.setupComponent(cType, system, component, extensionsByPriority(), module) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + if component == 'ComponentMonitoring': + result = MonitoringUtilities.monitorInstallation('DB', system, 'InstalledComponentsDB') if not result['OK']: gLogger.error(result['Message']) DIRACexit(1) - else: - gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) - result = gComponentInstaller.setupComponent(cType, system, component, getCSExtensions(), module) - if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - if component == 'ComponentMonitoring': - result = MonitoringUtilities.monitorInstallation('DB', system, 'InstalledComponentsDB') - if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - result = MonitoringUtilities.monitorInstallation(cType, system, component, module) - if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) - DIRACexit() + result = MonitoringUtilities.monitorInstallation(cType, system, component, module) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) + DIRACexit() if __name__ == "__main__": diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py b/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py index 30cb0d94458..5ca24aa9110 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py @@ -20,8 +20,8 @@ from DIRAC import gConfig, gLogger, S_OK from DIRAC.Core.Base import Script from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.Core.Utilities.Extensions import extensionsByPriority from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities -from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions __RCSID__ = "$Id$" @@ -81,7 +81,7 @@ def main(): compOrMod = module if module else component result = gComponentInstaller.addDefaultOptionsToCS(gConfig, 'service', system, component, - getCSExtensions(), + extensionsByPriority(), specialOptions=specialOptions, overwrite=overwrite) @@ -100,7 +100,7 @@ def main(): DIRACexit(1) gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) - result = gComponentInstaller.setupTornadoService(system, component, getCSExtensions(), module) + result = gComponentInstaller.setupTornadoService(system, component, extensionsByPriority(), module) if not result['OK']: gLogger.error(result['Message']) DIRACexit(1) diff --git a/src/DIRAC/ResourceStatusSystem/Agent/CacheFeederAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/CacheFeederAgent.py index 3eeea12d150..4140bb08ee2 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/CacheFeederAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/CacheFeederAgent.py @@ -47,15 +47,13 @@ def initialize(self): """ Define the commands to be executed, and instantiate the clients that will be used. """ - res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient', - 'ResourceStatusClient') + res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient') if not res['OK']: self.log.error('Failed to load ResourceStatusClient class: %s' % res['Message']) return res rsClass = res['Value'] - res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient', - 'ResourceManagementClient') + res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient') if not res['OK']: self.log.error('Failed to load ResourceManagementClient class: %s' % res['Message']) return res diff --git a/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py index d62227ba4f0..d6f3d4c803d 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py @@ -76,15 +76,13 @@ def initialize(self): self.elementType = self.am_getOption('elementType', self.elementType) - res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient', - 'ResourceStatusClient') + res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient') if not res['OK']: self.log.error('Failed to load ResourceStatusClient class: %s' % res['Message']) return res rsClass = res['Value'] - res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient', - 'ResourceManagementClient') + res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient') if not res['OK']: self.log.error('Failed to load ResourceManagementClient class: %s' % res['Message']) return res diff --git a/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py index 443c41cfe8a..4e8faaa58be 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py @@ -66,15 +66,13 @@ def initialize(self): maxNumberOfThreads = self.am_getOption('maxNumberOfThreads', self.__maxNumberOfThreads) self.threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads) - res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.SiteStatus', - 'SiteStatus') + res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.SiteStatus') if not res['OK']: self.log.error('Failed to load SiteStatus class: %s' % res['Message']) return res siteStatusClass = res['Value'] - res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient', - 'ResourceManagementClient') + res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient') if not res['OK']: self.log.error('Failed to load ResourceManagementClient class: %s' % res['Message']) return res diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py index e2b559bf370..8e6100585b0 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py @@ -46,22 +46,19 @@ def __init__(self, clients=dict()): # Creating the client in the PEP is a convenience for the PDP, that uses internally the RSS clients - res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient', - 'ResourceStatusClient') + res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient') if not res['OK']: self.log.error('Failed to load ResourceStatusClient class: %s' % res['Message']) raise ImportError(res['Message']) rsClass = res['Value'] - res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient', - 'ResourceManagementClient') + res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient') if not res['OK']: self.log.error('Failed to load ResourceManagementClient class: %s' % res['Message']) raise ImportError(res['Message']) rmClass = res['Value'] - res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.SiteStatus', - 'SiteStatus') + res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.SiteStatus') if not res['OK']: self.log.error('Failed to load SiteStatus class: %s' % res['Message']) raise ImportError(res['Message']) diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py index de13f39f9ec..8dc8d1664b8 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py @@ -178,10 +178,10 @@ def unpack(switchDict): statusTypes = [] if switchDict['name'] is not None: - names = filter(None, switchDict['name'].split(',')) + names = list(filter(None, switchDict['name'].split(','))) if switchDict['statusType'] is not None: - statusTypes = filter(None, switchDict['statusType'].split(',')) + statusTypes = list(filter(None, switchDict['statusType'].split(','))) statusTypes = checkStatusTypes(statusTypes) if names and statusTypes: diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py index 45125e5205d..29aa166c090 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py @@ -121,10 +121,10 @@ def unpack(switchDict): statusTypes = [] if switchDict['name'] is not None: - names = filter(None, switchDict['name'].split(',')) + names = list(filter(None, switchDict['name'].split(','))) if switchDict['statusType'] is not None: - statusTypes = filter(None, switchDict['statusType'].split(',')) + statusTypes = list(filter(None, switchDict['statusType'].split(','))) statusTypes = checkStatusTypes(statusTypes) if len(names) > 0 and len(statusTypes) > 0: diff --git a/src/DIRAC/Resources/Catalog/FCConditionParser.py b/src/DIRAC/Resources/Catalog/FCConditionParser.py index 92adf86d964..57e915256c5 100644 --- a/src/DIRAC/Resources/Catalog/FCConditionParser.py +++ b/src/DIRAC/Resources/Catalog/FCConditionParser.py @@ -171,7 +171,7 @@ def __init__(self, tokens): # Load the plugin, and give it the condition objLoader = ObjectLoader() - _class = objLoader.loadObject('Resources.Catalog.ConditionPlugins.%s' % self.pluginName, self.pluginName) + _class = objLoader.loadObject('Resources.Catalog.ConditionPlugins.%s' % self.pluginName) if not _class['OK']: raise Exception(_class['Message']) diff --git a/src/DIRAC/Resources/Catalog/FileCatalogFactory.py b/src/DIRAC/Resources/Catalog/FileCatalogFactory.py index 8bc3ddfe8e4..988f17a3b9e 100644 --- a/src/DIRAC/Resources/Catalog/FileCatalogFactory.py +++ b/src/DIRAC/Resources/Catalog/FileCatalogFactory.py @@ -8,7 +8,7 @@ from DIRAC import gLogger, gConfig, S_OK, S_ERROR from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCatalogPath from DIRAC.Resources.Catalog.FileCatalogProxyClient import FileCatalogProxyClient -from DIRAC.Core.Utilities import ObjectLoader +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader __RCSID__ = "$Id$" @@ -45,12 +45,9 @@ def createCatalog(self, catalogName, useProxy=False): return self.__createCatalog(catalogName, catalogType, catalogURL, optionsDict) def __getCatalogClass(self, catalogType): - - objectLoader = ObjectLoader.ObjectLoader() - result = objectLoader.loadObject('Resources.Catalog.%sClient' % catalogType, catalogType + 'Client') + result = ObjectLoader().loadObject('Resources.Catalog.%sClient' % catalogType) if not result['OK']: gLogger.error('Failed to load catalog object', '%s' % result['Message']) - return result def __createCatalog(self, catalogName, catalogType, catalogURL, optionsDict): diff --git a/src/DIRAC/Resources/Computing/ComputingElement.py b/src/DIRAC/Resources/Computing/ComputingElement.py index bab96d81995..a08c56ceafa 100755 --- a/src/DIRAC/Resources/Computing/ComputingElement.py +++ b/src/DIRAC/Resources/Computing/ComputingElement.py @@ -203,8 +203,7 @@ def loadBatchSystem(self): """ if self.batchSystem is None: self.batchSystem = self.ceParameters['BatchSystem'] - objectLoader = ObjectLoader() - result = objectLoader.loadObject('Resources.Computing.BatchSystems.%s' % self.batchSystem, self.batchSystem) + result = ObjectLoader().loadObject('Resources.Computing.BatchSystems.%s' % self.batchSystem) if not result['OK']: self.log.error('Failed to load batch object: %s' % result['Message']) return result diff --git a/src/DIRAC/Resources/Computing/ComputingElementFactory.py b/src/DIRAC/Resources/Computing/ComputingElementFactory.py index f42782fc745..d696e24c4d2 100755 --- a/src/DIRAC/Resources/Computing/ComputingElementFactory.py +++ b/src/DIRAC/Resources/Computing/ComputingElementFactory.py @@ -11,7 +11,7 @@ from __future__ import print_function from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.Resources.Computing.ComputingElement import getCEConfigDict -from DIRAC.Core.Utilities import ObjectLoader +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader __RCSID__ = "$Id$" @@ -46,8 +46,7 @@ def getCE(self, ceType='', ceName='', ceParametersDict={}): return S_ERROR(error) subClassName = "%sComputingElement" % (ceTypeLocal) - objectLoader = ObjectLoader.ObjectLoader() - result = objectLoader.loadObject('Resources.Computing.%s' % subClassName, subClassName) + result = ObjectLoader().loadObject('Resources.Computing.%s' % subClassName) if not result['OK']: self.log.error('Failed to load object', '%s: %s' % (subClassName, result['Message'])) return result diff --git a/src/DIRAC/Resources/Computing/SSHComputingElement.py b/src/DIRAC/Resources/Computing/SSHComputingElement.py index 9d5cd116e94..bd4f4177f1d 100644 --- a/src/DIRAC/Resources/Computing/SSHComputingElement.py +++ b/src/DIRAC/Resources/Computing/SSHComputingElement.py @@ -21,6 +21,7 @@ from six.moves.urllib.parse import quote as urlquote from six.moves.urllib.parse import unquote as urlunquote +import DIRAC from DIRAC import S_OK, S_ERROR from DIRAC import rootPath from DIRAC import gLogger @@ -424,7 +425,7 @@ def _generateControlScript(self): :return: a path containing the script generated """ # Get the batch system module to use - batchSystemDir = os.path.join(rootPath, "DIRAC", "Resources", "Computing", "BatchSystems") + batchSystemDir = os.path.join(os.path.dirname(DIRAC.__file__), "Resources", "Computing", "BatchSystems") batchSystemScript = os.path.join(batchSystemDir, '%s.py' % self.batchSystem) # Get the executeBatch.py content: an str variable composed of code content that has to be extracted diff --git a/src/DIRAC/Resources/Computing/SingularityComputingElement.py b/src/DIRAC/Resources/Computing/SingularityComputingElement.py index ab57a629991..ccbd5bb2bbf 100644 --- a/src/DIRAC/Resources/Computing/SingularityComputingElement.py +++ b/src/DIRAC/Resources/Computing/SingularityComputingElement.py @@ -36,7 +36,7 @@ __RCSID__ = "$Id$" -DIRAC_INSTALL = os.path.join(DIRAC.rootPath, 'DIRAC', 'Core', 'scripts', 'dirac-install.py') +DIRAC_INSTALL = os.path.join(os.path.dirname(DIRAC.__file__), 'Core', 'scripts', 'dirac-install.py') # Default container to use if it isn't specified in the CE options CONTAINER_DEFROOT = "/cvmfs/cernvm-prod.cern.ch/cvm3" CONTAINER_WORKDIR = "DIRAC_containers" diff --git a/src/DIRAC/Resources/Computing/test/Test_SSHComputingElement.py b/src/DIRAC/Resources/Computing/test/Test_SSHComputingElement.py index 97849e498c9..a51a559e9fa 100644 --- a/src/DIRAC/Resources/Computing/test/Test_SSHComputingElement.py +++ b/src/DIRAC/Resources/Computing/test/Test_SSHComputingElement.py @@ -12,7 +12,7 @@ import shlex import pytest -from DIRAC import rootPath +import DIRAC from DIRAC.Resources.Computing.SSHComputingElement import SSHComputingElement from DIRAC.Resources.Computing.BatchSystems.executeBatch import executeBatchContent @@ -53,7 +53,7 @@ def test_generateControlScript(batchSystem): with open(dest, 'r') as dst: dataDest = dst.read() - batchSystemDir = os.path.join(rootPath, "DIRAC", "Resources", "Computing", "BatchSystems") + batchSystemDir = os.path.join(os.path.dirname(DIRAC.__file__), "Resources", "Computing", "BatchSystems") batchSystemScript = os.path.join(batchSystemDir, '%s.py' % batchSystem) with open(batchSystemScript, 'r') as bsc: dataBatchSystemScript = bsc.read() diff --git a/src/DIRAC/Resources/IdProvider/IdProviderFactory.py b/src/DIRAC/Resources/IdProvider/IdProviderFactory.py index 38bdb15612f..1134654f218 100644 --- a/src/DIRAC/Resources/IdProvider/IdProviderFactory.py +++ b/src/DIRAC/Resources/IdProvider/IdProviderFactory.py @@ -11,7 +11,7 @@ from __future__ import print_function from DIRAC import S_OK, S_ERROR, gLogger -from DIRAC.Core.Utilities import ObjectLoader +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getInfoAboutProviders __RCSID__ = "$Id$" @@ -43,8 +43,7 @@ def getIdProvider(self, idProvider): self.log.verbose('Creating IdProvider', 'of %s type with the name %s' % (pType, idProvider)) subClassName = "%sIdProvider" % (pType) - objectLoader = ObjectLoader.ObjectLoader() - result = objectLoader.loadObject('Resources.IdProvider.%s' % subClassName, subClassName) + result = ObjectLoader().loadObject('Resources.IdProvider.%s' % subClassName) if not result['OK']: self.log.error('Failed to load object', '%s: %s' % (subClassName, result['Message'])) return result diff --git a/src/DIRAC/Resources/MessageQueue/MQConnector.py b/src/DIRAC/Resources/MessageQueue/MQConnector.py index a802078882f..f3bd0da69d9 100644 --- a/src/DIRAC/Resources/MessageQueue/MQConnector.py +++ b/src/DIRAC/Resources/MessageQueue/MQConnector.py @@ -6,7 +6,7 @@ from __future__ import print_function from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Core.Utilities import ObjectLoader +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Utilities.DErrno import EMQUKN __RCSID__ = "$Id$" @@ -50,8 +50,7 @@ def getMQConnectorClass(mqType): S_OK or S_ERROR: with loaded specialized class of MQConnector. """ subClassName = mqType + 'MQConnector' - objectLoader = ObjectLoader.ObjectLoader() - result = objectLoader.loadObject('Resources.MessageQueue.%s' % subClassName, subClassName) + result = ObjectLoader().loadObject('Resources.MessageQueue.%s' % subClassName) if not result['OK']: gLogger.error('Failed to load object', '%s: %s' % (subClassName, result['Message'])) return result diff --git a/src/DIRAC/Resources/ProxyProvider/ProxyProviderFactory.py b/src/DIRAC/Resources/ProxyProvider/ProxyProviderFactory.py index 15276455bf4..7f831ea1c32 100644 --- a/src/DIRAC/Resources/ProxyProvider/ProxyProviderFactory.py +++ b/src/DIRAC/Resources/ProxyProvider/ProxyProviderFactory.py @@ -10,7 +10,7 @@ from __future__ import division from __future__ import print_function from DIRAC import S_OK, S_ERROR, gLogger -from DIRAC.Core.Utilities import ObjectLoader +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getInfoAboutProviders __RCSID__ = "$Id$" @@ -46,8 +46,7 @@ def getProxyProvider(self, proxyProvider): self.log.verbose('Creating ProxyProvider of %s type with the name %s' % (ppType, proxyProvider)) subClassName = "%sProxyProvider" % (ppType) - objectLoader = ObjectLoader.ObjectLoader() - result = objectLoader.loadObject('Resources.ProxyProvider.%s' % subClassName, subClassName) + result = ObjectLoader().loadObject('Resources.ProxyProvider.%s' % subClassName) if not result['OK']: self.log.error('Failed to load object', '%s: %s' % (subClassName, result['Message'])) return result diff --git a/src/DIRAC/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py b/src/DIRAC/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py index 466b18415e6..b5ff1c24ebf 100644 --- a/src/DIRAC/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py +++ b/src/DIRAC/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py @@ -20,12 +20,13 @@ import pytest -from DIRAC import gLogger, rootPath +import DIRAC +from DIRAC import gLogger from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Resources.ProxyProvider.DIRACCAProxyProvider import DIRACCAProxyProvider -certsPath = os.path.join(rootPath, 'DIRAC/Core/Security/test/certs') +certsPath = os.path.join(os.path.dirname(DIRAC.__file__), 'Core/Security/test/certs') testCAPath = os.path.join(tempfile.mkdtemp(dir='/tmp'), 'ca') testCAConfigFile = os.path.join(testCAPath, 'openssl_config_ca.cnf') diff --git a/src/DIRAC/Resources/ProxyProvider/test/Test_ProxyProviderFactory.py b/src/DIRAC/Resources/ProxyProvider/test/Test_ProxyProviderFactory.py index 453db2fcd1d..8166df1c0e2 100644 --- a/src/DIRAC/Resources/ProxyProvider/test/Test_ProxyProviderFactory.py +++ b/src/DIRAC/Resources/ProxyProvider/test/Test_ProxyProviderFactory.py @@ -7,11 +7,12 @@ import mock import unittest -from DIRAC import S_OK, S_ERROR, gLogger, rootPath +import DIRAC +from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.Resources.ProxyProvider.ProxyProviderFactory import ProxyProviderFactory -certsPath = os.path.join(rootPath, 'DIRAC/Core/Security/test/certs') +certsPath = os.path.join(os.path.dirname(DIRAC.__file__), 'Core/Security/test/certs') def sf_getInfoAboutProviders(of, providerName, option, section): diff --git a/src/DIRAC/Resources/Storage/StorageElement.py b/src/DIRAC/Resources/Storage/StorageElement.py index c1ee256e70d..f454ffe746d 100755 --- a/src/DIRAC/Resources/Storage/StorageElement.py +++ b/src/DIRAC/Resources/Storage/StorageElement.py @@ -419,7 +419,7 @@ def getOccupancy(self, unit='MB', **kwargs): # Call occupancy plugin if requested occupancyPlugin = self.options.get('OccupancyPlugin') if occupancyPlugin: - res = ObjectLoader().loadObject('Resources.Storage.OccupancyPlugins.%s' % occupancyPlugin, occupancyPlugin) + res = ObjectLoader().loadObject('Resources.Storage.OccupancyPlugins.%s' % occupancyPlugin) if not res['OK']: return S_ERROR(errno.EPROTONOSUPPORT, 'Failed to load occupancy plugin %s' % occupancyPlugin) log.verbose('Use occupancy plugin %s' % occupancyPlugin) diff --git a/src/DIRAC/Resources/Storage/StorageFactory.py b/src/DIRAC/Resources/Storage/StorageFactory.py index 8bae9cb467b..2d524b59124 100755 --- a/src/DIRAC/Resources/Storage/StorageFactory.py +++ b/src/DIRAC/Resources/Storage/StorageFactory.py @@ -402,7 +402,7 @@ def __generateStorageObject(self, storageName, pluginName, parameters, hideExcep storageType = 'Proxy' objectLoader = ObjectLoader() - result = objectLoader.loadObject('Resources.Storage.%sStorage' % storageType, storageType + 'Storage', + result = objectLoader.loadObject('Resources.Storage.%sStorage' % storageType, hideExceptions=hideExceptions) if not result['OK']: gLogger.error('Failed to load storage object: %s' % result['Message']) diff --git a/src/DIRAC/Workflow/Utilities/Utils.py b/src/DIRAC/Workflow/Utilities/Utils.py index 72ceb1127ea..23f016dc397 100644 --- a/src/DIRAC/Workflow/Utilities/Utils.py +++ b/src/DIRAC/Workflow/Utilities/Utils.py @@ -4,56 +4,45 @@ from __future__ import division from __future__ import print_function +import importlib import os import time -from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO, getCSExtensions - from DIRAC.Core.Workflow.Module import ModuleDefinition +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Workflow.Parameter import Parameter from DIRAC.Core.Workflow.Step import StepDefinition ############################################################################# -def getStepDefinition(stepName, modulesNameList=None, importLine="""""", parametersList=None): +def getStepDefinition(stepName, modulesNameList=None, importLine="", parametersList=None): """ Given a name, a list of modules name, and a list of parameters, returns a step definition. Remember that Step definition = Parameters + Module Instances """ - if modulesNameList is None: modulesNameList = [] if parametersList is None: parametersList = [] - # In case the importLine is not set, this is looking for a DIRAC extension, if any. - # The extension is supposed to be called ExtDIRAC. - if not importLine: - importLine = "DIRAC.Workflow.Modules" - for ext in getCSExtensions(): - if ext.lower() == getVO(): - importLine = ext + "DIRAC.Workflow.Modules" - break - stepDef = StepDefinition(stepName) for moduleName in modulesNameList: + module = None + if importLine: + try: + module = importlib.import_module(importLine + "." + moduleName) + except ImportError: + pass + # In case the importLine is not set, this is looking for a DIRAC extension, if any + if module is None: + module = ObjectLoader().loadModule("Workflow.Modules." + moduleName)["Value"] # create the module definition moduleDef = ModuleDefinition(moduleName) - try: - # Look in the importLine given, or the DIRAC if the given location can't be imported - moduleDef.setDescription(getattr(__import__("%s.%s" % (importLine, moduleName), - globals(), locals(), ['__doc__']), - "__doc__")) - moduleDef.setBody("""\nfrom %s.%s import %s\n""" % (importLine, moduleName, moduleName)) - except ImportError: - alternativeImportLine = "DIRAC.Workflow.Modules" - moduleDef.setDescription(getattr(__import__("%s.%s" % (alternativeImportLine, moduleName), - globals(), locals(), ['__doc__']), - "__doc__")) - moduleDef.setBody("""\nfrom %s.%s import %s\n""" % (alternativeImportLine, moduleName, moduleName)) + moduleDef.setDescription(module.__doc__) + moduleDef.setBody("\nfrom %s import %s\n" % (module.__name__, moduleName)) # add the module to the step, and instance it stepDef.addModule(moduleDef) @@ -88,11 +77,9 @@ def getStepCPUTimes(step_commons): cputime = 0 if 'StartStats' in step_commons: - # 5-tuple: utime, stime, cutime, cstime, elapsed_time - stats = os.times() - cputimeNow = stats[0] + stats[1] + stats[2] + stats[3] - cputimeBefore = step_commons['StartStats'][0] + step_commons['StartStats'][1] \ - + step_commons['StartStats'][2] + step_commons['StartStats'][3] + # os.times() returns a 5-tuple (utime, stime, cutime, cstime, elapsed_time) + cputimeNow = sum(os.times()[:4]) + cputimeBefore = sum(step_commons['StartStats'][:4]) cputime = cputimeNow - cputimeBefore return exectime, cputime diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py index 6f40099dcbe..9e332c66156 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py @@ -53,7 +53,7 @@ from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus # dirac install file -DIRAC_INSTALL = os.path.join(DIRAC.rootPath, 'DIRAC', 'Core', 'scripts', 'dirac-install.py') +DIRAC_INSTALL = os.path.join(os.path.dirname(DIRAC.__file__), 'Core', 'scripts', 'dirac-install.py') # status TRANSIENT_PILOT_STATUS = ['Submitted', 'Waiting', 'Running', 'Scheduled', 'Ready', 'Unknown'] diff --git a/src/DIRAC/WorkloadManagementSystem/private/SharesCorrector.py b/src/DIRAC/WorkloadManagementSystem/private/SharesCorrector.py index 7ca32a17e01..dbf84643fbd 100644 --- a/src/DIRAC/WorkloadManagementSystem/private/SharesCorrector.py +++ b/src/DIRAC/WorkloadManagementSystem/private/SharesCorrector.py @@ -7,7 +7,7 @@ __RCSID__ = "$Id$" from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Core.Utilities import ObjectLoader +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.WorkloadManagementSystem.private.correctors.BaseCorrector import BaseCorrector @@ -22,7 +22,6 @@ def __init__(self, opsHelper): self.__shareCorrectors = {} self.__correctorsOrder = [] self.__baseCS = "JobScheduling/ShareCorrections" - self.__objLoader = ObjectLoader.ObjectLoader() def __getCSValue(self, path, defaultValue=''): return self.__opsHelper.getValue("%s/%s" % (self.__baseCS, path), defaultValue) @@ -30,7 +29,7 @@ def __getCSValue(self, path, defaultValue=''): def __getCorrectorClass(self, correctorName): baseImport = "WorkloadManagementSystem.private.correctors" fullCN = "%s.%sCorrector" % (baseImport, correctorName) - result = self.__objLoader.getObjects(baseImport, ".*Corrector", parentClass=BaseCorrector) + result = ObjectLoader().getObjects(baseImport, ".*Corrector", parentClass=BaseCorrector) if not result['OK']: return result data = result['Value'] @@ -45,7 +44,7 @@ def instantiateRequiredCorrectors(self): for corrector in self.__shareCorrectors: if corrector not in correctorsToStart: self.__log.info("Stopping corrector %s" % corrector) - del(self.__shareCorrectors[corrector]) + del self.__shareCorrectors[corrector] for corrector in correctorsToStart: if corrector not in self.__shareCorrectors: self.__log.info("Starting corrector %s" % corrector) diff --git a/src/DIRAC/__init__.py b/src/DIRAC/__init__.py index 3d73fd8c876..9ebca2f5097 100755 --- a/src/DIRAC/__init__.py +++ b/src/DIRAC/__init__.py @@ -122,9 +122,11 @@ alarmMail = "dirac.alarms@gmail.com" # Set rootPath of DIRAC installation - -pythonPath = os.path.realpath(__path__[0]) -rootPath = os.path.dirname(pythonPath) +if six.PY3: + rootPath = sys.base_prefix # pylint: disable=no-member +else: + pythonPath = os.path.realpath(__path__[0]) + rootPath = os.path.dirname(pythonPath) # Import DIRAC.Core.Utils modules @@ -152,32 +154,6 @@ __siteName = False -# # Update DErrno with the extensions errors -# from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader -# from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals -# allExtensions = CSGlobals.getCSExtensions() -# -# # Update for each extension. Careful to conflict :-) -# for extension in allExtensions: -# ol = ObjectLoader( baseModules = ["%sDIRAC" % extension] ) -# extraErrorModule = ol.loadModule( 'Core.Utilities.DErrno' ) -# if extraErrorModule['OK']: -# extraErrorModule = extraErrorModule['Value'] -# -# # The next 3 dictionary MUST be present for consistency -# -# # Global name of errors -# DErrno.__dict__.update( extraErrorModule.extra_dErrName ) -# # Dictionary with the error codes -# DErrno.dErrorCode.update( extraErrorModule.extra_dErrorCode ) -# # Error description string -# DErrno.dStrError.update( extraErrorModule.extra_dStrError ) -# -# # extra_compatErrorString is optional -# for err in getattr( extraErrorModule, 'extra_compatErrorString', [] ) : -# DErrno.compatErrorString.setdefault( err, [] ).extend( extraErrorModule.extra_compatErrorString[err] ) - - def siteName(): """ Determine and return DIRAC name for current site diff --git a/tests/Integration/AccountingSystem/plots/qualityplot1.png.py3k b/tests/Integration/AccountingSystem/plots/qualityplot1.png.py3k index c50125c7c3e..b7487b57bee 100644 Binary files a/tests/Integration/AccountingSystem/plots/qualityplot1.png.py3k and b/tests/Integration/AccountingSystem/plots/qualityplot1.png.py3k differ diff --git a/tests/Integration/AccountingSystem/plots/qualityplot2.png.py3k b/tests/Integration/AccountingSystem/plots/qualityplot2.png.py3k index c3ba03fc141..d5578f293d0 100644 Binary files a/tests/Integration/AccountingSystem/plots/qualityplot2.png.py3k and b/tests/Integration/AccountingSystem/plots/qualityplot2.png.py3k differ diff --git a/tests/Integration/DataManagementSystem/Test_Client_DFC.py b/tests/Integration/DataManagementSystem/Test_Client_DFC.py index 60d083988db..c90151d58f8 100644 --- a/tests/Integration/DataManagementSystem/Test_Client_DFC.py +++ b/tests/Integration/DataManagementSystem/Test_Client_DFC.py @@ -837,15 +837,10 @@ def test_directoryOperations(self): (parentDir, result)) self.assertEqual( - result2['Value'].get( - 'Successful', - {}).get( - parentDir, - {}).get('Owner'), + result2['Value'].get('Successful', {}).get(parentDir, {}).get('Owner'), proxyUser, - "parentDir should not have changed Owner from %s ==> %s)" % - (proxyUser, - result2)) + "parentDir should not have changed Owner from %s ==> %s)" % (proxyUser, result2) + ) self.assertEqual( result2['Value'].get( 'Successful', diff --git a/tests/Integration/Framework/Test_ProxyDB.py b/tests/Integration/Framework/Test_ProxyDB.py index db93264223a..8ed10903a4c 100644 --- a/tests/Integration/Framework/Test_ProxyDB.py +++ b/tests/Integration/Framework/Test_ProxyDB.py @@ -25,12 +25,13 @@ from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() +import DIRAC from DIRAC import gLogger, gConfig, S_OK, S_ERROR from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.FrameworkSystem.DB.ProxyDB import ProxyDB from DIRAC.Resources.ProxyProvider.DIRACCAProxyProvider import DIRACCAProxyProvider -certsPath = os.path.join(os.environ['DIRAC'], 'DIRAC/Core/Security/test/certs') +certsPath = os.path.join(os.path.dirname(DIRAC.__file__), 'Core/Security/test/certs') ca = DIRACCAProxyProvider() ca.setParameters({'CertFile': os.path.join(certsPath, 'ca/ca.cert.pem'), 'KeyFile': os.path.join(certsPath, 'ca/ca.key.pem')}) diff --git a/tests/Integration/Resources/ProxyProvider/Test_DIRACCAProxyProvider.py b/tests/Integration/Resources/ProxyProvider/Test_DIRACCAProxyProvider.py index 9245972ce55..6fcfdb63956 100644 --- a/tests/Integration/Resources/ProxyProvider/Test_DIRACCAProxyProvider.py +++ b/tests/Integration/Resources/ProxyProvider/Test_DIRACCAProxyProvider.py @@ -12,11 +12,12 @@ from diraccfg import CFG +import DIRAC from DIRAC import gConfig from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Resources.ProxyProvider.ProxyProviderFactory import ProxyProviderFactory -certsPath = os.path.join(os.environ['DIRAC'], 'DIRAC/Core/Security/test/certs') +certsPath = os.path.join(os.path.dirname(DIRAC.__file__), 'Core/Security/test/certs') diracTestCACFG = """ Resources diff --git a/tests/Integration/all_integration_client_tests.sh b/tests/Integration/all_integration_client_tests.sh index 368e26f4e3a..b443bff0e56 100644 --- a/tests/Integration/all_integration_client_tests.sh +++ b/tests/Integration/all_integration_client_tests.sh @@ -2,8 +2,7 @@ #------------------------------------------------------------------------------- # A convenient way to run all the integration tests for client -> server interaction # -# It supposes that DIRAC client is installed in "${CLIENTINSTALLDIR}" -# and that there's a DIRAC server running with all the services running. +# It supposes that there's a DIRAC server running with all the services running. #------------------------------------------------------------------------------- echo -e '****************************************' diff --git a/tests/Jenkins/dirac-cfg-add-option.py b/tests/Jenkins/dirac-cfg-add-option.py index 2af12c094e7..8119ff9368c 100644 --- a/tests/Jenkins/dirac-cfg-add-option.py +++ b/tests/Jenkins/dirac-cfg-add-option.py @@ -34,14 +34,14 @@ from DIRAC import gConfig from DIRAC import exit as DIRACexit -from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions +from DIRAC.Core.Utilities.Extensions import extensionsByPriority from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller # gComponentInstaller.exitOnError = True result = gComponentInstaller.addDefaultOptionsToCS(gConfig, componentType, system, component, - getCSExtensions(), + extensionsByPriority(), specialOptions={}, overwrite=False) if not result['OK']: diff --git a/tests/Jenkins/dirac_ci.sh b/tests/Jenkins/dirac_ci.sh index 11356615fb4..8beec165e0d 100644 --- a/tests/Jenkins/dirac_ci.sh +++ b/tests/Jenkins/dirac_ci.sh @@ -118,37 +118,60 @@ installSite() { echo "==> Started installing" - if [[ -n "${DEBUG+x}" ]]; then - INSTALLOPTIONS+=("${DEBUG}") - fi - - if [[ "${DIRACOSVER}" ]]; then - INSTALLOPTIONS+=("--dirac-os") - INSTALLOPTIONS+=("--dirac-os-version=${DIRACOSVER}") - fi - - if [[ "$DIRACOS_TARBALL_PATH" ]]; then - { - echo "DIRACOS = $DIRACOS_TARBALL_PATH" - } >> "${SERVERINSTALLDIR}/dirac-ci-install.cfg" - fi - - if [[ -n "${ALTERNATIVE_MODULES+x}" ]]; then - echo "Installing from non-release code" - option="--module=" + if [[ "${USE_PYTHON3:-}" == "Yes" ]]; then + if [[ -n "${DIRACOSVER:-}" ]] && [[ "${DIRACOSVER}" != "master" ]]; then + DIRACOS2_URL="https://github.com/DIRACGrid/DIRACOS2/releases/download/${DIRACOSVER}/DIRACOS-Linux-x86_64.sh" + else + DIRACOS2_URL="https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-x86_64.sh" + fi + cd "$SERVERINSTALLDIR" + curl -L "${DIRACOS2_URL}" > "installer.sh" + bash "installer.sh" + rm "installer.sh" + echo "source \"$PWD/diracos/diracosrc\"" > "$PWD/bashrc" + # TODO: This will be fixed properly as part of https://github.com/DIRACGrid/DIRAC/issues/5082 + mv "${SERVERINSTALLDIR}/etc/grid-security/"* "${SERVERINSTALLDIR}/diracos/etc/grid-security/" + rm -rf "${SERVERINSTALLDIR}/etc" + ln -s "${SERVERINSTALLDIR}/diracos/etc" "${SERVERINSTALLDIR}/etc" + source diracos/diracosrc + pip install git+https://gitlab.cern.ch/chaen/fts-rest-flask.git@packaging for module_path in "${ALTERNATIVE_MODULES[@]}"; do - if [[ -d "${module_path}" ]]; then - option+="${module_path}:::$(basename "${module_path}"):::local," - else - option+="${module_path}," - fi + pip install ${PIP_INSTALL_EXTRA_ARGS:-} "${module_path}[server]" done - INSTALLOPTIONS+=("${option: :$((${#option} - 1))}") - fi - - if ! "${SERVERINSTALLDIR}/dirac-install.py" "${INSTALLOPTIONS[@]}" "${SERVERINSTALLDIR}/install.cfg" "${SERVERINSTALLDIR}/dirac-ci-install.cfg"; then - echo "ERROR: dirac-install.py failed" >&2 - exit 1 + cd - + else + if [[ -n "${DEBUG+x}" ]]; then + INSTALLOPTIONS+=("${DEBUG}") + fi + + if [[ "${DIRACOSVER}" ]]; then + INSTALLOPTIONS+=("--dirac-os") + INSTALLOPTIONS+=("--dirac-os-version=${DIRACOSVER}") + fi + + if [[ "$DIRACOS_TARBALL_PATH" ]]; then + { + echo "DIRACOS = $DIRACOS_TARBALL_PATH" + } >> "${SERVERINSTALLDIR}/dirac-ci-install.cfg" + fi + + if [[ -n "${ALTERNATIVE_MODULES+x}" ]]; then + echo "Installing from non-release code" + option="--module=" + for module_path in "${ALTERNATIVE_MODULES[@]}"; do + if [[ -d "${module_path}" ]]; then + option+="${module_path}:::$(basename "${module_path}"):::local," + else + option+="${module_path}," + fi + done + INSTALLOPTIONS+=("${option: :$((${#option} - 1))}") + fi + + if ! "${SERVERINSTALLDIR}/dirac-install.py" "${INSTALLOPTIONS[@]}" "${SERVERINSTALLDIR}/install.cfg" "${SERVERINSTALLDIR}/dirac-ci-install.cfg"; then + echo "ERROR: dirac-install.py failed" >&2 + exit 1 + fi fi echo "==> Done installing, now configuring"