[v7r3] Start running integration tests with a Python 3 server

.github/workflows/integration.yml

@@ -31,6 +31,10 @@ jobs:
           ###### Python 3
           - TEST_NAME: "Python 3 client"
             ARGS: CLIENT_USE_PYTHON3=Yes
+          - TEST_NAME: "Python 3 server"
+            ARGS: SERVER_USE_PYTHON3=Yes
+          - TEST_NAME: "Python 3 server and client"
+            ARGS: CLIENT_USE_PYTHON3=Yes SERVER_USE_PYTHON3=Yes MYSQL_VER=8.0
 
     steps:
     - uses: actions/checkout@v2

setup.cfg

@@ -51,6 +51,11 @@ install_requires =
     six
     sqlalchemy
     subprocess32
+zip_safe = False
+include_package_data = True
+
+[options.package_data]
+* = ConfigTemplate.cfg, *.sql
 
 [options.packages.find]
 where=src

src/DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py

@@ -12,7 +12,9 @@
 
 import imp
 import six
+from DIRAC.Core.Utilities.Decorators import deprecated
 from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton
+from DIRAC.Core.Utilities.Extensions import extensionsByPriority
 
 
 @six.add_metaclass(DIRACSingleton)
@@ -25,33 +27,36 @@ def __init__(self):
   def __load(self):
     if self.__orderedExtNames:
       return
-    for extName in self.getCSExtensions() + ['']:
+    for extName in extensionsByPriority():
       try:
-        if not extName.endswith("DIRAC"):
-          extension = '%sDIRAC' % extName
-        res = imp.find_module(extension)
+        res = imp.find_module(extName)
         if res[0]:
           res[0].close()
-        self.__orderedExtNames.append(extension)
-        self.__modules[extension] = res
+        self.__orderedExtNames.append(extName)
+        self.__modules[extName] = res
       except ImportError:
         pass
 
   def getCSExtensions(self):
     if not self.__csExt:
-      from DIRAC.ConfigurationSystem.Client.Config import gConfig
-      exts = gConfig.getValue('/DIRAC/Extensions', [])
-      for iP in range(len(exts)):
-        ext = exts[iP]
+      if six.PY3:
+        exts = extensionsByPriority()
+      else:
+        from DIRAC.ConfigurationSystem.Client.Config import gConfig
+        exts = gConfig.getValue('/DIRAC/Extensions', [])
+
+      self.__csExt = []
+      for ext in exts:
         if ext.endswith("DIRAC"):
           ext = ext[:-5]
-          exts[iP] = ext
-      self.__csExt = exts
+        # If the extension is now "" (i.e. vanilla DIRAC), don't include it
+        if ext:
+          self.__csExt.append(ext)
     return self.__csExt
 
+  @deprecated("Use DIRAC.Core.Utilities.Extensions.extensionsByPriority instead")
   def getInstalledExtensions(self):
-    self.__load()
-    return list(self.__orderedExtNames)
+    return extensionsByPriority()
 
   def getExtensionPath(self, extName):
     self.__load()
@@ -83,11 +88,12 @@ def getCSExtensions():
   return Extensions().getCSExtensions()
 
 
+@deprecated("Use DIRAC.Core.Utilities.Extensions.extensionsByPriority instead")
 def getInstalledExtensions():
   """
     Return list of extensions registered in the CS and available in local installation
   """
-  return Extensions().getInstalledExtensions()
+  return extensionsByPriority()
 
 
 def skipCACheck():

src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py

@@ -404,7 +404,7 @@ def enableCS(self):
     """
     Force the connection the Configuration Server
 
-    (And incidentaly reinitialize the ObjectLoader and logger)
+    (And incidentally reinitialize the ObjectLoader and logger)
     """
     res = gRefresher.enable()
 

src/DIRAC/Core/Base/private/ModuleLoader.py

@@ -4,17 +4,14 @@
 from __future__ import division
 from __future__ import print_function
 
-import six
 import os
-import imp
 from DIRAC.Core.Utilities import List
 from DIRAC import gConfig, S_ERROR, S_OK, gLogger
-from DIRAC.ConfigurationSystem.Client.Helpers import getInstalledExtensions
 from DIRAC.ConfigurationSystem.Client import PathFinder
+from DIRAC.Core.Utilities.Extensions import extensionsByPriority, recurseImport
 
 
 class ModuleLoader(object):
-
   def __init__(self, importLocation, sectionFinder, superClass, csSuffix=False, moduleSuffix=False):
     self.__modules = {}
     self.__loadedModules = {}
@@ -43,7 +40,7 @@ def loadModules(self, modulesList, hideExceptions=False):
     for modName in modulesList:
       gLogger.verbose("Checking %s" % modName)
       # if it's a executor modName name just load it and be done with it
-      if modName.find("/") > -1:
+      if "/" in modName:
         gLogger.verbose("Module %s seems to be a valid name. Try to load it!" % modName)
         result = self.loadModule(modName, hideExceptions=hideExceptions)
         if not result['OK']:
@@ -64,13 +61,12 @@ def loadModules(self, modulesList, hideExceptions=False):
             return result
       # Look what is installed
       parentModule = None
-      for rootModule in getInstalledExtensions():
-        if system.find("System") != len(system) - 6:
-          parentImport = "%s.%sSystem.%s" % (rootModule, system, self.__csSuffix)
-        else:
-          parentImport = "%s.%s.%s" % (rootModule, system, self.__csSuffix)
+      for rootModule in extensionsByPriority():
+        if not system.endswith("System"):
+          system += "System"
+        parentImport = "%s.%s.%s" % (rootModule, system, self.__csSuffix)
         # HERE!
-        result = self.__recurseImport(parentImport)
+        result = recurseImport(parentImport)
         if not result['OK']:
           return result
         parentModule = result['Value']
@@ -81,7 +77,7 @@ def loadModules(self, modulesList, hideExceptions=False):
       parentPath = parentModule.__path__[0]
       gLogger.notice("Found modules path at %s" % parentImport)
       for entry in os.listdir(parentPath):
-        if entry[-3:] != ".py" or entry == "__init__.py":
+        if entry == "__init__.py" or not entry.endswith(".py"):
           continue
         if not os.path.isfile(os.path.join(parentPath, entry)):
           continue
@@ -111,9 +107,9 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False):
     if loadGroup:
       gLogger.info("Found load group %s. Will load %s" % (modName, ", ".join(loadGroup)))
       for loadModName in loadGroup:
-        if loadModName.find("/") == -1:
+        if "/" not in loadModName:
           loadModName = "%s/%s" % (modList[0], loadModName)
-        result = self.loadModule(loadModName, hideExceptions=hideExceptions, parentModule=False)
+        result = self.loadModule(loadModName, hideExceptions=hideExceptions)
         if not result['OK']:
           return result
       return S_OK()
@@ -123,7 +119,7 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False):
       loadName = modName
       gLogger.info("Loading %s" % (modName))
     else:
-      if loadName.find("/") == -1:
+      if "/" not in loadName:
         loadName = "%s/%s" % (modList[0], loadName)
       gLogger.info("Loading %s (%s)" % (modName, loadName))
     # If already loaded, skip
@@ -143,10 +139,10 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False):
         gLogger.info("Trying to %s from CS defined path %s" % (loadName, handlerPath))
         gLogger.verbose("Found handler for %s: %s" % (loadName, handlerPath))
         handlerPath = handlerPath.replace("/", ".")
-        if handlerPath.find(".py", len(handlerPath) - 3) > -1:
+        if handlerPath.endswith(".py"):
           handlerPath = handlerPath[:-3]
         className = List.fromChar(handlerPath, ".")[-1]
-        result = self.__recurseImport(handlerPath)
+        result = recurseImport(handlerPath)
         if not result['OK']:
           return S_ERROR("Cannot load user defined handler %s: %s" % (handlerPath, result['Message']))
         gLogger.verbose("Loaded %s" % handlerPath)
@@ -156,17 +152,17 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False):
         modImport = module
         if self.__modSuffix:
           modImport = "%s%s" % (modImport, self.__modSuffix)
-        result = self.__recurseImport(modImport, parentModule, hideExceptions=hideExceptions)
+        result = recurseImport(modImport, parentModule, hideExceptions=hideExceptions)
       else:
         # Check to see if the module exists in any of the root modules
         gLogger.info("Trying to autodiscover %s" % loadName)
-        rootModulesToLook = getInstalledExtensions()
+        rootModulesToLook = extensionsByPriority()
         for rootModule in rootModulesToLook:
           importString = '%s.%sSystem.%s.%s' % (rootModule, system, self.__importLocation, module)
           if self.__modSuffix:
             importString = "%s%s" % (importString, self.__modSuffix)
           gLogger.verbose("Trying to load %s" % importString)
-          result = self.__recurseImport(importString, hideExceptions=hideExceptions)
+          result = recurseImport(importString, hideExceptions=hideExceptions)
           # Error while loading
           if not result['OK']:
             return result
@@ -203,29 +199,3 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False):
     gLogger.notice("Loaded module %s" % modName)
 
     return S_OK()
-
-  def __recurseImport(self, modName, parentModule=None, hideExceptions=False):
-    gLogger.debug("importing recursively %s, parentModule=%s, hideExceptions=%s" % (modName,
-                                                                                    parentModule,
-                                                                                    hideExceptions))
-    if isinstance(modName, six.string_types):
-      modName = List.fromChar(modName, ".")
-    try:
-      if parentModule:
-        impData = imp.find_module(modName[0], parentModule.__path__)
-      else:
-        impData = imp.find_module(modName[0])
-      impModule = imp.load_module(modName[0], *impData)
-      if impData[0]:
-        impData[0].close()
-    except ImportError as excp:
-      strExcp = str(excp)
-      if strExcp.find("No module named") == 0 and strExcp.find(modName[0]) == len(strExcp) - len(modName[0]):
-        return S_OK()
-      errMsg = "Can't load %s" % ".".join(modName)
-      if not hideExceptions:
-        gLogger.exception(errMsg)
-      return S_ERROR(errMsg)
-    if len(modName) == 1:
-      return S_OK(impModule)
-    return self.__recurseImport(modName[1:], impModule)

src/DIRAC/Core/DISET/private/MessageFactory.py

@@ -7,7 +7,7 @@
 from DIRAC import S_OK, S_ERROR
 from DIRAC.FrameworkSystem.Client.Logger import gLogger
 from DIRAC.Core.Utilities import List
-from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals
+from DIRAC.Core.Utilities.Extensions import extensionsByPriority
 
 
 class MessageFactory(object):
@@ -256,10 +256,9 @@ def loadObjects(path, reFilter=None, parentClass=None):
     reFilter = re.compile(r".*[a-z1-9]\.py$")
   pathList = List.fromChar(path, "/")
 
-  parentModuleList = ["%sDIRAC" % ext for ext in CSGlobals.getCSExtensions()] + ['DIRAC']
   objectsToLoad = {}
   # Find which object files match
-  for parentModule in parentModuleList:
+  for parentModule in extensionsByPriority():
     objDir = os.path.join(DIRAC.rootPath, parentModule, *pathList)
     if not os.path.isdir(objDir):
       continue

src/DIRAC/Core/Utilities/DErrno.py

@@ -43,9 +43,11 @@
 
 import six
 import os
-import imp
+import importlib
 import sys
 
+from DIRAC.Core.Utilities.Extensions import extensionsByPriority
+
 # To avoid conflict, the error numbers should be greater than 1000
 # We decided to group the by range of 100 per system
 
@@ -344,50 +346,22 @@ def includeExtensionErrors():
       Should be called only at the initialization of DIRAC, so by the parseCommandLine,
       dirac-agent.py, dirac-service.py, dirac-executor.py
   """
-
-  def __recurseImport(modName, parentModule=None, fullName=False):
-    """ Internal function to load modules
-    """
-    if isinstance(modName, six.string_types):
-      modName = modName.split(".")
-    if not fullName:
-      fullName = ".".join(modName)
+  for extension in reversed(extensionsByPriority()):
+    if extension == "DIRAC":
+      continue
     try:
-      if parentModule:
-        impData = imp.find_module(modName[0], parentModule.__path__)
-      else:
-        impData = imp.find_module(modName[0])
-      impModule = imp.load_module(modName[0], *impData)
-      if impData[0]:
-        impData[0].close()
+      ext_derrno = importlib.import_module('%s.Core.Utilities.DErrno' % extension)
     except ImportError:
-      return None
-    if len(modName) == 1:
-      return impModule
-    return __recurseImport(modName[1:], impModule, fullName=fullName)
-
-  from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals
-  allExtensions = CSGlobals.getCSExtensions()
-
-  for extension in allExtensions:
-    ext_derrno = None
-    try:
-
-      ext_derrno = __recurseImport('%sDIRAC.Core.Utilities.DErrno' % extension)
-
-      if ext_derrno:
-        # The next 3 dictionary MUST be present for consistency
-
-        # Global name of errors
-        sys.modules[__name__].__dict__.update(ext_derrno.extra_dErrName)
-        # Dictionary with the error codes
-        sys.modules[__name__].dErrorCode.update(ext_derrno.extra_dErrorCode)
-        # Error description string
-        sys.modules[__name__].dStrError.update(ext_derrno.extra_dStrError)
-
-        # extra_compatErrorString is optional
-        for err in getattr(ext_derrno, 'extra_compatErrorString', []):
-          sys.modules[__name__].compatErrorString.setdefault(err, []).extend(ext_derrno.extra_compatErrorString[err])
-
-    except Exception:
       pass
+    else:
+      # The next 3 dictionary MUST be present for consistency
+      # Global name of errors
+      sys.modules[__name__].__dict__.update(ext_derrno.extra_dErrName)
+      # Dictionary with the error codes
+      sys.modules[__name__].dErrorCode.update(ext_derrno.extra_dErrorCode)
+      # Error description string
+      sys.modules[__name__].dStrError.update(ext_derrno.extra_dStrError)
+
+      # extra_compatErrorString is optional
+      for err in getattr(ext_derrno, 'extra_compatErrorString', []):
+        sys.modules[__name__].compatErrorString.setdefault(err, []).extend(ext_derrno.extra_compatErrorString[err])

src/DIRAC/Core/Utilities/DIRACScript.py

@@ -75,47 +75,3 @@ def __call__(self, func=None):
     )
 
     return entrypoint.load()._func()
-
-
-def _entrypointToExtension(entrypoint):
-  """"Get the extension name from an EntryPoint object"""
-  # In Python 3.9 this can be "entrypoint.module"
-  module = entrypoint.pattern.match(entrypoint.value).groupdict()["module"]
-  extensionName = module.split(".")[0]
-  return extensionName
-
-
-def _extensionsByPriority():
-  """Discover extensions using the setuptools metadata
-
-  TODO: This should move into a function which can also be called to fill the CS
-  """
-  # This is only available in Python 3.8+ so it has to be here for now
-  from importlib import metadata  # pylint: disable=no-name-in-module
-
-  priorities = defaultdict(list)
-  for entrypoint in metadata.entry_points()['dirac']:
-    extensionName = _entrypointToExtension(entrypoint)
-    extension_metadata = entrypoint.load()()
-    priorities[extension_metadata["priority"]].append(extensionName)
-
-  extensions = []
-  for priority, extensionNames in sorted(priorities.items()):
-    if len(extensionNames) != 1:
-      print(
-          "WARNING: Found multiple extensions with priority",
-          "{} ({})".format(priority, extensionNames),
-      )
-    # If multiple are passed, sort the extensions so things are deterministic at least
-    extensions.extend(sorted(extensionNames))
-  return extensions
-
-
-def _getExtensionMetadata(extensionName):
-  """Get the metadata for a given extension name"""
-  # This is only available in Python 3.8+ so it has to be here for now
-  from importlib import metadata  # pylint: disable=no-name-in-module
-
-  for entrypoint in metadata.entry_points()['dirac']:
-    if extensionName == _entrypointToExtension(entrypoint):
-      return entrypoint.load()()