From 0d72a7de32725733dc8486aec639a265274997a9 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 30 Oct 2023 04:41:09 +0100 Subject: [PATCH] feat: Support using both HTTPS and DiracX --- dirac.cfg | 4 ++-- .../environment_variable_configuration.rst | 3 --- integration_tests.py | 2 +- .../ConfigurationSystem/Client/PathFinder.py | 19 +++++++++++++++++++ src/DIRAC/Core/Base/Client.py | 5 ++++- .../Core/Tornado/Client/ClientSelector.py | 10 +++++++++- .../FrameworkSystem/scripts/dirac_login.py | 4 ++-- .../scripts/dirac_proxy_init.py | 4 ++-- .../Client/JobMonitoringClient.py | 14 ++++++++------ .../Service/SandboxStoreHandler.py | 4 ++-- 10 files changed, 49 insertions(+), 20 deletions(-) diff --git a/dirac.cfg b/dirac.cfg index d67f02c2959..242467ce2e5 100644 --- a/dirac.cfg +++ b/dirac.cfg @@ -134,8 +134,8 @@ DiracX URL = https://diracx.invalid:8000 # A key used to have priviledged interactions with diracx. see LegacyExchangeApiKey = diracx:legacy:InsecureChangeMe - # List of VOs which should use DiracX via the legacy compatibility mechanism - EnabledVOs = gridpp,cta + # List of VOs which should not use DiracX via the legacy compatibility mechanism + DisabledVOs = dteam,cta } ### Registry section: # Sections to register VOs, groups, users and hosts diff --git a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst index ab494abfb0d..09d0d29b3fa 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst @@ -18,9 +18,6 @@ DIRAC_DEPRECATED_FAIL If set, the use of functions or objects that are marked ``@deprecated`` will fail. Useful for example in continuous integration tests against future versions of DIRAC -DIRAC_ENABLE_DIRACX_JOB_MONITORING - If set, calls the diracx job monitoring service. Off by default. - DIRAC_FEWER_CFG_LOCKS If ``true`` or ``yes`` or ``on`` or ``1`` or ``y`` or ``t``, DIRAC will reduce the number of locks used when accessing the CS for better performance (default, ``no``). diff --git a/integration_tests.py b/integration_tests.py index 7e6a1c80389..6377a48fc9c 100755 --- a/integration_tests.py +++ b/integration_tests.py @@ -35,7 +35,7 @@ "DIRAC_USE_JSON_ENCODE": None, "INSTALLATION_BRANCH": "", } -DIRACX_OPTIONS = ("DIRAC_ENABLE_DIRACX_JOB_MONITORING",) +DIRACX_OPTIONS = () DEFAULT_MODULES = {"DIRAC": Path(__file__).parent.absolute()} # Static configuration diff --git a/src/DIRAC/ConfigurationSystem/Client/PathFinder.py b/src/DIRAC/ConfigurationSystem/Client/PathFinder.py index 941cf8dec41..f5470659cca 100755 --- a/src/DIRAC/ConfigurationSystem/Client/PathFinder.py +++ b/src/DIRAC/ConfigurationSystem/Client/PathFinder.py @@ -248,6 +248,19 @@ def getServiceURLs(system, service=None, setup=False, failover=False): return resList +def useLegacyAdapter(system, service=None) -> bool: + """Should DiracX be used for this service via the legacy adapter mechanism + + :param str system: system name or full name e.g.: Framework/ProxyManager + :param str service: service name, like 'ProxyManager'. + + :return: bool -- True if DiracX should be used + """ + system, service = divideFullName(system, service) + value = gConfigurationData.extractOptionFromCFG(f"/DiracX/LegacyClientEnabled/{system}/{service}") + return (value or "no").lower() in ("y", "yes", "true", "1") + + def getServiceURL(system, service=None, setup=False): """Generate url. @@ -297,3 +310,9 @@ def getGatewayURLs(system="", service=None): return False gateways = List.randomize(List.fromChar(gateways, ",")) return [checkComponentURL(u, system, service) for u in gateways if u] if system and service else gateways + + +def getDisabledDiracxVOs() -> list[str]: + """Get the list of VOs for which DiracX is enabled""" + vos = gConfigurationData.extractOptionFromCFG("/DiracX/DisabledVOs") + return List.fromChar(vos or "", ",") diff --git a/src/DIRAC/Core/Base/Client.py b/src/DIRAC/Core/Base/Client.py index 032c3542bd6..a487f9aa2d1 100644 --- a/src/DIRAC/Core/Base/Client.py +++ b/src/DIRAC/Core/Base/Client.py @@ -109,7 +109,10 @@ def _getRPC(self, rpc=None, url="", timeout=None): timeout = self.timeout self.__kwargs["timeout"] = timeout - rpc = RPCClientSelector(url, httpsClient=self.httpsClient, **self.__kwargs) + + rpc = RPCClientSelector( + url, httpsClient=self.httpsClient, diracxClient=getattr(self, "diracxClient", None), **self.__kwargs + ) return rpc diff --git a/src/DIRAC/Core/Tornado/Client/ClientSelector.py b/src/DIRAC/Core/Tornado/Client/ClientSelector.py index db64d438dac..741a6dc20bf 100644 --- a/src/DIRAC/Core/Tornado/Client/ClientSelector.py +++ b/src/DIRAC/Core/Tornado/Client/ClientSelector.py @@ -12,7 +12,7 @@ import functools from DIRAC import gLogger -from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL +from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL, useLegacyAdapter from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.Core.DISET.TransferClient import TransferClient from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient @@ -54,6 +54,7 @@ def ClientSelector(disetClient, *args, **kwargs): # We use same interface as RP # We detect if we need to use a specific class for the HTTPS client tornadoClient = kwargs.pop("httpsClient", TornadoClient) + diracxClient = kwargs.pop("diracxClient", None) # We have to make URL resolution BEFORE the RPCClient or TornadoClient to determine which one we want to use # URL is defined as first argument (called serviceName) in RPCClient @@ -65,6 +66,13 @@ def ClientSelector(disetClient, *args, **kwargs): # We use same interface as RP # If we are not already given a URL, resolve it if serviceName.startswith(("http", "dip")): completeUrl = serviceName + elif useLegacyAdapter(serviceName): + sLog.debug(f"Using legacy adapter for service {serviceName}") + if diracxClient is None: + raise NotImplementedError( + "DiracX is enabled but no diracxClient is provided, do you need to update your client?" + ) + return diracxClient() else: completeUrl = getServiceURL(serviceName) sLog.debug(f"URL resolved: {completeUrl}") diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_login.py b/src/DIRAC/FrameworkSystem/scripts/dirac_login.py index 55ff8ba528a..d0be47e34ec 100644 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_login.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_login.py @@ -315,8 +315,8 @@ def loginWithCertificate(self): # Get a token for use with diracx vo = getVOMSVOForGroup(self.group) - enabledVOs = gConfig.getValue("/DiracX/EnabledVOs", []) - if vo in enabledVOs: + disabledVOs = gConfig.getValue("/DiracX/DisabledVOs", []) + if vo not in disabledVOs: from diracx.core.utils import write_credentials # pylint: disable=import-error from diracx.core.models import TokenResponse # pylint: disable=import-error from diracx.core.preferences import DiracxPreferences # pylint: disable=import-error diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py index 3c936b494d9..235dbf4b59a 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py @@ -239,8 +239,8 @@ def doTheMagic(self): return resultProxyUpload vo = Registry.getVOMSVOForGroup(self.__piParams.diracGroup) - enabledVOs = gConfig.getValue("/DiracX/EnabledVOs", []) - if vo in enabledVOs: + disabledVOs = gConfig.getValue("/DiracX/DisabledVOs", []) + if vo not in disabledVOs: from diracx.core.utils import write_credentials # pylint: disable=import-error from diracx.core.models import TokenResponse # pylint: disable=import-error from diracx.core.preferences import DiracxPreferences # pylint: disable=import-error diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py b/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py index ecf53fa97b9..3c967b1be6b 100755 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py @@ -5,6 +5,13 @@ from DIRAC.Core.Utilities.DEncode import ignoreEncodeWarning from DIRAC.Core.Utilities.JEncode import strToIntDict +try: + from DIRAC.WorkloadManagementSystem.FutureClient.JobMonitoringClient import ( + JobMonitoringClient as futureJobMonitoringClient, + ) +except ImportError: + futureJobMonitoringClient = None + @createClient("WorkloadManagement/JobMonitoring") class JobMonitoringClient(Client): @@ -12,12 +19,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.setServer("WorkloadManagement/JobMonitoring") - if os.getenv("DIRAC_ENABLE_DIRACX_JOB_MONITORING", "No").lower() in ("yes", "true"): - from DIRAC.WorkloadManagementSystem.FutureClient.JobMonitoringClient import ( - JobMonitoringClient as futureJobMonitoringClient, - ) - - httpsClient = futureJobMonitoringClient + diracxClient = futureJobMonitoringClient @ignoreEncodeWarning def getJobsStatus(self, jobIDs): diff --git a/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py index c6489b6c1fd..bf617f3326c 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py @@ -111,8 +111,8 @@ def _getFromClient(self, fileId, token, fileSize, fileHelper=None, data=""): credDict = self.getRemoteCredentials() vo = Registry.getVOForGroup(credDict["group"]) - enabledVOs = gConfig.getValue("/DiracX/EnabledVOs", []) - if self._useDiracXBackend and vo in enabledVOs: + disabledVOs = gConfig.getValue("/DiracX/DisabledVOs", []) + if self._useDiracXBackend and vo not in disabledVOs: from DIRAC.FrameworkSystem.Utilities.diracx import TheImpersonator from diracx.client.models import SandboxInfo # pylint: disable=import-error