From f5d1c1c77236343d8086dc1915264ce6ca6fa58d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Manrique=20Ardila?= Date: Tue, 10 Oct 2023 09:52:18 -0500 Subject: [PATCH 1/4] Get default PNN from SITECONF file --- docker/CMSRucioClient/scripts/cmslinks.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docker/CMSRucioClient/scripts/cmslinks.py b/docker/CMSRucioClient/scripts/cmslinks.py index 01daa314..819be5c4 100755 --- a/docker/CMSRucioClient/scripts/cmslinks.py +++ b/docker/CMSRucioClient/scripts/cmslinks.py @@ -3,7 +3,8 @@ Class definition for the distances (links) among CMS RSEs. And script for updating the distances. """ - +import gitlab +import base64 import argparse import json import logging @@ -45,6 +46,8 @@ def __init__(self, account, auth_type=None, exclude=DEFAULT_EXCLUDE_LINKS, self._get_matrix(distance, exclude) def _get_rselist(self, rselist=None): + private_token = os.environ['GITLAB_TOKEN'] + gl = gitlab.Gitlab('https://gitlab.cern.ch', private_token=private_token) self.rselist = [] @@ -53,11 +56,20 @@ def _get_rselist(self, rselist=None): for rse in rselist: attrs = self.rcli.list_rse_attributes(rse=rse) + pnn = attrs.get('pnn') + if pnn is None: + project = gl.projects.get('SITECONF/'+rse) + f = project.files.get('storage.json', 'master') + sites = json.loads(base64.b64decode(f.content)) + for site in sites: + if site.get('rse') == rse: + pnn = site.get('site') + break try: self.rselist.append({ 'rse': rse, - 'pnn': attrs.get('pnn'), + 'pnn': pnn, 'type': attrs['cms_type'], 'country': attrs['country'], 'region': attrs.get('region', None) From ab94248b36e078d019d08e1211134bbbd2a3ccc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Manrique=20Ardila?= Date: Tue, 10 Oct 2023 14:36:48 -0500 Subject: [PATCH 2/4] Fixes to keep only relevant _ info --- docker/CMSRucioClient/scripts/cmslinks.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/docker/CMSRucioClient/scripts/cmslinks.py b/docker/CMSRucioClient/scripts/cmslinks.py index 819be5c4..a4299451 100755 --- a/docker/CMSRucioClient/scripts/cmslinks.py +++ b/docker/CMSRucioClient/scripts/cmslinks.py @@ -46,8 +46,12 @@ def __init__(self, account, auth_type=None, exclude=DEFAULT_EXCLUDE_LINKS, self._get_matrix(distance, exclude) def _get_rselist(self, rselist=None): - private_token = os.environ['GITLAB_TOKEN'] - gl = gitlab.Gitlab('https://gitlab.cern.ch', private_token=private_token) + try: + private_token = os.environ['GITLAB_TOKEN'] + gl = gitlab.Gitlab('https://gitlab.cern.ch', private_token=private_token) + except Exception as e: + logging.warning(f'Could not connect to gitlab. Error: {str(e)}') + gl = None self.rselist = [] @@ -58,11 +62,17 @@ def _get_rselist(self, rselist=None): attrs = self.rcli.list_rse_attributes(rse=rse) pnn = attrs.get('pnn') if pnn is None: - project = gl.projects.get('SITECONF/'+rse) - f = project.files.get('storage.json', 'master') - sites = json.loads(base64.b64decode(f.content)) + sites = [] + try: + project_rse = rse.split('_')[:3] + project_rse = '_'.join(project_rse) + project = gl.projects.get('SITECONF/'+project_rse) + f = project.files.get('storage.json', 'master') + sites = json.loads(base64.b64decode(f.content)) + except Exception as e: + logging.warning(f'No PNN for RSE {rse}. Trying to get it from gitlab. Error: {str(e)}') for site in sites: - if site.get('rse') == rse: + if site.get('rse') in rse: pnn = site.get('site') break From 7fd073e6ddf6d46a37c557abbcecf92594df098c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Manrique=20Ardila?= Date: Mon, 16 Oct 2023 21:10:44 -0500 Subject: [PATCH 3/4] Get usage to avoid KeyError Exception --- docker/CMSRucioClient/scripts/cmslinks.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/docker/CMSRucioClient/scripts/cmslinks.py b/docker/CMSRucioClient/scripts/cmslinks.py index a4299451..b1b30f7a 100755 --- a/docker/CMSRucioClient/scripts/cmslinks.py +++ b/docker/CMSRucioClient/scripts/cmslinks.py @@ -76,17 +76,14 @@ def _get_rselist(self, rselist=None): pnn = site.get('site') break - try: - self.rselist.append({ - 'rse': rse, - 'pnn': pnn, - 'type': attrs['cms_type'], - 'country': attrs['country'], - 'region': attrs.get('region', None) - }) - except KeyError: - logging.warning('No expected attributes for RSE %s. Skipping', - rse) + self.rselist.append({ + 'rse': rse, + 'pnn': pnn, + 'type': attrs.get('cms_type'), + 'country': attrs.get('country'), + 'region': attrs.get('region', None) + }) + def _get_matrix(self, distance, exclude): From 8329b3ca974d6e9733b130e512dfd2919ee9f997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Manrique=20Ardila?= Date: Thu, 2 Nov 2023 17:14:32 -0500 Subject: [PATCH 4/4] Missing key error logging --- docker/CMSRucioClient/scripts/cmslinks.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docker/CMSRucioClient/scripts/cmslinks.py b/docker/CMSRucioClient/scripts/cmslinks.py index b1b30f7a..6517e949 100755 --- a/docker/CMSRucioClient/scripts/cmslinks.py +++ b/docker/CMSRucioClient/scripts/cmslinks.py @@ -75,14 +75,16 @@ def _get_rselist(self, rselist=None): if site.get('rse') in rse: pnn = site.get('site') break - - self.rselist.append({ - 'rse': rse, - 'pnn': pnn, - 'type': attrs.get('cms_type'), - 'country': attrs.get('country'), - 'region': attrs.get('region', None) - }) + try: + self.rselist.append({ + 'rse': rse, + 'pnn': pnn, + 'type': attrs.get('cms_type'), + 'country': attrs.get('country'), + 'region': attrs.get('region') + }) + except Exception as e: + logging.warning(f'Could not get attributes for RSE {rse}. Error: {str(e)}') def _get_matrix(self, distance, exclude):