From a12c05c785c4833ab9f4025d439823a36eab947e Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Thu, 19 Sep 2024 12:47:02 +0100 Subject: [PATCH 1/5] Fallback to site config if no property --- .../cloud_info_catchall/config_generator.py | 1 + .../cloud_info_catchall/share_discovery.py | 28 +++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/cloud-info/cloud_info_catchall/config_generator.py b/cloud-info/cloud_info_catchall/config_generator.py index a7aa79b0..c27196b0 100755 --- a/cloud-info/cloud_info_catchall/config_generator.py +++ b/cloud-info/cloud_info_catchall/config_generator.py @@ -75,6 +75,7 @@ def main(): "site_name": os.environ["SITE_NAME"], "token_url": os.environ.get("TOKEN_URL", ""), "vo_dir": os.environ.get("VO_SECRETS_PATH", ""), + "site_config": os.environ.get("CLOUD_INFO_CONFIG", ""), } secrets = read_secrets(secrets_file) shares_config = generate_shares_config(config, secrets) diff --git a/cloud-info/cloud_info_catchall/share_discovery.py b/cloud-info/cloud_info_catchall/share_discovery.py index 57de67db..b33df417 100644 --- a/cloud-info/cloud_info_catchall/share_discovery.py +++ b/cloud-info/cloud_info_catchall/share_discovery.py @@ -14,10 +14,29 @@ def __init__(self, config, secret): self.identity_provider = config["identity_provider"] self.protocol = config["protocol"] self.secret = secret + self.site_config = config.get("site_config", None) + self._site_mapping = {} + + def get_site_mapping(self): + if self._site_mapping: + return self._site_mapping + if not self.site_config: + return {} + with open(self.site_config, "r") as f: + cloud_info_config = yaml.load(f.read(), Loader=yaml.SafeLoader) + shares = cloud_info_config["compute"]["shares"] + for share in shares.values(): + project = share.get("auth", {}).get("project_id", None) + if project: + self._site_mapping[project] = share.get("name", None) + return self._site_mapping def build_share(self, project, access_token): return {"auth": {"project_id": project["id"]}} + def site_config_vo(self, project): + return self.get_site_mapping().get(project["id"], None) + def get_project_vos(self, project): if not project.get("enabled", False): logging.warning( @@ -29,9 +48,14 @@ def get_project_vos(self, project): vo = project.get("VO", None) if not vo: logging.warning( - f"Discarding project {project['name']} as it does not have VO property" + f"Project {project['name']} as it does not have VO property" ) - return [] + vo = self.site_config_vo(project) + if not vo: + logging.warning( + f"Discarding project {project['name']} as it's not known" + ) + return [] return vo.split(",") def get_token_shares(self): From 01e74a5c2a35110a98b2b10f47170a84b8194a5e Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Thu, 19 Sep 2024 12:54:21 +0100 Subject: [PATCH 2/5] Add missing import --- cloud-info/cloud_info_catchall/share_discovery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloud-info/cloud_info_catchall/share_discovery.py b/cloud-info/cloud_info_catchall/share_discovery.py index b33df417..4b1aee37 100644 --- a/cloud-info/cloud_info_catchall/share_discovery.py +++ b/cloud-info/cloud_info_catchall/share_discovery.py @@ -6,6 +6,7 @@ import fedcloudclient.endpoint as fedcli from cloud_info_provider.auth_refreshers.oidc_refresh import OidcRefreshToken +import yaml class ShareDiscovery: From 3172145652ddb7135fc71a5f05f50e72c142ae10 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Thu, 19 Sep 2024 12:54:57 +0100 Subject: [PATCH 3/5] Improve warning message --- cloud-info/cloud_info_catchall/share_discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud-info/cloud_info_catchall/share_discovery.py b/cloud-info/cloud_info_catchall/share_discovery.py index 4b1aee37..bf4103ea 100644 --- a/cloud-info/cloud_info_catchall/share_discovery.py +++ b/cloud-info/cloud_info_catchall/share_discovery.py @@ -49,7 +49,7 @@ def get_project_vos(self, project): vo = project.get("VO", None) if not vo: logging.warning( - f"Project {project['name']} as it does not have VO property" + f"Project {project['name']} does not have VO property" ) vo = self.site_config_vo(project) if not vo: From 9681acf5618c5a3f66e2593a14e1663660507fcc Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Thu, 19 Sep 2024 13:36:33 +0100 Subject: [PATCH 4/5] Reorganise code --- .../cloud_info_catchall/config_generator.py | 19 ++++++++++++- .../cloud_info_catchall/share_discovery.py | 27 +++---------------- .../test_share_discovery.py | 9 +++++++ 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/cloud-info/cloud_info_catchall/config_generator.py b/cloud-info/cloud_info_catchall/config_generator.py index c27196b0..2f739a60 100755 --- a/cloud-info/cloud_info_catchall/config_generator.py +++ b/cloud-info/cloud_info_catchall/config_generator.py @@ -20,6 +20,8 @@ TOKEN_URL: URL to refresh tokens OS_AUTH_URL, OS_IDENTITY_PROVIDER, OS_PROTOCOL: OpenStack endpoint config SITE_NAME: site name +CLOUD_INFO_CONFIG: the file with the existing cloud-info provider config + it will be used as a fallback if the projects do not have the properties """ import logging @@ -59,6 +61,20 @@ def generate_shares(config, secrets): return shares +def get_fallback_mapping(site_config): + mapping = {} + if not site_config: + return mapping + with open(self.site_config, "r") as f: + cloud_info_config = yaml.load(f.read(), Loader=yaml.SafeLoader) + shares = cloud_info_config.get("compute", {}).get("shares", {}) + for share in shares.values(): + project = share.get("auth", {}).get("project_id", None) + if project and "name" in share: + mapping[project] = share["name"] + return mapping + + def generate_shares_config(config, secrets): shares = generate_shares(config, secrets) return {"site": {"name": config["site_name"]}, "compute": {"shares": shares}} @@ -68,6 +84,7 @@ def main(): logging.basicConfig() # get config from env secrets_file = os.environ["SECRETS_FILE"] + site_config = os.environ.get("CLOUD_INFO_CONFIG", "") config = { "auth_url": os.environ["OS_AUTH_URL"], "identity_provider": os.environ["OS_IDENTITY_PROVIDER"], @@ -75,7 +92,7 @@ def main(): "site_name": os.environ["SITE_NAME"], "token_url": os.environ.get("TOKEN_URL", ""), "vo_dir": os.environ.get("VO_SECRETS_PATH", ""), - "site_config": os.environ.get("CLOUD_INFO_CONFIG", ""), + "vo_fallback": get_fallback_mapping(site_config), } secrets = read_secrets(secrets_file) shares_config = generate_shares_config(config, secrets) diff --git a/cloud-info/cloud_info_catchall/share_discovery.py b/cloud-info/cloud_info_catchall/share_discovery.py index bf4103ea..b2ed90e9 100644 --- a/cloud-info/cloud_info_catchall/share_discovery.py +++ b/cloud-info/cloud_info_catchall/share_discovery.py @@ -6,7 +6,6 @@ import fedcloudclient.endpoint as fedcli from cloud_info_provider.auth_refreshers.oidc_refresh import OidcRefreshToken -import yaml class ShareDiscovery: @@ -15,29 +14,11 @@ def __init__(self, config, secret): self.identity_provider = config["identity_provider"] self.protocol = config["protocol"] self.secret = secret - self.site_config = config.get("site_config", None) - self._site_mapping = {} - - def get_site_mapping(self): - if self._site_mapping: - return self._site_mapping - if not self.site_config: - return {} - with open(self.site_config, "r") as f: - cloud_info_config = yaml.load(f.read(), Loader=yaml.SafeLoader) - shares = cloud_info_config["compute"]["shares"] - for share in shares.values(): - project = share.get("auth", {}).get("project_id", None) - if project: - self._site_mapping[project] = share.get("name", None) - return self._site_mapping + self.vo_fallback = config.get("vo_fallback", {}) def build_share(self, project, access_token): return {"auth": {"project_id": project["id"]}} - def site_config_vo(self, project): - return self.get_site_mapping().get(project["id"], None) - def get_project_vos(self, project): if not project.get("enabled", False): logging.warning( @@ -48,10 +29,8 @@ def get_project_vos(self, project): if not vo: vo = project.get("VO", None) if not vo: - logging.warning( - f"Project {project['name']} does not have VO property" - ) - vo = self.site_config_vo(project) + logging.warning(f"Project {project['name']} does not have VO property") + vo = self.vo_fallback.get(project.get("id", None), None) if not vo: logging.warning( f"Discarding project {project['name']} as it's not known" diff --git a/cloud-info/cloud_info_catchall/test_share_discovery.py b/cloud-info/cloud_info_catchall/test_share_discovery.py index ff5b5f70..21f13edd 100644 --- a/cloud-info/cloud_info_catchall/test_share_discovery.py +++ b/cloud-info/cloud_info_catchall/test_share_discovery.py @@ -19,6 +19,7 @@ class ShareDiscoveryTest(unittest.TestCase): "protocol": "oidc", "token_url": "https://aai.egi.eu", "vo_dir": "vo", + "vo_fallback": {"123": "cloud.egi.eu"}, } SECRET = {"foo": "bar"} @@ -57,6 +58,14 @@ def test_get_project_no_vo_property(self): } self.assertEqual(self.discoverer.get_project_vos(p), []) + def test_get_project_no_vo_property_fallback(self): + p = { + "enabled": True, + "name": "baz", + "id": "123", + } + self.assertEqual(self.discoverer.get_project_vos(p), ["cloud.egi.eu"]) + def test_get_project_multiple_vo_property(self): p = {"enabled": True, "name": "foo.eu", "egi.VO": "foo,bar"} self.assertEqual(self.discoverer.get_project_vos(p), ["foo", "bar"]) From a01f189177fe352794b96ba35325cf265a1a8551 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Thu, 19 Sep 2024 13:40:17 +0100 Subject: [PATCH 5/5] Copy pasting like crazy never helps --- cloud-info/cloud_info_catchall/config_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud-info/cloud_info_catchall/config_generator.py b/cloud-info/cloud_info_catchall/config_generator.py index 2f739a60..a6a337c7 100755 --- a/cloud-info/cloud_info_catchall/config_generator.py +++ b/cloud-info/cloud_info_catchall/config_generator.py @@ -65,7 +65,7 @@ def get_fallback_mapping(site_config): mapping = {} if not site_config: return mapping - with open(self.site_config, "r") as f: + with open(site_config, "r") as f: cloud_info_config = yaml.load(f.read(), Loader=yaml.SafeLoader) shares = cloud_info_config.get("compute", {}).get("shares", {}) for share in shares.values():