Fallback to site config if no property (#374)

* Fallback to site config if no property

* Add missing import

* Improve warning message

* Reorganise code

* Copy pasting like crazy never helps

cloud-info/cloud_info_catchall/config_generator.py

@@ -20,6 +20,8 @@
 TOKEN_URL: URL to refresh tokens
 OS_AUTH_URL, OS_IDENTITY_PROVIDER, OS_PROTOCOL: OpenStack endpoint config
 SITE_NAME: site name
+CLOUD_INFO_CONFIG: the file with the existing cloud-info provider config
+ it will be used as a fallback if the projects do not have the properties
 """
 
 import logging
@@ -59,6 +61,20 @@ def generate_shares(config, secrets):
     return shares
 
 
+def get_fallback_mapping(site_config):
+    mapping = {}
+    if not site_config:
+        return mapping
+    with open(site_config, "r") as f:
+        cloud_info_config = yaml.load(f.read(), Loader=yaml.SafeLoader)
+        shares = cloud_info_config.get("compute", {}).get("shares", {})
+        for share in shares.values():
+            project = share.get("auth", {}).get("project_id", None)
+            if project and "name" in share:
+                mapping[project] = share["name"]
+    return mapping
+
+
 def generate_shares_config(config, secrets):
     shares = generate_shares(config, secrets)
     return {"site": {"name": config["site_name"]}, "compute": {"shares": shares}}
@@ -68,13 +84,15 @@ def main():
     logging.basicConfig()
     # get config from env
     secrets_file = os.environ["SECRETS_FILE"]
+    site_config = os.environ.get("CLOUD_INFO_CONFIG", "")
     config = {
         "auth_url": os.environ["OS_AUTH_URL"],
         "identity_provider": os.environ["OS_IDENTITY_PROVIDER"],
         "protocol": os.environ["OS_PROTOCOL"],
         "site_name": os.environ["SITE_NAME"],
         "token_url": os.environ.get("TOKEN_URL", ""),
         "vo_dir": os.environ.get("VO_SECRETS_PATH", ""),
+        "vo_fallback": get_fallback_mapping(site_config),
     }
     secrets = read_secrets(secrets_file)
     shares_config = generate_shares_config(config, secrets)

cloud-info/cloud_info_catchall/share_discovery.py

@@ -14,6 +14,7 @@ def __init__(self, config, secret):
         self.identity_provider = config["identity_provider"]
         self.protocol = config["protocol"]
         self.secret = secret
+        self.vo_fallback = config.get("vo_fallback", {})
 
     def build_share(self, project, access_token):
         return {"auth": {"project_id": project["id"]}}
@@ -28,10 +29,13 @@ def get_project_vos(self, project):
         if not vo:
             vo = project.get("VO", None)
             if not vo:
-                logging.warning(
-                    f"Discarding project {project['name']} as it does not have VO property"
-                )
-                return []
+                logging.warning(f"Project {project['name']} does not have VO property")
+                vo = self.vo_fallback.get(project.get("id", None), None)
+                if not vo:
+                    logging.warning(
+                        f"Discarding project {project['name']} as it's not known"
+                    )
+                    return []
         return vo.split(",")
 
     def get_token_shares(self):

cloud-info/cloud_info_catchall/test_share_discovery.py

@@ -19,6 +19,7 @@ class ShareDiscoveryTest(unittest.TestCase):
         "protocol": "oidc",
         "token_url": "https://aai.egi.eu",
         "vo_dir": "vo",
+        "vo_fallback": {"123": "cloud.egi.eu"},
     }
     SECRET = {"foo": "bar"}
 
@@ -57,6 +58,14 @@ def test_get_project_no_vo_property(self):
         }
         self.assertEqual(self.discoverer.get_project_vos(p), [])
 
+    def test_get_project_no_vo_property_fallback(self):
+        p = {
+            "enabled": True,
+            "name": "baz",
+            "id": "123",
+        }
+        self.assertEqual(self.discoverer.get_project_vos(p), ["cloud.egi.eu"])
+
     def test_get_project_multiple_vo_property(self):
         p = {"enabled": True, "name": "foo.eu", "egi.VO": "foo,bar"}
         self.assertEqual(self.discoverer.get_project_vos(p), ["foo", "bar"])