Merge pull request #3341 from mwestphall/SOFTWARE-5642-auto-gen-topol…

…ogy-ids

SOFTWARE-5642: Auto-generate topology ids

src/tests/verify_resources.py

@@ -369,27 +369,30 @@ def test_7_fqdn_unique(rgs, rgfns):
 
 
 def test_8_res_ids(rgs, rgfns):
-    # Check that resources/resource groups have a numeric ID/GroupID
+    # Check that resources/resource groups have a unique, numeric ID/GroupID
+    # in the case that an ID is manually assigned
 
     errors = 0
     ridres = autodict()
     gidrgs = autodict()
 
     for rg,rgfn in zip(rgs,rgfns):
-        if not isinstance(rg.get('GroupID'), int):
+        group_id = rg.get('GroupID')
+        if group_id is not None and not isinstance(group_id, int):
             print_emsg_once('ResGrpID')
             print("ERROR: Resource Group missing numeric GroupID: '%s'" % rgfn)
             errors += 1
-        else:
+        elif group_id:
             gidrgs[rg['GroupID']] += [rgfn]
 
         for resname,res in sorted(rg['Resources'].items()):
-            if not isinstance(res.get('ID'), int):
+            resource_id = res.get('ID')
+            if resource_id is not None and not isinstance(resource_id, int):
                 print_emsg_once('ResID')
                 print("ERROR: Resource '%s' missing numeric ID in '%s'"
                       % (resname, rgfn))
                 errors += 1
-            else:
+            elif resource_id:
                 ridres[res['ID']] += [(rgfn, resname)]
 
     for gid,rglist in sorted(gidrgs.items()):

src/webapp/common.py

@@ -222,7 +222,6 @@ def trim_space(s: str) -> str:
     ret = re.sub(r"(?m)^[ \t]+", "", ret)
     return ret
 
-
 def run_git_cmd(cmd: List, dir=None, git_dir=None, ssh_key=None) -> bool:
     """
     Run git command, optionally specifying ssh key and/or git dirs
@@ -287,9 +286,19 @@ def git_clone_or_fetch_mirror(repo, git_dir, ssh_key=None) -> bool:
     return ok
 
 
-def gen_id(instr: AnyStr, digits, minimum=1, hashfn=hashlib.md5) -> int:
+def gen_id_from_yaml(data: dict, alternate_name: str, id_key = "ID", mod = 2 ** 31 - 1, minimum = 1, hashfn=hashlib.md5) -> int:
+    """
+    Given a yaml object, return its existing ID if an ID is present, or generate a new ID for the object
+    based on the md5sum of an alternate string value (usually the key of the object in its parent dictionary)
+    """
+    return data.get(id_key) or gen_id(alternate_name, mod, minimum, hashfn)
+
+def gen_id(instr: AnyStr, mod = 2 ** 31 - 1, minimum=1, hashfn=hashlib.md5) -> int:
+    """
+    Convert a string to its integer md5sum, used to autogenerate unique IDs for entities where
+    not otherwise specified
+    """
     instr_b = instr if isinstance(instr, bytes) else instr.encode("utf-8", "surrogateescape")
-    mod = (10 ** digits) - minimum
     return minimum + (int(hashfn(instr_b).hexdigest(), 16) % mod)
 
 

src/webapp/project_reader.py

@@ -13,7 +13,7 @@
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from webapp.common import load_yaml_file, to_xml, is_null
+from webapp.common import load_yaml_file, to_xml, is_null, gen_id_from_yaml
 from webapp.vo_reader import get_vos_data
 from webapp.vos_data import VOsData
 
@@ -64,16 +64,15 @@ def get_one_project(file: str, campus_grid_ids: Dict, vos_data: VOsData) -> Dict
         if 'ResourceAllocations' in data:
             resource_allocations = [get_resource_allocation(ra, idx) for idx, ra in enumerate(data['ResourceAllocations'])]
             data['ResourceAllocations'] = {"ResourceAllocation": resource_allocations}
-        if 'ID' not in data:
-            del project['ID']
-
         name_from_filename = os.path.basename(file)[:-5]  # strip '.yaml'
         if not is_null(data, 'Name'):
             if data['Name'] != name_from_filename:
                 log.warning("%s: 'Name' %r does not match filename" % (file, data['Name']))
         else:
             data['Name'] = name_from_filename
 
+        data["ID"] = str(gen_id_from_yaml(data, data["Name"]))
+
     except Exception as e:
         log.error("%r adding project %s", e, file)
         log.error("Data:\n%s", pprint.pformat(data))

src/webapp/rg_reader.py

@@ -18,12 +18,11 @@
 from pathlib import Path
 import yaml
 
-
 # thanks stackoverflow
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from webapp.common import ensure_list, to_xml, Filters, load_yaml_file
+from webapp.common import ensure_list, to_xml, Filters, load_yaml_file, gen_id_from_yaml
 from webapp.contacts_reader import get_contacts_data
 from webapp.topology import CommonData, Topology
 
@@ -72,7 +71,8 @@ def get_topology(indir="../topology", contacts_data=None, strict=False):
 
     for facility_path in root.glob("*/FACILITY.yaml"):
         name = facility_path.parts[-2]
-        id_ = load_yaml_file(facility_path)["ID"]
+        facility_data = load_yaml_file(facility_path)
+        id_ = gen_id_from_yaml(facility_data or {}, name)
         topology.add_facility(name, id_)
     for site_path in root.glob("*/*/SITE.yaml"):
         facility, name = site_path.parts[-3:-1]
@@ -85,7 +85,7 @@ def get_topology(indir="../topology", contacts_data=None, strict=False):
                 log.error(skip_msg)
                 continue
         site_info = load_yaml_file(site_path)
-        id_ = site_info["ID"]
+        id_ = gen_id_from_yaml(site_info, name)
         topology.add_site(facility, name, id_, site_info)
     for yaml_path in root.glob("*/*/*.yaml"):
         facility, site, name = yaml_path.parts[-3:]

src/webapp/topology.py

@@ -8,7 +8,7 @@
 import icalendar
 
 from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml,\
-    is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER
+    is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER, gen_id_from_yaml
 from .contacts_reader import ContactsData, User
 from .exceptions import DataError
 
@@ -34,6 +34,13 @@ def __init__(self, contacts: ContactsData, service_types: Dict, support_centers:
         self.service_types = service_types
         self.support_centers = support_centers
 
+        # Auto-generate IDs for any services and support centers that don't have them
+        for key, val in self.service_types.items():
+            self.service_types[key] = val or gen_id_from_yaml({}, key)
+
+        for key, val in self.support_centers.items():
+            val['ID'] = gen_id_from_yaml(val, key)
+
 
 class Facility(object):
     def __init__(self, name: str, id: int):
@@ -112,6 +119,7 @@ def __init__(self, name: str, yaml_data: ParsedYaml, common_data: CommonData):
             self.services = []
         self.service_names = [n["Name"] for n in self.services if "Name" in n]
         self.data = yaml_data
+        self.data["ID"] = gen_id_from_yaml(self.data, self.name)
         if is_null(yaml_data, "FQDN"):
             raise ValueError(f"Resource {name} does not have an FQDN")
         self.fqdn = self.data["FQDN"]
@@ -420,7 +428,7 @@ def get_tree(self, authorized=False, filters: Filters = None) -> Optional[Ordere
 
     @property
     def id(self):
-        return self.data["GroupID"]
+        return gen_id_from_yaml(self.data, self.name, "GroupID")
 
     @property
     def key(self):
@@ -439,6 +447,7 @@ def _expand_rg(self) -> OrderedDict:
                                        "SupportCenter", "GroupDescription", "IsCCStar"])
         new_rg.update({"Disable": False})
         new_rg.update(self.data)
+        new_rg['GroupID'] = gen_id_from_yaml(self.data, self.name, "GroupID")
 
         new_rg["Facility"] = self.site.facility.get_tree()
         new_rg["Site"] = self.site.get_tree()

src/webapp/vos_data.py

@@ -4,7 +4,7 @@
 from logging import getLogger
 from typing import Dict, List, Optional
 
-from .common import Filters, ParsedYaml, VOSUMMARY_SCHEMA_URL, is_null, expand_attr_list, order_dict, escape
+from .common import Filters, ParsedYaml, VOSUMMARY_SCHEMA_URL, is_null, expand_attr_list, order_dict, escape, gen_id_from_yaml
 from .data_federation import StashCache
 from .contacts_reader import ContactsData
 
@@ -23,6 +23,7 @@ def get_vo_id_to_name(self) -> Dict[str, str]:
         return {self.vos[name]["ID"]: name for name in self.vos}
 
     def add_vo(self, vo_name: str, vo_data: ParsedYaml):
+        vo_data["ID"] = gen_id_from_yaml(vo_data, vo_name)
         self.vos[vo_name] = vo_data
         stashcache_data = vo_data.get('DataFederations', {}).get('StashCache')
         if stashcache_data:
@@ -133,6 +134,7 @@ def _expand_vo(self, name: str, authorized: bool, filters: Filters) -> Optional[
         if not is_null(vo, "ParentVO"):
             parentvo = OrderedDict.fromkeys(["ID", "Name"])
             parentvo.update(vo["ParentVO"])
+            parentvo['ID'] = gen_id_from_yaml(parentvo, parentvo["Name"])
             new_vo["ParentVO"] = parentvo
 
         if not is_null(vo, "Credentials"):