diff --git a/src/tests/verify_resources.py b/src/tests/verify_resources.py index bc79dd70b..a7de19e74 100755 --- a/src/tests/verify_resources.py +++ b/src/tests/verify_resources.py @@ -369,27 +369,30 @@ def test_7_fqdn_unique(rgs, rgfns): def test_8_res_ids(rgs, rgfns): - # Check that resources/resource groups have a numeric ID/GroupID + # Check that resources/resource groups have a unique, numeric ID/GroupID + # in the case that an ID is manually assigned errors = 0 ridres = autodict() gidrgs = autodict() for rg,rgfn in zip(rgs,rgfns): - if not isinstance(rg.get('GroupID'), int): + group_id = rg.get('GroupID') + if group_id is not None and not isinstance(group_id, int): print_emsg_once('ResGrpID') print("ERROR: Resource Group missing numeric GroupID: '%s'" % rgfn) errors += 1 - else: + elif group_id: gidrgs[rg['GroupID']] += [rgfn] for resname,res in sorted(rg['Resources'].items()): - if not isinstance(res.get('ID'), int): + resource_id = res.get('ID') + if resource_id is not None and not isinstance(resource_id, int): print_emsg_once('ResID') print("ERROR: Resource '%s' missing numeric ID in '%s'" % (resname, rgfn)) errors += 1 - else: + elif resource_id: ridres[res['ID']] += [(rgfn, resname)] for gid,rglist in sorted(gidrgs.items()): diff --git a/src/webapp/common.py b/src/webapp/common.py index 8e0ed8957..8a9c4fb08 100644 --- a/src/webapp/common.py +++ b/src/webapp/common.py @@ -222,7 +222,6 @@ def trim_space(s: str) -> str: ret = re.sub(r"(?m)^[ \t]+", "", ret) return ret - def run_git_cmd(cmd: List, dir=None, git_dir=None, ssh_key=None) -> bool: """ Run git command, optionally specifying ssh key and/or git dirs @@ -287,9 +286,19 @@ def git_clone_or_fetch_mirror(repo, git_dir, ssh_key=None) -> bool: return ok -def gen_id(instr: AnyStr, digits, minimum=1, hashfn=hashlib.md5) -> int: +def gen_id_from_yaml(data: dict, alternate_name: str, id_key = "ID", mod = 2 ** 31 - 1, minimum = 1, hashfn=hashlib.md5) -> int: + """ + Given a yaml object, return its existing ID if an ID is present, or generate a new ID for the object + based on the md5sum of an alternate string value (usually the key of the object in its parent dictionary) + """ + return data.get(id_key) or gen_id(alternate_name, mod, minimum, hashfn) + +def gen_id(instr: AnyStr, mod = 2 ** 31 - 1, minimum=1, hashfn=hashlib.md5) -> int: + """ + Convert a string to its integer md5sum, used to autogenerate unique IDs for entities where + not otherwise specified + """ instr_b = instr if isinstance(instr, bytes) else instr.encode("utf-8", "surrogateescape") - mod = (10 ** digits) - minimum return minimum + (int(hashfn(instr_b).hexdigest(), 16) % mod) diff --git a/src/webapp/project_reader.py b/src/webapp/project_reader.py index e50399c76..29e4b3fea 100755 --- a/src/webapp/project_reader.py +++ b/src/webapp/project_reader.py @@ -13,7 +13,7 @@ if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from webapp.common import load_yaml_file, to_xml, is_null +from webapp.common import load_yaml_file, to_xml, is_null, gen_id_from_yaml from webapp.vo_reader import get_vos_data from webapp.vos_data import VOsData @@ -64,9 +64,6 @@ def get_one_project(file: str, campus_grid_ids: Dict, vos_data: VOsData) -> Dict if 'ResourceAllocations' in data: resource_allocations = [get_resource_allocation(ra, idx) for idx, ra in enumerate(data['ResourceAllocations'])] data['ResourceAllocations'] = {"ResourceAllocation": resource_allocations} - if 'ID' not in data: - del project['ID'] - name_from_filename = os.path.basename(file)[:-5] # strip '.yaml' if not is_null(data, 'Name'): if data['Name'] != name_from_filename: @@ -74,6 +71,8 @@ def get_one_project(file: str, campus_grid_ids: Dict, vos_data: VOsData) -> Dict else: data['Name'] = name_from_filename + data["ID"] = str(gen_id_from_yaml(data, data["Name"])) + except Exception as e: log.error("%r adding project %s", e, file) log.error("Data:\n%s", pprint.pformat(data)) diff --git a/src/webapp/rg_reader.py b/src/webapp/rg_reader.py index 4e8182b4d..1d74e8c99 100755 --- a/src/webapp/rg_reader.py +++ b/src/webapp/rg_reader.py @@ -18,12 +18,11 @@ from pathlib import Path import yaml - # thanks stackoverflow if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from webapp.common import ensure_list, to_xml, Filters, load_yaml_file +from webapp.common import ensure_list, to_xml, Filters, load_yaml_file, gen_id_from_yaml from webapp.contacts_reader import get_contacts_data from webapp.topology import CommonData, Topology @@ -72,7 +71,8 @@ def get_topology(indir="../topology", contacts_data=None, strict=False): for facility_path in root.glob("*/FACILITY.yaml"): name = facility_path.parts[-2] - id_ = load_yaml_file(facility_path)["ID"] + facility_data = load_yaml_file(facility_path) + id_ = gen_id_from_yaml(facility_data or {}, name) topology.add_facility(name, id_) for site_path in root.glob("*/*/SITE.yaml"): facility, name = site_path.parts[-3:-1] @@ -85,7 +85,7 @@ def get_topology(indir="../topology", contacts_data=None, strict=False): log.error(skip_msg) continue site_info = load_yaml_file(site_path) - id_ = site_info["ID"] + id_ = gen_id_from_yaml(site_info, name) topology.add_site(facility, name, id_, site_info) for yaml_path in root.glob("*/*/*.yaml"): facility, site, name = yaml_path.parts[-3:] diff --git a/src/webapp/topology.py b/src/webapp/topology.py index dee616829..6630baff6 100644 --- a/src/webapp/topology.py +++ b/src/webapp/topology.py @@ -8,7 +8,7 @@ import icalendar from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml,\ - is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER + is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER, gen_id_from_yaml from .contacts_reader import ContactsData, User from .exceptions import DataError @@ -34,6 +34,13 @@ def __init__(self, contacts: ContactsData, service_types: Dict, support_centers: self.service_types = service_types self.support_centers = support_centers + # Auto-generate IDs for any services and support centers that don't have them + for key, val in self.service_types.items(): + self.service_types[key] = val or gen_id_from_yaml({}, key) + + for key, val in self.support_centers.items(): + val['ID'] = gen_id_from_yaml(val, key) + class Facility(object): def __init__(self, name: str, id: int): @@ -112,6 +119,7 @@ def __init__(self, name: str, yaml_data: ParsedYaml, common_data: CommonData): self.services = [] self.service_names = [n["Name"] for n in self.services if "Name" in n] self.data = yaml_data + self.data["ID"] = gen_id_from_yaml(self.data, self.name) if is_null(yaml_data, "FQDN"): raise ValueError(f"Resource {name} does not have an FQDN") self.fqdn = self.data["FQDN"] @@ -420,7 +428,7 @@ def get_tree(self, authorized=False, filters: Filters = None) -> Optional[Ordere @property def id(self): - return self.data["GroupID"] + return gen_id_from_yaml(self.data, self.name, "GroupID") @property def key(self): @@ -439,6 +447,7 @@ def _expand_rg(self) -> OrderedDict: "SupportCenter", "GroupDescription", "IsCCStar"]) new_rg.update({"Disable": False}) new_rg.update(self.data) + new_rg['GroupID'] = gen_id_from_yaml(self.data, self.name, "GroupID") new_rg["Facility"] = self.site.facility.get_tree() new_rg["Site"] = self.site.get_tree() diff --git a/src/webapp/vos_data.py b/src/webapp/vos_data.py index 2804eeb76..ed077d670 100644 --- a/src/webapp/vos_data.py +++ b/src/webapp/vos_data.py @@ -4,7 +4,7 @@ from logging import getLogger from typing import Dict, List, Optional -from .common import Filters, ParsedYaml, VOSUMMARY_SCHEMA_URL, is_null, expand_attr_list, order_dict, escape +from .common import Filters, ParsedYaml, VOSUMMARY_SCHEMA_URL, is_null, expand_attr_list, order_dict, escape, gen_id_from_yaml from .data_federation import StashCache from .contacts_reader import ContactsData @@ -23,6 +23,7 @@ def get_vo_id_to_name(self) -> Dict[str, str]: return {self.vos[name]["ID"]: name for name in self.vos} def add_vo(self, vo_name: str, vo_data: ParsedYaml): + vo_data["ID"] = gen_id_from_yaml(vo_data, vo_name) self.vos[vo_name] = vo_data stashcache_data = vo_data.get('DataFederations', {}).get('StashCache') if stashcache_data: @@ -133,6 +134,7 @@ def _expand_vo(self, name: str, authorized: bool, filters: Filters) -> Optional[ if not is_null(vo, "ParentVO"): parentvo = OrderedDict.fromkeys(["ID", "Name"]) parentvo.update(vo["ParentVO"]) + parentvo['ID'] = gen_id_from_yaml(parentvo, parentvo["Name"]) new_vo["ParentVO"] = parentvo if not is_null(vo, "Credentials"):