From 998ffc3fdd8afb19caf7b2a4fe8d9dcdae86a6fb Mon Sep 17 00:00:00 2001 From: William Swanson Date: Thu, 7 Dec 2023 11:36:54 -0600 Subject: [PATCH] project_groups_setup refactoring --- project_group_setup.py | 193 +++++++++++++++++++++++++---------------- 1 file changed, 120 insertions(+), 73 deletions(-) diff --git a/project_group_setup.py b/project_group_setup.py index f794d9d..3a6ce83 100644 --- a/project_group_setup.py +++ b/project_group_setup.py @@ -11,7 +11,7 @@ SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" OSG_CO_ID = 8 -CLUSTER_ID = 10 +UNIX_CLUSTER_ID = 10 LDAP_TARGET_ID = 9 MINTIMEOUT = 5 MAXTIMEOUT = 625 @@ -32,7 +32,7 @@ OPTIONS: -u USER[:PASS] specify USER and optionally PASS on command line -c OSG_CO_ID specify OSG CO ID (default = {OSG_CO_ID}) - -g CLUSTER_ID specify UNIX Cluster ID (default = {CLUSTER_ID}) + -g CLUSTER_ID specify UNIX Cluster ID (default = {UNIX_CLUSTER_ID}) -l LDAP_TARGET specify LDAP Provsion ID (defult = {LDAP_TARGET_ID}) -d passfd specify open fd to read PASS -f passfile specify path to file to open and read PASS @@ -63,7 +63,7 @@ class Options: endpoint = ENDPOINT user = "co_8.william_test" osg_co_id = OSG_CO_ID - unix_id = CLUSTER_ID + ucid = UNIX_CLUSTER_ID provision_target = LDAP_TARGET_ID outfile = None authstr = None @@ -129,7 +129,7 @@ def call_api3(method, target, data, **kw): currentTimeout *= TIMEOUTMULTIPLE else: sys.exit( - f"Exception raised after maximum timeout {options.max_timeout} seconds reached. " + f"Exception raised after maximum retrys and timeout {options.max_timeout} seconds reached. " + f"Exception reason: {exception.reason}.\n Request: {req.full_url}" ) @@ -155,22 +155,32 @@ def get_co_person_identifiers(pid): return call_api("identifiers.json", copersonid=pid) +def get_unix_cluster_groups(ucid): + return call_api("unix_cluster/unix_cluster_groups.json", unix_cluster_id=ucid) + + +def get_unix_cluster_groups_ids(ucid): + unix_cluster_groups = get_unix_cluster_groups(ucid) + return set(group["CoGroupId"] for group in unix_cluster_groups["UnixClusterGroups"]) + + def get_datalist(data, listname): return data[listname] if data else [] -def identifier_index(id_list, id_type): +def identifier_from_list(id_list, id_type): id_type_list = [id["Type"] for id in id_list] try: - return id_type_list.index(id_type) + id_index = id_type_list.index(id_type) + return id_list[id_index]["Identifier"] except ValueError: - return -1 + return None def identifier_matches(id_list, id_type, regex_string): pattern = re.compile(regex_string) - index = identifier_index(id_list, id_type) - return (index != -1) & (pattern.match(id_list[index]["Identifier"]) is not None) + value = identifier_from_list(id_list, id_type) + return (value is not None) & (pattern.match(value) is not None) def add_identifier_to_group(gid, type, identifier_name): @@ -190,6 +200,19 @@ def add_identifier_to_group(gid, type, identifier_name): return call_api3(POST, "identifiers.json", data) +def add_unix_cluster_group(gid): + request = { + "RequestType": "UnixClusterGroups", + "Version": "1.0", + "UnixClusterGroups": [{"Version": "1.0", "UnixClusterId": options.ucid, "CoGroupId": gid}], + } + call_api3(POST, "unix_cluster/unix_cluster_groups.json", request) + + +def ldap_provision_group(gid): + call_api2(POST, f"co_provisioning_targets/provision/{options.provision_target}/cogroupid:{gid}.json") + + def parse_options(args): try: ops, args = getopt.getopt(args, "u:c:d:f:e:o:t:T:h") @@ -210,7 +233,7 @@ def parse_options(args): if op == "-c": options.osg_co_id = int(arg) if op == "-g": - options.unix_id = int(arg) + options.ucid = int(arg) if op == "-l": options.provision_target = int(arg) if op == "-d": @@ -230,78 +253,102 @@ def parse_options(args): options.authstr = mkauthstr(user, passwd) -def main(args): - parse_options(args) - - # get groups with 'OSPool project name' matching "Yes-*" that don't have a 'OSG GID' - +def get_projects_and_highest_osggid(): co_groups = get_osg_co_groups()["CoGroups"] + project_groups = [] highest_osggid = 0 - project_groups = set() - projects_to_assign_identifiers = [] - - unix_cluster_groups = call_api("unix_cluster/unix_cluster_groups.json", unix_cluster_id=options.unix_id) - clustered_group_ids = set(group["CoGroupId"] for group in unix_cluster_groups["UnixClusterGroups"]) - projects_needing_cluster_groups = set() for group in co_groups: gid = group["Id"] - identifier_data = get_co_group_identifiers(gid) - - if identifier_data: - identifier_list = identifier_data["Identifiers"] - - project_id_index = identifier_index(identifier_list, "ospoolproject") - if project_id_index != -1: - project_id = str(identifier_list[project_id_index]["Identifier"]) - if re.compile(OSPOOL_PROJECT_PREFIX_STR + "*").match(project_id) is not None: - project_groups.add(gid) - - osggid_index = identifier_index(identifier_list, "osggid") - if osggid_index != -1: - highest_osggid = max(highest_osggid, int(identifier_list[osggid_index]["Identifier"])) - - if gid in project_groups: - if osggid_index == -1: - project_name = project_id.replace(OSPOOL_PROJECT_PREFIX_STR, "", 1).lower() - project_data = ( - gid, - project_name, - ) - projects_to_assign_identifiers.append(project_data) - - if not gid in clustered_group_ids: - projects_needing_cluster_groups.add(gid) - - for gid, project_name in projects_to_assign_identifiers: - # for each, set a 'OSG GID' starting from 200000 and a 'OSG Group Name' that is the group name - osggid_to_assign = max(highest_osggid + 1, options.project_gid_startval) - highest_osggid = osggid_to_assign - add_identifier_to_group(gid, type="osggid", identifier_name=osggid_to_assign) - add_identifier_to_group(gid, type="osggroup", identifier_name=project_name) - print(f"project {project_name}: added osggid {osggid_to_assign} and osg project name {project_name}") - - for gid in projects_needing_cluster_groups: - request = { - "RequestType": "UnixClusterGroups", - "Version": "1.0", - "UnixClusterGroups": [{"Version": "1.0", "UnixClusterId": options.unix_id, "CoGroupId": gid}], - } - call_api3( - POST, - "unix_cluster/unix_cluster_groups.json", - request, - ) - print(f"project group {gid}: added UNIX Cluster Group") - - for project_gid in project_groups: - #Provision all project groups - call_api2(POST, f"co_provisioning_targets/provision/{options.provision_target}/cogroupid:{project_gid}.json") + name = group["Name"] + try: + identifier_list = get_co_group_identifiers(gid)["Identifiers"] + + # Get the value of the osggid identifier, if this group has one + osggid = identifier_from_list(identifier_list, "osggid") + # If this group has a osggid, keep a hold of the highest one we've seen so far + if osggid is not None: + highest_osggid = max(highest_osggid, int(osggid)) + + # If this group has a ospoolproject id, and it starts with "Yes-", it's a project + if identifier_matches(identifier_list, "ospoolproject", (OSPOOL_PROJECT_PREFIX_STR + "*")): + # Add a tuple of the relavent data for this project to the project set + data = tuple((gid, name, identifier_list)) + project_groups.append(data) + except TypeError: + pass + return (project_groups, highest_osggid) + + +def get_projects_to_setup(project_groups): + projects_to_assign_identifiers = [] + + # CO Groups associated with a UNIX Cluster Group + clustered_group_ids = get_unix_cluster_groups_ids(options.ucid) + projects_needing_cluster_groups = [] + + for project_gid, project_name, project_identifiers in project_groups: + # If this project doesn't have an osggid already assigned to it... + if identifier_from_list(project_identifiers, "osggid") is None: + # Prep the project to have the proper identifiers added to it + data = tuple((project_gid, project_name, project_identifiers)) + projects_to_assign_identifiers.append(data) + + # If this project doesn't have a UNIX Cluster Group associated with it... + if not project_gid in clustered_group_ids: + # Prep it to have one made for it and to be provisioned in LDAP + projects_needing_cluster_groups.append(project_gid) + + return (projects_to_assign_identifiers, projects_needing_cluster_groups) + + +def add_missing_group_identifier(gid, id_list, id_type, value): + # If the group doesn't already have an id of this type... + if identifier_from_list(id_list, id_type) is None: + add_identifier_to_group(gid, id_type, value) + print(f"project {gid}: aded id {value} of type {id_type}") + + +def assign_identifiers_to_group(gid, id_list, project_name, osggid_to_assign): + # Add an osggid and an osggoup identifier to this group, if it dones't have them already + add_missing_group_identifier(gid, id_list, "osggid", osggid_to_assign) + add_missing_group_identifier(gid, id_list, "osggroup", project_name) + + +def assign_identifiers(project_list, highest_osggid): + highest = highest_osggid + for gid, name, id_list in project_list: + # Project name identifier is the CO Group name in lower case + project_name = name.lower() + + # Determine what osggid to assign this project, + # based on the starting range and the highest osggid seen in existing groups + osggid_to_assign = max(highest + 1, options.project_gid_startval) + highest = osggid_to_assign + + assign_identifiers_to_group(gid, id_list, project_name, osggid_to_assign) + + +def create_unix_cluster_groups(gid_list): + for gid in gid_list: + add_unix_cluster_group(gid) + ldap_provision_group(gid) + print(f"project group {gid}: added UNIX Cluster Group and provisioned") + + +def main(args): + parse_options(args) + + project_groups, highest_osggid = get_projects_and_highest_osggid() + projects_to_assign_identifiers, projects_needing_cluster_groups = get_projects_to_setup(project_groups) + + assign_identifiers(projects_to_assign_identifiers, highest_osggid) + create_unix_cluster_groups(projects_needing_cluster_groups) if __name__ == "__main__": try: main(sys.argv[1:]) - except urllib.error.HTTPError as e: + except OSError as e: print(e, file=sys.stderr) sys.exit(1)