From 31d4f00b5378e370af17e563c059c366d62a5385 Mon Sep 17 00:00:00 2001 From: Dennis Lee Date: Tue, 5 Dec 2023 22:15:15 +0100 Subject: [PATCH 1/3] Added script find and convert legacydn to comma DN --- .../scripts/legacydn_converter.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docker/CMSRucioClient/scripts/legacydn_converter.py diff --git a/docker/CMSRucioClient/scripts/legacydn_converter.py b/docker/CMSRucioClient/scripts/legacydn_converter.py new file mode 100644 index 00000000..c709e897 --- /dev/null +++ b/docker/CMSRucioClient/scripts/legacydn_converter.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +import os + +from rucio.client.accountclient import AccountClient + +PROXY = os.getenv('X509_USER_PROXY') + +client = AccountClient() + +def rfc2253dn(legacy_dn: str) -> str: + """Converts legacy slash DB to comma separated DN""" + if not legacy_dn.startswith('/'): + return legacy_dn + + legacy_dn = legacy_dn.replace(',', r'\,') + parts = legacy_dn.split('/')[1:] + + return ','.join(parts) + + +def convert_identities(account_type: str, dry_run: bool=True): + """Fetches and converts identities to rfc2253dn""" + accounts = client.list_accounts(account_type=account_type) + + for account in accounts: + identities = [i for i in client.list_identities(account["account"]) if i['type'] == 'X509'] + for identity in identities: + new_dn = rfc2253dn(identity["identity"]) + print(f"old_dn: {identity['identity']} new_dn: {new_dn}") + + if not dry_run: + # add identities + # client.add_identity(account['account'], new_dn, 'X509', identity['email']) + + # delete old identity(?) + # client.del_identity(account['account], identity['identity'], 'X509') + pass + + +def main(): + # convert identities + convert_identities("GROUP", dry_run=True) + convert_identities("SERVICE", dry_run=True) + + + +if __name__ == "__main__": + main() From ffae13d75b3b906fadc51702c4e995a52c7c5a06 Mon Sep 17 00:00:00 2001 From: Dennis Lee Date: Thu, 7 Dec 2023 11:46:27 -0600 Subject: [PATCH 2/3] Updated rfc2253 converter to be in correct order --- .../scripts/legacydn_converter.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/docker/CMSRucioClient/scripts/legacydn_converter.py b/docker/CMSRucioClient/scripts/legacydn_converter.py index c709e897..6bbbb6c9 100644 --- a/docker/CMSRucioClient/scripts/legacydn_converter.py +++ b/docker/CMSRucioClient/scripts/legacydn_converter.py @@ -15,7 +15,20 @@ def rfc2253dn(legacy_dn: str) -> str: legacy_dn = legacy_dn.replace(',', r'\,') parts = legacy_dn.split('/')[1:] - return ','.join(parts) + return ','.join(parts[::-1]) + + +def get_identities_to_add(identities): + # get only legacy dns and convert them to new dn + dns_to_check = [rfc2253dn(i['identity']) for i in identities if "/" in i['identity']] + + identities_to_add = [] + for identity in identities: + if not identity['identity'] in dns_to_check: + # add identity + identities_to_add.append(identity) + + return identities_to_add def convert_identities(account_type: str, dry_run: bool=True): @@ -23,11 +36,13 @@ def convert_identities(account_type: str, dry_run: bool=True): accounts = client.list_accounts(account_type=account_type) for account in accounts: + # filter only X509 identities identities = [i for i in client.list_identities(account["account"]) if i['type'] == 'X509'] - for identity in identities: + identities_to_add = get_identities_to_add(identities) + for identity in identities_to_add: new_dn = rfc2253dn(identity["identity"]) - print(f"old_dn: {identity['identity']} new_dn: {new_dn}") + print(f"adding identity. account: {account['account']}, new_dn: {new_dn}, type: X509, email: {identity['email']}") if not dry_run: # add identities # client.add_identity(account['account'], new_dn, 'X509', identity['email']) From 8b77fcbf628338b65331a492dbe0104a634b2a9a Mon Sep 17 00:00:00 2001 From: Dennis Lee Date: Mon, 22 Jan 2024 16:24:55 -0600 Subject: [PATCH 3/3] Added argparsing and sorting the DN attribute order --- .../scripts/legacydn_converter.py | 143 +++++++++++++++--- 1 file changed, 118 insertions(+), 25 deletions(-) diff --git a/docker/CMSRucioClient/scripts/legacydn_converter.py b/docker/CMSRucioClient/scripts/legacydn_converter.py index 6bbbb6c9..21e650f4 100644 --- a/docker/CMSRucioClient/scripts/legacydn_converter.py +++ b/docker/CMSRucioClient/scripts/legacydn_converter.py @@ -1,62 +1,155 @@ #!/usr/bin/env python3 + +"""legacydn_converter.py + +A script that retrieves identities from a Rucio instance, converts it to +an RFC2253 compliant DN, then adds it back to the account identity. +""" + +import argparse +from itertools import groupby import os +from pprint import pprint + from rucio.client.accountclient import AccountClient PROXY = os.getenv('X509_USER_PROXY') +RFC_ATTRIBUTE_ORDER = ["CN", "L", "ST", "O", "OU", "C", "STREET", "DC", "UID"] + client = AccountClient() -def rfc2253dn(legacy_dn: str) -> str: + +def get_attribute(attribute: str) -> str: + """Extracts attribute key""" + try: + return attribute[:attribute.index("=")] + except ValueError: + # print(f"element lacks attribute, {attribute}, {dn}") + return "" + + +def rfc2253dn(legacy_dn: str, verbose: bool = False) -> str: """Converts legacy slash DB to comma separated DN""" if not legacy_dn.startswith('/'): + if verbose: + print(f'DN does not start with /: {legacy_dn}') return legacy_dn + # replace commas with an escape character legacy_dn = legacy_dn.replace(',', r'\,') parts = legacy_dn.split('/')[1:] - return ','.join(parts[::-1]) + # parts are reversed. See https://datatracker.ietf.org/doc/html/rfc2253#section-2.3 + elements = parts[::-1] + attributes = [get_attribute(e) for e in elements] + # skip any DNs who have an element without an attribute + if "" in attributes: + return "" -def get_identities_to_add(identities): - # get only legacy dns and convert them to new dn - dns_to_check = [rfc2253dn(i['identity']) for i in identities if "/" in i['identity']] + if verbose: + print(f"Attributes: {attributes}") + print(f"Expected order: {RFC_ATTRIBUTE_ORDER}") - identities_to_add = [] - for identity in identities: - if not identity['identity'] in dns_to_check: - # add identity - identities_to_add.append(identity) + indexes = [] + for attr in attributes: + try: + indexes.append(RFC_ATTRIBUTE_ORDER.index(attr)) + except ValueError: + # skips any DNs that don't have a attribute in the RFC_ATTRIBUTE_ORDER + pass - return identities_to_add + # sort existing attributes based on expected attribute order + result = [a[0] for a in sorted(zip(elements, attributes, indexes), key=lambda x: x[2])] + if verbose: + print(f"original: {legacy_dn}\nindexes: {indexes}\nconverted: {','.join(result)}") -def convert_identities(account_type: str, dry_run: bool=True): + return ','.join(result) + + +def convert_identities(account_type: str, dry_run: bool=True, verbose: bool=False): """Fetches and converts identities to rfc2253dn""" accounts = client.list_accounts(account_type=account_type) for account in accounts: # filter only X509 identities - identities = [i for i in client.list_identities(account["account"]) if i['type'] == 'X509'] - identities_to_add = get_identities_to_add(identities) - for identity in identities_to_add: - new_dn = rfc2253dn(identity["identity"]) + print(f"\n##### Checking account: {account['account']} #####") - print(f"adding identity. account: {account['account']}, new_dn: {new_dn}, type: X509, email: {identity['email']}") - if not dry_run: - # add identities - # client.add_identity(account['account'], new_dn, 'X509', identity['email']) + # get only X509 identities + identities = [i for i in client.list_identities(account["account"]) if i['type'] == 'X509'] + print("Current identities\n-------------------") + pprint(identities) + + # get a list of only the identities for an account + ids = [i['identity'] for i in identities] + + added = [] + existing = [] + failed = [] + for identity in identities: + if not identity['identity'].startswith('/'): + continue + + new_dn = rfc2253dn(identity["identity"], verbose) + + if not new_dn: + if verbose: + print(f"DN was invalid {identity['identity']}, will skip") + continue + + if new_dn in ids: + idx = ids.index(identity['identity']) + if verbose: + print(f"Identity exists, skipping:\n{new_dn}\n{ids[idx]}") + if not ids[idx].startswith('/'): + existing.append(new_dn) + continue + + + print(f"{'DRY RUN ' if dry_run else ''}Will add identity for account {account['account']}: '{new_dn}', type: X509, email: {identity['email']}") + + print("Adding to Rucio") + # add identities + try: + if not dry_run: + client.add_identity(account['account'], new_dn, 'X509', identity['email']) + except Exception as e: + print(f"Could not add identity: {new_dn}, Exception: {e}") + failed.append(identity) + continue # delete old identity(?) # client.del_identity(account['account], identity['identity'], 'X509') - pass + + print(f"Successfully Added Identity, {'DRY RUN' if dry_run else ''}") + added.append(new_dn) + + print(f"{len(existing)} identities already exist") + print(f"{len(added)} identiites added") + print(f"{len(failed)} identities failed to be added") + print(f"##### Account {account['account']} Done #####") -def main(): - # convert identities - convert_identities("GROUP", dry_run=True) - convert_identities("SERVICE", dry_run=True) +def main(): + parser = argparse.ArgumentParser( + description='LegacyDN Converter', + epilog='Converts legacy DN to rfc2253 filtered by account type') + parser.add_argument('--account_type', + action='store', + required=True, + choices=['GROUP', 'SERVICE']) + parser.add_argument('--dry_run', + action='store_true') + parser.add_argument('--verbose', + action='store_true') + + args = parser.parse_args() + + convert_identities(args.account_type, dry_run=args.dry_run, verbose=args.verbose) if __name__ == "__main__":