uuid_util.py

#!/usr/bin/env python3

# This script can be used to verify or replace UUIDs generated by file-copier
# UUID should be generated by using project's domain name, node type of 'file' and file's file_location (new) or MD5
# (legacy used before UBC01 and UBC02) as signature

import argparse
import csv
import os

from bento.common.utils import LOG_PREFIX, APP_NAME, get_logger, get_uuid

if LOG_PREFIX not in os.environ:
    os.environ[LOG_PREFIX] = 'UUID_util'

os.environ[APP_NAME] = 'UUID_util'
log = get_logger('UUID_util')


def get_new_manifest_name(manifest):
    folder = os.path.dirname(manifest)
    org_name, ext = os.path.basename(manifest).split('.')
    new_name = f"{org_name}_corrected.{ext}"
    return os.path.join(folder, new_name)


def process_file(file_obj, signature_column, uuid_column, domain, indexd_mode):
    file_name = file_obj.name
    log.info(f"Processing {file_name}")
    data = []

    reader = csv.DictReader(file_obj, delimiter='\t')
    total = 0
    succeeded = 0
    failed = 0
    for obj in reader:
        total += 1
        fname = os.path.basename(obj['url']) if indexd_mode else obj['file_name']
        log.info(f"Comparing {fname}")
        signature = obj.get(signature_column)
        current_uuid = obj.get(uuid_column)
        if indexd_mode:
            guid_prefix, current_uuid = current_uuid.split('/')
        new_uuid = get_uuid(domain, 'file', signature)
        if current_uuid != new_uuid:
            log.error(f"UUIDs don't match! current: {current_uuid}, new: {new_uuid}")
            failed += 1
            if indexd_mode:
                obj[uuid_column] = f'{guid_prefix}/{new_uuid}'
            else:
                obj[uuid_column] = new_uuid
        else:
            log.info(f"UUID match!")
            succeeded += 1

        data.append(obj)

    log.info(f'{total} rows compared')
    log.info(f"{succeeded} match, {failed} don't match")

    if failed > 0:
        new_manifest = get_new_manifest_name(file_name)
        log.info(f"Saving corrected manifest to: {new_manifest}")
        with open(new_manifest, 'w') as of:
            writer = csv.DictWriter(of, fieldnames=reader.fieldnames, delimiter='\t')
            writer.writeheader()
            for obj in data:
                writer.writerow(obj)
        log.info("Done")
    log.info("=" * 70)
    return total, succeeded, failed


def main():
    valid_projects = {
        'ICDC': 'caninecommons.cancer.gov',
        'CTDC': 'trialcommons.cancer.gov',
        'BENTO': 'bento-tools.org'
    }
    parser = argparse.ArgumentParser(description='Script to validate or replace UUID generated by file-copier')
    parser.add_argument('manifests', type=argparse.FileType('r'),
                        help='Neo4j manifest file generated by file-copier',
                        nargs='+'
                        )
    parser.add_argument('-p', '--project', help='Project name', choices=valid_projects.keys(), default='ICDC')
    parser.add_argument('-u', '--uuid-column', help='column that contains UUID', default='uuid')
    parser.add_argument('-s', '--signature-column', help='column that contains file signature new files should use '
                                                         'file_location, legacy files (before UBC01) should use '
                                                         'md5sum', default='file_location')
    parser.add_argument('-i', '--indexd-mode', help='IndexD Mode', action='store_true')

    args = parser.parse_args()

    domain = valid_projects[args.project]
    uuid_column = args.uuid_column
    signature_column = args.signature_column
    if args.indexd_mode:
        if uuid_column == 'uuid':
            uuid_column = 'GUID'
        if signature_column == 'file_location':
            signature_column = 'url'

    if args.indexd_mode:
        log.info('IndexD mode on, will process IndexD GUID properly')
    log.info(f'Domain name: {domain}')
    log.info(f'UUID column name: {uuid_column}, Signature column name: {signature_column}')

    total = 0
    succeeded = 0
    failed = 0
    for file_obj in args.manifests:
        tot, suc, fai = process_file(file_obj, signature_column, uuid_column, domain, args.indexd_mode)
        total += tot
        succeeded += suc
        failed += fai

    log.info(f'Total: {total} rows compared')
    log.info(f"{succeeded} match, {failed} don't match")


if __name__ == '__main__':
    main()