From 34b27cf55c89eade8fb298b0dc8a021168e00fa9 Mon Sep 17 00:00:00 2001 From: idelcano Date: Fri, 15 Feb 2019 15:13:58 +0100 Subject: [PATCH 1/9] Added --remove-object option --- .../change_user_groups.py | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/DHIS2/metadata_manipulation/change_user_groups.py b/DHIS2/metadata_manipulation/change_user_groups.py index 2d23794..b4f7019 100755 --- a/DHIS2/metadata_manipulation/change_user_groups.py +++ b/DHIS2/metadata_manipulation/change_user_groups.py @@ -36,11 +36,20 @@ def main(): args = get_args() + + validate_args(args) + dhis_objects = [item for item in args.apply if item not in args.exclude] - if args.replace_objects: - replaceable_objects = [item for item in args.replace_objects] + + if args.remove_objects: + modifiable_objects = [item for item in args.remove_objects] + action = remove_objects + elif args.replace_objects: + modifiable_objects = [item for item in args.replace_objects] + action = replace_objects else: - replaceable_objects = all_replaceable_objects + modifiable_objects = all_replaceable_objects + action = replace_objects output = {} replacements = {} @@ -59,8 +68,9 @@ def main(): continue elements_new = [] + for element in dhis_object_old: - replace_objects(args, element, replaceable_objects) + action(args, element, modifiable_objects) remove_ous_and_catcombos(args, dhis_object, element) @@ -75,6 +85,12 @@ def main(): json.dump(output, open(args.output, 'wt'), indent=2) +def validate_args(args): + if args.remove_objects and args.replace_objects: + print('Error The options --replace-objects and --remove-objects are mutually exclusive.') + exit(0) + + def replace_ids(args, element, replacements): if args.replace_ids and element['id'] in replacements.keys(): element['id'] = replacements.get(element['id']) @@ -118,6 +134,25 @@ def replace_objects(args, element, replaceable_objects): exit(0) +def remove_objects(args, element, removable_object): + for replaceable in removable_object: + if replaceable in all_replaceable_objects: + remove_ids = get_list_of_removable_uids(args, replaceable) + new_element = [] + for iter_element in element[replaceable]: + if iter_element['id'] not in remove_ids: + new_element.append(iter_element) + element[replaceable] = new_element + + +def get_list_of_removable_uids(args, replaceable): + file = json.load(open(args.__getattribute__(replaceable))) + remove_ids = [] + for removable_element in file[replaceable]: + remove_ids.append(removable_element['id']) + return remove_ids + + def get_args(): "Return arguments" parser = argparse.ArgumentParser(description=__doc__) @@ -130,8 +165,10 @@ def get_args(): help='userAccesses file (read from userAccesses.json if not given)') add('-uga', '--userGroupAccesses', dest="userGroupAccesses", default="userGroupAccesses.json", help='userGroupAccesses file (read from userGroupAccesses.json if not given)') - add('-ro', '--replace-objects', nargs='+', default=all_replaceable_objects, + add('-ro', '--replace-objects', nargs='+', help='replace only the provided objects default: publicAccess, userAccesses, userGroupAccesses') + add('--remove-objects', nargs='+', + help='delete the objects in the provided .json using its id. Example: userGroupAccesses') add('--public-access', default='--------', dest="publicAccess", help='set public permissions. Default: no public access (--------)') add('--remove-ous', action='store_true', From f73dc7294fab853c052f7e740547d87b5e6acd1c Mon Sep 17 00:00:00 2001 From: idelcano Date: Wed, 20 Feb 2019 22:25:40 +0100 Subject: [PATCH 2/9] added recursive support --- .../change_user_groups.py | 56 +++++++++++++++---- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/DHIS2/metadata_manipulation/change_user_groups.py b/DHIS2/metadata_manipulation/change_user_groups.py index b4f7019..ffa985e 100755 --- a/DHIS2/metadata_manipulation/change_user_groups.py +++ b/DHIS2/metadata_manipulation/change_user_groups.py @@ -68,15 +68,17 @@ def main(): continue elements_new = [] + if args.recursive: + elements_new = recursive_action(action, args, dhis_object_old, modifiable_objects) + else: + for element in dhis_object_old: + action(args, element, modifiable_objects) - for element in dhis_object_old: - action(args, element, modifiable_objects) + remove_ous_and_catcombos(args, dhis_object, element) - remove_ous_and_catcombos(args, dhis_object, element) + replace_ids(args, element, replacements) - replace_ids(args, element, replacements) - - elements_new.append(element) + elements_new.append(element) output[dhis_object] = elements_new else: @@ -112,6 +114,30 @@ def get_replacements(ids): return replacements +def recursive_action(action, args, dhis_objects, modifiable_objects): + action(args, dhis_objects, modifiable_objects) + if isinstance(dhis_objects, list): + return recursive_list(action, args, dhis_objects, modifiable_objects) + elif isinstance(dhis_objects, dict): + return recursive_dict(action, args, dhis_objects, modifiable_objects) + else: + return dhis_objects + + +def recursive_dict(action, args, dhis_objects, modifiable_objects): + new_dict = dict() + for key in dhis_objects.keys(): + new_dict[key] = recursive_action(action, args, dhis_objects.get(key), modifiable_objects) + return new_dict + + +def recursive_list(action, args, dhis_objects, modifiable_objects): + new_list = list() + for item in iter(dhis_objects): + new_list.append(recursive_action(action, args, item, modifiable_objects)) + return new_list + + def remove_ous_and_catcombos(args, dhis_object, element): if dhis_object in ['programs', 'dataElements']: if args.remove_ous and element.get('organisationUnits'): @@ -126,23 +152,29 @@ def replace_objects(args, element, replaceable_objects): element['publicAccess'] = args.publicAccess elif replaceable in all_replaceable_objects: file = json.load(open(args.__getattribute__(replaceable))) - element[replaceable] = file[replaceable] + if isinstance(element, dict) and replaceable in element.keys(): + element[replaceable] = file[replaceable] else: print("Error: " + replaceable + " is an invalid replaceable object)") print("List of valid replaceable objects: ") print(all_replaceable_objects) exit(0) + return element + def remove_objects(args, element, removable_object): for replaceable in removable_object: if replaceable in all_replaceable_objects: remove_ids = get_list_of_removable_uids(args, replaceable) new_element = [] - for iter_element in element[replaceable]: - if iter_element['id'] not in remove_ids: - new_element.append(iter_element) - element[replaceable] = new_element + if isinstance(element, dict) and replaceable in element.keys(): + for iter_element in element[replaceable]: + if iter_element['id'] not in remove_ids: + new_element.append(iter_element) + element[replaceable] = new_element + + return element def get_list_of_removable_uids(args, replaceable): @@ -181,6 +213,8 @@ def get_args(): help='DHIS2 objects to include. Default: All') add('--exclude', nargs='+', default=[], help='DHIS2 objects to exclude. Default: None') + add('-r', '--recursive', dest='recursive', action="store_true", + help='DHIS2 objects to exclude. Default: None') return parser.parse_args() From 002e03d705023724535b68dee94cb5d3a2c4b657 Mon Sep 17 00:00:00 2001 From: idelcano Date: Wed, 20 Feb 2019 22:30:45 +0100 Subject: [PATCH 3/9] fix json.dump codification --- DHIS2/metadata_manipulation/change_user_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHIS2/metadata_manipulation/change_user_groups.py b/DHIS2/metadata_manipulation/change_user_groups.py index acd3d78..27fff12 100755 --- a/DHIS2/metadata_manipulation/change_user_groups.py +++ b/DHIS2/metadata_manipulation/change_user_groups.py @@ -84,7 +84,7 @@ def main(): else: output[dhis_object] = dhis_object_old - json.dump(output, open(args.output, 'wt'), indent=2) + json.dump(output, open(args.output, 'wt'), ensure_ascii=False, indent=2) def validate_args(args): From 3c553fb72d86098bbe55248e06ed01fb7990d09c Mon Sep 17 00:00:00 2001 From: idelcano Date: Thu, 28 Feb 2019 08:45:06 +0100 Subject: [PATCH 4/9] make replace ids recursive --- DHIS2/metadata_manipulation/change_user_groups.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/DHIS2/metadata_manipulation/change_user_groups.py b/DHIS2/metadata_manipulation/change_user_groups.py index 27fff12..a29b4d5 100755 --- a/DHIS2/metadata_manipulation/change_user_groups.py +++ b/DHIS2/metadata_manipulation/change_user_groups.py @@ -62,6 +62,9 @@ def main(): for dhis_object in input_objects: dhis_object_old = json.load(open(args.input, encoding="utf-8")).get(dhis_object) + if len(replacements) > 0: + dhis_object_old = recursive_action(replace_ids, args, dhis_object_old, replacements) + if dhis_object in dhis_objects: if not dhis_object_old: print('WARN: Ignoring not found object %s' % dhis_object) @@ -76,8 +79,6 @@ def main(): remove_ous_and_catcombos(args, dhis_object, element) - replace_ids(args, element, replacements) - elements_new.append(element) output[dhis_object] = elements_new @@ -94,8 +95,11 @@ def validate_args(args): def replace_ids(args, element, replacements): - if args.replace_ids and element['id'] in replacements.keys(): - element['id'] = replacements.get(element['id']) + if isinstance(element, dict): + for key in element.keys(): + for replacement_key in replacements.keys(): + if isinstance(element[key], str) and replacement_key in element[key]: + element[key] = element[key].replace(replacement_key, replacements[replacement_key]) def get_input_objects(input, excluded_objects): @@ -214,7 +218,7 @@ def get_args(): add('--exclude', nargs='+', default=[], help='DHIS2 objects to exclude. Default: None') add('-r', '--recursive', dest='recursive', action="store_true", - help='DHIS2 objects to exclude. Default: None') + help='apply replace or remove actions recursively.') return parser.parse_args() From 74e1a306e589bafb49ff96829429c8e2366cee22 Mon Sep 17 00:00:00 2001 From: idelcano Date: Thu, 28 Feb 2019 08:49:11 +0100 Subject: [PATCH 5/9] make replace objects and remove objects mutually exclusive using ArgumentParser --- DHIS2/metadata_manipulation/change_user_groups.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/DHIS2/metadata_manipulation/change_user_groups.py b/DHIS2/metadata_manipulation/change_user_groups.py index a29b4d5..9ddbb41 100755 --- a/DHIS2/metadata_manipulation/change_user_groups.py +++ b/DHIS2/metadata_manipulation/change_user_groups.py @@ -197,14 +197,15 @@ def get_args(): help='input file (read from input.json if not given)') add('-o', '--output', default="output.json", help='output file (write to output.json if not given)') + group = parser.add_mutually_exclusive_group() + group.add_argument('-ro', '--replace-objects', nargs='+', + help='replace only the provided objects default: publicAccess, userAccesses, userGroupAccesses') + group.add_argument('--remove-objects', nargs='+', + help='delete the objects in the provided .json using its id. Example: userGroupAccesses') add('-ua', '--userAccesses', dest="userAccesses", default="userAccesses.json", help='userAccesses file (read from userAccesses.json if not given)') add('-uga', '--userGroupAccesses', dest="userGroupAccesses", default="userGroupAccesses.json", help='userGroupAccesses file (read from userGroupAccesses.json if not given)') - add('-ro', '--replace-objects', nargs='+', - help='replace only the provided objects default: publicAccess, userAccesses, userGroupAccesses') - add('--remove-objects', nargs='+', - help='delete the objects in the provided .json using its id. Example: userGroupAccesses') add('--public-access', default='--------', dest="publicAccess", help='set public permissions. Default: no public access (--------)') add('--remove-ous', action='store_true', From d29d23d6e2ea639120a435a8bd8302265bd29955 Mon Sep 17 00:00:00 2001 From: idelcano Date: Tue, 5 Mar 2019 10:01:51 +0100 Subject: [PATCH 6/9] Added remove recursive ids --- .../change_user_groups.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/DHIS2/metadata_manipulation/change_user_groups.py b/DHIS2/metadata_manipulation/change_user_groups.py index 9ddbb41..9a1a495 100755 --- a/DHIS2/metadata_manipulation/change_user_groups.py +++ b/DHIS2/metadata_manipulation/change_user_groups.py @@ -53,15 +53,22 @@ def main(): output = {} replacements = {} + removable_ids = {} if args.replace_ids: replacements = get_replacements(args.replace_ids) + if args.remove_ids: + removable_ids = args.remove_ids + input_objects = get_input_objects(args.input, args.exclude) for dhis_object in input_objects: dhis_object_old = json.load(open(args.input, encoding="utf-8")).get(dhis_object) + if len(removable_ids) > 0: + dhis_object_old = recursive_action(remove_ids, args, dhis_object_old, removable_ids) + if len(replacements) > 0: dhis_object_old = recursive_action(replace_ids, args, dhis_object_old, replacements) @@ -102,6 +109,16 @@ def replace_ids(args, element, replacements): element[key] = element[key].replace(replacement_key, replacements[replacement_key]) +def remove_ids(args, element, removable_ids): + if isinstance(element, list): + for child in element[:]: + if isinstance(child, dict) and "id" in child.keys() and child["id"] in removable_ids: + element.remove(child) + elif isinstance(element, dict) and "id" in element.keys() and element["id"] in removable_ids: + element["id"] = None + element.pop("id", None) + + def get_input_objects(input, excluded_objects): input_file = json.load(open(input, encoding="utf-8")) input_objects = [item for item in input_file.keys() if item not in excluded_objects] @@ -214,6 +231,8 @@ def get_args(): help='remove catcombos assignment from programs') add('--replace-ids', nargs='+', metavar='FROM TO', default=[], help='ids to replace. NOTE: references are not updated!!!') + add('--remove-ids', nargs='+', metavar='FROM TO', default=[], + help='ids to remove. NOTE: Remove the entire object') add('--apply-to-objects', dest='apply', nargs='+', default=dhis_objects, help='DHIS2 objects to include. Default: All') add('--exclude', nargs='+', default=[], From b5948156ee28697c22ae2af5041971ffb21bb565 Mon Sep 17 00:00:00 2001 From: idelcano Date: Tue, 5 Mar 2019 10:06:52 +0100 Subject: [PATCH 7/9] Remove unnecesary validation --- DHIS2/metadata_manipulation/change_user_groups.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/DHIS2/metadata_manipulation/change_user_groups.py b/DHIS2/metadata_manipulation/change_user_groups.py index 9a1a495..76055b4 100755 --- a/DHIS2/metadata_manipulation/change_user_groups.py +++ b/DHIS2/metadata_manipulation/change_user_groups.py @@ -37,8 +37,6 @@ def main(): args = get_args() - validate_args(args) - dhis_objects = [item for item in args.apply if item not in args.exclude] if args.remove_objects: @@ -95,12 +93,6 @@ def main(): json.dump(output, open(args.output, 'wt'), ensure_ascii=False, indent=2) -def validate_args(args): - if args.remove_objects and args.replace_objects: - print('Error The options --replace-objects and --remove-objects are mutually exclusive.') - exit(0) - - def replace_ids(args, element, replacements): if isinstance(element, dict): for key in element.keys(): From adc628d72077cac0e65f8d8d32923d04037a12fb Mon Sep 17 00:00:00 2001 From: idelcano Date: Tue, 5 Mar 2019 10:08:52 +0100 Subject: [PATCH 8/9] remove confuse key --- DHIS2/metadata_manipulation/change_user_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHIS2/metadata_manipulation/change_user_groups.py b/DHIS2/metadata_manipulation/change_user_groups.py index 76055b4..4731ccc 100755 --- a/DHIS2/metadata_manipulation/change_user_groups.py +++ b/DHIS2/metadata_manipulation/change_user_groups.py @@ -207,7 +207,7 @@ def get_args(): add('-o', '--output', default="output.json", help='output file (write to output.json if not given)') group = parser.add_mutually_exclusive_group() - group.add_argument('-ro', '--replace-objects', nargs='+', + group.add_argument('--replace-objects', nargs='+', help='replace only the provided objects default: publicAccess, userAccesses, userGroupAccesses') group.add_argument('--remove-objects', nargs='+', help='delete the objects in the provided .json using its id. Example: userGroupAccesses') From 5781aa4842b774470aee1b046854db8042124e64 Mon Sep 17 00:00:00 2001 From: ifoche Date: Wed, 3 Apr 2019 18:22:09 +0000 Subject: [PATCH 9/9] fix encoding problem --- DHIS2/metadata_manipulation/change_user_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DHIS2/metadata_manipulation/change_user_groups.py b/DHIS2/metadata_manipulation/change_user_groups.py index 4731ccc..3f22157 100755 --- a/DHIS2/metadata_manipulation/change_user_groups.py +++ b/DHIS2/metadata_manipulation/change_user_groups.py @@ -90,7 +90,7 @@ def main(): else: output[dhis_object] = dhis_object_old - json.dump(output, open(args.output, 'wt'), ensure_ascii=False, indent=2) + json.dump(output, open(args.output, 'wt', encoding="utf-8"), ensure_ascii=False, indent=2) def replace_ids(args, element, replacements):