From 52bfe202d2c171948dc5f044119d591de0d5aa5d Mon Sep 17 00:00:00 2001 From: Wambere Date: Tue, 24 Sep 2024 12:59:43 +0300 Subject: [PATCH 1/3] Initial update to report logging --- importer/README.md | 4 +- importer/importer/builder.py | 62 +++++++++++++++++- importer/importer/users.py | 59 +++++++++++------ importer/main.py | 121 +++++++++++++++++++++-------------- 4 files changed, 175 insertions(+), 71 deletions(-) diff --git a/importer/README.md b/importer/README.md index 970239c..03f816d 100644 --- a/importer/README.md +++ b/importer/README.md @@ -1,6 +1,6 @@ # Setup Keycloak Roles -This script is used to set up keycloak roles and groups. It takes in a CSV file with the following columns: +This script is used to set up keycloak roles and groups. It takes in a csv file with the following columns: - **role**: The actual names of the roles you would like to create - **composite**: A boolean value that tells if the role has composite roles or not @@ -56,7 +56,7 @@ and then posts them to the API for creation 4. Run script - `python3 main.py --csv_file csv/locations.csv --resource_type locations` 5. You can turn on logging by passing a `--log_level` to the command line as `info`, `debug` or `error`. For example `python3 main.py --csv_file csv/locations.csv --resource_type locations --log_level info` 6. There is a progress bar that shows the read_csv and build_payload progress as it is going on -7. You can get only the response from the api after the import is done by passing `--only_response true` +7. You can get a nicely formatted report response from the api after the import is done by passing `--report_response true`. See example csvs in the csv folder diff --git a/importer/importer/builder.py b/importer/importer/builder.py index b92781a..3f8255c 100644 --- a/importer/importer/builder.py +++ b/importer/importer/builder.py @@ -1,4 +1,5 @@ import base64 +import csv import json import logging import os @@ -11,8 +12,7 @@ import requests from dateutil.relativedelta import relativedelta -from importer.config.settings import (api_service, fhir_base_url, - product_access_token) +from importer.config.settings import api_service, fhir_base_url, product_access_token from importer.request import handle_request dir_path = str(pathlib.Path(__file__).parent.resolve()) @@ -1065,3 +1065,61 @@ def link_to_location(resource_list): return build_assign_payload(arr, "List", "subject=Location/") else: return "" + + +def count_records(csv_filepath): + with open(csv_filepath, newline="") as csvfile: + reader = csv.reader(csvfile) + return sum(1 for _ in reader) - 1 + + +def process_response(response): + json_response = json.loads(response) + issues = json_response["issue"] + return issues + + +def build_report(csv_file, response, error_details, fail_count, fail_all): + # Get total number of records + total_records = count_records(csv_file) + issues = [] + + # Get status code + if hasattr(response, "status_code") and response.status_code > 201: + status = "Failed" + processed_records = 0 + + if response.text: + issues = process_response(response.text) + for issue in issues: + del issue["code"] + else: + if fail_count > 0: + status = "Failed" + if fail_all: + processed_records = total_records + else: + processed_records = total_records - fail_count + else: + status = "Completed" + processed_records = total_records + + report = { + "status": status, + "totalRecords": total_records, + "processedRecords": processed_records, + } + if len(issues) > 0: + report["failedRecords"] = len(issues) + + all_errors = issues + error_details + + if len(all_errors) > 0: + report["errorDetails"] = all_errors + + string_report = json.dumps(report, indent=4) + logging.info("============================================================") + logging.info("============================================================") + logging.info(string_report) + logging.info("============================================================") + logging.info("============================================================") diff --git a/importer/importer/users.py b/importer/importer/users.py index 2585d2a..b9c5f7f 100644 --- a/importer/importer/users.py +++ b/importer/importer/users.py @@ -69,10 +69,11 @@ def create_user(user): logging.info("Setting user password") r = handle_request("PUT", payload, url) - return user_id + return user_id, {} else: logging.info(r.text) - return 0 + obj = {"task": "Create user", "row": str(user), "error": r.text} + return 0, obj # This function build the FHIR resources related to a @@ -178,21 +179,23 @@ def confirm_keycloak_user(user): try: response_username = json_response[0]["username"] except IndexError: - logging.error("Skipping user: " + str(user)) - logging.error("Username not found!") - return 0 + message = "Username not found! Skipping user: " + str(user[2]) + logging.error(message) + obj = {"task": "Confirm Keycloak user", "row": str(user), "error": message} + return 0, obj if response_username != user_username: - logging.error("Skipping user: " + str(user)) - logging.error("Username does not match") - return 0 + message = "Username does not match! Skipping user: " + str(user[2]) + logging.error(message) + obj = {"task": "Confirm Keycloak user", "row": str(user), "error": message} + return 0, obj if len(response_email) > 0 and response_email != user_email: - logging.error("Email does not match for user: " + str(user)) + logging.error("Email does not match for user: " + str(user[2])) keycloak_id = json_response[0]["id"] logging.info("User confirmed with id: " + keycloak_id) - return keycloak_id + return keycloak_id, {} def confirm_practitioner(user, user_id): @@ -204,18 +207,24 @@ def confirm_practitioner(user, user_id): json_r = json.loads(r[0]) counter = json_r["total"] if counter > 0: - logging.info( - str(counter) + " Practitioner(s) exist, linked to the provided user" + message = ( + "User " + + str(user[2]) + + " is linked to " + + str(counter) + + " Practitioner(s)" ) - return True + logging.info(message) + obj = {"task": "Confirm Practitioner", "row": str(user), "error": message} + return True, obj else: - return False + return False, {} r = handle_request("GET", "", base_url + "/Practitioner/" + practitioner_uuid) if r[1] == 404: logging.info("Practitioner does not exist, proceed to creation") - return False + return False, {} else: try: json_r = json.loads(r[0]) @@ -229,16 +238,26 @@ def confirm_practitioner(user, user_id): logging.info( "The Keycloak user and Practitioner are linked as expected" ) - return True + return True, {} else: - logging.error( + message = ( "The Keycloak user and Practitioner are not linked as expected" ) - return True + logging.error(message) + obj = { + "task": "Confirm Practitioner", + "row": str(user), + "error": message, + } + return True, obj except Exception as err: - logging.error("Error occurred trying to find Practitioner: " + str(err)) - return True + message = ( + "Error occurred trying to find Practitioner. The error is: " + str(err) + ) + logging.error(message) + obj = {"task": "Confirm Practitioner", "row": str(user), "error": message} + return True, obj def create_roles(role_list, roles_max): diff --git a/importer/main.py b/importer/main.py index eba4fcc..54c7264 100644 --- a/importer/main.py +++ b/importer/main.py @@ -1,52 +1,42 @@ import logging -import logging.config import pathlib from datetime import datetime import click +import json -from importer.builder import (build_assign_payload, build_group_list_resource, - build_org_affiliation, build_payload, - extract_matches, extract_resources, link_to_location) +from importer.builder import ( + build_assign_payload, + build_group_list_resource, + build_org_affiliation, + build_payload, + build_report, + extract_matches, + extract_resources, + link_to_location, +) from importer.config.settings import fhir_base_url from importer.request import handle_request -from importer.users import (assign_default_groups_roles, assign_group_roles, - confirm_keycloak_user, confirm_practitioner, - create_roles, create_user, create_user_resources) -from importer.utils import (build_mapped_payloads, clean_duplicates, - export_resources_to_csv, read_csv, - read_file_in_chunks) +from importer.users import ( + assign_default_groups_roles, + assign_group_roles, + confirm_keycloak_user, + confirm_practitioner, + create_roles, + create_user, + create_user_resources, +) +from importer.utils import ( + build_mapped_payloads, + clean_duplicates, + export_resources_to_csv, + read_csv, + read_file_in_chunks, +) dir_path = str(pathlib.Path(__file__).parent.resolve()) -class ResponseFilter(logging.Filter): - def __init__(self, param=None): - self.param = param - - def filter(self, record): - if self.param is None: - allow = True - else: - allow = self.param in record.msg - return allow - - -LOGGING = { - "version": 1, - "filters": { - "custom-filter": { - "()": ResponseFilter, - "param": "final-response", - } - }, - "handlers": { - "console": {"class": "logging.StreamHandler", "filters": ["custom-filter"]} - }, - "root": {"level": "INFO", "handlers": ["console"]}, -} - - @click.command() @click.option("--csv_file", required=False) @click.option("--json_file", required=False) @@ -57,7 +47,7 @@ def filter(self, record): @click.option("--roles_max", required=False, default=500) @click.option("--default_groups", required=False, default=True) @click.option("--cascade_delete", required=False, default=False) -@click.option("--only_response", required=False) +@click.option("--report_response", required=False) @click.option("--export_resources", required=False) @click.option("--parameter", required=False, default="_lastUpdated") @click.option("--value", required=False, default="gt2023-01-01") @@ -90,7 +80,7 @@ def main( roles_max, default_groups, cascade_delete, - only_response, + report_response, log_level, export_resources, parameter, @@ -117,11 +107,11 @@ def main( ) logging.getLogger().addHandler(logging.StreamHandler()) - if only_response: - logging.config.dictConfig(LOGGING) - start_time = datetime.now() logging.info("Start time: " + start_time.strftime("%H:%M:%S")) + issues = [] + fail_count = 0 + fail_all = False if export_resources == "True": logging.info("Starting export...") @@ -153,19 +143,30 @@ def main( resource_list, label="Progress:Processing users " ) as process_user_progress: for user in process_user_progress: - user_id = create_user(user) + user_id, create_issue = create_user(user) if user_id == 0: + fail_count = fail_count + 1 + if create_issue: + issues.append(create_issue) # user was not created above, check if it already exists - user_id = confirm_keycloak_user(user) + user_id, confirm_issue = confirm_keycloak_user(user) + if confirm_issue: + issues.append(confirm_issue) if user_id != 0: # user_id has been retrieved # check practitioner - practitioner_exists = confirm_practitioner(user, user_id) + practitioner_exists, practitioner_issue = confirm_practitioner( + user, user_id + ) + if practitioner_issue: + issues.append(practitioner_issue) if not practitioner_exists: payload = create_user_resources(user_id, user) final_response = handle_request( "POST", payload, fhir_base_url ) + if final_response.status_code > 201: + issues.append(final_response.text) logging.info("Processing complete!") elif resource_type == "locations": logging.info("Processing locations") @@ -199,7 +200,6 @@ def main( matches = extract_matches(resource_list) json_payload = build_org_affiliation(matches, resource_list) final_response = handle_request("POST", json_payload, fhir_base_url) - logging.info(final_response) logging.info("Processing complete!") elif assign == "users-organizations": logging.info("Assigning practitioner to Organization") @@ -244,7 +244,13 @@ def main( final_response = handle_request("POST", "", fhir_base_url, list_payload) logging.info("Processing complete!") else: - logging.error(product_creation_response.text) + fail_count = fail_count + 1 + fail_all = True + json_response = json.loads(product_creation_response.text) + for _ in json_response["issue"]: + del _["code"] + issues.append(_) + logging.error(json_response) elif setup == "inventories": logging.info("Importing inventories as FHIR Group resources") json_payload = build_payload( @@ -258,6 +264,14 @@ def main( groups_created = extract_resources( groups_created, inventory_creation_response.text ) + else: + fail_count = fail_count + 1 + fail_all = True + json_response = json.loads(inventory_creation_response.text) + for _ in json_response["issue"]: + del _["code"] + issues.append(_) + logging.error(json_response) lists_created = [] link_payload = link_to_location(resource_list) @@ -265,6 +279,13 @@ def main( link_response = handle_request("POST", link_payload, fhir_base_url) if link_response.status_code == 200: lists_created = extract_resources(lists_created, link_response.text) + else: + fail_count = fail_count + 1 + fail_all = True + json_response = json.loads(link_response.text) + for _ in json_response["issue"]: + del _["code"] + issues.append(_) logging.info(link_response.text) full_list_created_resources = groups_created + lists_created @@ -278,6 +299,9 @@ def main( final_response = handle_request("POST", "", fhir_base_url, list_payload) logging.info("Processing complete!") else: + message = "Unsupported request!" + fail_all = True + issues.append({"Error": message}) logging.error("Unsupported request!") else: logging.error("Empty csv file!") @@ -290,6 +314,9 @@ def main( total_time = end_time - start_time logging.info("Total time: " + str(total_time.total_seconds()) + " seconds") + if report_response: + build_report(csv_file, final_response, issues, fail_count, fail_all) + if __name__ == "__main__": main() From 007e30edb03b71b1cda9ce5305081253ace7a263 Mon Sep 17 00:00:00 2001 From: Wambere Date: Tue, 24 Sep 2024 13:09:18 +0300 Subject: [PATCH 2/3] sort imports --- importer/importer/builder.py | 3 ++- importer/main.py | 38 +++++++++++------------------------- 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/importer/importer/builder.py b/importer/importer/builder.py index 3f8255c..684f149 100644 --- a/importer/importer/builder.py +++ b/importer/importer/builder.py @@ -12,7 +12,8 @@ import requests from dateutil.relativedelta import relativedelta -from importer.config.settings import api_service, fhir_base_url, product_access_token +from importer.config.settings import (api_service, fhir_base_url, + product_access_token) from importer.request import handle_request dir_path = str(pathlib.Path(__file__).parent.resolve()) diff --git a/importer/main.py b/importer/main.py index 54c7264..8fe6a8c 100644 --- a/importer/main.py +++ b/importer/main.py @@ -1,38 +1,22 @@ +import json import logging import pathlib from datetime import datetime import click -import json -from importer.builder import ( - build_assign_payload, - build_group_list_resource, - build_org_affiliation, - build_payload, - build_report, - extract_matches, - extract_resources, - link_to_location, -) +from importer.builder import (build_assign_payload, build_group_list_resource, + build_org_affiliation, build_payload, + build_report, extract_matches, extract_resources, + link_to_location) from importer.config.settings import fhir_base_url from importer.request import handle_request -from importer.users import ( - assign_default_groups_roles, - assign_group_roles, - confirm_keycloak_user, - confirm_practitioner, - create_roles, - create_user, - create_user_resources, -) -from importer.utils import ( - build_mapped_payloads, - clean_duplicates, - export_resources_to_csv, - read_csv, - read_file_in_chunks, -) +from importer.users import (assign_default_groups_roles, assign_group_roles, + confirm_keycloak_user, confirm_practitioner, + create_roles, create_user, create_user_resources) +from importer.utils import (build_mapped_payloads, clean_duplicates, + export_resources_to_csv, read_csv, + read_file_in_chunks) dir_path = str(pathlib.Path(__file__).parent.resolve()) From a8493f79b06bc67da1c92cf73e59ab816ff6f0c6 Mon Sep 17 00:00:00 2001 From: Wambere Date: Tue, 24 Sep 2024 16:39:44 +0300 Subject: [PATCH 3/3] Update tests --- importer/tests/test_users.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/importer/tests/test_users.py b/importer/tests/test_users.py index 3278223..bab9d0a 100644 --- a/importer/tests/test_users.py +++ b/importer/tests/test_users.py @@ -105,7 +105,7 @@ def test_create_user( "demo", "pa$$word", ) - user_id = create_user(mocked_user_data) + user_id, obj = create_user(mocked_user_data) self.assertEqual(user_id, "6cd50351-3ddb-4296-b1db-aac2273e35f3") mock_logging.info.assert_called_with("Setting user password") @@ -132,7 +132,7 @@ def test_create_user_already_exists( "demo", "pa$$word", ) - user_id = create_user(mocked_user_data) + user_id, obj = create_user(mocked_user_data) self.assertEqual(user_id, 0) # Test the confirm_keycloak function @@ -158,7 +158,7 @@ def test_confirm_keycloak_user( "demo", "pa$$word", ) - user_id = create_user(mocked_user_data) + user_id, obj = create_user(mocked_user_data) self.assertEqual(user_id, 0) mock_response = ( @@ -172,7 +172,7 @@ def test_confirm_keycloak_user( ) mock_handle_request.return_value = mock_response mock_json_response = json.loads(mock_response[0]) - keycloak_id = confirm_keycloak_user(mocked_user_data) + keycloak_id, obj = confirm_keycloak_user(mocked_user_data) self.assertEqual(mock_json_response[0]["username"], "Jenny") self.assertEqual(mock_json_response[0]["email"], "jeendoe@example.com")