Skip to content

Commit

Permalink
Use click progress bar to show progress for csv read and building pay…
Browse files Browse the repository at this point in the history 
…load (#151)
  • Loading branch information
@Wambere
Wambere authored Feb 26, 2024
1 parent 98852b0 commit 3f9c508
Show file tree
Hide file tree
Showing 2 changed files with 72 additions and 66 deletions.
2 changes: 2 additions & 0 deletions importer/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ and then posts them to the API for creation
3. Create a `config.py` file. The `sample_config.py` is an example of what this should look like. Populate it with the right credentials
4. Run script - `python3 main.py --csv_file csv/locations.csv --resource_type locations`
5. You can turn on logging by passing a `--log_level` to the command line as `info`, `debug` or `error`. For example `python3 main.py --csv_file csv/locations.csv --resource_type locations --log_level info`
6. There is a progress bar that shows the read_csv and build_payload progress as it is going on


See example csvs in the csv folder

Expand Down
136 changes: 70 additions & 66 deletions importer/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ def read_csv(csv_file):
next(records)
all_records = []

for record in records:
all_records.append(record)
with click.progressbar(records, label='Progress::Reading csv ') as read_csv_progress:
for record in read_csv_progress:
all_records.append(record)

logging.info("Returning records from csv file")
return all_records
Expand Down Expand Up @@ -75,7 +76,7 @@ def get_access_token():
# to create resources
@backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_time=180)
def post_request(request_type, payload, url):
logging.info("Posting request-----------------")
logging.info("Posting request")
logging.info("Request type: " + request_type)
logging.info("Url: " + url)
logging.debug("Payload: " + payload)
Expand Down Expand Up @@ -514,63 +515,65 @@ def build_payload(resource_type, resources, resource_payload_file):
with open(resource_payload_file) as json_file:
payload_string = json_file.read()

for resource in resources:
name, status, method, id, *_ = resource
try:
if method == "create":
version = "1"
if len(id.strip()) > 0:
# use the provided id
unique_uuid = id.strip()
identifier_uuid = id.strip()
else:
# generate a new uuid
unique_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, name))
identifier_uuid = unique_uuid
except IndexError:
# default if method is not provided
unique_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, name))
identifier_uuid = unique_uuid
version = "1"

try:
if method == "update":
if len(id.strip()) > 0:
version = get_resource(id, resource_type)
if version != "0":
with click.progressbar(resources, label='Progress::Building payload ') as build_payload_progress:
for resource in build_payload_progress:
logging.info("\t")
name, status, method, id, *_ = resource
try:
if method == "create":
version = "1"
if len(id.strip()) > 0:
# use the provided id
unique_uuid = id.strip()
identifier_uuid = id.strip()
else:
logging.info("Failed to get resource!")
raise ValueError("Trying to update a Non-existent resource")
else:
logging.info("The id is required!")
raise ValueError("The id is required to update a resource")
except IndexError:
raise ValueError("The id is required to update a resource")

# ps = payload_string
ps = (
payload_string.replace("$name", name)
.replace("$unique_uuid", unique_uuid)
.replace("$identifier_uuid", identifier_uuid)
.replace("$version", version)
)
# generate a new uuid
unique_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, name))
identifier_uuid = unique_uuid
except IndexError:
# default if method is not provided
unique_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, name))
identifier_uuid = unique_uuid
version = "1"

try:
ps = ps.replace("$status", status)
except IndexError:
ps = ps.replace("$status", "active")
try:
if method == "update":
if len(id.strip()) > 0:
version = get_resource(id, resource_type)
if version != "0":
# use the provided id
unique_uuid = id.strip()
identifier_uuid = id.strip()
else:
logging.info("Failed to get resource!")
raise ValueError("Trying to update a Non-existent resource")
else:
logging.info("The id is required!")
raise ValueError("The id is required to update a resource")
except IndexError:
raise ValueError("The id is required to update a resource")

# ps = payload_string
ps = (
payload_string.replace("$name", name)
.replace("$unique_uuid", unique_uuid)
.replace("$identifier_uuid", identifier_uuid)
.replace("$version", version)
)

if resource_type == "organizations":
ps = organization_extras(resource, ps)
elif resource_type == "locations":
ps = location_extras(resource, ps)
elif resource_type == "careTeams":
ps = care_team_extras(resource, ps, "min", [], [], "orgs & users")
try:
ps = ps.replace("$status", status)
except IndexError:
ps = ps.replace("$status", "active")

if resource_type == "organizations":
ps = organization_extras(resource, ps)
elif resource_type == "locations":
ps = location_extras(resource, ps)
elif resource_type == "careTeams":
ps = care_team_extras(resource, ps, "min", [], [], "orgs & users")

final_string = final_string + ps + ","
final_string = final_string + ps + ","

final_string = initial_string + final_string[:-1] + " ] } "
return final_string
Expand Down Expand Up @@ -897,19 +900,20 @@ def main(
if resource_list:
if resource_type == "users":
logging.info("Processing users")
for user in resource_list:
user_id = create_user(user)
if user_id == 0:
# user was not created above, check if it already exists
user_id = confirm_keycloak_user(user)
if user_id != 0:
# user_id has been retrieved
# check practitioner
practitioner_exists = confirm_practitioner(user, user_id)
if not practitioner_exists:
payload = create_user_resources(user_id, user)
handle_request("POST", payload, config.fhir_base_url)
logging.info("Processing complete!")
with click.progressbar(resource_list, label="Progress:Processing users ") as process_user_progress:
for user in process_user_progress:
user_id = create_user(user)
if user_id == 0:
# user was not created above, check if it already exists
user_id = confirm_keycloak_user(user)
if user_id != 0:
# user_id has been retrieved
# check practitioner
practitioner_exists = confirm_practitioner(user, user_id)
if not practitioner_exists:
payload = create_user_resources(user_id, user)
handle_request("POST", payload, config.fhir_base_url)
logging.info("Processing complete!")
elif resource_type == "locations":
logging.info("Processing locations")
json_payload = build_payload(
Expand Down

0 comments on commit 3f9c508

Please sign in to comment.