diff --git a/backend/importer/tasks.py b/backend/importer/tasks.py index 202517d9..e8360ab9 100644 --- a/backend/importer/tasks.py +++ b/backend/importer/tasks.py @@ -84,10 +84,15 @@ def run_import(import_obj): f"with a batch size of {settings.IMPORT_BATCH_SIZE}" ) + import_data_into_db(import_data, import_model) + + import_obj.status = ImportStatusChoices.DONE + import_obj.save() + + +def import_data_into_db(import_data, import_model): batch_number = 0 items_imported = 0 - - errors = [] for i in range(0, len(import_data), settings.IMPORT_BATCH_SIZE): import_batch = import_data[i : i + settings.IMPORT_BATCH_SIZE] try: @@ -104,13 +109,11 @@ def run_import(import_obj): import_model.objects.create(**item) except IntegrityError as e: - errors.append(e) logger.exception(f"Error importing item #{items_imported + 1} {item}: {e}") batch_number += 1 - import_obj.status = ImportStatusChoices.DONE - import_obj.save() + logger.info(f"Imported {items_imported} items into {import_model}") def process_raw_data(import_obj: ImportJob, import_model: ModelBase, raw_data: List[Dict]) -> List[Dict]: