From 071959ccbe6ea1250835a7759197ff50c51523bd Mon Sep 17 00:00:00 2001 From: Wambere Date: Thu, 17 Oct 2024 22:42:47 +0300 Subject: [PATCH] Data generator (#226) * Initial attempt at creating a script to generate mock data * Rename stub data generatr * min fix * Update data generator * Update data gen for all resources * Refactor when log message is added. * Clean up * Update version of Faker used * Fix column ordering in inventory generation * Shebang * Rename cli.py to faker-clie * Ignore default folder where fake data is generated * Make cli file executable * Update Readme * Remove duplicate package in requirements * Update setup instructions in Readme --------- Co-authored-by: Peter Muriuki Co-authored-by: Peter Lubell-Doughtie --- importer/.gitignore | 1 + importer/requirements.txt | 1 + importer/stub_data_gen/Readme.md | 42 ++ importer/stub_data_gen/faker-cli.py | 172 +++++++ importer/stub_data_gen/fixtures.py | 329 +++++++++++++ importer/stub_data_gen/product_samples.json | 505 ++++++++++++++++++++ importer/stub_data_gen/stub_gen.py | 235 +++++++++ importer/stub_data_gen/utils.py | 42 ++ 8 files changed, 1327 insertions(+) create mode 100644 importer/stub_data_gen/Readme.md create mode 100755 importer/stub_data_gen/faker-cli.py create mode 100644 importer/stub_data_gen/fixtures.py create mode 100644 importer/stub_data_gen/product_samples.json create mode 100644 importer/stub_data_gen/stub_gen.py create mode 100644 importer/stub_data_gen/utils.py diff --git a/importer/.gitignore b/importer/.gitignore index 82639c78..e44652dd 100644 --- a/importer/.gitignore +++ b/importer/.gitignore @@ -1,5 +1,6 @@ config.py importer.log +out # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/importer/requirements.txt b/importer/requirements.txt index 77f04966..be1f7ad2 100644 --- a/importer/requirements.txt +++ b/importer/requirements.txt @@ -12,6 +12,7 @@ jwt python-dotenv==1.0.1 pytest-env==1.1.3 python-dateutil==2.9.0 +Faker==0.7.4 # Windows requirements python-magic-bin==0.4.14; sys_platform == 'win32' diff --git a/importer/stub_data_gen/Readme.md b/importer/stub_data_gen/Readme.md new file mode 100644 index 00000000..a69c3aad --- /dev/null +++ b/importer/stub_data_gen/Readme.md @@ -0,0 +1,42 @@ +# usage + +Ofcourse you need to prepare your python environment and install the package dependencies + +```commandline +python3 -m venv myenv +source myenv/bin/activate +pip instal -r ../requirements.txt +cd stub_data_gen +``` + +**To generate data for all resources** + +```commandline +python faker-cli generate +``` + +or + +```commandline +./faker-cli generate +``` + +**Generate data for a single resource** + +```commandline +python faker-cli generate users --count=100 +``` + +**Generate single resource that requires pre-generated data like assignments** + +e.g. When assigning users to organizations, provide individual csvs containing the users and organizations data + +```commandline +python faker-cli generate users-orgs --orgs-csv= --users-csv= +``` + +**Learn more** + +```commandline +python faker-cli generate --help +``` \ No newline at end of file diff --git a/importer/stub_data_gen/faker-cli.py b/importer/stub_data_gen/faker-cli.py new file mode 100755 index 00000000..243472f1 --- /dev/null +++ b/importer/stub_data_gen/faker-cli.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +import click +import os + +from utils import default_out_dir +from utils import write_resource, read_full_csv +from stub_gen import generate_users, generate_products, generate_care_teams, generate_locations, \ + generate_org_to_locations, generate_users_orgs, generate_inventory, generate_organizations + + +@click.group() +def faker(): + pass + + +@click.group(invoke_without_command=True) +@click.option('--out-dir', default=default_out_dir, help='Output directory for generated files') +@click.option('--count', default=20, help='Count of records to generate') +@click.pass_context +def generate(ctx, out_dir, count): + ctx.obj = {'OUT_DIR': out_dir, 'COUNT': count} + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + if ctx.invoked_subcommand is None: + # import ipdb; ipdb.set_trace() + # Call each subcommand's logic + user_data = generate_users(count) + write_resource("users", out_dir, user_data) + + location_data = generate_locations(count) + write_resource("locations", out_dir, location_data) + + products_data = generate_products(count) + write_resource("products", out_dir, products_data) + + organization_data = generate_organizations(count) + write_resource("organizations", out_dir, organization_data) + + care_team_data = generate_care_teams(organization_data[1], user_data[1], count) + write_resource("careteams", out_dir, care_team_data) + + inventory_data = generate_inventory(location_data[1], products_data[1], count) + write_resource("inventories", out_dir, inventory_data) + + user_orgs_data = generate_users_orgs(user_data[1], organization_data[1], count) + write_resource("users_orgs", out_dir, user_orgs_data) + + orgs_locs_data = generate_org_to_locations(organization_data[1], location_data[1], count) + write_resource("orgs_locs", out_dir, orgs_locs_data) + + click.echo(f"All data types generated and saved in {out_dir}") + + +@click.command() +@click.pass_context +def users(ctx): + out_dir = ctx.obj['OUT_DIR'] + count = ctx.obj["COUNT"] + data = generate_users(count) + write_resource("users", out_dir, data) + click.echo(f"User data generated and saved in {out_dir}") + + +@click.command() +@click.option('--orgs-csv', help='Organizations CSV file') +@click.option('--users-csv', help='Locations CSV file') +@click.pass_context +def careteams(ctx, orgs_csv, users_csv): + out_dir = ctx.obj['OUT_DIR'] + count = ctx.obj["COUNT"] + organizations_data = [] + locations_data = [] + if orgs_csv: + [_, organizations_data] = read_full_csv(orgs_csv) + if users_csv: + [_, locations_data] = read_full_csv(users_csv) + data = generate_care_teams( organizations_data, locations_data, count) + write_resource("careteams", out_dir, data) + click.echo(f"Care team data generated and saved in {out_dir}") + + +@click.command() +@click.pass_context +def locations(ctx): + out_dir = ctx.obj['OUT_DIR'] + count = ctx.obj["COUNT"] + data = generate_locations(count) + write_resource("locations", out_dir, data) + click.echo(f"Location data generated and saved in {out_dir}") + + +@click.command() +@click.option('--orgs-csv', help='Organizations CSV file', required=True) +@click.option('--locs-csv', help='Locations CSV file', required=True) +@click.pass_context +def orgs_locs(ctx, orgs_csv, locs_csv): + out_dir = ctx.obj['OUT_DIR'] + count = ctx.obj['COUNT'] + # Idea: assume orgs and locations files exist in the out folder then we can use them as source + # if not out_dir.startswith('./out') and (not orgs_csv or not locs_csv): + # click.echo("Error: --orgs-csv and --locs-csv are required when --out-dir is not './out'") + # return + [_, organizations] = read_full_csv(orgs_csv) + [_, locations] = read_full_csv(locs_csv) + + data = generate_org_to_locations(organizations, locations, count) + write_resource("orgs_locs", out_dir, data) + click.echo(f"Organization and location data generated and saved in {out_dir}") + + +@click.command() +@click.option('--orgs-csv', help='Organizations CSV file', required=True) +@click.option('--users-csv', help='users CSV file', required=True) +@click.pass_context +def users_orgs(ctx, orgs_csv, users_csv): + out_dir = ctx.obj['OUT_DIR'] + count = ctx.obj['COUNT'] + [_, organizations] = read_full_csv(orgs_csv) + [_, users] = read_full_csv(users_csv) + data = generate_users_orgs(users, organizations, count) + write_resource("users_orgs", out_dir, data) + click.echo(f"User and organization data generated and saved in {out_dir}") + + +@click.command() +@click.pass_context +def products(ctx): + out_dir = ctx.obj['OUT_DIR'] + count = ctx.obj["COUNT"] + data = generate_products(count) + write_resource("products", out_dir, data) + click.echo(f"Product data generated and saved in {out_dir}") + + +@click.command() +@click.option('--locs-csv', help='Organizations CSV file', required=True) +@click.option('--prods-csv', help='users CSV file', required=True) +@click.pass_context +def inventories(ctx, locs_csv, prods_csv): + out_dir = ctx.obj['OUT_DIR'] + count = ctx.obj["COUNT"] + [_, locations] = read_full_csv(locs_csv) + [_, products] = read_full_csv(prods_csv) + data = generate_inventory(locations, products, count) + write_resource("inventories", out_dir, data) + click.echo(f"Inventory data generated and saved in {out_dir}") + + +@click.command() +@click.pass_context +def organizations(ctx): + out_dir = ctx.obj['OUT_DIR'] + count = ctx.obj['COUNT'] + data = generate_organizations(count) + write_resource("organizations", out_dir, data) + click.echo(f"Organization data generated and saved in {out_dir}") + + +faker.add_command(generate) + +generate.add_command(users) +generate.add_command(careteams) +generate.add_command(locations) +generate.add_command(orgs_locs) +generate.add_command(users_orgs) +generate.add_command(products) +generate.add_command(inventories) +generate.add_command(organizations) + +if __name__ == '__main__': + faker() diff --git a/importer/stub_data_gen/fixtures.py b/importer/stub_data_gen/fixtures.py new file mode 100644 index 00000000..b6c39387 --- /dev/null +++ b/importer/stub_data_gen/fixtures.py @@ -0,0 +1,329 @@ +inventory_donors = [ + { + "code": "adb", + "display": "ADB" + }, + { + "code": "natcom_belgium", + "display": "NatCom Belgium" + }, + { + "code": "bmgf", + "display": "BMGF" + }, + { + "code": "govt_of_canada", + "display": "Govt of Canada" + }, + { + "code": "natcom_canada", + "display": "NatCom Canada" + }, + { + "code": "natcom_denmark", + "display": "NatCom Denmark" + }, + { + "code": "ecw", + "display": "ECW" + }, + { + "code": "end_violence_fund", + "display": "End Violence Fund" + }, + { + "code": "echo", + "display": "ECHO" + }, + { + "code": "ec", + "display": "EC" + }, + { + "code": "natcom_finland", + "display": "NatCom Finland" + }, + { + "code": "govt_of_france", + "display": "Govt of France" + }, + { + "code": "natcom_france", + "display": "NatCom France" + }, + { + "code": "gavi", + "display": "GAVI" + }, + { + "code": "natcom_germany", + "display": "NatCom Germany" + }, + { + "code": "govt_of_germany", + "display": "Govt of Germany" + }, + { + "code": "natcom_iceland", + "display": "NatCom Iceland" + }, + { + "code": "natcom_italy", + "display": "NatCom Italy" + }, + { + "code": "govt_of_japan", + "display": "Govt of Japan" + }, + { + "code": "natcom_japan", + "display": "NatCom Japan" + }, + { + "code": "natcom_luxembourg", + "display": "NatCom Luxembourg" + }, + { + "code": "monaco", + "display": "Monaco" + }, + { + "code": "natcom_netherlands", + "display": "NatCom Netherlands" + }, + { + "code": "govt_of_norway", + "display": "Govt of Norway" + }, + { + "code": "natcom_norway", + "display": "NatCom Norway" + }, + { + "code": "nutrition_international", + "display": "Nutrition International" + }, + { + "code": "natcom_poland", + "display": "NatCom Poland" + }, + { + "code": "govt_of_korea", + "display": "Govt of Korea" + }, + { + "code": "natcom_spain", + "display": "NatCom Spain" + }, + { + "code": "natcom_sweden", + "display": "NatCom Sweden" + }, + { + "code": "natcom_switzerland", + "display": "NatCom_Switzerland" + }, + { + "code": "govt_of_uk", + "display": "Govt of UK" + }, + { + "code": "natcom_uk", + "display": "NatCom UK" + }, + { + "code": "natcom_usa", + "display": "NatCom USA" + }, + { + "code": "ofda", + "display": "OFDA" + }, + { + "code": "cdc", + "display": "CDC" + }, + { + "code": "usaid", + "display": "USAID" + }, + { + "code": "usaid_ffp", + "display": "USAID FFP" + }, + { + "code": "world_bank", + "display": "World Bank" + }, + { + "code": "unicef", + "display": "UNICEF" + } +] + +unicef_sections = [ + { + "code": "health", + "display": "Health" + }, + { + "code": "wash", + "display": "WASH" + }, + { + "code": "nutrition", + "display": "Nutrition" + }, + { + "code": "education", + "display": "Education" + }, + { + "code": "child_protection", + "display": "Child Protection" + }, + { + "code": "social_policy", + "display": "Social Policy" + }, + { + "code": "c4d", + "display": "C4D" + }, + { + "code": "drr", + "display": "DRR" + }, + { + "code": "several_sections", + "display": "Several sections" + } +] + +service_point_types = [ + { + "code": "csb2", + "display": "CSB2" + }, + { + "code": "csb1", + "display": "CSB1" + }, + { + "code": "bsd", + "display": "BSD" + }, + { + "code": "chrd1", + "display": "CHRD1" + }, + { + "code": "chrd2", + "display": "CHRD2" + }, + { + "code": "chrr", + "display": "CHRR" + }, + { + "code": "sdsp", + "display": "SDSP" + }, + { + "code": "drsp", + "display": "DRSP" + }, + { + "code": "msp", + "display": "MSP" + }, + { + "code": "epp", + "display": "EPP" + }, + { + "code": "ceg", + "display": "CEG" + }, + { + "code": "warehouse", + "display": "Warehouse" + }, + { + "code": "water_point", + "display": "Water Point" + }, + { + "code": "presco", + "display": "Presco" + }, + { + "code": "meah", + "display": "MEAH" + }, + { + "code": "dreah", + "display": "DREAH" + }, + { + "code": "men", + "display": "MEN" + }, + { + "code": "dren", + "display": "DREN" + }, + { + "code": "mppspf", + "display": "MPPSPF" + }, + { + "code": "drppspf", + "display": "DRPPSPF" + }, + { + "code": "ngo_partner", + "display": "NGO Partner" + }, + { + "code": "site_communautaire", + "display": "Site Communautaire" + }, + { + "code": "drjs", + "display": "DRJS" + }, + { + "code": "instat", + "display": "INSTAT" + }, + { + "code": "mairie", + "display": "Mairie" + }, + { + "code": "ecole_privé", + "display": "Ecole privé" + }, + { + "code": "ecole_communautaire", + "display": "Ecole communautaire" + }, + { + "code": "lycée", + "display": "Lycée" + }, + { + "code": "chu", + "display": "CHU" + }, + { + "code": "district_ppspf", + "display": "District PPSPF" + } +] + +location_physical_types = [ + {"code": "bu", + "display": "building"}, {"code": "jdn", "display": "jurisdiction"} +] diff --git a/importer/stub_data_gen/product_samples.json b/importer/stub_data_gen/product_samples.json new file mode 100644 index 00000000..ceeaab68 --- /dev/null +++ b/importer/stub_data_gen/product_samples.json @@ -0,0 +1,505 @@ +[ + { + "id": "dfdd3b8d-b4a1-4f8c-8d73-8458ab8d5b29", + "productName": "Portable Ultrasound Machine", + "active": true, + "materialNumber": "SKU001", + "isAttractiveItem": true, + "availability": "Check stock levels in the main warehouse or contact the supplier for immediate availability.", + "condition": "The unit is in excellent condition, with all accessories included and fully operational.", + "appropriateUsage": "Ensure the device is used by trained personnel only and is regularly calibrated for accurate readings.", + "accountabilityPeriod": 48, + "imageSourceUrl": "http://example.com/images/1.jpg" + }, + { + "id": "6b8c7ef0-2b3f-11ed-861d-0242ac120002", + "productName": "Digital Thermometer", + "active": false, + "materialNumber": "SKU002", + "isAttractiveItem": false, + "availability": "Currently out of stock. New shipment expected within two weeks.", + "condition": "Some wear and tear visible; the device is functioning but may require recalibration.", + "appropriateUsage": "Ideal for routine temperature checks in clinics. Ensure proper cleaning after each use.", + "accountabilityPeriod": 12, + "imageSourceUrl": "http://example.com/images/2.jpg" + }, + { + "id": "3f9d8e19-e72b-4338-a2b0-5672d7161f85", + "productName": "Blood Pressure Monitor", + "active": true, + "materialNumber": "SKU003", + "isAttractiveItem": true, + "availability": "Available in all branches; check the nearest branch for in-store availability.", + "condition": "Brand new, unused, and sealed in original packaging.", + "appropriateUsage": "Use as per the included user manual. Regular maintenance and recalibration are recommended.", + "accountabilityPeriod": 24, + "imageSourceUrl": "http://example.com/images/3.jpg" + }, + { + "id": "492d7f9a-c731-4bd9-8f6f-70860e2e356a", + "productName": "Infant Weighing Scale", + "active": true, + "materialNumber": "SKU004", + "isAttractiveItem": false, + "availability": "Stock levels are sufficient for all orders. Check the supply room for immediate use.", + "condition": "In excellent condition with no visible damage. Accurate weighing guaranteed.", + "appropriateUsage": "Suitable for weighing infants up to 20kg. Clean and sanitize after each use.", + "accountabilityPeriod": 36, + "imageSourceUrl": "http://example.com/images/4.jpg" + }, + { + "id": "9264deac-78f4-11ec-90d6-0242ac120003", + "productName": "Surgical Mask Pack", + "active": true, + "materialNumber": "SKU005", + "isAttractiveItem": false, + "availability": "Bulk quantities available. Ready for immediate dispatch upon order.", + "condition": "Sterile and individually wrapped. No damage to packaging.", + "appropriateUsage": "Ensure masks are used once and disposed of properly. Suitable for medical and non-medical environments.", + "accountabilityPeriod": 6, + "imageSourceUrl": "http://example.com/images/5.jpg" + }, + { + "id": "d15dbf10-52d1-4a38-9c18-f5f72b1a8b1c", + "productName": "Oxygen Concentrator", + "active": true, + "materialNumber": "SKU006", + "isAttractiveItem": true, + "availability": "Available in limited quantities. Contact supplier for bulk orders.", + "condition": "In working condition. Requires annual maintenance for optimal performance.", + "appropriateUsage": "Ensure the device is used in a well-ventilated area and is regularly cleaned.", + "accountabilityPeriod": 60, + "imageSourceUrl": "http://example.com/images/6.jpg" + }, + { + "id": "a3b9c7a5-4b8f-4e6b-b3f8-42e38c7d5072", + "productName": "Stethoscope", + "active": true, + "materialNumber": "SKU007", + "isAttractiveItem": false, + "availability": "Readily available in all clinics. No current supply issues.", + "condition": "New and in excellent condition. Comes with a carry case.", + "appropriateUsage": "Ensure proper sanitization after each patient examination.", + "accountabilityPeriod": 18, + "imageSourceUrl": "http://example.com/images/7.jpg" + }, + { + "id": "eb9f7c08-5c34-4a79-bd7d-8a298e4c905b", + "productName": "Patient Monitor", + "active": true, + "materialNumber": "SKU008", + "isAttractiveItem": true, + "availability": "Available on special order. Contact supplier for delivery times.", + "condition": "Good working condition. Calibrate annually for accuracy.", + "appropriateUsage": "Only to be used by certified medical professionals in ICU settings.", + "accountabilityPeriod": 24, + "imageSourceUrl": "http://example.com/images/8.jpg" + }, + { + "id": "f2b5c216-1f2d-4e74-a07e-e05ad7d6f9e8", + "productName": "Sterile Surgical Gloves", + "active": false, + "materialNumber": "SKU009", + "isAttractiveItem": false, + "availability": "Out of stock. Expected restock in 3 weeks.", + "condition": "Sealed packaging, sterile and ready for use.", + "appropriateUsage": "Use in sterile surgical environments only. Dispose of after single use.", + "accountabilityPeriod": 12, + "imageSourceUrl": "http://example.com/images/9.jpg" + }, + { + "id": "4f3b9d5b-3b1f-41cd-8b2c-89a2de3c7f9f", + "productName": "IV Drip Stand", + "active": true, + "materialNumber": "SKU010", + "isAttractiveItem": false, + "availability": "In stock and available for immediate use.", + "condition": "Well-maintained, with adjustable height settings.", + "appropriateUsage": "Ensure the stand is stable and properly sanitized before use with each patient.", + "accountabilityPeriod": 36, + "imageSourceUrl": "http://example.com/images/10.jpg" + }, + { + "id": "127c61b4-bfb4-4200-8b0b-1783d5892c44", + "productName": "Portable ECG Machine", + "active": true, + "materialNumber": "SKU011", + "isAttractiveItem": true, + "availability": "Available in main storage. Can be ordered for rapid deployment.", + "condition": "New and fully operational with user manual included.", + "appropriateUsage": "Only trained professionals should use the machine for accurate ECG readings.", + "accountabilityPeriod": 36, + "imageSourceUrl": "http://example.com/images/11.jpg" + }, + { + "id": "259e2cd7-d9f6-450f-bc15-5b072c15e7e3", + "productName": "Digital Blood Glucose Meter", + "active": true, + "materialNumber": "SKU012", + "isAttractiveItem": false, + "availability": "Limited stock available, restock is expected in 2 weeks.", + "condition": "Brand new with spare test strips included in the package.", + "appropriateUsage": "Ensure regular calibration for accurate blood glucose readings.", + "accountabilityPeriod": 24, + "imageSourceUrl": "http://example.com/images/12.jpg" + }, + { + "id": "2c736b90-2bf7-4f1b-9616-267726c755d0", + "productName": "Wheelchair", + "active": false, + "materialNumber": "SKU013", + "isAttractiveItem": true, + "availability": "Currently on backorder due to high demand. Check alternative suppliers.", + "condition": "Used but in good condition. Regular maintenance required.", + "appropriateUsage": "Ensure the wheelchair is locked when stationary to avoid accidents.", + "accountabilityPeriod": 60, + "imageSourceUrl": "http://example.com/images/13.jpg" + }, + { + "id": "3d03a209-7a4e-49d5-9c33-745ad72e71e4", + "productName": "Surgical Scalpel Set", + "active": true, + "materialNumber": "SKU014", + "isAttractiveItem": false, + "availability": "In stock and ready for use in all surgical kits.", + "condition": "Sterile and sharp. Each set is individually packed.", + "appropriateUsage": "To be used only in sterile environments by trained surgeons.", + "accountabilityPeriod": 12, + "imageSourceUrl": "http://example.com/images/14.jpg" + }, + { + "id": "418cedab-93f5-4e45-b769-61f65ef90d4e", + "productName": "Digital Thermometer - Contactless", + "active": true, + "materialNumber": "SKU015", + "isAttractiveItem": true, + "availability": "High availability in all regional branches.", + "condition": "New and fully operational with quick response time.", + "appropriateUsage": "Suitable for use in high-traffic areas for quick temperature checks.", + "accountabilityPeriod": 24, + "imageSourceUrl": "http://example.com/images/15.jpg" + }, + { + "id": "46c0e1fb-1321-4d83-b748-8ed2e780dfaa", + "productName": "Disposable Surgical Gown", + "active": true, + "materialNumber": "SKU016", + "isAttractiveItem": false, + "availability": "Available in bulk. Contact supply chain for large orders.", + "condition": "Sterile and individually packed. Ready for use in surgical procedures.", + "appropriateUsage": "Dispose of after each use to maintain hygiene standards.", + "accountabilityPeriod": 6, + "imageSourceUrl": "http://example.com/images/16.jpg" + }, + { + "id": "526c835e-b92f-4f67-8a87-52adfa92d321", + "productName": "Emergency Oxygen Cylinder", + "active": true, + "materialNumber": "SKU017", + "isAttractiveItem": true, + "availability": "Available for immediate dispatch. Check expiry date before use.", + "condition": "Newly filled and sealed. Ensure valves are intact.", + "appropriateUsage": "Only certified professionals should handle and operate the cylinder.", + "accountabilityPeriod": 36, + "imageSourceUrl": "http://example.com/images/17.jpg" + }, + { + "id": "5bf6b9ec-9a0a-44d5-bc3a-9b34db5d872f", + "productName": "Electric Hospital Bed", + "active": false, + "materialNumber": "SKU018", + "isAttractiveItem": true, + "availability": "Currently out of stock due to high demand. Estimated restock in 4 weeks.", + "condition": "Used but well-maintained with all electronic components working.", + "appropriateUsage": "Regular maintenance checks are required to ensure safety and functionality.", + "accountabilityPeriod": 60, + "imageSourceUrl": "http://example.com/images/18.jpg" + }, + { + "id": "63e3efeb-3e9f-45c9-83bc-0c065cf8bbff", + "productName": "Digital Weighing Scale", + "active": true, + "materialNumber": "SKU019", + "isAttractiveItem": false, + "availability": "Stock available in central warehouse. Ready for deployment.", + "condition": "Brand new with accuracy guarantee up to 150kg.", + "appropriateUsage": "Ensure regular calibration and proper cleaning after each use.", + "accountabilityPeriod": 36, + "imageSourceUrl": "http://example.com/images/19.jpg" + }, + { + "id": "7343fba6-88c2-4f08-b6ff-d5148d90b702", + "productName": "IV Fluid Stand", + "active": true, + "materialNumber": "SKU020", + "isAttractiveItem": false, + "availability": "Available in all clinics. Contact supply manager for bulk orders.", + "condition": "Good condition with adjustable height settings.", + "appropriateUsage": "Ensure stand stability to prevent tipping during IV administration.", + "accountabilityPeriod": 36, + "imageSourceUrl": "http://example.com/images/20.jpg" + }, + { + "id": "86aaf109-43a6-403d-8b6f-46c056d65d5b", + "productName": "Hand Sanitizer Dispenser", + "active": true, + "materialNumber": "SKU021", + "isAttractiveItem": false, + "availability": "Widely available. Stock levels are sufficient for large orders.", + "condition": "New and ready to use. Comes with refill instructions.", + "appropriateUsage": "Place in high-traffic areas for easy access to hand hygiene.", + "accountabilityPeriod": 12, + "imageSourceUrl": "http://example.com/images/21.jpg" + }, + { + "id": "964a09b7-b59a-4f5d-bc48-6fada6e9c8db", + "productName": "Medical Examination Gloves", + "active": true, + "materialNumber": "SKU022", + "isAttractiveItem": false, + "availability": "High availability. Ready for dispatch in small or bulk quantities.", + "condition": "Sterile, latex-free, and ready for immediate use.", + "appropriateUsage": "Dispose of after a single use to maintain hygiene standards.", + "accountabilityPeriod": 6, + "imageSourceUrl": "http://example.com/images/22.jpg" + }, + { + "id": "a45c8e5b-0ff8-4d7e-89b1-9f76a7d87b63", + "productName": "Sterile Surgical Drapes", + "active": true, + "materialNumber": "SKU023", + "isAttractiveItem": false, + "availability": "Stock available. Contact supply chain for large orders.", + "condition": "New, sterile, and individually packed for surgery use.", + "appropriateUsage": "Use in sterile surgical environments. Dispose of after single use.", + "accountabilityPeriod": 6, + "imageSourceUrl": "http://example.com/images/23.jpg" + }, + { + "id": "b7e8cfc2-e4e7-43a4-9c2a-729c7efebf1f", + "productName": "Hospital Bedside Table", + "active": true, + "materialNumber": "SKU024", + "isAttractiveItem": false, + "availability": "Available in limited quantities. Contact supplier for large orders.", + "condition": "Good condition with no visible damage. Easy to clean surface.", + "appropriateUsage": "Ideal for use in patient rooms for storing personal items and medical supplies", + "accountabilityPeriod": 24 + }, + { + "id": "c24d3bb3-8fc1-4f87-89e1-638c5f8c99b7", + "productName": "Sterile Cotton Swabs", + "active": true, + "materialNumber": "SKU025", + "isAttractiveItem": false, + "availability": "Readily available in all medical supply stores and warehouses.", + "condition": "Sterile, individually packed, and ready for immediate use.", + "appropriateUsage": "Use for cleaning wounds or applying medication. Dispose after single use.", + "accountabilityPeriod": 6, + "imageSourceUrl": "http://example.com/images/25.jpg" + }, + { + "id": "d1f4a67c-bb32-4988-9454-9c06c126eeaf", + "productName": "Sterilization Pouch", + "active": true, + "materialNumber": "SKU026", + "isAttractiveItem": false, + "availability": "Available in bulk. Contact supply chain for large quantity orders.", + "condition": "New and sterile, ready for use in autoclaves.", + "appropriateUsage": "Place surgical instruments inside before sterilizing in an autoclave.", + "accountabilityPeriod": 24, + "imageSourceUrl": "http://example.com/images/26.jpg" + }, + { + "id": "e7c45f2f-d9d8-4f19-83a7-34a215627594", + "productName": "Syringe Disposal Container", + "active": true, + "materialNumber": "SKU027", + "isAttractiveItem": false, + "availability": "Stock available in all clinics and hospitals.", + "condition": "New, puncture-resistant containers with secure lids.", + "appropriateUsage": "Use to safely dispose of used syringes and needles. Replace when full.", + "accountabilityPeriod": 36, + "imageSourceUrl": "http://example.com/images/27.jpg" + }, + { + "id": "f9a3d2c4-ec65-4b56-9204-1a4356765b24", + "productName": "Disposable Face Shield", + "active": true, + "materialNumber": "SKU028", + "isAttractiveItem": false, + "availability": "Widely available for all healthcare personnel.", + "condition": "New and sterile. Each face shield is individually wrapped.", + "appropriateUsage": "Use to protect face from exposure to fluids. Dispose after single use.", + "accountabilityPeriod": 12, + "imageSourceUrl": "http://example.com/images/28.jpg" + }, + { + "id": "07d4c57e-60f4-4b28-93c0-cc6f96f4352a", + "productName": "Infrared Thermometer", + "active": true, + "materialNumber": "SKU029", + "isAttractiveItem": true, + "availability": "Available in limited quantities. Bulk orders may take 2 weeks.", + "condition": "New and accurate for non-contact temperature readings.", + "appropriateUsage": "Use for quick and hygienic temperature checks. Regular calibration recommended.", + "accountabilityPeriod": 18, + "imageSourceUrl": "http://example.com/images/29.jpg" + }, + { + "id": "186d01c3-07b7-4a74-917b-4db3e9ec9d52", + "productName": "Patient Identification Wristbands", + "active": true, + "materialNumber": "SKU030", + "isAttractiveItem": false, + "availability": "In stock and ready for distribution across all facilities.", + "condition": "New, adjustable, and easy to use with writable surfaces.", + "appropriateUsage": "Use to accurately identify patients in a hospital setting. Replace as needed.", + "accountabilityPeriod": 24, + "imageSourceUrl": "http://example.com/images/30.jpg" + }, + { + "id": "2a3fe17c-1d67-411b-bdf8-19fa3f7614d7", + "productName": "Nebulizer Machine", + "active": true, + "materialNumber": "SKU031", + "isAttractiveItem": true, + "availability": "Currently in stock. Limited units available for immediate use.", + "condition": "Good working condition. Comes with mask and tubing.", + "appropriateUsage": "For use with prescribed medication to administer aerosol treatments.", + "accountabilityPeriod": 36, + "imageSourceUrl": "http://example.com/images/31.jpg" + }, + { + "id": "39fc654e-1cc8-41c0-9246-b8a971da8d93", + "productName": "Disposable Syringes", + "active": true, + "materialNumber": "SKU032", + "isAttractiveItem": false, + "availability": "High availability in all medical supply chains.", + "condition": "New, sterile, and individually packed for single use.", + "appropriateUsage": "Use for injections and then safely dispose of in a sharps container.", + "accountabilityPeriod": 12, + "imageSourceUrl": "http://example.com/images/32.jpg" + }, + { + "id": "437e3de0-f2a7-4b6e-9ba4-399e95f6a45e", + "productName": "First Aid Kit", + "active": true, + "materialNumber": "SKU033", + "isAttractiveItem": false, + "availability": "Available in all departments. Ready for immediate use.", + "condition": "New with all necessary first aid items included.", + "appropriateUsage": "For emergency response in medical and non-medical settings.", + "accountabilityPeriod": 24, + "imageSourceUrl": "http://example.com/images/33.jpg" + }, + { + "id": "521c09c5-5074-47d7-b22c-0fefc3bb8f98", + "productName": "Surgical Mask with Ear Loops", + "active": true, + "materialNumber": "SKU034", + "isAttractiveItem": false, + "availability": "Available in bulk. High demand, but stock is sufficient.", + "condition": "New, sterile, and individually wrapped.", + "appropriateUsage": "Wear during surgical procedures or to prevent disease transmission.", + "accountabilityPeriod": 12, + "imageSourceUrl": "http://example.com/images/34.jpg" + }, + { + "id": "6491bfa0-1d72-4f03-906f-0c15475d92d6", + "productName": "Antiseptic Wipes", + "active": true, + "materialNumber": "SKU035", + "isAttractiveItem": false, + "availability": "Readily available in all storage facilities.", + "condition": "New, sterile, and individually packed for single use.", + "appropriateUsage": "Use for cleaning and disinfecting surfaces. Dispose of after use.", + "accountabilityPeriod": 6, + "imageSourceUrl": "http://example.com/images/35.jpg" + }, + { + "id": "754ebc5a-06bc-4e3f-8df8-f27ec4a5dd8a", + "productName": "Medical Waste Disposal Bag", + "active": true, + "materialNumber": "SKU036", + "isAttractiveItem": false, + "availability": "Stock available. Contact for bulk orders.", + "condition": "New and strong, made for safe disposal of medical waste.", + "appropriateUsage": "Use to safely dispose of medical waste. Ensure bags are securely tied.", + "accountabilityPeriod": 12, + "imageSourceUrl": "http://example.com/images/36.jpg" + }, + { + "id": "86b27e8f-e5cd-4702-9119-c3cdefbde308", + "productName": "Surgical Blade", + "active": true, + "materialNumber": "SKU037", + "isAttractiveItem": true, + "availability": "Available in surgical kits. Individual purchase available on request.", + "condition": "Sterile and sharp. Ready for use in surgical procedures.", + "appropriateUsage": "Use for precision cutting during surgeries. Dispose of after single use.", + "accountabilityPeriod": 18, + "imageSourceUrl": "http://example.com/images/37.jpg" + }, + { + "id": "97a24d38-9513-4cc2-bc69-61ef0191fbb3", + "productName": "Sterile Bandage Rolls", + "active": true, + "materialNumber": "SKU038", + "isAttractiveItem": false, + "availability": "High availability. Stock is sufficient for emergency needs.", + "condition": "New, sterile, and individually wrapped.", + "appropriateUsage": "Use to cover wounds and protect from infection. Replace as needed.", + "accountabilityPeriod": 24, + "imageSourceUrl": "http://example.com/images/38.jpg" + }, + { + "id": "a91c4eab-50df-4fa4-97f5-3404f0f729f3", + "productName": "Handheld Pulse Oximeter", + "active": true, + "materialNumber": "SKU039", + "isAttractiveItem": true, + "availability": "Available for immediate use in all departments.", + "condition": "New and accurate with digital display.", + "appropriateUsage": "Use to monitor blood oxygen levels. Clean sensor after each use.", + "accountabilityPeriod": 36, + "imageSourceUrl": "http://example.com/images/39.jpg" + }, + { + "id": "b9c7e5bc-91b8-47d5-92d6-0e8b2a67e027", + "productName": "Wheelchair Cushion", + "active": true, + "materialNumber": "SKU040", + "isAttractiveItem": false, + "availability": "Stock available. Can be ordered in different sizes.", + "condition": "New, foam cushion with washable cover.", + "appropriateUsage": "Place on wheelchair seat for added comfort. Regular cleaning recommended.", + "accountabilityPeriod": 48, + "imageSourceUrl": "http://example.com/images/40.jpg" + }, + { + "id": "c9724e8c-8d84-4a73-9d5b-79f74d0b94ed", + "productName": "Surgical Sutures", + "active": true, + "materialNumber": "SKU041", + "isAttractiveItem": false, + "availability": "Available in all surgical departments. Check packaging for expiry.", + "condition": "Sterile, with needle attached for quick use.", + "appropriateUsage": "Use to close surgical incisions. Dispose of needle safely after use.", + "accountabilityPeriod": 24, + "imageSourceUrl": "http://example.com/images/41.jpg" + }, + { + "id": "d9823fb6-2fb4-45ad-9b8c-6e0cddef3e6e", + "productName": "IV Cannula", + "active": true, + "materialNumber": "SKU042", + "isAttractiveItem": false, + "availability": "Stock available in all healthcare facilities.", + "condition": "New and sterile. Ready for use in IV administration.", + "appropriateUsage": "Use to insert into a vein for fluid administration. Dispose after use.", + "accountabilityPeriod": 18, + "imageSourceUrl": "http://example.com/images/42.jpg" + } + ] diff --git a/importer/stub_data_gen/stub_gen.py b/importer/stub_data_gen/stub_gen.py new file mode 100644 index 00000000..8c89d517 --- /dev/null +++ b/importer/stub_data_gen/stub_gen.py @@ -0,0 +1,235 @@ +import json +from faker import Faker +import math +import os + +from fixtures import unicef_sections, inventory_donors, location_physical_types, service_point_types + +fake = Faker() + + +def generate_users(count: int): + header = [ + "firstName", "lastName", "username", "email", "practitionerId", + "userType", "enableUser", "keycloakGroupId", "keycloakGroupName", + "appId", "password" + ] + + # Function to generate random row data + def generate_user_row(index: int): + f_name = fake.first_name() + l_name = fake.last_name() + u_name = fake.user_name() + email = fake.email() + user_id = fake.uuid4() + user_type = fake.random_element(["Practitioner", "Supervisor", ""]) + enable_user = fake.random_element(["true", "false"]) + group_ids = "" + group_names = "" + app_id = fake.random_element(["quest", "core", "anc", ""]) + password = fake.password() + return [f_name, l_name, u_name, email, user_id, user_type, enable_user, group_ids, group_names, app_id, + password] + + rows = [] + for cursor in range(1, count+1): + rows.append(generate_user_row(cursor)) + return [header, rows] + + +def generate_organizations(count: int): + header = [ + "orgName", "orgActive", "method", "orgId", "identifier" + ] + + # Function to generate random row data + def generate_row(idx: int): + org_name = fake.company() + active = fake.random_element(["true", "false"]) + method = "create" + org_id = fake.uuid4() + identifier = fake.uuid4() + + return [org_name, active, method, org_id, identifier] + + rows = [] + for cursor in range(1, count+1): + rows.append(generate_row(cursor)) + return [header, rows] + + +def generate_locations(count: int): + header = [ + "locationName", "locationStatus", "method", "locationId", "locationParentName", "locationParentId", + "locationType", "locationTypeCode", "locationAdminLevel", "locationPhysicalType", "locationPhysicalTypeCode", + "longitude", "latitude" + ] + + # Function to generate random row data + def generate_row(cursor: int, generated_locations): + name = fake.city() + parent_location = None + if len(generated_locations): + parent_location = fake.random_element(generated_locations) + physical_type_code = fake.random_element(location_physical_types) + type_code = fake.random_element(service_point_types) + status = fake.random_element(["active", "inactive", "suspended"]) + method = "create" + id = fake.uuid4() + loc_type = type_code.get("display") + loc_type_code = type_code.get("code") + loc_admin_level = 1 + loc_physical_type = physical_type_code.get("display") + loc_physical_type_code = physical_type_code.get("code") + parent_loc_name = "" + parent_loc_id = "" + latitude = "" + longitude = "" + if parent_location: + parent_admin_level = int(parent_location[8]) + parent_loc_name = parent_location[0] + parent_loc_id = parent_location[3] + loc_admin_level = str(parent_admin_level + 1) + if int(loc_admin_level) > 3: + latitude = fake.geo_coordinate() + longitude = fake.geo_coordinate() + return [name, status, method, id, parent_loc_name, parent_loc_id, loc_type, loc_type_code, loc_admin_level, loc_physical_type, + loc_physical_type_code, longitude, latitude] + + rows = [] + for idx in range(1, count + 1): + rows.append(generate_row(idx, rows)) + return [header, rows] + + +def generate_org_to_locations(organizations, locations, count: int): + header = [ + "orgName", "orgId", "locationName", "locationId", + ] + + # Function to generate random row data + def generate_row(_: int): + loc = fake.random_element(locations) + org = fake.random_element(organizations) + + return [org[0], org[3], loc[0], loc[3]] + + rows = [] + for cursor in range(1, count+1): + rows.append(generate_row(cursor)) + return [header, rows] + + +def generate_care_teams(organizations, practitioner_users, count: int): + header = [ + "name", "status", "method", "id", "identifier", "organizations", + "participants" + ] + + # Function to generate random row data + def generate_row(_): + name = fake.name() + status = fake.random_element(["active", "inactive"]) + method = "create" + id = fake.uuid4() + identifier = fake.uuid4() + try: + random_orgs = fake.random_sample(organizations, math.floor(fake.random.random() * 5)) + random_parts = fake.random_sample(practitioner_users, math.floor(fake.random.random() * 5)) + except IndexError: + random_parts = [] + random_orgs = [] + + assigned_orgs = "|".join([f"{org[3]}:{org[0]}" for org in random_orgs]) + assigned_parts = "|".join([f"{user[4]}:{user[0]} {user[1]}" for user in random_parts]) + + + return [name, status, method, id, identifier, assigned_orgs, assigned_parts] + + rows = [] + for cursor in range(1, count + 1): + rows.append(generate_row(cursor)) + return [header, rows] + +products_samples_lookup = os.path.normpath(os.path.join(os.path.realpath(__file__), "../product_samples.json")) +with open(products_samples_lookup, "r") as file: + + json_data = file.read() + productSample = json.loads(json_data) + + +def generate_products(count): + header = [ + "name", "active", "method", "id", "materialNumber", "previousId", "isAttractiveItem", "availability", "condition" + , "appropriateUsage", "accountabilityPeriod", "imageSourceUrl" + ] + + rows = [] + for i in range(1, count + 1): + template = fake.random_element(productSample) + identifier = fake.uuid4() + name = template.get("productName") + material_number = f"SKU00{i}" + name = f"{material_number}-{name}" + attractive_item = template.get("isAttractiveItem") + availability = template.get("availability") + condition = template.get("condition") + appropriate_usage = template.get("appropriateUsage") + accountability = template.get("accountabilityPeriod") + image_source_url = "" + status = fake.random_element(["true", "false"]) + rows.append( + [name, status, "create", identifier, material_number, "", attractive_item, availability, condition, appropriate_usage, + accountability, image_source_url]) + + return [header, rows] + + +def generate_inventory(locations, products, count: int): + header = [ + "name", "active", "method", "id", "poNumber", "serialNumber", "usualId", "actual", "productId", "deliveryDate" + , "accountabilityDate", "quantity", "unicefSection", "donor", "location" + ] + rows = [] + for i in range(1, count + 1): + location_record = fake.random_element(locations) + product_record = fake.random_element(products) + location_name = location_record[0] + product_name = product_record[0] + product_name = f"{location_name} - {product_name}" + status = fake.random_element(["active", "inactive"]) + attractive_item = "true" if product_record[6] else "false" + inventory_id = fake.uuid4() + serial_number = "" + po_umber = f"PONUM{i}" + if attractive_item: + serial_number = f"SKNO{i}" + delivery_date = "" + product_id = product_record[3] + quantity = math.ceil(fake.random.random() * 50) + unicef_section = fake.random_element(unicef_sections).get("code") + donor = fake.random_element(inventory_donors).get("code") + location = location_record[3] + + to_append = [product_name, status, "create", inventory_id, po_umber, serial_number,"","true", product_id, delivery_date, "", quantity, unicef_section, donor, location] + rows.append(to_append) + return [header, rows] + + +def generate_users_orgs(users, organizations, count: int): + header = [ + "username", "userId", "orgName", "orgId" + ] + def generate_row(): + random_org = fake.random_element(organizations) + random_user = fake.random_element(users) + org_id = random_org[3] + org_name = random_org[0] + user_id = random_user[4] + user_name = f"{random_user[0]} {random_user[1]}" + return [user_name, user_id, org_name, org_id, ] + + rows = [] + for cursor in range(1, count + 1): + rows.append(generate_row()) + return [header, rows] diff --git a/importer/stub_data_gen/utils.py b/importer/stub_data_gen/utils.py new file mode 100644 index 00000000..86be1345 --- /dev/null +++ b/importer/stub_data_gen/utils.py @@ -0,0 +1,42 @@ +import csv +import os +from typing import Literal + +default_out_dir = os.path.normpath(os.path.join(os.path.realpath(__file__), "../../out")) + +def write_csv(csv_path, data): + [header, rows] = data + with open(csv_path, mode='w', newline='') as file: + writer = csv.writer(file) + writer.writerow(header) + writer.writerows(rows) + print(f"CSV file '{csv_path}' with {len(data[1])} rows has been created.") + +def read_full_csv(csv_path): + with open(csv_path, "r") as file: + reader = csv.reader(file) + header = reader.__next__() + rows = [row for row in reader] + return [header, rows] + +ResourceTypes = Literal["users", "careteams", "locations", "orgs_locs", "users_orgs", "products", "inventories", "organizations"] +def write_resource(resource_type: ResourceTypes, out_dir, data): + if resource_type == "users": + csv_path = os.path.join(out_dir, 'users.csv') + elif resource_type == "careteams": + csv_path = os.path.join(out_dir, 'careteams.csv') + elif resource_type == "locations": + csv_path = os.path.join(out_dir, 'locations.csv') + elif resource_type == "orgs_locs": + csv_path = os.path.join(out_dir, 'orgs_locs.csv') + elif resource_type == "users_orgs": + csv_path = os.path.join(out_dir, 'users_orgs.csv') + elif resource_type == "products": + csv_path = os.path.join(out_dir, 'products.csv') + elif resource_type == "inventories": + csv_path = os.path.join(out_dir, 'inventories.csv') + elif resource_type == "organizations": + csv_path = os.path.join(out_dir, 'organizations.csv') + else: + raise TypeError("resource type is unknown") + write_csv(csv_path, data)