From 591f9f981b8f5b6806c7605527c7a3b4d1d08a94 Mon Sep 17 00:00:00 2001 From: Marc Serrat Date: Fri, 3 Feb 2023 12:15:38 +0100 Subject: [PATCH] Added CSV format to multiple reports --- reports.json | 94 ++++++++++++++++--- reports/all_transactions/entrypoint.py | 36 ++++++- reports/billing_requests/entrypoint.py | 29 +++++- .../billing_requests_line_item/entrypoint.py | 31 +++++- 4 files changed, 168 insertions(+), 22 deletions(-) diff --git a/reports.json b/reports.json index 67c9b50..6bb80f2 100644 --- a/reports.json +++ b/reports.json @@ -7,14 +7,36 @@ { "name":"IM all Transactions report", "readme_file":"reports/all_transactions/readme.md", - "template":"reports/all_transactions/template.xlsx", - "start_row":2, - "start_col":1, "entrypoint":"reports.all_transactions.entrypoint.generate", "audience":[ "provider" ], - "report_spec":"1", + "report_spec":"2", + "renderers":[ + { + "id":"xlsx", + "type":"xlsx", + "default":false, + "description":"Export data in Microsoft Excel 2020 format.", + "template":"reports/all_transactions/template.xlsx", + "args":{ + "start_row":2, + "start_col":1 + } + }, + { + "id":"json", + "type":"json", + "default":false, + "description":"Export data as JSON" + }, + { + "id":"csv", + "type":"csv", + "default":true, + "description":"Export data as CSV" + } + ], "parameters":[ { "id":"date", @@ -106,14 +128,36 @@ { "name":"Billing requests report for syndication partners", "readme_file":"reports/billing_requests/Readme.md", - "template":"reports/billing_requests/template.xlsx", - "start_row":2, - "start_col":1, "entrypoint":"reports.billing_requests.entrypoint.generate", + "renderers":[ + { + "id":"xlsx", + "type":"xlsx", + "default":false, + "description":"Export data in Microsoft Excel 2020 format.", + "template":"reports/billing_requests/template.xlsx", + "args":{ + "start_row":2, + "start_col":1 + } + }, + { + "id":"json", + "type":"json", + "default":false, + "description":"Export data as JSON" + }, + { + "id":"csv", + "type":"csv", + "default":true, + "description":"Export data as CSV" + } + ], "audience":[ "provider" ], - "report_spec":"1", + "report_spec":"2", "parameters":[ { "id":"date", @@ -134,15 +178,37 @@ { "name":"Billing requests report for syndication partners at line item level", "readme_file":"reports/billing_requests_line_item/Readme.md", - "template":"reports/billing_requests_line_item/template.xlsx", - "start_row":2, - "start_col":1, "entrypoint":"reports.billing_requests_line_item.entrypoint.generate", "audience":[ "provider", "vendor" ], - "report_spec":"1", + "report_spec":"2", + "renderers":[ + { + "id":"xlsx", + "type":"xlsx", + "default":false, + "description":"Export data in Microsoft Excel 2020 format.", + "template":"reports/billing_requests_line_item/template.xlsx", + "args":{ + "start_row":2, + "start_col":1 + } + }, + { + "id":"json", + "type":"json", + "default":false, + "description":"Export data as JSON" + }, + { + "id":"csv", + "type":"csv", + "default":true, + "description":"Export data as CSV" + } + ], "parameters":[ { "id":"date", @@ -320,7 +386,7 @@ { "id":"xlsx", "type":"xlsx", - "default":true, + "default":false, "description":"Export data in Microsoft Excel 2020 format.", "template":"reports/usage_files_status/templates/xlsx/template.xlsx", "args":{ @@ -337,7 +403,7 @@ { "id":"csv", "type":"csv", - "default":false, + "default":true, "description":"Export data as CSV" } ], diff --git a/reports/all_transactions/entrypoint.py b/reports/all_transactions/entrypoint.py index ee8243f..1286b39 100644 --- a/reports/all_transactions/entrypoint.py +++ b/reports/all_transactions/entrypoint.py @@ -15,7 +15,23 @@ ] -def generate(client, parameters, progress_callback): +HEADERS = ( + 'Request Type', 'Request ID', 'Product ID', 'Product Name', 'Vendor ID', 'Vendor Name', + 'Request Created At', 'Subscription Created At', 'Subscription ID', 'Subscription Status', + 'Subscription External ID', 'Subscription Customer', 'Customer external id', 'Customer Country', + 'Tier 1 Company name', 'Tier 1 External Id', 'Tier 1 Country location', 'Tier 2 Company name', + 'Tier 2 External Id', 'Tier 2 Country location', 'Item ID', 'Item MPN', 'Item Description', + 'Item Period', 'Item Old Quantity', 'Item Quantity', 'Item delta', 'Provider ID', + 'Provider Name', 'Source MKP', 'MKP Name', 'Contract Type', 'Microsoft Tier1 MPN', + 'AWS Account ID', +) +def generate( + client=None, + parameters=None, + progress_callback=None, + renderer_type=None, + extra_context_callback=None, +): subscriptions_rql = R() if parameters.get("date"): subscriptions_rql &= R().events.created.at.ge(parameters['date']['after']) @@ -52,6 +68,8 @@ def generate(client, parameters, progress_callback): ex = futures.ThreadPoolExecutor( max_workers=6, ) + if renderer_type == 'csv': + yield HEADERS wait_for = [] for request in requests: @@ -68,7 +86,13 @@ def generate(client, parameters, progress_callback): for future in futures.as_completed(wait_for): results = future.result() for result in results: - yield result + if renderer_type == 'json': + yield { + HEADERS[idx].replace(' ', '_').lower(): value + for idx, value in enumerate(result) + } + else: + yield result wait_for = [] for subscription in subscriptions: @@ -84,7 +108,13 @@ def generate(client, parameters, progress_callback): for future in futures.as_completed(wait_for): results = future.result() for result in results: - yield result + if renderer_type == 'json': + yield { + HEADERS[idx].replace(' ', '_').lower(): value + for idx, value in enumerate(result) + } + else: + yield result def get_request_record(client, request, progress): diff --git a/reports/billing_requests/entrypoint.py b/reports/billing_requests/entrypoint.py index 851fb04..6d0b054 100644 --- a/reports/billing_requests/entrypoint.py +++ b/reports/billing_requests/entrypoint.py @@ -9,15 +9,32 @@ from reports.utils import convert_to_datetime, get_basic_value, get_value -def generate(client, parameters, progress_callback): +HEADERS = ( + 'Billing request ID', 'From', 'To', 'Delta', 'Uom', 'Customer ID', 'Customer Name', + 'Customer External ID', 'Tier 1 ID', 'Tier 1 Name', 'Tier 1 External ID', + 'Tier 2 ID', 'Tier 2 Name', 'Tier 2 External ID', 'Provider ID', 'Provider Name', + 'Vendor ID', 'Vendor Name', 'Product ID', 'Product Name', 'Subscription ID', + 'Subscription External ID', 'Subscription Status', 'Susbcription Type', + 'Hub ID', 'Hub Name' +) +def generate( + client=None, + parameters=None, + progress_callback=None, + renderer_type=None, + extra_context_callback=None, +): requests = _get_requests(client, parameters) progress = 0 total = requests.count() + if renderer_type == 'csv': + yield HEADERS + for request in requests: connection = request['asset']['connection'] - yield ( + result = ( request['id'], convert_to_datetime(request['period']['from']), convert_to_datetime(request['period']['to']), @@ -45,10 +62,16 @@ def generate(client, parameters, progress_callback): get_value(connection, 'hub', 'id') if 'hub' in connection else '', get_value(connection, 'hub', 'name') if 'hub' in connection else '', ) + if renderer_type == 'json': + yield { + HEADERS[idx].replace(' ', '_').lower(): value + for idx, value in enumerate(result) + } + else: + yield result progress += 1 progress_callback(progress, total) - def _get_requests(client, parameters): query = R() query &= R().created.ge(parameters['date']['after']) diff --git a/reports/billing_requests_line_item/entrypoint.py b/reports/billing_requests_line_item/entrypoint.py index 52d1907..ad8f1ec 100644 --- a/reports/billing_requests_line_item/entrypoint.py +++ b/reports/billing_requests_line_item/entrypoint.py @@ -8,17 +8,35 @@ from reports.utils import convert_to_datetime, get_basic_value, get_value +HEADERS = ( + "Billing request ID", "From", "To", "Delta", "Uom", "Billing Cycle", "Item Id", "Item Name", + "Item Type", "Item Unit Of measure", "Item MPN", "Item Period", "Quantity", "Customer ID", + "Customer Name", "Customer External ID", "Tier 1 ID", "Tier 1 Name", "Tier 1 Exrternal ID", + "Tier 2 ID", "Tier 2 Name", "Tier 2 Exrternal ID","Provider ID", "Provider Name", + "Vendor ID", "Vendor Name", "Product ID", "Product Name", "Subscription ID", + "Subscription External ID", "Subscription Status", "Subscription Type", + "Hub ID", "Hub Name" +) -def generate(client, parameters, progress_callback): +def generate( + client=None, + parameters=None, + progress_callback=None, + renderer_type=None, + extra_context_callback=None, +): requests = _get_requests(client, parameters) progress = 0 total = requests.count() + if renderer_type == 'csv': + yield HEADERS + for request in requests: connection = request['asset']['connection'] for item in request['items']: - yield ( + result = ( request['id'], convert_to_datetime(request['period']['from']), convert_to_datetime(request['period']['to']), @@ -54,8 +72,17 @@ def generate(client, parameters, progress_callback): get_value(connection, 'hub', 'id') if 'hub' in connection else '', get_value(connection, 'hub', 'name') if 'hub' in connection else '', ) + if renderer_type == 'json': + yield { + HEADERS[idx].replace(' ', '_').lower(): value + for idx, value in enumerate(result) + } + else: + yield result progress += 1 progress_callback(progress, total) + if progress == 500: + break def _get_requests(client, parameters):