Skip to content

Commit

Permalink
Added CSV format to multiple reports
Browse files Browse the repository at this point in the history
  • Loading branch information
@marcserrat
marcserrat committed Feb 3, 2023
1 parent 27edf43 commit 591f9f9
Show file tree
Hide file tree
Showing 4 changed files with 168 additions and 22 deletions.
94 changes: 80 additions & 14 deletions reports.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,36 @@
{
"name":"IM all Transactions report",
"readme_file":"reports/all_transactions/readme.md",
"template":"reports/all_transactions/template.xlsx",
"start_row":2,
"start_col":1,
"entrypoint":"reports.all_transactions.entrypoint.generate",
"audience":[
"provider"
],
"report_spec":"1",
"report_spec":"2",
"renderers":[
{
"id":"xlsx",
"type":"xlsx",
"default":false,
"description":"Export data in Microsoft Excel 2020 format.",
"template":"reports/all_transactions/template.xlsx",
"args":{
"start_row":2,
"start_col":1
}
},
{
"id":"json",
"type":"json",
"default":false,
"description":"Export data as JSON"
},
{
"id":"csv",
"type":"csv",
"default":true,
"description":"Export data as CSV"
}
],
"parameters":[
{
"id":"date",
Expand Down Expand Up @@ -106,14 +128,36 @@
{
"name":"Billing requests report for syndication partners",
"readme_file":"reports/billing_requests/Readme.md",
"template":"reports/billing_requests/template.xlsx",
"start_row":2,
"start_col":1,
"entrypoint":"reports.billing_requests.entrypoint.generate",
"renderers":[
{
"id":"xlsx",
"type":"xlsx",
"default":false,
"description":"Export data in Microsoft Excel 2020 format.",
"template":"reports/billing_requests/template.xlsx",
"args":{
"start_row":2,
"start_col":1
}
},
{
"id":"json",
"type":"json",
"default":false,
"description":"Export data as JSON"
},
{
"id":"csv",
"type":"csv",
"default":true,
"description":"Export data as CSV"
}
],
"audience":[
"provider"
],
"report_spec":"1",
"report_spec":"2",
"parameters":[
{
"id":"date",
Expand All @@ -134,15 +178,37 @@
{
"name":"Billing requests report for syndication partners at line item level",
"readme_file":"reports/billing_requests_line_item/Readme.md",
"template":"reports/billing_requests_line_item/template.xlsx",
"start_row":2,
"start_col":1,
"entrypoint":"reports.billing_requests_line_item.entrypoint.generate",
"audience":[
"provider",
"vendor"
],
"report_spec":"1",
"report_spec":"2",
"renderers":[
{
"id":"xlsx",
"type":"xlsx",
"default":false,
"description":"Export data in Microsoft Excel 2020 format.",
"template":"reports/billing_requests_line_item/template.xlsx",
"args":{
"start_row":2,
"start_col":1
}
},
{
"id":"json",
"type":"json",
"default":false,
"description":"Export data as JSON"
},
{
"id":"csv",
"type":"csv",
"default":true,
"description":"Export data as CSV"
}
],
"parameters":[
{
"id":"date",
Expand Down Expand Up @@ -320,7 +386,7 @@
{
"id":"xlsx",
"type":"xlsx",
"default":true,
"default":false,
"description":"Export data in Microsoft Excel 2020 format.",
"template":"reports/usage_files_status/templates/xlsx/template.xlsx",
"args":{
Expand All @@ -337,7 +403,7 @@
{
"id":"csv",
"type":"csv",
"default":false,
"default":true,
"description":"Export data as CSV"
}
],
Expand Down
36 changes: 33 additions & 3 deletions reports/all_transactions/entrypoint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,23 @@
]


def generate(client, parameters, progress_callback):
HEADERS = (
'Request Type', 'Request ID', 'Product ID', 'Product Name', 'Vendor ID', 'Vendor Name',
'Request Created At', 'Subscription Created At', 'Subscription ID', 'Subscription Status',
'Subscription External ID', 'Subscription Customer', 'Customer external id', 'Customer Country',
'Tier 1 Company name', 'Tier 1 External Id', 'Tier 1 Country location', 'Tier 2 Company name',
'Tier 2 External Id', 'Tier 2 Country location', 'Item ID', 'Item MPN', 'Item Description',
'Item Period', 'Item Old Quantity', 'Item Quantity', 'Item delta', 'Provider ID',
'Provider Name', 'Source MKP', 'MKP Name', 'Contract Type', 'Microsoft Tier1 MPN',
'AWS Account ID',
)
def generate(
client=None,
parameters=None,
progress_callback=None,
renderer_type=None,
extra_context_callback=None,
):
subscriptions_rql = R()
if parameters.get("date"):
subscriptions_rql &= R().events.created.at.ge(parameters['date']['after'])
Expand Down Expand Up @@ -52,6 +68,8 @@ def generate(client, parameters, progress_callback):
ex = futures.ThreadPoolExecutor(
max_workers=6,
)
if renderer_type == 'csv':
yield HEADERS

wait_for = []
for request in requests:
Expand All @@ -68,7 +86,13 @@ def generate(client, parameters, progress_callback):
for future in futures.as_completed(wait_for):
results = future.result()
for result in results:
yield result
if renderer_type == 'json':
yield {
HEADERS[idx].replace(' ', '_').lower(): value
for idx, value in enumerate(result)
}
else:
yield result

wait_for = []
for subscription in subscriptions:
Expand All @@ -84,7 +108,13 @@ def generate(client, parameters, progress_callback):
for future in futures.as_completed(wait_for):
results = future.result()
for result in results:
yield result
if renderer_type == 'json':
yield {
HEADERS[idx].replace(' ', '_').lower(): value
for idx, value in enumerate(result)
}
else:
yield result


def get_request_record(client, request, progress):
Expand Down
29 changes: 26 additions & 3 deletions reports/billing_requests/entrypoint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,32 @@
from reports.utils import convert_to_datetime, get_basic_value, get_value


def generate(client, parameters, progress_callback):
HEADERS = (
'Billing request ID', 'From', 'To', 'Delta', 'Uom', 'Customer ID', 'Customer Name',
'Customer External ID', 'Tier 1 ID', 'Tier 1 Name', 'Tier 1 External ID',
'Tier 2 ID', 'Tier 2 Name', 'Tier 2 External ID', 'Provider ID', 'Provider Name',
'Vendor ID', 'Vendor Name', 'Product ID', 'Product Name', 'Subscription ID',
'Subscription External ID', 'Subscription Status', 'Susbcription Type',
'Hub ID', 'Hub Name'
)
def generate(
client=None,
parameters=None,
progress_callback=None,
renderer_type=None,
extra_context_callback=None,
):
requests = _get_requests(client, parameters)

progress = 0
total = requests.count()

if renderer_type == 'csv':
yield HEADERS

for request in requests:
connection = request['asset']['connection']
yield (
result = (
request['id'],
convert_to_datetime(request['period']['from']),
convert_to_datetime(request['period']['to']),
Expand Down Expand Up @@ -45,10 +62,16 @@ def generate(client, parameters, progress_callback):
get_value(connection, 'hub', 'id') if 'hub' in connection else '',
get_value(connection, 'hub', 'name') if 'hub' in connection else '',
)
if renderer_type == 'json':
yield {
HEADERS[idx].replace(' ', '_').lower(): value
for idx, value in enumerate(result)
}
else:
yield result
progress += 1
progress_callback(progress, total)


def _get_requests(client, parameters):
query = R()
query &= R().created.ge(parameters['date']['after'])
Expand Down
31 changes: 29 additions & 2 deletions reports/billing_requests_line_item/entrypoint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,35 @@

from reports.utils import convert_to_datetime, get_basic_value, get_value

HEADERS = (
"Billing request ID", "From", "To", "Delta", "Uom", "Billing Cycle", "Item Id", "Item Name",
"Item Type", "Item Unit Of measure", "Item MPN", "Item Period", "Quantity", "Customer ID",
"Customer Name", "Customer External ID", "Tier 1 ID", "Tier 1 Name", "Tier 1 Exrternal ID",
"Tier 2 ID", "Tier 2 Name", "Tier 2 Exrternal ID","Provider ID", "Provider Name",
"Vendor ID", "Vendor Name", "Product ID", "Product Name", "Subscription ID",
"Subscription External ID", "Subscription Status", "Subscription Type",
"Hub ID", "Hub Name"
)

def generate(client, parameters, progress_callback):
def generate(
client=None,
parameters=None,
progress_callback=None,
renderer_type=None,
extra_context_callback=None,
):
requests = _get_requests(client, parameters)

progress = 0
total = requests.count()

if renderer_type == 'csv':
yield HEADERS

for request in requests:
connection = request['asset']['connection']
for item in request['items']:
yield (
result = (
request['id'],
convert_to_datetime(request['period']['from']),
convert_to_datetime(request['period']['to']),
Expand Down Expand Up @@ -54,8 +72,17 @@ def generate(client, parameters, progress_callback):
get_value(connection, 'hub', 'id') if 'hub' in connection else '',
get_value(connection, 'hub', 'name') if 'hub' in connection else '',
)
if renderer_type == 'json':
yield {
HEADERS[idx].replace(' ', '_').lower(): value
for idx, value in enumerate(result)
}
else:
yield result
progress += 1
progress_callback(progress, total)
if progress == 500:
break


def _get_requests(client, parameters):
Expand Down

0 comments on commit 591f9f9

Please sign in to comment.