diff --git a/python/pdstools/adm/CDH_Guidelines.py b/python/pdstools/adm/CDH_Guidelines.py index fa4ee043..a145df0a 100644 --- a/python/pdstools/adm/CDH_Guidelines.py +++ b/python/pdstools/adm/CDH_Guidelines.py @@ -7,8 +7,8 @@ _data = { "Issues": [1, 5, 25, None], "Groups per Issue": [1, 5, 25, None], - "Treatments": [1, 2500, 5000, 5000], - "Treatments per Channel": [1, 1000, 2500, 2500], + "Treatments": [2, 2500, 5000, 5000], + "Treatments per Channel": [2, 1000, 2500, 2500], "Treatments per Channel per Action": [1, 1, 5, None], "Actions": [10, 1000, 2500, 2500], "Actions per Group": [1, 100, 250, None], @@ -16,11 +16,15 @@ "Configurations per Channel": [1, 1, 2, None], "Predictors": [50, 200, 700, 2000], "Active Predictors per Model": [2, 5, 100, None], + + # below are not part of the standard cloud limits but used in the reports + "Model Performance": [52, 55, 80, 90], - "Engagement Lift": [0.0, 0.2, 2.0, None], "Responses": [1.0, 200, None, None], "Positive Responses": [1.0, 200, None, None], "Engagement Lift": [0.0, 1.0, None, None], + "CTR": [0.0, 0.000001, 0.999999, 1.0], + "OmniChannel": [0.0, 0.5, 1.0, 1.0], } _pega_cloud_limits = pl.DataFrame(data=_data).transpose(include_header=True) diff --git a/python/pdstools/reports/HealthCheck.qmd b/python/pdstools/reports/HealthCheck.qmd index e157165d..82442ee8 100644 --- a/python/pdstools/reports/HealthCheck.qmd +++ b/python/pdstools/reports/HealthCheck.qmd @@ -265,24 +265,21 @@ formatted_channel_overview = ( df_channel_overview, cdh_guidelines=cdh_guidelines, highlight_limits={ - "Positives": "Positive Responses", - "Performance": "Model Performance", - "ResponseCount": "Responses", - "Total Number of Actions": "Actions", - "Used Actions": "Actions", - "Total Number of Treatments": "Treatments", - "Used Treatments": "Treatments", + "Positive Responses": "Positives", + "Model Performance": "Performance", + "Responses": "ResponseCount", + "Actions": ["Total Number of Actions", "Used Actions"], + "Treatments": ["Total Number of Treatments", "Used Treatments"], "Issues": "Issues", + "OmniChannel": "OmniChannel Actions", + "CTR" : "CTR", }, - highlight_lists = { - "Channel" : cdh_guidelines.standard_channels, - "Direction" : cdh_guidelines.standard_directions, + highlight_lists={ + "Channel": cdh_guidelines.standard_channels, + "Direction": cdh_guidelines.standard_directions, }, - highlight_configurations = ["Configuration"], + highlight_configurations=["Configuration"], ) - .fmt_percent(decimals=0, columns=["OmniChannel Actions"]) - .fmt_number(decimals=2, columns=["Performance"]) - .fmt_percent(decimals=2, columns=["CTR"]) .cols_label( CTR="Base Rate", ResponseCount="Total Responses", @@ -418,21 +415,24 @@ if prediction_file_path: highlight_limits={ # "Actions": "Actions", # "Unique Treatments": "Treatments", - "Positives": "Positive Responses", - "Negatives": "Responses", - "Positives_Test": "Positive Responses", - "Negatives_Test": "Responses", - "Positives_Control": "Positive Responses", - "Negatives_Control": "Responses", - "Positives_NBA": "Positive Responses", - "Negatives_NBA": "Responses", - "ResponseCount": "Responses", + "Positive Responses": [ + "Positives", + "Positives_Test", + "Positives_Control", + "Positives_NBA", + ], + "Responses": [ + "ResponseCount", + "Negatives", + "Negatives_Test", + "Negatives_Control", + "Negatives_NBA", + ], + "Model Performance": "Performance", + "CTR": ["CTR", "CTR_Test", "CTR_Control", "CTR_NBA"], + "Engagement Lift": "Lift", }, ) - .fmt_number(decimals=2, columns=["Performance"]) - .fmt_percent( - decimals=2, columns=["CTR", "CTR_Test", "CTR_Control", "CTR_NBA", "Lift"] - ) .fmt_percent( decimals=2, scale_values=False, @@ -811,7 +811,9 @@ else: model_overview = ( last_data.group_by( ["Configuration"] - + report_utils.polars_subset_to_existing_cols(datamart_all_columns, ["Channel", "Direction"]) + + report_utils.polars_subset_to_existing_cols( + datamart_all_columns, ["Channel", "Direction"] + ) ) .agg( [ @@ -835,18 +837,19 @@ model_overview = ( display( report_utils.table_standard_formatting( - model_overview, title="Model Overview", + model_overview, + title="Model Overview", cdh_guidelines=cdh_guidelines, highlight_limits={ "Actions": "Actions", - "Unique Treatments": "Treatments", - "Positives": "Positive Responses", - "ResponseCount": "Responses", + "Treatments": "Unique Treatments", + "Positive Responses": "Positives", + "Responses": "ResponseCount", }, - highlight_lists = { - "Channel" : cdh_guidelines.standard_channels, - "Direction" : cdh_guidelines.standard_directions, - "Configuration" : cdh_guidelines.standard_configurations, + highlight_lists={ + "Channel": cdh_guidelines.standard_channels, + "Direction": cdh_guidelines.standard_directions, + "Configuration": cdh_guidelines.standard_configurations, }, ) .tab_style( @@ -1301,7 +1304,7 @@ If predictors perform poorly across all models, that may be because of data sour ```{python} # weighted performance -# TODO apply highlighting in the std way like in the R version + if datamart.predictor_data is not None: bad_predictors = ( @@ -1329,7 +1332,7 @@ if datamart.predictor_data is not None: # .with_columns(MeanPlotData=pl.col("Mean")), rowname_col="PredictorName", cdh_guidelines=cdh_guidelines, - highlight_limits = {"Response Count" : "Responses"} + highlight_limits = {"Responses" : "Response Count"} ) .tab_options(container_height="400px", container_overflow_y=True) .tab_spanner( diff --git a/python/pdstools/reports/ModelReport.qmd b/python/pdstools/reports/ModelReport.qmd index f3f7bcc8..b4e677ff 100644 --- a/python/pdstools/reports/ModelReport.qmd +++ b/python/pdstools/reports/ModelReport.qmd @@ -230,28 +230,20 @@ try: classifier.collect().select( pl.last("ResponseCount"), pl.last("Positives"), - (pl.last("Positives") / pl.last("ResponseCount")).alias("Base Propensity"), - pl.last("Performance"), + (pl.last("Positives") / pl.last("ResponseCount")).alias( + "Base Propensity" + ), + pl.last("Performance") * 100, ), - highlight_limits = { - "ResponseCount" : "Responses", - "Positives" : "Positive Responses", - "Performance" : "Model Performance" - } - ) - .cols_label( + highlight_limits={ + "Responses": "ResponseCount", + "Positive Responses": "Positives", + "Model Performance": "Performance", + "CTR": "Base Propensity", + }, + ).cols_label( ResponseCount="Responses", ) - .fmt_number( - decimals=0, - columns=["ResponseCount", "Positives"], - ) - .fmt_percent(decimals=3, columns="Base Propensity") - .fmt_number( - decimals=2, - scale_by=100, - columns=["Performance"], - ) ) display(gt) diff --git a/python/pdstools/utils/report_utils.py b/python/pdstools/utils/report_utils.py index ab9c4b54..5a880d4d 100644 --- a/python/pdstools/utils/report_utils.py +++ b/python/pdstools/utils/report_utils.py @@ -1,5 +1,5 @@ import traceback -from typing import Dict, List +from typing import Dict, List, Literal, Optional, Union from IPython.display import display, Markdown from great_tables import GT, style, loc from ..adm.CDH_Guidelines import CDHGuidelines @@ -64,6 +64,54 @@ def polars_subset_to_existing_cols(all_columns, cols): return [col for col in cols if col in all_columns] +def rag_background_styler( + rag: Optional[Literal["Red", "Amber", "Yellow", "Green"]] = None +): + match rag[0].upper() if len(rag) > 0 else None: + case "R": + return style.fill(color="orangered") + case "A": + return style.fill(color="orange") + case "Y": + return style.fill(color="yellow") + case "G": + return None # no green background to keep it light + case _: + raise ValueError(f"Not a supported RAG value: {rag}") + + +def rag_background_styler_dense( + rag: Optional[Literal["Red", "Amber", "Yellow", "Green"]] = None +): + match rag[0].upper() if len(rag) > 0 else None: + case "R": + return style.fill(color="orangered") + case "A": + return style.fill(color="orange") + case "Y": + return style.fill(color="yellow") + case "G": + return style.fill(color="green") + case _: + raise ValueError(f"Not a supported RAG value: {rag}") + + +def rag_textcolor_styler( + rag: Optional[Literal["Red", "Amber", "Yellow", "Green"]] = None +): + match rag[0].upper() if len(rag) > 0 else None: + case "R": + return style.text(color="orangered") + case "A": + return style.text(color="orange") + case "Y": + return style.text(color="yellow") + case "G": + return style.text(color="green") + case _: + raise ValueError(f"Not a supported RAG value: {rag}") + + def table_standard_formatting( source_table, title=None, @@ -71,13 +119,12 @@ def table_standard_formatting( rowname_col=None, groupname_col=None, cdh_guidelines=CDHGuidelines(), - # TODO generalize highlight_lims so the dict can als be List[str] to str - highlight_limits: Dict[str, str] = {}, + highlight_limits: Dict[str, Union[str, List[str]]] = {}, highlight_lists: Dict[str, List[str]] = {}, highlight_configurations: List[str] = [], - color_styler=style.fill, + rag_styler: callable = rag_background_styler, ): - def apply_metric_style(gt, col_name, metric): + def apply_rag_styling(gt, col_name, metric): if col_name in source_table.collect_schema().names(): min_val = cdh_guidelines.min(metric) max_val = cdh_guidelines.max(metric) @@ -108,8 +155,8 @@ def apply_metric_style(gt, col_name, metric): or ( max_val is not None and best_practice_max is not None - and v >= max_val - and v < best_practice_max + and v > best_practice_max + and v <= max_val ) ) ] @@ -122,18 +169,18 @@ def apply_metric_style(gt, col_name, metric): ] # TODO consider that bad / warning rows are exclusive - gt = gt.tab_style( - style=color_styler(color="green"), - locations=loc.body(columns=col_name, rows=good_rows), - ) - gt = gt.tab_style( - style=color_styler(color="orange"), - locations=loc.body(columns=col_name, rows=warning_rows), - ) - gt = gt.tab_style( - style=color_styler(color="orangered"), - locations=loc.body(columns=col_name, rows=bad_rows), - ) + def apply_style(gt, rag, rows): + style = rag_styler(rag) + if style is not None: + gt = gt.tab_style( + style=style, + locations=loc.body(columns=col_name, rows=rows), + ) + return gt + + gt = apply_style(gt, "green", good_rows) + gt = apply_style(gt, "amber", warning_rows) + gt = apply_style(gt, "red", bad_rows) return gt def apply_standard_name_style(gt, col_name, standard_list): @@ -143,7 +190,7 @@ def apply_standard_name_style(gt, col_name, standard_list): i for i, v in enumerate(values) if v not in standard_list ] gt = gt.tab_style( - style=color_styler(color="yellow"), + style=rag_styler("yellow"), locations=loc.body(columns=col_name, rows=non_standard_rows), ) return gt @@ -153,45 +200,87 @@ def apply_configuration_style(gt, col_name): values = source_table[col_name].to_list() multiple_config_rows = [i for i, v in enumerate(values) if v.count(",") > 1] gt = gt.tab_style( - style=color_styler(color="yellow"), + style=rag_styler("yellow"), locations=loc.body(columns=col_name, rows=multiple_config_rows), ) return gt - gt = GT( - source_table, rowname_col=rowname_col, groupname_col=groupname_col - ).tab_options(table_font_size=8) + gt = ( + GT(source_table, rowname_col=rowname_col, groupname_col=groupname_col) + .tab_options(table_font_size=8) + .sub_missing(missing_text="") + ) if title is not None: gt = gt.tab_header(title=title, subtitle=subtitle) - for c in highlight_limits.keys(): - gt = apply_metric_style(gt, c, highlight_limits[c]) - gt = gt.fmt_number( - columns=c, decimals=0, compact=True - ) # default number formatting + def metric_styling_model_performance(gt, cols): + return gt.fmt_number( + decimals=2, + columns=cols, + ) + + def metric_styling_percentage(gt, cols): + return gt.fmt_percent( + decimals=0, + columns=cols, + ) - for c in highlight_lists.keys(): - gt = apply_standard_name_style(gt, c, highlight_lists[c]) + def metric_styling_ctr(gt, cols): + return gt.fmt_percent( + decimals=3, + columns=cols, + ) + + def metric_styling_default(gt, cols): + return gt.fmt_number( + decimals=0, + compact=True, + columns=cols, + ) - for c in highlight_configurations: - gt = apply_configuration_style(gt, c) + for metric in highlight_limits.keys(): + cols = highlight_limits[metric] + if isinstance(cols, str): + cols = [cols] + for col_name in cols: + gt = apply_rag_styling(gt, col_name=col_name, metric=metric) + # gt = gt.fmt_number( + # columns=col_name, decimals=0, compact=True + # ) # default number formatting applied to everything - consider being smarter, in config + match metric: + case "Model Performance": + gt = metric_styling_model_performance(gt, cols) + case "Engagement Lift": + gt = metric_styling_percentage(gt, cols) + case "OmniChannel": + gt = metric_styling_percentage(gt, cols) + case "CTR": + gt = metric_styling_ctr(gt, cols) + case _: + gt = metric_styling_default(gt, cols) + + for metric in highlight_lists.keys(): + gt = apply_standard_name_style(gt, metric, highlight_lists[metric]) + + for metric in highlight_configurations: + gt = apply_configuration_style(gt, metric) return gt def table_style_predictor_count( - gt: GT, flds, cdh_guidelines=CDHGuidelines(), color_styler=style.fill + gt: GT, flds, cdh_guidelines=CDHGuidelines(), rag_styler=rag_textcolor_styler ): for col in flds: gt = gt.tab_style( - style=color_styler(color="orange"), + style=rag_styler("amber"), locations=loc.body( columns=col, rows=(pl.col(col) < 200) | (pl.col(col) > 700) & (pl.col(col) > 0), ), ).tab_style( - style=color_styler(color="orangered"), + style=rag_styler("red"), locations=loc.body( columns=col, rows=(pl.col(col) == 0),