Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Show if datasets are identical #4029

Merged
merged 1 commit into from
Dec 14, 2023
Merged

Show if datasets are identical #4029

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions lib/plugins/compare/pug/index.pug
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,8 @@ table
td
a(href=createGraphLink(groupName, metricName))
b #{groupName + '.' + metricName}
if values.statisticalTestU === "N/A"
td N/A
if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical"
td #{values.statisticalTestU}
else
td #{h.decimals(values.statisticalTestU)}
td #{h.decimals(values.baseline.mean)}
Expand All @@ -84,7 +84,7 @@ table
td #{h.decimals(values.current.median)}
td #{h.decimals(values.baseline.stdev)}
td #{h.decimals(values.current.stdev)}
if values.statisticalTestU === "N/A"
if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical"
td No Test Conducted
else
td #{values.statisticalTestU < 0.05 ? 'Yes - ' + cliffDeltaHelper(values.cliffsDelta) : 'No'}
Expand All @@ -96,7 +96,7 @@ each metricGroup, groupName in compare.metrics
each values, metricName in metricGroup
- var fullMetricName = groupName + '.' + metricName
- var metricId = fullMetricName.replace(/\./g, '_')
h3 #{fullMetricName} #{values.statisticalTestU === "N/A"? '' : values.statisticalTestU < 0.05 ? '(significant change)' : ''}
h3 #{fullMetricName} #{values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical" ? '' : values.statisticalTestU < 0.05 ? '(significant change)' : ''}
.ct-chart(id=`chart-${metricId}`)
.ct-legend
span.ct-legend-item
Expand Down
12 changes: 6 additions & 6 deletions lib/plugins/compare/statistical.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,17 @@
import json
from scipy.stats import wilcoxon, mannwhitneyu


def has_variability(sample):
"""Check if the sample has more than one unique value."""
return len(set(sample)) > 1


def perform_test(test_type, baseline, current, **kwargs):
"""Perform the statistical test based on the test type."""
if not has_variability(baseline) or not has_variability(current):
return None, "No variability"
if baseline == current:
return None, "Datasets are identical"
else:
return None, "No variability"

if test_type == 'wilcoxon':
return wilcoxon(current, baseline, **kwargs)
Expand All @@ -20,7 +21,6 @@ def perform_test(test_type, baseline, current, **kwargs):
else:
raise ValueError("Invalid test type. Choose 'wilcoxon' or 'mannwhitneyu'.")


input_data = json.loads(sys.stdin.read())
options = input_data['options']
test_type = options.pop('test_type')
Expand All @@ -31,8 +31,8 @@ def perform_test(test_type, baseline, current, **kwargs):
group_results = {}
for metric_name, metric_data in metrics.items():
stat, p = perform_test(test_type, metric_data['baseline'], metric_data['current'], **options)
if p == "No variability":
group_results[metric_name] = {'statistic': "N/A", 'p-value': "N/A"}
if p == "No variability" or p == "Datasets are identical":
group_results[metric_name] = {'statistic': "N/A", 'p-value': p}
else:
group_results[metric_name] = {'statistic': stat, 'p-value': p}
final_results[group_name] = group_results
Expand Down