diff --git a/report/common.py b/report/common.py
index 6165890a1..408dd4946 100644
--- a/report/common.py
+++ b/report/common.py
@@ -79,6 +79,7 @@ class Project:
"""Results for a project entire."""
name: str
count: int = 0
+ success: int = 0
coverage_gain: float = 0.0
coverage_relative_gain: float = 0.0
coverage_ofg_total_new_covered_lines = 0
@@ -446,7 +447,7 @@ def get_results(self,
Returns results of all samples. Items can be None if they're not complete.
"""
targets = self._get_generated_targets(
- benchmark) + self._get_agent_generated_targets(benchmark)
+ benchmark) or self._get_agent_generated_targets(benchmark)
results = []
status_dir = os.path.join(self._results_dir, benchmark, 'status')
@@ -505,6 +506,8 @@ def get_project_summary(self, benchmarks: list[Benchmark]) -> list[Project]:
if benchmark.project not in project_summary_dict:
project_summary_dict[benchmark.project] = Project(benchmark.project)
project_summary_dict[benchmark.project].count += 1
+ project_summary_dict[benchmark.project].success += (
+ benchmark.result.build_success_count > 0)
# Retrieve coverage gain information
coverage_dict = {}
diff --git a/report/templates/index.html b/report/templates/index.html
index 568c3f60e..955ff89d2 100644
--- a/report/templates/index.html
+++ b/report/templates/index.html
@@ -104,7 +104,8 @@
Project summary
|
Project |
- Total generated harnesses |
+ Total benchmarks |
+ Successful benchmarks |
Total coverage gain |
Total relative gain |
OSS-Fuzz-gen total covered lines |
@@ -119,6 +120,7 @@ Project summary
{{ loop.index }} |
{{ project.name }} |
{{ project.count }} |
+ {{ project.success }} |
{{ project.coverage_gain|percent }}% |
{{ project.coverage_relative_gain | percent }}% |
{{project.coverage_ofg_total_covered_lines}} |
diff --git a/run_one_experiment.py b/run_one_experiment.py
index 10cbc59ea..c29bb018a 100644
--- a/run_one_experiment.py
+++ b/run_one_experiment.py
@@ -61,6 +61,7 @@
@dataclasses.dataclass
class AggregatedResult:
"""Aggregated evaluation result."""
+ build_success_count: int = 0
build_success_rate: float = 0.0
crash_rate: float = 0.0
found_bug: int = 0
@@ -110,11 +111,14 @@ def from_experiment_result(
if isinstance(trial_final_result.textcov_diff, textcov.Textcov):
all_textcov.merge(trial_final_result.textcov_diff)
- build_success_rate = (sum(compilable_trials) /
+ build_success_count = sum(compilable_trials)
+ build_success_rate = (build_success_count /
len(compilable_trials) if compilable_trials else 0)
+
crash_rate = sum(crash_trials) / len(crash_trials) if crash_trials else 0
- return AggregatedResult(build_success_rate=build_success_rate,
+ return AggregatedResult(build_success_count=build_success_count,
+ build_success_rate=build_success_rate,
crash_rate=crash_rate,
max_coverage=max_coverage,
max_line_coverage_diff=max_line_coverage_diff,
@@ -169,8 +173,8 @@ def fix_code(work_dirs: WorkDirs, generated_targets: List[str]) -> List[str]:
def aggregate_results(target_stats: list[tuple[int, exp_evaluator.Result]],
generated_targets: list[str]) -> AggregatedResult:
"""Aggregates experiment status and results of a targets."""
- build_success_rate = sum([int(stat.compiles) for _, stat in target_stats
- ]) / len(target_stats)
+ build_success_count = sum([int(stat.compiles) for _, stat in target_stats])
+ build_success_rate = build_success_count / len(target_stats)
crash_rate = sum([int(stat.crashes) for _, stat in target_stats
]) / len(target_stats)
found_bug = sum([
@@ -197,8 +201,8 @@ def aggregate_results(target_stats: list[tuple[int, exp_evaluator.Result]],
if isinstance(stat.textcov_diff, textcov.Textcov):
all_textcov.merge(stat.textcov_diff)
- return AggregatedResult(build_success_rate, crash_rate, found_bug,
- max_coverage, max_line_coverage_diff,
+ return AggregatedResult(build_success_count, build_success_rate, crash_rate,
+ found_bug, max_coverage, max_line_coverage_diff,
max_coverage_sample, max_coverage_diff_sample,
max_coverage_diff_report, all_textcov)