Show successful benchmarks per project (#748)

This shows which projects cannot benefit from OFG.

report/common.py

@@ -79,6 +79,7 @@ class Project:
   """Results for a project entire."""
   name: str
   count: int = 0
+  success: int = 0
   coverage_gain: float = 0.0
   coverage_relative_gain: float = 0.0
   coverage_ofg_total_new_covered_lines = 0
@@ -446,7 +447,7 @@ def get_results(self,
     Returns results of all samples. Items can be None if they're not complete.
     """
     targets = self._get_generated_targets(
-        benchmark) + self._get_agent_generated_targets(benchmark)
+        benchmark) or self._get_agent_generated_targets(benchmark)
 
     results = []
     status_dir = os.path.join(self._results_dir, benchmark, 'status')
@@ -505,6 +506,8 @@ def get_project_summary(self, benchmarks: list[Benchmark]) -> list[Project]:
       if benchmark.project not in project_summary_dict:
         project_summary_dict[benchmark.project] = Project(benchmark.project)
       project_summary_dict[benchmark.project].count += 1
+      project_summary_dict[benchmark.project].success += (
+          benchmark.result.build_success_count > 0)
 
     # Retrieve coverage gain information
     coverage_dict = {}

report/templates/index.html

@@ -104,7 +104,8 @@ <h2>Project summary</h2>
         <tr>
             <th></th>
             <th data-sorted="asc">Project</th>
-            <th data-sort-number>Total generated harnesses</th>
+            <th data-sort-number>Total benchmarks</th>
+            <th data-sort-number>Successful benchmarks</th>
             <th data-sort-number>Total coverage gain</th>
             <th data-sort-number>Total relative gain</th>
             <th data-sort-number>OSS-Fuzz-gen total covered lines</th>
@@ -119,6 +120,7 @@ <h2>Project summary</h2>
             <td class="table-index">{{ loop.index }}</td>
             <td data-sort-value="{{ project.name }}">{{ project.name }}</td>
             <td data-sort-value="{{ project.count }}">{{ project.count }}</td>
+            <td data-sort-value="{{ project.success }}">{{ project.success }}</td>
             <td data-sort-value="{{ project.coverage_gain|percent }}">{{ project.coverage_gain|percent }}%</td>
             <td data-sort-value="{{ project.coverage_relative_gain|percent}}">{{ project.coverage_relative_gain | percent }}%</td>
             <td data-sort-value="{{ project.coverage_ofg_total_covered_lines}}">{{project.coverage_ofg_total_covered_lines}}</td>

run_one_experiment.py

@@ -61,6 +61,7 @@
 @dataclasses.dataclass
 class AggregatedResult:
   """Aggregated evaluation result."""
+  build_success_count: int = 0
   build_success_rate: float = 0.0
   crash_rate: float = 0.0
   found_bug: int = 0
@@ -110,11 +111,14 @@ def from_experiment_result(
         if isinstance(trial_final_result.textcov_diff, textcov.Textcov):
           all_textcov.merge(trial_final_result.textcov_diff)
 
-    build_success_rate = (sum(compilable_trials) /
+    build_success_count = sum(compilable_trials)
+    build_success_rate = (build_success_count /
                           len(compilable_trials) if compilable_trials else 0)
+
     crash_rate = sum(crash_trials) / len(crash_trials) if crash_trials else 0
 
-    return AggregatedResult(build_success_rate=build_success_rate,
+    return AggregatedResult(build_success_count=build_success_count,
+                            build_success_rate=build_success_rate,
                             crash_rate=crash_rate,
                             max_coverage=max_coverage,
                             max_line_coverage_diff=max_line_coverage_diff,
@@ -169,8 +173,8 @@ def fix_code(work_dirs: WorkDirs, generated_targets: List[str]) -> List[str]:
 def aggregate_results(target_stats: list[tuple[int, exp_evaluator.Result]],
                       generated_targets: list[str]) -> AggregatedResult:
   """Aggregates experiment status and results of a targets."""
-  build_success_rate = sum([int(stat.compiles) for _, stat in target_stats
-                           ]) / len(target_stats)
+  build_success_count = sum([int(stat.compiles) for _, stat in target_stats])
+  build_success_rate = build_success_count / len(target_stats)
   crash_rate = sum([int(stat.crashes) for _, stat in target_stats
                    ]) / len(target_stats)
   found_bug = sum([
@@ -197,8 +201,8 @@ def aggregate_results(target_stats: list[tuple[int, exp_evaluator.Result]],
     if isinstance(stat.textcov_diff, textcov.Textcov):
       all_textcov.merge(stat.textcov_diff)
 
-  return AggregatedResult(build_success_rate, crash_rate, found_bug,
-                          max_coverage, max_line_coverage_diff,
+  return AggregatedResult(build_success_count, build_success_rate, crash_rate,
+                          found_bug, max_coverage, max_line_coverage_diff,
                           max_coverage_sample, max_coverage_diff_sample,
                           max_coverage_diff_report, all_textcov)