diff --git a/report/common.py b/report/common.py index 6165890a1..408dd4946 100644 --- a/report/common.py +++ b/report/common.py @@ -79,6 +79,7 @@ class Project: """Results for a project entire.""" name: str count: int = 0 + success: int = 0 coverage_gain: float = 0.0 coverage_relative_gain: float = 0.0 coverage_ofg_total_new_covered_lines = 0 @@ -446,7 +447,7 @@ def get_results(self, Returns results of all samples. Items can be None if they're not complete. """ targets = self._get_generated_targets( - benchmark) + self._get_agent_generated_targets(benchmark) + benchmark) or self._get_agent_generated_targets(benchmark) results = [] status_dir = os.path.join(self._results_dir, benchmark, 'status') @@ -505,6 +506,8 @@ def get_project_summary(self, benchmarks: list[Benchmark]) -> list[Project]: if benchmark.project not in project_summary_dict: project_summary_dict[benchmark.project] = Project(benchmark.project) project_summary_dict[benchmark.project].count += 1 + project_summary_dict[benchmark.project].success += ( + benchmark.result.build_success_count > 0) # Retrieve coverage gain information coverage_dict = {} diff --git a/report/templates/index.html b/report/templates/index.html index 568c3f60e..955ff89d2 100644 --- a/report/templates/index.html +++ b/report/templates/index.html @@ -104,7 +104,8 @@

Project summary

Project - Total generated harnesses + Total benchmarks + Successful benchmarks Total coverage gain Total relative gain OSS-Fuzz-gen total covered lines @@ -119,6 +120,7 @@

Project summary

{{ loop.index }} {{ project.name }} {{ project.count }} + {{ project.success }} {{ project.coverage_gain|percent }}% {{ project.coverage_relative_gain | percent }}% {{project.coverage_ofg_total_covered_lines}} diff --git a/run_one_experiment.py b/run_one_experiment.py index 10cbc59ea..c29bb018a 100644 --- a/run_one_experiment.py +++ b/run_one_experiment.py @@ -61,6 +61,7 @@ @dataclasses.dataclass class AggregatedResult: """Aggregated evaluation result.""" + build_success_count: int = 0 build_success_rate: float = 0.0 crash_rate: float = 0.0 found_bug: int = 0 @@ -110,11 +111,14 @@ def from_experiment_result( if isinstance(trial_final_result.textcov_diff, textcov.Textcov): all_textcov.merge(trial_final_result.textcov_diff) - build_success_rate = (sum(compilable_trials) / + build_success_count = sum(compilable_trials) + build_success_rate = (build_success_count / len(compilable_trials) if compilable_trials else 0) + crash_rate = sum(crash_trials) / len(crash_trials) if crash_trials else 0 - return AggregatedResult(build_success_rate=build_success_rate, + return AggregatedResult(build_success_count=build_success_count, + build_success_rate=build_success_rate, crash_rate=crash_rate, max_coverage=max_coverage, max_line_coverage_diff=max_line_coverage_diff, @@ -169,8 +173,8 @@ def fix_code(work_dirs: WorkDirs, generated_targets: List[str]) -> List[str]: def aggregate_results(target_stats: list[tuple[int, exp_evaluator.Result]], generated_targets: list[str]) -> AggregatedResult: """Aggregates experiment status and results of a targets.""" - build_success_rate = sum([int(stat.compiles) for _, stat in target_stats - ]) / len(target_stats) + build_success_count = sum([int(stat.compiles) for _, stat in target_stats]) + build_success_rate = build_success_count / len(target_stats) crash_rate = sum([int(stat.crashes) for _, stat in target_stats ]) / len(target_stats) found_bug = sum([ @@ -197,8 +201,8 @@ def aggregate_results(target_stats: list[tuple[int, exp_evaluator.Result]], if isinstance(stat.textcov_diff, textcov.Textcov): all_textcov.merge(stat.textcov_diff) - return AggregatedResult(build_success_rate, crash_rate, found_bug, - max_coverage, max_line_coverage_diff, + return AggregatedResult(build_success_count, build_success_rate, crash_rate, + found_bug, max_coverage, max_line_coverage_diff, max_coverage_sample, max_coverage_diff_sample, max_coverage_diff_report, all_textcov)