From 4b25746fc31b84ac2163c297e589cfd26fff7e3e Mon Sep 17 00:00:00 2001 From: DavidKorczynski Date: Wed, 6 Nov 2024 23:14:01 +0000 Subject: [PATCH] cloud_runner: prepare Dockerfile according to ccache (#696) Signed-off-by: David Korczynski --- experiment/builder_runner.py | 6 ++++- experiment/oss_fuzz_checkout.py | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/experiment/builder_runner.py b/experiment/builder_runner.py index 8bc80e9e9..959244518 100644 --- a/experiment/builder_runner.py +++ b/experiment/builder_runner.py @@ -912,9 +912,13 @@ def build_and_run_cloud( if oss_fuzz_checkout.ENABLE_CACHING and ( oss_fuzz_checkout.is_image_cached(project_name, 'address') and oss_fuzz_checkout.is_image_cached(project_name, 'coverage')): - logger.info(f'Using cached image for {project_name}.') + logger.info('Using cached image for %s', project_name) command.append('--use_cached_image') + # Overwrite the Dockerfile to be caching friendly + oss_fuzz_checkout.rewrite_project_to_cached_project_chronos( + generated_project) + if cloud_build_tags: command += ['--tags'] + cloud_build_tags command += ['--'] + self._libfuzzer_args() diff --git a/experiment/oss_fuzz_checkout.py b/experiment/oss_fuzz_checkout.py index 4b9ff5b65..19b62cd17 100644 --- a/experiment/oss_fuzz_checkout.py +++ b/experiment/oss_fuzz_checkout.py @@ -296,6 +296,47 @@ def is_image_cached(project_name: str, sanitizer: str) -> bool: return False +def rewrite_project_to_cached_project_chronos(generated_project) -> None: + """Rewrites Dockerfile to work with Chronos builds""" + + generated_project_folder = os.path.join(OSS_FUZZ_DIR, 'projects', + generated_project) + + # Check if there is an original Dockerfile, because we should use that in + # case,as otherwise the "Dockerfile" may be a copy of another sanitizer. + original_dockerfile = os.path.join(generated_project_folder, 'Dockerfile') + with open(original_dockerfile, 'r') as f: + docker_content = f.read() + + arg_line = 'ARG CACHE_IMAGE=gcr.io/MUST_PROVIDE_IMAGE' + docker_content = arg_line + '\n' + docker_content + docker_content = docker_content.replace( + 'FROM gcr.io/oss-fuzz-base/base-builder', 'FROM $CACHE_IMAGE') + + # Now comment out everything except the first FROM and the last COPY that + # was added earlier in the OFG process. + from_line = -1 + copy_fuzzer_line = -1 + + for line_idx, line in enumerate(docker_content.split('\n')): + if line.startswith('FROM') and from_line == -1: + from_line = line_idx + if line.startswith('COPY'): + copy_fuzzer_line = line_idx + + lines_to_keep = {from_line, copy_fuzzer_line} + new_content = '' + for line_idx, line in enumerate(docker_content.split('\n')): + if line_idx not in lines_to_keep: + new_content += f'# {line}\n' + else: + new_content += f'{line}\n' + + # Overwrite the existing one + with open(original_dockerfile, 'w') as f: + f.write(new_content) + + def rewrite_project_to_cached_project(project_name: str, generated_project: str, sanitizer: str) -> None: """Rewrites Dockerfile of a project to enable cached build scripts."""