Use the new cached images in prototyper (#710)

agent/prototyper.py

@@ -2,6 +2,8 @@
 Use it as a usual module locally, or as script in cloud builds.
 """
 import subprocess as sp
+import time
+from datetime import timedelta
 from typing import Optional
 
 import logger
@@ -84,14 +86,14 @@ def _validate_fuzz_target_and_build_script(self, cur_round: int,
 
     logger.info('First compile fuzz target without modifying build script.')
     build_result.build_script_source = ''
-    self._validate_fuzz_target_and_build_script_via_recompile(
+    self._validate_fuzz_target_and_build_script_via_compile(
         cur_round, build_result)
 
     if not build_result.success and build_script_source:
       logger.info('Then compile fuzz target with modified build script.')
       build_result.build_script_source = build_script_source
-      self._validate_fuzz_target_and_build_script_via_recompile(
-          cur_round, build_result, use_recompile=False)
+      self._validate_fuzz_target_and_build_script_via_compile(
+          cur_round, build_result)
 
   def _validate_fuzz_target_references_function(
       self, compilation_tool: ProjectContainerTool, benchmark: Benchmark,
@@ -110,11 +112,8 @@ def _validate_fuzz_target_references_function(
                    cur_round)
     return function_referenced
 
-  def _validate_fuzz_target_and_build_script_via_recompile(
-      self,
-      cur_round: int,
-      build_result: BuildResult,
-      use_recompile: bool = True) -> None:
+  def _validate_fuzz_target_and_build_script_via_compile(
+      self, cur_round: int, build_result: BuildResult) -> None:
     """Validates the new fuzz target and build script by recompiling them."""
     benchmark = build_result.benchmark
     compilation_tool = ProjectContainerTool(benchmark=benchmark)
@@ -135,9 +134,13 @@ def _validate_fuzz_target_and_build_script_via_recompile(
 
     # Recompile.
     logger.info('===== ROUND %02d Recompile =====', cur_round)
-    compile_process = compilation_tool.compile(use_recompile=use_recompile)
+    start_time = time.time()
+    compile_process = compilation_tool.compile()
+    end_time = time.time()
+    logger.debug('ROUND %02d compilation time: %s', cur_round,
+                 timedelta(seconds=end_time - start_time))
     compile_succeed = compile_process.returncode == 0
-    logger.debug('ROUND %02d Fuzz target compile Succeessfully: %s', cur_round,
+    logger.debug('ROUND %02d Fuzz target compiles: %s', cur_round,
                  compile_succeed)
 
     # Double-check binary.

experiment/oss_fuzz_checkout.py

@@ -27,6 +27,7 @@
 from experiment import benchmark as benchmarklib
 
 logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
 
 BUILD_DIR: str = 'build'
 GLOBAL_TEMP_DIR: str = ''
@@ -443,13 +444,6 @@ def _image_exists_online(image_name: str, project_name: str) -> bool:
            check=True)
     logger.info('Pulled online cached images of %s: %s', project_name,
                 online_image_name)
-    sp.run([
-        'docker', 'run', '--entrypoint', '/usr/local/bin/recompile',
-        online_image_name
-    ],
-           stdout=sp.PIPE,
-           text=True,
-           check=True)
 
     sp.run(['docker', 'tag', online_image_name, image_name],
            stdout=sp.PIPE,
@@ -462,22 +456,30 @@ def _image_exists_online(image_name: str, project_name: str) -> bool:
     return False
 
 
-def prepare_project_image(project: str) -> str:
-  """Prepares original image of the |project|'s fuzz target build container."""
-  image_name = f'gcr.io/oss-fuzz/{project}'
-  if (_image_exists_locally(image_name, project_name=project) or
-      _image_exists_online(image_name, project_name=project)):
-    logger.info('Using existing project image for %s', project)
-    return image_name
-  logger.info('Unable to find existing project image for %s', project)
+def _build_image(project_name: str) -> str:
+  """Builds project image in OSS-Fuzz"""
   adjusted_env = os.environ | {
-      'FUZZING_LANGUAGE': get_project_language(project)
+      'FUZZING_LANGUAGE': get_project_language(project_name)
   }
-  command = ['python3', 'infra/helper.py', 'build_image', '--pull', project]
+  command = [
+      'python3', 'infra/helper.py', 'build_image', '--pull', project_name
+  ]
   try:
     sp.run(command, cwd=OSS_FUZZ_DIR, env=adjusted_env, check=True)
-    logger.info('Successfully build project image for %s', project)
-    return image_name
+    logger.info('Successfully build project image for %s', project_name)
+    return f'gcr.io/oss-fuzz/{project_name}'
   except sp.CalledProcessError:
-    logger.info('Failed to build project image for %s', project)
+    logger.info('Failed to build project image for %s', project_name)
     return ''
+
+
+def prepare_project_image(project: str) -> str:
+  """Prepares original image of the |project|'s fuzz target build container."""
+  image_name = f'gcr.io/oss-fuzz/{project}'
+  if (ENABLE_CACHING and is_image_cached(project, 'address') and
+      (_image_exists_locally(image_name, project_name=project) or
+       _image_exists_online(image_name, project_name=project))):
+    logger.info('Using cached project image for %s', project)
+    return image_name
+  logger.warning('Unable to find cached project image for %s', project)
+  return _build_image(project)

tool/container_tool.py

@@ -96,18 +96,8 @@ def execute(self, command: str) -> sp.CompletedProcess:
     process.args = command
     return process
 
-  def compile(self,
-              use_recompile: bool = True,
-              extra_commands: str = '') -> sp.CompletedProcess:
-    """Compiles or recompiles the fuzz target."""
-    if use_recompile:
-      logger.info('Will attempt to use recompile')
-      self.execute(
-          '[ -f /usr/local/bin/recompile ] && echo "Will use recompile" '
-          '&& mv /usr/local/bin/recompile /usr/local/bin/compile')
-    else:
-      logger.info('Will use the original compile')
-
+  def compile(self, extra_commands: str = '') -> sp.CompletedProcess:
+    """Compiles the fuzz target."""
     command = 'compile > /dev/null' + extra_commands
     return self.execute(command)