infra: enable building projects using cached images (google#12597)

Signed-off-by: David Korczynski <[email protected]>
Co-authored-by: Oliver Chang <[email protected]>

Author: DavidKorczynski

Diff for: infra/build/functions/build_project.py

@@ -61,6 +61,10 @@
 LOCAL_BUILD_LOG_PATH = '/workspace/build.log'
 BUILD_SUCCESS_MARKER = '/workspace/build.succeeded'
 
+_CACHED_IMAGE = ('us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/'
+                 '{name}-ofg-cached-{sanitizer}')
+_CACHED_SANITIZERS = ('address', 'coverage')
+
 
 @dataclass
 class Config:
@@ -163,6 +167,8 @@ def __init__(self, name, project_yaml, dockerfile):
     else:
       self.main_repo = ''
 
+    self.cached_sanitizer = None
+
   @property
   def sanitizers(self):
     """Returns processed sanitizers."""
@@ -172,8 +178,14 @@ def sanitizers(self):
   @property
   def image(self):
     """Returns the docker image for the project."""
+    if self.cached_sanitizer:
+      return self.cached_image(self.cached_sanitizer)
+
     return f'gcr.io/{build_lib.IMAGE_PROJECT}/{self.name}'
 
+  def cached_image(self, sanitizer):
+    return _CACHED_IMAGE.format(name=self.name, sanitizer=sanitizer)
+
 
 def get_last_step_id(steps):
   """Returns the id of the last step in |steps|."""
@@ -313,30 +325,37 @@ def get_build_steps(  # pylint: disable=too-many-locals, too-many-statements, to
     project_yaml,
     dockerfile,
     config,
-    additional_env=None):
+    additional_env=None,
+    use_caching=False):
   """Returns build steps for project."""
 
   project = Project(project_name, project_yaml, dockerfile)
-
   if project.disabled:
     logging.info('Project "%s" is disabled.', project.name)
     return []
 
   timestamp = get_datetime_now().strftime('%Y%m%d%H%M')
-  build_steps = build_lib.get_project_image_steps(
-      project.name,
-      project.image,
-      project.fuzzing_language,
-      config=config,
-      architectures=project.architectures,
-      experiment=config.experiment)
+  if use_caching:
+    # Use cached built image.
+    build_steps = []
+  else:
+    build_steps = build_lib.get_project_image_steps(
+        project.name,
+        project.image,
+        project.fuzzing_language,
+        config=config,
+        architectures=project.architectures,
+        experiment=config.experiment)
 
   # Sort engines to make AFL first to test if libFuzzer has an advantage in
   # finding bugs first since it is generally built first.
   for fuzzing_engine in sorted(project.fuzzing_engines):
     # Sort sanitizers and architectures so order is determinisitic (good for
     # tests).
     for sanitizer in sorted(project.sanitizers):
+      if use_caching and sanitizer in _CACHED_SANITIZERS:
+        project.cached_sanitizer = sanitizer
+
       # Build x86_64 before i386.
       for architecture in reversed(sorted(project.architectures)):
         build = Build(fuzzing_engine, sanitizer, architecture)

Diff for: infra/build/functions/target_experiment.py

@@ -30,7 +30,8 @@
 
 def run_experiment(project_name, target_name, args, output_path, errlog_path,
                    build_output_path, upload_corpus_path, upload_coverage_path,
-                   experiment_name, upload_reproducer_path, tags):
+                   experiment_name, upload_reproducer_path, tags,
+                   use_cached_image):
   config = build_project.Config(testing=True,
                                 test_image_suffix='',
                                 repo=build_project.DEFAULT_OSS_FUZZ_REPO,
@@ -68,7 +69,8 @@ def run_experiment(project_name, target_name, args, output_path, errlog_path,
                                         project_yaml,
                                         dockerfile_contents,
                                         config,
-                                        additional_env=jcc_env)
+                                        additional_env=jcc_env,
+                                        use_caching=use_cached_image)
 
   build = build_project.Build('libfuzzer', 'address', 'x86_64')
   local_output_path = '/workspace/output.log'
@@ -209,6 +211,9 @@ def run_experiment(project_name, target_name, args, output_path, errlog_path,
   env = build_project.get_env(project_yaml['language'], build)
   env.extend(jcc_env)
 
+  if use_cached_image:
+    project.cached_sanitizer = 'coverage'
+
   steps.append(
       build_project.get_compile_step(project, build, env, config.parallel))
 
@@ -330,12 +335,15 @@ def main():
                       nargs='*',
                       help='Tags for cloud build.',
                       default=[])
+  parser.add_argument('--use_cached_image',
+                      action='store_true',
+                      help='Use cached images post build.')
   args = parser.parse_args()
 
   run_experiment(args.project, args.target, args.args, args.upload_output_log,
                  args.upload_err_log, args.upload_build_log, args.upload_corpus,
                  args.upload_coverage, args.experiment_name,
-                 args.upload_reproducer, args.tags)
+                 args.upload_reproducer, args.tags, args.use_cached_image)
 
 
 if __name__ == '__main__':