Merge branch 'master' into upgrade-go

CITATION.cff

@@ -0,0 +1,46 @@
+cff-version: 1.2.0
+title: OSS-Fuzz
+message: >-
+  If you use this software, please cite it using the
+  metadata from this file.
+type: software
+authors:
+  - given-names: Abhishek
+    family-names: Arya
+    affiliation: Google LLC
+    email: [email protected]
+    orcid: 'https://orcid.org/0009-0009-4558-4314'
+  - given-names: Oliver
+    family-names: Chang
+    email: [email protected]
+    affiliation: Google LLC
+    orcid: 'https://orcid.org/0009-0006-3181-4551'
+  - given-names: Jonathan
+    family-names: Metzman
+    email: [email protected]
+    affiliation: Google LLC
+    orcid: 'https://orcid.org/0000-0002-7042-0444'
+  - given-names: Kostya
+    family-names: Serebryany
+    email: [email protected]
+    affiliation: Google LLC
+    orcid: 'https://orcid.org/0009-0009-2379-3641'
+  - given-names: Dongge
+    family-names: Liu
+    email: [email protected]
+    affiliation: Google LLC
+    orcid: 'https://orcid.org/0000-0003-4821-7033'
+repository-code: 'https://github.com/google/oss-fuzz'
+abstract: >-
+  OSS-Fuzz is an open-source project by Google that provides
+  continuous fuzzing for open-source software. It aims to
+  make common open-source software more secure and stable by
+  combining modern fuzzing techniques with scalable,
+  distributed execution. As of August 2023, OSS-Fuzz has
+  helped identify and fix over 10,000 vulnerabilities and
+  36,000 bugs across 1,000 projects.
+keywords:
+  - open-source
+  - fuzzing
+license: Apache-2.0
+

infra/base-images/base-builder/bazel_build_fuzz_tests

@@ -48,7 +48,6 @@ for oss_fuzz_test in "${OSS_FUZZ_TESTS[@]}"; do
 done
 
 declare -r BAZEL_BUILD_FLAGS=(
-    "-c" "opt"
     "--@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_engine" \
     "--@rules_fuzzing//fuzzing:java_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_java_engine" \
     "--@rules_fuzzing//fuzzing:cc_engine_instrumentation=oss-fuzz" \

infra/base-images/base-builder/compile

@@ -17,12 +17,9 @@
 
 echo "---------------------------------------------------------------"
 
-OSS_FUZZ_ON_DEMAND="${OSS_FUZZ_ON_DEMAND:-0}"
+sysctl -w vm.mmap_rnd_bits=28
 
-# This is a temporary fix: fall back to LLVM14's old pass manager
-if [ -n "${OLD_LLVMPASS-}" ]; then
-  export SANITIZER_FLAGS_introspector=$(echo $SANITIZER_FLAGS_introspector | sed -r 's/-O0/-flegacy-pass-manager/')
-fi
+OSS_FUZZ_ON_DEMAND="${OSS_FUZZ_ON_DEMAND:-0}"
 
 if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
   if [ "$FUZZING_ENGINE" != "libfuzzer" ] && [ "$FUZZING_ENGINE" != "wycheproof" ]; then
@@ -77,6 +74,7 @@ fi
 if [[ $ARCHITECTURE == "i386" ]]; then
     export CFLAGS="-m32 $CFLAGS"
     cp -R /usr/i386/lib/* /usr/local/lib
+    cp -R /usr/i386/include/* /usr/local/include
 fi
 
 # Don't use a fuzzing engine with Jazzer which has libFuzzer built-in or with
@@ -131,6 +129,15 @@ export CUSTOM_LIBFUZZER_STD_CXX=c++
 export CFLAGS="$CFLAGS $SANITIZER_FLAGS $COVERAGE_FLAGS"
 export CXXFLAGS="$CFLAGS $CXXFLAGS_EXTRA"
 
+if [ "$SANITIZER" = "undefined" ]; then
+  # Disable "function" sanitizer for C code for now, because many projects,
+  # possibly via legacy C code are affected.
+  # The projects should be fixed and this workaround be removed in the future.
+  # TODO(#11778):
+  # https://github.com/google/oss-fuzz/issues/11778
+  export CFLAGS="$CFLAGS -fno-sanitize=function"
+fi
+
 if [ "$FUZZING_LANGUAGE" = "go" ]; then
   # required by Go 1.20
   export CXX="${CXX} -lresolv"

infra/base-images/base-builder/compile_libfuzzer

@@ -21,6 +21,5 @@ if [ "$FUZZING_LANGUAGE" = "go" ]; then
     export LIB_FUZZING_ENGINE="$LIB_FUZZING_ENGINE $GOPATH/gosigfuzz/gosigfuzz.o"
 fi
 
-cp /usr/local/lib/clang/*/lib/linux/libclang_rt.fuzzer-$ARCHITECTURE.a \
-  $LIB_FUZZING_ENGINE_DEPRECATED
+cp /usr/local/lib/clang/*/lib/$ARCHITECTURE-unknown-linux-gnu/libclang_rt.fuzzer.a $LIB_FUZZING_ENGINE_DEPRECATED
 echo " done."

infra/base-images/base-builder/install_python.sh

@@ -19,5 +19,5 @@ echo "ATHERIS INSTALL"
 unset CFLAGS CXXFLAGS
 # PYI_STATIC_ZLIB=1 is needed for installing pyinstaller 5.0
 export PYI_STATIC_ZLIB=1
-pip3 install -v --no-cache-dir "atheris>=2.1.1" "pyinstaller==5.0.1" "coverage==6.3.2"
+LIBFUZZER_LIB=$( echo /usr/local/lib/clang/*/lib/x86_64-unknown-linux-gnu/libclang_rt.fuzzer_no_main.a ) pip3 install -v --no-cache-dir "atheris>=2.1.1" "pyinstaller==5.0.1" "setuptools==42.0.2" "coverage==6.3.2"
 rm -rf /tmp/*

infra/base-images/base-builder/install_rust.sh

@@ -15,7 +15,8 @@
 #
 ################################################################################
 
-curl https://sh.rustup.rs | sh -s -- -y --default-toolchain=nightly --profile=minimal
+curl https://sh.rustup.rs | sh -s -- -y --default-toolchain=nightly-2023-12-28 --profile=minimal
 cargo install cargo-fuzz && rm -rf /rust/registry
 # Needed to recompile rust std library for MSAN
-rustup component add rust-src --toolchain nightly
+rustup component add rust-src --toolchain nightly-2023-12-28
+cp -r /usr/local/lib/x86_64-unknown-linux-gnu/* /usr/local/lib/

infra/base-images/base-builder/jcc/jcc.go

@@ -357,7 +357,7 @@ func WriteStdErrOut(outstr string, errstr string) {
 	// Prints |outstr| to stdout, prints |errstr| to stderr, and saves |errstr| to err.log.
 	fmt.Print(outstr)
 	fmt.Fprint(os.Stderr, errstr)
-	AppendStringToFile("/out/err.log", errstr)
+	AppendStringToFile("/workspace/err.log", errstr)
 }
 
 func main() {

infra/base-images/base-clang/Dockerfile

@@ -22,7 +22,6 @@ ARG arch=x86_64
 
 ENV FUZZINTRO_OUTDIR=$SRC
 
-
 # Install newer cmake.
 ENV CMAKE_VERSION 3.24.2
 RUN apt-get update && apt-get install -y wget sudo && \
@@ -36,7 +35,7 @@ RUN apt-get update && apt-get install -y wget sudo && \
 RUN apt-get update && apt-get install -y git && \
     git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \
     cd fuzz-introspector && \
-    git checkout 2188913aac0f423ec2aa6f4efdd5de6df0752376 && \
+    git checkout cfb5266a4c45cbec8663bb1b215c7fd326c60901 && \
     git submodule init && \
     git submodule update && \
     apt-get autoremove --purge -y git && \
@@ -56,6 +55,12 @@ ENV CCC "clang++"
 # FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION is described at
 # https://llvm.org/docs/LibFuzzer.html#fuzzer-friendly-build-mode
 
-ENV CFLAGS "-O1 -fno-omit-frame-pointer -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION"
+# The implicit-function-declaration and implicit-int errors are downgraded to a
+# warning, to allow compiling legacy code.
+# See https://releases.llvm.org/16.0.0/tools/clang/docs/ReleaseNotes.html#potentially-breaking-changes
+# Same for deprecated-declarations, int-conversion,
+# incompatible-function-pointer-types, enum-constexpr-conversion
+
+ENV CFLAGS "-O1 -fno-omit-frame-pointer -gline-tables-only -Wno-error=enum-constexpr-conversion -Wno-error=incompatible-function-pointer-types -Wno-error=int-conversion -Wno-error=deprecated-declarations -Wno-error=implicit-function-declaration -Wno-error=implicit-int -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION"
 ENV CXXFLAGS_EXTRA "-stdlib=libc++"
 ENV CXXFLAGS "$CFLAGS $CXXFLAGS_EXTRA"

infra/base-images/base-clang/checkout_build_install_llvm.sh

@@ -50,14 +50,14 @@ LLVM_DEP_PACKAGES="build-essential make ninja-build git python3 python3-distutil
 apt-get update && apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends
 
 # For manual bumping.
-OUR_LLVM_REVISION=llvmorg-15-init-1464-gbf7f8d6f
+OUR_LLVM_REVISION=llvmorg-18-init-4631-gd50b56d1
 
 mkdir $SRC/chromium_tools
 cd $SRC/chromium_tools
 git clone https://chromium.googlesource.com/chromium/src/tools/clang
 cd clang
-# Pin clang due to https://github.com/google/oss-fuzz/issues/7617
-git checkout 946a41a51f44207941b3729a0733dfc1e236644e
+# Pin clang script due to https://github.com/google/oss-fuzz/issues/7617
+git checkout 9eb79319239629c1b23cf7a59e5ebb2bab319a34
 
 # To allow for manual downgrades. Set to 0 to use Chrome's clang version (i.e.
 # *not* force a manual downgrade). Set to 1 to force a manual downgrade.
@@ -91,14 +91,15 @@ function clone_with_retries {
 }
 clone_with_retries https://github.com/llvm/llvm-project.git $LLVM_SRC
 
-PROJECTS_TO_BUILD="libcxx;libcxxabi;compiler-rt;clang;lld"
+PROJECTS_TO_BUILD="clang;lld"
 function cmake_llvm {
   extra_args="$@"
   cmake -G "Ninja" \
       -DLIBCXX_ENABLE_SHARED=OFF \
       -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
       -DLIBCXXABI_ENABLE_SHARED=OFF \
       -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi" \
       -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \
       -DLLVM_ENABLE_PROJECTS="$PROJECTS_TO_BUILD" \
       -DLLVM_BINUTILS_INCDIR="/usr/include/" \
@@ -145,7 +146,8 @@ rm -rf $WORK/llvm-stage1 $WORK/llvm-stage2
 cp -r $LLVM_SRC/compiler-rt/lib/fuzzer $SRC/libfuzzer
 
 # Use the clang we just built from now on.
-CMAKE_EXTRA_ARGS="-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++"
+export CC=clang
+export CXX=clang++
 
 function free_disk_space {
     rm -rf $LLVM_SRC $SRC/chromium_tools
@@ -222,16 +224,16 @@ function cmake_libcxx {
       -DLIBCXXABI_ENABLE_SHARED=OFF \
       -DCMAKE_BUILD_TYPE=Release \
       -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \
-      -DLLVM_ENABLE_PROJECTS="libcxx;libcxxabi" \
+      -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \
       -DLLVM_BINUTILS_INCDIR="/usr/include/" \
       $extra_args \
-      $LLVM_SRC/llvm
+      -S $LLVM_SRC/runtimes
 }
 
 # 32-bit libraries.
 mkdir -p $WORK/i386
 cd $WORK/i386
-cmake_libcxx $CMAKE_EXTRA_ARGS \
+cmake_libcxx \
     -DCMAKE_INSTALL_PREFIX=/usr/i386/ \
     -DCMAKE_C_FLAGS="-m32" \
     -DCMAKE_CXX_FLAGS="-m32"
@@ -249,7 +251,7 @@ cat <<EOF > $WORK/msan/blocklist.txt
 fun:__gxx_personality_*
 EOF
 
-cmake_libcxx $CMAKE_EXTRA_ARGS \
+cmake_libcxx \
     -DLLVM_USE_SANITIZER=Memory \
     -DCMAKE_INSTALL_PREFIX=/usr/msan/ \
     -DCMAKE_CXX_FLAGS="-fsanitize-blacklist=$WORK/msan/blocklist.txt"

infra/base-images/base-runner/bad_build_check

@@ -44,8 +44,8 @@ MSAN_CALLS_THRESHOLD_FOR_NON_MSAN_BUILD=3
 # Usually, a non UBSan build (e.g. ASan) has 165 calls to UBSan runtime. The
 # majority of targets built with UBSan have 200+ UBSan calls, but there are
 # some very small targets that may have < 200 UBSan calls even in a UBSan build.
-# Use the threshold value of 169 (slightly > 165) for UBSan build.
-UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD=169
+# Use the threshold value of 168 (slightly > 165) for UBSan build.
+UBSAN_CALLS_THRESHOLD_FOR_UBSAN_BUILD=168
 
 # It would be risky to use the threshold value close to 165 for non UBSan build,
 # as UBSan runtime may change any time and thus we could have different number

infra/base-images/base-runner/run_fuzzer

@@ -18,6 +18,8 @@
 # Fuzzer runner. Appends .options arguments and seed corpus to users args.
 # Usage: $0 <fuzzer_name> <fuzzer_args>
 
+sysctl -w vm.mmap_rnd_bits=28
+
 export PATH=$OUT:$PATH
 cd $OUT
 

infra/build/functions/build_lib.py

@@ -23,13 +23,16 @@
 import sys
 import time
 import subprocess
+import tarfile
 import tempfile
 import json
+import uuid
 
 from googleapiclient.discovery import build as cloud_build
 import googleapiclient.discovery
 import google.api_core.client_options
 import google.auth
+from google.cloud import storage
 from oauth2client import service_account as service_account_lib
 import requests
 import yaml
@@ -547,6 +550,15 @@ def get_build_body(steps,
   return build_body
 
 
+def _tgz_local_build(oss_fuzz_project, temp_tgz_path):
+  """Prepare a .tgz containing the files required to build `oss_fuzz_project`."""
+  # Just the projects/<project> dir should be sufficient.
+  project_rel_path = os.path.join('projects', oss_fuzz_project)
+  with tarfile.open(temp_tgz_path, 'w:gz') as tar:
+    tar.add(os.path.join(OSS_FUZZ_ROOT, project_rel_path),
+            arcname=project_rel_path)
+
+
 def run_build(  # pylint: disable=too-many-arguments
     oss_fuzz_project,
     steps,
@@ -566,23 +578,24 @@ def run_build(  # pylint: disable=too-many-arguments
                               use_build_pool=use_build_pool,
                               experiment=experiment)
   if experiment:
-    with tempfile.NamedTemporaryFile(suffix='build.json') as config_file:
-      config_file.write(bytes(json.dumps(build_body), 'utf-8'))
-      config_file.seek(0)
-      result = subprocess.run([
-          'gcloud',
-          'builds',
-          'submit',
-          '--project=oss-fuzz',
-          f'--config={config_file.name}',
-          '--async',
-          '--format=get(id)',
-      ],
-                              stdout=subprocess.PIPE,
-                              cwd=OSS_FUZZ_ROOT,
-                              encoding='utf-8',
-                              check=True)
-      return result.stdout.strip()
+    with tempfile.NamedTemporaryFile(suffix='source.tgz') as tgz_file:
+      # Archive the necessary files for the build.
+      _tgz_local_build(oss_fuzz_project, tgz_file.name)
+      gcs_client = storage.Client()
+      # This is the automatically created Cloud Build bucket for Cloud Build.
+      bucket_name = gcs_client.project + '_cloudbuild'
+      bucket = gcs_client.bucket(bucket_name)
+      blob_name = f'source/{str(uuid.uuid4())}.tgz'
+      blob = bucket.blob(blob_name)
+      logging.info(f'Uploading project to {bucket_name}/{blob_name}')
+      blob.upload_from_filename(tgz_file.name)
+
+      build_body['source'] = {
+          'storageSource': {
+              'bucket': bucket_name,
+              'object': blob_name,
+          }
+      }
 
   cloudbuild = cloud_build('cloudbuild',
                            'v1',
@@ -616,7 +629,7 @@ def wait_for_build(build_id, credentials, cloud_project):
                                   'INTERNAL_ERROR', 'EXPIRED', 'CANCELLED'):
         # Build done.
         return
-    except googleapiclient.errors.HttpError:
+    except (googleapiclient.errors.HttpError, BrokenPipeError):
       pass
 
     time.sleep(15)  # Avoid rate limiting.
@@ -630,4 +643,4 @@ def cancel_build(build_id, credentials, cloud_project):
                            cache_discovery=False,
                            client_options=US_CENTRAL_CLIENT_OPTIONS)
   cloudbuild.projects().builds().cancel(projectId=cloud_project,
-                                        id=build_id).execute()
+                                        id=build_id).execute()