Update Clang toolchain from 18.0.0 to 18.1.8 (#12365)

Follow-up on #12077 by @alexcrichton cc @maflcko Main difference is to update infra/base-images/base-runner/profraw_update.py so that oss-fuzz converts profraw version 8 to 9 (and llvm-cov seems more tolerant in older version reading cf llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp This way, it should be more transparent for projects, that can be updated individually or not --------- Co-authored-by: Alex Crichton <[email protected]>
google · Oct 1, 2024 · 1778ebe · 1778ebe
1 parent 10779de
commit 1778ebe
Show file tree

Hide file tree

Showing 14 changed files with 128 additions and 50 deletions.
diff --git a/infra/base-images/base-builder-rust/Dockerfile b/infra/base-images/base-builder-rust/Dockerfile
@@ -26,7 +26,7 @@ ENV OSSFUZZ_RUSTPATH /rust
 # manually specifying what toolchain to use. Note that this environment variable
 # is additionally used by `install_rust.sh` as the toolchain to install.
 # cf https://rust-lang.github.io/rustup/overrides.html
-ENV RUSTUP_TOOLCHAIN nightly-2024-02-12
+ENV RUSTUP_TOOLCHAIN nightly-2024-07-12
 
 # Configure the linker used by default for x86_64 linux to be `clang` instead of
 # rustc's default of `cc` which is able to find custom-built libraries like

diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile
@@ -60,8 +60,19 @@ ENV CCC "clang++"
 # warning, to allow compiling legacy code.
 # See https://releases.llvm.org/16.0.0/tools/clang/docs/ReleaseNotes.html#potentially-breaking-changes
 # Same for deprecated-declarations, int-conversion,
-# incompatible-function-pointer-types, enum-constexpr-conversion
+# incompatible-function-pointer-types, enum-constexpr-conversion,
+# vla-cxx-extension
 
-ENV CFLAGS "-O1 -fno-omit-frame-pointer -gline-tables-only -Wno-error=enum-constexpr-conversion -Wno-error=incompatible-function-pointer-types -Wno-error=int-conversion -Wno-error=deprecated-declarations -Wno-error=implicit-function-declaration -Wno-error=implicit-int -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION"
+ENV CFLAGS -O1 \
+  -fno-omit-frame-pointer \
+  -gline-tables-only \
+  -Wno-error=enum-constexpr-conversion \
+  -Wno-error=incompatible-function-pointer-types \
+  -Wno-error=int-conversion \
+  -Wno-error=deprecated-declarations \
+  -Wno-error=implicit-function-declaration \
+  -Wno-error=implicit-int \
+  -Wno-error=vla-cxx-extension \
+  -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
 ENV CXXFLAGS_EXTRA "-stdlib=libc++"
 ENV CXXFLAGS "$CFLAGS $CXXFLAGS_EXTRA"
diff --git a/infra/base-images/base-clang/checkout_build_install_llvm.sh b/infra/base-images/base-clang/checkout_build_install_llvm.sh
@@ -54,7 +54,7 @@ apt-get update && apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends
 # languages, projects, ...) is needed.
 # Check CMAKE_VERSION infra/base-images/base-clang/Dockerfile was released
 # recently enough to fully support this clang version.
-OUR_LLVM_REVISION=llvmorg-18-init-4631-gd50b56d1
+OUR_LLVM_REVISION=llvmorg-18.1.8
 
 mkdir $SRC/chromium_tools
 cd $SRC/chromium_tools
@@ -116,6 +116,7 @@ cmake -G "Ninja" \
   -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \
   -DLLVM_ENABLE_PROJECTS="clang;lld" \
   -DLLVM_BINUTILS_INCDIR="/usr/include/" \
+  -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \
   $LLVM_SRC/llvm
 
 ninja -j $NPROC
@@ -202,6 +203,7 @@ function cmake_libcxx {
       -DLIBCXX_ENABLE_SHARED=OFF \
       -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
       -DLIBCXXABI_ENABLE_SHARED=OFF \
+      -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \
       -DCMAKE_BUILD_TYPE=Release \
       -DLLVM_ENABLE_PIC=ON \
       -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \

diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage
@@ -115,7 +115,7 @@ function run_fuzz_target {
     # Extract fuzztest binary name from fuzztest wrapper script.
     target=(${target//@/ }[0])
   fi
-  profraw_update.py $OUT/$target $profraw_file_mask $profraw_file_mask
+  profraw_update.py $OUT/$target -i $profraw_file_mask
   llvm-profdata merge -j=1 -sparse $profraw_file_mask -o $profdata_file
 
   # Delete unnecessary and (potentially) large .profraw files.

diff --git a/infra/base-images/base-runner/profraw_update.py b/infra/base-images/base-runner/profraw_update.py
@@ -22,10 +22,11 @@
 import sys
 
 HeaderGeneric = namedtuple('HeaderGeneric', 'magic version')
-HeaderVersion7 = namedtuple(
-    'HeaderVersion7',
+HeaderVersion9 = namedtuple(
+    'HeaderVersion9',
     'BinaryIdsSize DataSize PaddingBytesBeforeCounters CountersSize \
-    PaddingBytesAfterCounters NamesSize CountersDelta NamesDelta ValueKindLast')
+    PaddingBytesAfterCounters NumBitmapBytes PaddingBytesAfterBitmapBytes NamesSize CountersDelta BitmapDelta NamesDelta ValueKindLast'
+)
 
 PROFRAW_MAGIC = 0xff6c70726f667281
 
@@ -39,65 +40,100 @@ def relativize_address(data, offset, databegin, sect_prf_cnts, sect_prf_data):
     value = struct.pack('Q', value)
     for i in range(8):
       data[offset + i] = value[i]
+    # address was made relative
+    return True
+  # no changes done
+  return False
 
 
 def upgrade(data, sect_prf_cnts, sect_prf_data):
   """Upgrades profraw data, knowing the sections addresses."""
   generic_header = HeaderGeneric._make(struct.unpack('QQ', data[:16]))
   if generic_header.magic != PROFRAW_MAGIC:
     raise Exception('Bad magic.')
+  base_version = generic_header.version
+
+  if base_version >= 9:
+    # Nothing to do.
+    return data
+  if base_version < 5 or base_version == 6:
+    raise Exception('Unhandled version.')
+
   if generic_header.version == 5:
     generic_header = generic_header._replace(version=7)
     # Upgrade from version 5 to 7 by adding binaryids field.
     data = data[:8] + struct.pack('Q', generic_header.version) + struct.pack(
         'Q', 0) + data[16:]
-  if generic_header.version < 7:
-    raise Exception('Unhandled version.')
   if generic_header.version == 7:
     # cf https://reviews.llvm.org/D111123
     generic_header = generic_header._replace(version=8)
     data = data[:8] + struct.pack('Q', generic_header.version) + data[16:]
-  v7_header = HeaderVersion7._make(struct.unpack('QQQQQQQQQ', data[16:88]))
+  if generic_header.version == 8:
+    # see https://reviews.llvm.org/D138846
+    generic_header = generic_header._replace(version=9)
+    # Upgrade from version 8 to 9 by adding NumBitmapBytes, PaddingBytesAfterBitmapBytes and BitmapDelta fields.
+    data = data[:8] + struct.pack(
+        'Q', generic_header.version) + data[16:56] + struct.pack(
+            'QQ', 0, 0) + data[56:72] + struct.pack('Q', 0) + data[72:]
+
+  v9_header = HeaderVersion9._make(struct.unpack('QQQQQQQQQQQQ', data[16:112]))
 
-  if v7_header.BinaryIdsSize % 8 != 0:
+  if base_version <= 8 and v9_header.BinaryIdsSize % 8 != 0:
     # Adds padding for binary ids.
     # cf commit b9f547e8e51182d32f1912f97a3e53f4899ea6be
     # cf https://reviews.llvm.org/D110365
-    padlen = 8 - (v7_header.BinaryIdsSize % 8)
-    v7_header = v7_header._replace(BinaryIdsSize=v7_header.BinaryIdsSize +
+    padlen = 8 - (v9_header.BinaryIdsSize % 8)
+    v7_header = v9_header._replace(BinaryIdsSize=v9_header.BinaryIdsSize +
                                    padlen)
-    data = data[:16] + struct.pack('Q', v7_header.BinaryIdsSize) + data[24:]
-    data = data[:88 + v7_header.BinaryIdsSize] + bytes(
-        padlen) + data[88 + v7_header.BinaryIdsSize:]
-
-  if v7_header.CountersDelta != (sect_prf_cnts -
-                                 sect_prf_data) & 0xffffffffffffffff:
-    # Rust linking seems to add an offset...
-    sect_prf_data = v7_header.CountersDelta - sect_prf_cnts + sect_prf_data
-    sect_prf_cnts = v7_header.CountersDelta
-
+    data = data[:16] + struct.pack('Q', v9_header.BinaryIdsSize) + data[24:]
+    data = data[:112 + v9_header.BinaryIdsSize] + bytes(
+        padlen) + data[112 + v9_header.BinaryIdsSize:]
+
+  if base_version <= 8:
+    offset = 112 + v9_header.BinaryIdsSize
+    for d in range(v9_header.DataSize):
+      # Add BitmapPtr and aligned u32(NumBitmapBytes)
+      data = data[:offset + 3 * 8] + struct.pack(
+          'Q', 0) + data[offset + 3 * 8:offset + 6 * 8] + struct.pack(
+              'Q', 0) + data[offset + 6 * 8:]
+      value = struct.unpack('Q',
+                            data[offset + 2 * 8:offset + 3 * 8])[0] - 16 * d
+      data = data[:offset + 2 * 8] + struct.pack('Q',
+                                                 value) + data[offset + 3 * 8:]
+      offset += 8 * 8
+
+  if base_version >= 8:
+    # Nothing more to do.
+    return data
+
+  # Last changes are relaed to bump from 7 to version 8 making CountersPtr relative.
   dataref = sect_prf_data
-  relativize_address(data, 64, dataref, sect_prf_cnts, sect_prf_data)
+  # 80 is offset of CountersDelta.
+  if not relativize_address(data, 80, dataref, sect_prf_cnts, sect_prf_data):
+    return data
 
-  offset = 88 + v7_header.BinaryIdsSize
+  offset = 112 + v9_header.BinaryIdsSize
   # This also works for C+Rust binaries compiled with
   # clang-14/rust-nightly-clang-13.
-  for _ in range(v7_header.DataSize):
+  for _ in range(v9_header.DataSize):
     # 16 is the offset of CounterPtr in ProfrawData structure.
     relativize_address(data, offset + 16, dataref, sect_prf_cnts, sect_prf_data)
     # We need this because of CountersDelta -= sizeof(*SrcData);
     # seen in __llvm_profile_merge_from_buffer.
-    dataref += 44 + 2 * (v7_header.ValueKindLast + 1)
+    dataref += 44 + 2 * (v9_header.ValueKindLast + 1)
+    if was8:
+      #profraw9 added RelativeBitmapPtr and NumBitmapBytes (8+4 rounded up to 16)
+      dataref -= 16
     # This is the size of one ProfrawData structure.
-    offset += 44 + 2 * (v7_header.ValueKindLast + 1)
+    offset += 44 + 2 * (v9_header.ValueKindLast + 1)
 
   return data
 
 
 def main():
   """Helper script for upgrading a profraw file to latest version."""
-  if len(sys.argv) != 4:
-    sys.stderr.write('Usage: %s <binary> <profraw> <output>\n' % sys.argv[0])
+  if len(sys.argv) < 3:
+    sys.stderr.write('Usage: %s <binary> options? <profraw>...\n' % sys.argv[0])
     return 1
 
   # First find llvm profile sections addresses in the elf, quick and dirty.
@@ -113,14 +149,31 @@ def main():
     elif b'__llvm_prf_data' in line:
       sect_prf_data = int(line.split()[3], 16)
 
-  # Then open and read the input profraw file.
-  with open(sys.argv[2], 'rb') as input_file:
-    profraw_base = bytearray(input_file.read())
-  # Do the upgrade, returning a bytes object.
-  profraw_latest = upgrade(profraw_base, sect_prf_cnts, sect_prf_data)
-  # Write the output to the file given to the command line.
-  with open(sys.argv[3], 'wb') as output_file:
-    output_file.write(profraw_latest)
+  out_name = "default.profup"
+  in_place = False
+  start = 2
+  if sys.argv[2] == "-i":
+    in_place = True
+    start = start + 1
+  elif sys.argv[2] == "-o":
+    out_name = sys.argv[3]
+    start = 4
+
+  if len(sys.argv) < start:
+    sys.stderr.write('Usage: %s <binary> options <profraw>...\n' % sys.argv[0])
+    return 1
+
+  for i in range(start, len(sys.argv)):
+    # Then open and read the input profraw file.
+    with open(sys.argv[i], 'rb') as input_file:
+      profraw_base = bytearray(input_file.read())
+    # Do the upgrade, returning a bytes object.
+    profraw_latest = upgrade(profraw_base, sect_prf_cnts, sect_prf_data)
+    # Write the output to the file given to the command line.
+    if in_place:
+      out_name = sys.argv[i]
+    with open(out_name, 'wb') as output_file:
+      output_file.write(profraw_latest)
 
   return 0
 

diff --git a/projects/elfutils/Dockerfile b/projects/elfutils/Dockerfile
@@ -14,7 +14,10 @@
 #
 ################################################################################
 
-FROM gcr.io/oss-fuzz-base/base-builder
+FROM gcr.io/oss-fuzz-base/base-builder@sha256:56905c98ae0083d14da0e7371184e694560a74750533f321ac0e9145af0e8d2e
+# ! This project was pinned after a clang bump. Please remove the pin, Try to fix any build warnings and errors, as well as runtime errors
+# see https://github.com/google/oss-fuzz/pull/12365
+
 RUN apt-get update && \
     apt-get install -y pkg-config make autoconf autopoint zlib1g-dev zlib1g-dev:i386 flex gawk bison
 RUN git clone --depth 1 https://sourceware.org/git/elfutils.git

diff --git a/projects/envoy/Dockerfile b/projects/envoy/Dockerfile
@@ -15,7 +15,10 @@
 ################################################################################
 
 
-FROM gcr.io/oss-fuzz-base/base-builder
+FROM gcr.io/oss-fuzz-base/base-builder@sha256:56905c98ae0083d14da0e7371184e694560a74750533f321ac0e9145af0e8d2e
+# ! This project was pinned after a clang bump. Please remove the pin, Try to fix any build warnings and errors, as well as runtime errors
+# see https://github.com/google/oss-fuzz/pull/12365
+
 
 RUN apt-get update && apt-get -y install  \
 	build-essential \

diff --git a/projects/icu/project.yaml b/projects/icu/project.yaml
@@ -17,5 +17,10 @@ sanitizers:
  - undefined 
 # Disabled MSAN because of https://github.com/google/oss-fuzz/issues/6294
 #  - memory
+fuzzing_engines:
+  - libfuzzer
+  - honggfuzz
+  - afl
+  # - centipede disabled due to https://github.com/google/oss-fuzz/pull/12365 clang 18 update
 
 main_repo: 'https://github.com/unicode-org/icu.git'
diff --git a/projects/librawspeed/build.sh b/projects/librawspeed/build.sh
@@ -15,4 +15,6 @@
 #
 ################################################################################
 
+export CFLAGS="$CFLAGS -Wno-error=nan-infinity-disabled"
+export CXXFLAGS="$CXXFLAGS -Wno-error=nan-infinity-disabled"
 $SRC/librawspeed/.ci/oss-fuzz.sh
diff --git a/projects/rust-lexical/Dockerfile b/projects/rust-lexical/Dockerfile
@@ -13,7 +13,10 @@
 # limitations under the License.
 #
 ################################################################################
-FROM gcr.io/oss-fuzz-base/base-builder-rust
+FROM gcr.io/oss-fuzz-base/base-builder-rust@sha256:b9a45fecf0d9be6559fca019e90577632242be120ee2d97cec5c2045c1440710
+# ! This project was pinned after a clang bump. Please remove the pin, Try to fix any build warnings and errors, as well as runtime errors
+# /usr/bin/ld: /src/rust-lexical/fuzz/target/x86_64-unknown-linux-gnu/release/deps/parse_integer_u16-53e4bc89ab30e724.parse_integer_u16.9056e4c0a19617b4-cgu.0.rcgu.o: in function `asan.module_dtor.204':
+#          parse_integer_u16.9056e4c0a19617b4-cgu.0:(.text.asan.module_dtor.204[asan.module_dtor]+0x6): undefined reference to `__sancov_gen_.998'
 
 RUN git clone --depth 1 https://github.com/Alexhuszagh/rust-lexical
 COPY build.sh $SRC/
diff --git a/projects/samba/build.sh b/projects/samba/build.sh
@@ -15,6 +15,6 @@
 #
 ################################################################################
 
-export CFLAGS="$CFLAGS -Wno-error=strict-prototypes"
+export CFLAGS="$CFLAGS -Wno-error=strict-prototypes -Wno-error=format-truncation"
 # The real script is maintained in the Samba repo
 exec lib/fuzzing/oss-fuzz/build_samba.sh
diff --git a/projects/suricata/Dockerfile b/projects/suricata/Dockerfile
@@ -29,13 +29,9 @@ RUN git clone --depth=1 https://github.com/catenacyber/fuzzpcap
 
 ADD https://rules.emergingthreats.net/open/suricata/emerging.rules.zip emerging.rules.zip
 
-ENV RUSTUP_TOOLCHAIN nightly
 RUN cargo install --force cbindgen
-# TODO remove once we have clang with coverage version 9 as rustc
-ENV RUSTUP_TOOLCHAIN nightly-2024-02-12
 
 RUN git clone --depth 1 https://github.com/OISF/suricata.git suricata
-RUN git clone --depth 1 --branch master-6.0.x https://github.com/OISF/suricata.git suricata6
 RUN git clone --depth 1 --branch main-7.0.x https://github.com/OISF/suricata.git suricata7
 RUN git clone --depth 1 https://github.com/OISF/libhtp.git libhtp
 RUN git clone --depth 1 https://github.com/OISF/suricata-verify suricata-verify

diff --git a/projects/suricata/build.sh b/projects/suricata/build.sh
@@ -160,7 +160,6 @@ cat $t/*.rules > corpus/$i || true; echo -ne '\0' >> corpus/$i; cat $t/*.pcap >>
 done
 set -x
 zip -q -r $OUT/fuzz_sigpcap_seed_corpus.zip corpus
-cp $OUT/fuzz_sigpcap_seed_corpus.zip $OUT/fuzz_sigpcap6_seed_corpus.zip
 rm -Rf corpus
 mkdir corpus
 set +x
@@ -170,7 +169,6 @@ echo -ne '\0' >> corpus/$i; python3 $SRC/fuzzpcap/tcptofpc.py $t/*.pcap >> corpu
 done
 set -x
 zip -q -r $OUT/fuzz_sigpcap_aware_seed_corpus.zip corpus
-cp $OUT/fuzz_sigpcap_aware_seed_corpus.zip $OUT/fuzz_sigpcap_aware6_seed_corpus.zip
 rm -Rf corpus
 mkdir corpus
 set +x
@@ -180,4 +178,3 @@ python3 $SRC/fuzzpcap/tcptofpc.py $t/*.pcap >> corpus/$i || rm corpus/$i; i=$((i
 done
 set -x
 zip -q -r $OUT/fuzz_predefpcap_aware_seed_corpus.zip corpus
-cp $OUT/fuzz_predefpcap_aware_seed_corpus.zip $OUT/fuzz_predefpcap_aware6_seed_corpus.zip
diff --git a/projects/wasmer/Dockerfile b/projects/wasmer/Dockerfile
@@ -27,4 +27,7 @@ RUN mkdir -p $SRC/.llvm && curl --proto '=https' --tlsv1.2 -sSf \
 
 WORKDIR wasmer
 
+# dead code warnings with nightly-2024-07-12
+ENV RUSTUP_TOOLCHAIN nightly-2024-02-12
+
 COPY build.sh default.options $SRC/