Add FuzzTest and use it to fuzz types

.flake8

@@ -6,4 +6,4 @@ ignore =
   E241,
   ; line break after binary operator
   W504
-exclude = third_party,./test/emscripten,./test/spec,./test/wasm-install,./test/lit
+exclude = third_party,./test/emscripten,./test/spec,./test/wasm-install,./test/lit,./_deps

.github/workflows/ci.yml

@@ -147,6 +147,34 @@ jobs:
     - name: test
       run: python check.py --binaryen-bin=out/bin
 
+  # Copied and modified from build-clang
+  build-fuzztest:
+    name: clang with fuzztest
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: install ninja
+      run: sudo apt-get install ninja-build
+    - name: install v8
+      run: |
+        npm install jsvu -g
+        jsvu --os=default --engines=v8
+    - name: install Python dev dependencies
+      run: pip3 install -r requirements-dev.txt
+    - name: cmake
+      run: |
+        mkdir -p out
+        cmake -S . -B out -G Ninja -DCMAKE_INSTALL_PREFIX=out/install -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DBUILD_FUZZTEST=ON
+    - name: build
+      run: cmake --build out -v
+    - name: test
+      run: python check.py --binaryen-bin=out/bin
+
   # TODO(sbc): Find a way to reduce the duplicate between these sanitizer jobs
   build-asan:
     name: asan

.gitignore

@@ -21,6 +21,8 @@ CMakeFiles
 /.ninja_log
 /bin/
 /lib/
+/_deps/
+/dist/
 /config.h
 /emcc-build
 compile_commands.json

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "test/spec/testsuite"]
 	path = test/spec/testsuite
 	url = https://github.com/WebAssembly/testsuite.git
+[submodule "third_party/fuzztest"]
+	path = third_party/fuzztest
+	url = https://github.com/google/fuzztest

CMakeLists.txt

@@ -36,6 +36,8 @@ option(BYN_ENABLE_LTO "Build with LTO" Off)
 # Turn this off to avoid the dependency on gtest.
 option(BUILD_TESTS "Build GTest-based tests" ON)
 
+option(BUILD_FUZZTEST "Build fuzztest-based tests and fuzzers" OFF)
+
 # Turn this off to build only the library.
 option(BUILD_TOOLS "Build tools" ON)
 
@@ -161,8 +163,8 @@ endfunction()
 
 function(binaryen_add_executable name sources)
   add_executable(${name} ${sources})
-  target_link_libraries(${name} Threads::Threads)
-  target_link_libraries(${name} binaryen)
+  target_link_libraries(${name} PRIVATE Threads::Threads)
+  target_link_libraries(${name} PRIVATE binaryen)
   binaryen_setup_rpath(${name})
   install(TARGETS ${name} DESTINATION ${CMAKE_INSTALL_BINDIR})
 endfunction()
@@ -258,7 +260,10 @@ if(MSVC)
 else() # MSVC
 
   add_compile_flag("-fno-omit-frame-pointer")
-  add_compile_flag("-fno-rtti")
+  if(NOT BUILD_FUZZTEST)
+    # fuzztest depends on RTTIs.
+    add_compile_flag("-fno-rtti")
+  endif()
   if(WIN32)
     add_compile_flag("-D_GNU_SOURCE")
     add_compile_flag("-D__STDC_FORMAT_MACROS")
@@ -276,10 +281,6 @@ else() # MSVC
     # explicitly undefine it:
     add_nondebug_compile_flag("-UNDEBUG")
   endif()
-  if(NOT APPLE AND NOT "${CMAKE_CXX_FLAGS}" MATCHES "-fsanitize")
-    # This flag only applies to shared libraries so don't use add_link_flag
-    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
-  endif()
 endif()
 
 if(EMSCRIPTEN)
@@ -416,6 +417,25 @@ else() # MSVC
     add_compile_flag("-Wno-deprecated-declarations")
   endif()
 
+  if(BUILD_FUZZTEST)
+    add_compile_flag("-DFUZZTEST")
+    fuzztest_setup_fuzzing_flags()
+
+    # Enabling fuzzing mode turns on sanitizers, which turn on additional
+    # warnings. To keep the build working, do not treat these warnings as
+    # errors.
+    add_compile_flag("-Wno-error=maybe-uninitialized")
+    add_compile_flag("-Wno-error=uninitialized")
+    add_compile_flag("-Wno-error=array-bounds")
+    add_compile_flag("-Wno-error=stringop-overread")
+    add_compile_flag("-Wno-error=missing-field-initializers")
+  endif()
+
+  if(NOT APPLE AND NOT "${CMAKE_CXX_FLAGS}" MATCHES "-fsanitize")
+    # This flag only applies to shared libraries so don't use add_link_flag
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
+  endif()
+
 endif()
 
 # Declare libbinaryen
@@ -483,83 +503,84 @@ endif()
 if(EMSCRIPTEN)
   # binaryen.js WebAssembly variant
   add_executable(binaryen_wasm ${binaryen_SOURCES})
-  target_link_libraries(binaryen_wasm binaryen)
-  target_link_libraries(binaryen_wasm "-sFILESYSTEM")
-  target_link_libraries(binaryen_wasm "-sEXPORT_NAME=Binaryen")
-  target_link_libraries(binaryen_wasm "-sNODERAWFS=0")
+  target_link_libraries(binaryen_wasm PRIVATE binaryen)
+  target_link_libraries(binaryen_wasm PRIVATE "-sFILESYSTEM")
+  target_link_libraries(binaryen_wasm PRIVATE "-sEXPORT_NAME=Binaryen")
+  target_link_libraries(binaryen_wasm PRIVATE "-sNODERAWFS=0")
   # Do not error on the repeated NODERAWFS argument
-  target_link_libraries(binaryen_wasm "-Wno-unused-command-line-argument")
+  target_link_libraries(binaryen_wasm PRIVATE "-Wno-unused-command-line-argument")
   # Emit a single file for convenience of people using binaryen.js as a library,
   # so they only need to distribute a single file.
   if(EMSCRIPTEN_ENABLE_SINGLE_FILE)
-    target_link_libraries(binaryen_wasm "-sSINGLE_FILE")
-  endif()
-  target_link_libraries(binaryen_wasm "-sEXPORT_ES6")
-  target_link_libraries(binaryen_wasm "-sEXPORTED_RUNTIME_METHODS=stringToUTF8OnStack,stringToAscii")
-  target_link_libraries(binaryen_wasm "-sEXPORTED_FUNCTIONS=_malloc,_free")
-  target_link_libraries(binaryen_wasm "--post-js=${CMAKE_CURRENT_SOURCE_DIR}/src/js/binaryen.js-post.js")
-  target_link_libraries(binaryen_wasm "-msign-ext")
-  target_link_libraries(binaryen_wasm "-mbulk-memory")
-  target_link_libraries(binaryen_wasm optimized "--closure=1")
+    target_link_libraries(binaryen_wasm PRIVATE "-sSINGLE_FILE")
+  endif()
+  target_link_libraries(binaryen_wasm PRIVATE "-sEXPORT_ES6")
+  target_link_libraries(binaryen_wasm PRIVATE "-sEXPORTED_RUNTIME_METHODS=stringToUTF8OnStack,stringToAscii")
+  target_link_libraries(binaryen_wasm PRIVATE "-sEXPORTED_FUNCTIONS=_malloc,_free")
+  target_link_libraries(binaryen_wasm PRIVATE "--post-js=${CMAKE_CURRENT_SOURCE_DIR}/src/js/binaryen.js-post.js")
+  target_link_libraries(binaryen_wasm PRIVATE "-msign-ext")
+  target_link_libraries(binaryen_wasm PRIVATE "-mbulk-memory")
+  target_link_libraries(binaryen_wasm PRIVATE optimized "--closure=1")
   # TODO: Fix closure warnings! (#5062)
-  target_link_libraries(binaryen_wasm optimized "-Wno-error=closure")
-  target_link_libraries(binaryen_wasm optimized "-flto")
-  target_link_libraries(binaryen_wasm debug "--profiling")
+  target_link_libraries(binaryen_wasm PRIVATE optimized "-Wno-error=closure")
+  target_link_libraries(binaryen_wasm PRIVATE optimized "-flto")
+  target_link_libraries(binaryen_wasm PRIVATE debug "--profiling")
   # Avoid catching exit as that can confuse error reporting in Node,
   # see https://github.com/emscripten-core/emscripten/issues/17228
-  target_link_libraries(binaryen_wasm "-sNODEJS_CATCH_EXIT=0")
+  target_link_libraries(binaryen_wasm PRIVATE "-sNODEJS_CATCH_EXIT=0")
   install(TARGETS binaryen_wasm DESTINATION ${CMAKE_INSTALL_BINDIR})
 
   # binaryen.js JavaScript variant
   add_executable(binaryen_js ${binaryen_SOURCES})
-  target_link_libraries(binaryen_js binaryen)
-  target_link_libraries(binaryen_js "-sWASM=0")
-  target_link_libraries(binaryen_js "-sWASM_ASYNC_COMPILATION=0")
+  target_link_libraries(binaryen_js PRIVATE binaryen)
+  target_link_libraries(binaryen_js PRIVATE "-sWASM=0")
+  target_link_libraries(binaryen_js PRIVATE "-sWASM_ASYNC_COMPILATION=0")
+
   if(${CMAKE_CXX_COMPILER_VERSION} STREQUAL "6.0.1")
     # only valid with fastcomp and WASM=0
-    target_link_libraries(binaryen_js "-sELIMINATE_DUPLICATE_FUNCTIONS")
+    target_link_libraries(binaryen_js PRIVATE "-sELIMINATE_DUPLICATE_FUNCTIONS")
   endif()
   # Disabling filesystem and setting web environment for js_of_ocaml
   # so it doesn't try to detect the "node" environment
   if(JS_OF_OCAML)
-    target_link_libraries(binaryen_js "-sFILESYSTEM=0")
-    target_link_libraries(binaryen_js "-sENVIRONMENT=web,worker")
+    target_link_libraries(binaryen_js PRIVATE "-sFILESYSTEM=0")
+    target_link_libraries(binaryen_js PRIVATE "-sENVIRONMENT=web,worker")
   else()
-    target_link_libraries(binaryen_js "-sFILESYSTEM=1")
+    target_link_libraries(binaryen_js PRIVATE "-sFILESYSTEM=1")
   endif()
-  target_link_libraries(binaryen_js "-sNODERAWFS=0")
+  target_link_libraries(binaryen_js PRIVATE "-sNODERAWFS=0")
   # Do not error on the repeated NODERAWFS argument
-  target_link_libraries(binaryen_js "-Wno-unused-command-line-argument")
+  target_link_libraries(binaryen_js PRIVATE "-Wno-unused-command-line-argument")
   if(EMSCRIPTEN_ENABLE_SINGLE_FILE)
-    target_link_libraries(binaryen_js "-sSINGLE_FILE")
+    target_link_libraries(binaryen_js PRIVATE "-sSINGLE_FILE")
   endif()
-  target_link_libraries(binaryen_js "-sEXPORT_NAME=Binaryen")
+  target_link_libraries(binaryen_js PRIVATE "-sEXPORT_NAME=Binaryen")
   # Currently, js_of_ocaml can only process ES5 code
   if(JS_OF_OCAML)
-    target_link_libraries(binaryen_js "-sEXPORT_ES6=0")
+    target_link_libraries(binaryen_js PRIVATE "-sEXPORT_ES6=0")
   else()
-    target_link_libraries(binaryen_js "-sEXPORT_ES6=1")
+    target_link_libraries(binaryen_js PRIVATE "-sEXPORT_ES6=1")
   endif()
-  target_link_libraries(binaryen_js "-sEXPORTED_RUNTIME_METHODS=stringToUTF8OnStack,stringToAscii")
-  target_link_libraries(binaryen_js "-sEXPORTED_FUNCTIONS=_malloc,_free")
-  target_link_libraries(binaryen_js "--post-js=${CMAKE_CURRENT_SOURCE_DIR}/src/js/binaryen.js-post.js")
+  target_link_libraries(binaryen_js PRIVATE "-sEXPORTED_RUNTIME_METHODS=stringToUTF8OnStack,stringToAscii")
+  target_link_libraries(binaryen_js PRIVATE "-sEXPORTED_FUNCTIONS=_malloc,_free")
+  target_link_libraries(binaryen_js PRIVATE "--post-js=${CMAKE_CURRENT_SOURCE_DIR}/src/js/binaryen.js-post.js")
   # js_of_ocaml needs a specified variable with special comment to provide the library to consumers
   if(JS_OF_OCAML)
-    target_link_libraries(binaryen_js "--extern-pre-js=${CMAKE_CURRENT_SOURCE_DIR}/src/js/binaryen.jsoo-extern-pre.js")
+    target_link_libraries(binaryen_js PRIVATE "--extern-pre-js=${CMAKE_CURRENT_SOURCE_DIR}/src/js/binaryen.jsoo-extern-pre.js")
   endif()
-  target_link_libraries(binaryen_js optimized "--closure=1")
+  target_link_libraries(binaryen_js PRIVATE optimized "--closure=1")
   # Currently, js_of_ocaml can only process ES5 code
   if(JS_OF_OCAML)
-    target_link_libraries(binaryen_js optimized "--closure-args=\"--language_out=ECMASCRIPT5\"")
+    target_link_libraries(binaryen_js PRIVATE optimized "--closure-args=\"--language_out=ECMASCRIPT5\"")
   endif()
   # TODO: Fix closure warnings! (#5062)
-  target_link_libraries(binaryen_js optimized "-Wno-error=closure")
-  target_link_libraries(binaryen_js optimized "-flto")
-  target_link_libraries(binaryen_js debug "--profiling")
-  target_link_libraries(binaryen_js debug "-sASSERTIONS")
+  target_link_libraries(binaryen_js PRIVATE optimized "-Wno-error=closure")
+  target_link_libraries(binaryen_js PRIVATE optimized "-flto")
+  target_link_libraries(binaryen_js PRIVATE debug "--profiling")
+  target_link_libraries(binaryen_js PRIVATE debug "-sASSERTIONS")
   # Avoid catching exit as that can confuse error reporting in Node,
   # see https://github.com/emscripten-core/emscripten/issues/17228
-  target_link_libraries(binaryen_js "-sNODEJS_CATCH_EXIT=0")
+  target_link_libraries(binaryen_js PRIVATE "-sNODEJS_CATCH_EXIT=0")
   install(TARGETS binaryen_js DESTINATION ${CMAKE_INSTALL_BINDIR})
 endif()
 

check.py

@@ -43,11 +43,11 @@ def run_version_tests():
     print('[ checking --version ... ]\n')
 
     not_executable_suffix = ['.DS_Store', '.txt', '.js', '.ilk', '.pdb', '.dll', '.wasm', '.manifest']
-    not_executable_prefix = ['binaryen-lit', 'binaryen-unittests']
+    executable_prefix = ['wasm']
     bin_files = [os.path.join(shared.options.binaryen_bin, f) for f in os.listdir(shared.options.binaryen_bin)]
     executables = [f for f in bin_files if os.path.isfile(f) and
                    not any(f.endswith(s) for s in not_executable_suffix) and
-                   not any(os.path.basename(f).startswith(s) for s in not_executable_prefix)]
+                   any(os.path.basename(f).startswith(s) for s in executable_prefix)]
     executables = sorted(executables)
     assert len(executables)
 

test/gtest/CMakeLists.txt

@@ -1,4 +1,8 @@
-include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third_party/googletest/googletest/include)
+if(BUILD_FUZZTEST)
+  include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third_party/fuzztest)
+else()
+  include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third_party/googletest/googletest/include)
+endif()
 
 set(unittest_SOURCES
   arena.cpp
@@ -21,10 +25,21 @@ set(unittest_SOURCES
   validator.cpp
 )
 
+if(BUILD_FUZZTEST)
+  set(unittest_SOURCES ${unittest_SOURCES} type-domains.cpp)
+endif()
+
 # suffix_tree.cpp includes LLVM header using std::iterator (deprecated in C++17)
 if (NOT MSVC)
   set_source_files_properties(suffix_tree.cpp PROPERTIES COMPILE_FLAGS -Wno-deprecated-declarations)
 endif()
 
+enable_testing()
+include(GoogleTest)
 binaryen_add_executable(binaryen-unittests "${unittest_SOURCES}")
-target_link_libraries(binaryen-unittests gtest gtest_main)
+if(BUILD_FUZZTEST)
+  link_fuzztest(binaryen-unittests)
+  gtest_discover_tests(binaryen-unittests)
+else()
+  target_link_libraries(binaryen-unittests PRIVATE gtest gtest_main)
+endif()

test/gtest/type-builder.cpp

@@ -5,6 +5,10 @@
 #include "wasm-type.h"
 #include "gtest/gtest.h"
 
+#ifdef FUZZTEST
+#include "type-domains.h"
+#endif
+
 using namespace wasm;
 
 TEST_F(TypeTest, TypeBuilderGrowth) {
@@ -1056,6 +1060,41 @@ TEST_F(TypeTest, TestHeapTypeRelations) {
   }
 }
 
+#ifdef FUZZTEST
+
+void TestHeapTypeRelationsFuzz(std::pair<HeapType, HeapType> pair) {
+  auto [a, b] = pair;
+  auto lub = HeapType::getLeastUpperBound(a, b);
+  auto otherLub = HeapType::getLeastUpperBound(b, a);
+  EXPECT_EQ(lub, otherLub);
+  if (lub) {
+    EXPECT_EQ(a.getTop(), b.getTop());
+    EXPECT_EQ(a.getBottom(), b.getBottom());
+    EXPECT_TRUE(HeapType::isSubType(a, *lub));
+    EXPECT_TRUE(HeapType::isSubType(b, *lub));
+  } else {
+    EXPECT_NE(a.getTop(), b.getTop());
+    EXPECT_NE(a.getBottom(), b.getBottom());
+  }
+  if (a == b) {
+    EXPECT_EQ(lub, a);
+    EXPECT_EQ(lub, b);
+  } else if (lub && *lub == b) {
+    EXPECT_TRUE(HeapType::isSubType(a, b));
+    EXPECT_FALSE(HeapType::isSubType(b, a));
+  } else if (lub && *lub == a) {
+    EXPECT_FALSE(HeapType::isSubType(a, b));
+    EXPECT_TRUE(HeapType::isSubType(b, a));
+  } else if (lub) {
+    EXPECT_FALSE(HeapType::isSubType(a, b));
+    EXPECT_FALSE(HeapType::isSubType(b, a));
+  }
+}
+FUZZ_TEST(TypeFuzzTest, TestHeapTypeRelationsFuzz)
+  .WithDomains(ArbitraryHeapTypePair());
+
+#endif // FUZZTEST
+
 TEST_F(TypeTest, TestSubtypeErrors) {
   Type anyref = Type(HeapType::any, Nullable);
   Type eqref = Type(HeapType::eq, Nullable);