[libc++] Build Google Benchmark directly from Lit

libcxx/test/CMakeLists.txt

@@ -63,7 +63,6 @@ set(SERIALIZED_LIT_PARAMS "# Lit parameters serialized here for llvm-lit to pick
 serialize_lit_string_param(SERIALIZED_LIT_PARAMS compiler "${CMAKE_CXX_COMPILER}")
 
 if (LIBCXX_INCLUDE_BENCHMARKS)
-  add_subdirectory(benchmarks)
   set(_libcxx_benchmark_mode "dry-run")
 else()
   serialize_lit_string_param(SERIALIZED_LIT_PARAMS enable_benchmarks "no")

libcxx/test/configs/cmake-bridge.cfg.in

@@ -33,5 +33,4 @@ config.substitutions.append(('%{target-include-dir}', '@LIBCXX_TESTING_INSTALL_P
 config.substitutions.append(('%{lib-dir}', '@LIBCXX_TESTING_INSTALL_PREFIX@/@LIBCXX_INSTALL_LIBRARY_DIR@'))
 config.substitutions.append(('%{module-dir}', '@LIBCXX_TESTING_INSTALL_PREFIX@/@LIBCXX_INSTALL_MODULES_DIR@'))
 config.substitutions.append(('%{test-tools-dir}', '@LIBCXX_TEST_TOOLS_PATH@'))
-config.substitutions.append(('%{benchmark_flags}', '-I @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/include -L @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/lib -L @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/lib64 -l benchmark'))
 config.substitutions.append(("%{python}", shlex.quote(sys.executable)))

libcxx/utils/libcxx/test/config.py

@@ -52,7 +52,7 @@ def configure(parameters, features, config, lit_config):
                 )
 
     # Print the basic substitutions
-    for sub in ("%{cxx}", "%{flags}", "%{compile_flags}", "%{link_flags}", "%{benchmark_flags}", "%{exec}"):
+    for sub in ("%{cxx}", "%{flags}", "%{compile_flags}", "%{link_flags}", "%{exec}"):
         note("Using {} substitution: '{}'".format(sub, _getSubstitution(sub, config)))
 
     # Print all available features

libcxx/utils/libcxx/test/format.py

@@ -31,7 +31,7 @@ def _getTempPaths(test):
 
 def _checkBaseSubstitutions(substitutions):
     substitutions = [s for (s, _) in substitutions]
-    for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{benchmark_flags}", "%{flags}", "%{exec}"]:
+    for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]:
         assert s in substitutions, "Required substitution {} was not provided".format(s)
 
 def _executeScriptInternal(test, litConfig, commands):
@@ -230,10 +230,6 @@ class CxxStandardLibraryTest(lit.formats.FileBasedTest):
         %{compile_flags}   - Flags to use when compiling a test case
         %{link_flags}      - Flags to use when linking a test case
         %{flags}           - Flags to use either when compiling or linking a test case
-        %{benchmark_flags} - Flags to use when compiling benchmarks. These flags should provide access to
-                             GoogleBenchmark but shouldn't hardcode any optimization level or other settings,
-                             since the benchmarks should be run under the same configuration as the rest of
-                             the test suite.
         %{exec}            - A command to prefix the execution of executables
 
     Note that when building an executable (as opposed to only compiling a source
@@ -350,8 +346,23 @@ def execute(self, test, litConfig):
                         test.getFullName()
                     ),
                 )
+            # TODO: %{libcxx-dir} is not a base substitution
+            # TODO: We're building Google Benchmark for every test, which is super wasteful
+            configure = ['cmake', '-S', '%{libcxx-dir}/../third-party/benchmark', '-B', '%T/gbench-build']
+            configure += ['-D', 'CMAKE_INSTALL_PREFIX=%T/gbench']
+            configure += ['-D', 'CMAKE_CXX_COMPILER=%{cxx}']
+            configure += ['-D', 'CMAKE_BUILD_TYPE=Release']
+            configure += ['-D', 'CMAKE_CXX_FLAGS="%{flags} %{compile_flags} %{link_flags} -Wno-error"']
+            configure += ['-D', 'BENCHMARK_ENABLE_TESTING=OFF']
+            configure += ['-D', 'BENCHMARK_ENABLE_WERROR=OFF']
+            configure += ['-D', 'BENCHMARK_INSTALL_DOCS=OFF']
+            build = ['cmake', '--build', '%T/gbench-build', '--target', 'install']
+
             steps = [
-                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{benchmark_flags} %{link_flags} -o %t.exe",
+                "rm -rf %T/gbench %T/gbench-build",
+                "%dbg(CONFIGURING GBENCH) " + " ".join(configure),
+                "%dbg(BUILDING GBENCH) " + " ".join(build),
+                "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -I %T/gbench/include -L %T/gbench/lib -l benchmark -o %t.exe",
             ]
             if "enable-benchmarks=run" in test.config.available_features:
                 steps += ["%dbg(EXECUTED AS) %{exec} %t.exe --benchmark_out=%T/benchmark-result.json --benchmark_out_format=json"]

libcxxabi/test/configs/cmake-bridge.cfg.in

@@ -34,7 +34,6 @@ config.substitutions.append(('%{include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/i
 config.substitutions.append(('%{cxx-include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_INCLUDE_DIR@'))
 config.substitutions.append(('%{cxx-target-include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_INCLUDE_TARGET_DIR@'))
 config.substitutions.append(('%{lib}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_LIBRARY_DIR@'))
-config.substitutions.append(('%{benchmark_flags}', ''))
 
 if @LIBCXXABI_USE_LLVM_UNWINDER@:
     config.substitutions.append(('%{maybe-include-libunwind}', '-I "@LIBCXXABI_LIBUNWIND_INCLUDES_INTERNAL@"'))

libunwind/test/configs/cmake-bridge.cfg.in

@@ -32,4 +32,3 @@ if not @LIBUNWIND_ENABLE_THREADS@:
 config.substitutions.append(('%{install-prefix}', '@LIBUNWIND_TESTING_INSTALL_PREFIX@'))
 config.substitutions.append(('%{include}', '@LIBUNWIND_TESTING_INSTALL_PREFIX@/include'))
 config.substitutions.append(('%{lib}', '@LIBUNWIND_TESTING_INSTALL_PREFIX@/@LIBUNWIND_INSTALL_LIBRARY_DIR@'))
-config.substitutions.append(('%{benchmark_flags}', ''))

third-party/benchmark/CMakeLists.txt

@@ -197,8 +197,8 @@ else()
     # Disable warning when compiling tests as gtest does not use 'override'.
     add_cxx_compiler_flag(-Wsuggest-override)
   endif()
-  add_cxx_compiler_flag(-pedantic)
-  add_cxx_compiler_flag(-pedantic-errors)
+  # add_cxx_compiler_flag(-pedantic)
+  # add_cxx_compiler_flag(-pedantic-errors)
   add_cxx_compiler_flag(-Wshorten-64-to-32)
   add_cxx_compiler_flag(-fstrict-aliasing)
   # Disable warnings regarding deprecated parts of the library while building