Skip to content

Commit

Permalink
Enable Monolithic build (facebookincubator#9948)
Browse files Browse the repository at this point in the history
Summary:
This PR adds an option that combines all targets into a single `velox` target and build a single unified `libvelox.a`. This is entirely optional and defaults to off. This will make it easier for downstream users to use Velox by just linking to `velox::velox` in their own cmake instead of having to look through our cmake which targets they need. (Once we have a config etc.).

I have yet to test the install functionality (it's strictly a bonus, the monolithic build is the goal for now).

Pull Request resolved: facebookincubator#9948

Reviewed By: kgpai

Differential Revision: D59974408

Pulled By: kevinwilfong

fbshipit-source-id: 67459767d247a1264e1c5465c248586ac047ef4f
  • Loading branch information
assignUser authored and facebook-github-bot committed Jul 23, 2024
1 parent db784f6 commit 9f64e42
Show file tree
Hide file tree
Showing 94 changed files with 781 additions and 356 deletions.
84 changes: 84 additions & 0 deletions .cmake-format.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Currently this config mostly mirrors the default with the addition of custom functions
format:
line_width: 80
tab_size: 2
use_tabchars: false
max_pargs_hwrap: 4
max_subgroups_hwrap: 2
min_prefix_chars: 4
max_prefix_chars: 6
separate_ctrl_name_with_space: false
separate_fn_name_with_space: false
dangle_parens: false
command_case: "canonical"
keyword_case: "unchanged"
always_wrap:
- set_target_properties
- target_sources
- target_link_libraries

parse:
# We define these for our custom
# functions so they get formatted correctly
additional_commands:
velox_add_library:
pargs:
nargs: 1+
flags:
- OBJECT
- STATIC
- SHARED
- INTERFACE
kwargs: {}

velox_base_add_library:
pargs:
nargs: 1+
flags:
- OBJECT
- STATIC
- SHARED
- INTERFACE
kwargs: {}

velox_compile_definitions:
pargs: 1
kwargs:
PRIVATE: '*'
PUBLIC: '*'
INTERFACE: '*'

velox_include_directories:
pargs: '1+'
flags:
- SYSTEM
- BEFORE
- AFTER
kwargs:
PRIVATE: '*'
PUBLIC: '*'
INTERFACE: '*'

velox_link_libraries:
pargs: '1+'
kwargs:
PRIVATE: '*'
PUBLIC: '*'
INTERFACE: '*'

markup:
first_comment_is_literal: true
1 change: 1 addition & 0 deletions .github/workflows/linux-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ jobs:
"-DVELOX_ENABLE_ABFS=ON"
"-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON"
"-DVELOX_ENABLE_GPU=ON"
"-DVELOX_MONO_LIBRARY=ON"
)
make release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}"
Expand Down
6 changes: 4 additions & 2 deletions .github/workflows/preliminary_checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,15 @@ jobs:
}
- { name: "Code Format",
command: "format-fix",
message: "Found format issues",
message: "Found format issues"
}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- run: |
apt -y install python3-pip
pip3 install --break-system-packages pyyaml
- name: Fix git permissions
# Usually actions/checkout does this but as we run in a container
# it doesn't work
Expand Down
132 changes: 132 additions & 0 deletions CMake/VeloxUtils.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
include_guard(GLOBAL)

# TODO use file sets
function(velox_install_library_headers)
# Find any headers and install them relative to the source tree in include.
file(GLOB _hdrs "*.h")
if(NOT "${_hdrs}" STREQUAL "")
cmake_path(
RELATIVE_PATH
CMAKE_CURRENT_SOURCE_DIR
BASE_DIRECTORY
"${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE
_hdr_dir)
install(FILES ${_hdrs} DESTINATION include/${_hdr_dir})
endif()
endfunction()

# Base add velox library call to add a library and install it.
function(velox_base_add_library TARGET)
add_library(${TARGET} ${ARGN})
install(TARGETS ${TARGET} DESTINATION lib/velox)
velox_install_library_headers()
endfunction()

# This is extremely hackish but presents an easy path to installation.
function(velox_add_library TARGET)
set(options OBJECT STATIC SHARED INTERFACE)
set(oneValueArgs)
set(multiValueArgs)
cmake_parse_arguments(
VELOX
"${options}"
"${oneValueArgs}"
"${multiValueArgs}"
${ARGN})

# Remove library type specifiers from ARGN
set(library_type)
if(VELOX_OBJECT)
set(library_type OBJECT)
elseif(VELOX_STATIC)
set(library_type STATIC)
elseif(VELOX_SHARED)
set(library_type SHARED)
elseif(VELOX_INTERFACE)
set(library_type INTERFACE)
endif()

list(REMOVE_ITEM ARGN OBJECT)
list(REMOVE_ITEM ARGN STATIC)
list(REMOVE_ITEM ARGN SHARED)
list(REMOVE_ITEM ARGN INTERFACE)
# Propagate to the underlying add_library and then install the target.
if(VELOX_MONO_LIBRARY)
if(TARGET velox)
# Target already exists, append sources to it.
target_sources(velox PRIVATE ${ARGN})
else()
# Create the target if this is the first invocation.
add_library(velox ${ARGN})
set_target_properties(velox PROPERTIES LIBRARY_OUTPUT_DIRECTORY
${CMAKE_BINARY_DIR}/lib)
set_target_properties(velox PROPERTIES ARCHIVE_OUTPUT_DIRECTORY
${CMAKE_BINARY_DIR}/lib)
install(TARGETS velox DESTINATION lib/velox)
endif()
# create alias for compatability
if(NOT TARGET ${TARGET})
add_library(${TARGET} ALIAS velox)
endif()
else()
# Create a library for each invocation.
velox_base_add_library(${TARGET} ${library_type} ${ARGN})
endif()
velox_install_library_headers()
endfunction()

function(velox_link_libraries TARGET)
# TODO(assignUser): Handle scope keywords (they currently are empty calls ala
# target_link_libraries(target PRIVATE))
if(VELOX_MONO_LIBRARY)
message(DEBUG "${TARGET}: ${ARGN}")
foreach(_lib ${ARGN})
if("${_lib}" MATCHES "^velox_*")
message(DEBUG "\t\tDROP: ${_lib}")
else()
message(DEBUG "\t\tADDING: ${_lib}")
target_link_libraries(velox ${_lib})
endif()
endforeach()
else()
target_link_libraries(${TARGET} ${ARGN})
endif()
endfunction()

function(velox_include_directories TARGET)
if(VELOX_MONO_LIBRARY)
target_include_directories(velox ${ARGN})
else()
target_include_directories(${TARGET} ${ARGN})
endif()
endfunction()

function(velox_compile_definitions TARGET)
if(VELOX_MONO_LIBRARY)
target_compile_definitions(velox ${ARGN})
else()
target_compile_definitions(${TARGET} ${ARGN})
endif()
endfunction()

function(velox_sources TARGET)
if(VELOX_MONO_LIBRARY)
target_sources(velox ${ARGN})
else()
target_sources(${TARGET} ${ARGN})
endif()
endfunction()
8 changes: 5 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ list(PREPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/CMake"

# Include our ThirdPartyToolchain dependencies macros
include(ResolveDependency)
include(VeloxUtils)

set_with_default(VELOX_DEPENDENCY_SOURCE_DEFAULT VELOX_DEPENDENCY_SOURCE AUTO)
message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
Expand All @@ -68,6 +69,7 @@ option(
"Build a minimal set of components, including DWIO (file format readers/writers).
This will override other build options."
OFF)
option(VELOX_MONO_LIBRARY "Build single unified library." OFF)

# option() always creates a BOOL variable so we have to use a normal cache
# variable with STRING type for this option.
Expand All @@ -94,9 +96,9 @@ option(VELOX_ENABLE_TPCH_CONNECTOR "Build TPC-H connector." ON)
option(VELOX_ENABLE_PRESTO_FUNCTIONS "Build Presto SQL functions." ON)
option(VELOX_ENABLE_SPARK_FUNCTIONS "Build Spark SQL functions." ON)
option(VELOX_ENABLE_EXPRESSION "Build expression." ON)
option(VELOX_ENABLE_EXAMPLES
"Build examples. This will enable VELOX_ENABLE_EXPRESSION automatically."
OFF)
option(
VELOX_ENABLE_EXAMPLES
"Build examples. This will enable VELOX_ENABLE_EXPRESSION automatically." OFF)
option(VELOX_ENABLE_SUBSTRAIT "Build Substrait-to-Velox converter." OFF)
option(VELOX_ENABLE_BENCHMARKS "Enable Velox top level benchmarks." OFF)
option(VELOX_ENABLE_BENCHMARKS_BASIC "Enable Velox basic benchmarks." OFF)
Expand Down
9 changes: 9 additions & 0 deletions scripts/check.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,15 @@ def fix(self, commit):


class CMakeFormatter(str):
def __init__(self, commit) -> None:
super().__init__()
try:
import yaml
except ModuleNotFoundError:
# We need pyyaml so cmake-format can read '.cmake-format.yml'
# otherwise it will run with default
raise SystemExit("Please install 'pyyaml' for the CMake formatter.")

def diff(self, commit):
return get_diff(
self, util.run(f"cmake-format --first-comment-is-literal True {self}")[1]
Expand Down
2 changes: 1 addition & 1 deletion scripts/setup-check.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ set -x
export DEBIAN_FRONTEND=noninteractive
apt update
apt install --no-install-recommends -y clang-format-18 python3-pip git make ssh
pip3 install --break-system-packages cmake==3.28.3 cmake_format black regex
pip3 install --break-system-packages cmake==3.28.3 cmake_format black pyyaml regex
pip3 cache purge
apt purge --auto-remove -y python3-pip
update-alternatives --install /usr/bin/clang-format clang-format "$(command -v clang-format-18)" 18
Expand Down
4 changes: 2 additions & 2 deletions velox/buffer/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

add_library(velox_buffer StringViewBufferHolder.cpp)
velox_add_library(velox_buffer StringViewBufferHolder.cpp)

target_link_libraries(velox_buffer velox_memory velox_common_base Folly::folly)
velox_link_libraries(velox_buffer velox_memory velox_common_base Folly::folly)

if(${VELOX_BUILD_TESTING})
add_subdirectory(tests)
Expand Down
26 changes: 16 additions & 10 deletions velox/common/base/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.

add_library(velox_exception Exceptions.cpp VeloxException.cpp Exceptions.h)
target_link_libraries(
velox_exception PUBLIC velox_flag_definitions velox_process Folly::folly
fmt::fmt gflags::gflags glog::glog)
velox_add_library(velox_exception Exceptions.cpp VeloxException.cpp
Exceptions.h)
velox_link_libraries(
velox_exception
PUBLIC velox_flag_definitions
velox_process
Folly::folly
fmt::fmt
gflags::gflags
glog::glog)

add_library(
velox_add_library(
velox_common_base
BitUtil.cpp
Counters.cpp
Expand All @@ -32,7 +38,7 @@ add_library(
StatsReporter.cpp
SuccinctPrinter.cpp)

target_link_libraries(
velox_link_libraries(
velox_common_base
PUBLIC velox_exception Folly::folly fmt::fmt xsimd
PRIVATE velox_common_compression velox_process velox_test_util glog::glog)
Expand All @@ -45,8 +51,8 @@ if(${VELOX_ENABLE_BENCHMARKS})
add_subdirectory(benchmarks)
endif()

add_library(velox_id_map BigintIdMap.cpp)
target_link_libraries(
velox_add_library(velox_id_map BigintIdMap.cpp)
velox_link_libraries(
velox_id_map
velox_memory
velox_flag_definitions
Expand All @@ -56,8 +62,8 @@ target_link_libraries(
fmt::fmt
gflags::gflags)

add_library(velox_status Status.cpp)
target_link_libraries(
velox_add_library(velox_status Status.cpp)
velox_link_libraries(
velox_status
PUBLIC fmt::fmt Folly::folly
PRIVATE glog::glog)
4 changes: 2 additions & 2 deletions velox/common/caching/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

add_library(
velox_add_library(
velox_caching
AsyncDataCache.cpp
CacheTTLController.cpp
Expand All @@ -22,7 +22,7 @@ add_library(
SsdFile.cpp
SsdFileTracker.cpp
StringIdMap.cpp)
target_link_libraries(
velox_link_libraries(
velox_caching
PUBLIC velox_common_base
velox_exception
Expand Down
4 changes: 2 additions & 2 deletions velox/common/compression/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ if(${VELOX_BUILD_TESTING})
add_subdirectory(tests)
endif()

add_library(velox_common_compression Compression.cpp LzoDecompressor.cpp)
target_link_libraries(
velox_add_library(velox_common_compression Compression.cpp LzoDecompressor.cpp)
velox_link_libraries(
velox_common_compression
PUBLIC Folly::folly
PRIVATE velox_exception)
Loading

0 comments on commit 9f64e42

Please sign in to comment.