Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/mohsaka/velox into ipprefix…
Browse files Browse the repository at this point in the history
…_final
  • Loading branch information
mohsaka committed Sep 13, 2024
2 parents 0aa11fe + 580c356 commit 481164d
Show file tree
Hide file tree
Showing 491 changed files with 16,310 additions and 5,760 deletions.
1 change: 1 addition & 0 deletions .github/workflows/build-metrics.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ on:
pull_request:
paths:
- ".github/workflows/build-metrics.yml"
- "script/bm-report/**"

workflow_dispatch:
inputs:
Expand Down
19 changes: 12 additions & 7 deletions .github/workflows/macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ jobs:
runs-on: ${{ matrix.os }}
env:
CCACHE_DIR: '${{ github.workspace }}/.ccache'
# The arm runners have only 7GB RAM
BUILD_TYPE: "${{ matrix.os == 'macos-14' && 'Release' || 'Debug' }}"
steps:
- name: Checkout
uses: actions/checkout@v4
Expand All @@ -62,7 +64,8 @@ jobs:
HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "TRUE"
run: |
source scripts/setup-macos.sh
brew install $MACOS_BUILD_DEPS $MACOS_VELOX_DEPS
install_build_prerequisites
install_velox_deps_from_brew
echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV
brew unlink protobuf || echo "protobuf not installed"
Expand All @@ -72,30 +75,32 @@ jobs:
uses: assignUser/stash/restore@v1
with:
path: '${{ env.CCACHE_DIR }}'
key: ccache-macos-${{ matrix.os }}
key: ccache-macos-1-${{ matrix.os }}

- name: Configure Build
env:
folly_SOURCE: BUNDLED
folly_SOURCE: BUNDLED #brew folly does not have int128
fmt_SOURCE: BUNDLED #brew fmt11 is not supported
run: |
ccache -sz -M 5Gi
cmake \
-B _build/debug \
-B _build/$BUILD_TYPE \
-GNinja \
-DTREAT_WARNINGS_AS_ERRORS=1 \
-DENABLE_ALL_WARNINGS=1 \
-DVELOX_ENABLE_PARQUET=ON \
-DCMAKE_BUILD_TYPE=Debug
-DVELOX_MONO_LIBRARY=ON \
-DCMAKE_BUILD_TYPE=$BUILD_TYPE
- name: Build
run: |
cmake --build _build/debug -j $NJOBS
cmake --build _build/$BUILD_TYPE -j $NJOBS
ccache -s
- uses: assignUser/stash/save@v1
with:
path: '${{ env.CCACHE_DIR }}'
key: ccache-macos-${{ matrix.os }}
key: ccache-macos-1-${{ matrix.os }}

- name: Run Tests
if: false
Expand Down
34 changes: 30 additions & 4 deletions .github/workflows/scheduled.yml
Original file line number Diff line number Diff line change
Expand Up @@ -573,21 +573,44 @@ jobs:
path: |
/tmp/spark_fuzzer_repro
join-fuzzer-run:
presto-java-join-fuzzer-run:
name: Join Fuzzer
runs-on: ubuntu-latest
container: ghcr.io/facebookincubator/velox-dev:centos9
container: ghcr.io/facebookincubator/velox-dev:presto-java
needs: compile
timeout-minutes: 120
env:
CCACHE_DIR: "${{ github.workspace }}/.ccache/"
LINUX_DISTRO: "centos"
steps:

- name: Download join fuzzer
uses: actions/download-artifact@v4
with:
name: join

- name: "Checkout Repo"
uses: actions/checkout@v4
with:
path: velox
submodules: 'recursive'
ref: "${{ inputs.ref }}"

- name: Fix git permissions
# Usually actions/checkout does this but as we run in a container
# it doesn't work
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

- name: Run Join Fuzzer
run: |
cd velox
cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
ls -lR $PRESTO_HOME/etc
$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
# Sleep for 60 seconds to allow Presto server to start.
sleep 60
/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;'
cd -
mkdir -p /tmp/join_fuzzer_repro/logs/
chmod -R 777 /tmp/join_fuzzer_repro
chmod +x velox_join_fuzzer_test
Expand All @@ -597,15 +620,18 @@ jobs:
--minloglevel=0 \
--stderrthreshold=2 \
--log_dir=/tmp/join_fuzzer_repro/logs \
--presto_url=http://127.0.0.1:8080 \
--req_timeout_ms=2000 \
&& echo -e "\n\nJoin fuzzer run finished successfully."
- name: Archive aggregate production artifacts
- name: Archive join production artifacts
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: join-fuzzer-failure-artifacts
name: presto-sot-join-fuzzer-failure-artifacts
path: |
/tmp/join_fuzzer_repro
/tmp/server.log
exchange-fuzzer-run:
name: Exchange Fuzzer
Expand Down
27 changes: 15 additions & 12 deletions CMake/FindArrow.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,19 @@ add_library(thrift ALIAS thrift::thrift)

set(Arrow_FOUND true)

add_library(arrow STATIC IMPORTED GLOBAL)
add_library(parquet STATIC IMPORTED GLOBAL)
add_library(arrow_testing STATIC IMPORTED GLOBAL)
# Only add the libraries once.
if(NOT TARGET arrow)
add_library(arrow STATIC IMPORTED GLOBAL)
add_library(parquet STATIC IMPORTED GLOBAL)
add_library(arrow_testing STATIC IMPORTED GLOBAL)

find_path(ARROW_INCLUDE_PATH arrow/api.h)
set_target_properties(
arrow arrow_testing parquet PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
${ARROW_INCLUDE_PATH})
set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ${ARROW_LIB}
INTERFACE_LINK_LIBRARIES thrift)
set_target_properties(parquet PROPERTIES IMPORTED_LOCATION ${PARQUET_LIB})
set_target_properties(arrow_testing PROPERTIES IMPORTED_LOCATION
${ARROW_TESTING_LIB})
find_path(ARROW_INCLUDE_PATH arrow/api.h)
set_target_properties(
arrow arrow_testing parquet PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
${ARROW_INCLUDE_PATH})
set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ${ARROW_LIB}
INTERFACE_LINK_LIBRARIES thrift)
set_target_properties(parquet PROPERTIES IMPORTED_LOCATION ${PARQUET_LIB})
set_target_properties(arrow_testing PROPERTIES IMPORTED_LOCATION
${ARROW_TESTING_LIB})
endif()
40 changes: 40 additions & 0 deletions CMake/Findc-ares.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

find_package(c-ares CONFIG)
if(c-ares_FOUND)
if(TARGET c-ares::cares)
return()
endif()
endif()

find_path(
C_ARES_INCLUDE_DIR
NAMES ares.h
PATH_SUFFIXES c-ares)
find_library(C_ARES_LIBRARY NAMES c-ares)

include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(c-ares DEFAULT_MSG C_ARES_LIBRARY
C_ARES_INCLUDE_DIR)

if(c-ares_FOUND AND NOT TARGET c-ares::cares)
add_library(c-ares::cares UNKNOWN IMPORTED)
set_target_properties(
c-ares::cares
PROPERTIES IMPORTED_LOCATION "${C_ARES_LIBRARY}"
INTERFACE_INCLUDE_DIRECTORIES "${C_ARES_INCLUDE_DIR}")
endif()

mark_as_advanced(C_ARES_INCLUDE_DIR C_ARES_LIBRARY)
1 change: 1 addition & 0 deletions CMake/resolve_dependency_modules/arrow/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ if(VELOX_ENABLE_ARROW)
set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep")
set(ARROW_CMAKE_ARGS
-DARROW_PARQUET=OFF
-DARROW_DEPENDENCY_SOURCE=AUTO
-DARROW_WITH_THRIFT=ON
-DARROW_WITH_LZ4=ON
-DARROW_WITH_SNAPPY=ON
Expand Down
43 changes: 43 additions & 0 deletions CMake/resolve_dependency_modules/c-ares.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
include_guard(GLOBAL)

set(VELOX_CARES_BUILD_VERSION cares-1_13_0)
set(VELOX_CARES_BUILD_SHA256_CHECKSUM
7c48c57706a38691041920e705d2a04426ad9c68d40edd600685323f214b2d57)
string(
CONCAT VELOX_CARES_SOURCE_URL
"https://github.com/c-ares/c-ares/archive/refs/tags/"
"${VELOX_CARES_BUILD_VERSION}.tar.gz")

resolve_dependency_url(CARES)

message(STATUS "Building C-ARES from source")

FetchContent_Declare(
c-ares
URL ${VELOX_CARES_SOURCE_URL}
URL_HASH ${VELOX_CARES_BUILD_SHA256_CHECKSUM}
PATCH_COMMAND
git init && git apply
${CMAKE_CURRENT_LIST_DIR}/c-ares/c-ares-random-file.patch
OVERRIDE_FIND_PACKAGE EXCLUDE_FROM_ALL SYSTEM)

set(CARES_STATIC ON)
set(CARES_INSTALL ON)
set(CARES_SHARED OFF)
FetchContent_MakeAvailable(c-ares)
if(NOT TARGET c-ares::cares)
add_library(c-ares::cares ALIAS c-ares)
endif()
73 changes: 73 additions & 0 deletions CMake/resolve_dependency_modules/c-ares/c-ares-random-file.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 364b6c7c..c96a5b6c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -333,6 +333,8 @@ SET (CMAKE_EXTRA_INCLUDE_FILES)
SET (CMAKE_REQUIRED_DEFINITIONS)
SET (CMAKE_REQUIRED_LIBRARIES)

+find_file(CARES_RANDOM_FILE urandom /dev)
+mark_as_advanced(CARES_RANDOM_FILE)

################################################################################
# recv, recvfrom, send, getnameinfo, gethostname
diff --git a/ares_config.h.cmake b/ares_config.h.cmake
index 0cb2f6ae..b5da36d3 100644
--- a/ares_config.h.cmake
+++ b/ares_config.h.cmake
@@ -338,7 +338,7 @@
#cmakedefine NEED_MEMORY_H

/* a suitable file/device to read random data from */
-#cmakedefine RANDOM_FILE
+#cmakedefine CARES_RANDOM_FILE "@CARES_RANDOM_FILE@"

/* Define to the type qualifier pointed by arg 5 for recvfrom. */
#define RECVFROM_QUAL_ARG5 @RECVFROM_QUAL_ARG5@
diff --git a/ares_init.c b/ares_init.c
index f7b700bf..29bb9784 100644
--- a/ares_init.c
+++ b/ares_init.c
@@ -2376,8 +2376,8 @@ static void randomize_key(unsigned char* key,int key_data_len)
randomized = 1;
}
#else /* !WIN32 */
-#ifdef RANDOM_FILE
- FILE *f = fopen(RANDOM_FILE, "rb");
+#ifdef CARES_RANDOM_FILE
+ FILE *f = fopen(CARES_RANDOM_FILE, "rb");
if(f) {
counter = aresx_uztosi(fread(key, 1, key_data_len, f));
fclose(f);
diff --git a/configure.ac b/configure.ac
index 7e86ad67..ffdcfdf7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -829,22 +829,22 @@ dnl Check for user-specified random device
AC_ARG_WITH(random,
AC_HELP_STRING([--with-random=FILE],
[read randomness from FILE (default=/dev/urandom)]),
- [ RANDOM_FILE="$withval" ],
+ [ CARES_RANDOM_FILE="$withval" ],
[
dnl Check for random device. If we're cross compiling, we can't
dnl check, and it's better to assume it doesn't exist than it is
dnl to fail on AC_CHECK_FILE or later.
if test "$cross_compiling" = "no"; then
- AC_CHECK_FILE("/dev/urandom", [ RANDOM_FILE="/dev/urandom"] )
+ AC_CHECK_FILE("/dev/urandom", [ CARES_RANDOM_FILE="/dev/urandom"] )
else
AC_MSG_WARN([cannot check for /dev/urandom while cross compiling; assuming none])
fi

]
)
-if test -n "$RANDOM_FILE" && test X"$RANDOM_FILE" != Xno ; then
- AC_SUBST(RANDOM_FILE)
- AC_DEFINE_UNQUOTED(RANDOM_FILE, "$RANDOM_FILE",
+if test -n "$CARES_RANDOM_FILE" && test X"$CARES_RANDOM_FILE" != Xno ; then
+ AC_SUBST(CARES_RANDOM_FILE)
+ AC_DEFINE_UNQUOTED(CARES_RANDOM_FILE, "$CARES_RANDOM_FILE",
[a suitable file/device to read random data from])
fi

9 changes: 5 additions & 4 deletions CMake/resolve_dependency_modules/grpc.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,14 @@ resolve_dependency(absl CONFIG REQUIRED)
set(VELOX_GRPC_BUILD_VERSION 1.48.1)
set(VELOX_GRPC_BUILD_SHA256_CHECKSUM
320366665d19027cda87b2368c03939006a37e0388bfd1091c8d2a96fbc93bd8)
string(CONCAT VELOX_GRPC_SOURCE_URL
"https://github.com/grpc/grpc/archive/refs/tags/"
"v${VELOX_GRPC_BUILD_VERSION}.tar.gz")
string(
CONCAT VELOX_GRPC_SOURCE_URL
"https://github.com/grpc/grpc/archive/refs/tags/"
"v${VELOX_GRPC_BUILD_VERSION}.tar.gz")

resolve_dependency_url(GRPC)

message(STATUS "Building Google Cloud CPP from source")
message(STATUS "Building gRPC from source")

FetchContent_Declare(
gRPC
Expand Down
16 changes: 15 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ if(${VELOX_BUILD_TESTING})
set(VELOX_ENABLE_TPCH_CONNECTOR ON)
set(VELOX_ENABLE_SPARK_FUNCTIONS ON)
set(VELOX_ENABLE_EXAMPLES ON)
set(VELOX_ENABLE_PARQUET ON)
endif()

if(${VELOX_ENABLE_BENCHMARKS})
Expand Down Expand Up @@ -344,7 +345,6 @@ if("${ENABLE_ALL_WARNINGS}")
-Wno-unused-parameter \
-Wno-sign-compare \
-Wno-ignored-qualifiers \
-Wnon-virtual-dtor \
${KNOWN_COMPILER_SPECIFIC_WARNINGS}")

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra ${KNOWN_WARNINGS}")
Expand Down Expand Up @@ -461,6 +461,20 @@ add_compile_definitions(FOLLY_HAVE_INT128_T=1)
set_source(folly)
resolve_dependency(folly)

if(${VELOX_BUILD_TESTING})
# Spark qury runner depends on absl, gRPC.
set_source(absl)
resolve_dependency(absl)

# 'gRPC_CARES_PROVIDER' is set as 'package', which means c-ares library needs
# to be installed on the system, instead of being built by gRPC.
set_source(c-ares)
resolve_dependency(c-ares)

set_source(gRPC)
resolve_dependency(gRPC)
endif()

if(VELOX_ENABLE_REMOTE_FUNCTIONS)
# TODO: Move this to use resolve_dependency(). For some reason, FBThrift
# requires clients to explicitly install fizz and wangle.
Expand Down
Loading

0 comments on commit 481164d

Please sign in to comment.