From ad1a55e5a8a97aaed8ef288c33721df100e0fa37 Mon Sep 17 00:00:00 2001 From: acezen Date: Mon, 28 Aug 2023 18:06:13 +0800 Subject: [PATCH 1/9] Support build GraphAr with system installed arrow Signed-off-by: acezen Update Update Fix update Update update Update --- .github/workflows/ci.yml | 62 +++++++++++++++------ cpp/CMakeLists.txt | 102 ++++++++++++++++++++++++++--------- cpp/README.md | 12 +++++ cpp/cmake/apache-arrow.cmake | 2 +- cpp/src/filesystem.cc | 4 +- cpp/test/test_info.cc | 3 ++ 6 files changed, 142 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f582db8b..43f97c2e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,30 +29,22 @@ concurrency: cancel-in-progress: true jobs: - GraphAr-on-ubuntu: - runs-on: ubuntu-20.04 + GraphAr-ubuntu-arrow-installed: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: submodules: true - - name: Cache for ccache - uses: actions/cache@v3 - with: - path: ~/.ccache - key: ${{ matrix.os }}-build-ccache-${{ hashFiles('**/git-modules.txt') }} - restore-keys: | - ${{ matrix.os }}-build-ccache- - - name: Install dependencies run: | # install the latest arrow deb to test arrow wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \ -P /tmp/ - sudo apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb + sudo apt-get install -y /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb sudo apt-get update -y - sudo apt-get install -y libarrow-dev + sudo apt install -y libarrow-dev libarrow-dataset-dev libarrow-acero-dev libparquet-dev sudo apt-get install -y libboost-graph-dev ccache libcurl4-openssl-dev - name: CMake @@ -111,6 +103,46 @@ jobs: popd + - name: Build GraphAr + run: | + pushd build + make -j$(nproc) + popd + + - name: Test + run: | + cd build + export GAR_TEST_DATA=$PWD/../testing/ + make test + + GraphAr-ubuntu-arrow-from-source: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Cache for ccache + uses: actions/cache@v3 + with: + path: ~/.ccache + key: ${{ matrix.os }}-build-ccache-${{ hashFiles('**/git-modules.txt') }} + restore-keys: | + ${{ matrix.os }}-build-ccache- + + - name: Install dependencies + run: | + + sudo apt-get update -y + sudo apt-get install -y libboost-graph-dev ccache libcurl4-openssl-dev + + - name: CMake + run: | + mkdir build + pushd build + cmake ../cpp -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DBUILD_EXAMPLES=ON -DBUILD_ARROW_FROM_SOURCE=ON + popd + - name: Build GraphAr run: | pushd build @@ -124,8 +156,8 @@ jobs: export GAR_TEST_DATA=$PWD/../testing/ make test - GraphAr-on-centos8: - runs-on: ubuntu-22.04 + GraphAr-centos8-arrow-from-source: + runs-on: ubuntu-latest container: image: centos:latest steps: @@ -145,6 +177,6 @@ jobs: run: | mkdir build pushd build - cmake ../cpp + cmake ../cpp -DBUILD_ARROW_FROM_SOURCE=ON make -j$(nproc) popd diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 37e0e2835..cb2f1b357 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -26,6 +26,7 @@ project(graph-archive LANGUAGES C CXX VERSION ${GAR_VERSION}) option(NAMESPACE "User specific namespace, default if GraphArchive" OFF) option(BUILD_TESTS "Build unit tests" OFF) option(BUILD_EXAMPLES "Build examples" OFF) +option(BUILD_ARROW_FROM_SOURCE "Build Arrow from source (ON) or use system-installed Arrow (OFF)" OFF) if (NAMESPACE) add_definitions(-DGAR_NAMESPACE=${NAMESPACE}) @@ -159,8 +160,16 @@ if(OPENSSL_FOUND) endif() endif() -include(apache-arrow) -build_arrow() +if(BUILD_ARROW_FROM_SOURCE) + include(apache-arrow) + build_arrow() +else() + find_package(Arrow REQUIRED) + find_package(ArrowDataset REQUIRED) + find_package(ArrowAcero REQUIRED) + find_package(Parquet REQUIRED) +endif() + macro(get_target_location var target) if(TARGET ${target}) @@ -185,21 +194,37 @@ macro(build_gar) $ $ ) - target_include_directories(gar SYSTEM BEFORE PRIVATE ${GAR_ARROW_INCLUDE_DIR}) + if(BUILD_ARROW_FROM_SOURCE) + target_include_directories(gar SYSTEM BEFORE PRIVATE ${GAR_ARROW_INCLUDE_DIR}) + endif() target_link_libraries(gar PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) if(APPLE) - target_link_libraries(gar PRIVATE -Wl,-force_load gar_arrow_static - "${GAR_PARQUET_STATIC_LIB}" - "${GAR_DATASET_STATIC_LIB}" - "${GAR_ACERO_STATIC_LIB}" - "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") + if(BUILD_ARROW_FROM_SOURCE) + target_link_libraries(gar PRIVATE -Wl,-force_load gar_arrow_static + "${GAR_PARQUET_STATIC_LIB}" + "${GAR_DATASET_STATIC_LIB}" + "${GAR_ACERO_STATIC_LIB}" + "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") + else() + target_link_libraries(gar PRIVATE -Wl,-force_load Arrow::arrow_static + Parquet::parquet_static + ArrowDataset::arrow_dataset_static + ArrowAcero::arrow_acero_static) + endif() else() - target_link_libraries(gar PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive gar_arrow_static - "${GAR_PARQUET_STATIC_LIB}" - "${GAR_DATASET_STATIC_LIB}" - "${GAR_ARROW_ACERO_STATIC_LIB}" - "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}" -Wl,--no-whole-archive) + if(BUILD_ARROW_FROM_SOURCE) + target_link_libraries(gar PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive gar_arrow_static + "${GAR_PARQUET_STATIC_LIB}" + "${GAR_DATASET_STATIC_LIB}" + "${GAR_ARROW_ACERO_STATIC_LIB}" + "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}" -Wl,--no-whole-archive) + else() + target_link_libraries(gar PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive Arrow::arrow_static + Parquet::parquet_static + ArrowDataset::arrow_dataset_static + ArrowAcero::arrow_acero_static -Wl,--no-whole-archive) + endif() endif() # if OpenSSL library exists, link the OpenSSL library. @@ -231,16 +256,28 @@ if (BUILD_EXAMPLES) add_executable(${E_NAME} examples/${E_NAME}.cc) target_include_directories(${E_NAME} PRIVATE examples ${PROJECT_SOURCE_DIR}/include $) target_include_directories(${E_NAME} SYSTEM PRIVATE ${Boost_INCLUDE_DIRS}) - target_include_directories(${E_NAME} SYSTEM BEFORE PRIVATE ${GAR_ARROW_INCLUDE_DIR}) + if(BUILD_ARROW_FROM_SOURCE) + target_include_directories(${E_NAME} SYSTEM BEFORE PRIVATE ${GAR_ARROW_INCLUDE_DIR}) + endif() target_link_libraries(${E_NAME} PRIVATE gar ${Boost_LIBRARIES} Threads::Threads ${CMAKE_DL_LIBS}) if(APPLE) - target_link_libraries(${E_NAME} PRIVATE -Wl,-force_load gar_arrow_static - "${GAR_PARQUET_STATIC_LIB}" - "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") + if(BUILD_ARROW_FROM_SOURCE) + target_link_libraries(${E_NAME} PRIVATE gar_arrow_static + "${GAR_PARQUET_STATIC_LIB}" + "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") + else() + target_link_libraries(${E_NAME} PRIVATE Arrow::arrow_static + Parquet::parquet_static) + endif() else() - target_link_libraries(${E_NAME} PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive gar_arrow_static - "${GAR_PARQUET_STATIC_LIB}" - "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}" -Wl,--no-whole-archive) + if(BUILD_ARROW_FROM_SOURCE) + target_link_libraries(${E_NAME} PRIVATE gar_arrow_static + "${GAR_PARQUET_STATIC_LIB}" + "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") + else() + target_link_libraries(${E_NAME} PRIVATE Arrow::arrow_static + Parquet::parquet_static) + endif() endif() # if OpenSSL library exists, link the OpenSSL library. @@ -300,15 +337,28 @@ if (BUILD_TESTS) cmake_parse_arguments(add_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_executable(${target} ${add_test_SRCS}) target_compile_features(${target} PRIVATE cxx_std_17) + if(BUILD_ARROW_FROM_SOURCE) + target_include_directories(${target} SYSTEM BEFORE PRIVATE ${GAR_ARROW_INCLUDE_DIR}) + endif() target_link_libraries(${target} PRIVATE Catch2::Catch2 gar Threads::Threads ${CMAKE_DL_LIBS}) if(APPLE) - target_link_libraries(${target} PRIVATE -Wl,-force_load gar_arrow_static - "${GAR_PARQUET_STATIC_LIB}" - "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") + if(BUILD_ARROW_FROM_SOURCE) + target_link_libraries(${target} PRIVATE gar_arrow_static + "${GAR_PARQUET_STATIC_LIB}" + "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") + else() + target_link_libraries(${target} Arrow::arrow_static + Parquet::parquet_static) + endif() else() - target_link_libraries(${target} PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive gar_arrow_static - "${GAR_PARQUET_STATIC_LIB}" - "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}" -Wl,--no-whole-archive) + if(BUILD_ARROW_FROM_SOURCE) + target_link_libraries(${target} PRIVATE gar_arrow_static + "${GAR_PARQUET_STATIC_LIB}" + "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") + else() + target_link_libraries(${target} PRIVATE Arrow::arrow_static + Parquet::parquet_static) + endif() endif() target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR}/include $) target_include_directories(${target} SYSTEM BEFORE PRIVATE ${GAR_ARROW_INCLUDE_DIR}) diff --git a/cpp/README.md b/cpp/README.md index ce20ee491..d732a2b5a 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -25,6 +25,7 @@ Building requires: - CMake 3.5 or higher - On Linux and macOS, ``make`` build utilities - curl-devel with SSL (Linux) or curl (macOS), for s3 filesystem support +- Apache Arrow C++ (>= 12.0.0, requires `arrow-dev`, `arrow-dataset`, `arrow-acero` and `parquet` modules) for Arrow filesystem support and can use `BUILD_ARROW_FROM_SOURCE` option to build with GraphAr automatically. You can refer to [Apache Arrow Installation](https://arrow.apache.org/install/) to install directly too. Dependencies for optional features: @@ -68,6 +69,17 @@ setting `NAMESPACE` option with cmake: $ make -j8 # if you have 8 CPU cores, otherwise adjust, use -j`nproc` for all cores ``` +Build the Apache Arrow dependency from source: + +By default, GraphAr try to find Apache arrow in the system. This can be configured to build arrow dependency automatically from source: + +```bash + $ mkdir build + $ cd build + $ cmake -DBUILD_ARROW_FROM_SOURCE=ON .. + $ make -j8 +``` + Debug build with unit tests: ```bash diff --git a/cpp/cmake/apache-arrow.cmake b/cpp/cmake/apache-arrow.cmake index 4a37486c6..9e8f60001 100644 --- a/cpp/cmake/apache-arrow.cmake +++ b/cpp/cmake/apache-arrow.cmake @@ -90,7 +90,7 @@ function(build_arrow) find_package(Threads) find_package(Arrow QUIET) - set(ARROW_VERSION_TO_BUILD "10.0.1" CACHE INTERNAL "arrow version") + set(ARROW_VERSION_TO_BUILD "12.0.0" CACHE INTERNAL "arrow version") if (Arrow_FOUND) # arrow is installed, build the same version as the installed one message(STATUS "Found Arrow installed, align to version: ${Arrow_VERSION}") set(ARROW_VERSION_TO_BUILD "${Arrow_VERSION}" CACHE INTERNAL "arrow version") diff --git a/cpp/src/filesystem.cc b/cpp/src/filesystem.cc index f4f7c3eb2..2579ba714 100644 --- a/cpp/src/filesystem.cc +++ b/cpp/src/filesystem.cc @@ -267,11 +267,13 @@ Result FileSystem::GetFileNumOfDir(const std::string& dir_path, Result> FileSystemFromUriOrPath( const std::string& uri_string, std::string* out_path) { - if (arrow::fs::internal::DetectAbsolutePath(uri_string)) { + if (uri_string.length() >= 1 && uri_string[0] == '/') { // if the uri_string is an absolute path, we need to create a local file GAR_RETURN_ON_ARROW_ERROR_AND_ASSIGN( auto arrow_fs, arrow::fs::FileSystemFromUriOrPath(uri_string, out_path)); + // arrow would delete the last slash, so use uri string + *out_path = uri_string; return std::make_shared(arrow_fs); } diff --git a/cpp/test/test_info.cc b/cpp/test/test_info.cc index 941e34714..bf90c5630 100644 --- a/cpp/test/test_info.cc +++ b/cpp/test/test_info.cc @@ -368,6 +368,8 @@ TEST_CASE("test_graph_info_load_from_file") { REQUIRE(edge_infos.size() == 1); } +// ISSUE-187 +#if defined(ARROW_VERSION) && ARROW_VERSION < 12000000 TEST_CASE("test_graph_info_load_from_s3") { std::string path = "s3://graphar/ldbc/ldbc.graph.yml" @@ -381,3 +383,4 @@ TEST_CASE("test_graph_info_load_from_s3") { REQUIRE(vertex_infos.size() == 8); REQUIRE(edge_infos.size() == 23); } +#endif From 87dfe6c179338ace6ebf5bde54a882a17528da7c Mon Sep 17 00:00:00 2001 From: acezen Date: Tue, 29 Aug 2023 11:00:52 +0800 Subject: [PATCH 2/9] Fix --- cpp/CMakeLists.txt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cb2f1b357..00396da4c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -198,6 +198,7 @@ macro(build_gar) target_include_directories(gar SYSTEM BEFORE PRIVATE ${GAR_ARROW_INCLUDE_DIR}) endif() target_link_libraries(gar PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) + message(STATUS "LIBS: ${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB} ${GAR_ARROW_ACERO_STATIC_LIB} ${GAR_DATASET_STATIC_LIB} ${GAR_PARQUET_STATIC_LIB} gar_arrow_static") if(APPLE) if(BUILD_ARROW_FROM_SOURCE) @@ -262,7 +263,7 @@ if (BUILD_EXAMPLES) target_link_libraries(${E_NAME} PRIVATE gar ${Boost_LIBRARIES} Threads::Threads ${CMAKE_DL_LIBS}) if(APPLE) if(BUILD_ARROW_FROM_SOURCE) - target_link_libraries(${E_NAME} PRIVATE gar_arrow_static + target_link_libraries(${E_NAME} PRIVATE -Wl,-force_load gar_arrow_static "${GAR_PARQUET_STATIC_LIB}" "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") else() @@ -271,9 +272,9 @@ if (BUILD_EXAMPLES) endif() else() if(BUILD_ARROW_FROM_SOURCE) - target_link_libraries(${E_NAME} PRIVATE gar_arrow_static + target_link_libraries(${E_NAME} PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive gar_arrow_static "${GAR_PARQUET_STATIC_LIB}" - "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") + "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}" -Wl,--no-whole-archive) else() target_link_libraries(${E_NAME} PRIVATE Arrow::arrow_static Parquet::parquet_static) @@ -343,7 +344,7 @@ if (BUILD_TESTS) target_link_libraries(${target} PRIVATE Catch2::Catch2 gar Threads::Threads ${CMAKE_DL_LIBS}) if(APPLE) if(BUILD_ARROW_FROM_SOURCE) - target_link_libraries(${target} PRIVATE gar_arrow_static + target_link_libraries(${target} PRIVATE -Wl,-force_load gar_arrow_static "${GAR_PARQUET_STATIC_LIB}" "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") else() @@ -352,9 +353,9 @@ if (BUILD_TESTS) endif() else() if(BUILD_ARROW_FROM_SOURCE) - target_link_libraries(${target} PRIVATE gar_arrow_static + target_link_libraries(${target} PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive gar_arrow_static "${GAR_PARQUET_STATIC_LIB}" - "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") + "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}" -Wl,--no-whole-archive) else() target_link_libraries(${target} PRIVATE Arrow::arrow_static Parquet::parquet_static) From 21fd30ac9012fd157f9de0a88ce8980a4c4db593 Mon Sep 17 00:00:00 2001 From: acezen Date: Tue, 29 Aug 2023 11:29:56 +0800 Subject: [PATCH 3/9] Update java ci --- .github/workflows/java.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index f13197dd1..c47435f7f 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -24,25 +24,21 @@ concurrency: jobs: GraphAr-java: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: submodules: true - # install GrahpAr C++ library first - - name: Cache for ccache - uses: actions/cache@v3 - with: - path: ~/.ccache - key: ${{ matrix.os }}-build-ccache-${{ hashFiles('**/git-modules.txt') }} - restore-keys: | - ${{ matrix.os }}-build-ccache- - - name: Install dependencies run: | + # install the latest arrow deb to test arrow + wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \ + -P /tmp/ + sudo apt-get install -y /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb sudo apt-get update -y - sudo apt-get install ccache libcurl4-openssl-dev -y + sudo apt install -y libarrow-dev libarrow-dataset-dev libarrow-acero-dev libparquet-dev + sudo apt-get install libcurl4-openssl-dev -y sudo apt-get install llvm-11 clang-11 lld-11 libclang-11-dev libz-dev -y - name: Build and Install cpp From 421627c999cb0f087f753bea5af94b8b5c9ccb0f Mon Sep 17 00:00:00 2001 From: acezen Date: Tue, 29 Aug 2023 11:53:14 +0800 Subject: [PATCH 4/9] Fix curl link --- cpp/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 00396da4c..c7f66f54c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -198,7 +198,6 @@ macro(build_gar) target_include_directories(gar SYSTEM BEFORE PRIVATE ${GAR_ARROW_INCLUDE_DIR}) endif() target_link_libraries(gar PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) - message(STATUS "LIBS: ${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB} ${GAR_ARROW_ACERO_STATIC_LIB} ${GAR_DATASET_STATIC_LIB} ${GAR_PARQUET_STATIC_LIB} gar_arrow_static") if(APPLE) if(BUILD_ARROW_FROM_SOURCE) @@ -234,7 +233,7 @@ macro(build_gar) target_link_libraries(gar PRIVATE OpenSSL::SSL) endif() if (CURL_FOUND) - target_link_libraries(gar PRIVATE CURL::libcurl) + target_link_libraries(gar PRIVATE ${CURL_LIBRARIES}) endif() if (APPLE) target_link_libraries(gar "-framework CoreFoundation") From 8b4de85432420eb4bb6d9dc6523130a7c09688ff Mon Sep 17 00:00:00 2001 From: acezen Date: Tue, 29 Aug 2023 17:11:53 +0800 Subject: [PATCH 5/9] Move build arrow from source ci to nightly --- .github/workflows/ci-nightly.yml | 50 ++++++++++++++++++++++ .github/workflows/ci.yml | 71 ++++++++++---------------------- 2 files changed, 71 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/ci-nightly.yml diff --git a/.github/workflows/ci-nightly.yml b/.github/workflows/ci-nightly.yml new file mode 100644 index 000000000..a685c9e5e --- /dev/null +++ b/.github/workflows/ci-nightly.yml @@ -0,0 +1,50 @@ +name: GraphAr C++ CI Nightly + +on: + schedule: + # The notifications for scheduled workflows are sent to the user who + # last modified the cron syntax in the workflow file. + # Trigger the workflow at 03:00(CST) every day. + - cron: '00 19 * * *' +jobs: + GraphAr-ubuntu-arrow-from-source: + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Cache for ccache + uses: actions/cache@v3 + with: + path: ~/.ccache + key: ${{ matrix.os }}-build-ccache-${{ hashFiles('**/git-modules.txt') }} + restore-keys: | + ${{ matrix.os }}-build-ccache- + + - name: Install dependencies + run: | + + sudo apt-get update -y + sudo apt-get install -y libboost-graph-dev ccache libcurl4-openssl-dev + + - name: CMake + run: | + mkdir build + pushd build + cmake ../cpp -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DBUILD_EXAMPLES=ON -DBUILD_ARROW_FROM_SOURCE=ON + popd + + - name: Build GraphAr + run: | + pushd build + make -j$(nproc) + make gar-ccache-stats + popd + + - name: Test + run: | + cd build + export GAR_TEST_DATA=$PWD/../testing/ + make test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43f97c2e2..e730fecac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,68 +115,39 @@ jobs: export GAR_TEST_DATA=$PWD/../testing/ make test - GraphAr-ubuntu-arrow-from-source: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Cache for ccache - uses: actions/cache@v3 - with: - path: ~/.ccache - key: ${{ matrix.os }}-build-ccache-${{ hashFiles('**/git-modules.txt') }} - restore-keys: | - ${{ matrix.os }}-build-ccache- - - - name: Install dependencies - run: | - - sudo apt-get update -y - sudo apt-get install -y libboost-graph-dev ccache libcurl4-openssl-dev - - - name: CMake - run: | - mkdir build - pushd build - cmake ../cpp -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DBUILD_EXAMPLES=ON -DBUILD_ARROW_FROM_SOURCE=ON - popd - - - name: Build GraphAr - run: | - pushd build - make -j$(nproc) - make gar-ccache-stats - popd - - - name: Test - run: | - cd build - export GAR_TEST_DATA=$PWD/../testing/ - make test - GraphAr-centos8-arrow-from-source: runs-on: ubuntu-latest container: - image: centos:latest + image: centos:7 steps: - uses: actions/checkout@v3 - name: Install dependencies run: | - pushd /etc/yum.repos.d/ - sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - popd - yum update -y - dnf groupinstall -y "Development Tools" - yum install -y boost-devel libcurl-devel openssl-devel cmake + # install gcc and g++ 8 + yum install -y centos-release-scl + yum install -y devtoolset-8-gcc* + scl enable devtoolset-8 bash + + # install cmake + yum install -y wget + wget https://cmake.org/files/v3.12/cmake-3.12.3.tar.gz -P /tmp/ && \ + tar -zxvf /tmp/cmake-3.12.3.tar.gz -C /tmp/ && \ + cd /tmp/cmake-3.12.3 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && \ + make install + echo "cmake version: $(cmake --version)" + + #install arrow + yum install -y epel-release || yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1).noarch.rpm + yum install -y https://apache.jfrog.io/artifactory/arrow/centos/$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1)/apache-arrow-release-latest.rpm + yum install -y --enablerepo=epel arrow-devel arrow-dataset-devel arrow-acero-devel parquet-devel - name: Build GraphAr run: | mkdir build pushd build - cmake ../cpp -DBUILD_ARROW_FROM_SOURCE=ON + cmake ../cpp -DBUILD_TESTS=ON make -j$(nproc) popd From 244e4a32605b92e49a07a83616cca10315935afa Mon Sep 17 00:00:00 2001 From: acezen Date: Tue, 29 Aug 2023 17:19:40 +0800 Subject: [PATCH 6/9] update --- .github/workflows/ci.yml | 4 ++-- cpp/README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e730fecac..976c57dd6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,7 +115,7 @@ jobs: export GAR_TEST_DATA=$PWD/../testing/ make test - GraphAr-centos8-arrow-from-source: + GraphAr-centos8-arrow-installed: runs-on: ubuntu-latest container: image: centos:7 @@ -126,7 +126,7 @@ jobs: run: | # install gcc and g++ 8 yum install -y centos-release-scl - yum install -y devtoolset-8-gcc* + yum install -y devtoolset-8 scl enable devtoolset-8 bash # install cmake diff --git a/cpp/README.md b/cpp/README.md index d732a2b5a..5d832d71a 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -25,7 +25,7 @@ Building requires: - CMake 3.5 or higher - On Linux and macOS, ``make`` build utilities - curl-devel with SSL (Linux) or curl (macOS), for s3 filesystem support -- Apache Arrow C++ (>= 12.0.0, requires `arrow-dev`, `arrow-dataset`, `arrow-acero` and `parquet` modules) for Arrow filesystem support and can use `BUILD_ARROW_FROM_SOURCE` option to build with GraphAr automatically. You can refer to [Apache Arrow Installation](https://arrow.apache.org/install/) to install directly too. +- Apache Arrow C++ (>= 12.0.0, requires `arrow-dev`, `arrow-dataset`, `arrow-acero` and `parquet` modules) for Arrow filesystem support and can use `BUILD_ARROW_FROM_SOURCE` option to build with GraphAr automatically. You can refer to [Apache Arrow Installation](https://arrow.apache.org/install/) to install Arrow directly too. Dependencies for optional features: @@ -71,7 +71,7 @@ setting `NAMESPACE` option with cmake: Build the Apache Arrow dependency from source: -By default, GraphAr try to find Apache arrow in the system. This can be configured to build arrow dependency automatically from source: +By default, GraphAr try to find Apache Arrow in the system. This can be configured to build Arrow dependency automatically from source: ```bash $ mkdir build From 9cf6f254c6f8aa90a03d68fd79a3afe47d3f64cc Mon Sep 17 00:00:00 2001 From: acezen Date: Tue, 29 Aug 2023 17:27:32 +0800 Subject: [PATCH 7/9] Test --- .github/workflows/ci.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 976c57dd6..76abaea75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,6 +117,9 @@ jobs: GraphAr-centos8-arrow-installed: runs-on: ubuntu-latest + defaults: + run: + shell: yum install -y centos-release-scl devtoolset-8 && scl enable devtoolset-8 -- bash --noprofile --norc -eo pipefail {0} container: image: centos:7 steps: @@ -125,18 +128,19 @@ jobs: - name: Install dependencies run: | # install gcc and g++ 8 - yum install -y centos-release-scl - yum install -y devtoolset-8 - scl enable devtoolset-8 bash + # yum install -y centos-release-scl + # yum install -y devtoolset-8 + # scl enable devtoolset-8 bash # install cmake yum install -y wget wget https://cmake.org/files/v3.12/cmake-3.12.3.tar.gz -P /tmp/ && \ - tar -zxvf /tmp/cmake-3.12.3.tar.gz -C /tmp/ && \ - cd /tmp/cmake-3.12.3 && \ + tar -zxf /tmp/cmake-3.12.3.tar.gz -C /tmp/ && \ + pushd /tmp/cmake-3.12.3 && \ ./bootstrap --prefix=/usr/local && \ make -j$(nproc) && \ - make install + make install && \ + popd echo "cmake version: $(cmake --version)" #install arrow From 0f7093740437031c5fc0ce3e8e0680cb088d4e13 Mon Sep 17 00:00:00 2001 From: acezen Date: Tue, 29 Aug 2023 17:40:14 +0800 Subject: [PATCH 8/9] setup shell --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76abaea75..c88b78321 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,21 +117,20 @@ jobs: GraphAr-centos8-arrow-installed: runs-on: ubuntu-latest - defaults: - run: - shell: yum install -y centos-release-scl devtoolset-8 && scl enable devtoolset-8 -- bash --noprofile --norc -eo pipefail {0} container: image: centos:7 steps: - uses: actions/checkout@v3 - - name: Install dependencies + - name: Set up devtoolset-8 run: | # install gcc and g++ 8 - # yum install -y centos-release-scl - # yum install -y devtoolset-8 - # scl enable devtoolset-8 bash + yum install -y centos-release-scl + yum install -y devtoolset-8 + - name: Install dependencies + shell: scl enable devtoolset-8 -- bash --noprofile --norc -eo pipefail {0} + run: | # install cmake yum install -y wget wget https://cmake.org/files/v3.12/cmake-3.12.3.tar.gz -P /tmp/ && \ @@ -149,6 +148,7 @@ jobs: yum install -y --enablerepo=epel arrow-devel arrow-dataset-devel arrow-acero-devel parquet-devel - name: Build GraphAr + shell: scl enable devtoolset-8 -- bash --noprofile --norc -eo pipefail {0} run: | mkdir build pushd build From 0696cde7e37c9306f5e311d901f079918394e583 Mon Sep 17 00:00:00 2001 From: acezen Date: Tue, 29 Aug 2023 18:01:35 +0800 Subject: [PATCH 9/9] Update --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c88b78321..88c25e55f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,6 +152,6 @@ jobs: run: | mkdir build pushd build - cmake ../cpp -DBUILD_TESTS=ON + cmake ../cpp make -j$(nproc) popd