update sys deps

Signed-off-by: rmdg88 <[email protected]>
DS4SD · Oct 6, 2024 · 6fa3bcb · 6fa3bcb
1 parent c7ec4b6
commit 6fa3bcb
Show file tree

Hide file tree

Showing 13 changed files with 79 additions and 44 deletions.
diff --git a/.github/scripts/build_rhel.sh b/.github/scripts/build_rhel.sh
@@ -6,31 +6,71 @@ set -x  # display command on output
 # Build the Python package with Poetry
 poetry build -f sdist
 
-# Docker build
-docker build -f - . <<EOF
+USE_SYSTEM_DEPS="ON"
+
+USE_TEST_LOGURU="ON"
+
+docker build --progress=plain \
+             --build-arg USE_SYSTEM_DEPS="$USE_SYSTEM_DEPS" \
+             --build-arg USE_TEST_LOGURU="$USE_TEST_LOGURU" \
+             -f - . <<EOF
+    # syntax=docker/dockerfile:1
+
     FROM quay.io/centos/centos:stream9
+
     RUN dnf config-manager --set-enabled crb
+    
     RUN dnf copr -y enable cheimes/deepsearch-glm rhel-9-x86_64
+    
     RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \
         && dnf clean all
+    
     RUN dnf install -y --nodocs \
-            gcc gcc-c++ git make cmake pkgconfig ninja-build glibc-devel \
+            autoconf automake binutils cmake gcc gcc-c++ git glibc-devel glibc-headers glibc-static kernel-devel libtool libstdc++-devel make ninja-build pkgconfig zlib-devel \
             python3.11 python3.11-pip python3.11-devel \
-            libjpeg-turbo-devel libpng-devel zlib-devel \
-            cxxopts-devel fasttext-devel fmt-devel json-schema-validator-devel \
-            loguru-devel sentencepiece-devel pcre2-devel utf8cpp-devel \
+            cxxopts-devel fasttext-devel fmt-devel json-schema-validator-devel pcre2-devel sentencepiece-devel utf8cpp-devel \
         && dnf clean all
 
+    ## USE_TEST_LOGURU
+
+    # RUN dns install -y --nodocs \
+    #         loguru-devel \
+    #     && dnf clean all
+
+    RUN dnf install -y --nodocs \
+            gperftools gperftools-devel gperftools-libs protobuf protobuf-c protobuf-lite protobuf-lite-devel \
+            && dnf clean all
+
+    ARG USE_TEST_LOGURU
+
+    RUN if [ "\$USE_TEST_LOGURU" = "ON" ]; then \
+        git clone https://github.com/emilk/loguru.git /loguru \
+        && cd /loguru \
+        && mkdir build && cd build \
+        && cmake .. \
+        && make \
+        && make install \
+        && cp /loguru/loguru.cpp /usr/local/include/loguru/ \
+        && ls /usr/local/include/loguru/; \
+    fi
+
+    RUN ldconfig -p | grep pthread
+
+    ## USE_TEST_LOGURU
+
     RUN mkdir /src
+
     COPY ./dist/*.tar.gz /src/
 
-    RUN USE_SYSTEM_DEPS=ON pip3.11 install /src/deepsearch_glm*.tar.gz
+    ARG USE_SYSTEM_DEPS
+
+    RUN USE_SYSTEM_DEPS=\$USE_SYSTEM_DEPS pip3.11 install /src/deepsearch_glm*.tar.gz
 
     COPY ./tests /src/tests
+    
     RUN cd /src \
         && pip3.11 install pytest \
         && pytest ./tests/test_structs.py -v \
         && pytest ./tests/test_nlp.py -v \
-        && pytest ./tests/test_glm.py -v \
-        || { echo "Tests failed"; exit 1; }
+        && pytest ./tests/test_glm.py -v
 EOF
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -95,8 +95,12 @@ message(STATUS "cxx-standard: " ${CMAKE_CXX_STANDARD})
 
 option(USE_SYSTEM_DEPS OFF "If enabled, the build will find and link to system dependencies, otherwise they are sourced from the original repos and compiled on the fly.")
 
-# prepare setup for building external packages
-if(NOT USE_SYSTEM_DEPS)
+if(USE_SYSTEM_DEPS)
+    add_definitions(-DUSE_SYSTEM_DEPS)
+    set(DEPENDENCIES fmt json loguru cxxopts pcre2 fasttext utf8 sentencepiece)
+else()
+    add_definitions(-DNOT_USE_SYSTEM_DEPS)
+    set(DEPENDENCIES fmt json loguru cxxopts pcre2 fasttext utf8 sentencepiece sentencepiece_train) # sentencepiece in the compiled version, uses 2 static libs
     if(NOT EXISTS ${EXTERNALS_PREFIX_PATH})
         file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH})
         file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH}/bin)
@@ -106,7 +110,6 @@ if(NOT USE_SYSTEM_DEPS)
     endif()
 endif()
 
-
 # include dependencies
 include(cmake/extlib_fmt.cmake)
 include(cmake/extlib_json.cmake)
@@ -117,9 +120,6 @@ include(cmake/extlib_fasttext.cmake)
 include(cmake/extlib_utf8.git.cmake)
 include(cmake/extlib_sentencepiece.cmake)
 
-# aggregate the targets created by the dependencies
-set(DEPENDENCIES fmt json loguru cxxopts pcre2 fasttext utf8 sentencepiece sentencepiece_train)
-
 # ************************
 # *** libraries        ***
 # ************************

diff --git a/cmake/extlib_cxxopts.cmake b/cmake/extlib_cxxopts.cmake
@@ -6,8 +6,8 @@ set(ext_name "cxxopts")
 if(USE_SYSTEM_DEPS)
     find_package(PkgConfig)
     pkg_check_modules(libcxxopts REQUIRED IMPORTED_TARGET cxxopts)
-
     add_library(${ext_name} ALIAS PkgConfig::libcxxopts)
+
 else()
     include(ExternalProject)
     include(CMakeParseArguments)

diff --git a/cmake/extlib_fasttext.cmake b/cmake/extlib_fasttext.cmake
@@ -6,8 +6,8 @@ set(ext_name "fasttext")
 if(USE_SYSTEM_DEPS)
     find_package(PkgConfig)
     pkg_check_modules(libfasttext_pic REQUIRED IMPORTED_TARGET fasttext)
-
     add_library(${ext_name} ALIAS PkgConfig::libfasttext_pic)
+
 else()
     include(ExternalProject)
     include(CMakeParseArguments)

diff --git a/cmake/extlib_fmt.cmake b/cmake/extlib_fmt.cmake
@@ -6,8 +6,8 @@ set(ext_name "fmt")
 if(USE_SYSTEM_DEPS)
     find_package(PkgConfig)
     pkg_check_modules(libfmt REQUIRED IMPORTED_TARGET fmt)
-
     add_library(${ext_name} ALIAS PkgConfig::libfmt)
+
 else()
     include(ExternalProject)
     include(CMakeParseArguments)

diff --git a/cmake/extlib_json.cmake b/cmake/extlib_json.cmake
@@ -6,8 +6,8 @@ set(ext_name "json")
 if(USE_SYSTEM_DEPS)
     find_package(PkgConfig)
     pkg_check_modules(libjson REQUIRED IMPORTED_TARGET nlohmann_json)
-
     add_library(${ext_name} ALIAS PkgConfig::libjson)
+
 else()
     include(ExternalProject)
     include(CMakeParseArguments)

diff --git a/cmake/extlib_loguru.cmake b/cmake/extlib_loguru.cmake
@@ -4,10 +4,12 @@ message(STATUS "entering in extlib_loguru.cmake")
 set(ext_name "loguru")
 
 if(USE_SYSTEM_DEPS)
-    find_package(PkgConfig)
-    pkg_check_modules(libloguru REQUIRED IMPORTED_TARGET loguru)
+    find_package(PkgConfig REQUIRED)
+    pkg_check_modules(loguru REQUIRED loguru)
+    add_library(${ext_name} INTERFACE)
+    target_include_directories(${ext_name} INTERFACE ${loguru_INCLUDE_DIRS})
+    target_link_libraries(${ext_name} INTERFACE ${loguru_LIBRARIES})
 
-    add_library(${ext_name} ALIAS PkgConfig::libloguru)
 else()
     include(ExternalProject)
     include(CMakeParseArguments)
@@ -17,7 +19,6 @@ else()
 
     set(LOGURU_INCLUDE_DIR ${EXTERNALS_PREFIX_PATH}/include/loguru)
 
-    # Ensure the directory for headers exists
     execute_process(COMMAND mkdir -p ${LOGURU_INCLUDE_DIR})
 
     ExternalProject_Add(extlib_loguru

diff --git a/cmake/extlib_pcre2.cmake b/cmake/extlib_pcre2.cmake
@@ -6,8 +6,8 @@ set(ext_name "pcre2")
 if(USE_SYSTEM_DEPS)
     find_package(PkgConfig)
     pkg_check_modules(libpcre2-8 REQUIRED IMPORTED_TARGET libpcre2-8)
-
     add_library(${ext_name} ALIAS PkgConfig::libpcre2-8)
+
 else()
     include(ExternalProject)
     include(CMakeParseArguments)

diff --git a/cmake/extlib_sentencepiece.cmake b/cmake/extlib_sentencepiece.cmake
@@ -4,12 +4,11 @@ message(STATUS "entering in extlib_sentencepiece.cmake")
 set(ext_name "sentencepiece")
 
 if(USE_SYSTEM_DEPS)
-    find_package(PkgConfig)
-    pkg_check_modules(libsentencepiece REQUIRED IMPORTED_TARGET sentencepiece)
+    find_package(PkgConfig REQUIRED)
+    pkg_check_modules(sentencepiece REQUIRED IMPORTED_TARGET sentencepiece)
+    add_library(${ext_name} ALIAS PkgConfig::sentencepiece)
 
-    add_library(${ext_name} ALIAS PkgConfig::libsentencepiece)
 else()
-    include(ExternalProject)
     include(CMakeParseArguments)
 
     set(SENTENCEPIECE_URL https://github.com/google/sentencepiece.git)

diff --git a/cmake/extlib_utf8.git.cmake b/cmake/extlib_utf8.git.cmake
@@ -4,9 +4,7 @@ message(STATUS "entering in extlib_utf8.cmake")
 set(ext_name "utf8")
 
 if(USE_SYSTEM_DEPS)
-    # this will define the utf8cpp target
     find_package(utf8cpp REQUIRED)
-
     add_library(${ext_name} INTERFACE IMPORTED)
     add_dependencies(${ext_name} utf8cpp)
 

diff --git a/cmake/os_opts.cmake b/cmake/os_opts.cmake
@@ -10,27 +10,16 @@ if(WIN32)
 elseif(APPLE)
    message(STATUS "compiling on mac-osx")
 
-   #set(CMAKE_MACOSX_RPATH 1)
-
    find_library(FoundationLib Foundation)
    find_package(ZLIB)
-   #message("LIB: ${FoundationLib}")
-
    find_library(SystemConfigurationLib SystemConfiguration)
-   #message("LIB: ${SystemConfigurationLib}")
-
-   # set(LIB_LINK qpdf jpeg utf8 z)	
    set(OS_DEPENDENCIES ZLIB::ZLIB)
-
-
    set(LIB_LINK ${FoundationLib} ${SystemConfigurationLib})
 
 elseif(UNIX)
    message(STATUS "compiling on linux")
 
-   # set(LIB_LINK qpdf jpeg utf8 z)
-   find_package(ZLIB)
-   set(LIB_LINK dl m pthread rt resolv)
-   set(OS_DEPENDENCIES ZLIB::ZLIB)
+   set(LIB_LINK dl m pthread rt resolv z)
+   set(OS_DEPENDENCIES)
 
 endif()
diff --git a/src/libraries.h b/src/libraries.h
@@ -25,3 +25,7 @@
 //#include <sentencepiece.pb.h>
 #include <sentencepiece_processor.h>
 #include <sentencepiece_trainer.h>
+
+#include <iostream>
+#include <mutex>
+#include <thread>
diff --git a/src/pybind/base_log.h b/src/pybind/base_log.h
@@ -3,7 +3,11 @@
 #ifndef PYBIND_ANDROMEDA_BASE_LOG_H
 #define PYBIND_ANDROMEDA_BASE_LOG_H
 
-#include <loguru.hpp>
+#ifdef USE_SYSTEM_DEPS
+    #include <loguru/loguru.hpp>
+#else
+    #include <loguru.hpp>
+#endif
 
 namespace andromeda_py
 {