Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
replace intel mkl with openblas
Browse files Browse the repository at this point in the history
ljcui committed Nov 25, 2024
1 parent f6fc539 commit 1a1d20d
Showing 7 changed files with 26 additions and 37 deletions.
23 changes: 10 additions & 13 deletions ci/images/tugraph-compile-centos7-Dockerfile
Original file line number Diff line number Diff line change
@@ -129,7 +129,7 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/leveldb-v1
&& ln -s /usr/local/lib/libleveldb.so.1.20 /usr/local/lib/libleveldb.so \
&& rm -rf /leveldb*

# install node.js && pnpm
# install node.js
# if it is not accessible, replace it with the link below
# https://registry.npmmirror.com/-/binary/node/v16.6.0/node-v16.20.0-linux-x64.tar.gz
RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/node-v16.20.0-linux-x64.tar.gz \
@@ -139,11 +139,8 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/node-v16.2
&& npm config set registry https://registry.npmmirror.com \
&& rm -rf /node-v16.20.0-linux-x64.tar.gz \
&& npm install --global yarn \
&& npm install --global pnpm \
&& ln -s /node-v16.20.0-linux-x64/lib/node_modules/yarn/bin/yarn /usr/local/bin/yarn \
&& ln -s /node-v16.20.0-linux-x64/bin/pnpm /usr/local/bin/pnpm \
&& yarn config set registry https://registry.npmmirror.com \
&& pnpm config set registry https://registry.npmmirror.com
&& yarn config set registry https://registry.npmmirror.com

# install lcov
# if it is not accessible, replace it with the link below
@@ -359,17 +356,17 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/swig-4.0.2

# install faiss
RUN wget -O /tmp/openblas-faiss.tar.gz https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/openblas-faiss.tar.gz \
&& tar -xzvf /tmp/openblas-faiss.tar.gz -C /tmp && cd /tmp/openblas-faiss/OpenBLAS && make && make install \
&& cd /tmp/openblas-faiss/faiss && cmake -B build -DCMAKE_BUILD_TYPE=Release -DFAISS_ENABLE_GPU=OFF -DFAISS_OPT_LEVEL=avx2 -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_INSTALL_LIBDIR=lib -DOpenMP_CXX_FLAGS="-fopenmp" -DOpenMP_CXX_LIB_NAMES="gomp" -DOpenMP_gomp_LIBRARY="/usr/lib64/libgomp.so.1" -DBLAS_LIBRARIES=/opt/OpenBLAS/lib/libopenblas.so . && make -C build && make -C build install \
&& tar -xzvf /tmp/openblas-faiss.tar.gz -C /tmp && cd /tmp/openblas-faiss/OpenBLAS && make ${JFLAG} && make install \
&& cd /tmp/openblas-faiss/faiss && cmake -B build -DCMAKE_BUILD_TYPE=Release -DFAISS_ENABLE_GPU=OFF -DFAISS_OPT_LEVEL=avx2 -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_INSTALL_LIBDIR=lib -DOpenMP_CXX_FLAGS="-fopenmp" -DOpenMP_CXX_LIB_NAMES="gomp" -DOpenMP_gomp_LIBRARY="/usr/lib64/libgomp.so.1" -DBLAS_LIBRARIES=/opt/OpenBLAS/lib/libopenblas.so . && make -C build ${JFLAG} && make -C build install \
&& rm -rf /tmp/openblas-faiss /tmp/openblas-faiss.tar.gz && ldconfig

# install vsag
RUN wget -O /tmp/vsag.tar.gz https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/vsag.tar.gz \
&& tar -xzvf /tmp/vsag.tar.gz -C /tmp \
&& cd /tmp/vsag && sh ./scripts/deps/install_deps_centos.sh \
&& yum install -y libgfortran5.x86_64 && ln -s /usr/lib64/libgfortran.so.5.0.0 /usr/local/lib/libgfortran.so \
&& ldconfig && make release VSAG_CMAKE_ARGS="-S. -Bbuild -DDISABLE_SSE_FORCE=OFF -DDISABLE_AVX_FORCE=OFF -DDISABLE_AVX2_FORCE=OFF -DDISABLE_AVX512_FORCE=ON" \
&& make install && rm -rf /tmp/vsag /tmp/vsag.tar.gz /opt/intel
RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/vsag-0.11.5.tar.gz -O /tmp/vsag.tar.gz && \
cd /tmp && mkdir vsag && tar -xzf vsag.tar.gz --strip-components=1 -C vsag && cd vsag && \
yum install -y libgfortran5.x86_64 && ln -s /usr/lib64/libgfortran.so.5.0.0 /usr/local/lib/libgfortran.so && ldconfig && \
mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_INTEL_MKL=OFF -DDISABLE_AVX2_FORCE=ON -DDISABLE_AVX512_FORCE=ON .. && \
make -j10 && make install && \
cd / && rm -rf /tmp/vsag*

ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk \
LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk/lib/server:$LD_LIBRARY_PATH \
19 changes: 8 additions & 11 deletions ci/images/tugraph-compile-centos8-Dockerfile
Original file line number Diff line number Diff line change
@@ -129,7 +129,7 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/leveldb-v1
&& ln -s /usr/local/lib/libleveldb.so.1.20 /usr/local/lib/libleveldb.so \
&& rm -rf /leveldb*

# install node.js && pnpm
# install node.js
# if it is not accessible, replace it with the link below
# https://registry.npmmirror.com/-/binary/node/v16.6.0/node-v16.20.0-linux-x64.tar.gz
RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/node-v16.20.0-linux-x64.tar.gz \
@@ -139,11 +139,8 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/node-v16.2
&& npm config set registry https://registry.npmmirror.com \
&& rm -rf /node-v16.20.0-linux-x64.tar.gz \
&& npm install --global yarn \
&& npm install --global pnpm \
&& ln -s /node-v16.20.0-linux-x64/lib/node_modules/yarn/bin/yarn /usr/local/bin/yarn \
&& ln -s /node-v16.20.0-linux-x64/bin/pnpm /usr/local/bin/pnpm \
&& yarn config set registry https://registry.npmmirror.com \
&& pnpm config set registry https://registry.npmmirror.com
&& yarn config set registry https://registry.npmmirror.com

# install lcov
# if it is not accessible, replace it with the link below
@@ -348,12 +345,12 @@ RUN wget -O /tmp/openblas-faiss.tar.gz https://tugraph-web.oss-cn-beijing.aliyun
&& rm -rf /tmp/openblas-faiss /tmp/openblas-faiss.tar.gz && ldconfig

# install vsag
RUN wget -O /tmp/vsag.tar.gz https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/vsag.tar.gz \
&& tar -xzvf /tmp/vsag.tar.gz -C /tmp \
&& cd /tmp/vsag && sh ./scripts/deps/install_deps_centos.sh \
&& yum install -y libgfortran && ln -s /usr/lib64/libgfortran.so.5 /usr/local/lib/libgfortran.so \
&& ldconfig && make ${JFLAG} release VSAG_CMAKE_ARGS="-S. -Bbuild -DDISABLE_SSE_FORCE=OFF -DDISABLE_AVX_FORCE=OFF -DDISABLE_AVX2_FORCE=OFF -DDISABLE_AVX512_FORCE=ON" \
&& make install && rm -rf /tmp/vsag /tmp/vsag.tar.gz /opt/intel
RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/vsag-0.11.5.tar.gz -O /tmp/vsag.tar.gz && \
cd /tmp && mkdir vsag && tar -xzf vsag.tar.gz --strip-components=1 -C vsag && cd vsag && \
yum install -y libgfortran5.x86_64 && ln -s /usr/lib64/libgfortran.so.5.0.0 /usr/local/lib/libgfortran.so && ldconfig && \
mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_INTEL_MKL=OFF -DDISABLE_AVX2_FORCE=ON -DDISABLE_AVX512_FORCE=ON .. && \
make -j10 && make install && \
cd / && rm -rf /tmp/vsag*

ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk \
LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/lib64:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk/lib/server:$LD_LIBRARY_PATH \
6 changes: 5 additions & 1 deletion include/fma-common/rw_lock.h
Original file line number Diff line number Diff line change
@@ -29,6 +29,7 @@ enum class LockStatus {
class InvalidThreadIdError : public std::runtime_error {
public:
InvalidThreadIdError() : std::runtime_error("Invalid thread id.") {}
explicit InvalidThreadIdError(const std::string& msg) : std::runtime_error(msg) {}
};

// RWLock using thread-local storage.
@@ -51,7 +52,10 @@ class InterruptableTLSRWLock {
DISABLE_MOVE(InterruptableTLSRWLock);

void ThrowOnInvalidTID(int tid) {
if (tid < 0 || tid >= FMA_MAX_THREADS) throw InvalidThreadIdError();
if (tid < 0 || tid >= FMA_MAX_THREADS) {
throw InvalidThreadIdError(FMA_FMT(
"Invalid thread id, tid:{}, FMA_MAX_THREADS:{}", tid, FMA_MAX_THREADS));
}
}

#define RETURN_IF_INTERRUPTED() \
2 changes: 1 addition & 1 deletion include/fma-common/thread_id.h
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@
#include "fma-common/assert.h"

namespace fma_common {
static const int FMA_MAX_THREADS = 480;
static const int FMA_MAX_THREADS = 65000;

class ThreadIdAssigner {
static const int N = FMA_MAX_THREADS;
9 changes: 0 additions & 9 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -46,15 +46,6 @@ install(FILES
DESTINATION etc)
install(FILES
/usr/local/lib64/libvsag.so
/usr/local/lib64/libmkl_intel_lp64.so
/usr/local/lib64/libmkl_sequential.so
/usr/local/lib64/libmkl_core.so
/usr/local/lib64/libmkl_def.so
/usr/local/lib64/libmkl_avx2.so
/usr/local/lib64/libmkl_mc3.so
/usr/local/lib64/libmkl_gf_lp64.so
/usr/local/lib64/libmkl_intel_thread.so
/usr/local/lib64/libiomp5.so
/lib64/libgfortran.so.5
/lib64/libgfortran.so.5.0.0
DESTINATION lib64/lgraph)
2 changes: 1 addition & 1 deletion src/bolt/io_service.h
Original file line number Diff line number Diff line change
@@ -93,7 +93,7 @@ class IOService : private boost::asio::noncopyable {
: handler_(handler),
acceptor_(service, tcp::endpoint(tcp::v4(), port),
/*reuse_addr*/true),
io_service_pool_(thread_num), interval_(10), timer_(service) {
io_service_pool_(thread_num), interval_(5), timer_(service) {
io_service_pool_.Run();
invoke_async_accept();
clean_closed_conn();

0 comments on commit 1a1d20d

Please sign in to comment.