diff --git a/.gitmodules b/.gitmodules index 91a0a8c3ea..30ddd2e55d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,6 @@ [submodule "deps/tugraph-web"] path = deps/tugraph-web url = https://github.com/TuGraph-family/tugraph-web.git -[submodule "deps/tugraph-db-client-java"] - path = deps/tugraph-db-client-java - url = https://github.com/TuGraph-family/tugraph-db-client-java.git -[submodule "deps/tugraph-db-management"] - path = deps/tugraph-db-management - url = https://github.com/TuGraph-family/tugraph-db-management.git [submodule "deps/tugraph-db-browser"] path = deps/tugraph-db-browser url = https://github.com/TuGraph-family/tugraph-db-browser.git diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..8b2187a902 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community includes using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting opensource@github.com, which is a shared team inbox. If the incident involves someone who receives that shared inbox, you can contact an individual maintainer @qishipengqsp at qishipengqsp@gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [https://contributor-covenant.org/version/1/4][version] + +[homepage]: https://contributor-covenant.org +[version]: https://contributor-covenant.org/version/1/4/ diff --git a/README.md b/README.md index 123a8222fc..47092f4aa1 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ NOTICE: If you want to contribute code, you should sign a [cla doc](https://cla- ## 6. Contact -Official Website: [www.tugraph.org](https://www.tugraph.org) +Official Website: [tugraph.tech](https://tugraph.tech) Slack (For developer quick communication): [TuGraph.slack](https://join.slack.com/t/tugraph/shared_invite/zt-1hha8nuli-bqdkwn~w4zH1vlk0QvqIfg) @@ -109,5 +109,15 @@ Slack (For developer quick communication): Contact us via dingtalk, wechat, email and telephone: ![contacts](./docs/images/contact-en.png) +## 7. Acknowledgement + +Thanks to all the individual developers who have contributed to this repository, which are listed below. + + + + + +Made with [contrib.rocks](https://contrib.rocks). + diff --git a/README_CN.md b/README_CN.md index 01f8773323..131afb8bdd 100644 --- a/README_CN.md +++ b/README_CN.md @@ -95,9 +95,9 @@ $ make package -## 5. 联系我们 +## 6. 联系我们 -官网: [www.tugraph.org](https://www.tugraph.org) +官网: [tugraph.tech](https://tugraph.tech) Slack (在线开发沟通): [TuGraph.slack](https://join.slack.com/t/tugraph/shared_invite/zt-1hha8nuli-bqdkwn~w4zH1vlk0QvqIfg) @@ -105,3 +105,14 @@ Slack (在线开发沟通): 通过钉钉群、微信群、微信公众号、邮箱和电话联系我们: ![contacts](./docs/images/contact-zh.png) +## 7. 致谢 + +感谢对这个项目做过贡献的个人开发者，名单如下： + + + + + +生成 By [contrib.rocks](https://contrib.rocks). + + diff --git a/ci/images/tugraph-compile-arm64v8-centos7-Dockerfile b/ci/images/tugraph-compile-arm64v8-centos7-Dockerfile index c8d6a06681..a2af698255 100644 --- a/ci/images/tugraph-compile-arm64v8-centos7-Dockerfile +++ b/ci/images/tugraph-compile-arm64v8-centos7-Dockerfile @@ -336,8 +336,8 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/tabulate-3 RUN sed -i '3 s/-lgomp/-l:libgomp.a/' /usr/local/lib64/libgomp.spec ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk \ - LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk/lib/server:$LD_LIBRARY_PATH \ - PYTHONPATH=/usr/local/lgraph/lib64:/usr/local/lib64:$PYTHONPATH \ + LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk/lib/server:$LD_LIBRARY_PATH \ + PYTHONPATH=/usr/local/lib64/lgraph:/usr/local/lib64:$PYTHONPATH \ PATH=/opt/apache-maven-3.8.7/bin:$PATH # set locale diff --git a/ci/images/tugraph-compile-centos7-Dockerfile b/ci/images/tugraph-compile-centos7-Dockerfile index 498eed3eca..51278907c4 100644 --- a/ci/images/tugraph-compile-centos7-Dockerfile +++ b/ci/images/tugraph-compile-centos7-Dockerfile @@ -129,7 +129,7 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/leveldb-v1 && ln -s /usr/local/lib/libleveldb.so.1.20 /usr/local/lib/libleveldb.so \ && rm -rf /leveldb* -# install node.js && pnpm +# install node.js # if it is not accessible, replace it with the link below # https://registry.npmmirror.com/-/binary/node/v16.6.0/node-v16.20.0-linux-x64.tar.gz RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/node-v16.20.0-linux-x64.tar.gz \ @@ -139,11 +139,8 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/node-v16.2 && npm config set registry https://registry.npmmirror.com \ && rm -rf /node-v16.20.0-linux-x64.tar.gz \ && npm install --global yarn \ - && npm install --global pnpm \ && ln -s /node-v16.20.0-linux-x64/lib/node_modules/yarn/bin/yarn /usr/local/bin/yarn \ - && ln -s /node-v16.20.0-linux-x64/bin/pnpm /usr/local/bin/pnpm \ - && yarn config set registry https://registry.npmmirror.com \ - && pnpm config set registry https://registry.npmmirror.com + && yarn config set registry https://registry.npmmirror.com # install lcov # if it is not accessible, replace it with the link below @@ -343,37 +340,19 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/tabulate-3 && tar zxf tabulate-3a5830.tar.gz && cp -rf tabulate/include/tabulate /usr/local/include \ && rm -rf /tabulate* -RUN sed -i '3 s/-lgomp/-l:libgomp.a/' /usr/local/lib64/libgomp.spec - -# install epel -RUN yum install -y epel-release && yum makecache - -# install openblas -RUN yum install -y openblas-devel - -# install swig -RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/swig-4.0.2.tar.gz \ - && tar -xzvf swig-4.0.2.tar.gz && cd swig-4.0.2 && ./configure \ - && make && make install && cd .. \ - && rm -rf swig-4.0.2 swig-4.0.2.tar.gz - -# install faiss -RUN wget -O /tmp/openblas-faiss.tar.gz https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/openblas-faiss.tar.gz \ - && tar -xzvf /tmp/openblas-faiss.tar.gz -C /tmp && cd /tmp/openblas-faiss/OpenBLAS && make && make install \ - && cd /tmp/openblas-faiss/faiss && cmake -B build -DCMAKE_BUILD_TYPE=Release -DFAISS_ENABLE_GPU=OFF -DFAISS_OPT_LEVEL=avx2 -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_INSTALL_LIBDIR=lib -DOpenMP_CXX_FLAGS="-fopenmp" -DOpenMP_CXX_LIB_NAMES="gomp" -DOpenMP_gomp_LIBRARY="/usr/lib64/libgomp.so.1" -DBLAS_LIBRARIES=/opt/OpenBLAS/lib/libopenblas.so . && make -C build && make -C build install \ - && rm -rf /tmp/openblas-faiss /tmp/openblas-faiss.tar.gz && ldconfig - # install vsag -RUN wget -O /tmp/vsag.tar.gz https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/vsag.tar.gz \ - && tar -xzvf /tmp/vsag.tar.gz -C /tmp \ - && cd /tmp/vsag && sh ./scripts/deps/install_deps_centos.sh \ - && yum install -y libgfortran5.x86_64 && ln -s /usr/lib64/libgfortran.so.5.0.0 /usr/local/lib/libgfortran.so \ - && ldconfig && make release VSAG_CMAKE_ARGS="-S. -Bbuild -DDISABLE_SSE_FORCE=OFF -DDISABLE_AVX_FORCE=OFF -DDISABLE_AVX2_FORCE=OFF -DDISABLE_AVX512_FORCE=ON" \ - && make install && rm -rf /tmp/vsag /tmp/vsag.tar.gz /opt/intel +RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/vsag-0.11.5.tar.gz -O /tmp/vsag.tar.gz && \ + cd /tmp && mkdir vsag && tar -xzf vsag.tar.gz --strip-components=1 -C vsag && cd vsag && \ + yum install -y libgfortran5.x86_64 && ln -s /usr/lib64/libgfortran.so.5.0.0 /usr/local/lib/libgfortran.so && ldconfig && \ + mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_INTEL_MKL=OFF -DDISABLE_AVX2_FORCE=ON -DDISABLE_AVX512_FORCE=ON .. && \ + make ${JFLAG} && make install && \ + cd / && rm -rf /tmp/vsag* + +RUN sed -i '3 s/-lgomp/-l:libgomp.a/' /usr/local/lib64/libgomp.spec ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk \ - LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk/lib/server:$LD_LIBRARY_PATH \ - PYTHONPATH=/usr/local/lgraph/lib64:/usr/local/lib64:$PYTHONPATH \ + LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk/lib/server:$LD_LIBRARY_PATH \ + PYTHONPATH=/usr/local/lib64/lgraph:/usr/local/lib64:$PYTHONPATH \ PATH=/opt/apache-maven-3.8.7/bin:$PATH # set locale diff --git a/ci/images/tugraph-compile-centos8-Dockerfile b/ci/images/tugraph-compile-centos8-Dockerfile index 6879f9c025..5c69b5f948 100644 --- a/ci/images/tugraph-compile-centos8-Dockerfile +++ b/ci/images/tugraph-compile-centos8-Dockerfile @@ -6,9 +6,11 @@ FROM reg.docker.alibaba-inc.com/fma/centos:8.4.2105 ARG JFLAG=-j8 ARG PYPI="-i https://pypi.antfin-inc.com/simple/ --trusted-host pypi.antfin-inc.com" -RUN rm -rf /etc/yum.repos.d \ - && curl -O https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/yum.repos.d.tar.gz \ - && tar -xf yum.repos.d.tar.gz && mv yum.repos.d /etc/yum.repos.d && rm yum.repos.d.tar.gz +RUN sed -e "s|^mirrorlist=|#mirrorlist=|g" \ + -e "s|^#baseurl=http://mirror.centos.org/centos/\$releasever|baseurl=http://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.4.2105|g" \ + -e "s|^#baseurl=http://mirror.centos.org/\$contentdir/\$releasever|baseurl=http://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.4.2105|g" \ + -i.bak \ + /etc/yum.repos.d/CentOS-*.repo && yum clean all && yum makecache RUN yum update -y && yum install -y \ git \ @@ -129,7 +131,7 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/leveldb-v1 && ln -s /usr/local/lib/libleveldb.so.1.20 /usr/local/lib/libleveldb.so \ && rm -rf /leveldb* -# install node.js && pnpm +# install node.js # if it is not accessible, replace it with the link below # https://registry.npmmirror.com/-/binary/node/v16.6.0/node-v16.20.0-linux-x64.tar.gz RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/node-v16.20.0-linux-x64.tar.gz \ @@ -139,11 +141,8 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/node-v16.2 && npm config set registry https://registry.npmmirror.com \ && rm -rf /node-v16.20.0-linux-x64.tar.gz \ && npm install --global yarn \ - && npm install --global pnpm \ && ln -s /node-v16.20.0-linux-x64/lib/node_modules/yarn/bin/yarn /usr/local/bin/yarn \ - && ln -s /node-v16.20.0-linux-x64/bin/pnpm /usr/local/bin/pnpm \ - && yarn config set registry https://registry.npmmirror.com \ - && pnpm config set registry https://registry.npmmirror.com + && yarn config set registry https://registry.npmmirror.com # install lcov # if it is not accessible, replace it with the link below @@ -246,6 +245,19 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/rocksdb-v7 && tar zxf rocksdb-v7.8.3.tar.gz && cd rocksdb-7.8.3 && cmake -DCMAKE_BUILD_TYPE=Release -DPORTABLE=ON -DFORCE_SSE42=ON -DWITH_JEMALLOC=ON \ && make ${JFLAG} && make install && rm -rf /rocksdb* +# set maven mirror +RUN mkdir -p ~/.m2 \ + && echo '' > ~/.m2/settings.xml \ + && echo ' ' >> ~/.m2/settings.xml \ + && echo ' ' >> ~/.m2/settings.xml \ + && echo ' alimaven' >> ~/.m2/settings.xml \ + && echo ' central' >> ~/.m2/settings.xml \ + && echo ' https://maven.aliyun.com/nexus/content/groups/public/' >> ~/.m2/settings.xml \ + && echo ' ' >> ~/.m2/settings.xml \ + && echo ' ' >> ~/.m2/settings.xml \ + && echo '' >> ~/.m2/settings.xml \ + && cat ~/.m2/settings.xml + # install antlr4-4.13.0 # if it is not accessible, replace it with the link below # https://github.com/antlr/antlr4/archive/refs/tags/4.13.0.tar.gz @@ -331,33 +343,19 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/tabulate-3 && tar zxf tabulate-3a5830.tar.gz && cp -rf tabulate/include/tabulate /usr/local/include \ && rm -rf /tabulate* -RUN sed -i '3 s/-lgomp/-l:libgomp.a/' /usr/local/lib64/libgomp.spec - -RUN yum install -y epel-release pcre-devel yum-utils - -# install swig -RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/swig-4.0.2.tar.gz \ - && tar -xzvf swig-4.0.2.tar.gz && cd swig-4.0.2 && ./configure \ - && make && make install && cd .. \ - && rm -rf swig-4.0.2 swig-4.0.2.tar.gz - -# install faiss -RUN wget -O /tmp/openblas-faiss.tar.gz https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/openblas-faiss.tar.gz \ - && tar -xzvf /tmp/openblas-faiss.tar.gz -C /tmp && cd /tmp/openblas-faiss/OpenBLAS && make && make install \ - && cd /tmp/openblas-faiss/faiss && cmake -B build -DCMAKE_BUILD_TYPE=Release -DFAISS_ENABLE_GPU=OFF -DFAISS_OPT_LEVEL=avx2 -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_INSTALL_LIBDIR=lib -DOpenMP_CXX_FLAGS="-fopenmp" -DOpenMP_CXX_LIB_NAMES="gomp" -DOpenMP_gomp_LIBRARY="/usr/lib64/libgomp.so.1" -DBLAS_LIBRARIES=/opt/OpenBLAS/lib/libopenblas.so . && make -C build && make -C build install \ - && rm -rf /tmp/openblas-faiss /tmp/openblas-faiss.tar.gz && ldconfig - # install vsag -RUN wget -O /tmp/vsag.tar.gz https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/vsag.tar.gz \ - && tar -xzvf /tmp/vsag.tar.gz -C /tmp \ - && cd /tmp/vsag && sh ./scripts/deps/install_deps_centos.sh \ - && yum install -y libgfortran && ln -s /usr/lib64/libgfortran.so.5 /usr/local/lib/libgfortran.so \ - && ldconfig && make ${JFLAG} release VSAG_CMAKE_ARGS="-S. -Bbuild -DDISABLE_SSE_FORCE=OFF -DDISABLE_AVX_FORCE=OFF -DDISABLE_AVX2_FORCE=OFF -DDISABLE_AVX512_FORCE=ON" \ - && make install && rm -rf /tmp/vsag /tmp/vsag.tar.gz /opt/intel +RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/vsag-0.11.5.tar.gz -O /tmp/vsag.tar.gz && \ + cd /tmp && mkdir vsag && tar -xzf vsag.tar.gz --strip-components=1 -C vsag && cd vsag && \ + yum install -y libgfortran.x86_64 && ln -s /usr/lib64/libgfortran.so.5.0.0 /usr/local/lib/libgfortran.so && ldconfig && \ + mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_INTEL_MKL=OFF -DDISABLE_AVX2_FORCE=ON -DDISABLE_AVX512_FORCE=ON .. && \ + make ${JFLAG} && make install && \ + cd / && rm -rf /tmp/vsag* + +RUN sed -i '3 s/-lgomp/-l:libgomp.a/' /usr/local/lib64/libgomp.spec ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk \ - LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/lib64:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk/lib/server:$LD_LIBRARY_PATH \ - PYTHONPATH=/usr/local/lgraph/lib64:/usr/local/lib64:$PYTHONPATH \ + LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/lib64:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk/lib/server:$LD_LIBRARY_PATH \ + PYTHONPATH=/usr/local/lib64/lgraph:/usr/local/lib64:$PYTHONPATH \ PATH=/opt/apache-maven-3.8.7/bin:$PATH # set locale diff --git a/ci/images/tugraph-compile-ubuntu18.04-Dockerfile b/ci/images/tugraph-compile-ubuntu18.04-Dockerfile index 594ab07d20..3b3df93386 100644 --- a/ci/images/tugraph-compile-ubuntu18.04-Dockerfile +++ b/ci/images/tugraph-compile-ubuntu18.04-Dockerfile @@ -128,7 +128,7 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/leveldb-v1 && ln -s /usr/local/lib/libleveldb.so.1.20 /usr/local/lib/libleveldb.so \ && rm -rf /leveldb* -# install node.js && pnpm +# install node.js # if it is not accessible, replace it with the link below # https://registry.npmmirror.com/-/binary/node/v16.6.0/node-v16.20.0-linux-x64.tar.gz RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/node-v16.20.0-linux-x64.tar.gz \ @@ -138,11 +138,8 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/node-v16.2 && npm config set registry https://registry.npmmirror.com \ && rm -rf /node-v16.20.0-linux-x64.tar.gz \ && npm install --global yarn \ - && npm install --global pnpm \ && ln -s /node-v16.20.0-linux-x64/lib/node_modules/yarn/bin/yarn /usr/local/bin/yarn \ - && ln -s /node-v16.20.0-linux-x64/bin/pnpm /usr/local/bin/pnpm \ - && yarn config set registry https://registry.npmmirror.com \ - && pnpm config set registry https://registry.npmmirror.com + && yarn config set registry https://registry.npmmirror.com # install lcov # if it is not accessible, replace it with the link below @@ -244,6 +241,19 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/rocksdb-v7 && tar zxf rocksdb-v7.8.3.tar.gz && cd rocksdb-7.8.3 && cmake -DCMAKE_BUILD_TYPE=Release -DPORTABLE=ON -DFORCE_SSE42=ON -DWITH_JEMALLOC=ON \ && make ${JFLAG} && make install && rm -rf /rocksdb* +# set maven mirror +RUN mkdir -p ~/.m2 \ + && echo '' > ~/.m2/settings.xml \ + && echo ' ' >> ~/.m2/settings.xml \ + && echo ' ' >> ~/.m2/settings.xml \ + && echo ' alimaven' >> ~/.m2/settings.xml \ + && echo ' central' >> ~/.m2/settings.xml \ + && echo ' https://maven.aliyun.com/nexus/content/groups/public/' >> ~/.m2/settings.xml \ + && echo ' ' >> ~/.m2/settings.xml \ + && echo ' ' >> ~/.m2/settings.xml \ + && echo '' >> ~/.m2/settings.xml \ + && cat ~/.m2/settings.xml + # install antlr4-4.13.0 # if it is not accessible, replace it with the link below # https://github.com/antlr/antlr4/archive/refs/tags/4.13.0.tar.gz @@ -329,11 +339,20 @@ RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/tabulate-3 && tar zxf tabulate-3a5830.tar.gz && cp -rf tabulate/include/tabulate /usr/local/include \ && rm -rf /tabulate* +# install vsag +RUN apt-get install -y gfortran && ln -s /usr/lib/x86_64-linux-gnu/libgfortran.so.4 /usr/local/lib/libgfortran.so + +RUN wget https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/vsag-0.11.5.tar.gz -O /tmp/vsag.tar.gz && \ + cd /tmp && mkdir vsag && tar -xzf vsag.tar.gz --strip-components=1 -C vsag && cd vsag && \ + mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib64 -DENABLE_INTEL_MKL=OFF -DDISABLE_AVX2_FORCE=ON -DDISABLE_AVX512_FORCE=ON .. && \ + make -j10 && make install && \ + cd / && rm -rf /tmp/vsag* + RUN sed -i '3 s/-lgomp/-l:libgomp.a/' /usr/local/lib64/libgomp.spec ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 \ - LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk-amd64/lib/server:$LD_LIBRARY_PATH \ - PYTHONPATH=/usr/local/lgraph/lib64:/usr/local/lib64:$PYTHONPATH \ + LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk-amd64/lib/server:$LD_LIBRARY_PATH \ + PYTHONPATH=/usr/local/lib64/lgraph:/usr/local/lib64:$PYTHONPATH \ PATH=/opt/apache-maven-3.8.7/bin:$PATH ENV LANG zh_CN.UTF-8 diff --git a/ci/images/tugraph-mini-runtime-centos7-Dockerfile b/ci/images/tugraph-mini-runtime-centos7-Dockerfile index 163add4dbf..caffe94b78 100644 --- a/ci/images/tugraph-mini-runtime-centos7-Dockerfile +++ b/ci/images/tugraph-mini-runtime-centos7-Dockerfile @@ -12,7 +12,9 @@ RUN sed -e "s|^mirrorlist=|#mirrorlist=|g" \ RUN yum -x filesystem update -y && yum install -y \ gcc \ openssl-static \ - wget + wget \ + libgfortran5.x86_64 \ + libgomp # install tugraph # specifies the path of the object storage where the installation package resides @@ -22,6 +24,6 @@ ARG FILENAME RUN wget ${FILEPATH}/${FILENAME} RUN rpm -ivh ${FILENAME} && rm /${FILENAME} -ENV LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/local/lib64:/usr/local/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:/usr/local/lib:$LD_LIBRARY_PATH CMD lgraph_server -d run diff --git a/ci/images/tugraph-mini-runtime-centos8-Dockerfile b/ci/images/tugraph-mini-runtime-centos8-Dockerfile index 6952dd1258..f2d773a3a4 100644 --- a/ci/images/tugraph-mini-runtime-centos8-Dockerfile +++ b/ci/images/tugraph-mini-runtime-centos8-Dockerfile @@ -3,14 +3,18 @@ # Alibaba Image FROM reg.docker.alibaba-inc.com/fma/centos:8.4.2105 -RUN rm -rf /etc/yum.repos.d \ - && curl -O https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/yum.repos.d.tar.gz \ - && tar -xf yum.repos.d.tar.gz && mv yum.repos.d /etc/yum.repos.d && rm yum.repos.d.tar.gz +RUN sed -e "s|^mirrorlist=|#mirrorlist=|g" \ + -e "s|^#baseurl=http://mirror.centos.org/centos/\$releasever|baseurl=http://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.4.2105|g" \ + -e "s|^#baseurl=http://mirror.centos.org/\$contentdir/\$releasever|baseurl=http://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.4.2105|g" \ + -i.bak \ + /etc/yum.repos.d/CentOS-*.repo && yum clean all && yum makecache RUN yum update -y && yum install -y \ gcc \ openssl-devel.x86_64 \ - wget + wget \ + libgfortran.x86_64 \ + libgomp # install tugraph # specifies the path of the object storage where the installation package resides @@ -20,6 +24,6 @@ ARG FILENAME RUN wget ${FILEPATH}/${FILENAME} RUN rpm -ivh ${FILENAME} && rm /${FILENAME} -ENV LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/local/lib64:/usr/local/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:/usr/local/lib:$LD_LIBRARY_PATH CMD lgraph_server -d run diff --git a/ci/images/tugraph-mini-runtime-ubuntu18.04-Dockerfile b/ci/images/tugraph-mini-runtime-ubuntu18.04-Dockerfile index 981276d756..c4f374806d 100644 --- a/ci/images/tugraph-mini-runtime-ubuntu18.04-Dockerfile +++ b/ci/images/tugraph-mini-runtime-ubuntu18.04-Dockerfile @@ -12,7 +12,8 @@ RUN rm -rf /et/apt/sources.list \ RUN apt-get update && apt-get install -y \ gcc \ libssl-dev \ - wget + wget \ + gfortran # install tugraph # specifies the path of the object storage where the installation package resides @@ -22,4 +23,4 @@ ARG FILENAME RUN wget ${FILEPATH}/${FILENAME} RUN dpkg -i ${FILENAME} && rm /${FILENAME} -ENV LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/local/lib64:/usr/local/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:/usr/local/lib:$LD_LIBRARY_PATH diff --git a/ci/images/tugraph-runtime-arm64v8-centos7-Dockerfile b/ci/images/tugraph-runtime-arm64v8-centos7-Dockerfile index b17edd5631..07aa3c9ea3 100644 --- a/ci/images/tugraph-runtime-arm64v8-centos7-Dockerfile +++ b/ci/images/tugraph-runtime-arm64v8-centos7-Dockerfile @@ -104,7 +104,7 @@ RUN cd /usr/local/share/lgraph/tugraph-db-browser \ && pnpm run build ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk/jre/ \ - LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/local/lib64:/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server:$LD_LIBRARY_PATH \ - PYTHONPATH=/usr/local/lgraph/lib64:/usr/local/lib64:$PYTHONPATH \ + LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server:$LD_LIBRARY_PATH \ + PYTHONPATH=/usr/local/lib64/lgraph:/usr/local/lib64:$PYTHONPATH \ PATH=/opt/apache-maven-3.8.7/bin:$PATH diff --git a/ci/images/tugraph-runtime-centos7-Dockerfile b/ci/images/tugraph-runtime-centos7-Dockerfile index 2aa8972c4f..8aba18872f 100644 --- a/ci/images/tugraph-runtime-centos7-Dockerfile +++ b/ci/images/tugraph-runtime-centos7-Dockerfile @@ -18,6 +18,8 @@ RUN yum -x filesystem update -y && yum install -y \ bzip2 \ unzip \ openssl-static \ + libgfortran5.x86_64 \ + libgomp \ libcurl-devel.x86_64 && yum clean all # install g++ 8.4.0 @@ -68,7 +70,7 @@ ARG FILEPATH ARG FILENAME RUN wget ${FILEPATH}/${FILENAME} && rpm -ivh ${FILENAME} && rm -f /${FILENAME} -ENV LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/local/lib64:$LD_LIBRARY_PATH \ - PYTHONPATH=/usr/local/lgraph/lib64:/usr/local/lib64:$PYTHONPATH +ENV LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:$LD_LIBRARY_PATH \ + PYTHONPATH=/usr/local/lib64/lgraph:/usr/local/lib64:$PYTHONPATH CMD lgraph_server -d run diff --git a/ci/images/tugraph-runtime-centos8-Dockerfile b/ci/images/tugraph-runtime-centos8-Dockerfile index d86120bf43..f66448d3cd 100644 --- a/ci/images/tugraph-runtime-centos8-Dockerfile +++ b/ci/images/tugraph-runtime-centos8-Dockerfile @@ -3,9 +3,11 @@ # Alibaba Image FROM reg.docker.alibaba-inc.com/fma/centos:8.4.2105 -RUN rm -rf /etc/yum.repos.d \ - && curl -O https://tugraph-web.oss-cn-beijing.aliyuncs.com/tugraph/deps/yum.repos.d.tar.gz \ - && tar -xf yum.repos.d.tar.gz && mv yum.repos.d /etc/yum.repos.d && rm yum.repos.d.tar.gz +RUN sed -e "s|^mirrorlist=|#mirrorlist=|g" \ + -e "s|^#baseurl=http://mirror.centos.org/centos/\$releasever|baseurl=http://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.4.2105|g" \ + -e "s|^#baseurl=http://mirror.centos.org/\$contentdir/\$releasever|baseurl=http://mirrors.tuna.tsinghua.edu.cn/centos-vault/8.4.2105|g" \ + -i.bak \ + /etc/yum.repos.d/CentOS-*.repo && yum clean all && yum makecache RUN yum update -y && yum install -y \ git \ @@ -16,6 +18,8 @@ RUN yum update -y && yum install -y \ bzip2 \ unzip \ openssl-devel.x86_64 \ + libgfortran.x86_64 \ + libgomp \ libcurl-devel.x86_64 && yum clean all # install g++ 8.4.0 @@ -66,7 +70,7 @@ ARG FILEPATH ARG FILENAME RUN wget ${FILEPATH}/${FILENAME} && rpm -ivh ${FILENAME} && rm -f /${FILENAME} -ENV LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/local/lib64:/usr/local/lib:$LD_LIBRARY_PATH \ - PYTHONPATH=/usr/local/lgraph/lib64:/usr/local/lib64:$PYTHONPATH +ENV LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:/usr/local/lib:$LD_LIBRARY_PATH \ + PYTHONPATH=/usr/local/lib64/lgraph:/usr/local/lib64:$PYTHONPATH CMD lgraph_server -d run diff --git a/ci/images/tugraph-runtime-ubuntu18.04-Dockerfile b/ci/images/tugraph-runtime-ubuntu18.04-Dockerfile index bbe3997a2f..814a2bfc2e 100644 --- a/ci/images/tugraph-runtime-ubuntu18.04-Dockerfile +++ b/ci/images/tugraph-runtime-ubuntu18.04-Dockerfile @@ -21,7 +21,8 @@ RUN apt-get update && apt-get install -y \ zlib1g-dev \ libssl-dev \ openjdk-8-jdk \ - unzip + unzip \ + gfortran # install g++ 8.4.0 # if it is not accessible, replace it with the link below @@ -111,7 +112,7 @@ RUN cd /usr/local/share/lgraph/tugraph-db-browser \ && pnpm run build ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre \ - LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/:$LD_LIBRARY_PATH \ - PYTHONPATH=/usr/local/lgraph/lib64:/usr/local/lib64:$PYTHONPATH \ + LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/:$LD_LIBRARY_PATH \ + PYTHONPATH=/usr/local/lib64/lgraph:/usr/local/lib64:$PYTHONPATH \ PATH=/opt/apache-maven-3.8.7/bin:/usr/lib/jvm/java-8-openjdk-amd64/jre/bin:$PATH diff --git a/ci/start_dev.sh b/ci/start_dev.sh index 7c69c5d5da..cbf557a275 100644 --- a/ci/start_dev.sh +++ b/ci/start_dev.sh @@ -77,7 +77,7 @@ if [ -z "$CONTAINER_ID" ]; then ${DOCKER_IMAGE} \ bash -c "chown ${USER_NAME}:${GROUP_NAME} /home/${USER_NAME} && echo 'export JAVA_HOME=/usr/lib/jvm/java-11-openjdk' > /home/${USER_NAME}/.bashrc && - echo 'export LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/lib64:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk/lib/server' > /home/${USER_NAME}/.bashrc && + echo 'export LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/lib64:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-11-openjdk/lib/server' > /home/${USER_NAME}/.bashrc && echo 'export JAVA_HOME=/usr/lib/jvm/java-11-openjdk' > /home/${USER_NAME}/.bash_profile && /usr/sbin/sshd -D" fi diff --git a/community/CONTRIBUTING.md b/community/CONTRIBUTING.md new file mode 100644 index 0000000000..7173e8b8ab --- /dev/null +++ b/community/CONTRIBUTING.md @@ -0,0 +1,52 @@ +# Contributing to TuGraph DB + +Thank you for considering contributing to TuGraph DB! We welcome contributions from the community and are grateful for your support. + +## How to Contribute + +### 1. Fork the Repository +Fork the repository to your own GitHub account by clicking the "Fork" button at the top right of the repository page. + +### 2. Clone the Repository +Clone the forked repository to your local machine: +```bash +git clone https://github.com/TuGraph-family/tugraph-db.git +cd tugraph-db +``` + +### 3. Create a Branch +Create a new branch for your feature or bugfix: +```bash +git checkout -b my-feature-branch +``` + +### 4. Make Changes +Make your changes to the codebase. Ensure your code follows the project's coding standards and includes appropriate tests. + +### 5. Commit Changes +Commit your changes with a clear and concise commit message: +```bash +git add . +git commit -m "Description of your changes" +``` + +### 6. Push Changes +Push your changes to your forked repository: +```bash +git push origin my-feature-branch +``` + +### 7. Create a Pull Request +Create a pull request from your forked repository to the main repository. Provide a detailed description of your changes and any relevant information. + +### 8. Review Process +Your pull request will be reviewed by the project maintainers. Please be responsive to any feedback or requests for changes. + +## Code of Conduct +Please adhere to our [Code of Conduct](../CODE_OF_CONDUCT.md) in all your interactions with the project. + +## Additional Resources +- [Issue Tracker](https://github.com/TuGraph-family/tugraph-db/issues) +- [Project Documentation](../docs) + +Thank you for your contributions! diff --git a/community/ROLES.md b/community/ROLES.md new file mode 100644 index 0000000000..7011e6d866 --- /dev/null +++ b/community/ROLES.md @@ -0,0 +1,44 @@ +# Roles and Responsibilities + +## Project Roles + +### Project Management Committee (PMC) +- **Description**: The PMC is responsible for the overall management and oversight of the project. +- **Responsibilities**: + - Ensure the project adheres to the Apache Way. + - Make decisions regarding project direction and policies. + - Manage releases and ensure quality standards are met. + - Handle legal and licensing issues. + +### Committers +- **Description**: Committers are contributors who have earned the right to commit code directly to the project repository. +- **Responsibilities**: + - Review and commit code contributions from the community. + - Maintain the project's codebase and documentation. + - Mentor new contributors and facilitate their integration into the project. + - Ensure contributions adhere to project guidelines and standards. + +### Contributors +- **Description**: Contributors are individuals who contribute to the project by submitting patches, documentation, or other improvements. +- **Responsibilities**: + - Submit patches and improvements via pull requests. + - Participate in discussions and provide feedback on issues and proposals. + - Follow the project's contribution guidelines and coding standards. + - Engage with the community and assist other contributors. + +### Users +- **Description**: Users are individuals or organizations that use the project. +- **Responsibilities**: + - Provide feedback and report issues. + - Participate in the community by asking questions and sharing experiences. + - Contribute to the project's documentation and knowledge base. + +## The Apache Way + +The project follows the principles of the Apache Way, which include: +- **Community Over Code**: Prioritizing a healthy and vibrant community over the code itself. +- **Meritocracy**: Recognizing and rewarding contributions based on merit. +- **Openness**: Conducting all project discussions and decisions in public forums. +- **Consensus**: Striving for consensus in decision-making processes. + +For more information on the Apache Way, please refer to the [Apache Software Foundation](https://www.apache.org/foundation/how-it-works.html). diff --git a/demo/Bolt/go_example.go b/demo/Bolt/go_example.go index edce5f98cc..df3087c4a5 100644 --- a/demo/Bolt/go_example.go +++ b/demo/Bolt/go_example.go @@ -23,7 +23,7 @@ func main() { panic(err) return } - _, err = session.Run(ctx, "CALL db.createVertexLabel('person', 'id' , 'id' ,INT32, false, 'name' ,STRING, false)", nil) + _, err = session.Run(ctx, "CALL db.createVertexLabel('person', 'id' , 'id' ,'INT32', false, 'name' ,'STRING', false)", nil) if err != nil { panic(err) return diff --git a/demo/Bolt/java_example.java b/demo/Bolt/java_example.java index a8c3a771f3..d604ba5398 100644 --- a/demo/Bolt/java_example.java +++ b/demo/Bolt/java_example.java @@ -10,7 +10,7 @@ public static void main( final String[] args ) { Driver driver = GraphDatabase.driver("bolt://localhost:7687", AuthTokens.basic("admin", "73@TuGraph")); Session session = driver.session(SessionConfig.forDatabase("default")); session.run("CALL db.dropDB()"); - session.run("CALL db.createVertexLabel('person', 'id' , 'id' ,INT32, false, 'name' ,STRING, false)"); + session.run("CALL db.createVertexLabel('person', 'id' , 'id' ,'INT32', false, 'name', 'STRING', false)"); session.run("CALL db.createEdgeLabel('is_friend','[[\"person\",\"person\"]]')"); session.run("create (n1:person {name:'jack',id:1}), (n2:person {name:'lucy',id:2})"); session.run("match (n1:person {id:1}), (n2:person {id:2}) create (n1)-[r:is_friend]->(n2)"); diff --git a/demo/Bolt/js_example.js b/demo/Bolt/js_example.js index 8fa9d01f86..c4d3068e76 100644 --- a/demo/Bolt/js_example.js +++ b/demo/Bolt/js_example.js @@ -4,7 +4,7 @@ var session = driver.session({database: 'default'}) await session.run("CALL db.dropDB()"); console.log("clean db"); - await session.run("CALL db.createVertexLabel('person', 'id' , 'id' ,INT32, false, 'name' ,STRING, false)"); + await session.run("CALL db.createVertexLabel('person', 'id' , 'id', 'INT32', false, 'name', 'STRING', false)"); console.log("add vertex label"); await session.run("CALL db.createEdgeLabel('is_friend','[[\"person\",\"person\"]]')"); console.log("add edge label"); diff --git a/demo/Bolt/python_example.py b/demo/Bolt/python_example.py index ef62aea129..cf7acaaf79 100644 --- a/demo/Bolt/python_example.py +++ b/demo/Bolt/python_example.py @@ -5,7 +5,7 @@ with GraphDatabase.driver(URI, auth=AUTH) as client: session = client.session(database="default") session.run("CALL db.dropDB()") - session.run("CALL db.createVertexLabel('person', 'id' , 'id' ,INT32, false, 'name' ,STRING, false)") + session.run("CALL db.createVertexLabel('person', 'id' , 'id', 'INT32', false, 'name', 'STRING', false)") session.run("CALL db.createEdgeLabel('is_friend','[[\"person\",\"person\"]]')") session.run("create (n1:person {name:'jack',id:1}), (n2:person {name:'lucy',id:2})") session.run("match (n1:person {id:1}), (n2:person {id:2}) create (n1)-[r:is_friend]->(n2)") diff --git a/demo/Bolt/rust_example.rs b/demo/Bolt/rust_example.rs index a05b11f4b3..6ecacc0890 100644 --- a/demo/Bolt/rust_example.rs +++ b/demo/Bolt/rust_example.rs @@ -14,7 +14,7 @@ async fn main() { let graph = Graph::connect(config).await.unwrap(); { graph.run(query("CALL db.dropDB()")).await.unwrap(); - graph.run(query("CALL db.createVertexLabel('person', 'id' , 'id' ,INT32, false, 'name' ,STRING, false)")).await.unwrap(); + graph.run(query("CALL db.createVertexLabel('person', 'id' , 'id', 'INT32', false, 'name', 'STRING', false)")).await.unwrap(); graph.run(query("CALL db.createEdgeLabel('is_friend','[[\"person\",\"person\"]]')")).await.unwrap(); graph.run(query("create (n1:person {name:'jack',id:1}), (n2:person {name:'lucy',id:2})")).await.unwrap(); graph.run(query("match (n1:person {id:1}), (n2:person {id:2}) create (n1)-[r:is_friend]->(n2)")).await.unwrap(); diff --git a/deps/geax-front-end/include/geax-front-end/GEAXErrorCode.h b/deps/geax-front-end/include/geax-front-end/GEAXErrorCode.h index 551c9dae2b..6622153d97 100644 --- a/deps/geax-front-end/include/geax-front-end/GEAXErrorCode.h +++ b/deps/geax-front-end/include/geax-front-end/GEAXErrorCode.h @@ -26,6 +26,7 @@ enum class GEAXErrorCode : uint32_t { GEAX_ERROR = 1, GEAX_OOPS = 4, GEAX_OPTIMIZATION_PASS = 5, + GEAX_OPTIMIZATION_NOT_PASS = 6, GEAX_COMMON_INVALID_ARGUMENT = 30002, GEAX_COMMON_NULLPTR = 30003, diff --git a/deps/geax-front-end/include/geax-front-end/ast/clause/EdgeLike.h b/deps/geax-front-end/include/geax-front-end/ast/clause/EdgeLike.h index 62c5343371..855557268b 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/clause/EdgeLike.h +++ b/deps/geax-front-end/include/geax-front-end/ast/clause/EdgeLike.h @@ -26,7 +26,7 @@ namespace frontend { class EdgeLike : public AstNode { public: explicit EdgeLike(AstNodeType type) : AstNode(type) {} - ~EdgeLike() = default; + virtual ~EdgeLike() = default; }; } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/clause/ElementPredicate.h b/deps/geax-front-end/include/geax-front-end/ast/clause/ElementPredicate.h index e4afca1ccd..bba4871afb 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/clause/ElementPredicate.h +++ b/deps/geax-front-end/include/geax-front-end/ast/clause/ElementPredicate.h @@ -26,7 +26,7 @@ namespace frontend { class ElementPredicate : public AstNode { public: explicit ElementPredicate(AstNodeType type) : AstNode(type) {} - ~ElementPredicate() = default; + virtual ~ElementPredicate() = default; }; } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/clause/Hint.h b/deps/geax-front-end/include/geax-front-end/ast/clause/Hint.h index 2049c5b381..3d8e04a1a4 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/clause/Hint.h +++ b/deps/geax-front-end/include/geax-front-end/ast/clause/Hint.h @@ -26,7 +26,7 @@ namespace frontend { class Hint : public AstNode { public: explicit Hint(AstNodeType type) : AstNode(type) {} - ~Hint() = default; + virtual ~Hint() = default; }; // class Hint } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/clause/LabelTree.h b/deps/geax-front-end/include/geax-front-end/ast/clause/LabelTree.h index d2fdef52ca..52464682f5 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/clause/LabelTree.h +++ b/deps/geax-front-end/include/geax-front-end/ast/clause/LabelTree.h @@ -26,7 +26,7 @@ namespace frontend { class LabelTree : public AstNode { public: explicit LabelTree(AstNodeType type) : AstNode(type) {} - ~LabelTree() = default; + virtual ~LabelTree() = default; friend bool operator==(const LabelTree& lhs, const LabelTree& rhs) { return lhs.type() == rhs.type() && lhs.equals(rhs); diff --git a/deps/geax-front-end/include/geax-front-end/ast/clause/PathPrefix.h b/deps/geax-front-end/include/geax-front-end/ast/clause/PathPrefix.h index 74610c8aa5..82925a434d 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/clause/PathPrefix.h +++ b/deps/geax-front-end/include/geax-front-end/ast/clause/PathPrefix.h @@ -26,7 +26,7 @@ namespace frontend { class PathPrefix : public AstNode { public: explicit PathPrefix(AstNodeType type) : AstNode(type) {} - ~PathPrefix() = default; + virtual ~PathPrefix() = default; }; } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/clause/QueryConjunctionType.h b/deps/geax-front-end/include/geax-front-end/ast/clause/QueryConjunctionType.h index 35c1be1d3c..257179cefa 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/clause/QueryConjunctionType.h +++ b/deps/geax-front-end/include/geax-front-end/ast/clause/QueryConjunctionType.h @@ -26,7 +26,7 @@ namespace frontend { class QueryConjunctionType : public AstNode { public: explicit QueryConjunctionType(AstNodeType nodeType) : AstNode(nodeType) {} - ~QueryConjunctionType() = default; + virtual ~QueryConjunctionType() = default; }; // class QueryConjunctionType } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/clause/RemoveItem.h b/deps/geax-front-end/include/geax-front-end/ast/clause/RemoveItem.h index a4872a7888..2839ce2f89 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/clause/RemoveItem.h +++ b/deps/geax-front-end/include/geax-front-end/ast/clause/RemoveItem.h @@ -13,7 +13,7 @@ namespace frontend { class RemoveItem : public AstNode { public: explicit RemoveItem(AstNodeType type) : AstNode(type) {} - ~RemoveItem() = default; + virtual ~RemoveItem() = default; }; } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/clause/SchemaRef.h b/deps/geax-front-end/include/geax-front-end/ast/clause/SchemaRef.h index 2b155f5767..6792a6e122 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/clause/SchemaRef.h +++ b/deps/geax-front-end/include/geax-front-end/ast/clause/SchemaRef.h @@ -26,7 +26,7 @@ namespace frontend { class SchemaRef : public AstNode { public: explicit SchemaRef(AstNodeType nodeType) : AstNode(nodeType) {} - ~SchemaRef() = default; + virtual ~SchemaRef() = default; }; // class SchemaRef } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/clause/SessionResetCommand.h b/deps/geax-front-end/include/geax-front-end/ast/clause/SessionResetCommand.h index 80cc90a3b6..02a9926457 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/clause/SessionResetCommand.h +++ b/deps/geax-front-end/include/geax-front-end/ast/clause/SessionResetCommand.h @@ -26,7 +26,7 @@ namespace frontend { class SessionResetCommand : public AstNode { public: explicit SessionResetCommand(AstNodeType type) : AstNode(type) {} - ~SessionResetCommand() = default; + virtual ~SessionResetCommand() = default; }; } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/clause/SessionSetCommand.h b/deps/geax-front-end/include/geax-front-end/ast/clause/SessionSetCommand.h index ded3949c28..49839c8d75 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/clause/SessionSetCommand.h +++ b/deps/geax-front-end/include/geax-front-end/ast/clause/SessionSetCommand.h @@ -26,7 +26,7 @@ namespace frontend { class SessionSetCommand : public AstNode { public: explicit SessionSetCommand(AstNodeType type) : AstNode(type) {} - ~SessionSetCommand() = default; + virtual ~SessionSetCommand() = default; }; } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/clause/SetItem.h b/deps/geax-front-end/include/geax-front-end/ast/clause/SetItem.h index 7a886e0900..f8819cd708 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/clause/SetItem.h +++ b/deps/geax-front-end/include/geax-front-end/ast/clause/SetItem.h @@ -26,7 +26,7 @@ namespace frontend { class SetItem : public AstNode { public: explicit SetItem(AstNodeType type) : AstNode(type) {} - ~SetItem() = default; + virtual ~SetItem() = default; }; } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/expr/BinaryOp.h b/deps/geax-front-end/include/geax-front-end/ast/expr/BinaryOp.h index 584649b011..2739a280cc 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/expr/BinaryOp.h +++ b/deps/geax-front-end/include/geax-front-end/ast/expr/BinaryOp.h @@ -26,7 +26,7 @@ namespace frontend { class BinaryOp : public Expr { public: explicit BinaryOp(AstNodeType type) : Expr(type), left_(nullptr), right_(nullptr) {} - ~BinaryOp() = default; + virtual ~BinaryOp() = default; void setLeft(Expr* left) { left_ = left; } Expr* left() const { return left_; } diff --git a/deps/geax-front-end/include/geax-front-end/ast/expr/Expr.h b/deps/geax-front-end/include/geax-front-end/ast/expr/Expr.h index 51d04789ff..36e66f7d02 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/expr/Expr.h +++ b/deps/geax-front-end/include/geax-front-end/ast/expr/Expr.h @@ -31,7 +31,7 @@ class GQLExpr; class Expr : public AstNode { public: explicit Expr(AstNodeType type) : AstNode(type) {} - ~Expr() = default; + virtual ~Expr() = default; friend bool operator==(const Expr& lhs, const Expr& rhs) { return lhs.type() == rhs.type() && lhs.equals(rhs); diff --git a/deps/geax-front-end/include/geax-front-end/ast/expr/Literal.h b/deps/geax-front-end/include/geax-front-end/ast/expr/Literal.h index abd3eba72a..13e1aa210e 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/expr/Literal.h +++ b/deps/geax-front-end/include/geax-front-end/ast/expr/Literal.h @@ -26,7 +26,7 @@ namespace frontend { class Literal : public Expr { public: explicit Literal(AstNodeType type) : Expr(type) {} - ~Literal() = default; + virtual ~Literal() = default; }; // class Literal } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/expr/UnaryOp.h b/deps/geax-front-end/include/geax-front-end/ast/expr/UnaryOp.h index c158d6251b..03a199dbfb 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/expr/UnaryOp.h +++ b/deps/geax-front-end/include/geax-front-end/ast/expr/UnaryOp.h @@ -26,7 +26,7 @@ namespace frontend { class UnaryOp : public Expr { public: explicit UnaryOp(AstNodeType type) : Expr(type), expr_(nullptr) {} - ~UnaryOp() = default; + virtual ~UnaryOp() = default; void setExpr(Expr* expr) { expr_ = expr; } Expr* expr() const { return expr_; } diff --git a/deps/geax-front-end/include/geax-front-end/ast/stmt/BindingDefinition.h b/deps/geax-front-end/include/geax-front-end/ast/stmt/BindingDefinition.h index 35c6fd3348..7c4ed039c9 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/stmt/BindingDefinition.h +++ b/deps/geax-front-end/include/geax-front-end/ast/stmt/BindingDefinition.h @@ -26,7 +26,7 @@ namespace frontend { class BindingDefinition : public AstNode { public: explicit BindingDefinition(AstNodeType nodeType) : AstNode(nodeType) {} - ~BindingDefinition() = default; + virtual ~BindingDefinition() = default; }; // class BindingDefinition } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/stmt/BindingTableExpr.h b/deps/geax-front-end/include/geax-front-end/ast/stmt/BindingTableExpr.h index cdff84fe53..f35e357806 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/stmt/BindingTableExpr.h +++ b/deps/geax-front-end/include/geax-front-end/ast/stmt/BindingTableExpr.h @@ -26,7 +26,7 @@ namespace frontend { class BindingTableExpr : public AstNode { public: explicit BindingTableExpr(AstNodeType nodeType) : AstNode(nodeType) {} - ~BindingTableExpr() = default; + virtual ~BindingTableExpr() = default; }; // class BindingTableExpr } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/stmt/EndTransaction.h b/deps/geax-front-end/include/geax-front-end/ast/stmt/EndTransaction.h index 785d0a74be..2dd8ab338b 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/stmt/EndTransaction.h +++ b/deps/geax-front-end/include/geax-front-end/ast/stmt/EndTransaction.h @@ -26,7 +26,7 @@ namespace frontend { class EndTransaction : public Transaction { public: explicit EndTransaction(AstNodeType type) : Transaction(type) {} - ~EndTransaction() = default; + virtual ~EndTransaction() = default; }; } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/stmt/LinearQueryStatement.h b/deps/geax-front-end/include/geax-front-end/ast/stmt/LinearQueryStatement.h index 1e71516c0f..0e9e95a04b 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/stmt/LinearQueryStatement.h +++ b/deps/geax-front-end/include/geax-front-end/ast/stmt/LinearQueryStatement.h @@ -26,7 +26,7 @@ namespace frontend { class LinearQueryStatement : public AstNode { public: explicit LinearQueryStatement(AstNodeType nodeType) : AstNode(nodeType) {} - ~LinearQueryStatement() = default; + virtual ~LinearQueryStatement() = default; }; // class LinearQueryStatement } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/stmt/PrimitiveDataModifyStatement.h b/deps/geax-front-end/include/geax-front-end/ast/stmt/PrimitiveDataModifyStatement.h index c318accba1..c267ab6631 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/stmt/PrimitiveDataModifyStatement.h +++ b/deps/geax-front-end/include/geax-front-end/ast/stmt/PrimitiveDataModifyStatement.h @@ -26,7 +26,7 @@ namespace frontend { class PrimitiveDataModifyStatement : public AstNode { public: explicit PrimitiveDataModifyStatement(AstNodeType nodeType) : AstNode(nodeType) {} - ~PrimitiveDataModifyStatement() = default; + virtual ~PrimitiveDataModifyStatement() = default; }; // class PrimitiveDataModifyStatement } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/stmt/ProcedureCall.h b/deps/geax-front-end/include/geax-front-end/ast/stmt/ProcedureCall.h index ee7e8a077b..b5d9ae9fef 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/stmt/ProcedureCall.h +++ b/deps/geax-front-end/include/geax-front-end/ast/stmt/ProcedureCall.h @@ -26,7 +26,7 @@ namespace frontend { class ProcedureCall : public AstNode { public: explicit ProcedureCall(AstNodeType nodeType) : AstNode(nodeType) {} - ~ProcedureCall() = default; + virtual ~ProcedureCall() = default; }; // class ProcedureCall } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/stmt/Session.h b/deps/geax-front-end/include/geax-front-end/ast/stmt/Session.h index 1bb9f6ff72..9ebd3c1a4e 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/stmt/Session.h +++ b/deps/geax-front-end/include/geax-front-end/ast/stmt/Session.h @@ -26,7 +26,7 @@ namespace frontend { class Session : public AstNode { public: explicit Session(AstNodeType type) : AstNode(type) {} - ~Session() = default; + virtual ~Session() = default; }; } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/stmt/SimpleQueryStatement.h b/deps/geax-front-end/include/geax-front-end/ast/stmt/SimpleQueryStatement.h index 04abe52889..6d5cd05c07 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/stmt/SimpleQueryStatement.h +++ b/deps/geax-front-end/include/geax-front-end/ast/stmt/SimpleQueryStatement.h @@ -26,7 +26,7 @@ namespace frontend { class SimpleQueryStatement : public AstNode { public: explicit SimpleQueryStatement(AstNodeType nodeType) : AstNode(nodeType) {} - ~SimpleQueryStatement() = default; + virtual ~SimpleQueryStatement() = default; }; // class SimpleQueryStatement } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/stmt/Statement.h b/deps/geax-front-end/include/geax-front-end/ast/stmt/Statement.h index 1108ace0e3..140af80992 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/stmt/Statement.h +++ b/deps/geax-front-end/include/geax-front-end/ast/stmt/Statement.h @@ -42,7 +42,7 @@ namespace frontend { class Statement : public AstNode { public: explicit Statement(AstNodeType nodeType) : AstNode(nodeType) {} - ~Statement() = default; + virtual ~Statement() = default; }; // class Statement } // namespace frontend diff --git a/deps/geax-front-end/include/geax-front-end/ast/stmt/Transaction.h b/deps/geax-front-end/include/geax-front-end/ast/stmt/Transaction.h index 06d1d562da..c4c840c1cf 100644 --- a/deps/geax-front-end/include/geax-front-end/ast/stmt/Transaction.h +++ b/deps/geax-front-end/include/geax-front-end/ast/stmt/Transaction.h @@ -26,7 +26,7 @@ namespace frontend { class Transaction : public AstNode { public: explicit Transaction(AstNodeType type) : AstNode(type) {} - ~Transaction() = default; + virtual ~Transaction() = default; }; } // namespace frontend diff --git a/deps/tugraph-db-browser b/deps/tugraph-db-browser index 2e36e4c06b..176dfdfc4b 160000 --- a/deps/tugraph-db-browser +++ b/deps/tugraph-db-browser @@ -1 +1 @@ -Subproject commit 2e36e4c06b1f46e58c793df78eee812ff06ff5fb +Subproject commit 176dfdfc4be6e08b1a9c983ff7b6434b9c4d0157 diff --git a/deps/tugraph-db-client-java b/deps/tugraph-db-client-java deleted file mode 160000 index 059fda24bf..0000000000 --- a/deps/tugraph-db-client-java +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 059fda24bf22bf57f7cf7033105c19bdad61e1dd diff --git a/deps/tugraph-db-management b/deps/tugraph-db-management deleted file mode 160000 index 2b2aff9fbd..0000000000 --- a/deps/tugraph-db-management +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2b2aff9fbdfc0722a5b154e17677f0208ed254d1 diff --git a/docs/en-US/source/15.contacts.md b/docs/en-US/source/15.contacts.md index fb28a5f1aa..edde354f9a 100644 --- a/docs/en-US/source/15.contacts.md +++ b/docs/en-US/source/15.contacts.md @@ -2,7 +2,7 @@ If you have any feedback or suggestions for the product, You are welcome to join the discussion or make suggestions through the following contact methods. -Official website: [www.tugraph.org](https://www.tugraph.org) +Official website: [tugraph.tech](https://tugraph.tech) Github Issue: [Issue](https://github.com/TuGraph-db/tugraph-db/issues) diff --git a/docs/en-US/source/7.client-tools/7.restful-api.md b/docs/en-US/source/7.client-tools/7.restful-api.md index b5679e97fd..fed821e48b 100644 --- a/docs/en-US/source/7.client-tools/7.restful-api.md +++ b/docs/en-US/source/7.client-tools/7.restful-api.md @@ -29,23 +29,24 @@ The user sends the login request to the server with the user name and password. - **METHOD**: POST - **REQUEST**: -| body parameter | parameter description | parameter type | necessary | -| ------- |-----------------------|-------|------------| -| userName | name of the user | string | yes | -| password | password of the user | string | yes | +| body parameter | parameter description | parameter type | necessary | +|----------------|-----------------------|-------|------------| +| user | name of the user | string | yes | +| password | password of the user | string | yes | - **RESPONSE**: -If successful, the success field in the returned response message will be set to 00 and the token will be included in data + If successful, the success field in the returned response message will be set to 00 and the token will be included in data -| body parameter | parameter description | parameter type | necessary | -| ------- |-------|--------|------| -| authorization | token | string | yes | -| default_password | whether the password is the default password | bool | yes | +| body parameter | parameter description | parameter type | +|------------------|------------------|----------------| +| jwt | token | string | +| is_admin | Whether the user is an admin | bool | +| default_password | Whether it is the default password | bool | **Example request.** ``` - {"userName" : "test", "password" : "123456"} + {"user" : "admin", "password" : "73@TuGraph"} ``` #### 2.2.2. User Logout @@ -54,10 +55,18 @@ When a user logs out, the authenticated token is deleted. and the user needs to - **URI**: /logout - **METHOD**: POST - **REQUEST**: -The http request header carries the token returned by login interface, and the body has no parameters + The http request header carries the token returned by login interface. The specific string to be filled is `Bearer ${jwt}`, where `${jwt}` is the "jwt" returned from the login interface，and the body has no parameters + +| header parameter | parameter description | parameter type | +|------------------|-----------------------|------------| +| Authorization | Bearer ${jwt} | string | - **RESPONSE**: -If successful, the success field in the returned response message will be set to 00, and data is empty + If successful, the success field in the returned response message will be set to 00, and data is + +| body parameter | parameter description | parameter type | +|------------------|--------------------|----------------| +| is_admin | Whether the user is an admin | bool | #### 2.2.3. Refresh Token If the delivered token becomes invalid, you need to invoke this interface for re-authentication. The token is valid within one hour after the first login and needs to be refreshed @@ -65,14 +74,19 @@ If the delivered token becomes invalid, you need to invoke this interface for re - **URI**: /refresh - **METHOD**: POST - **REQUEST**: -The http request header carries the token returned by login interface, and the body has no parameters + The http request header carries the token returned by login interface. The specific string to be filled is `Bearer ${jwt}`, where `${jwt}` is the "jwt" returned from the login interface，and the body has no parameters + +| header parameter | parameter description | parameter type | +|------------------|-----------------------|------------| +| Authorization | Bearer ${jwt} | string | - **RESPONSE**: -If successful, the success field in the returned response message will be set to 00, and the token will be included in data + If successful, the success field in the returned response message will be set to 00, and the token will be included in data -| body parameter | parameter description | parameter type | necessary | -| ------- |------|----------------|------------| -| authorization | token | string | yes | +| body parameter | parameter description | parameter type | +|----------------|------|----------------| +| jwt | token | string | +| is_admin | Whether the user is an admin | bool | #### 2.2.4. Call Cypher User manipulation of data and models in tugraph requires calling the cypher interface and is initiated through the standard cypher query language @@ -81,13 +95,17 @@ User manipulation of data and models in tugraph requires calling the cypher inte - **METHOD**: POST - **REQUEST**: +| header parameter | parameter description | parameter type | +|------------------|-----------------------|------------| +| Authorization | Bearer ${jwt} | string | + | body parameter | parameter description | parameter type | necessary | | ------- |------------------|---------|-----------| | graph | the name of the subgraph to be queried | string | yes | | script | query statement | string | yes | - **RESPONSE**: -If successful, the success field in the returned response message will be set to 00, and the query results will be included in data + If successful, the success field in the returned response message will be set to 00, and the query results will be included in data | body parameter | parameter description | parameter type | necessary | | ------- |------|----------------|------------| @@ -97,245 +115,4 @@ If successful, the success field in the returned response message will be set to ``` {"script" : "Match (n) return n", "graph" : "default"} -``` - -#### 2.2.5. Upload File -This interface is used to upload local files to the TuGraph machine. You can upload text/binary files, large files, and small files. For large files, the client must split the files, and each file fragment must not be larger than 1MB. Parameters Begin-Pos and Size correspond to the offset and fragment size of this fragment in the complete file. The parameter must be placed in the header of the http request, and the request body contains only the file fragment content. The request header of this interface contains more than token parameters - -- **URI**: /upload_file -- **METHOD**: POST -- **REQUEST**: - -| header parameter | parameter description | parameter type | necessary | -|------------------|--------------------------------------|---------|-------| -| File-Name | the name of the file | string | yes | -| Begin-Pos | Offset of the start position of the current file fragment within the file | string | yes | -| Size | the current file fragment size | string | yes | - -- **RESPONSE**: -If successful, the success field in the returned response message will be set to 00 - -#### 2.2.6. Check File -this interface is used to check the correctness of uploaded files. If the check succeeds, the system returns a success message when the same file is uploaded again - -- **URI**: /check_file -- **METHOD**: POST -- **REQUEST**: - -| body parameter | parameter description | parameter type | necessary | -| ------- |-----------------------------|----------------|-----------| -| fileName | the name of the file | string | yes | -| checkSum | the checksum of the file | string | yes when flag set to "1" | -| fileSize | the size of the file | string | yes when flag set to "2" | -| flag | If flag is "1", check md5. If flag is "2"，check file size | string | yes | - -- **RESPONSE**: -If successful, the success field in the returned response message will be set to 00 - -| body parameter | parameter description | parameter type | necessary | -| ------- |------|------|------| -| pass | true on success, false otherwise | bool | yes | - -**Example request.** - -``` -{"fileName" : "test.csv", "checkSum" : "$MD5", "flag" : “1”} -``` - -#### 2.2.7. Delete Cached File -The admin user can delete all the files uploaded by anyone. Other users can delete their own files. You can delete a file with a specified name, a file uploaded by a specified user, or all files - -- **URI**: /clear_cache -- **METHOD**: POST -- **REQUEST**: - -| body parameter | parameter description | parameter type | necessary | -| ------- |-----------------------|--------|--------| -| fileName | the name of the file | string | yes when flag set to "0" | -| userName | the name of the user | string | yes when flag set to "1" | -| flag | When flag is set to 0, the file specified by fileName is deleted; when flag is set to 1, all files uploaded by userName are deleted; when flag is set to 2, all files are deleted | string | yes | - -- **RESPONSE**: -If successful, the success field in the returned response message will be set to 00 - -**Example request.** - -``` -{"fileName" : "test.csv", "userName" : "test", "flag" : “1”} -``` - -#### 2.2.8. Import Schema -This interface can create a schema model based on the schema information specified by description parameter. For details about the schema format, refer to data-import.md - -- **URI**: /import_schema -- **METHOD**: POST -- **REQUEST**: - -| body parameter | parameter description | parameter type | necessary | -| ------- |-----------------------|----------------|-----------| -| graph | name of the subgraph | string | yes | -| description | schema infomation | json string | yes | - -- **RESPONSE**: - If successful, the success field in the returned response message will be set to 00 - -**Example request.** - -``` -{ - "graph": "test_graph", - "description": { - "schema": [{ - "label": "Person", - "type": "VERTEX", - "primary": "name", - "properties": [{ - "name": "name", - "type": "STRING" - }, { - "name": "birthyear", - "type": "INT16", - "optional": true - }, { - "name": "phone", - "type": "INT16", - "unique": true, - "index": true - }] - }, { - "label": "City", - "type": "VERTEX", - "primary": "name", - "properties": [{ - "name": "name", - "type": "STRING" - }] - }, { - "label": "Film", - "type": "VERTEX", - "primary": "title", - "properties": [{ - "name": "title", - "type": "STRING" - }] - }, { - "label": "HAS_CHILD", - "type": "EDGE" - }, { - "label": "MARRIED", - "type": "EDGE" - }, { - "label": "BORN_IN", - "type": "EDGE", - "properties": [{ - "name": "weight", - "type": "FLOAT", - "optional": true - }] - }, { - "label": "DIRECTED", - "type": "EDGE" - }, { - "label": "WROTE_MUSIC_FOR", - "type": "EDGE" - }, { - "label": "ACTED_IN", - "type": "EDGE", - "properties": [{ - "name": "charactername", - "type": "STRING" - }] - }, { - "label": "PLAY_IN", - "type": "EDGE", - "properties": [{ - "name": "role", - "type": "STRING", - "optional": true - }], - "constraints": [ - ["Person", "Film"] - ] - }] - } -} -``` - -#### 2.2.9. Import Data -This interface allows users to specify uploaded and verified files as data files and import them to the subgraph specified by the graph parameter. The import process is asynchronous, and the server returns a task id after receiving the import request - -- **URI**: /import_data -- **METHOD**: POST -- **REQUEST**: - -| body parameter | parameter description | parameter type | necessary | -| ------- |---------------------------|--------|-----| -| graph | name of the subgraph | string | yes | -| schema | schema infomation | json string | yes | -| delimiter | column delimiter | string | yes | -| continueOnError | Whether to skip the error and continue when an error occurs | boolean | no | -| skipPackages | number of packets skipped | string | no | -| taskId | used to restart the failed task | string | no | -| flag | If flag is set to 1, the data file is deleted after the import is successful | string | no | -| other | other parameter | string | no | - -- **RESPONSE**: - If successful, the success field in the returned response message will be set to 00 - -| body parameter | parameter description | parameter type | necessary | -| ------- |---------------------------------------|--------|------------| -| taskId | task id is used to find a import task | string | yes | - -**Example request.** - -``` -{ - "graph": "default", //导入的子图名称 - "delimiter": ",", //数据分隔符 - "continueOnError": true, //遇到错误时是否跳过错误数据并继续导入 - "skipPackages": “0”, //跳过的包个数 - "flag" : "1", - "schema": { - "files": [{ - "DST_ID": "Film", //终点label类型 - "SRC_ID": "Person", //起点label类型 - "columns": [ //数据格式说明 - "SRC_ID", //起点id - "DST_ID", //终点id - "SKIP", //表示跳过此列数据 - "charactername" //属性名称 - ], - "format": "CSV", //数据文件格式类型,支持csv和json - "path": "acted_in.csv", //数据文件名称 - "header": 0, //数据从第几行开始 - "label": "ACTED_IN" //边的类型 - }] - } -} -``` - -#### 2.2.10. Import Progress Query -This interface is used to query the execution progress of a import task - -- **URI**: /import_progress -- **METHOD**: POST -- **REQUEST**: - -| body parameter | parameter description | parameter type | necessary | -| ------- |-------------------------------------------|---------|----| -| taskId | task id returned by import_data interface | string | 是 | - -- **RESPONSE**: - If successful, the success field in the returned response message will be set to 00 - -| body parameter | parameter description | parameter type | necessary | -| ------- |--------------------------------------|----------------|-----------| -| success | whether import is successful | boolean | yes | -| reason | reason of the import failure | string | yes if success is false | -| progress | import progress at the current time | string | yes | - -**Example request.** - -``` -{"taskId" : "$taskId"} ``` \ No newline at end of file diff --git a/docs/zh-CN/source/15.contacts.md b/docs/zh-CN/source/15.contacts.md index e003ef4098..787d8c1e49 100644 --- a/docs/zh-CN/source/15.contacts.md +++ b/docs/zh-CN/source/15.contacts.md @@ -2,7 +2,7 @@ 您有任何对产品的意见和建议，欢迎通过以下联系方式加入讨论，或提出建议。 -官网: [www.tugraph.org](https://www.tugraph.org) +官网: [tugraph.tech](https://tugraph.tech) Github Issue (错误反馈、功能讨论) [Issue](https://github.com/TuGraph-db/tugraph-db/issues) diff --git a/docs/zh-CN/source/7.client-tools/7.restful-api.md b/docs/zh-CN/source/7.client-tools/7.restful-api.md index beb44d41aa..e8f090a1c8 100644 --- a/docs/zh-CN/source/7.client-tools/7.restful-api.md +++ b/docs/zh-CN/source/7.client-tools/7.restful-api.md @@ -29,23 +29,24 @@ TuGraph HTTP Server 接收json格式的请求，经过鉴权后开始提取请 - **METHOD**: POST - **REQUEST**: -| body参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |------|-------|------------| -| userName | 用户名 | 字符串类型 | 是 | -| password | 用户密码 | 字符串类型 | 是 | +| body参数 | 参数说明 | 参数类型 | 是否必填 | +| ------ |------|-------|------------| +| user | 用户名 | 字符串类型 | 是 | +| password | 用户密码 | 字符串类型 | 是 | - **RESPONSE**: -如果成功，返回的响应信息中success为00，data中包含令牌 + 如果成功，返回的响应信息中success为00，data中包含令牌 -| body参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |------|-------|------------| -| authorization | 令牌 | 字符串类型 | 是 | -| default_password | 默认密码 | 布尔类型 | 是 | +| body参数 | 参数说明 | 参数类型 | +|------------------|--------------------|------------| +| jwt | 令牌 | 字符串类型 | +| is_admin | 是否是admin用户 | 布尔类型 | +| default_password | 默认密码 | 布尔类型 | **Example request.** ``` - {"userName" : "test", "password" : "123456"} + {"user" : "admin", "password" : "73@TuGraph"} ``` #### 2.2.2. 用户登出 @@ -53,11 +54,19 @@ TuGraph HTTP Server 接收json格式的请求，经过鉴权后开始提取请 - **URI**: /logout - **METHOD**: POST -- **REQUEST**: - http request header中携带调用login接口时返回的token，body中没有参数 + - **REQUEST**: + http request header中携带调用login接口时返回的token，具体填写的字符串是`Bearer ${jwt}`，`${jwt}`就是login接口返回的jwt，body中没有参数 + +| header参数 | 参数说明 | 参数类型 | +|------------------|---------------|------------| +| Authorization | Bearer ${jwt} | 字符串类型 | - **RESPONSE**: -如果成功，返回的响应信息中success为00，data为空 + 如果成功，返回的响应信息中success为00，data中包含 + +| body参数 | 参数说明 | 参数类型 | +|------------------|--------------------|------------| +| is_admin | 是否是admin用户 | 布尔类型 | #### 2.2.3. 身份刷新 已下发的token失效后，需要调用本接口重新认证。后端验证token合法性。token在初次登录后，1小时内有效，过期需要刷新 @@ -65,14 +74,19 @@ TuGraph HTTP Server 接收json格式的请求，经过鉴权后开始提取请 - **URI**: /refresh - **METHOD**: POST - **REQUEST**: - http request header中携带调用login接口时返回的token，body中没有参数 + http request header中携带调用login接口返回的token，具体填写的字符串是`Bearer ${jwt}`，`${jwt}`就是login接口返回的jwt，body中没有参数 + +| header参数 | 参数说明 | 参数类型 | +|------------------|---------------|------------| +| Authorization | Bearer ${jwt} | 字符串类型 | - **RESPONSE**: 如果成功，返回的响应信息中success为00，data中包含令牌 -| body参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |------|-------|------------| -| authorization | 令牌 | 字符串类型 | 是 | +| body参数 | 参数说明 | 参数类型 | +|------------------|--------------------|------------| +| jwt | 令牌 | 字符串类型 | +| is_admin | 是否是admin用户 | 布尔类型 | #### 2.2.4. 调用cypher 用户对TuGraph的增删改查请求需要调用cypher接口，并通过标准的cypher查询语言发起 @@ -81,6 +95,10 @@ TuGraph HTTP Server 接收json格式的请求，经过鉴权后开始提取请 - **METHOD**: POST - **REQUEST**: +| header参数 | 参数说明 | 参数类型 | +|------------------|---------------|------------| +| Authorization | Bearer ${jwt} | 字符串类型 | + | body参数 | 参数说明 | 参数类型 | 是否必填 | | ------- |----------|-------|------------| | graph | 查询的子图名称 | 字符串类型 | 是 | @@ -97,244 +115,4 @@ TuGraph HTTP Server 接收json格式的请求，经过鉴权后开始提取请 ``` {"script" : "Match (n) return n", "graph" : "default"} -``` - -#### 2.2.5. 上传文件 -接口用于将本地文件上传至TuGraph所在机器。可以上传文本文件，二进制文件，可以上传大文件，也可以上传小文件，对于大文件，客户端在上传时应该对文件做切分，每个文件分片不大于1MB，参数Begin-Pos和Size对应本次分片在完整文件中的偏移量与分片大小。参数需要放在http请求的报文头，请求内容对应文件分片内容。本接口请求头不止有token参数 - -- **URI**: /upload_file -- **METHOD**: POST -- **REQUEST**: - -| header参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |-------------|------------------|------------| -| File-Name | 文件名称 | 字符串类型 | 是 | -| Begin-Pos | 开始位置在文件内的偏移 | 可以转成size_t类型的字符串 | 是 | -| Size | 本次请求文件分片大小 | 可以转成size_t类型的字符串 | 是 | - -- **RESPONSE**: - 如果成功，返回的响应信息中success为00 - -#### 2.2.6. 检查文件 -本接口用于检查已上传文件正确性，如果成功通过检查，再次上传同一文件时，直接返回成功 - -- **URI**: /check_file -- **METHOD**: POST -- **REQUEST**: - -| body参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |----------------|---------------|-------------| -| fileName | 文件名称 | 字符串类型 | 是 | -| checkSum | 文件对应md5值 | 可以转成int类型的字符串 | flag为"1"时必填 | -| fileSize | 文件长度(以字节计算) | 可以转成int类型的字符串 | flag为"2"时必填 | -| flag | 标记位，flag为1时校验md5。flag为2时校验文件长度 | 可以转成int类型的字符串 | 是 | - -- **RESPONSE**: - 如果成功，返回的响应信息中success为00 - -| body参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |------|--------|------------| -| pass | 检查成功为true，否则为false | bool类型 | 是 | - -**Example request.** - -``` -{"fileName" : "test.csv", "checkSum" : "$MD5", "flag" : “1”} -``` - -#### 2.2.7. 清理缓存文件 -admin用户可以删除所有用户上传的文件，其他用户可以删除自己的上传的文件，可以删除指定名称的文件，指定用户上传的文件，也可以删除所有文件 - -- **URI**: /clear_cache -- **METHOD**: POST -- **REQUEST**: - -| body参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |-------------|---------------|-------------| -| fileName | 文件名称 | 字符串类型 | flag为"0"时必填 | -| userName | 用户名称 | 字符串类型 | flag为"1"时必填 | -| flag | 标记位，flag为0时删除fileName指定文件, flag为1时删除userName指定用户已经上传的所有文件，flag为2时删除所有用户上传的文件 | 可以转成int类型的字符串 | 是 | - -- **RESPONSE**: - 如果成功，返回中success为00 - -**Example request.** - -``` -{"fileName" : "test.csv", "userName" : "test", "flag" : “1”} -``` - -#### 2.2.8. 导入schema -本接口可以根据用户指定的schema信息创建schema模型，schema的详细格式信息请参考data-import.md - -- **URI**: /import_schema -- **METHOD**: POST -- **REQUEST**: - -| body参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |------------|---------------|-------------| -| graph | 子图名称 | 字符串 | 是 | -| description | schema描述信息 | json字符串 | 是 | - -- **RESPONSE**: - 如果成功，返回中success为00 - -**Example request.** - -``` -{ - "graph": "test_graph", - "description": { - "schema": [{ - "label": "Person", - "type": "VERTEX", - "primary": "name", - "properties": [{ - "name": "name", - "type": "STRING" - }, { - "name": "birthyear", - "type": "INT16", - "optional": true - }, { - "name": "phone", - "type": "INT16", - "unique": true, - "index": true - }] - }, { - "label": "City", - "type": "VERTEX", - "primary": "name", - "properties": [{ - "name": "name", - "type": "STRING" - }] - }, { - "label": "Film", - "type": "VERTEX", - "primary": "title", - "properties": [{ - "name": "title", - "type": "STRING" - }] - }, { - "label": "HAS_CHILD", - "type": "EDGE" - }, { - "label": "MARRIED", - "type": "EDGE" - }, { - "label": "BORN_IN", - "type": "EDGE", - "properties": [{ - "name": "weight", - "type": "FLOAT", - "optional": true - }] - }, { - "label": "DIRECTED", - "type": "EDGE" - }, { - "label": "WROTE_MUSIC_FOR", - "type": "EDGE" - }, { - "label": "ACTED_IN", - "type": "EDGE", - "properties": [{ - "name": "charactername", - "type": "STRING" - }] - }, { - "label": "PLAY_IN", - "type": "EDGE", - "properties": [{ - "name": "role", - "type": "STRING", - "optional": true - }], - "constraints": [ - ["Person", "Film"] - ] - }] - } -} -``` - -#### 2.2.9. 导入数据 -本接口允许用户指定已经通过上传，校验的文件作为数据文件，按照schema参数描述的配置信息，导入到graph参数指定的子图中。导入过程是异步的，server在接收到导入请求后，返回一个任务id - -- **URI**: /import_data -- **METHOD**: POST -- **REQUEST**: - -| body参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |-------|---------|----| -| graph | 子图名称 | 字符串类型 | 是 | -| schema | 导入schema描述 | json字符串 | 是 | -| delimiter | 分隔符 | 字符串类型 | 是 | -| continueOnError | 单行数据出错是否跳过错误并继续 | boolean类型 | 否 | -| skipPackages | 跳过的包个数 | 可以转成int类型的字符串 | 否 | -| taskId | 任务id，用于重启出错的任务 | 字符串类型 | 否 | -| flag | 标记位，flag为1时导入成功将删除数据文件 | 可以转成int类型的字符串 | 否 | -| other | 其他参数 | 可以转成int类型的字符串 | 否 | - -- **RESPONSE**: - 如果成功，返回的响应信息中success为00 - -| body参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |------|--------|------------| -| taskId | 任务编号 | 字符串类型 | 是 | - -**Example request.** - -``` -{ - "graph": "default", //导入的子图名称 - "delimiter": ",", //数据分隔符 - "continueOnError": true, //遇到错误时是否跳过错误数据并继续导入 - "skipPackages": “0”, //跳过的包个数 - "flag" : "1", - "schema": { - "files": [{ - "DST_ID": "Film", //终点label类型 - "SRC_ID": "Person", //起点label类型 - "columns": [ //数据格式说明 - "SRC_ID", //起点id - "DST_ID", //终点id - "SKIP", //表示跳过此列数据 - "charactername" //属性名称 - ], - "format": "CSV", //数据文件格式类型,支持csv和json - "path": "acted_in.csv", //数据文件名称 - "header": 0, //数据从第几行开始 - "label": "ACTED_IN" //边的类型 - }] - } -} -``` - -#### 2.2.10. 导入进度查询 -本接口用于查询导入任务的执行进度 - -- **URI**: /import_progress -- **METHOD**: POST -- **REQUEST**: - -| body参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |----------------------|---------|----| -| taskId | import_data接口返回的任务id | 字符串类型 | 是 | -- **RESPONSE**: - 如果成功，返回的响应信息中success为00 - -| body参数 | 参数说明 | 参数类型 | 是否必填 | -| ------- |---------|------------------|---------| -| success | 是否成功导入 | boolean类型 | 是 | -| reason | 导入失败原因 | 字符串类型 | success为false时必填 | -| progress | 当前导入进度 | 可以转成double类型的字符串 | 是 | - -**Example request.** - -``` -{"taskId" : "$taskId"} ``` \ No newline at end of file diff --git a/include/fma-common/rw_lock.h b/include/fma-common/rw_lock.h index b3de444991..b3009d5a1a 100644 --- a/include/fma-common/rw_lock.h +++ b/include/fma-common/rw_lock.h @@ -17,6 +17,7 @@ #include "fma-common/cache_aligned_vector.h" #include "fma-common/thread_id.h" #include "fma-common/utils.h" +#include "fma-common/string_formatter.h" namespace fma_common { @@ -29,6 +30,7 @@ enum class LockStatus { class InvalidThreadIdError : public std::runtime_error { public: InvalidThreadIdError() : std::runtime_error("Invalid thread id.") {} + explicit InvalidThreadIdError(const std::string& msg) : std::runtime_error(msg) {} }; // RWLock using thread-local storage. @@ -51,7 +53,10 @@ class InterruptableTLSRWLock { DISABLE_MOVE(InterruptableTLSRWLock); void ThrowOnInvalidTID(int tid) { - if (tid < 0 || tid >= FMA_MAX_THREADS) throw InvalidThreadIdError(); + if (tid < 0 || tid >= FMA_MAX_THREADS) { + throw InvalidThreadIdError(FMA_FMT( + "Invalid thread id, tid:{}, FMA_MAX_THREADS:{}", tid, FMA_MAX_THREADS)); + } } #define RETURN_IF_INTERRUPTED() \ diff --git a/include/fma-common/thread_id.h b/include/fma-common/thread_id.h index 4c7332365f..452cd9a408 100644 --- a/include/fma-common/thread_id.h +++ b/include/fma-common/thread_id.h @@ -14,7 +14,7 @@ #include "fma-common/assert.h" namespace fma_common { -static const int FMA_MAX_THREADS = 480; +static const int FMA_MAX_THREADS = 65000; class ThreadIdAssigner { static const int N = FMA_MAX_THREADS; diff --git a/include/lgraph/lgraph_types.h b/include/lgraph/lgraph_types.h index 9094d507c2..4fd07fce27 100644 --- a/include/lgraph/lgraph_types.h +++ b/include/lgraph/lgraph_types.h @@ -1198,6 +1198,83 @@ struct FieldData { /** @brief Query if this object is float vector*/ bool IsFloatVector() const { return type == FieldType::FLOAT_VECTOR; } + struct Hash { + size_t operator()(const FieldData& fd) const { + switch (fd.type) { + case FieldType::NUL: + return 0; + case FieldType::BOOL: + return std::hash()(fd.AsBool()); + case FieldType::INT8: + return std::hash()(fd.AsInt8()); + case FieldType::INT16: + return std::hash()(fd.AsInt16()); + case FieldType::INT32: + return std::hash()(fd.AsInt32()); + case FieldType::INT64: + return std::hash()(fd.AsInt64()); + case FieldType::FLOAT: + return std::hash()(fd.AsFloat()); + case FieldType::DOUBLE: + return std::hash()(fd.AsDouble()); + case FieldType::DATE: + return std::hash()(fd.AsDate().DaysSinceEpoch()); + case FieldType::DATETIME: + return std::hash()(fd.AsDateTime().MicroSecondsSinceEpoch()); + case FieldType::STRING: + return std::hash()(fd.AsString()); + case FieldType::BLOB: + return std::hash()(fd.AsBlob()); + case FieldType::POINT: + { + switch (fd.GetSRID()) { + case ::lgraph_api::SRID::WGS84: + return std::hash()(fd.AsWgsPoint().AsEWKB()); + case ::lgraph_api::SRID::CARTESIAN: + return std::hash()(fd.AsCartesianPoint().AsEWKB()); + default: + THROW_CODE(InputError, "unsupported spatial srid"); + } + } + case FieldType::LINESTRING: + { + switch (fd.GetSRID()) { + case ::lgraph_api::SRID::WGS84: + return std::hash()(fd.AsWgsLineString().AsEWKB()); + case ::lgraph_api::SRID::CARTESIAN: + return std::hash()(fd.AsCartesianLineString().AsEWKB()); + default: + THROW_CODE(InputError, "unsupported spatial srid"); + } + } + case FieldType::POLYGON: + { + switch (fd.GetSRID()) { + case ::lgraph_api::SRID::WGS84: + return std::hash()(fd.AsWgsPolygon().AsEWKB()); + case ::lgraph_api::SRID::CARTESIAN: + return std::hash()(fd.AsCartesianPolygon().AsEWKB()); + default: + THROW_CODE(InputError, "unsupported spatial srid"); + } + } + case FieldType::SPATIAL: + { + switch (fd.GetSRID()) { + case ::lgraph_api::SRID::WGS84: + return std::hash()(fd.AsWgsSpatial().AsEWKB()); + case ::lgraph_api::SRID::CARTESIAN: + return std::hash()(fd.AsCartesianSpatial().AsEWKB()); + default: + THROW_CODE(InputError, "unsupported spatial srid"); + } + } + default: + throw std::runtime_error("Unhandled data type, probably corrupted data."); + } + } + }; + private: /** @brief Query if 't' is BLOB or STRING */ static inline bool IsBufType(FieldType t) { @@ -1296,8 +1373,8 @@ struct VectorIndexSpec { std::string index_type; int dimension; std::string distance_type; - int hnsm_m; - int hnsm_ef_construction; + int hnsw_m; + int hnsw_ef_construction; }; struct EdgeUid { diff --git a/include/lgraph/olap_on_db.h b/include/lgraph/olap_on_db.h index aecd5b03db..175ed07dbc 100644 --- a/include/lgraph/olap_on_db.h +++ b/include/lgraph/olap_on_db.h @@ -515,6 +515,9 @@ class OlapOnDB : public OlapBase { for (size_t vi = partition_begin; vi < partition_end; vi++) { if (vi % 64 == 0 && ShouldKillThisTask(task_ctx)) break; vit.Goto(vi); + if (!vit.IsValid() || vit.GetNumOutEdges() == 0) { + continue; + } for (auto eit = vit.GetOutEdgeIterator(); eit.IsValid(); eit.Next()) { size_t dst = eit.GetDst(); EdgeData edata; @@ -982,7 +985,7 @@ class OlapOnDB : public OlapBase { } Init(txn.GetNumVertices()); - if (flags_ & SNAPSHOT_IDMAPPING) { + /*if (flags_ & SNAPSHOT_IDMAPPING) { Construct(); } else { if ((out_edge_filter == nullptr) && (flags_ & SNAPSHOT_PARALLEL) && txn_.IsReadOnly()) { @@ -990,7 +993,8 @@ class OlapOnDB : public OlapBase { } else { ConstructWithVid(); } - } + }*/ + Construct(); } /** @@ -1525,18 +1529,21 @@ class OlapOnDB : public OlapBase { if (output_filter != nullptr && !output_filter(i, vertex_data[i])) { continue; } - auto vit = txn_.GetVertexIterator(OriginalVid(i)); - auto vit_label = vit.GetLabel(); - auto primary_field = txn_.GetVertexPrimaryField(vit_label); - auto field_data = vit.GetField(primary_field); - json curJson; - curJson["vid"] = OriginalVid(i); - curJson["label"] = vit_label; - curJson["primary_field"] = primary_field; - curJson["field_data"] = field_data.ToString(); - curJson["result"] = vertex_data[i]; - auto content = curJson.dump() + "\n"; - fout.Write(content.c_str(), content.size()); + auto vit = txn_.GetVertexIterator(); + vit.Goto(OriginalVid(i)); + if (vit.IsValid()) { + auto vit_label = vit.GetLabel(); + auto primary_field = txn_.GetVertexPrimaryField(vit_label); + auto field_data = vit.GetField(primary_field); + json curJson; + curJson["vid"] = OriginalVid(i); + curJson["label"] = vit_label; + curJson["primary_field"] = primary_field; + curJson["field_data"] = field_data.ToString(); + curJson["result"] = vertex_data[i]; + auto content = curJson.dump() + "\n"; + fout.Write(content.c_str(), content.size()); + } } } diff --git a/learn/docker/dockerfile b/learn/docker/dockerfile index e9ba459e88..6a813ee130 100644 --- a/learn/docker/dockerfile +++ b/learn/docker/dockerfile @@ -237,8 +237,8 @@ RUN sed -i '3 s/-lgomp/-l:libgomp.a/' /usr/local/lib64/libgomp.spec # RUN sed -i '3 s/-lgomp/-l:libgomp.a/' /usr/lib/gcc/x86_64-linux-gnu/7/libgomp.spec ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre \ - LD_LIBRARY_PATH=/usr/local/lgraph/lib64:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/lib/amd64/:$LD_LIBRARY_PATH \ - PYTHONPATH=/usr/local/lgraph/lib64:/usr/local/lib64:$PYTHONPATH \ + LD_LIBRARY_PATH=/usr/local/lib64/lgraph:/usr/local/lib64:/usr/local/lib:/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/lib/amd64/:$LD_LIBRARY_PATH \ + PYTHONPATH=/usr/local/lib64/lgraph:/usr/local/lib64:$PYTHONPATH \ PATH=/opt/apache-maven-3.8.7/bin:/usr/lib/jvm/java-8-openjdk-amd64/jre/bin:$PATH diff --git a/procedures/CMakeLists.txt b/procedures/CMakeLists.txt index 0a2852a309..b90c0c3e1e 100644 --- a/procedures/CMakeLists.txt +++ b/procedures/CMakeLists.txt @@ -46,6 +46,15 @@ function(add_v2procedure APP) LIBRARY_OUTPUT_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/algo") endfunction() +function(add_standalone_from_community APP) + add_executable(${APP}_standalone community/${APP}_core.cpp community/${APP}_standalone.cpp ${LGRAPH_API_SRC}) + target_link_libraries(${APP}_standalone ${Boost_LIBRARIES} libstdc++fs.a libgomp.a dl) + set_target_properties( ${APP}_standalone PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/algo") + target_include_directories(${APP}_standalone PUBLIC ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/src) +endfunction() + add_standalone(apsp) add_standalone(bfs) add_standalone(pagerank) @@ -81,6 +90,7 @@ add_standalone(wlpa) add_standalone(subgraph_isomorphism) add_standalone(sybilrank) add_standalone(leiden) +add_standalone_from_community(khop) add_embed(apsp) add_embed(bfs) diff --git a/procedures/community/README.md b/procedures/community/README.md new file mode 100644 index 0000000000..17be30e3ae --- /dev/null +++ b/procedures/community/README.md @@ -0,0 +1,14 @@ +# community procedures + + In this directory, The procedures are contribution from the commmunity. + +**Note: Please be aware that the compilation of the procedures mentioned here has not been verified on GitHub CI. Ensure that you compile your procedures successfully before submitting a pull request (PR)** + +## K-hop Algorithm + The K-hop algorithm is used to find all nodes within K hops from a given starting node in a graph. It is particularly useful in social network analysis, recommendation systems, and network topology analysis. + +### Contributor + This algorithm was contributed by [Yingqi Zhao](https://github.com/AidenPearce-ZYQ), [Junjie Wang](https://github.com/iwanttoknowwhy) and [Haibo Zheng](https://github.com/ZhengHeber). + +### Usage + For detailed usage, please refer to the TuGraph-DB OLAP C++ API documentation. After installation, you can run `khop_standalone --help` for additional information. \ No newline at end of file diff --git a/procedures/community/khop_core.cpp b/procedures/community/khop_core.cpp new file mode 100644 index 0000000000..e6420c5a81 --- /dev/null +++ b/procedures/community/khop_core.cpp @@ -0,0 +1,57 @@ +/* + * Copyright 2024 Yingqi Zhao + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lgraph/olap_base.h" + +using namespace lgraph_api; +using namespace lgraph_api::olap; + +size_t k_hop(OlapBase& graph, size_t root_vid, ParallelVector& result, size_t k) { + size_t root = root_vid; + auto active_in = graph.AllocVertexSubset(); + active_in.Add(root); + auto active_out = graph.AllocVertexSubset(); + auto parent = graph.AllocVertexArray(); + parent.Fill(0); + parent[root] = root; + size_t num_activations = 1; + size_t discovered_vertices = 0, j = 0; + for (size_t ii = 0; ii < k; ii++) { + active_out.Clear(); + num_activations = graph.ProcessVertexActive( + [&](size_t vi) { + size_t num_activations = 0; + for (auto& edge : graph.OutEdges(vi)) { + size_t dst = edge.neighbour; + if (parent[dst] == 0) { + auto lock = graph.GuardVertexLock(dst); + if (parent[dst] == 0) { + parent[dst] = vi; + num_activations += 1; + active_out.Add(dst); + result[j++] = dst; + } + } + } + return num_activations; + }, + active_in); + printf("activates(%lu) <= %lu \n", ii+1, num_activations); + discovered_vertices += num_activations; + active_in.Swap(active_out); + } + return discovered_vertices; +} diff --git a/procedures/community/khop_standalone.cpp b/procedures/community/khop_standalone.cpp new file mode 100644 index 0000000000..47a1be5bd9 --- /dev/null +++ b/procedures/community/khop_standalone.cpp @@ -0,0 +1,99 @@ +/* + * Copyright 2024 Yingqi Zhao + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @brief The k-hop algorithm. + * + * @param root Identifier of root node. + * @param value_k Number of search layers (value of k in K-hop algorithm). + * @return Number of nodes in K-hop. + */ +#include "olap/olap_on_disk.h" +#include "tools/json.hpp" +using namespace lgraph_api; +using namespace lgraph_api::olap; +using json = nlohmann::json; + +class MyConfig : public ConfigBase { + public: + std::string root = "0"; + std::string name = std::string("khop"); + size_t value_k = 3; + void AddParameter(fma_common::Configuration& config) { + ConfigBase::AddParameter(config); + config.Add(root, "root", true).Comment("Identifier of the root node."); + config.Add(value_k, "value_k", true).Comment( + "Number of search layers(value of k in K-hop algorithm)."); + } + void Print() { + ConfigBase::Print(); + std::cout << " name: " << name << std::endl; + std::cout << " root: " << name << std::endl; + std::cout << " value_k: " << value_k << std::endl; + } + MyConfig(int& argc, char**& argv) : ConfigBase(argc, argv) { + fma_common::Configuration config; + AddParameter(config); + config.ExitAfterHelp(true); + config.ParseAndFinalize(argc, argv); + Print(); + } +}; + +extern size_t k_hop(OlapBase& graph, size_t root_vid, + ParallelVector& result, size_t k); + +int main(int argc, char** argv) { + double start_time; + MemUsage memUsage; + memUsage.startMemRecord(); + start_time = get_time(); + MyConfig config(argc, argv); + OlapOnDisk graph; + graph.Load(config, INPUT_SYMMETRIC); + size_t root_vid; + auto result = graph.AllocVertexArray(); + result.Fill(0); + if (config.id_mapping) + root_vid = graph.hash_list_.find(config.root); + else + root_vid = std::stoi(config.root); + size_t value_k = config.value_k; + memUsage.print(); + memUsage.reset(); + auto prepare_cost = get_time()- start_time; + printf("prepare_cost = %.2lf(s)\n", prepare_cost); + + start_time = get_time(); + size_t count_result = k_hop(graph, root_vid, result, value_k); + memUsage.print(); + memUsage.reset(); + auto core_cost = get_time()- start_time; + + start_time = get_time(); + if (config.output_dir != "") { + graph.Write(config, result, graph.NumVertices(), config.name); + } + printf("\n================\n"); + printf("Find %lu vertexes in %lu-hop from node NO.%lu", count_result, value_k, root_vid); + printf("\n================\n"); + auto output_cost = get_time()- start_time; + + printf("core_cost = %.2lf(s)\n", core_cost); + printf("output_cost = %.2lf(s)\n", output_cost); + printf("total_cost = %.2lf(s)\n", prepare_cost + core_cost + output_cost); + printf("DONE.\n"); + return 0; +} diff --git a/procedures/custom_cpp/louvain_procedure.cpp b/procedures/custom_cpp/louvain_procedure.cpp index a9d07d3667..329b500e47 100644 --- a/procedures/custom_cpp/louvain_procedure.cpp +++ b/procedures/custom_cpp/louvain_procedure.cpp @@ -66,28 +66,34 @@ extern "C" bool Process(GraphDB& db, const std::string& request, std::string& re json communityNode; for (size_t i = 0; i < olapondb.NumVertices(); i++) { if (label[i]) { - auto vit = txn.GetVertexIterator(i, false); - auto vit_label = vit.GetLabel(); - auto primary_field = txn.GetVertexPrimaryField(vit_label); - auto field_data = vit.GetField(primary_field); - curNode["vid"] = i; - curNode["label"] = vit_label; - curNode["primary_field"] = primary_field; - curNode["field_data"] = field_data.ToString(); + auto vit = txn.GetVertexIterator(); + vit.Goto(i); + if (vit.IsValid()) { + auto vit_label = vit.GetLabel(); + auto primary_field = txn.GetVertexPrimaryField(vit_label); + auto field_data = vit.GetField(primary_field); + curNode["vid"] = i; + curNode["label"] = vit_label; + curNode["primary_field"] = primary_field; + curNode["field_data"] = field_data.ToString(); - vit = txn.GetVertexIterator(label[i], false); - vit_label = vit.GetLabel(); - primary_field = txn.GetVertexPrimaryField(vit_label); - field_data = vit.GetField(primary_field); - communityNode["vid"] = label[i]; - communityNode["label"] = vit_label; - communityNode["primary_field"] = primary_field; - communityNode["field_data"] = field_data.ToString(); + vit = txn.GetVertexIterator(); + vit.Goto(label[i]); + if (vit.IsValid()) { + vit_label = vit.GetLabel(); + primary_field = txn.GetVertexPrimaryField(vit_label); + field_data = vit.GetField(primary_field); + communityNode["vid"] = label[i]; + communityNode["label"] = vit_label; + communityNode["primary_field"] = primary_field; + communityNode["field_data"] = field_data.ToString(); - cur["cur"] = curNode; - cur["community"] = communityNode; - auto content = cur.dump() + "\n"; - fout.Write(content.c_str(), content.size()); + cur["cur"] = curNode; + cur["community"] = communityNode; + auto content = cur.dump() + "\n"; + fout.Write(content.c_str(), content.size()); + } + } } } } diff --git a/src/BuildLGraphApi.cmake b/src/BuildLGraphApi.cmake index fec6bd677c..652a7b0b36 100644 --- a/src/BuildLGraphApi.cmake +++ b/src/BuildLGraphApi.cmake @@ -108,8 +108,6 @@ target_include_directories(${TARGET_LGRAPH} PUBLIC if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") target_link_libraries(${TARGET_LGRAPH} PUBLIC vsag - /opt/OpenBLAS/lib/libopenblas.a - libfaiss_avx2.a libgomp.a -static-libstdc++ -static-libgcc @@ -129,8 +127,6 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if (CMAKE_SYSTEM_NAME STREQUAL "Darwin") target_link_libraries(${TARGET_LGRAPH} PUBLIC vsag - /opt/OpenBLAS/lib/libopenblas.a - libfaiss_avx2.a ${Boost_LIBRARIES} omp pthread @@ -140,8 +136,6 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") else () target_link_libraries(${TARGET_LGRAPH} PUBLIC vsag - /opt/OpenBLAS/lib/libopenblas.a - libfaiss_avx2.a rt omp pthread diff --git a/src/BuildLGraphServer.cmake b/src/BuildLGraphServer.cmake index afc2a821b9..4c7dd1d979 100644 --- a/src/BuildLGraphServer.cmake +++ b/src/BuildLGraphServer.cmake @@ -73,8 +73,6 @@ if (NOT (CMAKE_SYSTEM_NAME STREQUAL "Darwin")) geax_isogql bolt vsag - /opt/OpenBLAS/lib/libopenblas.a - libfaiss_avx2.a # begin static linking -Wl,-Bstatic cpprest @@ -134,6 +132,4 @@ target_link_libraries(${TARGET_SERVER} ${TARGET_SERVER_LIB} librocksdb.a vsag - /opt/OpenBLAS/lib/libopenblas.a - libfaiss_avx2.a ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index abf6d04258..f3078b7992 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,7 +16,7 @@ include(BuildBoltLib.cmake) install(TARGETS lgraph lgraph_python_api lgraph_server lgraph_db_python RUNTIME DESTINATION bin - LIBRARY DESTINATION lgraph/lib64) + LIBRARY DESTINATION lib64/lgraph) install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/restful/server/resource DESTINATION share/lgraph) @@ -46,18 +46,7 @@ install(FILES DESTINATION etc) install(FILES /usr/local/lib64/libvsag.so - /usr/local/lib64/libmkl_intel_lp64.so - /usr/local/lib64/libmkl_sequential.so - /usr/local/lib64/libmkl_core.so - /usr/local/lib64/libmkl_def.so - /usr/local/lib64/libmkl_avx2.so - /usr/local/lib64/libmkl_mc3.so - /usr/local/lib64/libmkl_gf_lp64.so - /usr/local/lib64/libmkl_intel_thread.so - /usr/local/lib64/libiomp5.so - /lib64/libgfortran.so.5 - /lib64/libgfortran.so.5.0.0 - DESTINATION lgraph/lib64) + DESTINATION lib64/lgraph) install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../include DESTINATION ./) diff --git a/src/bolt/io_service.h b/src/bolt/io_service.h index 9d00e0af73..48dd4506ea 100644 --- a/src/bolt/io_service.h +++ b/src/bolt/io_service.h @@ -93,7 +93,7 @@ class IOService : private boost::asio::noncopyable { : handler_(handler), acceptor_(service, tcp::endpoint(tcp::v4(), port), /*reuse_addr*/true), - io_service_pool_(thread_num), interval_(10), timer_(service) { + io_service_pool_(thread_num), interval_(5), timer_(service) { io_service_pool_.Run(); invoke_async_accept(); clean_closed_conn(); diff --git a/src/bolt/to_string.h b/src/bolt/to_string.h index adab89e1ae..6ca490f360 100644 --- a/src/bolt/to_string.h +++ b/src/bolt/to_string.h @@ -132,8 +132,9 @@ nlohmann::json ToJsonObj(const bolt::Path& path) { ret.append(forward ? "-" : "<-"); ret.append(ToJsonObj(rel).get()); ret.append(forward ? "->" : "-"); - ret.append(ToJsonObj(path.nodes[nodes_index.at(rel.endId)]).get()); - nodeId = rel.endId; + ret.append(ToJsonObj(path.nodes[nodes_index.at( + forward ? rel.endId : rel.startId)]).get()); + nodeId = forward ? rel.endId : rel.startId; } return ret; } diff --git a/src/core/edge_index.cpp b/src/core/edge_index.cpp index c9395f263b..1642872d34 100644 --- a/src/core/edge_index.cpp +++ b/src/core/edge_index.cpp @@ -261,9 +261,9 @@ bool EdgeIndexIterator::PrevKV() { return true; } -bool EdgeIndexIterator::KeyEquals(Value& key, VertexId src, VertexId dst) { - // ust need to deal with NonuniqueIndex，because a key has a set of values in - // this kind index, and other types are guaranteed by lmdb +bool EdgeIndexIterator::KeyEquals(Value& key) { + // just need to deal with NonuniqueIndex，because a key has a set of values in + // this index type, and other types are guaranteed by lmdb if (type_ == IndexType::NonuniqueIndex) { auto key_euid = it_->GetKey(); if (key_euid.Size() - _detail::EUID_SIZE != key.Size()) { @@ -375,6 +375,7 @@ VertexId EdgeIndexIterator::GetPairUniqueSrcVertexId() { _detail::VID_SIZE); return vid; } + VertexId EdgeIndexIterator::GetPairUniqueDstVertexId() { VertexId vid = 0; Value key_vid = _detail::ReturnKeyEvenIfLong(it_->GetKey()); @@ -646,7 +647,7 @@ bool EdgeIndex::Add(KvTransaction& txn, const Value& k, const EdgeUid& euid) { EdgeIndexIterator it = GetUnmanagedIterator(txn, key, key, euid.src, euid.dst, euid.lid, euid.tid, euid.eid); if (!it.IsValid() || it.KeyOutOfRange()) { - if (!it.PrevKV() || !it.KeyEquals(key, euid.src, euid.dst)) { + if (!it.PrevKV() || !it.KeyEquals(key)) { // create a new VertexIndexValue EdgeIndexValue iv; uint8_t r = iv.InsertEUid({euid.src, euid.dst, euid.lid, euid.tid, euid.eid}); diff --git a/src/core/edge_index.h b/src/core/edge_index.h index bdf8b5cc2b..003d103776 100644 --- a/src/core/edge_index.h +++ b/src/core/edge_index.h @@ -38,7 +38,7 @@ class EdgeIndex; class EdgeIndexIterator; /** - * An EdgeIndexValue packs multiple src_vids ,dst_vids, lids, tids and eids into a single value. + * An EdgeIndexValue packs multiple src_vids, dst_vids, lids, tids and eids into a single value. */ class EdgeIndexValue { /** @@ -108,12 +108,8 @@ class EdgeIndexValue { /** * Search for the specified edgeuid * - * \param src_vid The src_vid. - * \param dst_vid The dst_vid. - * \param lid The label id. - * \param tid The temporal id. - * \param eid The eid. - * \param [in,out] found Is this vid found? + * \param euid The Edge unique id. + * \param [in,out] found Is this euid found? * * \return Position where the euid is found. * list. @@ -123,11 +119,7 @@ class EdgeIndexValue { /** * Insert the specified euid. * - * \param src_vid The src_vid. - * \param dst_vid The dst_vid. - * \param lid The label id. - * \param tid The temporal id. - * \param eid The edge id. + * \param euid The Edge unique id. * \return The status of the operation. * * return value: @@ -142,11 +134,7 @@ class EdgeIndexValue { /** * Delete the specified euid. * - * \param src_vid The vid. - * \param dst_vid The dst_vid. - * \param lid The label id. - * \param tid The temporal id. - * \param eid The edge id. + * \param euid The Edge unique id. * \return The status of the operation. * * return value: @@ -205,7 +193,7 @@ class EdgeIndexIterator : public ::lgraph::IteratorBase { * \param lid The label id form which to start searching. * \param tid The Temporal id form which to start searching. * \param eid The eid from which to start searching. - * \param unique Whether the index is a unique index. + * \param type Index type. */ EdgeIndexIterator(EdgeIndex* idx, Transaction* txn, KvTable& table, const Value& key_start, const Value& key_end, VertexId src_vid, VertexId dst_vid, LabelId lid, @@ -231,7 +219,7 @@ class EdgeIndexIterator : public ::lgraph::IteratorBase { * \param key The key. * \return Whether the iterator's current key starts with key. */ - bool KeyEquals(Value& key, VertexId src, VertexId dst); + bool KeyEquals(Value& key); /** Loads content from iterator, assuming iterator is already at the right * position. */ @@ -475,7 +463,10 @@ class EdgeIndex { } bool IsUnique() const { - return type_ == IndexType::GlobalUniqueIndex || type_ == IndexType::PairUniqueIndex; + return type_ == IndexType::GlobalUniqueIndex; + } + bool IsPairUnique() const { + return type_ == IndexType::PairUniqueIndex; } IndexType GetType() const { return type_; } @@ -503,13 +494,9 @@ class EdgeIndex { /** * Delete a specified vertex under a specified key. * - * \param [in,out] txn The transaction. - * \param key The key. - * \param src_vid The src vid. - * \param dst_vid The dst vid. - * \param lid The label id. - * \param tid the temporal id. - * \param eid The eid. + * \param [in,out] txn The transaction. + * \param k The key. + * \param euid The Edge unique id. * \return Whether the operation succeeds or not. */ bool Delete(KvTransaction& txn, const Value& k, const EdgeUid& euid); @@ -522,11 +509,7 @@ class EdgeIndex { * \param [in,out] txn The transaction. * \param old_key The old key. * \param new_key The new key. - * \param src_vid The src vid. - * \param dst_vid The dst vid. - * \param lid the label id. - * \param tid the temporal id. - * \param eid The eid. + * \param euid The Edge unique id. * \return Whether the operation succeeds or not. */ bool Update(KvTransaction& txn, const Value& old_key, const Value& new_key, @@ -537,13 +520,9 @@ class EdgeIndex { * * \exception IndexException Thrown when an EdgeIndex error condition occurs. * - * \param [in,out] txn The transaction. - * \param key The key. - * \param src_vid The vid. - * \param dst_vid The dst_vid. - * \param lid The label id. - * \param tid The temporal id. - * \param eid The eid. + * \param [in,out] txn The transaction. + * \param k The key. + * \param euid The Edge unique id. * \return Whether the operation succeeds or not. */ bool Add(KvTransaction& txn, const Value& k, const EdgeUid& euid); diff --git a/src/core/graph.h b/src/core/graph.h index 4b67d67ec7..0fc5aa0ff6 100644 --- a/src/core/graph.h +++ b/src/core/graph.h @@ -196,6 +196,7 @@ class Graph { * * \param [in,out] txn The transaction. * \param src Source of the edge. + * \param lid The label id. * \param dst Destination of the edge. * \param prop The edge property. * \param constraints edge constraints @@ -213,9 +214,7 @@ class Graph { * Adds an edge * * \param [in,out] txn The transaction. - * \param src Source of the edge. - * \param tid The Temporal id. - * \param dst Destination of the edge. + * \param esid Edge id without eid. * \param prop The edge property. * \param constraints edge constraints * @@ -269,12 +268,10 @@ class Graph { } /** - * Delete edge from src to dst with edge_id==eid + * Delete an edge * * \param [in,out] txn The transaction. - * \param src Source of the edge. - * \param dst Destination of the edge. - * \param eid Edge property. + * \param uid The Edge unique id. * * \return True if the edge is deleted successfully. False if the edge does not exist. Raises * exception if other errors occur. @@ -338,9 +335,7 @@ class Graph { * Sets edge property * * \param [in,out] txn The transaction. - * \param src Source for the. - * \param dst Destination for the. - * \param eid The old property. + * \param uid The Edge unique id. * \param new_prop The new property. * * \return True if it succeeds, false if it fails. diff --git a/src/core/index_manager.cpp b/src/core/index_manager.cpp index 4c46b0947e..0ea04fdde1 100644 --- a/src/core/index_manager.cpp +++ b/src/core/index_manager.cpp @@ -107,10 +107,8 @@ IndexManager::IndexManager(KvTransaction& txn, SchemaManager* v_schema_manager, std::unique_ptr vsag_index; vsag_index.reset(dynamic_cast ( new HNSW(idx.label, idx.field, idx.distance_type, idx.index_type, - idx.dimension, {idx.hnsm_m, idx.hnsm_ef_construction}))); + idx.dimension, {idx.hnsw_m, idx.hnsw_ef_construction}))); uint64_t count = 0; - std::vector> floatvector; - std::vector vids; auto kv_iter = schema->GetPropertyTable().GetIterator(txn); for (kv_iter->GotoFirstKey(); kv_iter->IsValid(); kv_iter->Next()) { auto prop = kv_iter->GetValue(); @@ -119,14 +117,14 @@ IndexManager::IndexManager(KvTransaction& txn, SchemaManager* v_schema_manager, } auto vid = graph::KeyPacker::GetVidFromPropertyTableKey(kv_iter->GetKey()); auto vector = (extractor->GetConstRef(prop)).AsType>(); - floatvector.emplace_back(vector); - vids.emplace_back(vid); + vsag_index->Add({std::move(vector)}, {vid}); count++; + if ((count % 10000) == 0) { + LOG_INFO() << "vector index count: " << count; + } } - vsag_index->Build(); - vsag_index->Add(floatvector, vids, count); kv_iter.reset(); - LOG_DEBUG() << "index count: " << count; + LOG_INFO() << "vector index count: " << count; schema->MarkVectorIndexed(extractor->GetFieldId(), vsag_index.release()); LOG_INFO() << FMA_FMT("end building vertex vector index for {}:{} in detached model", idx.label, idx.field); @@ -185,8 +183,8 @@ bool IndexManager::AddVectorIndex(KvTransaction& txn, const std::string& label, idx.index_type = index_type; idx.dimension = vec_dimension; idx.distance_type = distance_type; - idx.hnsm_m = index_spec[0]; - idx.hnsm_ef_construction = index_spec[1]; + idx.hnsw_m = index_spec[0]; + idx.hnsw_ef_construction = index_spec[1]; auto table_name = GetVertexVectorIndexTableName(label, field); auto it = index_list_table_->GetIterator(txn, Value::ConstRef(table_name)); if (it->IsValid()) return false; // already exist @@ -315,8 +313,8 @@ std::vector IndexManager::ListVectorIndex(KvTransaction& txn) { vs.index_type = vi.index_type; vs.dimension = vi.dimension; vs.distance_type = vi.distance_type; - vs.hnsm_m = vi.hnsm_m; - vs.hnsm_ef_construction = vi.hnsm_ef_construction; + vs.hnsw_m = vi.hnsw_m; + vs.hnsw_ef_construction = vi.hnsw_ef_construction; ret.emplace_back(vs); } } diff --git a/src/core/index_manager.h b/src/core/index_manager.h index 3e7f29bb8c..2406b341fe 100644 --- a/src/core/index_manager.h +++ b/src/core/index_manager.h @@ -111,20 +111,20 @@ struct VectorIndexEntry { std::string index_type; int dimension; std::string distance_type; - int hnsm_m; - int hnsm_ef_construction; + int hnsw_m; + int hnsw_ef_construction; template size_t Serialize(StreamT& buf) const { return BinaryWrite(buf, label) + BinaryWrite(buf, field) + BinaryWrite(buf, index_type) + BinaryWrite(buf, dimension) + BinaryWrite(buf, distance_type) + - BinaryWrite(buf, hnsm_m) + BinaryWrite(buf, hnsm_ef_construction); + BinaryWrite(buf, hnsw_m) + BinaryWrite(buf, hnsw_ef_construction); } template size_t Deserialize(StreamT& buf) { return BinaryRead(buf, label) + BinaryRead(buf, field) + BinaryRead(buf, index_type) + BinaryRead(buf, dimension) + BinaryRead(buf, distance_type) + - BinaryRead(buf, hnsm_m) + BinaryRead(buf, hnsm_ef_construction); + BinaryRead(buf, hnsw_m) + BinaryRead(buf, hnsw_ef_construction); } }; @@ -289,21 +289,23 @@ class IndexManager { std::vector ListVectorIndex(KvTransaction& txn); // vertex index - std::pair, std::vector> ListAllIndexes( + std::tuple, std::vector, + std::vector> ListAllIndexes( KvTransaction& txn) { std::vector indexes; std::vector compositeIndexes; - IndexSpec is; - CompositeIndexSpec cis; + std::vector vectorIndexes; size_t v_index_len = strlen(_detail::VERTEX_INDEX); size_t e_index_len = strlen(_detail::EDGE_INDEX); size_t c_index_len = strlen(_detail::COMPOSITE_INDEX); + size_t ve_index_len = strlen(_detail::VERTEX_VECTOR_INDEX); auto it = index_list_table_->GetIterator(txn); for (it->GotoFirstKey(); it->IsValid(); it->Next()) { std::string index_name = it->GetKey().AsString(); if (index_name.size() > v_index_len && index_name.substr(index_name.size() - v_index_len) == _detail::VERTEX_INDEX) { _detail::IndexEntry ent = LoadIndex(it->GetValue()); + IndexSpec is; is.label = ent.label; is.field = ent.field; is.type = ent.type; @@ -311,21 +313,36 @@ class IndexManager { } else if (index_name.size() > e_index_len && index_name.substr(index_name.size() - e_index_len) == _detail::EDGE_INDEX) { _detail::IndexEntry ent = LoadIndex(it->GetValue()); + IndexSpec is; is.label = ent.label; is.field = ent.field; is.type = ent.type; indexes.emplace_back(std::move(is)); + } else if (index_name.size() > ve_index_len && + index_name.substr(index_name.size() - ve_index_len) + == _detail::VERTEX_VECTOR_INDEX) { + _detail::VectorIndexEntry ent = LoadVectorIndex(it->GetValue()); + VectorIndexSpec vis; + vis.label = ent.label; + vis.field = ent.field; + vis.distance_type = ent.distance_type; + vis.dimension = ent.dimension; + vis.hnsw_ef_construction = ent.hnsw_ef_construction; + vis.hnsw_m = ent.hnsw_m; + vis.index_type = ent.index_type; + vectorIndexes.emplace_back(vis); } else if (index_name.size() > c_index_len && index_name.substr(index_name.size() - c_index_len) == _detail::COMPOSITE_INDEX) { _detail::CompositeIndexEntry idx = LoadCompositeIndex(it->GetValue()); + CompositeIndexSpec cis; cis.label = idx.label; cis.fields = idx.field_names; cis.type = idx.index_type; compositeIndexes.emplace_back(std::move(cis)); } } - return {indexes, compositeIndexes}; + return {std::move(indexes), std::move(compositeIndexes), std::move(vectorIndexes)}; } }; } // namespace lgraph diff --git a/src/core/kv_table_comparators.cpp b/src/core/kv_table_comparators.cpp index b7f3b6f386..24a16c0fff 100644 --- a/src/core/kv_table_comparators.cpp +++ b/src/core/kv_table_comparators.cpp @@ -47,9 +47,9 @@ struct KeyCompareFunc { * The comparator for bytes or strings. */ static int LexicalKeyVidCompareFunc(const MDB_val* a, const MDB_val* b) { - int diff; - int len_diff; - unsigned int len; + int diff = 0; + int len_diff = 0; + unsigned int len = 0; len = static_cast(a->mv_size) - VID_SIZE; len_diff = static_cast(a->mv_size) - static_cast(b->mv_size); @@ -59,12 +59,15 @@ static int LexicalKeyVidCompareFunc(const MDB_val* a, const MDB_val* b) { } diff = memcmp(a->mv_data, b->mv_data, len); - if (diff == 0 && len_diff == 0) { + if (diff) { + return diff; + } else if (len_diff) { + return len_diff; + } else { int64_t a_vid = GetVid((char*)a->mv_data + a->mv_size - VID_SIZE); int64_t b_vid = GetVid((char*)b->mv_data + b->mv_size - VID_SIZE); return a_vid < b_vid ? -1 : a_vid > b_vid ? 1 : 0; } - return static_cast(diff ? diff : len_diff < 0 ? -1 : len_diff); } /** @@ -114,7 +117,11 @@ static int LexicalKeyEuidCompareFunc(const MDB_val* a, const MDB_val* b) { } diff = memcmp(a->mv_data, b->mv_data, len); - if (diff == 0 && len_diff == 0) { + if (diff) { + return diff; + } else if (len_diff) { + return len_diff; + } else { int64_t a_vid1 = GetVid((char*)a->mv_data + a->mv_size - EUID_SIZE); int64_t a_vid2 = GetVid((char*)a->mv_data + a->mv_size - EUID_SIZE + VID_SIZE); int64_t a_lid = GetLabelId((char*)a->mv_data + a->mv_size - EUID_SIZE + LID_BEGIN); @@ -133,7 +140,6 @@ static int LexicalKeyEuidCompareFunc(const MDB_val* a, const MDB_val* b) { : a_eid > b_eid ? 1 : 0; } - return static_cast(diff ? diff : len_diff < 0 ? -1 : len_diff); } template diff --git a/src/core/lightning_graph.cpp b/src/core/lightning_graph.cpp index 2daf2da426..b4571382bd 100644 --- a/src/core/lightning_graph.cpp +++ b/src/core/lightning_graph.cpp @@ -66,7 +66,8 @@ void LightningGraph::DropAllVertex() { Transaction txn = CreateWriteTxn(false); ScopedRef curr_schema = schema_.GetScopedRef(); // clear indexes - auto [indexes, composite_indexes] = index_manager_->ListAllIndexes(txn.GetTxn()); + auto [indexes, composite_indexes, vector_indexes] + = index_manager_->ListAllIndexes(txn.GetTxn()); for (auto& idx : indexes) { auto v_schema = curr_schema->v_schema_manager.GetSchema(idx.label); auto e_schema = curr_schema->e_schema_manager.GetSchema(idx.label); @@ -89,6 +90,15 @@ void LightningGraph::DropAllVertex() { v_schema->GetCompositeIndex(idx.fields)->Clear(txn.GetTxn()); } } + for (auto& idx : vector_indexes) { + auto v_schema = curr_schema->v_schema_manager.GetSchema(idx.label); + FMA_DBG_ASSERT(v_schema); + if (v_schema) { + auto ext = v_schema->GetFieldExtractor(idx.field); + FMA_DBG_ASSERT(ext); + ext->GetVectorIndex()->Clear(); + } + } // clear detached property data for (auto& name : curr_schema->v_schema_manager.GetAllLabels()) { auto s = curr_schema->v_schema_manager.GetSchema(name); @@ -130,12 +140,8 @@ size_t LightningGraph::GetNumVertices() { * \param label The label. * \param n_fields Number of fields for this label. * \param fds The FieldDefs. - * \param is_vertex True if this is vertex label, otherwise - * it is edge label. - * \param primary_field The vertex primary property, must be - * set when is_vertex is true - * \param edge_constraints The edge constraints, can be set - * when is_vertex is false + * \param is_vertex True if this is vertex label, otherwise it is edge label. + * \param options Cast to VertexOptions when is_vertex is true, else cast to EdgeOptions. * * \return True if it succeeds, false if the label already exists. Throws exception on error. */ @@ -251,12 +257,8 @@ bool LightningGraph::AddLabel(const std::string& label, size_t n_fields, const F * * \param label The label name. * \param fds The FieldDefs. - * \param is_vertex True if this is vertex label, otherwise - * it is edge label. - * \param primary_field The vertex primary property, must be - * set when is_vertex is true - * \param edge_constraints The edge constraints, can be set - * when is_vertex is false + * \param is_vertex True if this is vertex label, otherwise it is edge label. + * \param options Cast to VertexOptions when is_vertex is true, else cast to EdgeOptions. * * \return True if it succeeds, false if the label already exists. Throws exception on error. */ @@ -1051,7 +1053,8 @@ bool LightningGraph::AlterLabelModFields(const std::string& label, * * \param label The label. * \param field The field. - * \param is_unique True if the field content is unique for each vertex. + * \param type The index type. + * \param is_vertex True if this is vertex label, otherwise it is edge label. * * \return True if it succeeds, false if the index already exists. Throws exception on error. */ @@ -2011,25 +2014,29 @@ bool LightningGraph::BlockingAddIndex(const std::string& label, const std::strin FMA_FMT("start building vertex index for {}:{} in detached model", label, field); VertexIndex* index = extractor->GetVertexIndex(); uint64_t count = 0; + uint64_t filter = 0; auto kv_iter = schema->GetPropertyTable().GetIterator(txn.GetTxn()); for (kv_iter->GotoFirstKey(); kv_iter->IsValid(); kv_iter->Next()) { auto vid = graph::KeyPacker::GetVidFromPropertyTableKey(kv_iter->GetKey()); - auto prop = kv_iter->GetValue(); - if (extractor->GetIsNull(prop)) { + auto props = kv_iter->GetValue(); + auto prop = extractor->GetConstRef(props); + if (prop.Empty()) { + filter++; continue; } - if (!index->Add(txn.GetTxn(), extractor->GetConstRef(prop), vid)) { + if (!index->Add(txn.GetTxn(), prop, vid)) { THROW_CODE(InternalError, "Failed to index vertex [{}] with field value [{}:{}]", - vid, extractor->Name(), extractor->FieldToString(prop)); + vid, extractor->Name(), extractor->FieldToString(props)); } count++; if (count % 100000 == 0) { - LOG_DEBUG() << "index count: " << count; + LOG_INFO() << "index count: " << count; } } kv_iter.reset(); - LOG_DEBUG() << "index count: " << count; + LOG_INFO() << "index count: " << count; + LOG_INFO() << FMA_FMT("{} records are skipped during adding index.", filter); txn.Commit(); schema_.Assign(new_schema.release()); LOG_INFO() << @@ -2219,8 +2226,6 @@ bool LightningGraph::BlockingAddVectorIndex(bool is_vertex, const std::string& l label, field); VectorIndex* index = extractor->GetVectorIndex(); uint64_t count = 0; - std::vector> floatvector; - std::vector vids; auto dim = index->GetVecDimension(); auto kv_iter = schema->GetPropertyTable().GetIterator(txn.GetTxn()); for (kv_iter->GotoFirstKey(); kv_iter->IsValid(); kv_iter->Next()) { @@ -2234,13 +2239,13 @@ bool LightningGraph::BlockingAddVectorIndex(bool is_vertex, const std::string& l THROW_CODE(VectorIndexException, "vector size error, size:{}, dim:{}", vector.size(), dim); } - floatvector.emplace_back(std::move(vector)); - vids.emplace_back(vid); + index->Add({std::move(vector)}, {vid}); count++; + if ((count % 10000) == 0) { + LOG_INFO() << "vector index count: " << count; + } } - index->Build(); - index->Add(floatvector, vids, count); - LOG_INFO() << "index count: " << count; + LOG_INFO() << "vector index count: " << count; LOG_INFO() << FMA_FMT("end building vertex vector index for {}:{} in detached model", label, field); kv_iter.reset(); @@ -2809,7 +2814,8 @@ void LightningGraph::DropAllIndex() { ScopedRef curr_schema = schema_.GetScopedRef(); std::unique_ptr new_schema(new SchemaInfo(*curr_schema.Get())); std::unique_ptr backup_schema(new SchemaInfo(*curr_schema.Get())); - auto [indexes, composite_indexes] = index_manager_->ListAllIndexes(txn.GetTxn()); + auto [indexes, composite_indexes, vector_indexes] + = index_manager_->ListAllIndexes(txn.GetTxn()); bool success = true; for (auto& idx : indexes) { @@ -2840,6 +2846,14 @@ void LightningGraph::DropAllIndex() { } v_schema->UnVertexCompositeIndex(idx.fields); } + for (auto& idx : vector_indexes) { + auto v_schema = new_schema->v_schema_manager.GetSchema(idx.label); + FMA_DBG_ASSERT(v_schema); + auto ret = index_manager_->DeleteVectorIndex(txn.GetTxn(), idx.label, idx.field); + FMA_DBG_ASSERT(ret); + auto ext = v_schema->GetFieldExtractor(idx.field); + v_schema->UnVectorIndex(ext->GetFieldId()); + } if (success) { schema_.Assign(new_schema.release()); AutoCleanupAction revert_assign_new_schema( @@ -2867,6 +2881,7 @@ const DBConfig& LightningGraph::GetConfig() const { return config_; } * Backups the current DB to the path specified. * * \param path Full pathname of the destination. + * \param compact True to enable compaction * * \return Transaction ID of the last committed transaction. */ diff --git a/src/core/lightning_graph.h b/src/core/lightning_graph.h index 0b7dca5a84..27730f83f4 100644 --- a/src/core/lightning_graph.h +++ b/src/core/lightning_graph.h @@ -126,10 +126,7 @@ class LightningGraph { * \param n_fields Number of fields for this label. * \param fds The FieldDefs. * \param is_vertex True if this is vertex label, otherwise it is edge label. - * \param primary_field The vertex primary property, must - * beset when is_vertex is true - * \param edge_constraints The edge constraints, can be set - * when is_vertex is false + * \param options Cast to VertexOptions when is_vertex is true, else cast to EdgeOptions. * \return True if it succeeds, false if the label already exists. Throws exception on error. */ @@ -141,12 +138,8 @@ class LightningGraph { * * \param label The label name. * \param fds The FieldDefs. - * \param is_vertex True if this is vertex label, otherwise - * it is edge label. - * \param primary_field The vertex primary property, must be - * set when is_vertex is true - * \param edge_constraints The edge constraints, can be set - * when is_vertex is false + * \param is_vertex True if this is vertex label, otherwise it is edge label. + * \param options Cast to VertexOptions when is_vertex is true, else cast to EdgeOptions. * \return True if it succeeds, false if the label already exists. Throws exception on error. */ @@ -183,7 +176,8 @@ class LightningGraph { * * \param label The label. * \param field The field. - * \param is_unique True if the field content is unique for each vertex. + * \param type The index type. + * \param is_vertex True if this is vertex label, otherwise it is edge label. * * \return True if it succeeds, false if the index already exists. Throws exception on error. */ @@ -240,7 +234,6 @@ class LightningGraph { void OfflineCreateBatchIndex(const std::vector& indexes, size_t commit_batch_size = 1 << 20, bool is_vertex = true); - /** * Is this index ready? When an index is added to a non-empty graph, it will require some time * to be built. This function returns the status of the index so user can choose to wait for @@ -248,21 +241,23 @@ class LightningGraph { * * \param label The label. * \param field The field. + * \param is_vertxt True for vertex index, False for edge index. * * \return True if it succeeds, false if it fails. */ + bool IsIndexed(const std::string& label, const std::string& field, bool is_vertex); + + bool IsCompositeIndexed(const std::string& label, const std::vector& fields); + /** * Deletes the index to 'label:field' * * \param label The label. * \param field The field. + * \param is_vertxt True for vertex index, False for edge index. * * \return True if it succeeds, false if the index does not exist. Throws exception on error. */ - bool IsIndexed(const std::string& label, const std::string& field, bool is_vertex); - - bool IsCompositeIndexed(const std::string& label, const std::vector& fields); - bool DeleteIndex(const std::string& label, const std::string& field, bool is_vertex); bool DeleteVectorIndex(bool is_vertex, const std::string& label, const std::string& field); diff --git a/src/core/lmdb_iterator.h b/src/core/lmdb_iterator.h index ff1c8b8bae..7f0f84f92e 100644 --- a/src/core/lmdb_iterator.h +++ b/src/core/lmdb_iterator.h @@ -76,7 +76,6 @@ class LMDBKvIterator final : public KvIterator { * exist. * @param [in,out] txn The transaction. * @param [in,out] table The table. - * @param refresh_callback Callback function when iterator was refreshed. * @param key The key. * @param closest True to closest. */ diff --git a/src/core/lmdb_table.h b/src/core/lmdb_table.h index 313d80f2df..21a1c2359e 100644 --- a/src/core/lmdb_table.h +++ b/src/core/lmdb_table.h @@ -83,7 +83,7 @@ class LMDBKvTable final : public KvTable { bool GetValue(KvTransaction& txn, const Value& key, Value& val) override; - /** + /** * Gets number of k-v pairs in the table. * * \param [in,out] txn The transaction. diff --git a/src/core/schema.cpp b/src/core/schema.cpp index 51a5bb4b3a..59aa549679 100644 --- a/src/core/schema.cpp +++ b/src/core/schema.cpp @@ -48,12 +48,15 @@ void Schema::DeleteVertexFullTextIndex(VertexId vid, std::vector& void Schema::DeleteVertexIndex(KvTransaction& txn, VertexId vid, const Value& record) { for (auto& idx : indexed_fields_) { auto& fe = fields_[idx]; - if (fe.GetIsNull(record)) continue; + auto prop = fe.GetConstRef(record); + if (prop.Empty()) { + continue; + } if (fe.Type() != FieldType::FLOAT_VECTOR) { VertexIndex* index = fe.GetVertexIndex(); FMA_ASSERT(index); // update field index - if (!index->Delete(txn, fe.GetConstRef(record), vid)) { + if (!index->Delete(txn, prop, vid)) { THROW_CODE(InputError, "Failed to un-index vertex [{}] with field " "value [{}:{}]: index value does not exist.", vid, fe.Name(), fe.FieldToString(record)); @@ -102,11 +105,14 @@ void Schema::DeleteCreatedVertexIndex(KvTransaction& txn, VertexId vid, const Va const std::vector& created) { for (auto& idx : created) { auto& fe = fields_[idx]; - if (fe.GetIsNull(record)) continue; + auto prop = fe.GetConstRef(record); + if (prop.Empty()) { + continue; + } VertexIndex* index = fe.GetVertexIndex(); FMA_ASSERT(index); // the aim of this method is delete the index that has been created - if (!index->Delete(txn, fe.GetConstRef(record), vid)) { + if (!index->Delete(txn, prop, vid)) { THROW_CODE(InputError, "Failed to un-index vertex [{}] with field " "value [{}:{}]: index value does not exist.", vid, fe.Name(), fe.FieldToString(record)); @@ -155,12 +161,15 @@ void Schema::AddVertexToIndex(KvTransaction& txn, VertexId vid, const Value& rec created.reserve(fields_.size()); for (auto& idx : indexed_fields_) { auto& fe = fields_[idx]; - if (fe.GetIsNull(record)) continue; + auto prop = fe.GetConstRef(record); + if (prop.Empty()) { + continue; + } if (fe.Type() != FieldType::FLOAT_VECTOR) { VertexIndex* index = fe.GetVertexIndex(); FMA_ASSERT(index); // update field index - if (!index->Add(txn, fe.GetConstRef(record), vid)) { + if (!index->Add(txn, prop, vid)) { THROW_CODE(InputError, "Failed to index vertex [{}] with field value [{}:{}]: index value already exists.", vid, fe.Name(), fe.FieldToString(record)); @@ -281,20 +290,6 @@ void Schema::DeleteCreatedEdgeIndex(KvTransaction& txn, const EdgeUid& euid, con } } -bool Schema::EdgeUniqueIndexConflict(KvTransaction& txn, const Value& record) { - for (auto& idx : indexed_fields_) { - auto& fe = fields_[idx]; - EdgeIndex* index = fe.GetEdgeIndex(); - FMA_ASSERT(index); - if (!index->IsUnique()) continue; - if (fe.GetIsNull(record)) continue; - if (index->UniqueIndexConflict(txn, fe.GetConstRef(record))) { - return true; - } - } - return false; -} - void Schema::AddEdgeToIndex(KvTransaction& txn, const EdgeUid& euid, const Value& record, std::vector& created) { created.reserve(fields_.size()); @@ -328,7 +323,7 @@ void Schema::AddVectorToVectorIndex(KvTransaction& txn, VertexId vid, const Valu "vector index dimension mismatch, vector size:{}, dim:{}", floatvector.back().size(), dim); } - index->Add(floatvector, vids, 1); + index->Add(floatvector, vids); } } @@ -337,9 +332,7 @@ void Schema::DeleteVectorIndex(KvTransaction& txn, VertexId vid, const Value& re auto& fe = fields_[idx]; if (fe.GetIsNull(record)) continue; VectorIndex* index = fe.GetVectorIndex(); - std::vector vids; - vids.push_back(vid); - index->Add({}, vids, 0); + index->Remove({vid}); } } @@ -536,9 +529,10 @@ void Schema::RefreshLayout() { /** * Creates an empty record * - * \param [in,out] v Value to store the result. * \param size_hint (Optional) Hint of size of the record, used to * reduce memory realloc. + * + * \return A Value. */ Value Schema::CreateEmptyRecord(size_t size_hint) const { Value v(size_hint); diff --git a/src/core/schema.h b/src/core/schema.h index 6390666f37..bce2cf2c59 100644 --- a/src/core/schema.h +++ b/src/core/schema.h @@ -499,7 +499,6 @@ class Schema { void AddEdgeToIndex(KvTransaction& txn, const EdgeUid& euid, const Value& record, std::vector& created); - bool EdgeUniqueIndexConflict(KvTransaction& txn, const Value& record); void AddVectorToVectorIndex(KvTransaction& txn, VertexId vid, const Value& record); diff --git a/src/core/schema_manager.h b/src/core/schema_manager.h index cf2d81518d..fb18d50c87 100644 --- a/src/core/schema_manager.h +++ b/src/core/schema_manager.h @@ -173,13 +173,12 @@ class SchemaManager { * \exception ::lgraph::SchemaException Thrown when a Schema error condition occurs. * * \param [in,out] txn The transaction. + * \param is_vertex True if this is vertex label, otherwise it is edge label. * \param label The label. * \param n_fields The fields. * \param fields The fields in the labeled data. - * \param primary_field The vertex primary property, - * must be set when is_vertex is true - * \param edge_constraints The edge constraints, can - * be set when is_vertex is false + * \param options Cast to VertexOptions when is_vertex is true, else cast to + * EdgeOptions. * * \return True if it succeeds, false if the label already exists. Throws exception on error. */ diff --git a/src/core/transaction.cpp b/src/core/transaction.cpp index 2a3476c02e..5f39191a31 100644 --- a/src/core/transaction.cpp +++ b/src/core/transaction.cpp @@ -968,19 +968,52 @@ Transaction::SetVertexProperty(VertexIterator& it, size_t n_fields, const FieldT // no need to update index since blob cannot be indexed } else if (fe->Type() == FieldType::FLOAT_VECTOR) { fe->ParseAndSet(new_prop, values[i]); - schema->DeleteVectorIndex(*txn_, vid, old_prop); - schema->AddVectorToVectorIndex(*txn_, vid, new_prop); + VectorIndex* index = fe->GetVectorIndex(); + if (index) { + auto old_v = fe->GetConstRef(old_prop); + auto new_v = fe->GetConstRef(new_prop); + std::vector vids {vid}; + if (!old_v.Empty() && !new_v.Empty()) { + if (old_v == new_v) { + continue; + } + // delete + index->Remove(vids); + // add + auto dim = index->GetVecDimension(); + std::vector> floatvector; + floatvector.emplace_back(new_v.AsFloatVector()); + if (floatvector.back().size() != (size_t)dim) { + THROW_CODE(InputError, + "vector index dimension mismatch, vector size:{}, dim:{}", + floatvector.back().size(), dim); + } + index->Add(floatvector, vids); + } else if (old_v.Empty() && !new_v.Empty()) { + // add + auto dim = index->GetVecDimension(); + std::vector> floatvector; + floatvector.emplace_back(new_v.AsFloatVector()); + if (floatvector.back().size() != (size_t)dim) { + THROW_CODE(InputError, + "vector index dimension mismatch, vector size:{}, dim:{}", + floatvector.back().size(), dim); + } + index->Add(floatvector, vids); + } else if (!old_v.Empty() && new_v.Empty()) { + // delete + index->Remove(vids); + } + } } else { fe->ParseAndSet(new_prop, values[i]); // update index if there is no error VertexIndex* index = fe->GetVertexIndex(); if (index && index->IsReady()) { - bool oldnull = fe->GetIsNull(old_prop); - bool newnull = fe->GetIsNull(new_prop); - if (!oldnull && !newnull) { + auto old_v = fe->GetConstRef(old_prop); + auto new_v = fe->GetConstRef(new_prop); + if (!old_v.Empty() && !new_v.Empty()) { // update - const auto& old_v = fe->GetConstRef(old_prop); - const auto& new_v = fe->GetConstRef(new_prop); if (old_v == new_v) { // If the values are equal, there is no need to update the index. continue; @@ -990,16 +1023,16 @@ Transaction::SetVertexProperty(VertexIterator& it, size_t n_fields, const FieldT THROW_CODE(InputError, "failed to update vertex index, {}:[{}] already exists", fe->Name(), fe->FieldToString(new_prop)); - } else if (oldnull && !newnull) { + } else if (old_v.Empty() && !new_v.Empty()) { // set to non-null, add index - bool r = index->Add(*txn_, fe->GetConstRef(new_prop), vid); + bool r = index->Add(*txn_, new_v, vid); if (!r) THROW_CODE(InputError, "failed to add vertex index, {}:[{}] already exists", fe->Name(), fe->FieldToString(new_prop)); - } else if (!oldnull && newnull) { + } else if (!old_v.Empty() && new_v.Empty()) { // set to null, delete index - bool r = index->Delete(*txn_, fe->GetConstRef(old_prop), vid); + bool r = index->Delete(*txn_, old_v, vid); FMA_DBG_ASSERT(r); } else { // both null, nothing to do diff --git a/src/core/transaction.h b/src/core/transaction.h index c4b000b6e7..10aec64743 100644 --- a/src/core/transaction.h +++ b/src/core/transaction.h @@ -81,6 +81,7 @@ class Transaction { std::vector iterators_; FullTextIndex* fulltext_index_; std::vector fulltext_buffers_; + std::vector vector_buffers_; std::unordered_map vertex_delta_count_; std::unordered_map edge_delta_count_; std::set vertex_label_delete_; diff --git a/src/core/value.h b/src/core/value.h index e316141d29..56ac518ef1 100644 --- a/src/core/value.h +++ b/src/core/value.h @@ -522,6 +522,8 @@ class Value { */ std::string AsString() const { return AsType(); } + std::vector AsFloatVector() const { return AsType>(); } + /** * Create a Value that is a const reference to the object t * diff --git a/src/core/vector_index.h b/src/core/vector_index.h index 320d8eea1d..e41fb2a2b4 100644 --- a/src/core/vector_index.h +++ b/src/core/vector_index.h @@ -20,6 +20,7 @@ namespace lgraph { + class VectorIndex { friend class Schema; friend class LightningGraph; @@ -66,12 +67,13 @@ class VectorIndex { // add vector to index and build index virtual void Add(const std::vector>& vectors, - const std::vector& vids, int64_t num_vectors) = 0; + const std::vector& vids) = 0; + + virtual void Remove(const std::vector& vids) = 0; - // build index - virtual void Build() = 0; + virtual void Clear() = 0; - // serialize index + // serialize index virtual std::vector Save() = 0; // load index form serialization @@ -83,5 +85,17 @@ class VectorIndex { virtual std::vector> RangeSearch(const std::vector& query, float radius, int ef_search, int limit) = 0; + + virtual int64_t GetElementsNum() = 0; + virtual int64_t GetMemoryUsage() = 0; + virtual int64_t GetDeletedIdsNum() = 0; }; + +struct VectorIndexEntry { + VectorIndex* index; + bool add; + std::vector vids; + std::vector> vectors; +}; + } // namespace lgraph diff --git a/src/core/vertex_index.h b/src/core/vertex_index.h index 59218bf287..13c6c10e6c 100644 --- a/src/core/vertex_index.h +++ b/src/core/vertex_index.h @@ -160,7 +160,7 @@ class VertexIndexIterator : public ::lgraph::IteratorBase { * \param key_start The start key. * \param key_end The end key. * \param vid The vid from which to start searching. - * \param unique Whether the index is a unique index. + * \param type The index type. */ VertexIndexIterator(VertexIndex* idx, Transaction* txn, KvTable& table, const Value& key_start, @@ -272,7 +272,6 @@ class VertexIndex { VertexIndex(const VertexIndex& rhs); - VertexIndex(VertexIndex&& rhs) = delete; VertexIndex& operator=(const VertexIndex& rhs) = delete; @@ -466,7 +465,7 @@ class VertexIndex { * \exception IndexException Thrown when an VertexIndex error condition occurs. * * \param [in,out] txn The transaction. - * \param key The key. + * \param k The key. * \param vid The vid. * * \return Whether the operation succeeds or not. diff --git a/src/core/vsag_hnsw.cpp b/src/core/vsag_hnsw.cpp index 03eec8909b..e7447d9a99 100644 --- a/src/core/vsag_hnsw.cpp +++ b/src/core/vsag_hnsw.cpp @@ -23,33 +23,42 @@ HNSW::HNSW(const std::string& label, const std::string& name, const std::string& index_type, int vec_dimension, std::vector index_spec) : VectorIndex(label, name, distance_type, index_type, - vec_dimension, std::move(index_spec)), - createindex_(nullptr), index_(createindex_.get()) {} + vec_dimension, std::move(index_spec)) { + Build(); + LOG_INFO() << FMA_FMT("Create HNSW instance, {}:{}", GetLabel(), GetName()); +} + +HNSW::~HNSW() { + LOG_INFO() << FMA_FMT("Destroy HNSW instance, {}:{}", GetLabel(), GetName()); + index_ = nullptr; +} // add vector to index void HNSW::Add(const std::vector>& vectors, - const std::vector& vids, int64_t num_vectors) { - // reduce dimension - if (num_vectors == 0) { - for (auto vid : vids) { - auto result = index_->Remove(vid); - if (result.has_value()) { - if (!result.value()) { - THROW_CODE(InputError, "failed to remove vector from index, vid:{}", vid); - } - } else { - THROW_CODE(InputError, "failed to remove vector from index, vid:{}, error:{}", - vid, result.error().message); - } - } + const std::vector& vids) { + if (vectors.size() != vids.size()) { + THROW_CODE(VectorIndexException, + "size mismatch, vectors.size:{}, vids.size:{}", vectors.size(), vids.size()); + } + if (vectors.empty()) { + return; } + auto num_vectors = vectors.size(); auto* index_vectors = new float[num_vectors * vec_dimension_]; auto* ids = new int64_t[num_vectors]; - for (int64_t i = 0; i < num_vectors; i++) { + for (size_t i = 0; i < num_vectors; i++) { std::copy(vectors[i].begin(), vectors[i].end(), &index_vectors[i * vec_dimension_]); } - for (int64_t i = 0; i < num_vectors; i++) { - ids[i] = vids[i]; + for (size_t i = 0; i < num_vectors; i++) { + vectorid_++; + ids[i] = vectorid_; + if (vid_vectorid_.count(vids[i])) { + delete[] ids; + delete[] index_vectors; + THROW_CODE(VectorIndexException, "[HNSW Add] vid {} already exists", vids[i]); + } + vid_vectorid_[vids[i]] = vectorid_; + vectorid_vid_[vectorid_] = {false, vids[i]}; } auto dataset = vsag::Dataset::Make(); dataset->Dim(vec_dimension_)->NumElements(num_vectors) @@ -65,6 +74,27 @@ void HNSW::Add(const std::vector>& vectors, } } +void HNSW::Clear() { + vectorid_ = 0; + index_ = nullptr; + deleted_vectorid_ = 0; + std::unordered_map().swap(vid_vectorid_); + std::unordered_map>().swap(vectorid_vid_); + Build(); +} + +void HNSW::Remove(const std::vector& vids) { + for (auto vid : vids) { + auto iter = vid_vectorid_.find(vid); + if (iter == vid_vectorid_.end()) { + THROW_CODE(VectorIndexException, "[HNSW Remove] vid {} does not exist", vid); + } + vectorid_vid_.at(iter->second) = {true, -1}; + deleted_vectorid_++; + vid_vectorid_.erase(iter); + } +} + void HNSW::Build() { nlohmann::json hnsw_parameters{ {"max_degree", index_spec_[0]}, @@ -78,8 +108,7 @@ void HNSW::Build() { }; auto temp = vsag::Factory::CreateIndex("hnsw", index_parameters.dump()); if (temp.has_value()) { - createindex_ = std::move(temp.value()); - index_ = createindex_.get(); + index_ = std::move(temp.value()); } else { THROW_CODE(VectorIndexException, temp.error().message); } @@ -185,10 +214,13 @@ HNSW::KnnSearch(const std::vector& query, int64_t top_k, int ef_search) { {"hnsw", {{"ef_search", ef_search}}}, }; std::vector> ret; - auto result = index_->KnnSearch(dataset, top_k, parameters.dump()); + auto result = index_->KnnSearch(dataset, top_k, parameters.dump(), [this](int64_t id)->bool { + return vectorid_vid_.at(id).first; + }); if (result.has_value()) { for (int64_t i = 0; i < result.value()->GetDim(); ++i) { - ret.emplace_back(result.value()->GetIds()[i], result.value()->GetDistances()[i]); + auto vector_id = result.value()->GetIds()[i]; + ret.emplace_back(vectorid_vid_.at(vector_id).second, result.value()->GetDistances()[i]); } } else { THROW_CODE(VectorIndexException, result.error().message); @@ -208,10 +240,13 @@ HNSW::RangeSearch(const std::vector& query, float radius, int ef_search, {"hnsw", {{"ef_search", ef_search}}}, }; std::vector> ret; - auto result = index_->RangeSearch(dataset, radius, parameters.dump(), limit); + auto result = index_->RangeSearch(dataset, radius, parameters.dump(), [this](int64_t id)->bool { + return vectorid_vid_.at(id).first; + }, limit); if (result.has_value()) { for (int64_t i = 0; i < result.value()->GetDim(); ++i) { - ret.emplace_back(result.value()->GetIds()[i], result.value()->GetDistances()[i]); + int64_t vector_id = result.value()->GetIds()[i]; + ret.emplace_back(vectorid_vid_.at(vector_id).second, result.value()->GetDistances()[i]); } } else { THROW_CODE(VectorIndexException, result.error().message); @@ -219,4 +254,16 @@ HNSW::RangeSearch(const std::vector& query, float radius, int ef_search, return ret; } +int64_t HNSW::GetElementsNum() { + return index_->GetNumElements(); +} + +int64_t HNSW::GetMemoryUsage() { + return index_->GetMemoryUsage(); +} + +int64_t HNSW::GetDeletedIdsNum() { + return deleted_vectorid_; +} + } // namespace lgraph diff --git a/src/core/vsag_hnsw.h b/src/core/vsag_hnsw.h index d520a8a40d..438807cfbb 100644 --- a/src/core/vsag_hnsw.h +++ b/src/core/vsag_hnsw.h @@ -30,9 +30,14 @@ class HNSW : public VectorIndex { friend class LightningGraph; friend class Transaction; friend class IndexManager; + int64_t vectorid_ = 0; + int64_t deleted_vectorid_ = 0; + std::unordered_map vid_vectorid_; + std::unordered_map> vectorid_vid_; + std::shared_ptr index_; - std::shared_ptr createindex_; - vsag::Index* index_; + // build index + void Build(); public: HNSW(const std::string& label, const std::string& name, @@ -43,7 +48,7 @@ class HNSW : public VectorIndex { HNSW(HNSW&& rhs) = delete; - ~HNSW() { index_ = nullptr; } + ~HNSW() override; HNSW& operator=(const HNSW& rhs) = delete; @@ -51,10 +56,11 @@ class HNSW : public VectorIndex { // add vector to index and build index void Add(const std::vector>& vectors, - const std::vector& vids, int64_t num_vectors) override; + const std::vector& vids) override; - // build index - void Build() override; + void Remove(const std::vector& vids) override; + + void Clear() override; // serialize index std::vector Save() override; @@ -69,6 +75,10 @@ class HNSW : public VectorIndex { std::vector> RangeSearch( const std::vector& query, float radius, int ef_search, int limit) override; + int64_t GetElementsNum() override; + int64_t GetMemoryUsage() override; + int64_t GetDeletedIdsNum() override; + template static void writeBinaryPOD(std::ostream& out, const T& podRef) { out.write((char*)&podRef, sizeof(T)); diff --git a/src/cypher/arithmetic/agg_funcs.h b/src/cypher/arithmetic/agg_funcs.h index 3d546510f2..600531e8bb 100644 --- a/src/cypher/arithmetic/agg_funcs.h +++ b/src/cypher/arithmetic/agg_funcs.h @@ -152,84 +152,11 @@ class MinAggCtx : public AggCtx { } }; -// TODO(anyone): remove duplicate, defined in filter.h -struct FieldDataHash { - size_t operator()(const lgraph_api::FieldData &fd) const { - using namespace lgraph_api; - switch (fd.type) { - case FieldType::BOOL: - return std::hash()(fd.AsBool()); - case FieldType::INT8: - return std::hash()(fd.AsInt8()); - case FieldType::INT16: - return std::hash()(fd.AsInt16()); - case FieldType::INT32: - return std::hash()(fd.AsInt32()); - case FieldType::INT64: - return std::hash()(fd.AsInt64()); - case FieldType::FLOAT: - return std::hash()(fd.AsFloat()); - case FieldType::DOUBLE: - return std::hash()(fd.AsDouble()); - case FieldType::DATE: - return std::hash()(fd.AsDate().DaysSinceEpoch()); - case FieldType::DATETIME: - return std::hash()(fd.AsDateTime().MicroSecondsSinceEpoch()); - case FieldType::STRING: - return std::hash()(fd.AsString()); - case FieldType::BLOB: - return std::hash()(fd.AsBlob()); - case FieldType::POINT: { - switch (fd.GetSRID()) { - case ::lgraph_api::SRID::WGS84: - return std::hash()(fd.AsWgsPoint().AsEWKB()); - case ::lgraph_api::SRID::CARTESIAN: - return std::hash()(fd.AsCartesianPoint().AsEWKB()); - default: - THROW_CODE(InputError, "unsupported spatial srid"); - } - } - case FieldType::LINESTRING: { - switch (fd.GetSRID()) { - case ::lgraph_api::SRID::WGS84: - return std::hash()(fd.AsWgsLineString().AsEWKB()); - case ::lgraph_api::SRID::CARTESIAN: - return std::hash()(fd.AsCartesianLineString().AsEWKB()); - default: - THROW_CODE(InputError, "unsupported spatial srid"); - } - } - case FieldType::POLYGON: { - switch (fd.GetSRID()) { - case ::lgraph_api::SRID::WGS84: - return std::hash()(fd.AsWgsPolygon().AsEWKB()); - case ::lgraph_api::SRID::CARTESIAN: - return std::hash()(fd.AsCartesianPolygon().AsEWKB()); - default: - THROW_CODE(InputError, "unsupported spatial srid"); - } - } - case FieldType::SPATIAL: { - switch (fd.GetSRID()) { - case ::lgraph_api::SRID::WGS84: - return std::hash()(fd.AsWgsSpatial().AsEWKB()); - case ::lgraph_api::SRID::CARTESIAN: - return std::hash()(fd.AsCartesianSpatial().AsEWKB()); - default: - THROW_CODE(InputError, "unsupported spatial srid"); - } - } - default: - throw std::runtime_error("Unhandled data type, probably corrupted data."); - } - } -}; - class CountAggCtx : public AggCtx { size_t count = 0; int distinct = -1; /* count(distinct sth), used to identify unique records. */ - std::unordered_set uset; + std::unordered_set uset; public: int Step(const std::vector &args) override { diff --git a/src/cypher/arithmetic/arithmetic_expression.cpp b/src/cypher/arithmetic/arithmetic_expression.cpp index 08e6e19a6d..cdc071465a 100644 --- a/src/cypher/arithmetic/arithmetic_expression.cpp +++ b/src/cypher/arithmetic/arithmetic_expression.cpp @@ -762,7 +762,7 @@ cypher::FieldData BuiltinFunction::DateTime(RTContext *ctx, const Record &record if (args.size() > 2) CYPHER_ARGUMENT_ERROR(); if (args.size() == 1) { // datetime() Returns the current DateTime. - return cypher::FieldData(::lgraph::FieldData(::lgraph::DateTime::Now())); + return cypher::FieldData(::lgraph::FieldData(::lgraph::DateTime::LocalNow())); } else { CYPHER_THROW_ASSERT(args.size() == 2); // datetime(string) Returns a DateTime by parsing a string. @@ -1292,11 +1292,140 @@ cypher::FieldData BuiltinFunction::Regexp(RTContext *ctx, const Record &record, } cypher::FieldData BuiltinFunction::Exists(RTContext *ctx, const Record &record, - const std::vector &args) { + const std::vector &args) { auto res = args[1].Evaluate(ctx, record); return cypher::FieldData(lgraph_api::FieldData(!res.IsNull())); } +cypher::FieldData BuiltinFunction::ToLower(RTContext *ctx, const Record &record, + const std::vector &args) { + if (args.size() != 2) CYPHER_ARGUMENT_ERROR(); + auto arg1_entry = args[1].Evaluate(ctx, record); + std::string arg1 = arg1_entry.constant.scalar.AsString(); + std::transform(arg1.begin(), arg1.end(), arg1.begin(), [](unsigned char c) { + return std::tolower(c); + }); + return cypher::FieldData(lgraph_api::FieldData(arg1)); +} + +cypher::FieldData BuiltinFunction::ToUpper(RTContext *ctx, const Record &record, + const std::vector &args) { + if (args.size() != 2) CYPHER_ARGUMENT_ERROR(); + auto arg1_entry = args[1].Evaluate(ctx, record); + std::string arg1 = arg1_entry.constant.scalar.AsString(); + std::transform(arg1.begin(), arg1.end(), arg1.begin(), [](unsigned char c) { + return std::toupper(c); + }); + return cypher::FieldData(lgraph_api::FieldData(arg1)); +} + +static std::string ltrim(const std::string& s) { + auto it = std::find_if(s.begin(), s.end(), [](unsigned char ch) { + return !std::isspace(ch); + }); + return {it, s.end()}; +} + +static std::string rtrim(const std::string& s) { + auto it = std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { + return !std::isspace(ch); + }); + return {s.begin(), it.base()}; +} + +static std::string trim(const std::string& s) { + return ltrim(rtrim(s)); +} + +cypher::FieldData BuiltinFunction::Trim(RTContext *ctx, const Record &record, + const std::vector &args) { + if (args.size() != 2) CYPHER_ARGUMENT_ERROR(); + auto arg1_entry = args[1].Evaluate(ctx, record); + std::string arg1 = arg1_entry.constant.scalar.AsString(); + return cypher::FieldData(lgraph_api::FieldData(trim(arg1))); +} + +cypher::FieldData BuiltinFunction::Ltrim(RTContext *ctx, const Record &record, + const std::vector &args) { + if (args.size() != 2) CYPHER_ARGUMENT_ERROR(); + auto arg1_entry = args[1].Evaluate(ctx, record); + std::string arg1 = arg1_entry.constant.scalar.AsString(); + return cypher::FieldData(lgraph_api::FieldData(ltrim(arg1))); +} + +cypher::FieldData BuiltinFunction::Rtrim(RTContext *ctx, const Record &record, + const std::vector &args) { + if (args.size() != 2) CYPHER_ARGUMENT_ERROR(); + auto arg1_entry = args[1].Evaluate(ctx, record); + std::string arg1 = arg1_entry.constant.scalar.AsString(); + return cypher::FieldData(lgraph_api::FieldData(rtrim(arg1))); +} + +cypher::FieldData BuiltinFunction::Replace(RTContext *ctx, const Record &record, + const std::vector &args) { + if (args.size() != 4) CYPHER_ARGUMENT_ERROR(); + auto arg1_entry = args[1].Evaluate(ctx, record); + std::string arg1 = arg1_entry.constant.scalar.AsString(); + auto arg2_entry = args[2].Evaluate(ctx, record); + std::string arg2 = arg2_entry.constant.scalar.AsString(); + auto arg3_entry = args[3].Evaluate(ctx, record); + std::string arg3 = arg3_entry.constant.scalar.AsString(); + size_t pos = 0; + while ((pos = arg1.find(arg2, pos)) != std::string::npos) { + arg1.replace(pos, arg2.length(), arg3); + pos += arg3.length(); + } + return cypher::FieldData(lgraph_api::FieldData(arg1)); +} + +cypher::FieldData BuiltinFunction::Split(RTContext *ctx, const Record &record, + const std::vector &args) { + if (args.size() != 3) CYPHER_ARGUMENT_ERROR(); + auto arg1_entry = args[1].Evaluate(ctx, record); + std::string arg1 = arg1_entry.constant.scalar.AsString(); + auto arg2_entry = args[2].Evaluate(ctx, record); + std::string arg2 = arg2_entry.constant.scalar.AsString(); + std::vector split_result; + boost::algorithm::split(split_result, arg1, boost::is_any_of(arg2)); + std::vector<::lgraph::FieldData> res; + for (auto& item : split_result) { + res.emplace_back(std::move(item)); + } + return cypher::FieldData(std::move(res)); +} + +cypher::FieldData BuiltinFunction::Left(RTContext *ctx, const Record &record, + const std::vector &args) { + if (args.size() != 3) CYPHER_ARGUMENT_ERROR(); + auto arg1_entry = args[1].Evaluate(ctx, record); + std::string arg1 = arg1_entry.constant.scalar.AsString(); + auto arg2_entry = args[2].Evaluate(ctx, record); + int64_t arg2 = arg2_entry.constant.scalar.AsInt64(); + if (arg2 < 0) THROW_CODE(InvalidParameter); + return cypher::FieldData(arg1.substr(0, std::min((size_t)arg2, arg1.size()))); +} + +cypher::FieldData BuiltinFunction::Right(RTContext *ctx, const Record &record, + const std::vector &args) { + if (args.size() != 3) CYPHER_ARGUMENT_ERROR(); + auto arg1_entry = args[1].Evaluate(ctx, record); + std::string arg1 = arg1_entry.constant.scalar.AsString(); + auto arg2_entry = args[2].Evaluate(ctx, record); + int64_t arg2 = arg2_entry.constant.scalar.AsInt64(); + if (arg2 < 0) THROW_CODE(InvalidParameter); + size_t len = std::min((size_t)arg2, arg1.size()); + return cypher::FieldData(arg1.substr(arg1.size() - len, len)); +} + +cypher::FieldData BuiltinFunction::Reverse(RTContext *ctx, const Record &record, + const std::vector &args) { + if (args.size() != 2) CYPHER_ARGUMENT_ERROR(); + auto arg1_entry = args[1].Evaluate(ctx, record); + std::string arg1 = arg1_entry.constant.scalar.AsString(); + std::reverse(arg1.begin(), arg1.end()); + return cypher::FieldData(lgraph_api::FieldData(arg1)); +} + cypher::FieldData BuiltinFunction::Bin(RTContext *ctx, const Record &record, const std::vector &args) { if (args.size() != 2) CYPHER_ARGUMENT_ERROR(); diff --git a/src/cypher/arithmetic/arithmetic_expression.h b/src/cypher/arithmetic/arithmetic_expression.h index ef996ae4ec..73c389eb85 100644 --- a/src/cypher/arithmetic/arithmetic_expression.h +++ b/src/cypher/arithmetic/arithmetic_expression.h @@ -361,6 +361,29 @@ struct BuiltinFunction { static cypher::FieldData Exists(RTContext *ctx, const Record &record, const std::vector &args); + static cypher::FieldData ToLower(RTContext *ctx, const Record &record, + const std::vector &args); + + static cypher::FieldData ToUpper(RTContext *ctx, const Record &record, + const std::vector &args); + + static cypher::FieldData Trim(RTContext *ctx, const Record &record, + const std::vector &args); + static cypher::FieldData Ltrim(RTContext *ctx, const Record &record, + const std::vector &args); + static cypher::FieldData Rtrim(RTContext *ctx, const Record &record, + const std::vector &args); + static cypher::FieldData Replace(RTContext *ctx, const Record &record, + const std::vector &args); + static cypher::FieldData Split(RTContext *ctx, const Record &record, + const std::vector &args); + static cypher::FieldData Left(RTContext *ctx, const Record &record, + const std::vector &args); + static cypher::FieldData Right(RTContext *ctx, const Record &record, + const std::vector &args); + static cypher::FieldData Reverse(RTContext *ctx, const Record &record, + const std::vector &args); + /* binary function (open cypher extension) */ static cypher::FieldData Bin(RTContext *ctx, const Record &record, const std::vector &args); @@ -585,11 +608,22 @@ struct ArithOpNode { ae_registered_funcs.emplace("polygon", BuiltinFunction::Polygon); ae_registered_funcs.emplace("polygonwkb", BuiltinFunction::PolygonWKB); ae_registered_funcs.emplace("polygonwkt", BuiltinFunction::PolygonWKT); + ae_registered_funcs.emplace("startswith", BuiltinFunction::StartsWith); ae_registered_funcs.emplace("endswith", BuiltinFunction::EndsWith); ae_registered_funcs.emplace("contains", BuiltinFunction::Contains); ae_registered_funcs.emplace("regexp", BuiltinFunction::Regexp); ae_registered_funcs.emplace("exists", BuiltinFunction::Exists); + ae_registered_funcs.emplace("tolower", BuiltinFunction::ToLower); + ae_registered_funcs.emplace("toupper", BuiltinFunction::ToUpper); + ae_registered_funcs.emplace("trim", BuiltinFunction::Trim); + ae_registered_funcs.emplace("ltrim", BuiltinFunction::Ltrim); + ae_registered_funcs.emplace("rtrim", BuiltinFunction::Rtrim); + ae_registered_funcs.emplace("replace", BuiltinFunction::Replace); + ae_registered_funcs.emplace("split", BuiltinFunction::Split); + ae_registered_funcs.emplace("left", BuiltinFunction::Left); + ae_registered_funcs.emplace("right", BuiltinFunction::Right); + ae_registered_funcs.emplace("reverse", BuiltinFunction::Reverse); /* native API-like functions */ ae_registered_funcs.emplace("native.getedgefield", BuiltinFunction::NativeGetEdgeField); diff --git a/src/cypher/cypher_types.h b/src/cypher/cypher_types.h index 1ad0a09f6a..928d0f0af1 100644 --- a/src/cypher/cypher_types.h +++ b/src/cypher/cypher_types.h @@ -26,7 +26,6 @@ namespace cypher { struct FieldData { typedef std::unordered_map CYPHER_FIELD_DATA_MAP; typedef std::vector CYPHER_FIELD_DATA_LIST; - // TODO(lingsu) : a default state should be added enum FieldType { SCALAR, ARRAY, MAP } type; lgraph::FieldData scalar; @@ -116,6 +115,7 @@ struct FieldData { map->emplace(kv); } } + explicit FieldData(const CYPHER_FIELD_DATA_MAP& rhs) { type = MAP; map = new CYPHER_FIELD_DATA_MAP(rhs); @@ -134,7 +134,7 @@ struct FieldData { map = new CYPHER_FIELD_DATA_MAP(std::move(rhs)); } - ~FieldData() { + ~FieldData() { switch (type) { case ARRAY: delete array; @@ -167,15 +167,16 @@ struct FieldData { switch (type) { case ARRAY: array = rhs.array; + rhs.array = nullptr; break; case MAP: map = rhs.map; + rhs.map = nullptr; break; case SCALAR: scalar = std::move(rhs.scalar); break; } - // TODO(lingsu) : rhs should return to the default state rhs.type = SCALAR; } diff --git a/src/cypher/execution_plan/ops/op_gql_traversal.h b/src/cypher/execution_plan/ops/op_gql_traversal.h index ecfc2a462c..ad38768935 100644 --- a/src/cypher/execution_plan/ops/op_gql_traversal.h +++ b/src/cypher/execution_plan/ops/op_gql_traversal.h @@ -64,7 +64,7 @@ class OpGqlTraversal : public OpBase { expands_; // /** result */ - cuckoohash_map map_; + cuckoohash_map map_; std::vector result_buffer_; size_t result_idx_; diff --git a/src/cypher/execution_plan/ops/op_node_by_label_scan.h b/src/cypher/execution_plan/ops/op_node_by_label_scan.h index 3a2163ffcf..184757eff1 100644 --- a/src/cypher/execution_plan/ops/op_node_by_label_scan.h +++ b/src/cypher/execution_plan/ops/op_node_by_label_scan.h @@ -26,8 +26,8 @@ class NodeByLabelScan : public OpBase { friend class LocateNodeByVidV2; friend class LocateNodeByIndexedProp; friend class LocateNodeByIndexedPropV2; + friend class LocateNodeByPropRangeFilter; - std::unique_ptr *txn_ = nullptr; Node *node_ = nullptr; lgraph::VIter *it_ = nullptr; // also cab be derived from node std::string alias_; // also can be derived from node @@ -36,21 +36,24 @@ class NodeByLabelScan : public OpBase { int rec_length_; // number of entries in a record. const SymbolTable *sym_tab_ = nullptr; // build time context bool consuming_ = false; // whether begin consuming + std::unordered_map> field_bounds_; public: - NodeByLabelScan(Node *node, const SymbolTable *sym_tab) + NodeByLabelScan(Node *node, const SymbolTable *sym_tab, + std::unordered_map> + field_bounds = {}) : OpBase(OpType::NODE_BY_LABEL_SCAN, "Node By Label Scan"), node_(node), sym_tab_(sym_tab) { - if (node) { - it_ = node->ItRef(); - alias_ = node->Alias(); - label_ = node->Label(); - modifies.emplace_back(alias_); - } + CYPHER_THROW_ASSERT(node); + it_ = node->ItRef(); + alias_ = node->Alias(); + label_ = node->Label(); + modifies.emplace_back(alias_); auto it = sym_tab->symbols.find(alias_); - CYPHER_THROW_ASSERT(node && it != sym_tab->symbols.end()); - if (it != sym_tab->symbols.end()) node_rec_idx_ = it->second.id; + CYPHER_THROW_ASSERT(it != sym_tab->symbols.end()); + node_rec_idx_ = it->second.id; rec_length_ = sym_tab->symbols.size(); consuming_ = false; + field_bounds_ = std::move(field_bounds); } OpResult Initialize(RTContext *ctx) override { @@ -59,11 +62,22 @@ class NodeByLabelScan : public OpBase { record->values[node_rec_idx_].type = Entry::NODE; record->values[node_rec_idx_].node = node_; // transaction allocated before in plan:execute - // TODO(anyone) remove patternGraph's state (ctx) - auto primary_filed = ctx->txn_->GetVertexPrimaryField(node_->Label()); - node_->ItRef()->Initialize(ctx->txn_->GetTxn().get(), lgraph::VIter::INDEX_ITER, - node_->Label(), primary_filed, - lgraph::FieldData(), lgraph::FieldData()); + auto primary_field = ctx->txn_->GetVertexPrimaryField(label_); + for (auto &[field, bounds] : field_bounds_) { + if (field == primary_field) { + break; + } + if (ctx->txn_->GetTxn()->IsIndexed(label_, field)) { + it_->Initialize(ctx->txn_->GetTxn().get(), lgraph::VIter::INDEX_ITER, label_, field, + bounds.first, bounds.second); + return OP_OK; + } + } + + // fallback to primary index + it_->Initialize(ctx->txn_->GetTxn().get(), lgraph::VIter::INDEX_ITER, label_, primary_field, + lgraph::FieldData(), lgraph::FieldData()); + return OP_OK; } @@ -104,7 +118,19 @@ class NodeByLabelScan : public OpBase { std::string ToString() const override { std::string str(name); + bool flag = false; str.append(" [").append(alias_).append(":").append(label_).append("]"); + + for (auto &[field, bounds] : field_bounds_) { + if (flag) { + str.append(","); + } + str.append(" ").append(alias_).append(".").append(field).append("["); + str.append(bounds.first.ToString()).append(","); + str.append(bounds.second.ToString()); + str.append(")"); + flag = true; + } return str; } diff --git a/src/cypher/execution_plan/ops/op_traversal.h b/src/cypher/execution_plan/ops/op_traversal.h index 98caa2652b..120d2ebf29 100644 --- a/src/cypher/execution_plan/ops/op_traversal.h +++ b/src/cypher/execution_plan/ops/op_traversal.h @@ -49,7 +49,7 @@ class Traversal : public OpBase { expands_; // /** result */ - cuckoohash_map map_; + cuckoohash_map map_; std::vector result_buffer_; size_t result_idx_; diff --git a/src/cypher/execution_plan/optimization/locate_node_by_indexed_prop_v2.h b/src/cypher/execution_plan/optimization/locate_node_by_indexed_prop_v2.h index 36aefc05a6..955916ff94 100644 --- a/src/cypher/execution_plan/optimization/locate_node_by_indexed_prop_v2.h +++ b/src/cypher/execution_plan/optimization/locate_node_by_indexed_prop_v2.h @@ -114,7 +114,7 @@ class LocateNodeByIndexedPropV2 : public OptPass { (op->children[0]->type == OpType::ALL_NODE_SCAN || op->children[0]->type == OpType::NODE_BY_LABEL_SCAN)) { op_filter = dynamic_cast(op); - if (_CheckPropFilter(op_filter, target_value_datas)) { + if (_CheckPropFilter(op_filter, target_value_datas)) { return true; } } diff --git a/src/cypher/execution_plan/optimization/locate_node_by_prop_range_filter.h b/src/cypher/execution_plan/optimization/locate_node_by_prop_range_filter.h new file mode 100644 index 0000000000..acbbc1f95b --- /dev/null +++ b/src/cypher/execution_plan/optimization/locate_node_by_prop_range_filter.h @@ -0,0 +1,121 @@ +/** + * Copyright 2024 AntGroup CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +#pragma once + +#include "core/data_type.h" +#include "cypher/execution_plan/ops/op_filter.h" +#include "cypher/execution_plan/ops/op_node_by_label_scan.h" +#include "cypher/execution_plan/optimization/opt_pass.h" +#include "cypher/execution_plan/optimization/property_range_filter_detector.h" + +namespace cypher { + +/* + +Before: + +Produce Results + Limit [5] + Project [n] + Filter [((n.born>1922) and (n.born<1989))] + Node By Label Scan [n:person] + +After: + +Produce Results + Limit [5] + Project [n] + Filter [((n.born>1922) and (n.born<1989))] + Node By Label Scan [n:person] n.born[1922,1989) +*/ + +class LocateNodeByPropRangeFilter : public OptPass { + public: + LocateNodeByPropRangeFilter() : OptPass(typeid(LocateNodeByPropRangeFilter).name()) {} + ~LocateNodeByPropRangeFilter() = default; + bool Gate() override { return true; } + int Execute(OpBase *root) override { + _AdjustNodePropRangeFilter(root); + return 0; + } + + private: + void _AdjustNodePropRangeFilter(OpBase *root) { + OpFilter *op_filter = nullptr; + std::unordered_map< + std::string, + std::unordered_map>> + target_value_datas; + + if (FindNodePropRangeFilter(root, op_filter, target_value_datas)) { + auto op_child = op_filter->children[0]; + CYPHER_THROW_ASSERT(op_child->type == OpType::NODE_BY_LABEL_SCAN); + + Node *node = dynamic_cast(op_child)->GetNode(); + const SymbolTable *symtab = dynamic_cast(op_child)->sym_tab_; + + auto it = target_value_datas.find(node->Alias()); + if (it == target_value_datas.end()) return; + + auto op_node_by_label_scan = + new NodeByLabelScan(node, symtab, std::move(it->second)); + op_filter->RemoveChild(op_child); + OpBase::FreeStream(op_child); + op_child = nullptr; + op_filter->AddChild(op_node_by_label_scan); + } + } + + bool _CheckPropRangeFilter( + OpFilter *&op_filter, + std::unordered_map< + std::string, + std::unordered_map>> + &target_value_datas) { + auto filter = op_filter->Filter(); + CYPHER_THROW_ASSERT(filter->Type() == lgraph::Filter::GEAX_EXPR_FILTER); + auto geax_filter = ((lgraph::GeaxExprFilter *)filter.get())->GetArithExpr(); + geax::frontend::Expr *expr = geax_filter.expr_; + PropertyRangeFilterDetector detector; + if (!detector.Build(expr)) return false; + target_value_datas = detector.GetProperties(); + if (target_value_datas.empty()) return false; + return true; + } + + bool FindNodePropRangeFilter( + OpBase *root, OpFilter *&op_filter, + std::unordered_map< + std::string, + std::unordered_map>> + &target_value_datas) { + auto op = root; + if (op->type == OpType::FILTER && op->children.size() == 1 && + op->children[0]->type == OpType::NODE_BY_LABEL_SCAN) { + op_filter = dynamic_cast(op); + if (_CheckPropRangeFilter(op_filter, target_value_datas)) { + return true; + } + } + + for (auto child : op->children) { + if (FindNodePropRangeFilter(child, op_filter, target_value_datas)) return true; + } + + return false; + } +}; + +} // namespace cypher diff --git a/src/cypher/execution_plan/optimization/pass_manager.h b/src/cypher/execution_plan/optimization/pass_manager.h index ccda354abc..8c546b4838 100644 --- a/src/cypher/execution_plan/optimization/pass_manager.h +++ b/src/cypher/execution_plan/optimization/pass_manager.h @@ -27,6 +27,7 @@ #include "cypher/execution_plan/optimization/opt_rewrite_with_schema_inference.h" #include "execution_plan/optimization/locate_node_by_vid_v2.h" #include "execution_plan/optimization/locate_node_by_indexed_prop_v2.h" +#include "execution_plan/optimization/locate_node_by_prop_range_filter.h" #include "execution_plan/optimization/parallel_traversal_v2.h" namespace cypher { @@ -48,6 +49,7 @@ class PassManager { all_passes_.emplace_back(new ParallelTraversalV2()); all_passes_.emplace_back(new LocateNodeByVidV2()); all_passes_.emplace_back(new LocateNodeByIndexedPropV2()); + all_passes_.emplace_back(new LocateNodeByPropRangeFilter()); } ~PassManager() { diff --git a/src/cypher/execution_plan/optimization/property_range_filter_detector.h b/src/cypher/execution_plan/optimization/property_range_filter_detector.h new file mode 100644 index 0000000000..bc7f4342fb --- /dev/null +++ b/src/cypher/execution_plan/optimization/property_range_filter_detector.h @@ -0,0 +1,205 @@ +/** + * Copyright 2024 AntGroup CO., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License") {} + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +#pragma once + +#include "core/data_type.h" +#include "cypher/execution_plan/optimization/optimization_filter_visitor_impl.h" + +namespace cypher { + +class PropertyRangeFilterDetector : public cypher::OptimizationFilterVisitorImpl { + public: + PropertyRangeFilterDetector() {} + + virtual ~PropertyRangeFilterDetector() = default; + + bool Build(geax::frontend::AstNode* astNode) { + try { + if (std::any_cast(astNode->accept(*this)) != + geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_PASS) { + return false; + } + } catch (const lgraph::CypherException& e) { + return false; + } + return true; + } + + std::unordered_map< + std::string, + std::unordered_map>>& + GetProperties() { + return properties_; + } + + private: + std::unordered_map< + std::string, + std::unordered_map>> + properties_; + std::string cur_symbol_; + std::string cur_field_; + lgraph::FieldData cur_value_; + bool is_conjunction_ = false; + bool is_disjunction_ = false; + std::pair cur_bounds_; + + bool in_bounds(const std::pair& bounds, + const lgraph::FieldData& value) { + CYPHER_THROW_ASSERT(!bounds.first.is_null() || !bounds.second.is_null()); + if (!bounds.first.is_null() && !bounds.second.is_null()) { + return value > bounds.first && value < bounds.second; + } + if (!bounds.first.is_null()) { + return value > bounds.first; + } + if (!bounds.second.is_null()) { + return value < bounds.second; + } + return false; + } + + /* merge cur_bounds_ into bounds, return true if the merge succeed, false else */ + bool merge_bounds(std::pair& bounds) { + CYPHER_THROW_ASSERT(is_conjunction_ != is_disjunction_); + if (is_conjunction_) { + if (!cur_bounds_.first.is_null()) { + if (in_bounds(bounds, cur_bounds_.first)) { + bounds.first = cur_bounds_.first; + } else if (cur_bounds_.first >= bounds.second) { + return false; + } + } else { + if (in_bounds(bounds, cur_bounds_.second)) { + bounds.second = cur_bounds_.second; + } else if (cur_bounds_.second <= bounds.first) { + return false; + } + } + } else { + if (!cur_bounds_.first.is_null()) { + if (bounds.first.is_null()) { + return false; + } else if (cur_bounds_.first < bounds.first) { + bounds.first = cur_bounds_.first; + } + } else { + if (bounds.second.is_null()) { + return false; + } else if (cur_bounds_.second > bounds.second) { + bounds.second = cur_bounds_.second; + } + } + } + // reset cur_bounds_ + cur_bounds_.first = lgraph::FieldData(); + cur_bounds_.second = lgraph::FieldData(); + return true; + } + + std::any visit(geax::frontend::BOr* node) override { + ACCEPT_AND_CHECK_WITH_PASS_MSG(node->left()); + ACCEPT_AND_CHECK_WITH_PASS_MSG(node->right()); + if (is_conjunction_) { + return geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_NOT_PASS; + } + is_disjunction_ = true; + if (!cur_bounds_.first.is_null() || !cur_bounds_.second.is_null()) { + // get the bookkeeping bounds and do the merge + auto& fields_map = properties_[cur_symbol_]; + auto& field_bounds = fields_map[cur_field_]; + // if merge failed, optimization not pass + if (!merge_bounds(field_bounds)) { + return geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_NOT_PASS; + } + } + return geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_PASS; + } + + std::any visit(geax::frontend::BAnd* node) override { + ACCEPT_AND_CHECK_WITH_PASS_MSG(node->left()); + ACCEPT_AND_CHECK_WITH_PASS_MSG(node->right()); + if (is_disjunction_) { + return geax::frontend::GEAXErrorCode::GEAX_COMMON_NOT_SUPPORT; + } + is_conjunction_ = true; + if (!cur_bounds_.first.is_null() || !cur_bounds_.second.is_null()) { + // get the bookkeeping bounds and do the merge + auto& fields_map = properties_[cur_symbol_]; + auto& field_bounds = fields_map[cur_field_]; + // if merge failed, optimization not pass + if (!merge_bounds(field_bounds)) { + return geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_NOT_PASS; + } + } + return geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_PASS; + } + + std::any visit(geax::frontend::GetField* node) override { + cur_field_ = node->fieldName(); + ACCEPT_AND_CHECK_WITH_PASS_MSG(node->expr()); + return geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_PASS; + } + + std::any visit(geax::frontend::Ref* node) override { + cur_symbol_ = node->name(); + return geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_PASS; + } + + std::any visit(geax::frontend::BSmallerThan* node) override { + ACCEPT_AND_CHECK_WITH_PASS_MSG(node->left()); + ACCEPT_AND_CHECK_WITH_PASS_MSG(node->right()); + auto& fields_map = properties_[cur_symbol_]; + auto& field_bounds = fields_map[cur_field_]; + if (field_bounds.first.is_null() && field_bounds.second.is_null()) { + // left bound and right bound both not set, set one + field_bounds.second = cur_value_; + } else { + // set cur_bounds and do the merge later + cur_bounds_.first = lgraph::FieldData(); + cur_bounds_.second = cur_value_; + } + return geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_PASS; + } + + std::any visit(geax::frontend::BGreaterThan* node) override { + ACCEPT_AND_CHECK_WITH_PASS_MSG(node->left()); + ACCEPT_AND_CHECK_WITH_PASS_MSG(node->right()); + auto& fields_map = properties_[cur_symbol_]; + auto& field_bounds = fields_map[cur_field_]; + if (field_bounds.first.is_null() && field_bounds.second.is_null()) { + // left bound and right bound both not set, set one + field_bounds.first = cur_value_; + } else { + // set cur_bounds and do the merge later + cur_bounds_.first = cur_value_; + cur_bounds_.second = lgraph::FieldData(); + } + + return geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_PASS; + } + + std::any visit(geax::frontend::VInt* node) override { + cur_value_ = lgraph::FieldData(node->val()); + return geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_PASS; + } + + std::any visit(geax::frontend::VDouble* node) override { + cur_value_ = lgraph::FieldData(node->val()); + return geax::frontend::GEAXErrorCode::GEAX_OPTIMIZATION_PASS; + } +}; + +} // namespace cypher diff --git a/src/cypher/filter/filter.h b/src/cypher/filter/filter.h index 2ab8d9bd70..22d3fbe32c 100644 --- a/src/cypher/filter/filter.h +++ b/src/cypher/filter/filter.h @@ -34,83 +34,6 @@ class LocateNodeByIndexedProp; namespace lgraph { -struct FieldDataHash { - size_t operator()(const lgraph::FieldData &fd) const { - switch (fd.type) { - case FieldType::NUL: - return 0; - case FieldType::BOOL: - return std::hash()(fd.AsBool()); - case FieldType::INT8: - return std::hash()(fd.AsInt8()); - case FieldType::INT16: - return std::hash()(fd.AsInt16()); - case FieldType::INT32: - return std::hash()(fd.AsInt32()); - case FieldType::INT64: - return std::hash()(fd.AsInt64()); - case FieldType::FLOAT: - return std::hash()(fd.AsFloat()); - case FieldType::DOUBLE: - return std::hash()(fd.AsDouble()); - case FieldType::DATE: - return std::hash()(fd.AsDate().DaysSinceEpoch()); - case FieldType::DATETIME: - return std::hash()(fd.AsDateTime().MicroSecondsSinceEpoch()); - case FieldType::STRING: - return std::hash()(fd.AsString()); - case FieldType::BLOB: - return std::hash()(fd.AsBlob()); - case FieldType::POINT: - { - switch (fd.GetSRID()) { - case ::lgraph_api::SRID::WGS84: - return std::hash()(fd.AsWgsPoint().AsEWKB()); - case ::lgraph_api::SRID::CARTESIAN: - return std::hash()(fd.AsCartesianPoint().AsEWKB()); - default: - THROW_CODE(InputError, "unsupported spatial srid"); - } - } - case FieldType::LINESTRING: - { - switch (fd.GetSRID()) { - case ::lgraph_api::SRID::WGS84: - return std::hash()(fd.AsWgsLineString().AsEWKB()); - case ::lgraph_api::SRID::CARTESIAN: - return std::hash()(fd.AsCartesianLineString().AsEWKB()); - default: - THROW_CODE(InputError, "unsupported spatial srid"); - } - } - case FieldType::POLYGON: - { - switch (fd.GetSRID()) { - case ::lgraph_api::SRID::WGS84: - return std::hash()(fd.AsWgsPolygon().AsEWKB()); - case ::lgraph_api::SRID::CARTESIAN: - return std::hash()(fd.AsCartesianPolygon().AsEWKB()); - default: - THROW_CODE(InputError, "unsupported spatial srid"); - } - } - case FieldType::SPATIAL: - { - switch (fd.GetSRID()) { - case ::lgraph_api::SRID::WGS84: - return std::hash()(fd.AsWgsSpatial().AsEWKB()); - case ::lgraph_api::SRID::CARTESIAN: - return std::hash()(fd.AsCartesianSpatial().AsEWKB()); - default: - THROW_CODE(InputError, "unsupported spatial srid"); - } - } - default: - throw std::runtime_error("Unhandled data type, probably corrupted data."); - } - } -}; - class Filter { public: enum Type { @@ -608,7 +531,7 @@ class TestNullFilter : public Filter { class TestInFilter : public Filter { cypher::ArithExprNode ae_left_; cypher::ArithExprNode ae_right_; - std::unordered_set right_set_; + std::unordered_set right_set_; cypher::OpBase *producer_op_ = nullptr; size_t timestamp; diff --git a/src/cypher/parser/cypher_base_visitor.h b/src/cypher/parser/cypher_base_visitor.h index 875a5c34d9..4a6c24598b 100644 --- a/src/cypher/parser/cypher_base_visitor.h +++ b/src/cypher/parser/cypher_base_visitor.h @@ -1418,7 +1418,7 @@ class CypherBaseVisitor : public LcypherVisitor { // result, here `_listcompr_placeholder` = "x" // // I try to add symbol in `visitOC_ListComprephension` function, a cypher todo - // exception is thrown during exection. Fix this problem makes a lot adjustment + // exception is thrown during execution. Fix this problem makes a lot adjustment // or refactor work, may break the whole semantic analysis robust. // // Also, the `_InClauseORDERBY()` condition is a WORKAROUND for that, `AddSymbol` cannot diff --git a/src/cypher/procedure/procedure.cpp b/src/cypher/procedure/procedure.cpp index 2d37b468f9..f39a9ca2a5 100644 --- a/src/cypher/procedure/procedure.cpp +++ b/src/cypher/procedure/procedure.cpp @@ -579,7 +579,7 @@ void BuiltinProcedure::DbUpsertVertex(RTContext *ctx, const Record *record, } else { fd = item.second.scalar; } - if (fd.IsNull()) { + if (fd.IsNull() && !iter->second.second.optional) { success = false; break; } @@ -910,7 +910,7 @@ void BuiltinProcedure::DbUpsertEdge(RTContext *ctx, const Record *record, } else { fd = item.second.scalar; } - if (fd.IsNull()) { + if (fd.IsNull() && !iter->second.second.optional) { success = false; break; } @@ -2241,6 +2241,7 @@ void BuiltinProcedure::DbmsGraphGetGraphSchema(RTContext *ctx, const Record *rec if (vi) { property["index"] = true; property["unique"] = vi->IsUnique(); + property["pair_unique"] = vi->IsPairUnique(); } edge["properties"].push_back(property); } @@ -4188,19 +4189,19 @@ void VectorFunc::AddVertexVectorIndex(RTContext *ctx, const cypher::Record *reco } CYPHER_ARG_CHECK((distance_type == "l2" || distance_type == "ip"), "Distance type should be one of them : l2, ip"); - int hnsm_m = 16; - if (parameter.count("hnsm_m")) { - hnsm_m = (int)parameter.at("hnsm_m").AsInt64(); - } - CYPHER_ARG_CHECK((hnsm_m <= 64 && hnsm_m >= 5), - "hnsm.m should be an integer in the range [5, 64]"); - int hnsm_ef_construction = 100; - if (parameter.count("hnsm_ef_construction")) { - hnsm_ef_construction = (int)parameter.at("hnsm_ef_construction").AsInt64(); - } - CYPHER_ARG_CHECK((hnsm_ef_construction <= 1000 && hnsm_ef_construction >= hnsm_m), - "hnsm.efConstruction should be an integer in the range [hnsm.m,1000]"); - std::vector index_spec = {hnsm_m, hnsm_ef_construction}; + int hnsw_m = 16; + if (parameter.count("hnsw_m")) { + hnsw_m = (int)parameter.at("hnsw_m").AsInt64(); + } + CYPHER_ARG_CHECK((hnsw_m <= 64 && hnsw_m >= 5), + "hnsw.m should be an integer in the range [5, 64]"); + int hnsw_ef_construction = 100; + if (parameter.count("hnsw_ef_construction")) { + hnsw_ef_construction = (int)parameter.at("hnsw_ef_construction").AsInt64(); + } + CYPHER_ARG_CHECK((hnsw_ef_construction <= 1000 && hnsw_ef_construction >= hnsw_m), + "hnsw.efConstruction should be an integer in the range [hnsw.m,1000]"); + std::vector index_spec = {hnsw_m, hnsw_ef_construction}; auto ac_db = ctx->galaxy_->OpenGraph(ctx->user_, ctx->graph_); bool success = ac_db.AddVectorIndex(true, label, field, index_type, dimension, distance_type, index_spec); @@ -4251,8 +4252,12 @@ void VectorFunc::ShowVertexVectorIndex(RTContext *ctx, const cypher::Record *rec r.AddConstant(lgraph::FieldData(item.index_type)); r.AddConstant(lgraph::FieldData(item.dimension)); r.AddConstant(lgraph::FieldData(item.distance_type)); - r.AddConstant(lgraph::FieldData(item.hnsm_m)); - r.AddConstant(lgraph::FieldData(item.hnsm_ef_construction)); + r.AddConstant(lgraph::FieldData(item.hnsw_m)); + r.AddConstant(lgraph::FieldData(item.hnsw_ef_construction)); + auto index = ctx->txn_->GetTxn()->GetVertexVectorIndex(item.label, item.field); + r.AddConstant(lgraph::FieldData(index->GetElementsNum())); + r.AddConstant(lgraph::FieldData(index->GetMemoryUsage())); + r.AddConstant(lgraph::FieldData(index->GetDeletedIdsNum())); records->emplace_back(r.Snapshot()); } FillProcedureYieldItem("db.showVertexVectorIndex", yield_items, records); diff --git a/src/cypher/procedure/procedure.h b/src/cypher/procedure/procedure.h index 3373c5e86c..23cd629dca 100644 --- a/src/cypher/procedure/procedure.h +++ b/src/cypher/procedure/procedure.h @@ -948,8 +948,11 @@ static std::vector global_procedures = { {"index_type", {2, lgraph_api::LGraphType::STRING}}, {"dimension", {3, lgraph_api::LGraphType::INTEGER}}, {"distance_type", {4, lgraph_api::LGraphType::STRING}}, - {"hnsm.m", {5, lgraph_api::LGraphType::INTEGER}}, - {"hnsm.ef_construction", {6, lgraph_api::LGraphType::INTEGER}}, + {"hnsw.m", {5, lgraph_api::LGraphType::INTEGER}}, + {"hnsw.ef_construction", {6, lgraph_api::LGraphType::INTEGER}}, + {"elements_num", {7, lgraph_api::LGraphType::INTEGER}}, + {"memory_usage", {8, lgraph_api::LGraphType::INTEGER}}, + {"deleted_ids_num", {9, lgraph_api::LGraphType::INTEGER}}, }), Procedure("db.vertexVectorKnnSearch", VectorFunc::VertexVectorKnnSearch, diff --git a/src/server/bolt_handler.cpp b/src/server/bolt_handler.cpp index c9d1f8398e..30c599e2f5 100644 --- a/src/server/bolt_handler.cpp +++ b/src/server/bolt_handler.cpp @@ -255,6 +255,17 @@ std::function&>(fields[0]); + if (!val.count("principal") || !val.count("credentials")) { + LOG_ERROR() << "Hello msg fields error, " + "'principal' or 'credentials' are missing."; + bolt::PackStream ps; + ps.AppendFailure({{"code", "error"}, + {"message", "Hello msg fields error, " + "'principal' or 'credentials' are missing."}}); + conn.Respond(std::move(ps.MutableBuffer())); + conn.Close(); + return; + } auto& principal = std::any_cast(val.at("principal")); auto& credentials = std::any_cast(val.at("credentials")); auto galaxy = BoltServer::Instance().StateMachine()->GetGalaxy(); @@ -285,9 +296,9 @@ std::functionstate = SessionState::READY; session->user = principal; + conn.SetContext(session); session->fsm_thread = std::thread(BoltFSM, conn.shared_from_this()); session->fsm_thread.detach(); - conn.SetContext(std::move(session)); bolt::PackStream ps; ps.AppendSuccess(meta); conn.Respond(std::move(ps.MutableBuffer())); diff --git a/src/server/lgraph_server.cpp b/src/server/lgraph_server.cpp index 0c23619e57..f989fc5796 100644 --- a/src/server/lgraph_server.cpp +++ b/src/server/lgraph_server.cpp @@ -39,31 +39,55 @@ namespace brpc { DECLARE_bool(usercode_in_pthread); } #endif +#include namespace lgraph { static Signal _kill_signal_; -void int_handler(int x) { - LOG_INFO() << "!!!!! Received signal " << x << ", exiting... !!!!!"; +void shutdown_handler(int sig) { + LOG_WARN() << FMA_FMT("Received signal {}, shutdown", std::string(strsignal(sig))); + lgraph_log::LoggerManager::GetInstance().FlushAllSinks(); _kill_signal_.Notify(); } +void crash_handler(int sig) { + LOG_ERROR() << FMA_FMT("Received signal {}, crash", std::string(strsignal(sig))); + lgraph_log::LoggerManager::GetInstance().FlushAllSinks(); + + struct sigaction sa{}; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sa.sa_handler = SIG_DFL; + sigaction(sig, &sa, nullptr); + kill(getpid(), sig); +} + #ifndef _WIN32 #include static void SetupSignalHandler() { - struct sigaction sa {}; - sa.sa_handler = int_handler; - sigemptyset(&sa.sa_mask); - sa.sa_flags = 0; - int r = sigaction(SIGINT, &sa, nullptr); - if (r != 0) { - LOG_ERROR() << "Error setting up SIGINT signal handler: " << strerror(errno); + { + // shutdown + struct sigaction sa{}; + sa.sa_handler = shutdown_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(SIGINT, &sa, nullptr); + sigaction(SIGTERM, &sa, nullptr); + sigaction(SIGUSR1, &sa, nullptr); } - r = sigaction(SIGQUIT, &sa, nullptr); - if (r != 0) { - LOG_ERROR() << "Error setting up SIGQUIT signal handler: " << strerror(errno); + { + // crash + struct sigaction sa{}; + sa.sa_handler = crash_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_NODEFER; + sigaction(SIGSEGV, &sa, nullptr); + sigaction(SIGBUS, &sa, nullptr); + sigaction(SIGFPE, &sa, nullptr); + sigaction(SIGILL, &sa, nullptr); + sigaction(SIGABRT, &sa, nullptr); } } @@ -164,6 +188,17 @@ int LGraphServer::Start() { << "Server is configured with the following parameters:\n" << config_->FormatAsString(); LOG_INFO() << header.str(); + struct rlimit rlim{}; + getrlimit(RLIMIT_CORE, &rlim); + LOG_INFO() << FMA_FMT("Core dump file limit size, soft limit: {}, hard limit: {}", + rlim.rlim_cur, rlim.rlim_max); + std::ifstream file("/proc/sys/kernel/core_pattern"); + if (file.is_open()) { + std::string content((std::istreambuf_iterator(file)), + (std::istreambuf_iterator())); + LOG_INFO() << "Core dump file path: " << content; + file.close(); + } // starting audit log if (config_->audit_log_dir.empty()) diff --git a/src/server/service.h b/src/server/service.h index 078969276c..a9f939eaf6 100644 --- a/src/server/service.h +++ b/src/server/service.h @@ -213,8 +213,7 @@ class Service { LOG_WARN() << "Service " << service_name_ << " is already dead."; return NO_PID_FILE; } - // int r = kill(pid, SIGINT); - int r = kill(pid, SIGQUIT); + int r = kill(pid, SIGUSR1); if (r == EPERM) { LOG_WARN() << "Failed to kill service " << service_name_ << ": " << strerror(errno); return OTHER_ERROR; diff --git a/test/cypher_plan_validate.json b/test/cypher_plan_validate.json index b8072431b5..5869805186 100644 --- a/test/cypher_plan_validate.json +++ b/test/cypher_plan_validate.json @@ -115,6 +115,26 @@ "query": "match(c:City)<-[r:BORN_IN]-(p:Person) where c.name = 'London' and p.name <> 'Christopher Nolan' return r", "plan": "ReadOnly:1\nExecution Plan:\nProduce Results\n Project [r]\n Filter [(p.name!=\"Christopher Nolan\")]\n Expand(All) [c <-- p ]\n Node Index Seek [c] IN []\n", "res": 2 + }, + { + "query": "match(n:Person) where n.birthyear > 1900 return n limit 5", + "plan": "ReadOnly:1\nExecution Plan:\nProduce Results\n Limit [5]\n Project [n]\n Filter [(n.birthyear>1900)]\n Node By Label Scan [n:Person] n.birthyear[1900,NUL)\n", + "res": 5 + }, + { + "query": "match(n:Person) where n.birthyear < 2000 return n limit 5", + "plan": "ReadOnly:1\nExecution Plan:\nProduce Results\n Limit [5]\n Project [n]\n Filter [(n.birthyear<2000)]\n Node By Label Scan [n:Person] n.birthyear[NUL,2000)\n", + "res": 5 + }, + { + "query": "match(n:Person) where n.birthyear > 1900 and n.birthyear < 2000 return n limit 5", + "plan": "ReadOnly:1\nExecution Plan:\nProduce Results\n Limit [5]\n Project [n]\n Filter [((n.birthyear>1900) and (n.birthyear<2000))]\n Node By Label Scan [n:Person] n.birthyear[1900,2000)\n", + "res": 5 + }, + { + "query": "match(n:Person) where n.birthyear > 1900 or n.birthyear > 2000 return n limit 5", + "plan": "ReadOnly:1\nExecution Plan:\nProduce Results\n Limit [5]\n Project [n]\n Filter [((n.birthyear>1900) or (n.birthyear>2000))]\n Node By Label Scan [n:Person] n.birthyear[1900,NUL)\n", + "res": 5 } ], "start_vertex": [ diff --git a/test/resource/unit_test/expression/cypher/expression.result b/test/resource/unit_test/expression/cypher/expression.result index a3d552ccc6..6094283e05 100644 --- a/test/resource/unit_test/expression/cypher/expression.result +++ b/test/resource/unit_test/expression/cypher/expression.result @@ -254,3 +254,23 @@ RETURN datetimeComponent(1582705717, 'hour'),datetimeComponent(1582705717, 'minu [{"datetimeComponent(1582705717, 'hour')":0,"datetimeComponent(1582705717, 'microsecond')":705717,"datetimeComponent(1582705717, 'minute')":26,"datetimeComponent(1582705717, 'second')":22}] RETURN datetimeComponent(1582705717000, 'year'),datetimeComponent(1582705717000, 'second'), datetimeComponent(1582705717000, 'microsecond'); [{"datetimeComponent(1582705717000, 'microsecond')":717000,"datetimeComponent(1582705717000, 'second')":25,"datetimeComponent(1582705717000, 'year')":1970}] +RETURN toLower("TuGraph"); +[{"toLower(\"TuGraph\")":"tugraph"}] +RETURN toUpper("TuGraph"); +[{"toUpper(\"TuGraph\")":"TUGRAPH"}] +RETURN trim(" TuGraph "); +[{"trim(\" TuGraph \")":"TuGraph"}] +RETURN replace("Hello TuGraph", "TuGraph", "World"); +[{"replace(\"Hello TuGraph\", \"TuGraph\", \"World\")":"Hello World"}] +RETURN split("TuGraph,Graph,Database", ","); +[{"split(\"TuGraph,Graph,Database\", \",\")":"[TuGraph,Graph,Database]"}] +RETURN left("TuGraph", 3); +[{"left(\"TuGraph\", 3)":"TuG"}] +RETURN right("TuGraph", 2); +[{"right(\"TuGraph\", 2)":"ph"}] +RETURN reverse("TuGraph"); +[{"reverse(\"TuGraph\")":"hparGuT"}] +RETURN lTrim(" TuGraph "); +[{"lTrim(\" TuGraph \")":"TuGraph "}] +RETURN rTrim(" TuGraph "); +[{"rTrim(\" TuGraph \")":" TuGraph"}] diff --git a/test/resource/unit_test/expression/cypher/expression.test b/test/resource/unit_test/expression/cypher/expression.test index 883a482143..ccec60faa7 100644 --- a/test/resource/unit_test/expression/cypher/expression.test +++ b/test/resource/unit_test/expression/cypher/expression.test @@ -126,4 +126,14 @@ WITH bin('MjAyMAo=') AS bin1, bin('MjAxOQo=') AS bin2 RETURN bin1>bin2,bin1":"created 1 vertices, created 0 edges."}] CREATE (n:person {id:2, name:'name2', embedding1: [2.0,2.0,2.0,2.0], embedding2: [12.0,12.0,12.0,12.0]}); @@ -31,17 +31,17 @@ match(n:person {id:2}) delete n; CALL db.vertexVectorKnnSearch('person','embedding1',[1,2,3,4], {top_k:2, hnsw_ef_search:10}) yield node return node.id; [{"node.id":3},{"node.id":4}] CALL db.vertexVectorRangeSearch('person','embedding1', [1.0,2.0,3.0,4.0], {radius:10.0, hnsw_ef_search:10}) yield node,distance return node.id, distance; -[VectorIndexException] failed to perofrm range_search(internalError): not support perform range search on a index that deleted some vectors +[{"distance":6.0,"node.id":3}] CALL db.alterLabelDelFields('vertex', 'person', ['embedding1']); [{"record_affected":3}] CALL db.showVertexVectorIndex(); -[{"dimension":4,"distance_type":"l2","field_name":"embedding2","hnsm.ef_construction":100,"hnsm.m":16,"index_type":"hnsw","label_name":"person"}] +[{"deleted_ids_num":1,"dimension":4,"distance_type":"l2","elements_num":4,"field_name":"embedding2","hnsw.ef_construction":100,"hnsw.m":16,"index_type":"hnsw","label_name":"person","memory_usage":244345476}] CALL db.vertexVectorKnnSearch('person','embedding1',[1,2,3,4], {top_k:2, hnsw_ef_search:10}) yield node return node.id; [FieldNotFound] Field [embedding1] does not exist. CALL db.vertexVectorKnnSearch('person','embedding2',[1,2,3,4], {top_k:2, hnsw_ef_search:10}) yield node return node.id; [{"node.id":1},{"node.id":3}] CALL db.vertexVectorRangeSearch('person','embedding2', [1.0,2.0,3.0,4.0], {radius:10.0, hnsw_ef_search:10}) yield node,distance return node.id, distance; -[VectorIndexException] failed to perofrm range_search(internalError): not support perform range search on a index that deleted some vectors +[] CALL db.createVertexLabelByJson('{"label":"student","primary":"id","type":"VERTEX","detach_property":true,"properties":[{"name":"id","type":"INT32","optional":false},{"name":"name","type":"STRING","optional":false,"index":false},{"name":"embedding","type":"FLOAT_VECTOR","optional":false}]}'); [] @@ -57,3 +57,70 @@ CALL db.vertexVectorRangeSearch('student','embedding', [1.0,2.0,3.0,4.0], {radiu [{"distance":6.0,"node.id":2},{"distance":6.0,"node.id":3}] CALL db.vertexVectorRangeSearch('student','embedding', [1.0,2.0,3.0,4.0], {radius:10.0, hnsw_ef_search:10, limit:1}) yield node,distance return node.id, distance; [{"distance":6.0,"node.id":2}] + +CALL db.createVertexLabelByJson('{"label":"Chunk","primary":"id","type":"VERTEX","detach_property":true,"properties":[{"name":"id","type":"INT32","optional":false},{"name":"embedding","type":"FLOAT_VECTOR","optional":true}]}'); +[] +CALL db.addVertexVectorIndex('Chunk','embedding', {dimension:4}); +[] +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":0,"elements_num":0,"label_name":"Chunk"}] +CALL db.upsertVertex('Chunk', [{id:1},{id:2}]); +[{"data_error":0,"index_conflict":0,"insert":2,"total":2,"update":0}] +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":0,"elements_num":0,"label_name":"Chunk"}] +CALL db.upsertVertex('Chunk', [{id:1, embedding:[0.1,0.1,0.1,0.1]},{id:2, embedding:[-0.2,-0.2,-0.2,-0.2]}]); +[{"data_error":0,"index_conflict":0,"insert":0,"total":2,"update":2}] +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":0,"elements_num":2,"label_name":"Chunk"}] +CALL db.vertexVectorKnnSearch('Chunk','embedding',[1,2,3,4], {top_k:10, hnsw_ef_search:10}); +[{"distance":28.04,"node":{"identity":7,"label":"Chunk","properties":{"embedding":[0.10000000149011612,0.10000000149011612,0.10000000149011612,0.10000000149011612],"id":1}}},{"distance":34.16,"node":{"identity":8,"label":"Chunk","properties":{"embedding":[-0.20000000298023224,-0.20000000298023224,-0.20000000298023224,-0.20000000298023224],"id":2}}}] +CALL db.upsertVertex('Chunk', [{id:1, embedding:[0.1,0.1,0.1,0.1]},{id:2, embedding:[-0.2,-0.2,-0.2,-0.2]}, {id:3, embedding:[0.3,0.3,0.3,0.3]}]); +[{"data_error":0,"index_conflict":0,"insert":1,"total":3,"update":2}] +CALL db.vertexVectorKnnSearch('Chunk','embedding',[1,2,3,4], {top_k:10, hnsw_ef_search:10}); +[{"distance":24.36,"node":{"identity":9,"label":"Chunk","properties":{"embedding":[0.30000001192092896,0.30000001192092896,0.30000001192092896,0.30000001192092896],"id":3}}},{"distance":28.04,"node":{"identity":7,"label":"Chunk","properties":{"embedding":[0.10000000149011612,0.10000000149011612,0.10000000149011612,0.10000000149011612],"id":1}}},{"distance":34.16,"node":{"identity":8,"label":"Chunk","properties":{"embedding":[-0.20000000298023224,-0.20000000298023224,-0.20000000298023224,-0.20000000298023224],"id":2}}}] +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":0,"elements_num":3,"label_name":"Chunk"}] +CALL db.upsertVertex('Chunk', [{id:1, embedding:[1.1,1.1,1.1,1.1]},{id:2, embedding:[-1.2,-1.2,-1.2,-1.2]}, {id:3, embedding:[1.3,1.3,1.3,1.3]}]); +[{"data_error":0,"index_conflict":0,"insert":0,"total":3,"update":3}] +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":3,"elements_num":6,"label_name":"Chunk"}] +CALL db.vertexVectorKnnSearch('Chunk','embedding',[1,2,3,4], {top_k:10, hnsw_ef_search:10}); +[{"distance":10.76,"node":{"identity":9,"label":"Chunk","properties":{"embedding":[1.2999999523162842,1.2999999523162842,1.2999999523162842,1.2999999523162842],"id":3}}},{"distance":12.84,"node":{"identity":7,"label":"Chunk","properties":{"embedding":[1.100000023841858,1.100000023841858,1.100000023841858,1.100000023841858],"id":1}}},{"distance":59.75999,"node":{"identity":8,"label":"Chunk","properties":{"embedding":[-1.2000000476837158,-1.2000000476837158,-1.2000000476837158,-1.2000000476837158],"id":2}}}] +CALL db.upsertVertex('Chunk', [{id:1, embedding:null},{id:2, embedding:[-1.2,-1.2,-1.2,-1.2]}, {id:3, embedding:null}]); +[{"data_error":0,"index_conflict":0,"insert":0,"total":3,"update":3}] +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":5,"elements_num":6,"label_name":"Chunk"}] +CALL db.vertexVectorKnnSearch('Chunk','embedding',[1,2,3,4], {top_k:10, hnsw_ef_search:10}); +[{"distance":59.75999,"node":{"identity":8,"label":"Chunk","properties":{"embedding":[-1.2000000476837158,-1.2000000476837158,-1.2000000476837158,-1.2000000476837158],"id":2}}}] +CALL db.vertexVectorRangeSearch('Chunk','embedding', [1,2,3,4], {radius:100.0, hnsw_ef_search:10}) yield node,distance return node.id, distance; +[{"distance":59.75999,"node.id":2}] + +CALL db.showVertexVectorIndex() yield label_name, field_name, elements_num, deleted_ids_num return label_name, field_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":5,"elements_num":6,"field_name":"embedding","label_name":"Chunk"},{"deleted_ids_num":1,"elements_num":4,"field_name":"embedding2","label_name":"person"},{"deleted_ids_num":0,"elements_num":3,"field_name":"embedding","label_name":"student"}] +CALL db.dropAllVertex(); +[] +CALL db.showVertexVectorIndex() yield label_name, field_name, elements_num, deleted_ids_num return label_name, field_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":0,"elements_num":0,"field_name":"embedding","label_name":"Chunk"},{"deleted_ids_num":0,"elements_num":0,"field_name":"embedding2","label_name":"person"},{"deleted_ids_num":0,"elements_num":0,"field_name":"embedding","label_name":"student"}] +CALL db.createVertexLabelByJson('{"label":"Chunk","primary":"id","type":"VERTEX","detach_property":true,"properties":[{"name":"id","type":"INT32","optional":false},{"name":"embedding","type":"FLOAT_VECTOR","optional":true}]}'); +[LabelExist] Vertex label [Chunk] already exists. +CALL db.addVertexVectorIndex('Chunk','embedding', {dimension:4}); +[IndexExist] VertexIndex [Chunk:embedding] already exist. +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":0,"elements_num":0,"label_name":"Chunk"}] +CALL db.upsertVertex('Chunk', [{id:1},{id:2}]); +[{"data_error":0,"index_conflict":0,"insert":2,"total":2,"update":0}] +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":0,"elements_num":0,"label_name":"Chunk"}] +CALL db.upsertVertex('Chunk', [{id:1, embedding:[0.1,0.1,0.1,0.1]},{id:2, embedding:[-0.2,-0.2,-0.2,-0.2]}]); +[{"data_error":0,"index_conflict":0,"insert":0,"total":2,"update":2}] +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":0,"elements_num":2,"label_name":"Chunk"}] + +CALL db.dropAllVertex(); +[] +CALL db.showVertexVectorIndex() yield label_name, field_name, elements_num, deleted_ids_num return label_name, field_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":0,"elements_num":0,"field_name":"embedding","label_name":"Chunk"},{"deleted_ids_num":0,"elements_num":0,"field_name":"embedding2","label_name":"person"},{"deleted_ids_num":0,"elements_num":0,"field_name":"embedding","label_name":"student"}] +Create(n1:Chunk {id:1, embedding:[0.1,0.1,0.1,0.1]}) Create(n2:Chunk {id:1, embedding:[0.2,0.2,0.2,0.2]}); +[InputError] Failed to index vertex [13] with field value [id:1]: index value already exists. +CALL db.showVertexVectorIndex() yield label_name, field_name, elements_num, deleted_ids_num return label_name, field_name, elements_num, deleted_ids_num; +[{"deleted_ids_num":0,"elements_num":1,"field_name":"embedding","label_name":"Chunk"},{"deleted_ids_num":0,"elements_num":0,"field_name":"embedding2","label_name":"person"},{"deleted_ids_num":0,"elements_num":0,"field_name":"embedding","label_name":"student"}] diff --git a/test/resource/unit_test/vector_index/cypher/vector_index.test b/test/resource/unit_test/vector_index/cypher/vector_index.test index b360f1ec28..6b89a73632 100644 --- a/test/resource/unit_test/vector_index/cypher/vector_index.test +++ b/test/resource/unit_test/vector_index/cypher/vector_index.test @@ -27,4 +27,39 @@ CREATE (n:student {id:1, name:'name1', embedding: [1.0,1.0,1.0,1.0]}); CREATE (n:student {id:2, name:'name2', embedding: [2.0,2.0,2.0,2.0]}); CREATE (n:student {id:3, name:'name3', embedding: [3.0,3.0,3.0,3.0]}); CALL db.vertexVectorRangeSearch('student','embedding', [1.0,2.0,3.0,4.0], {radius:10.0, hnsw_ef_search:10}) yield node,distance return node.id, distance; -CALL db.vertexVectorRangeSearch('student','embedding', [1.0,2.0,3.0,4.0], {radius:10.0, hnsw_ef_search:10, limit:1}) yield node,distance return node.id, distance; \ No newline at end of file +CALL db.vertexVectorRangeSearch('student','embedding', [1.0,2.0,3.0,4.0], {radius:10.0, hnsw_ef_search:10, limit:1}) yield node,distance return node.id, distance; + +CALL db.createVertexLabelByJson('{"label":"Chunk","primary":"id","type":"VERTEX","detach_property":true,"properties":[{"name":"id","type":"INT32","optional":false},{"name":"embedding","type":"FLOAT_VECTOR","optional":true}]}'); +CALL db.addVertexVectorIndex('Chunk','embedding', {dimension:4}); +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +CALL db.upsertVertex('Chunk', [{id:1},{id:2}]); +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +CALL db.upsertVertex('Chunk', [{id:1, embedding:[0.1,0.1,0.1,0.1]},{id:2, embedding:[-0.2,-0.2,-0.2,-0.2]}]); +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +CALL db.vertexVectorKnnSearch('Chunk','embedding',[1,2,3,4], {top_k:10, hnsw_ef_search:10}); +CALL db.upsertVertex('Chunk', [{id:1, embedding:[0.1,0.1,0.1,0.1]},{id:2, embedding:[-0.2,-0.2,-0.2,-0.2]}, {id:3, embedding:[0.3,0.3,0.3,0.3]}]); +CALL db.vertexVectorKnnSearch('Chunk','embedding',[1,2,3,4], {top_k:10, hnsw_ef_search:10}); +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +CALL db.upsertVertex('Chunk', [{id:1, embedding:[1.1,1.1,1.1,1.1]},{id:2, embedding:[-1.2,-1.2,-1.2,-1.2]}, {id:3, embedding:[1.3,1.3,1.3,1.3]}]); +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +CALL db.vertexVectorKnnSearch('Chunk','embedding',[1,2,3,4], {top_k:10, hnsw_ef_search:10}); +CALL db.upsertVertex('Chunk', [{id:1, embedding:null},{id:2, embedding:[-1.2,-1.2,-1.2,-1.2]}, {id:3, embedding:null}]); +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +CALL db.vertexVectorKnnSearch('Chunk','embedding',[1,2,3,4], {top_k:10, hnsw_ef_search:10}); +CALL db.vertexVectorRangeSearch('Chunk','embedding', [1,2,3,4], {radius:100.0, hnsw_ef_search:10}) yield node,distance return node.id, distance; + +CALL db.showVertexVectorIndex() yield label_name, field_name, elements_num, deleted_ids_num return label_name, field_name, elements_num, deleted_ids_num; +CALL db.dropAllVertex(); +CALL db.showVertexVectorIndex() yield label_name, field_name, elements_num, deleted_ids_num return label_name, field_name, elements_num, deleted_ids_num; +CALL db.createVertexLabelByJson('{"label":"Chunk","primary":"id","type":"VERTEX","detach_property":true,"properties":[{"name":"id","type":"INT32","optional":false},{"name":"embedding","type":"FLOAT_VECTOR","optional":true}]}'); +CALL db.addVertexVectorIndex('Chunk','embedding', {dimension:4}); +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +CALL db.upsertVertex('Chunk', [{id:1},{id:2}]); +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; +CALL db.upsertVertex('Chunk', [{id:1, embedding:[0.1,0.1,0.1,0.1]},{id:2, embedding:[-0.2,-0.2,-0.2,-0.2]}]); +CALL db.showVertexVectorIndex() yield label_name, elements_num, deleted_ids_num where label_name = 'Chunk' return label_name, elements_num, deleted_ids_num; + +CALL db.dropAllVertex(); +CALL db.showVertexVectorIndex() yield label_name, field_name, elements_num, deleted_ids_num return label_name, field_name, elements_num, deleted_ids_num; +Create(n1:Chunk {id:1, embedding:[0.1,0.1,0.1,0.1]}) Create(n2:Chunk {id:1, embedding:[0.2,0.2,0.2,0.2]}); +CALL db.showVertexVectorIndex() yield label_name, field_name, elements_num, deleted_ids_num return label_name, field_name, elements_num, deleted_ids_num; \ No newline at end of file diff --git a/test/test_ha_witness.cpp b/test/test_ha_witness.cpp index 2e9838eb6a..54a523c663 100644 --- a/test/test_ha_witness.cpp +++ b/test/test_ha_witness.cpp @@ -205,6 +205,7 @@ TEST_F(TestHAWitness, HAWitness) { } TEST_F(TestHAWitness, HAWitnessDisableLeader) { + GTEST_SKIP() << "Skipping HAWitnessDisableLeader"; start_server(this->host, true); build_so("./sortstr.so", "../../test/test_procedures/sortstr.cpp"); std::unique_ptr client = std::make_unique( diff --git a/test/test_lgraph_cli.cpp b/test/test_lgraph_cli.cpp index 60ed9a1f6d..e1f9b1c919 100644 --- a/test/test_lgraph_cli.cpp +++ b/test/test_lgraph_cli.cpp @@ -274,7 +274,7 @@ p WriteFile(file, statements); std::string lgraph_cli = "./lgraph_cli --ip 127.0.0.1 --port 7687 --graph default " - "--user admin --password 73@TuGraph"; + "--user admin --password 73@TuGraph --print_time false"; lgraph::SubProcess cli(FMA_FMT("{} < {}", lgraph_cli, file)); cli.Wait(); UT_EXPECT_EQ(cli.Stdout(), expected); diff --git a/test/test_vsag_index.cpp b/test/test_vsag_index.cpp index fadc71b7c1..50906221aa 100644 --- a/test/test_vsag_index.cpp +++ b/test/test_vsag_index.cpp @@ -57,16 +57,17 @@ class TestVsag : public TuGraphTest { void TearDown() override {} }; -TEST_F(TestVsag, BuildIndex) { EXPECT_NO_THROW(vector_index->Build()); } - TEST_F(TestVsag, AddVectors) { - EXPECT_NO_THROW(vector_index->Build()); - EXPECT_NO_THROW(vector_index->Add(vectors, vids, num_vectors)); + EXPECT_NO_THROW(vector_index->Add(vectors, vids)); +} + +TEST_F(TestVsag, AddVectorsException) { + EXPECT_NO_THROW(vector_index->Add(vectors, vids)); + UT_EXPECT_THROW_CODE(vector_index->Add(vectors, vids), VectorIndexException); } TEST_F(TestVsag, SearchIndex) { - EXPECT_NO_THROW(vector_index->Build()); - EXPECT_NO_THROW(vector_index->Add(vectors, vids, num_vectors)); + EXPECT_NO_THROW(vector_index->Add(vectors, vids)); std::vector query(vectors[0].begin(), vectors[0].end()); std::vector> ret; ret = vector_index->KnnSearch(query, 10, 10); @@ -74,25 +75,24 @@ TEST_F(TestVsag, SearchIndex) { ASSERT_EQ(ret[0].first, vids[0]); } +/* TEST_F(TestVsag, SaveAndLoadIndex) { - EXPECT_NO_THROW(vector_index->Build()); - EXPECT_NO_THROW(vector_index->Add(vectors, vids, num_vectors)); + EXPECT_NO_THROW(vector_index->Add(vectors, vids)); std::vector serialized_index = vector_index->Save(); ASSERT_FALSE(serialized_index.empty()); lgraph::HNSW vector_index_loaded("label", "name", "l2", "hnsw", dim, index_spec); - EXPECT_NO_THROW(vector_index_loaded.Build()); vector_index_loaded.Load(serialized_index); std::vector query(vectors[0].begin(), vectors[0].end()); auto ret = vector_index_loaded.KnnSearch(query, 10, 10); ASSERT_TRUE(!ret.empty()); ASSERT_EQ(ret[0].first, vids[0]); } +*/ TEST_F(TestVsag, DeleteVectors) { - EXPECT_NO_THROW(vector_index->Build()); - EXPECT_NO_THROW(vector_index->Add(vectors, vids, num_vectors)); + EXPECT_NO_THROW(vector_index->Add(vectors, vids)); std::vector delete_vids = {vids[0], vids[1]}; - EXPECT_NO_THROW(vector_index->Add({}, delete_vids, 0)); + EXPECT_NO_THROW(vector_index->Remove(delete_vids)); std::vector query(vectors[0].begin(), vectors[0].end()); auto ret = vector_index->KnnSearch(query, 10, 10); for (const auto& pair : ret) { diff --git a/toolkits/lgraph_cli.cpp b/toolkits/lgraph_cli.cpp index e4a80f263c..070dd53d92 100644 --- a/toolkits/lgraph_cli.cpp +++ b/toolkits/lgraph_cli.cpp @@ -60,7 +60,9 @@ std::any ReadMessage(asio::ip::tcp::socket& socket, bolt::Hydrator& hydrator) { return ret.first; } -bool FetchRecords(asio::ip::tcp::socket& socket, bolt::Hydrator& hydrator, OutputFormat of) { +bool FetchRecords(asio::ip::tcp::socket& socket, bolt::Hydrator& hydrator, + OutputFormat of, bool print_time) { + auto start = std::chrono::high_resolution_clock::now(); std::string error; std::optional> header; tabulate::Table table; @@ -122,12 +124,24 @@ bool FetchRecords(asio::ip::tcp::socket& socket, bolt::Hydrator& hydrator, Outpu } if (error.empty()) { + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = end - start; if (of == OutputFormat::TABLE) { if (table.size() > 0) { LOG_INFO() << table << "\n"; - LOG_INFO() << FMA_FMT("{} rows", table.size() - 1) << "\n"; + if (print_time) { + LOG_INFO() << FMA_FMT("{} rows ({} ms)", table.size() - 1, + static_cast(elapsed.count())) << "\n"; + } else { + LOG_INFO() << FMA_FMT("{} rows", table.size() - 1) << "\n"; + } } else { - LOG_INFO() << FMA_FMT("{} rows", table.size()) << "\n"; + if (print_time) { + LOG_INFO() << FMA_FMT("{} rows ({} ms)", table.size(), + static_cast(elapsed.count())) << "\n"; + } else { + LOG_INFO() << FMA_FMT("{} rows", table.size()) << "\n"; + } } } return true; @@ -184,6 +198,7 @@ int main(int argc, char** argv) { std::string graph = "default"; std::string username = "admin"; std::string password = "73@TuGraph"; + bool print_time = true; config.Add(format, "format", true). Comment("output format (table, csv, json)"). SetPossibleValues({"table", "csv", "json"}); @@ -192,6 +207,8 @@ int main(int argc, char** argv) { config.Add(graph, "graph", true).Comment("Graph to use"); config.Add(username, "user", true).Comment("User to login"); config.Add(password, "password", true).Comment("Password to use when connecting to server"); + config.Add(print_time, "print_time", true).Comment( + "Whether to print the execution time of the query"); try { config.ExitAfterHelp(true); config.ParseAndFinalize(argc, argv); @@ -282,7 +299,7 @@ int main(int argc, char** argv) { ps.AppendPullN(-1); asio::write(socket, asio::const_buffer(ps.ConstBuffer().data(), ps.ConstBuffer().size())); - bool ret = FetchRecords(socket, hydrator, of); + bool ret = FetchRecords(socket, hydrator, of, print_time); if (!ret) { // reset connection ps.Reset();