merge with gmaster branch

marian-nmt · Feb 26, 2021 · c2d5fcc · c2d5fcc
2 parents c0a84a4 + 8336bbd
commit c2d5fcc
Show file tree

Hide file tree

Showing 225 changed files with 229,153 additions and 69,881 deletions.
diff --git a/.gitignore b/.gitignore
@@ -30,6 +30,10 @@ CMakeFiles
 CMakeCache.txt
 config.h
 sentencepiece.pc
+CPackConfig.cmake
+CTestTestfile.cmake
+CPackSourceConfig.cmake
+DartConfiguration.tcl
 
 *.o
 *.lo
@@ -53,9 +57,6 @@ spm_train
 spm_normalize
 spm_test
 
-*.pb.cc
-*.pb.h
-
 .DS_Store
 *.egg-info/
 dist/
@@ -65,3 +66,9 @@ dist/
 
 m.model
 m.vocab
+
+cmake_install.cmake
+libsentencepiece.so*
+libsentencepiece_train.so*
+python/bundled
+_sentencepiece.*.so
diff --git a/.travis.yml b/.travis.yml
@@ -5,13 +5,16 @@ matrix:
       env: IMAGE=ubuntu:rolling COMMAND=build_linux_gcc_coverall_ubuntu RELEASE_FILES="$TRAVIS_BUILD_DIR/build/*.xz"
       services: docker
     - os: linux
-      env: IMAGE=i386/ubuntu:rolling COMMAND=build_linux_gcc_ubuntu_i386
+      env: IMAGE=i386/ubuntu:rolling COMMAND=build_linux_gcc_ubuntu
+      services: docker
+    - os: linux
+      env: IMAGE=ubuntu:bionic COMMAND=build_linux_gcc_ubuntu
       services: docker
     - os: linux
       env: IMAGE=ubuntu:xenial COMMAND=build_linux_gcc_ubuntu
       services: docker
     - os: linux
-      env: IMAGE=ubuntu:trusty COMMAND=build_linux_gcc_ubuntu_trusty
+      env: IMAGE=ubuntu:trusty COMMAND=build_linux_gcc_ubuntu
       services: docker
     - os: linux
       env: IMAGE=debian:stable COMMAND=build_linux_gcc_debian
@@ -22,6 +25,18 @@ matrix:
     - os: linux
       env: IMAGE=ubuntu:rolling COMMAND=build_linux_clang_ubuntu
       services: docker
+    - os: linux
+      arch: arm64
+      env: IMAGE=arm64v8/ubuntu:rolling COMMAND=build_linux_gcc_ubuntu
+      services: docker
+    - os: linux
+      arch: ppc64le
+      env: IMAGE=ppc64le/ubuntu:rolling COMMAND=build_linux_gcc_ubuntu
+      services: docker
+    - os: linux
+      arch: s390x
+      env: IMAGE=s390x/ubuntu:rolling COMMAND=build_linux_gcc_ubuntu
+      services: docker
     - os: linux
       env: IMAGE=x86_64 COMMAND=make_py_wheel_py RELEASE_FILES="$TRAVIS_BUILD_DIR/python/dist/*manylinux*.whl"
       script:
@@ -35,32 +50,35 @@ matrix:
         - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES          ; fi
       services: docker
     - os: linux
-      env: IMAGE=x86_64 COMMAND=make_py_wheel_tf RELEASE_FILES="$TRAVIS_BUILD_DIR/tensorflow/dist/*.whl"
+      arch: arm64
+      env: IMAGE=aarch64 COMMAND=make_py_wheel_py RELEASE_FILES="$TRAVIS_BUILD_DIR/python/dist/*manylinux*.whl"
       script:
-        - $TRAVIS_BUILD_DIR/tensorflow/make_py_wheel.sh
+        - $TRAVIS_BUILD_DIR/python/make_py_wheel.sh ${IMAGE}
+        - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES          ; fi
+      services: docker
+    - os: linux
+      arch: ppc64le
+      env: IMAGE=ppc64le COMMAND=make_py_wheel_py RELEASE_FILES="$TRAVIS_BUILD_DIR/python/dist/*manylinux*.whl"
+      script:
+        - $TRAVIS_BUILD_DIR/python/make_py_wheel.sh ${IMAGE}
+        - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES          ; fi
+      services: docker
+    - os: linux
+      arch: s390x
+      env: IMAGE=s390x COMMAND=make_py_wheel_py RELEASE_FILES="$TRAVIS_BUILD_DIR/python/dist/*manylinux*.whl"
+      script:
+        - $TRAVIS_BUILD_DIR/python/make_py_wheel.sh ${IMAGE}
         - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES          ; fi
       services: docker
-    - os: osx
-      osx_image: xcode9.3
-      env: IMAGE=native COMMAND=build_osx
     - os: osx
       osx_image: xcode9.4
       env: IMAGE=native COMMAND=build_osx
-    - os: osx
-      osx_image: xcode9.2
-      env: IMAGE=native COMMAND=build_osx
     - os: osx
       osx_image: xcode9.4
       env: IMAGE=native COMMAND=make_py_wheel_mac_py RELEASE_FILES="$TRAVIS_BUILD_DIR/python/dist/delocated_wheel/*.whl"
       script:
         - $TRAVIS_BUILD_DIR/python/make_py_wheel_mac.sh
         - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES          ; fi
-    - os: osx
-      osx_image: xcode9.4
-      env: IMAGE=native COMMAND=make_py_wheel_mac_tf RELEASE_FILES="$TRAVIS_BUILD_DIR/tensorflow/dist/*.whl"
-      script:
-        - $TRAVIS_BUILD_DIR/tensorflow/make_py_wheel_mac.sh
-        - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES          ; fi
 script:
   - $TRAVIS_BUILD_DIR/test.sh ${IMAGE} ${COMMAND}
   - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES          ; fi

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -15,7 +15,7 @@
 cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
 file(STRINGS "VERSION" SPM_VERSION)
 message(STATUS "VERSION: ${SPM_VERSION}")
-project(sentencepiece VERSION ${SPM_VERSION} LANGUAGES CXX)
+project(sentencepiece VERSION ${SPM_VERSION} LANGUAGES C CXX)
 
 option(SPM_ENABLE_NFKC_COMPILE "Enables NFKC compile" OFF)
 option(SPM_ENABLE_SHARED "Builds shared libaries in addition to static libraries." ON)
@@ -25,24 +25,38 @@ option(SPM_ENABLE_TENSORFLOW_SHARED "Makes a tensorflow compatible shared file."
 option(SPM_ENABLE_TCMALLOC "Enable TCMalloc if available." ON)
 option(SPM_TCMALLOC_STATIC "Link static library of TCMALLOC." OFF)
 option(SPM_NO_THREADLOCAL "Disable thread_local operator" OFF)
+option(SPM_USE_BUILTIN_PROTOBUF "Use built-in protobuf" ON)
 
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-
-set(prefix ${CMAKE_INSTALL_PREFIX})
-set(exec_prefix "\${prefix}")
-set(libdir "\${exec_prefix}/lib")
-set(includedir "\${prefix}/include")
+if (UNIX)
+  include(GNUInstallDirs)
+  set(prefix ${CMAKE_INSTALL_PREFIX})
+  set(exec_prefix "\${prefix}")
+  set(libdir "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}")
+  set(includedir "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}")
+else()
+  set(prefix ${CMAKE_INSTALL_PREFIX})
+  set(exec_prefix "\${prefix}")
+  set(libdir "\${exec_prefix}/lib")
+  set(includedir "\${prefix}/include")
+endif()
 set(GNUCXX_STD_SUPPORT_VERSION "4.3")
 
-if(MSVC)
-    string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_DEBUG          ${CMAKE_CXX_FLAGS_DEBUG})
-    string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_MINSIZEREL     ${CMAKE_CXX_FLAGS_MINSIZEREL})
-    string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE        ${CMAKE_CXX_FLAGS_RELEASE})
-    string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO})
-    add_definitions("/wd4267 /wd4244 /wd4305 /Zc:strictStrings /utf-8")
-endif(MSVC)
+if (SPM_USE_BUILTIN_PROTOBUF)
+  set(libprotobuf_lite "")
+else()
+  set(libprotobuf_lite "-lprotobuf-lite")
+endif()
+
+if (MSVC)
+  string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_DEBUG          ${CMAKE_CXX_FLAGS_DEBUG})
+  string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_MINSIZEREL     ${CMAKE_CXX_FLAGS_MINSIZEREL})
+  string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE        ${CMAKE_CXX_FLAGS_RELEASE})
+  string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO})
+  add_definitions("/wd4267 /wd4244 /wd4305 /Zc:strictStrings /utf-8")
+endif()
 
 if (APPLE)
   set(CMAKE_MACOSX_RPATH ON)
@@ -64,10 +78,6 @@ if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
   set(CMAKE_INSTALL_LIBDIR lib)
 endif()
 
-if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
-  set(CMAKE_INSTALL_LIBDIR lib)
-endif()
-
 if (NOT DEFINED CMAKE_INSTALL_INCDIR)
   set(CMAKE_INSTALL_INCDIR include)
 endif()
@@ -79,13 +89,14 @@ if (NOT MSVC)
   install(FILES "${CMAKE_CURRENT_BINARY_DIR}/sentencepiece.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
 endif()
 
-include_directories("." ${CMAKE_SOURCE_DIR} ${PROJECT_BINARY_DIR})
+include_directories("." ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_BINARY_DIR})
 
 if (SPM_BUILD_TEST)
   enable_testing()
 endif()
 
 add_subdirectory(src)
+add_subdirectory(third_party)
 
 set(CPACK_SOURCE_GENERATOR "TXZ")
 set(CPACK_GENERATOR "7Z")