diff --git a/examples/models/llama/runner/CMakeLists.txt b/examples/models/llama/runner/CMakeLists.txt index 04fe23e4d82..0807e6fa422 100644 --- a/examples/models/llama/runner/CMakeLists.txt +++ b/examples/models/llama/runner/CMakeLists.txt @@ -41,14 +41,6 @@ target_include_directories( extension_module INTERFACE ${_common_include_directories} ) -list( - APPEND _llama_runner__srcs - ${EXECUTORCH_ROOT}/extension/llm/tokenizers/src/tiktoken.cpp -) -list(APPEND _llama_runner__srcs - ${CMAKE_CURRENT_SOURCE_DIR}/../tokenizer/llama_tiktoken.cpp -) - if(CMAKE_TOOLCHAIN_IOS OR ANDROID OR APPLE @@ -60,23 +52,8 @@ else() add_library(llama_runner SHARED ${_llama_runner__srcs}) endif() -# find RE2 for tokenizer, build tiktoken -set(ABSL_ENABLE_INSTALL ON) -set(ABSL_PROPAGATE_CXX_STD ON) -set(_pic_flag ${CMAKE_POSITION_INDEPENDENT_CODE}) -set(CMAKE_POSITION_INDEPENDENT_CODE ON) -add_subdirectory( - ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/abseil-cpp - ${CMAKE_CURRENT_BINARY_DIR}/abseil-cpp -) -add_subdirectory( - ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/re2 - ${CMAKE_CURRENT_BINARY_DIR}/re2 -) -set(CMAKE_POSITION_INDEPENDENT_CODE ${_pic_flag}) - set(llama_runner_deps executorch extension_data_loader extension_module - extension_tensor re2::re2 + extension_tensor ) target_link_libraries(llama_runner PUBLIC ${llama_runner_deps}) @@ -85,6 +62,17 @@ target_include_directories( llama_runner INTERFACE ${_common_include_directories} ) + +# Include tokenizers dependency +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +add_subdirectory( + ${EXECUTORCH_ROOT}/extension/llm/tokenizers + ${CMAKE_CURRENT_BINARY_DIR}/tokenizers +) +target_link_libraries( + llama_runner PUBLIC tokenizers +) + target_include_directories( llama_runner PUBLIC ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include diff --git a/examples/models/llama/runner/runner.cpp b/examples/models/llama/runner/runner.cpp index 186c2013616..534738521a5 100644 --- a/examples/models/llama/runner/runner.cpp +++ b/examples/models/llama/runner/runner.cpp @@ -17,6 +17,7 @@ #include #include +#include #include namespace example { @@ -36,6 +37,30 @@ static constexpr auto kMaxContextLen = "get_max_context_len"; static constexpr auto kVocabSize = "get_vocab_size"; static constexpr auto kUseKVCache = "use_kv_cache"; static constexpr auto kUseSDPAWithKVCache = "use_sdpa_with_kv_cache"; + +std::unique_ptr<::tokenizers::Tokenizer> load_tokenizer( + const std::string& tokenizer_path) { + + auto json_tokenizer = std::make_unique(); + if (json_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) { + ET_LOG(Info, "Loaded json tokenizer"); + return json_tokenizer; + } + + auto tiktoken_tokenizer = get_tiktoken_for_llama(); + if (tiktoken_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) { + ET_LOG(Info, "Loaded TikToken tokenizer"); + return tiktoken_tokenizer; + } + + auto bpe_tokenizer = std::make_unique<::tokenizers::Llama2cTokenizer>(); + if (bpe_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) { + ET_LOG(Info, "Loaded BPE tokenizer"); + return bpe_tokenizer; + } + + return nullptr; +} } // namespace Runner::Runner( @@ -87,13 +112,10 @@ Error Runner::load() { return Error::Ok; } ET_CHECK_OK_OR_RETURN_ERROR(module_->load_method("forward")); - // load tokenizer. Assuming tiktoken is the default tokenizer - tokenizer_ = nullptr; - tokenizer_ = get_tiktoken_for_llama(); - ::tokenizers::Error err = tokenizer_->load(tokenizer_path_); - // Rely on tiktoken to throw error if the artifact is incompatible. Then we - // fallback to BPE tokenizer. - if (err != ::tokenizers::Error::Ok) { + + // Load tokenizer. + tokenizer_ = load_tokenizer(tokenizer_path_); + if (tokenizer_ == nullptr) { ET_LOG( Info, "Failed to load %s as a Tiktoken artifact, trying BPE tokenizer", @@ -101,11 +123,12 @@ Error Runner::load() { tokenizer_.reset(); // @lint-ignore CLANGTIDY facebook-hte-Deprecated tokenizer_ = std::make_unique<::tokenizers::Llama2cTokenizer>(); - err = tokenizer_->load(tokenizer_path_); + auto err = tokenizer_->load(tokenizer_path_); ET_CHECK_TK_OK_OR_RETURN_ERROR( err, "Failed to load %s as a llama2.c tokenizer artifact", tokenizer_path_.c_str()); + return ::executorch::runtime::Error::InvalidArgument; } ET_LOG(Info, "Reading metadata from model"); diff --git a/examples/models/llama/runner/targets.bzl b/examples/models/llama/runner/targets.bzl index 60fc1f2b74d..158202cf55a 100644 --- a/examples/models/llama/runner/targets.bzl +++ b/examples/models/llama/runner/targets.bzl @@ -49,6 +49,7 @@ def define_common_targets(): "//executorch/runtime/core/exec_aten/util:tensor_util" + aten_suffix, "//executorch/examples/models/llama/tokenizer:tiktoken", "//pytorch/tokenizers:llama2c_tokenizer", + "//pytorch/tokenizers:hf_tokenizer", ] + (_get_operator_lib(aten)) + ([ # Vulkan API currently cannot build on some platforms (e.g. Apple, FBCODE) # Therefore enable it explicitly for now to avoid failing tests diff --git a/examples/qualcomm/CMakeLists.txt b/examples/qualcomm/CMakeLists.txt index 7b2c43b3f46..4f338a23044 100644 --- a/examples/qualcomm/CMakeLists.txt +++ b/examples/qualcomm/CMakeLists.txt @@ -35,7 +35,7 @@ find_package(gflags REQUIRED) set(_common_compile_options -Wno-deprecated-declarations -fPIC) # Let files say "include ". -set(_common_include_directories ${EXECUTORCH_ROOT}/..) +set(_common_include_directories ${EXECUTORCH_ROOT}/.. ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/json/single_include) # # The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. @@ -67,6 +67,9 @@ target_include_directories( PUBLIC ${_common_include_directories} ${CMAKE_CURRENT_SOURCE_DIR}/../../extension/llm/tokenizers/include + ${CMAKE_CURRENT_SOURCE_DIR}/../../extension/llm/tokenizers/third-party/json/single_include + ${CMAKE_CURRENT_SOURCE_DIR}/../../extension/llm/tokenizers/third-party/llama.cpp-unicode/include + ${CMAKE_CURRENT_SOURCE_DIR}/../../extension/llm/tokenizers/third-party/llama.cpp-unicode/src ) # find RE2 for tokenizer diff --git a/examples/qualcomm/qaihub_scripts/llama/CMakeLists.txt b/examples/qualcomm/qaihub_scripts/llama/CMakeLists.txt index f96d0169809..16d91013349 100644 --- a/examples/qualcomm/qaihub_scripts/llama/CMakeLists.txt +++ b/examples/qualcomm/qaihub_scripts/llama/CMakeLists.txt @@ -27,7 +27,12 @@ list(PREPEND _qaihub_llama2_7b_runner__srcs # build qaihub llama2 7b runner add_executable(qaihub_llama2_7b_runner ${_qaihub_llama2_7b_runner__srcs}) target_include_directories( - qaihub_llama2_7b_runner PUBLIC ${_common_include_directories} ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizers/include + qaihub_llama2_7b_runner PUBLIC + ${_common_include_directories} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizers/include + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizers/third-party/json/single_include + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizers/third-party/llama.cpp-unicode/include + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizers/third-party/llama.cpp-unicode/src ) target_link_libraries( qaihub_llama2_7b_runner @@ -69,7 +74,12 @@ list( # build qaihub llama3 8b runner add_executable(qaihub_llama3_8b_runner ${_qaihub_llama3_8b_runner__srcs}) target_include_directories( - qaihub_llama3_8b_runner PUBLIC ${_common_include_directories} ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizers/include + qaihub_llama3_8b_runner PUBLIC + ${_common_include_directories} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizers/include + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizers/third-party/json/single_include + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizers/third-party/llama.cpp-unicode/include + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizers/third-party/llama.cpp-unicode/src ) target_link_libraries( diff --git a/extension/benchmark/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj b/extension/benchmark/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj index 9fc1d47cb22..dda5f2bc6fa 100644 --- a/extension/benchmark/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj +++ b/extension/benchmark/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj @@ -27,6 +27,13 @@ 03F181582D7262FC0058BDF9 /* kernels_optimized in Frameworks */ = {isa = PBXBuildFile; productRef = 03F181572D7262FC0058BDF9 /* kernels_optimized */; }; 03F1815A2D7262FC0058BDF9 /* kernels_portable in Frameworks */ = {isa = PBXBuildFile; productRef = 03F181592D7262FC0058BDF9 /* kernels_portable */; }; 03F1815C2D7262FC0058BDF9 /* kernels_quantized in Frameworks */ = {isa = PBXBuildFile; productRef = 03F1815B2D7262FC0058BDF9 /* kernels_quantized */; }; + 30AA4B602DC0766800B1BE50 /* pcre2_regex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 30AA4B5A2DC0766800B1BE50 /* pcre2_regex.cpp */; }; + 30AA4B612DC0766800B1BE50 /* regex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 30AA4B5D2DC0766800B1BE50 /* regex.cpp */; }; + 30AA4B622DC0766800B1BE50 /* hf_tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 30AA4B592DC0766800B1BE50 /* hf_tokenizer.cpp */; }; + 30AA4B632DC0766800B1BE50 /* token_decoder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 30AA4B5F2DC0766800B1BE50 /* token_decoder.cpp */; }; + 30AA4B642DC0766800B1BE50 /* std_regex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 30AA4B5E2DC0766800B1BE50 /* std_regex.cpp */; }; + 30AA4B652DC0766800B1BE50 /* pre_tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 30AA4B5B2DC0766800B1BE50 /* pre_tokenizer.cpp */; }; + 30AA4B662DC0766800B1BE50 /* re2_regex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 30AA4B5C2DC0766800B1BE50 /* re2_regex.cpp */; }; F292B01D2D88AF3500BE6839 /* bpe_tokenizer_base.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B0162D88AF3500BE6839 /* bpe_tokenizer_base.cpp */; }; F292B0202D88AF3500BE6839 /* llama2c_tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B0172D88AF3500BE6839 /* llama2c_tokenizer.cpp */; }; F292B0212D88AF3500BE6839 /* tiktoken.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B01A2D88AF3500BE6839 /* tiktoken.cpp */; }; @@ -73,6 +80,21 @@ 03B2D3792C8A515C0046936E /* GenericTests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = GenericTests.mm; sourceTree = ""; }; 03C7FA322C8AA24200E6E9AE /* Resources */ = {isa = PBXFileReference; lastKnownFileType = folder; path = Resources; sourceTree = SOURCE_ROOT; }; 03E7E6782CBDC1C900205E71 /* CoreMLTests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = CoreMLTests.mm; sourceTree = ""; }; + 30593C332DC02ED100AB308C /* regex.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = regex.h; sourceTree = ""; }; + 30593C342DC02EDD00AB308C /* re2_regex.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = re2_regex.h; sourceTree = ""; }; + 30593C3D2DC02FD400AB308C /* pcre2_regex.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pcre2_regex.h; sourceTree = ""; }; + 30593C3E2DC02FD400AB308C /* std_regex.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = std_regex.h; sourceTree = ""; }; + 30AA4B552DC0756E00B1BE50 /* hf_tokenizer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = hf_tokenizer.h; sourceTree = ""; }; + 30AA4B562DC075CE00B1BE50 /* pre_tokenizer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pre_tokenizer.h; sourceTree = ""; }; + 30AA4B572DC0760200B1BE50 /* token_decoder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = token_decoder.h; sourceTree = ""; }; + 30AA4B582DC0760C00B1BE50 /* string_integer_map.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = string_integer_map.h; sourceTree = ""; }; + 30AA4B592DC0766800B1BE50 /* hf_tokenizer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = hf_tokenizer.cpp; path = src/hf_tokenizer.cpp; sourceTree = ""; }; + 30AA4B5A2DC0766800B1BE50 /* pcre2_regex.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = pcre2_regex.cpp; path = src/pcre2_regex.cpp; sourceTree = ""; }; + 30AA4B5B2DC0766800B1BE50 /* pre_tokenizer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = pre_tokenizer.cpp; path = src/pre_tokenizer.cpp; sourceTree = ""; }; + 30AA4B5C2DC0766800B1BE50 /* re2_regex.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = re2_regex.cpp; path = src/re2_regex.cpp; sourceTree = ""; }; + 30AA4B5D2DC0766800B1BE50 /* regex.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = regex.cpp; path = src/regex.cpp; sourceTree = ""; }; + 30AA4B5E2DC0766800B1BE50 /* std_regex.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = std_regex.cpp; path = src/std_regex.cpp; sourceTree = ""; }; + 30AA4B5F2DC0766800B1BE50 /* token_decoder.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = token_decoder.cpp; path = src/token_decoder.cpp; sourceTree = ""; }; F292B0162D88AF3500BE6839 /* bpe_tokenizer_base.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = bpe_tokenizer_base.cpp; path = src/bpe_tokenizer_base.cpp; sourceTree = ""; }; F292B0172D88AF3500BE6839 /* llama2c_tokenizer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = llama2c_tokenizer.cpp; path = src/llama2c_tokenizer.cpp; sourceTree = ""; }; F292B01A2D88AF3500BE6839 /* tiktoken.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = tiktoken.cpp; path = src/tiktoken.cpp; sourceTree = ""; }; @@ -156,12 +178,19 @@ 032A74022CAFBB7800932D36 /* tokenizers */ = { isa = PBXGroup; children = ( + 30AA4B592DC0766800B1BE50 /* hf_tokenizer.cpp */, + 30AA4B5A2DC0766800B1BE50 /* pcre2_regex.cpp */, + 30AA4B5B2DC0766800B1BE50 /* pre_tokenizer.cpp */, + 30AA4B5C2DC0766800B1BE50 /* re2_regex.cpp */, + 30AA4B5D2DC0766800B1BE50 /* regex.cpp */, + 30AA4B5E2DC0766800B1BE50 /* std_regex.cpp */, + 30AA4B5F2DC0766800B1BE50 /* token_decoder.cpp */, F292B0302D88AF4800BE6839 /* include */, F292B0162D88AF3500BE6839 /* bpe_tokenizer_base.cpp */, + 032A74252CAFC34800932D36 /* llama_tiktoken.cpp */, F292B0172D88AF3500BE6839 /* llama2c_tokenizer.cpp */, F292B01A2D88AF3500BE6839 /* tiktoken.cpp */, 032A74242CAFC34800932D36 /* llama_tiktoken.h */, - 032A74252CAFC34800932D36 /* llama_tiktoken.cpp */, ); name = tokenizers; path = ../../../llm/tokenizers; @@ -222,12 +251,20 @@ F292B02E2D88AF4800BE6839 /* tokenizers */ = { isa = PBXGroup; children = ( + 30AA4B582DC0760C00B1BE50 /* string_integer_map.h */, + 30AA4B572DC0760200B1BE50 /* token_decoder.h */, + 30AA4B562DC075CE00B1BE50 /* pre_tokenizer.h */, + 30AA4B552DC0756E00B1BE50 /* hf_tokenizer.h */, F292B0222D88AF4800BE6839 /* base64.h */, F292B0232D88AF4800BE6839 /* bpe_tokenizer_base.h */, F292B0242D88AF4800BE6839 /* error.h */, F292B0262D88AF4800BE6839 /* llama2c_tokenizer.h */, F292B0272D88AF4800BE6839 /* log.h */, + 30593C3D2DC02FD400AB308C /* pcre2_regex.h */, + 30593C342DC02EDD00AB308C /* re2_regex.h */, + 30593C332DC02ED100AB308C /* regex.h */, F292B0292D88AF4800BE6839 /* result.h */, + 30593C3E2DC02FD400AB308C /* std_regex.h */, F292B02B2D88AF4800BE6839 /* tiktoken.h */, F292B02D2D88AF4800BE6839 /* tokenizer.h */, ); @@ -357,7 +394,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\nif ! command -v cmake &> /dev/null\nthen\n echo \"Cmake not found, please install Cmake. \\n1. Download Cmake.app from https://cmake.org/download with version > 3.19. \\n2. Install it to Applications/ folder and run sudo /Applications/CMake.app/Contents/bin/cmake-gui --install to install CMake commandline tools.\"\n exit 1\nfi\n\nCMAKE_DIR=\"$TEMP_DIR/cmake\"\nrm -rf \"$CMAKE_DIR\"\n\nPLATFORM=\"SIMULATORARM64\"\nDEPLOYMENT_TARGET=\"17.0\"\n\nif [[ \"$PLATFORM_NAME\" == *\"iphoneos\"* ]]; then\n PLATFORM=\"OS64\"\nelif [[ \"$PLATFORM_NAME\" == *\"macos\"* ]]; then\n PLATFORM=\"MAC_ARM64\"\n DEPLOYMENT_TARGET=\"10.15\"\nfi\n\ncmake_build() {\n local src_dir=$1\n local target=$2\n shift 2\n local extra_args=(\"$@\")\n local build_dir=\"$CMAKE_DIR/build/$(basename \"$src_dir\")\"\n\n mkdir -p \"$build_dir\" && cd \"$build_dir\"\n\n if [[ \"$PLATFORM\" == \"MAC_ARM64\" ]]; then\n extra_args+=(-DCMAKE_INSTALL_BUNDLEDIR=\"${CMAKE_DIR}/bin\")\n extra_args+=(-DCMAKE_MACOSX_BUNDLE=OFF)\n fi\n cmake -G Xcode \\\n -DCMAKE_BUILD_TYPE=\"Release\" \\\n -DCMAKE_CXX_STANDARD=17 \\\n -DCMAKE_TOOLCHAIN_FILE=\"$SRCROOT/../../../../third-party/ios-cmake/ios.toolchain.cmake\" \\\n -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD=\"c++17\" \\\n -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY=\"libc++\" \\\n -DPLATFORM=\"$PLATFORM\" \\\n -DDEPLOYMENT_TARGET=\"$DEPLOYMENT_TARGET\" \\\n -DCMAKE_INSTALL_PREFIX=\"$CMAKE_DIR\" \\\n \"${extra_args[@]}\" \\\n \"$src_dir\"\n cmake --build . --config \"Release\" --target \"$target\"\n if [[ \"$target\" == \"install\" ]]; then\n cmake --install . --prefix \"$CMAKE_DIR\"\n fi\n}\n\ncmake_build \"$SRCROOT/../../../llm/tokenizers/third-party/abseil-cpp\" \"install\" \\\n -DABSL_PROPAGATE_CXX_STD=ON\n\ncmake_build \"$SRCROOT/../../../llm/tokenizers/third-party/re2\" \"install\"\n\ncmake_build \"$SRCROOT/../../../llm/tokenizers/third-party/sentencepiece\" \"sentencepiece-static\" \\\n -DSPM_ENABLE_SHARED=OFF\n\necho \"$(find $CMAKE_DIR/lib -name \"*.a\" | sed -E 's|^.*/lib([^/]+)\\.a|-l\\1|g' | tr '\\n' ' ')\" > \"$CMAKE_DIR/linker_flags\"\n"; + shellScript = "set -e\n\nif ! command -v cmake &> /dev/null\nthen\n echo \"Cmake not found, please install Cmake. \\n1. Download Cmake.app from https://cmake.org/download with version > 3.19. \\n2. Install it to Applications/ folder and run sudo /Applications/CMake.app/Contents/bin/cmake-gui --install to install CMake commandline tools.\"\n exit 1\nfi\n\nCMAKE_DIR=\"$TEMP_DIR/cmake\"\nrm -rf \"$CMAKE_DIR\"\n\nPLATFORM=\"SIMULATORARM64\"\nDEPLOYMENT_TARGET=\"17.0\"\n\nif [[ \"$PLATFORM_NAME\" == *\"iphoneos\"* ]]; then\n PLATFORM=\"OS64\"\nelif [[ \"$PLATFORM_NAME\" == *\"macos\"* ]]; then\n PLATFORM=\"MAC_ARM64\"\n DEPLOYMENT_TARGET=\"10.15\"\nfi\n\ncmake_build() {\n local src_dir=$1\n local target=$2\n shift 2\n local extra_args=(\"$@\")\n local build_dir=\"$CMAKE_DIR/build/$(basename \"$src_dir\")\"\n\n mkdir -p \"$build_dir\" && cd \"$build_dir\"\n\n if [[ \"$PLATFORM\" == \"MAC_ARM64\" ]]; then\n extra_args+=(-DCMAKE_INSTALL_BUNDLEDIR=\"${CMAKE_DIR}/bin\")\n extra_args+=(-DCMAKE_MACOSX_BUNDLE=OFF)\n fi\n cmake -G Xcode \\\n -DCMAKE_BUILD_TYPE=\"Release\" \\\n -DCMAKE_CXX_STANDARD=17 \\\n -DCMAKE_TOOLCHAIN_FILE=\"$SRCROOT/../../../../third-party/ios-cmake/ios.toolchain.cmake\" \\\n -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD=\"c++17\" \\\n -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY=\"libc++\" \\\n -DPLATFORM=\"$PLATFORM\" \\\n -DDEPLOYMENT_TARGET=\"$DEPLOYMENT_TARGET\" \\\n -DCMAKE_INSTALL_PREFIX=\"$CMAKE_DIR\" \\\n \"${extra_args[@]}\" \\\n \"$src_dir\"\n cmake --build . --config \"Release\" --target \"$target\"\n if [[ \"$target\" == \"install\" ]]; then\n cmake --install . --prefix \"$CMAKE_DIR\"\n fi\n}\n\ncmake_build \"$SRCROOT/../../../llm/tokenizers/third-party/abseil-cpp\" \"install\" \\\n -DABSL_PROPAGATE_CXX_STD=ON\n\ncmake_build \"$SRCROOT/../../../llm/tokenizers/third-party/re2\" \"install\"\n\ncmake_build \"$SRCROOT/../../../llm/tokenizers/third-party/pcre2\" \"install\" \\\n -DPCRE2_BUILD_PCRE2_8=ON \\\n -DPCRE2_BUILD_PCRE2_16=OFF \\\n -DPCRE2_BUILD_PCRE2_32=OFF \\\n -DPCRE2_BUILD_TESTS=OFF \\\n -DPCRE2_BUILD_PCRE2GREP=OFF \\\n -DPCRE2_BUILD_PCRE2TEST=OFF \\\n -DPCRE2_BUILD_PCRE2GPERF=OFF \\\n -DPCRE2_BUILD_DOCS=OFF \\\n -DPCRE2_BUILD_LIBPCRE2_PDB=OFF\n \ncmake_build \"$SRCROOT/../../../llm/tokenizers/third-party/sentencepiece\" \"sentencepiece-static\" \\\n -DSPM_ENABLE_SHARED=OFF\n \ncmake_build \"$SRCROOT/../../../llm/tokenizers/third-party/llama.cpp-unicode\" \"install\"\n \n# Include the single header for json.\nmkdir -p \"$CMAKE_DIR/include/nlohmann\"\ncp \"$SRCROOT/../../../llm/tokenizers/third-party/json/single_include/nlohmann/json.hpp\" \"$CMAKE_DIR/include/nlohmann/json.hpp\"\n\necho \"$(find $CMAKE_DIR/lib -name \"*.a\" | sed -E 's|^.*/lib([^/]+)\\.a|-l\\1|g' | tr '\\n' ' ')\" > \"$CMAKE_DIR/linker_flags\"\n"; }; /* End PBXShellScriptBuildPhase section */ @@ -385,6 +422,13 @@ 03E7E6792CBDCAE900205E71 /* CoreMLTests.mm in Sources */, 032A74232CAFC1B300932D36 /* runner.cpp in Sources */, 03B2D37A2C8A515C0046936E /* GenericTests.mm in Sources */, + 30AA4B602DC0766800B1BE50 /* pcre2_regex.cpp in Sources */, + 30AA4B612DC0766800B1BE50 /* regex.cpp in Sources */, + 30AA4B622DC0766800B1BE50 /* hf_tokenizer.cpp in Sources */, + 30AA4B632DC0766800B1BE50 /* token_decoder.cpp in Sources */, + 30AA4B642DC0766800B1BE50 /* std_regex.cpp in Sources */, + 30AA4B652DC0766800B1BE50 /* pre_tokenizer.cpp in Sources */, + 30AA4B662DC0766800B1BE50 /* re2_regex.cpp in Sources */, 032A73CA2CAFBA8600932D36 /* LLaMATests.mm in Sources */, 032A74262CAFC34800932D36 /* llama_tiktoken.cpp in Sources */, ); diff --git a/extension/llm/runner/CMakeLists.txt b/extension/llm/runner/CMakeLists.txt index 75c30cff71b..e23f5c2c58d 100644 --- a/extension/llm/runner/CMakeLists.txt +++ b/extension/llm/runner/CMakeLists.txt @@ -49,6 +49,13 @@ set(runner_deps executorch extension_data_loader extension_module target_link_libraries(extension_llm_runner PUBLIC ${runner_deps}) +target_include_directories( + extension_llm_runner + PUBLIC + ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/llama.cpp-unicode/include + ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/llama.cpp-unicode/src +) + target_include_directories( extension_llm_runner INTERFACE ${_common_include_directories} ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index 3f9c458586e..0ed8e2e3448 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit 3f9c458586ee576a7ddafb48eb491f117187e178 +Subproject commit 0ed8e2e34486f119a87c15d000080c5e3eea7aea