From 86a5d96fc6c42932a2c978b66a0f60e4c93cba53 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Thu, 11 Apr 2024 14:27:15 +0200 Subject: [PATCH 01/36] feat: first things to do --- CMakeLists.txt | 8 +- Makefile | 2731 ++++++++++++++++++++------------ convert-hf-to-gguf.py | 88 +- gguf-py/gguf/constants.py | 16 + gguf-py/gguf/tensor_mapping.py | 3 + llama.cpp | 37 +- 6 files changed, 1875 insertions(+), 1008 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 19fdfa46ca4f1..a144e2cf323a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,9 +59,9 @@ option(LLAMA_GPROF "llama: enable gprof" option(LLAMA_FATAL_WARNINGS "llama: enable -Werror flag" OFF) # sanitizers -option(LLAMA_SANITIZE_THREAD "llama: enable thread sanitizer" OFF) -option(LLAMA_SANITIZE_ADDRESS "llama: enable address sanitizer" OFF) -option(LLAMA_SANITIZE_UNDEFINED "llama: enable undefined sanitizer" OFF) +option(LLAMA_SANITIZE_THREAD "llama: enable thread sanitizer" ON) +option(LLAMA_SANITIZE_ADDRESS "llama: enable address sanitizer" ON) +option(LLAMA_SANITIZE_UNDEFINED "llama: enable undefined sanitizer" ON) # instruction set specific if (LLAMA_NATIVE) @@ -126,7 +126,7 @@ option(LLAMA_CPU_HBM "llama: use memkind for CPU HBM" set(LLAMA_SCHED_MAX_COPIES "4" CACHE STRING "llama: max input copies for pipeline parallelism") option(LLAMA_BUILD_TESTS "llama: build tests" ${LLAMA_STANDALONE}) -option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE}) +option(LLAMA_BUILD_EXAMPLES "llama: build examples" ON) option(LLAMA_BUILD_SERVER "llama: build server example" ON) # add perf arguments diff --git a/Makefile b/Makefile index 11b31c5c84182..ec7edd425de0b 100644 --- a/Makefile +++ b/Makefile @@ -1,990 +1,1761 @@ -# Define the default target now so that it is always the first target -BUILD_TARGETS = \ - main quantize quantize-stats perplexity imatrix embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \ - simple batched batched-bench save-load-state server gguf gguf-split llama-bench libllava.a llava-cli baby-llama beam-search \ - retrieval speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead lookup passkey gritlm tests/test-c.o - -# Binaries only useful for tests -TEST_TARGETS = \ - tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt \ - tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama \ - tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe tests/test-rope \ - tests/test-backend-ops tests/test-model-load-cancel tests/test-autorelease \ - tests/test-json-schema-to-grammar tests/test-grammar-integration - -# Code coverage output files -COV_TARGETS = *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report - -ifndef UNAME_S -UNAME_S := $(shell uname -s) -endif - -ifndef UNAME_P -UNAME_P := $(shell uname -p) -endif - -ifndef UNAME_M -UNAME_M := $(shell uname -m) -endif - -# Mac OS + Arm can report x86_64 -# ref: https://github.com/ggerganov/whisper.cpp/issues/66#issuecomment-1282546789 -ifeq ($(UNAME_S),Darwin) - ifndef LLAMA_NO_METAL - LLAMA_METAL := 1 - endif - - ifneq ($(UNAME_P),arm) - SYSCTL_M := $(shell sysctl -n hw.optional.arm64 2>/dev/null) - ifeq ($(SYSCTL_M),1) - # UNAME_P := arm - # UNAME_M := arm64 - warn := $(warning Your arch is announced as x86_64, but it seems to actually be ARM64. Not fixing that can lead to bad performance. For more info see: https://github.com/ggerganov/whisper.cpp/issues/66\#issuecomment-1282546789) - endif - endif -endif - -default: $(BUILD_TARGETS) - -test: $(TEST_TARGETS) - @failures=0; \ - for test_target in $(TEST_TARGETS); do \ - if [ "$$test_target" = "tests/test-tokenizer-0-llama" ]; then \ - ./$$test_target $(CURDIR)/models/ggml-vocab-llama.gguf; \ - elif [ "$$test_target" = "tests/test-tokenizer-0-falcon" ]; then \ - ./$$test_target $(CURDIR)/models/ggml-vocab-falcon.gguf; \ - elif [ "$$test_target" = "tests/test-tokenizer-1-llama" ]; then \ - continue; \ - elif [ "$$test_target" = "tests/test-tokenizer-1-bpe" ]; then \ - continue; \ - else \ - echo "Running test $$test_target..."; \ - ./$$test_target; \ - fi; \ - if [ $$? -ne 0 ]; then \ - printf 'Test %s FAILED!\n\n' $$test_target; \ - failures=$$(( failures + 1 )); \ - else \ - printf 'Test %s passed.\n\n' $$test_target; \ - fi; \ - done; \ - if [ $$failures -gt 0 ]; then \ - printf '\n%s tests failed.\n' $$failures; \ - exit 1; \ - fi - @echo 'All tests passed.' - -all: $(BUILD_TARGETS) $(TEST_TARGETS) - -coverage: ## Run code coverage - gcov -pb tests/*.cpp - -lcov-report: coverage ## Generate lcov report - mkdir -p lcov-report - lcov --capture --directory . --output-file lcov-report/coverage.info - genhtml lcov-report/coverage.info --output-directory lcov-report - -gcovr-report: coverage ## Generate gcovr report - mkdir -p gcovr-report - gcovr --root . --html --html-details --output gcovr-report/coverage.html - -ifdef RISCV_CROSS_COMPILE -CC := riscv64-unknown-linux-gnu-gcc -CXX := riscv64-unknown-linux-gnu-g++ -endif - -# -# Compile flags -# - -# keep standard at C11 and C++11 -MK_CPPFLAGS = -I. -Icommon -MK_CFLAGS = -std=c11 -fPIC -MK_CXXFLAGS = -std=c++11 -fPIC -MK_NVCCFLAGS = -std=c++11 - -# -Ofast tends to produce faster code, but may not be available for some compilers. -ifdef LLAMA_FAST -MK_CFLAGS += -Ofast -HOST_CXXFLAGS += -Ofast -MK_NVCCFLAGS += -O3 -else -MK_CFLAGS += -O3 -MK_CXXFLAGS += -O3 -MK_NVCCFLAGS += -O3 -endif - -ifndef LLAMA_NO_CCACHE -CCACHE := $(shell which ccache) -ifdef CCACHE -export CCACHE_SLOPPINESS = time_macros -$(info I ccache found, compilation results will be cached. Disable with LLAMA_NO_CCACHE.) -CC := $(CCACHE) $(CC) -CXX := $(CCACHE) $(CXX) -else -$(info I ccache not found. Consider installing it for faster compilation.) -endif # CCACHE -endif # LLAMA_NO_CCACHE - -# clock_gettime came in POSIX.1b (1993) -# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional -# posix_memalign came in POSIX.1-2001 / SUSv3 -# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985) -MK_CPPFLAGS += -D_XOPEN_SOURCE=600 - -# Somehow in OpenBSD whenever POSIX conformance is specified -# some string functions rely on locale_t availability, -# which was introduced in POSIX.1-2008, forcing us to go higher -ifeq ($(UNAME_S),OpenBSD) - MK_CPPFLAGS += -U_XOPEN_SOURCE -D_XOPEN_SOURCE=700 -endif - -# Data types, macros and functions related to controlling CPU affinity and -# some memory allocation are available on Linux through GNU extensions in libc -ifeq ($(UNAME_S),Linux) - MK_CPPFLAGS += -D_GNU_SOURCE -endif - -# RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1, -# and on macOS its availability depends on enabling Darwin extensions -# similarly on DragonFly, enabling BSD extensions is necessary -ifeq ($(UNAME_S),Darwin) - MK_CPPFLAGS += -D_DARWIN_C_SOURCE -endif -ifeq ($(UNAME_S),DragonFly) - MK_CPPFLAGS += -D__BSD_VISIBLE -endif - -# alloca is a non-standard interface that is not visible on BSDs when -# POSIX conformance is specified, but not all of them provide a clean way -# to enable it in such cases -ifeq ($(UNAME_S),FreeBSD) - MK_CPPFLAGS += -D__BSD_VISIBLE -endif -ifeq ($(UNAME_S),NetBSD) - MK_CPPFLAGS += -D_NETBSD_SOURCE -endif -ifeq ($(UNAME_S),OpenBSD) - MK_CPPFLAGS += -D_BSD_SOURCE -endif - -ifdef LLAMA_SCHED_MAX_COPIES - MK_CPPFLAGS += -DGGML_SCHED_MAX_COPIES=$(LLAMA_SCHED_MAX_COPIES) -endif - -ifdef LLAMA_DEBUG - MK_CFLAGS += -O0 -g - MK_CXXFLAGS += -O0 -g - MK_LDFLAGS += -g - - ifeq ($(UNAME_S),Linux) - MK_CPPFLAGS += -D_GLIBCXX_ASSERTIONS - endif -else - MK_CPPFLAGS += -DNDEBUG -endif - -ifdef LLAMA_SANITIZE_THREAD - MK_CFLAGS += -fsanitize=thread -g - MK_CXXFLAGS += -fsanitize=thread -g - MK_LDFLAGS += -fsanitize=thread -g -endif - -ifdef LLAMA_SANITIZE_ADDRESS - MK_CFLAGS += -fsanitize=address -fno-omit-frame-pointer -g - MK_CXXFLAGS += -fsanitize=address -fno-omit-frame-pointer -g - MK_LDFLAGS += -fsanitize=address -fno-omit-frame-pointer -g -endif - -ifdef LLAMA_SANITIZE_UNDEFINED - MK_CFLAGS += -fsanitize=undefined -g - MK_CXXFLAGS += -fsanitize=undefined -g - MK_LDFLAGS += -fsanitize=undefined -g -endif - -ifdef LLAMA_SERVER_VERBOSE - MK_CPPFLAGS += -DSERVER_VERBOSE=$(LLAMA_SERVER_VERBOSE) -endif - -ifdef LLAMA_SERVER_SSL - MK_CPPFLAGS += -DCPPHTTPLIB_OPENSSL_SUPPORT - MK_LDFLAGS += -lssl -lcrypto -endif - -ifdef LLAMA_CODE_COVERAGE - MK_CXXFLAGS += -fprofile-arcs -ftest-coverage -dumpbase '' -endif - -ifdef LLAMA_DISABLE_LOGS - MK_CPPFLAGS += -DLOG_DISABLE_LOGS -endif # LLAMA_DISABLE_LOGS - -# warnings -WARN_FLAGS = -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -MK_CFLAGS += $(WARN_FLAGS) -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int \ - -Werror=implicit-function-declaration -MK_CXXFLAGS += $(WARN_FLAGS) -Wmissing-declarations -Wmissing-noreturn - -ifeq ($(LLAMA_FATAL_WARNINGS),1) - MK_CFLAGS += -Werror - MK_CXXFLAGS += -Werror -endif - -# this version of Apple ld64 is buggy -ifneq '' '$(findstring dyld-1015.7,$(shell $(CC) $(LDFLAGS) -Wl,-v 2>&1))' - MK_CPPFLAGS += -DHAVE_BUGGY_APPLE_LINKER -endif - -# OS specific -# TODO: support Windows -ifneq '' '$(filter $(UNAME_S),Linux Darwin FreeBSD NetBSD OpenBSD Haiku)' - MK_CFLAGS += -pthread - MK_CXXFLAGS += -pthread -endif - -# detect Windows -ifneq ($(findstring _NT,$(UNAME_S)),) - _WIN32 := 1 -endif - -# library name prefix -ifneq ($(_WIN32),1) - LIB_PRE := lib -endif - -# Dynamic Shared Object extension -ifneq ($(_WIN32),1) - DSO_EXT := .so -else - DSO_EXT := .dll -endif - -# Windows Sockets 2 (Winsock) for network-capable apps -ifeq ($(_WIN32),1) - LWINSOCK2 := -lws2_32 -endif - -ifdef LLAMA_GPROF - MK_CFLAGS += -pg - MK_CXXFLAGS += -pg -endif -ifdef LLAMA_PERF - MK_CPPFLAGS += -DGGML_PERF -endif - -# Architecture specific -# TODO: probably these flags need to be tweaked on some architectures -# feel free to update the Makefile for your architecture and send a pull request or issue - -ifndef RISCV - -ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) - # Use all CPU extensions that are available: - MK_CFLAGS += -march=native -mtune=native - HOST_CXXFLAGS += -march=native -mtune=native - - # Usage AVX-only - #MK_CFLAGS += -mfma -mf16c -mavx - #MK_CXXFLAGS += -mfma -mf16c -mavx - - # Usage SSSE3-only (Not is SSE3!) - #MK_CFLAGS += -mssse3 - #MK_CXXFLAGS += -mssse3 -endif - -ifneq '' '$(findstring mingw,$(shell $(CC) -dumpmachine))' - # The stack is only 16-byte aligned on Windows, so don't let gcc emit aligned moves. - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412 - # https://github.com/ggerganov/llama.cpp/issues/2922 - MK_CFLAGS += -Xassembler -muse-unaligned-vector-move - MK_CXXFLAGS += -Xassembler -muse-unaligned-vector-move - - # Target Windows 8 for PrefetchVirtualMemory - MK_CPPFLAGS += -D_WIN32_WINNT=0x602 -endif - -ifneq ($(filter aarch64%,$(UNAME_M)),) - # Apple M1, M2, etc. - # Raspberry Pi 3, 4, Zero 2 (64-bit) - # Nvidia Jetson - MK_CFLAGS += -mcpu=native - MK_CXXFLAGS += -mcpu=native - JETSON_RELEASE_INFO = $(shell jetson_release) - ifdef JETSON_RELEASE_INFO - ifneq ($(filter TX2%,$(JETSON_RELEASE_INFO)),) - JETSON_EOL_MODULE_DETECT = 1 - CC = aarch64-unknown-linux-gnu-gcc - cxx = aarch64-unknown-linux-gnu-g++ - endif - endif -endif - -ifneq ($(filter armv6%,$(UNAME_M)),) - # Raspberry Pi 1, Zero - MK_CFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access - MK_CXXFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access -endif - -ifneq ($(filter armv7%,$(UNAME_M)),) - # Raspberry Pi 2 - MK_CFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access -funsafe-math-optimizations - MK_CXXFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access -funsafe-math-optimizations -endif - -ifneq ($(filter armv8%,$(UNAME_M)),) - # Raspberry Pi 3, 4, Zero 2 (32-bit) - MK_CFLAGS += -mfp16-format=ieee -mno-unaligned-access - MK_CXXFLAGS += -mfp16-format=ieee -mno-unaligned-access -endif - -ifneq ($(filter ppc64%,$(UNAME_M)),) - POWER9_M := $(shell grep "POWER9" /proc/cpuinfo) - ifneq (,$(findstring POWER9,$(POWER9_M))) - MK_CFLAGS += -mcpu=power9 - MK_CXXFLAGS += -mcpu=power9 - endif -endif - -ifneq ($(filter ppc64le%,$(UNAME_M)),) - MK_CFLAGS += -mcpu=powerpc64le - MK_CXXFLAGS += -mcpu=powerpc64le - CUDA_POWER_ARCH = 1 -endif - -else - MK_CFLAGS += -march=rv64gcv -mabi=lp64d - MK_CXXFLAGS += -march=rv64gcv -mabi=lp64d -endif - -ifdef LLAMA_QKK_64 - MK_CPPFLAGS += -DGGML_QKK_64 -endif - -ifndef LLAMA_NO_ACCELERATE - # Mac OS - include Accelerate framework. - # `-framework Accelerate` works both with Apple Silicon and Mac Intel - ifeq ($(UNAME_S),Darwin) - MK_CPPFLAGS += -DGGML_USE_ACCELERATE - MK_CPPFLAGS += -DACCELERATE_NEW_LAPACK - MK_CPPFLAGS += -DACCELERATE_LAPACK_ILP64 - MK_LDFLAGS += -framework Accelerate - endif -endif # LLAMA_NO_ACCELERATE - -ifdef LLAMA_MPI - MK_CPPFLAGS += -DGGML_USE_MPI - MK_CFLAGS += -Wno-cast-qual - MK_CXXFLAGS += -Wno-cast-qual - OBJS += ggml-mpi.o -endif # LLAMA_MPI - -ifdef LLAMA_OPENBLAS - MK_CPPFLAGS += -DGGML_USE_OPENBLAS $(shell pkg-config --cflags-only-I openblas) - MK_CFLAGS += $(shell pkg-config --cflags-only-other openblas) - MK_LDFLAGS += $(shell pkg-config --libs openblas) -endif # LLAMA_OPENBLAS - -ifdef LLAMA_BLIS - MK_CPPFLAGS += -DGGML_USE_OPENBLAS -I/usr/local/include/blis -I/usr/include/blis - MK_LDFLAGS += -lblis -L/usr/local/lib -endif # LLAMA_BLIS - -ifdef LLAMA_CUBLAS -# LLAMA_CUBLAS is deprecated and will be removed in the future - LLAMA_CUDA := 1 -endif - -ifdef LLAMA_CUDA - ifneq ('', '$(wildcard /opt/cuda)') - CUDA_PATH ?= /opt/cuda - else - CUDA_PATH ?= /usr/local/cuda - endif - MK_CPPFLAGS += -DGGML_USE_CUDA -I$(CUDA_PATH)/include -I$(CUDA_PATH)/targets/$(UNAME_M)-linux/include - MK_LDFLAGS += -lcuda -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L$(CUDA_PATH)/lib64 -L/usr/lib64 -L$(CUDA_PATH)/targets/$(UNAME_M)-linux/lib -L/usr/lib/wsl/lib - OBJS += ggml-cuda.o - OBJS += $(patsubst %.cu,%.o,$(wildcard ggml-cuda/*.cu)) - MK_NVCCFLAGS += -use_fast_math -ifdef LLAMA_FATAL_WARNINGS - MK_NVCCFLAGS += -Werror all-warnings -endif # LLAMA_FATAL_WARNINGS -ifndef JETSON_EOL_MODULE_DETECT - MK_NVCCFLAGS += --forward-unknown-to-host-compiler -endif # JETSON_EOL_MODULE_DETECT -ifdef LLAMA_DEBUG - MK_NVCCFLAGS += -lineinfo -endif # LLAMA_DEBUG -ifdef LLAMA_CUDA_NVCC - NVCC = $(CCACHE) $(LLAMA_CUDA_NVCC) -else - NVCC = $(CCACHE) nvcc -endif #LLAMA_CUDA_NVCC -ifdef CUDA_DOCKER_ARCH - MK_NVCCFLAGS += -Wno-deprecated-gpu-targets -arch=$(CUDA_DOCKER_ARCH) -else ifndef CUDA_POWER_ARCH - MK_NVCCFLAGS += -arch=native -endif # CUDA_DOCKER_ARCH -ifdef LLAMA_CUDA_FORCE_DMMV - MK_NVCCFLAGS += -DGGML_CUDA_FORCE_DMMV -endif # LLAMA_CUDA_FORCE_DMMV -ifdef LLAMA_CUDA_FORCE_MMQ - MK_NVCCFLAGS += -DGGML_CUDA_FORCE_MMQ -endif # LLAMA_CUDA_FORCE_MMQ -ifdef LLAMA_CUDA_DMMV_X - MK_NVCCFLAGS += -DGGML_CUDA_DMMV_X=$(LLAMA_CUDA_DMMV_X) -else - MK_NVCCFLAGS += -DGGML_CUDA_DMMV_X=32 -endif # LLAMA_CUDA_DMMV_X -ifdef LLAMA_CUDA_MMV_Y - MK_NVCCFLAGS += -DGGML_CUDA_MMV_Y=$(LLAMA_CUDA_MMV_Y) -else ifdef LLAMA_CUDA_DMMV_Y - MK_NVCCFLAGS += -DGGML_CUDA_MMV_Y=$(LLAMA_CUDA_DMMV_Y) # for backwards compatibility -else - MK_NVCCFLAGS += -DGGML_CUDA_MMV_Y=1 -endif # LLAMA_CUDA_MMV_Y -ifdef LLAMA_CUDA_F16 - MK_NVCCFLAGS += -DGGML_CUDA_F16 -endif # LLAMA_CUDA_F16 -ifdef LLAMA_CUDA_DMMV_F16 - MK_NVCCFLAGS += -DGGML_CUDA_F16 -endif # LLAMA_CUDA_DMMV_F16 -ifdef LLAMA_CUDA_KQUANTS_ITER - MK_NVCCFLAGS += -DK_QUANTS_PER_ITERATION=$(LLAMA_CUDA_KQUANTS_ITER) -else - MK_NVCCFLAGS += -DK_QUANTS_PER_ITERATION=2 -endif -ifdef LLAMA_CUDA_PEER_MAX_BATCH_SIZE - MK_NVCCFLAGS += -DGGML_CUDA_PEER_MAX_BATCH_SIZE=$(LLAMA_CUDA_PEER_MAX_BATCH_SIZE) -else - MK_NVCCFLAGS += -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -endif # LLAMA_CUDA_PEER_MAX_BATCH_SIZE -ifdef LLAMA_CUDA_NO_PEER_COPY - MK_NVCCFLAGS += -DGGML_CUDA_NO_PEER_COPY -endif # LLAMA_CUDA_NO_PEER_COPY -ifdef LLAMA_CUDA_CCBIN - MK_NVCCFLAGS += -ccbin $(LLAMA_CUDA_CCBIN) -endif - -ifdef JETSON_EOL_MODULE_DETECT -define NVCC_COMPILE - $(NVCC) -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUDA -I/usr/local/cuda/include -I/opt/cuda/include -I/usr/local/cuda/targets/aarch64-linux/include -std=c++11 -O3 $(NVCCFLAGS) $(CPPFLAGS) -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ -endef # NVCC_COMPILE -else -define NVCC_COMPILE - $(NVCC) $(NVCCFLAGS) $(CPPFLAGS) -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ -endef # NVCC_COMPILE -endif # JETSON_EOL_MODULE_DETECT - -ggml-cuda/%.o: ggml-cuda/%.cu ggml-cuda/%.cuh ggml.h ggml-common.h ggml-cuda/common.cuh - $(NVCC_COMPILE) - -ggml-cuda.o: ggml-cuda.cu ggml-cuda.h ggml.h ggml-backend.h ggml-backend-impl.h ggml-common.h $(wildcard ggml-cuda/*.cuh) - $(NVCC_COMPILE) - -endif # LLAMA_CUDA - -ifdef LLAMA_CLBLAST - - MK_CPPFLAGS += -DGGML_USE_CLBLAST $(shell pkg-config --cflags-only-I clblast OpenCL) - MK_CFLAGS += $(shell pkg-config --cflags-only-other clblast OpenCL) - MK_CXXFLAGS += $(shell pkg-config --cflags-only-other clblast OpenCL) - - # Mac provides OpenCL as a framework - ifeq ($(UNAME_S),Darwin) - MK_LDFLAGS += -lclblast -framework OpenCL - else - MK_LDFLAGS += $(shell pkg-config --libs clblast OpenCL) - endif - OBJS += ggml-opencl.o - -ggml-opencl.o: ggml-opencl.cpp ggml-opencl.h - $(CXX) $(CXXFLAGS) -c $< -o $@ -endif # LLAMA_CLBLAST - -ifdef LLAMA_VULKAN - MK_CPPFLAGS += -DGGML_USE_VULKAN - MK_LDFLAGS += -lvulkan - OBJS += ggml-vulkan.o - -ifdef LLAMA_VULKAN_CHECK_RESULTS - MK_CPPFLAGS += -DGGML_VULKAN_CHECK_RESULTS -endif - -ifdef LLAMA_VULKAN_DEBUG - MK_CPPFLAGS += -DGGML_VULKAN_DEBUG -endif - -ifdef LLAMA_VULKAN_VALIDATE - MK_CPPFLAGS += -DGGML_VULKAN_VALIDATE -endif - -ifdef LLAMA_VULKAN_RUN_TESTS - MK_CPPFLAGS += -DGGML_VULKAN_RUN_TESTS -endif - -ggml-vulkan.o: ggml-vulkan.cpp ggml-vulkan.h - $(CXX) $(CXXFLAGS) -c $< -o $@ -endif # LLAMA_VULKAN - -ifdef LLAMA_HIPBLAS - ifeq ($(wildcard /opt/rocm),) - ROCM_PATH ?= /usr - GPU_TARGETS ?= $(shell $(shell which amdgpu-arch)) - else - ROCM_PATH ?= /opt/rocm - GPU_TARGETS ?= $(shell $(ROCM_PATH)/llvm/bin/amdgpu-arch) - endif - HIPCC ?= $(CCACHE) $(ROCM_PATH)/bin/hipcc - LLAMA_CUDA_DMMV_X ?= 32 - LLAMA_CUDA_MMV_Y ?= 1 - LLAMA_CUDA_KQUANTS_ITER ?= 2 - MK_CPPFLAGS += -DGGML_USE_HIPBLAS -DGGML_USE_CUDA -ifdef LLAMA_HIP_UMA - MK_CPPFLAGS += -DGGML_HIP_UMA -endif # LLAMA_HIP_UMA - MK_LDFLAGS += -L$(ROCM_PATH)/lib -Wl,-rpath=$(ROCM_PATH)/lib - MK_LDFLAGS += -lhipblas -lamdhip64 -lrocblas - HIPFLAGS += $(addprefix --offload-arch=,$(GPU_TARGETS)) - HIPFLAGS += -DGGML_CUDA_DMMV_X=$(LLAMA_CUDA_DMMV_X) - HIPFLAGS += -DGGML_CUDA_MMV_Y=$(LLAMA_CUDA_MMV_Y) - HIPFLAGS += -DK_QUANTS_PER_ITERATION=$(LLAMA_CUDA_KQUANTS_ITER) -ifdef LLAMA_CUDA_FORCE_DMMV - HIPFLAGS += -DGGML_CUDA_FORCE_DMMV -endif # LLAMA_CUDA_FORCE_DMMV -ifdef LLAMA_CUDA_NO_PEER_COPY - HIPFLAGS += -DGGML_CUDA_NO_PEER_COPY -endif # LLAMA_CUDA_NO_PEER_COPY - OBJS += ggml-cuda.o - OBJS += $(patsubst %.cu,%.o,$(wildcard ggml-cuda/*.cu)) - -ggml-cuda.o: ggml-cuda.cu ggml-cuda.h ggml.h ggml-backend.h ggml-backend-impl.h ggml-common.h $(wildcard ggml-cuda/*.cuh) - $(HIPCC) $(CXXFLAGS) $(HIPFLAGS) -x hip -c -o $@ $< - -ggml-cuda/%.o: ggml-cuda/%.cu ggml-cuda/%.cuh ggml.h ggml-common.h ggml-cuda/common.cuh - $(HIPCC) $(CXXFLAGS) $(HIPFLAGS) -x hip -c -o $@ $< - -endif # LLAMA_HIPBLAS - -ifdef LLAMA_METAL - MK_CPPFLAGS += -DGGML_USE_METAL - MK_LDFLAGS += -framework Foundation -framework Metal -framework MetalKit - OBJS += ggml-metal.o -ifdef LLAMA_METAL_NDEBUG - MK_CPPFLAGS += -DGGML_METAL_NDEBUG -endif -ifdef LLAMA_METAL_EMBED_LIBRARY - MK_CPPFLAGS += -DGGML_METAL_EMBED_LIBRARY - OBJS += ggml-metal-embed.o -endif -endif # LLAMA_METAL - -ifdef LLAMA_METAL -ggml-metal.o: ggml-metal.m ggml-metal.h ggml.h - $(CC) $(CFLAGS) -c $< -o $@ - -ifdef LLAMA_METAL_EMBED_LIBRARY -ggml-metal-embed.o: ggml-metal.metal ggml-common.h - @echo "Embedding Metal library" - @sed -e '/#include "ggml-common.h"/r ggml-common.h' -e '/#include "ggml-common.h"/d' < ggml-metal.metal > ggml-metal-embed.metal - $(eval TEMP_ASSEMBLY=$(shell mktemp)) - @echo ".section __DATA, __ggml_metallib" > $(TEMP_ASSEMBLY) - @echo ".globl _ggml_metallib_start" >> $(TEMP_ASSEMBLY) - @echo "_ggml_metallib_start:" >> $(TEMP_ASSEMBLY) - @echo ".incbin \"ggml-metal-embed.metal\"" >> $(TEMP_ASSEMBLY) - @echo ".globl _ggml_metallib_end" >> $(TEMP_ASSEMBLY) - @echo "_ggml_metallib_end:" >> $(TEMP_ASSEMBLY) - @$(AS) $(TEMP_ASSEMBLY) -o $@ - @rm -f ${TEMP_ASSEMBLY} -endif -endif # LLAMA_METAL - -ifdef LLAMA_MPI -ggml-mpi.o: ggml-mpi.c ggml-mpi.h - $(CC) $(CFLAGS) -c $< -o $@ -endif # LLAMA_MPI - -GF_CC := $(CC) -include scripts/get-flags.mk - -# combine build flags with cmdline overrides -override CPPFLAGS := $(MK_CPPFLAGS) $(CPPFLAGS) -override CFLAGS := $(CPPFLAGS) $(MK_CFLAGS) $(GF_CFLAGS) $(CFLAGS) -BASE_CXXFLAGS := $(MK_CXXFLAGS) $(CXXFLAGS) -override CXXFLAGS := $(BASE_CXXFLAGS) $(HOST_CXXFLAGS) $(GF_CXXFLAGS) $(CPPFLAGS) -override NVCCFLAGS := $(MK_NVCCFLAGS) $(NVCCFLAGS) -override LDFLAGS := $(MK_LDFLAGS) $(LDFLAGS) - -# identify CUDA host compiler -ifdef LLAMA_CUDA -GF_CC := $(NVCC) $(NVCCFLAGS) 2>/dev/null .c -Xcompiler -include scripts/get-flags.mk -CUDA_CXXFLAGS := $(BASE_CXXFLAGS) $(GF_CXXFLAGS) -Wno-pedantic -endif - -ifdef LLAMA_CURL -override CXXFLAGS := $(CXXFLAGS) -DLLAMA_USE_CURL -override LDFLAGS := $(LDFLAGS) -lcurl -endif - -# -# Print build information -# - -$(info I llama.cpp build info: ) -$(info I UNAME_S: $(UNAME_S)) -$(info I UNAME_P: $(UNAME_P)) -$(info I UNAME_M: $(UNAME_M)) -$(info I CFLAGS: $(CFLAGS)) -$(info I CXXFLAGS: $(CXXFLAGS)) -$(info I NVCCFLAGS: $(NVCCFLAGS)) -$(info I LDFLAGS: $(LDFLAGS)) -$(info I CC: $(shell $(CC) --version | head -n 1)) -$(info I CXX: $(shell $(CXX) --version | head -n 1)) -ifdef LLAMA_CUDA -$(info I NVCC: $(shell $(NVCC) --version | tail -n 1)) -CUDA_VERSION := $(shell $(NVCC) --version | grep -oP 'release (\K[0-9]+\.[0-9])') -ifeq ($(shell awk -v "v=$(CUDA_VERSION)" 'BEGIN { print (v < 11.7) }'),1) -ifndef CUDA_DOCKER_ARCH -ifndef CUDA_POWER_ARCH -$(error I ERROR: For CUDA versions < 11.7 a target CUDA architecture must be explicitly provided via CUDA_DOCKER_ARCH) -endif # CUDA_POWER_ARCH -endif # CUDA_DOCKER_ARCH -endif # eq ($(shell echo "$(CUDA_VERSION) < 11.7" | bc),1) -endif # LLAMA_CUDA -$(info ) - -ifdef LLAMA_CUBLAS -$(info !!!!) -$(info LLAMA_CUBLAS is deprecated and will be removed in the future. Use LLAMA_CUDA instead.) -$(info !!!!) -$(info ) -endif - -# -# Build library -# - -ggml.o: ggml.c ggml.h ggml-cuda.h - $(CC) $(CFLAGS) -c $< -o $@ - -ggml-alloc.o: ggml-alloc.c ggml.h ggml-alloc.h - $(CC) $(CFLAGS) -c $< -o $@ - -ggml-backend.o: ggml-backend.c ggml.h ggml-backend.h - $(CC) $(CFLAGS) -c $< -o $@ - -ggml-quants.o: ggml-quants.c ggml.h ggml-quants.h ggml-common.h - $(CC) $(CFLAGS) -c $< -o $@ - -unicode.o: unicode.cpp unicode.h - $(CXX) $(CXXFLAGS) -c $< -o $@ - -unicode-data.o: unicode-data.cpp unicode-data.h - $(CXX) $(CXXFLAGS) -c $< -o $@ - -OBJS += ggml-alloc.o ggml-backend.o ggml-quants.o unicode.o unicode-data.o - -llama.o: llama.cpp unicode.h ggml.h ggml-alloc.h ggml-backend.h ggml-cuda.h ggml-metal.h llama.h - $(CXX) $(CXXFLAGS) -c $< -o $@ - -COMMON_H_DEPS = common/common.h common/sampling.h common/log.h -COMMON_DEPS = common.o sampling.o grammar-parser.o build-info.o - -common.o: common/common.cpp $(COMMON_H_DEPS) - $(CXX) $(CXXFLAGS) -c $< -o $@ - -sampling.o: common/sampling.cpp $(COMMON_H_DEPS) - $(CXX) $(CXXFLAGS) -c $< -o $@ - -console.o: common/console.cpp common/console.h - $(CXX) $(CXXFLAGS) -c $< -o $@ - -grammar-parser.o: common/grammar-parser.cpp common/grammar-parser.h - $(CXX) $(CXXFLAGS) -c $< -o $@ - -json-schema-to-grammar.o: common/json-schema-to-grammar.cpp common/json-schema-to-grammar.h - $(CXX) $(CXXFLAGS) -c $< -o $@ - -train.o: common/train.cpp common/train.h - $(CXX) $(CXXFLAGS) -c $< -o $@ - -ngram-cache.o: common/ngram-cache.cpp common/ngram-cache.h - $(CXX) $(CXXFLAGS) -c $< -o $@ - -libllama.so: llama.o ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -shared -fPIC -o $@ $^ $(LDFLAGS) - -libllama.a: llama.o ggml.o $(OBJS) $(COMMON_DEPS) - ar rcs libllama.a llama.o ggml.o $(OBJS) $(COMMON_DEPS) +# CMAKE generated file: DO NOT EDIT! +# Generated by "Unix Makefiles" Generator, CMake Version 3.29 -clean: - rm -vrf *.o tests/*.o *.so *.a *.dll benchmark-matmult lookup-create lookup-merge lookup-stats common/build-info.cpp *.dot $(COV_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) - rm -vrf ggml-cuda/*.o - find examples pocs -type f -name "*.o" -delete - -# -# Examples -# - -# $< is the first prerequisite, i.e. the source file. -# Explicitly compile this to an object file so that it can be cached with ccache. -# The source file is then filtered out from $^ (the list of all prerequisites) and the object file is added instead. - -# Helper function that replaces .c, .cpp, and .cu file endings with .o: -GET_OBJ_FILE = $(patsubst %.c,%.o,$(patsubst %.cpp,%.o,$(patsubst %.cu,%.o,$(1)))) +# Default target executed when no arguments are given to make. +default_target: all +.PHONY : default_target + +# Allow only one "make -f Makefile2" at a time, but pass parallelism. +.NOTPARALLEL: + +#============================================================================= +# Special targets provided by cmake. + +# Disable implicit rules so canonical targets will work. +.SUFFIXES: + +# Disable VCS-based implicit rules. +% : %,v + +# Disable VCS-based implicit rules. +% : RCS/% + +# Disable VCS-based implicit rules. +% : RCS/%,v + +# Disable VCS-based implicit rules. +% : SCCS/s.% + +# Disable VCS-based implicit rules. +% : s.% + +.SUFFIXES: .hpux_make_needs_suffix_list + +# Command-line flag to silence nested $(MAKE). +$(VERBOSE)MAKESILENT = -s + +#Suppress display of executed commands. +$(VERBOSE).SILENT: + +# A target that is always out of date. +cmake_force: +.PHONY : cmake_force + +#============================================================================= +# Set environment variables for the build. -main: examples/main/main.cpp ggml.o llama.o $(COMMON_DEPS) console.o grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - @echo - @echo '==== Run ./main -h for help. ====' - @echo - -infill: examples/infill/infill.cpp ggml.o llama.o $(COMMON_DEPS) console.o grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -simple: examples/simple/simple.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tokenize: examples/tokenize/tokenize.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -batched: examples/batched/batched.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -batched-bench: examples/batched-bench/batched-bench.cpp build-info.o ggml.o llama.o common.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -quantize: examples/quantize/quantize.cpp build-info.o ggml.o llama.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -quantize-stats: examples/quantize-stats/quantize-stats.cpp build-info.o ggml.o llama.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -perplexity: examples/perplexity/perplexity.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -imatrix: examples/imatrix/imatrix.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -embedding: examples/embedding/embedding.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -gritlm: examples/gritlm/gritlm.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -server: examples/server/server.cpp examples/server/utils.hpp examples/server/httplib.h common/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp json-schema-to-grammar.o common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h %.hpp $<,$^) -Iexamples/server $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) $(LWINSOCK2) - -gguf: examples/gguf/gguf.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -gguf-split: examples/gguf-split/gguf-split.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -convert-llama2c-to-ggml: examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -llama-bench: examples/llama-bench/llama-bench.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -libllava.a: examples/llava/llava.cpp examples/llava/llava.h examples/llava/clip.cpp examples/llava/clip.h common/stb_image.h common/base64.hpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -static -fPIC -c $< -o $@ -Wno-cast-qual - -llava-cli: examples/llava/llava-cli.cpp examples/llava/clip.h examples/llava/clip.cpp examples/llava/llava.h examples/llava/llava.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) -c examples/llava/clip.cpp -o $(call GET_OBJ_FILE, examples/llava/clip.cpp) -Wno-cast-qual - $(CXX) $(CXXFLAGS) -c examples/llava/llava.cpp -o $(call GET_OBJ_FILE, examples/llava/llava.cpp) - $(CXX) $(CXXFLAGS) $(filter-out %.h $< examples/llava/clip.cpp examples/llava/llava.cpp,$^) $(call GET_OBJ_FILE, $<) $(call GET_OBJ_FILE, examples/llava/clip.cpp) $(call GET_OBJ_FILE, examples/llava/llava.cpp) -o $@ $(LDFLAGS) - -baby-llama: examples/baby-llama/baby-llama.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -beam-search: examples/beam-search/beam-search.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -finetune: examples/finetune/finetune.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -export-lora: examples/export-lora/export-lora.cpp ggml.o common/common.h $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -retrieval: examples/retrieval/retrieval.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -speculative: examples/speculative/speculative.cpp ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -parallel: examples/parallel/parallel.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -lookahead: examples/lookahead/lookahead.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -lookup: examples/lookup/lookup.cpp ggml.o llama.o ngram-cache.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - $(CXX) $(CXXFLAGS) -c examples/lookup/lookup-create.cpp -o $(call GET_OBJ_FILE, examples/lookup/lookup-create.cpp) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, examples/lookup/lookup-create.cpp) -o lookup-create $(LDFLAGS) - $(CXX) $(CXXFLAGS) -c examples/lookup/lookup-merge.cpp -o $(call GET_OBJ_FILE, examples/lookup/lookup-merge.cpp) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, examples/lookup/lookup-merge.cpp) -o lookup-merge $(LDFLAGS) - $(CXX) $(CXXFLAGS) -c examples/lookup/lookup-stats.cpp -o $(call GET_OBJ_FILE, examples/lookup/lookup-stats.cpp) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, examples/lookup/lookup-stats.cpp) -o lookup-stats $(LDFLAGS) - -passkey: examples/passkey/passkey.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -gbnf-validator: examples/gbnf-validator/gbnf-validator.cpp ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -ifeq ($(UNAME_S),Darwin) -swift: examples/batched.swift - (cd examples/batched.swift; make build) -endif - -common/build-info.cpp: $(wildcard .git/index) scripts/build-info.sh - @sh scripts/build-info.sh "$(CC)" > $@.tmp - @if ! cmp -s $@.tmp $@; then \ - mv $@.tmp $@; \ - else \ - rm $@.tmp; \ - fi - -build-info.o: common/build-info.cpp - $(CXX) $(CXXFLAGS) -c $(filter-out %.h,$^) -o $@ - -# -# Tests -# - -tests: $(TEST_TARGETS) - -benchmark-matmult: examples/benchmark/benchmark-matmult.cpp build-info.o ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -run-benchmark-matmult: benchmark-matmult - ./$@ - -.PHONY: run-benchmark-matmult swift - -vdot: pocs/vdot/vdot.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -q8dot: pocs/vdot/q8dot.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-llama-grammar: tests/test-llama-grammar.cpp ggml.o grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-grammar-parser: tests/test-grammar-parser.cpp ggml.o llama.o grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-grammar-integration: tests/test-grammar-integration.cpp ggml.o llama.o grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-double-float: tests/test-double-float.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-json-schema-to-grammar: tests/test-json-schema-to-grammar.cpp json-schema-to-grammar.o ggml.o llama.o grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) -Iexamples/server -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-grad0: tests/test-grad0.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-opt: tests/test-opt.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-quantize-fns: tests/test-quantize-fns.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-quantize-perf: tests/test-quantize-perf.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-sampling: tests/test-sampling.cpp ggml.o llama.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-tokenizer-0-falcon: tests/test-tokenizer-0-falcon.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) - -tests/test-tokenizer-0-llama: tests/test-tokenizer-0-llama.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +# The shell in which to execute make rules. +SHELL = /bin/sh -tests/test-tokenizer-1-bpe: tests/test-tokenizer-1-bpe.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +# The CMake executable. +CMAKE_COMMAND = /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -tests/test-tokenizer-1-llama: tests/test-tokenizer-1-llama.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +# The command to remove a file. +RM = /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -E rm -f -tests/test-rope: tests/test-rope.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +# Escaping for special characters. +EQUALS = = -tests/test-c.o: tests/test-c.c llama.h - $(CC) $(CFLAGS) -c $(filter-out %.h,$^) -o $@ +# The top-level source directory on which CMake was run. +CMAKE_SOURCE_DIR = /home/joan/workspace/llama.cpp + +# The top-level build directory on which CMake was run. +CMAKE_BINARY_DIR = /home/joan/workspace/llama.cpp + +#============================================================================= +# Targets provided globally by CMake. + +# Special rule for the target test +test: + @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Running tests..." + /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/ctest --force-new-ctest-process $(ARGS) +.PHONY : test + +# Special rule for the target test +test/fast: test +.PHONY : test/fast + +# Special rule for the target edit_cache +edit_cache: + @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "No interactive CMake dialog available..." + /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -E echo No\ interactive\ CMake\ dialog\ available. +.PHONY : edit_cache + +# Special rule for the target edit_cache +edit_cache/fast: edit_cache +.PHONY : edit_cache/fast + +# Special rule for the target rebuild_cache +rebuild_cache: + @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Running CMake to regenerate build system..." + /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake --regenerate-during-build -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) +.PHONY : rebuild_cache + +# Special rule for the target rebuild_cache +rebuild_cache/fast: rebuild_cache +.PHONY : rebuild_cache/fast + +# Special rule for the target list_install_components +list_install_components: + @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Available install components are: \"Unspecified\"" +.PHONY : list_install_components + +# Special rule for the target list_install_components +list_install_components/fast: list_install_components +.PHONY : list_install_components/fast + +# Special rule for the target install +install: preinstall + @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Install the project..." + /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -P cmake_install.cmake +.PHONY : install + +# Special rule for the target install +install/fast: preinstall/fast + @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Install the project..." + /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -P cmake_install.cmake +.PHONY : install/fast + +# Special rule for the target install/local +install/local: preinstall + @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing only the local directory..." + /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake +.PHONY : install/local + +# Special rule for the target install/local +install/local/fast: preinstall/fast + @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing only the local directory..." + /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake +.PHONY : install/local/fast + +# Special rule for the target install/strip +install/strip: preinstall + @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing the project stripped..." + /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake +.PHONY : install/strip + +# Special rule for the target install/strip +install/strip/fast: preinstall/fast + @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing the project stripped..." + /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake +.PHONY : install/strip/fast + +# The main all target +all: cmake_check_build_system + $(CMAKE_COMMAND) -E cmake_progress_start /home/joan/workspace/llama.cpp/CMakeFiles /home/joan/workspace/llama.cpp//CMakeFiles/progress.marks + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 all + $(CMAKE_COMMAND) -E cmake_progress_start /home/joan/workspace/llama.cpp/CMakeFiles 0 +.PHONY : all + +# The main clean target +clean: + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 clean +.PHONY : clean + +# The main clean target +clean/fast: clean +.PHONY : clean/fast + +# Prepare targets for installation. +preinstall: all + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 preinstall +.PHONY : preinstall + +# Prepare targets for installation. +preinstall/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 preinstall +.PHONY : preinstall/fast + +# clear depends +depend: + $(CMAKE_COMMAND) -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 1 +.PHONY : depend + +#============================================================================= +# Target rules for targets named ggml + +# Build rule for target. +ggml: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ggml +.PHONY : ggml + +# fast build rule for target. +ggml/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/build +.PHONY : ggml/fast + +#============================================================================= +# Target rules for targets named ggml_static + +# Build rule for target. +ggml_static: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ggml_static +.PHONY : ggml_static + +# fast build rule for target. +ggml_static/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml_static.dir/build.make CMakeFiles/ggml_static.dir/build +.PHONY : ggml_static/fast + +#============================================================================= +# Target rules for targets named llama + +# Build rule for target. +llama: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 llama +.PHONY : llama + +# fast build rule for target. +llama/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/build +.PHONY : llama/fast + +#============================================================================= +# Target rules for targets named Experimental + +# Build rule for target. +Experimental: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 Experimental +.PHONY : Experimental + +# fast build rule for target. +Experimental/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/Experimental.dir/build.make CMakeFiles/Experimental.dir/build +.PHONY : Experimental/fast + +#============================================================================= +# Target rules for targets named Nightly + +# Build rule for target. +Nightly: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 Nightly +.PHONY : Nightly + +# fast build rule for target. +Nightly/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/Nightly.dir/build.make CMakeFiles/Nightly.dir/build +.PHONY : Nightly/fast + +#============================================================================= +# Target rules for targets named Continuous + +# Build rule for target. +Continuous: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 Continuous +.PHONY : Continuous + +# fast build rule for target. +Continuous/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/Continuous.dir/build.make CMakeFiles/Continuous.dir/build +.PHONY : Continuous/fast + +#============================================================================= +# Target rules for targets named NightlyMemoryCheck + +# Build rule for target. +NightlyMemoryCheck: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyMemoryCheck +.PHONY : NightlyMemoryCheck + +# fast build rule for target. +NightlyMemoryCheck/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyMemoryCheck.dir/build.make CMakeFiles/NightlyMemoryCheck.dir/build +.PHONY : NightlyMemoryCheck/fast + +#============================================================================= +# Target rules for targets named NightlyStart + +# Build rule for target. +NightlyStart: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyStart +.PHONY : NightlyStart + +# fast build rule for target. +NightlyStart/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyStart.dir/build.make CMakeFiles/NightlyStart.dir/build +.PHONY : NightlyStart/fast + +#============================================================================= +# Target rules for targets named NightlyUpdate + +# Build rule for target. +NightlyUpdate: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyUpdate +.PHONY : NightlyUpdate + +# fast build rule for target. +NightlyUpdate/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyUpdate.dir/build.make CMakeFiles/NightlyUpdate.dir/build +.PHONY : NightlyUpdate/fast + +#============================================================================= +# Target rules for targets named NightlyConfigure + +# Build rule for target. +NightlyConfigure: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyConfigure +.PHONY : NightlyConfigure + +# fast build rule for target. +NightlyConfigure/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyConfigure.dir/build.make CMakeFiles/NightlyConfigure.dir/build +.PHONY : NightlyConfigure/fast + +#============================================================================= +# Target rules for targets named NightlyBuild + +# Build rule for target. +NightlyBuild: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyBuild +.PHONY : NightlyBuild + +# fast build rule for target. +NightlyBuild/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyBuild.dir/build.make CMakeFiles/NightlyBuild.dir/build +.PHONY : NightlyBuild/fast + +#============================================================================= +# Target rules for targets named NightlyTest + +# Build rule for target. +NightlyTest: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyTest +.PHONY : NightlyTest + +# fast build rule for target. +NightlyTest/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyTest.dir/build.make CMakeFiles/NightlyTest.dir/build +.PHONY : NightlyTest/fast + +#============================================================================= +# Target rules for targets named NightlyCoverage + +# Build rule for target. +NightlyCoverage: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyCoverage +.PHONY : NightlyCoverage + +# fast build rule for target. +NightlyCoverage/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyCoverage.dir/build.make CMakeFiles/NightlyCoverage.dir/build +.PHONY : NightlyCoverage/fast + +#============================================================================= +# Target rules for targets named NightlyMemCheck + +# Build rule for target. +NightlyMemCheck: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyMemCheck +.PHONY : NightlyMemCheck + +# fast build rule for target. +NightlyMemCheck/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyMemCheck.dir/build.make CMakeFiles/NightlyMemCheck.dir/build +.PHONY : NightlyMemCheck/fast + +#============================================================================= +# Target rules for targets named NightlySubmit + +# Build rule for target. +NightlySubmit: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlySubmit +.PHONY : NightlySubmit + +# fast build rule for target. +NightlySubmit/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlySubmit.dir/build.make CMakeFiles/NightlySubmit.dir/build +.PHONY : NightlySubmit/fast + +#============================================================================= +# Target rules for targets named ExperimentalStart + +# Build rule for target. +ExperimentalStart: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalStart +.PHONY : ExperimentalStart + +# fast build rule for target. +ExperimentalStart/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalStart.dir/build.make CMakeFiles/ExperimentalStart.dir/build +.PHONY : ExperimentalStart/fast + +#============================================================================= +# Target rules for targets named ExperimentalUpdate + +# Build rule for target. +ExperimentalUpdate: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalUpdate +.PHONY : ExperimentalUpdate + +# fast build rule for target. +ExperimentalUpdate/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalUpdate.dir/build.make CMakeFiles/ExperimentalUpdate.dir/build +.PHONY : ExperimentalUpdate/fast + +#============================================================================= +# Target rules for targets named ExperimentalConfigure + +# Build rule for target. +ExperimentalConfigure: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalConfigure +.PHONY : ExperimentalConfigure + +# fast build rule for target. +ExperimentalConfigure/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalConfigure.dir/build.make CMakeFiles/ExperimentalConfigure.dir/build +.PHONY : ExperimentalConfigure/fast + +#============================================================================= +# Target rules for targets named ExperimentalBuild + +# Build rule for target. +ExperimentalBuild: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalBuild +.PHONY : ExperimentalBuild + +# fast build rule for target. +ExperimentalBuild/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalBuild.dir/build.make CMakeFiles/ExperimentalBuild.dir/build +.PHONY : ExperimentalBuild/fast + +#============================================================================= +# Target rules for targets named ExperimentalTest + +# Build rule for target. +ExperimentalTest: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalTest +.PHONY : ExperimentalTest + +# fast build rule for target. +ExperimentalTest/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalTest.dir/build.make CMakeFiles/ExperimentalTest.dir/build +.PHONY : ExperimentalTest/fast + +#============================================================================= +# Target rules for targets named ExperimentalCoverage + +# Build rule for target. +ExperimentalCoverage: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalCoverage +.PHONY : ExperimentalCoverage + +# fast build rule for target. +ExperimentalCoverage/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalCoverage.dir/build.make CMakeFiles/ExperimentalCoverage.dir/build +.PHONY : ExperimentalCoverage/fast + +#============================================================================= +# Target rules for targets named ExperimentalMemCheck + +# Build rule for target. +ExperimentalMemCheck: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalMemCheck +.PHONY : ExperimentalMemCheck + +# fast build rule for target. +ExperimentalMemCheck/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalMemCheck.dir/build.make CMakeFiles/ExperimentalMemCheck.dir/build +.PHONY : ExperimentalMemCheck/fast + +#============================================================================= +# Target rules for targets named ExperimentalSubmit + +# Build rule for target. +ExperimentalSubmit: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalSubmit +.PHONY : ExperimentalSubmit + +# fast build rule for target. +ExperimentalSubmit/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalSubmit.dir/build.make CMakeFiles/ExperimentalSubmit.dir/build +.PHONY : ExperimentalSubmit/fast + +#============================================================================= +# Target rules for targets named ContinuousStart + +# Build rule for target. +ContinuousStart: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousStart +.PHONY : ContinuousStart + +# fast build rule for target. +ContinuousStart/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousStart.dir/build.make CMakeFiles/ContinuousStart.dir/build +.PHONY : ContinuousStart/fast + +#============================================================================= +# Target rules for targets named ContinuousUpdate + +# Build rule for target. +ContinuousUpdate: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousUpdate +.PHONY : ContinuousUpdate + +# fast build rule for target. +ContinuousUpdate/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousUpdate.dir/build.make CMakeFiles/ContinuousUpdate.dir/build +.PHONY : ContinuousUpdate/fast + +#============================================================================= +# Target rules for targets named ContinuousConfigure + +# Build rule for target. +ContinuousConfigure: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousConfigure +.PHONY : ContinuousConfigure + +# fast build rule for target. +ContinuousConfigure/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousConfigure.dir/build.make CMakeFiles/ContinuousConfigure.dir/build +.PHONY : ContinuousConfigure/fast + +#============================================================================= +# Target rules for targets named ContinuousBuild + +# Build rule for target. +ContinuousBuild: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousBuild +.PHONY : ContinuousBuild + +# fast build rule for target. +ContinuousBuild/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousBuild.dir/build.make CMakeFiles/ContinuousBuild.dir/build +.PHONY : ContinuousBuild/fast + +#============================================================================= +# Target rules for targets named ContinuousTest + +# Build rule for target. +ContinuousTest: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousTest +.PHONY : ContinuousTest + +# fast build rule for target. +ContinuousTest/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousTest.dir/build.make CMakeFiles/ContinuousTest.dir/build +.PHONY : ContinuousTest/fast + +#============================================================================= +# Target rules for targets named ContinuousCoverage + +# Build rule for target. +ContinuousCoverage: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousCoverage +.PHONY : ContinuousCoverage + +# fast build rule for target. +ContinuousCoverage/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousCoverage.dir/build.make CMakeFiles/ContinuousCoverage.dir/build +.PHONY : ContinuousCoverage/fast + +#============================================================================= +# Target rules for targets named ContinuousMemCheck + +# Build rule for target. +ContinuousMemCheck: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousMemCheck +.PHONY : ContinuousMemCheck + +# fast build rule for target. +ContinuousMemCheck/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousMemCheck.dir/build.make CMakeFiles/ContinuousMemCheck.dir/build +.PHONY : ContinuousMemCheck/fast + +#============================================================================= +# Target rules for targets named ContinuousSubmit + +# Build rule for target. +ContinuousSubmit: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousSubmit +.PHONY : ContinuousSubmit + +# fast build rule for target. +ContinuousSubmit/fast: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousSubmit.dir/build.make CMakeFiles/ContinuousSubmit.dir/build +.PHONY : ContinuousSubmit/fast + +#============================================================================= +# Target rules for targets named build_info + +# Build rule for target. +build_info: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 build_info +.PHONY : build_info + +# fast build rule for target. +build_info/fast: + $(MAKE) $(MAKESILENT) -f common/CMakeFiles/build_info.dir/build.make common/CMakeFiles/build_info.dir/build +.PHONY : build_info/fast + +#============================================================================= +# Target rules for targets named json-schema-to-grammar + +# Build rule for target. +json-schema-to-grammar: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 json-schema-to-grammar +.PHONY : json-schema-to-grammar + +# fast build rule for target. +json-schema-to-grammar/fast: + $(MAKE) $(MAKESILENT) -f common/CMakeFiles/json-schema-to-grammar.dir/build.make common/CMakeFiles/json-schema-to-grammar.dir/build +.PHONY : json-schema-to-grammar/fast + +#============================================================================= +# Target rules for targets named common + +# Build rule for target. +common: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 common +.PHONY : common + +# fast build rule for target. +common/fast: + $(MAKE) $(MAKESILENT) -f common/CMakeFiles/common.dir/build.make common/CMakeFiles/common.dir/build +.PHONY : common/fast + +#============================================================================= +# Target rules for targets named test-quantize-fns + +# Build rule for target. +test-quantize-fns: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-quantize-fns +.PHONY : test-quantize-fns + +# fast build rule for target. +test-quantize-fns/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-quantize-fns.dir/build.make tests/CMakeFiles/test-quantize-fns.dir/build +.PHONY : test-quantize-fns/fast + +#============================================================================= +# Target rules for targets named test-quantize-perf + +# Build rule for target. +test-quantize-perf: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-quantize-perf +.PHONY : test-quantize-perf + +# fast build rule for target. +test-quantize-perf/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-quantize-perf.dir/build.make tests/CMakeFiles/test-quantize-perf.dir/build +.PHONY : test-quantize-perf/fast + +#============================================================================= +# Target rules for targets named test-sampling + +# Build rule for target. +test-sampling: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-sampling +.PHONY : test-sampling + +# fast build rule for target. +test-sampling/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-sampling.dir/build.make tests/CMakeFiles/test-sampling.dir/build +.PHONY : test-sampling/fast + +#============================================================================= +# Target rules for targets named test-chat-template + +# Build rule for target. +test-chat-template: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-chat-template +.PHONY : test-chat-template + +# fast build rule for target. +test-chat-template/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-chat-template.dir/build.make tests/CMakeFiles/test-chat-template.dir/build +.PHONY : test-chat-template/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-0-llama + +# Build rule for target. +test-tokenizer-0-llama: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-0-llama +.PHONY : test-tokenizer-0-llama + +# fast build rule for target. +test-tokenizer-0-llama/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-0-llama.dir/build.make tests/CMakeFiles/test-tokenizer-0-llama.dir/build +.PHONY : test-tokenizer-0-llama/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-0-falcon + +# Build rule for target. +test-tokenizer-0-falcon: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-0-falcon +.PHONY : test-tokenizer-0-falcon + +# fast build rule for target. +test-tokenizer-0-falcon/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-0-falcon.dir/build.make tests/CMakeFiles/test-tokenizer-0-falcon.dir/build +.PHONY : test-tokenizer-0-falcon/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-1-llama + +# Build rule for target. +test-tokenizer-1-llama: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-llama +.PHONY : test-tokenizer-1-llama + +# fast build rule for target. +test-tokenizer-1-llama/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-llama.dir/build.make tests/CMakeFiles/test-tokenizer-1-llama.dir/build +.PHONY : test-tokenizer-1-llama/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-1-baichuan + +# Build rule for target. +test-tokenizer-1-baichuan: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-baichuan +.PHONY : test-tokenizer-1-baichuan + +# fast build rule for target. +test-tokenizer-1-baichuan/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-baichuan.dir/build.make tests/CMakeFiles/test-tokenizer-1-baichuan.dir/build +.PHONY : test-tokenizer-1-baichuan/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-1-falcon + +# Build rule for target. +test-tokenizer-1-falcon: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-falcon +.PHONY : test-tokenizer-1-falcon + +# fast build rule for target. +test-tokenizer-1-falcon/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-falcon.dir/build.make tests/CMakeFiles/test-tokenizer-1-falcon.dir/build +.PHONY : test-tokenizer-1-falcon/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-1-aquila + +# Build rule for target. +test-tokenizer-1-aquila: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-aquila +.PHONY : test-tokenizer-1-aquila + +# fast build rule for target. +test-tokenizer-1-aquila/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-aquila.dir/build.make tests/CMakeFiles/test-tokenizer-1-aquila.dir/build +.PHONY : test-tokenizer-1-aquila/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-1-mpt + +# Build rule for target. +test-tokenizer-1-mpt: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-mpt +.PHONY : test-tokenizer-1-mpt + +# fast build rule for target. +test-tokenizer-1-mpt/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-mpt.dir/build.make tests/CMakeFiles/test-tokenizer-1-mpt.dir/build +.PHONY : test-tokenizer-1-mpt/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-1-stablelm-3b-4e1t + +# Build rule for target. +test-tokenizer-1-stablelm-3b-4e1t: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-stablelm-3b-4e1t +.PHONY : test-tokenizer-1-stablelm-3b-4e1t + +# fast build rule for target. +test-tokenizer-1-stablelm-3b-4e1t/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-stablelm-3b-4e1t.dir/build.make tests/CMakeFiles/test-tokenizer-1-stablelm-3b-4e1t.dir/build +.PHONY : test-tokenizer-1-stablelm-3b-4e1t/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-1-gpt-neox + +# Build rule for target. +test-tokenizer-1-gpt-neox: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-gpt-neox +.PHONY : test-tokenizer-1-gpt-neox + +# fast build rule for target. +test-tokenizer-1-gpt-neox/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-gpt-neox.dir/build.make tests/CMakeFiles/test-tokenizer-1-gpt-neox.dir/build +.PHONY : test-tokenizer-1-gpt-neox/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-1-refact + +# Build rule for target. +test-tokenizer-1-refact: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-refact +.PHONY : test-tokenizer-1-refact + +# fast build rule for target. +test-tokenizer-1-refact/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-refact.dir/build.make tests/CMakeFiles/test-tokenizer-1-refact.dir/build +.PHONY : test-tokenizer-1-refact/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-1-starcoder + +# Build rule for target. +test-tokenizer-1-starcoder: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-starcoder +.PHONY : test-tokenizer-1-starcoder + +# fast build rule for target. +test-tokenizer-1-starcoder/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-starcoder.dir/build.make tests/CMakeFiles/test-tokenizer-1-starcoder.dir/build +.PHONY : test-tokenizer-1-starcoder/fast + +#============================================================================= +# Target rules for targets named test-tokenizer-1-gpt2 + +# Build rule for target. +test-tokenizer-1-gpt2: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-gpt2 +.PHONY : test-tokenizer-1-gpt2 + +# fast build rule for target. +test-tokenizer-1-gpt2/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-gpt2.dir/build.make tests/CMakeFiles/test-tokenizer-1-gpt2.dir/build +.PHONY : test-tokenizer-1-gpt2/fast + +#============================================================================= +# Target rules for targets named test-grammar-parser + +# Build rule for target. +test-grammar-parser: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-grammar-parser +.PHONY : test-grammar-parser + +# fast build rule for target. +test-grammar-parser/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-grammar-parser.dir/build.make tests/CMakeFiles/test-grammar-parser.dir/build +.PHONY : test-grammar-parser/fast + +#============================================================================= +# Target rules for targets named test-llama-grammar + +# Build rule for target. +test-llama-grammar: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-llama-grammar +.PHONY : test-llama-grammar + +# fast build rule for target. +test-llama-grammar/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-llama-grammar.dir/build.make tests/CMakeFiles/test-llama-grammar.dir/build +.PHONY : test-llama-grammar/fast + +#============================================================================= +# Target rules for targets named test-grammar-integration + +# Build rule for target. +test-grammar-integration: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-grammar-integration +.PHONY : test-grammar-integration + +# fast build rule for target. +test-grammar-integration/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-grammar-integration.dir/build.make tests/CMakeFiles/test-grammar-integration.dir/build +.PHONY : test-grammar-integration/fast + +#============================================================================= +# Target rules for targets named test-grad0 + +# Build rule for target. +test-grad0: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-grad0 +.PHONY : test-grad0 + +# fast build rule for target. +test-grad0/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-grad0.dir/build.make tests/CMakeFiles/test-grad0.dir/build +.PHONY : test-grad0/fast + +#============================================================================= +# Target rules for targets named test-backend-ops + +# Build rule for target. +test-backend-ops: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-backend-ops +.PHONY : test-backend-ops + +# fast build rule for target. +test-backend-ops/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-backend-ops.dir/build.make tests/CMakeFiles/test-backend-ops.dir/build +.PHONY : test-backend-ops/fast + +#============================================================================= +# Target rules for targets named test-rope + +# Build rule for target. +test-rope: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-rope +.PHONY : test-rope + +# fast build rule for target. +test-rope/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-rope.dir/build.make tests/CMakeFiles/test-rope.dir/build +.PHONY : test-rope/fast + +#============================================================================= +# Target rules for targets named test-model-load-cancel + +# Build rule for target. +test-model-load-cancel: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-model-load-cancel +.PHONY : test-model-load-cancel + +# fast build rule for target. +test-model-load-cancel/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-model-load-cancel.dir/build.make tests/CMakeFiles/test-model-load-cancel.dir/build +.PHONY : test-model-load-cancel/fast + +#============================================================================= +# Target rules for targets named test-autorelease + +# Build rule for target. +test-autorelease: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-autorelease +.PHONY : test-autorelease + +# fast build rule for target. +test-autorelease/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-autorelease.dir/build.make tests/CMakeFiles/test-autorelease.dir/build +.PHONY : test-autorelease/fast + +#============================================================================= +# Target rules for targets named test-json-schema-to-grammar + +# Build rule for target. +test-json-schema-to-grammar: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-json-schema-to-grammar +.PHONY : test-json-schema-to-grammar + +# fast build rule for target. +test-json-schema-to-grammar/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-json-schema-to-grammar.dir/build.make tests/CMakeFiles/test-json-schema-to-grammar.dir/build +.PHONY : test-json-schema-to-grammar/fast + +#============================================================================= +# Target rules for targets named test-c + +# Build rule for target. +test-c: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-c +.PHONY : test-c + +# fast build rule for target. +test-c/fast: + $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-c.dir/build.make tests/CMakeFiles/test-c.dir/build +.PHONY : test-c/fast + +#============================================================================= +# Target rules for targets named baby-llama + +# Build rule for target. +baby-llama: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 baby-llama +.PHONY : baby-llama + +# fast build rule for target. +baby-llama/fast: + $(MAKE) $(MAKESILENT) -f examples/baby-llama/CMakeFiles/baby-llama.dir/build.make examples/baby-llama/CMakeFiles/baby-llama.dir/build +.PHONY : baby-llama/fast + +#============================================================================= +# Target rules for targets named batched + +# Build rule for target. +batched: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 batched +.PHONY : batched + +# fast build rule for target. +batched/fast: + $(MAKE) $(MAKESILENT) -f examples/batched/CMakeFiles/batched.dir/build.make examples/batched/CMakeFiles/batched.dir/build +.PHONY : batched/fast + +#============================================================================= +# Target rules for targets named batched-bench + +# Build rule for target. +batched-bench: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 batched-bench +.PHONY : batched-bench + +# fast build rule for target. +batched-bench/fast: + $(MAKE) $(MAKESILENT) -f examples/batched-bench/CMakeFiles/batched-bench.dir/build.make examples/batched-bench/CMakeFiles/batched-bench.dir/build +.PHONY : batched-bench/fast + +#============================================================================= +# Target rules for targets named beam-search + +# Build rule for target. +beam-search: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 beam-search +.PHONY : beam-search + +# fast build rule for target. +beam-search/fast: + $(MAKE) $(MAKESILENT) -f examples/beam-search/CMakeFiles/beam-search.dir/build.make examples/beam-search/CMakeFiles/beam-search.dir/build +.PHONY : beam-search/fast + +#============================================================================= +# Target rules for targets named benchmark + +# Build rule for target. +benchmark: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 benchmark +.PHONY : benchmark + +# fast build rule for target. +benchmark/fast: + $(MAKE) $(MAKESILENT) -f examples/benchmark/CMakeFiles/benchmark.dir/build.make examples/benchmark/CMakeFiles/benchmark.dir/build +.PHONY : benchmark/fast + +#============================================================================= +# Target rules for targets named convert-llama2c-to-ggml + +# Build rule for target. +convert-llama2c-to-ggml: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 convert-llama2c-to-ggml +.PHONY : convert-llama2c-to-ggml + +# fast build rule for target. +convert-llama2c-to-ggml/fast: + $(MAKE) $(MAKESILENT) -f examples/convert-llama2c-to-ggml/CMakeFiles/convert-llama2c-to-ggml.dir/build.make examples/convert-llama2c-to-ggml/CMakeFiles/convert-llama2c-to-ggml.dir/build +.PHONY : convert-llama2c-to-ggml/fast + +#============================================================================= +# Target rules for targets named embedding + +# Build rule for target. +embedding: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 embedding +.PHONY : embedding + +# fast build rule for target. +embedding/fast: + $(MAKE) $(MAKESILENT) -f examples/embedding/CMakeFiles/embedding.dir/build.make examples/embedding/CMakeFiles/embedding.dir/build +.PHONY : embedding/fast + +#============================================================================= +# Target rules for targets named finetune + +# Build rule for target. +finetune: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 finetune +.PHONY : finetune + +# fast build rule for target. +finetune/fast: + $(MAKE) $(MAKESILENT) -f examples/finetune/CMakeFiles/finetune.dir/build.make examples/finetune/CMakeFiles/finetune.dir/build +.PHONY : finetune/fast + +#============================================================================= +# Target rules for targets named gritlm + +# Build rule for target. +gritlm: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 gritlm +.PHONY : gritlm + +# fast build rule for target. +gritlm/fast: + $(MAKE) $(MAKESILENT) -f examples/gritlm/CMakeFiles/gritlm.dir/build.make examples/gritlm/CMakeFiles/gritlm.dir/build +.PHONY : gritlm/fast + +#============================================================================= +# Target rules for targets named gguf-split + +# Build rule for target. +gguf-split: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 gguf-split +.PHONY : gguf-split + +# fast build rule for target. +gguf-split/fast: + $(MAKE) $(MAKESILENT) -f examples/gguf-split/CMakeFiles/gguf-split.dir/build.make examples/gguf-split/CMakeFiles/gguf-split.dir/build +.PHONY : gguf-split/fast + +#============================================================================= +# Target rules for targets named infill + +# Build rule for target. +infill: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 infill +.PHONY : infill + +# fast build rule for target. +infill/fast: + $(MAKE) $(MAKESILENT) -f examples/infill/CMakeFiles/infill.dir/build.make examples/infill/CMakeFiles/infill.dir/build +.PHONY : infill/fast + +#============================================================================= +# Target rules for targets named llama-bench + +# Build rule for target. +llama-bench: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 llama-bench +.PHONY : llama-bench + +# fast build rule for target. +llama-bench/fast: + $(MAKE) $(MAKESILENT) -f examples/llama-bench/CMakeFiles/llama-bench.dir/build.make examples/llama-bench/CMakeFiles/llama-bench.dir/build +.PHONY : llama-bench/fast + +#============================================================================= +# Target rules for targets named llava + +# Build rule for target. +llava: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 llava +.PHONY : llava + +# fast build rule for target. +llava/fast: + $(MAKE) $(MAKESILENT) -f examples/llava/CMakeFiles/llava.dir/build.make examples/llava/CMakeFiles/llava.dir/build +.PHONY : llava/fast + +#============================================================================= +# Target rules for targets named llava_static + +# Build rule for target. +llava_static: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 llava_static +.PHONY : llava_static + +# fast build rule for target. +llava_static/fast: + $(MAKE) $(MAKESILENT) -f examples/llava/CMakeFiles/llava_static.dir/build.make examples/llava/CMakeFiles/llava_static.dir/build +.PHONY : llava_static/fast + +#============================================================================= +# Target rules for targets named llava-cli + +# Build rule for target. +llava-cli: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 llava-cli +.PHONY : llava-cli + +# fast build rule for target. +llava-cli/fast: + $(MAKE) $(MAKESILENT) -f examples/llava/CMakeFiles/llava-cli.dir/build.make examples/llava/CMakeFiles/llava-cli.dir/build +.PHONY : llava-cli/fast + +#============================================================================= +# Target rules for targets named main + +# Build rule for target. +main: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 main +.PHONY : main + +# fast build rule for target. +main/fast: + $(MAKE) $(MAKESILENT) -f examples/main/CMakeFiles/main.dir/build.make examples/main/CMakeFiles/main.dir/build +.PHONY : main/fast + +#============================================================================= +# Target rules for targets named tokenize + +# Build rule for target. +tokenize: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 tokenize +.PHONY : tokenize + +# fast build rule for target. +tokenize/fast: + $(MAKE) $(MAKESILENT) -f examples/tokenize/CMakeFiles/tokenize.dir/build.make examples/tokenize/CMakeFiles/tokenize.dir/build +.PHONY : tokenize/fast + +#============================================================================= +# Target rules for targets named parallel + +# Build rule for target. +parallel: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 parallel +.PHONY : parallel + +# fast build rule for target. +parallel/fast: + $(MAKE) $(MAKESILENT) -f examples/parallel/CMakeFiles/parallel.dir/build.make examples/parallel/CMakeFiles/parallel.dir/build +.PHONY : parallel/fast + +#============================================================================= +# Target rules for targets named perplexity + +# Build rule for target. +perplexity: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 perplexity +.PHONY : perplexity + +# fast build rule for target. +perplexity/fast: + $(MAKE) $(MAKESILENT) -f examples/perplexity/CMakeFiles/perplexity.dir/build.make examples/perplexity/CMakeFiles/perplexity.dir/build +.PHONY : perplexity/fast + +#============================================================================= +# Target rules for targets named quantize + +# Build rule for target. +quantize: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 quantize +.PHONY : quantize + +# fast build rule for target. +quantize/fast: + $(MAKE) $(MAKESILENT) -f examples/quantize/CMakeFiles/quantize.dir/build.make examples/quantize/CMakeFiles/quantize.dir/build +.PHONY : quantize/fast + +#============================================================================= +# Target rules for targets named quantize-stats + +# Build rule for target. +quantize-stats: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 quantize-stats +.PHONY : quantize-stats + +# fast build rule for target. +quantize-stats/fast: + $(MAKE) $(MAKESILENT) -f examples/quantize-stats/CMakeFiles/quantize-stats.dir/build.make examples/quantize-stats/CMakeFiles/quantize-stats.dir/build +.PHONY : quantize-stats/fast + +#============================================================================= +# Target rules for targets named retrieval + +# Build rule for target. +retrieval: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 retrieval +.PHONY : retrieval + +# fast build rule for target. +retrieval/fast: + $(MAKE) $(MAKESILENT) -f examples/retrieval/CMakeFiles/retrieval.dir/build.make examples/retrieval/CMakeFiles/retrieval.dir/build +.PHONY : retrieval/fast + +#============================================================================= +# Target rules for targets named save-load-state + +# Build rule for target. +save-load-state: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 save-load-state +.PHONY : save-load-state + +# fast build rule for target. +save-load-state/fast: + $(MAKE) $(MAKESILENT) -f examples/save-load-state/CMakeFiles/save-load-state.dir/build.make examples/save-load-state/CMakeFiles/save-load-state.dir/build +.PHONY : save-load-state/fast + +#============================================================================= +# Target rules for targets named simple + +# Build rule for target. +simple: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 simple +.PHONY : simple + +# fast build rule for target. +simple/fast: + $(MAKE) $(MAKESILENT) -f examples/simple/CMakeFiles/simple.dir/build.make examples/simple/CMakeFiles/simple.dir/build +.PHONY : simple/fast + +#============================================================================= +# Target rules for targets named passkey + +# Build rule for target. +passkey: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 passkey +.PHONY : passkey + +# fast build rule for target. +passkey/fast: + $(MAKE) $(MAKESILENT) -f examples/passkey/CMakeFiles/passkey.dir/build.make examples/passkey/CMakeFiles/passkey.dir/build +.PHONY : passkey/fast + +#============================================================================= +# Target rules for targets named speculative + +# Build rule for target. +speculative: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 speculative +.PHONY : speculative + +# fast build rule for target. +speculative/fast: + $(MAKE) $(MAKESILENT) -f examples/speculative/CMakeFiles/speculative.dir/build.make examples/speculative/CMakeFiles/speculative.dir/build +.PHONY : speculative/fast + +#============================================================================= +# Target rules for targets named lookahead + +# Build rule for target. +lookahead: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 lookahead +.PHONY : lookahead + +# fast build rule for target. +lookahead/fast: + $(MAKE) $(MAKESILENT) -f examples/lookahead/CMakeFiles/lookahead.dir/build.make examples/lookahead/CMakeFiles/lookahead.dir/build +.PHONY : lookahead/fast + +#============================================================================= +# Target rules for targets named lookup + +# Build rule for target. +lookup: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 lookup +.PHONY : lookup + +# fast build rule for target. +lookup/fast: + $(MAKE) $(MAKESILENT) -f examples/lookup/CMakeFiles/lookup.dir/build.make examples/lookup/CMakeFiles/lookup.dir/build +.PHONY : lookup/fast + +#============================================================================= +# Target rules for targets named lookup-create + +# Build rule for target. +lookup-create: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 lookup-create +.PHONY : lookup-create + +# fast build rule for target. +lookup-create/fast: + $(MAKE) $(MAKESILENT) -f examples/lookup/CMakeFiles/lookup-create.dir/build.make examples/lookup/CMakeFiles/lookup-create.dir/build +.PHONY : lookup-create/fast + +#============================================================================= +# Target rules for targets named lookup-merge + +# Build rule for target. +lookup-merge: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 lookup-merge +.PHONY : lookup-merge + +# fast build rule for target. +lookup-merge/fast: + $(MAKE) $(MAKESILENT) -f examples/lookup/CMakeFiles/lookup-merge.dir/build.make examples/lookup/CMakeFiles/lookup-merge.dir/build +.PHONY : lookup-merge/fast + +#============================================================================= +# Target rules for targets named lookup-stats + +# Build rule for target. +lookup-stats: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 lookup-stats +.PHONY : lookup-stats + +# fast build rule for target. +lookup-stats/fast: + $(MAKE) $(MAKESILENT) -f examples/lookup/CMakeFiles/lookup-stats.dir/build.make examples/lookup/CMakeFiles/lookup-stats.dir/build +.PHONY : lookup-stats/fast + +#============================================================================= +# Target rules for targets named gguf + +# Build rule for target. +gguf: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 gguf +.PHONY : gguf + +# fast build rule for target. +gguf/fast: + $(MAKE) $(MAKESILENT) -f examples/gguf/CMakeFiles/gguf.dir/build.make examples/gguf/CMakeFiles/gguf.dir/build +.PHONY : gguf/fast + +#============================================================================= +# Target rules for targets named train-text-from-scratch + +# Build rule for target. +train-text-from-scratch: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 train-text-from-scratch +.PHONY : train-text-from-scratch + +# fast build rule for target. +train-text-from-scratch/fast: + $(MAKE) $(MAKESILENT) -f examples/train-text-from-scratch/CMakeFiles/train-text-from-scratch.dir/build.make examples/train-text-from-scratch/CMakeFiles/train-text-from-scratch.dir/build +.PHONY : train-text-from-scratch/fast + +#============================================================================= +# Target rules for targets named imatrix + +# Build rule for target. +imatrix: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 imatrix +.PHONY : imatrix + +# fast build rule for target. +imatrix/fast: + $(MAKE) $(MAKESILENT) -f examples/imatrix/CMakeFiles/imatrix.dir/build.make examples/imatrix/CMakeFiles/imatrix.dir/build +.PHONY : imatrix/fast + +#============================================================================= +# Target rules for targets named server + +# Build rule for target. +server: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 server +.PHONY : server + +# fast build rule for target. +server/fast: + $(MAKE) $(MAKESILENT) -f examples/server/CMakeFiles/server.dir/build.make examples/server/CMakeFiles/server.dir/build +.PHONY : server/fast + +#============================================================================= +# Target rules for targets named export-lora + +# Build rule for target. +export-lora: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 export-lora +.PHONY : export-lora + +# fast build rule for target. +export-lora/fast: + $(MAKE) $(MAKESILENT) -f examples/export-lora/CMakeFiles/export-lora.dir/build.make examples/export-lora/CMakeFiles/export-lora.dir/build +.PHONY : export-lora/fast + +#============================================================================= +# Target rules for targets named vdot + +# Build rule for target. +vdot: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 vdot +.PHONY : vdot + +# fast build rule for target. +vdot/fast: + $(MAKE) $(MAKESILENT) -f pocs/vdot/CMakeFiles/vdot.dir/build.make pocs/vdot/CMakeFiles/vdot.dir/build +.PHONY : vdot/fast + +#============================================================================= +# Target rules for targets named q8dot + +# Build rule for target. +q8dot: cmake_check_build_system + $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 q8dot +.PHONY : q8dot + +# fast build rule for target. +q8dot/fast: + $(MAKE) $(MAKESILENT) -f pocs/vdot/CMakeFiles/q8dot.dir/build.make pocs/vdot/CMakeFiles/q8dot.dir/build +.PHONY : q8dot/fast + +ggml-alloc.o: ggml-alloc.c.o +.PHONY : ggml-alloc.o + +# target to build an object file +ggml-alloc.c.o: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-alloc.c.o +.PHONY : ggml-alloc.c.o + +ggml-alloc.i: ggml-alloc.c.i +.PHONY : ggml-alloc.i + +# target to preprocess a source file +ggml-alloc.c.i: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-alloc.c.i +.PHONY : ggml-alloc.c.i + +ggml-alloc.s: ggml-alloc.c.s +.PHONY : ggml-alloc.s + +# target to generate assembly for a file +ggml-alloc.c.s: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-alloc.c.s +.PHONY : ggml-alloc.c.s + +ggml-backend.o: ggml-backend.c.o +.PHONY : ggml-backend.o + +# target to build an object file +ggml-backend.c.o: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-backend.c.o +.PHONY : ggml-backend.c.o + +ggml-backend.i: ggml-backend.c.i +.PHONY : ggml-backend.i + +# target to preprocess a source file +ggml-backend.c.i: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-backend.c.i +.PHONY : ggml-backend.c.i + +ggml-backend.s: ggml-backend.c.s +.PHONY : ggml-backend.s + +# target to generate assembly for a file +ggml-backend.c.s: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-backend.c.s +.PHONY : ggml-backend.c.s -tests/test-backend-ops: tests/test-backend-ops.cpp ggml.o $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +ggml-quants.o: ggml-quants.c.o +.PHONY : ggml-quants.o + +# target to build an object file +ggml-quants.c.o: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-quants.c.o +.PHONY : ggml-quants.c.o -tests/test-model-load-cancel: tests/test-model-load-cancel.cpp ggml.o llama.o tests/get-model.cpp $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +ggml-quants.i: ggml-quants.c.i +.PHONY : ggml-quants.i + +# target to preprocess a source file +ggml-quants.c.i: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-quants.c.i +.PHONY : ggml-quants.c.i -tests/test-autorelease: tests/test-autorelease.cpp ggml.o llama.o tests/get-model.cpp $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +ggml-quants.s: ggml-quants.c.s +.PHONY : ggml-quants.s + +# target to generate assembly for a file +ggml-quants.c.s: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-quants.c.s +.PHONY : ggml-quants.c.s + +ggml.o: ggml.c.o +.PHONY : ggml.o + +# target to build an object file +ggml.c.o: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml.c.o +.PHONY : ggml.c.o + +ggml.i: ggml.c.i +.PHONY : ggml.i + +# target to preprocess a source file +ggml.c.i: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml.c.i +.PHONY : ggml.c.i + +ggml.s: ggml.c.s +.PHONY : ggml.s + +# target to generate assembly for a file +ggml.c.s: + $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml.c.s +.PHONY : ggml.c.s + +llama.o: llama.cpp.o +.PHONY : llama.o + +# target to build an object file +llama.cpp.o: + $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/llama.cpp.o +.PHONY : llama.cpp.o + +llama.i: llama.cpp.i +.PHONY : llama.i + +# target to preprocess a source file +llama.cpp.i: + $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/llama.cpp.i +.PHONY : llama.cpp.i + +llama.s: llama.cpp.s +.PHONY : llama.s + +# target to generate assembly for a file +llama.cpp.s: + $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/llama.cpp.s +.PHONY : llama.cpp.s + +unicode-data.o: unicode-data.cpp.o +.PHONY : unicode-data.o + +# target to build an object file +unicode-data.cpp.o: + $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode-data.cpp.o +.PHONY : unicode-data.cpp.o + +unicode-data.i: unicode-data.cpp.i +.PHONY : unicode-data.i + +# target to preprocess a source file +unicode-data.cpp.i: + $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode-data.cpp.i +.PHONY : unicode-data.cpp.i + +unicode-data.s: unicode-data.cpp.s +.PHONY : unicode-data.s + +# target to generate assembly for a file +unicode-data.cpp.s: + $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode-data.cpp.s +.PHONY : unicode-data.cpp.s + +unicode.o: unicode.cpp.o +.PHONY : unicode.o + +# target to build an object file +unicode.cpp.o: + $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode.cpp.o +.PHONY : unicode.cpp.o + +unicode.i: unicode.cpp.i +.PHONY : unicode.i + +# target to preprocess a source file +unicode.cpp.i: + $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode.cpp.i +.PHONY : unicode.cpp.i + +unicode.s: unicode.cpp.s +.PHONY : unicode.s + +# target to generate assembly for a file +unicode.cpp.s: + $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode.cpp.s +.PHONY : unicode.cpp.s + +# Help Target +help: + @echo "The following are some of the valid targets for this Makefile:" + @echo "... all (the default if no target is provided)" + @echo "... clean" + @echo "... depend" + @echo "... edit_cache" + @echo "... install" + @echo "... install/local" + @echo "... install/strip" + @echo "... list_install_components" + @echo "... rebuild_cache" + @echo "... test" + @echo "... Continuous" + @echo "... ContinuousBuild" + @echo "... ContinuousConfigure" + @echo "... ContinuousCoverage" + @echo "... ContinuousMemCheck" + @echo "... ContinuousStart" + @echo "... ContinuousSubmit" + @echo "... ContinuousTest" + @echo "... ContinuousUpdate" + @echo "... Experimental" + @echo "... ExperimentalBuild" + @echo "... ExperimentalConfigure" + @echo "... ExperimentalCoverage" + @echo "... ExperimentalMemCheck" + @echo "... ExperimentalStart" + @echo "... ExperimentalSubmit" + @echo "... ExperimentalTest" + @echo "... ExperimentalUpdate" + @echo "... Nightly" + @echo "... NightlyBuild" + @echo "... NightlyConfigure" + @echo "... NightlyCoverage" + @echo "... NightlyMemCheck" + @echo "... NightlyMemoryCheck" + @echo "... NightlyStart" + @echo "... NightlySubmit" + @echo "... NightlyTest" + @echo "... NightlyUpdate" + @echo "... baby-llama" + @echo "... batched" + @echo "... batched-bench" + @echo "... beam-search" + @echo "... benchmark" + @echo "... build_info" + @echo "... common" + @echo "... convert-llama2c-to-ggml" + @echo "... embedding" + @echo "... export-lora" + @echo "... finetune" + @echo "... ggml" + @echo "... ggml_static" + @echo "... gguf" + @echo "... gguf-split" + @echo "... gritlm" + @echo "... imatrix" + @echo "... infill" + @echo "... json-schema-to-grammar" + @echo "... llama" + @echo "... llama-bench" + @echo "... llava" + @echo "... llava-cli" + @echo "... llava_static" + @echo "... lookahead" + @echo "... lookup" + @echo "... lookup-create" + @echo "... lookup-merge" + @echo "... lookup-stats" + @echo "... main" + @echo "... parallel" + @echo "... passkey" + @echo "... perplexity" + @echo "... q8dot" + @echo "... quantize" + @echo "... quantize-stats" + @echo "... retrieval" + @echo "... save-load-state" + @echo "... server" + @echo "... simple" + @echo "... speculative" + @echo "... test-autorelease" + @echo "... test-backend-ops" + @echo "... test-c" + @echo "... test-chat-template" + @echo "... test-grad0" + @echo "... test-grammar-integration" + @echo "... test-grammar-parser" + @echo "... test-json-schema-to-grammar" + @echo "... test-llama-grammar" + @echo "... test-model-load-cancel" + @echo "... test-quantize-fns" + @echo "... test-quantize-perf" + @echo "... test-rope" + @echo "... test-sampling" + @echo "... test-tokenizer-0-falcon" + @echo "... test-tokenizer-0-llama" + @echo "... test-tokenizer-1-aquila" + @echo "... test-tokenizer-1-baichuan" + @echo "... test-tokenizer-1-falcon" + @echo "... test-tokenizer-1-gpt-neox" + @echo "... test-tokenizer-1-gpt2" + @echo "... test-tokenizer-1-llama" + @echo "... test-tokenizer-1-mpt" + @echo "... test-tokenizer-1-refact" + @echo "... test-tokenizer-1-stablelm-3b-4e1t" + @echo "... test-tokenizer-1-starcoder" + @echo "... tokenize" + @echo "... train-text-from-scratch" + @echo "... vdot" + @echo "... ggml-alloc.o" + @echo "... ggml-alloc.i" + @echo "... ggml-alloc.s" + @echo "... ggml-backend.o" + @echo "... ggml-backend.i" + @echo "... ggml-backend.s" + @echo "... ggml-quants.o" + @echo "... ggml-quants.i" + @echo "... ggml-quants.s" + @echo "... ggml.o" + @echo "... ggml.i" + @echo "... ggml.s" + @echo "... llama.o" + @echo "... llama.i" + @echo "... llama.s" + @echo "... unicode-data.o" + @echo "... unicode-data.i" + @echo "... unicode-data.s" + @echo "... unicode.o" + @echo "... unicode.i" + @echo "... unicode.s" +.PHONY : help + + + +#============================================================================= +# Special targets to cleanup operation of make. + +# Special rule to run CMake to check the build system integrity. +# No rule that depends on this can have commands that come from listfiles +# because they might be regenerated. +cmake_check_build_system: + $(CMAKE_COMMAND) -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 0 +.PHONY : cmake_check_build_system -tests/test-chat-template: tests/test-chat-template.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) - $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) - $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 37af6328a1705..5f3b565be4b2c 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -53,7 +53,8 @@ def __init__(self, dir_model: Path, ftype: int, fname_out: Path, is_big_endian: self.num_parts = Model.count_model_parts(self.dir_model, ".safetensors" if self.is_safetensors else ".bin") self.part_names = self._get_part_names() self.hparams = Model.load_hparams(self.dir_model) - self.gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[self.model_arch], endianess=self.endianess, use_temp_file=False) + self.gguf_writer = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[self.model_arch], endianess=self.endianess, + use_temp_file=False) self.block_count = self.find_hparam(["n_layers", "num_hidden_layers", "n_layer"]) @property @@ -80,7 +81,8 @@ def get_tensors(self) -> Iterator[tuple[str, Tensor]]: from safetensors import safe_open ctx = cast(ContextManager[Any], safe_open(self.dir_model / part_name, framework="pt", device="cpu")) else: - ctx = contextlib.nullcontext(torch.load(str(self.dir_model / part_name), map_location="cpu", mmap=True, weights_only=True)) + ctx = contextlib.nullcontext( + torch.load(str(self.dir_model / part_name), map_location="cpu", mmap=True, weights_only=True)) with ctx as model_part: for name in model_part.keys(): @@ -117,7 +119,8 @@ def set_gguf_parameters(self): if (f_rms_eps := self.hparams.get("rms_norm_eps")) is not None: self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps) print(f"gguf: rms norm epsilon = {f_rms_eps}") - if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], optional=True)) is not None: + if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], + optional=True)) is not None: self.gguf_writer.add_layer_norm_eps(f_norm_eps) print(f"gguf: layer norm epsilon = {f_norm_eps}") if (n_experts := self.hparams.get("num_local_experts")) is not None: @@ -205,6 +208,7 @@ def func(modelcls: type[Model]): for name in names: cls._model_classes[name] = modelcls return modelcls + return func @classmethod @@ -286,7 +290,7 @@ def _set_vocab_qwen(self): # for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combined added_vocab = tokenizer.special_tokens - reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in (vocab | added_vocab).items()} + reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in (vocab | added_vocab).items()} for i in range(vocab_size): if i not in reverse_vocab: @@ -771,8 +775,8 @@ def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | Non return ( weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) - .swapaxes(1, 2) - .reshape(weights.shape) + .swapaxes(1, 2) + .reshape(weights.shape) ) def _reverse_hf_permute_part( @@ -923,8 +927,8 @@ def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | Non return ( weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) - .swapaxes(1, 2) - .reshape(weights.shape) + .swapaxes(1, 2) + .reshape(weights.shape) ) @@ -1201,9 +1205,11 @@ def set_gguf_parameters(self): self.gguf_writer.add_block_count(block_count) self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"]) rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"]) - self.gguf_writer.add_rope_dimension_count(int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))) + self.gguf_writer.add_rope_dimension_count( + int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))) self.gguf_writer.add_head_count(hparams["num_attention_heads"]) - self.gguf_writer.add_parallel_residual(hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True) + self.gguf_writer.add_parallel_residual( + hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True) self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_eps", "norm_eps"])) @@ -1213,7 +1219,7 @@ class LlamaModel(Model): def set_vocab(self): try: - self. _set_vocab_sentencepiece() + self._set_vocab_sentencepiece() except FileNotFoundError: self._set_vocab_llama_hf() @@ -1450,8 +1456,8 @@ def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | Non return ( weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) - .swapaxes(1, 2) - .reshape(weights.shape) + .swapaxes(1, 2) + .reshape(weights.shape) ) def write_tensors(self): @@ -1612,7 +1618,8 @@ def write_tensors(self): for name, data_torch in self.get_tensors(): # we don't need these - if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq", ".attn.bias", ".attn.masked_bias")): + if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq", + ".attn.bias", ".attn.masked_bias")): continue if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight", ".c_proj.weight")): @@ -1995,7 +2002,8 @@ def write_tensors(self): bid = re.findall(qkv_pattern, name)[0] qkv = data_torch qkv = rearrange(qkv.T, " o (g n i) ->o g n i", g=num_groups, n=q_per_kv + 2, i=head_dim) - q, k, v = qkv[..., : q_per_kv, :], qkv[..., q_per_kv: q_per_kv + 1, :], qkv[..., q_per_kv + 1: q_per_kv + 2, :] + q, k, v = qkv[..., : q_per_kv, :], qkv[..., q_per_kv: q_per_kv + 1, :], qkv[..., + q_per_kv + 1: q_per_kv + 2, :] # The model weights of q and k equire additional reshape. q = self._hf_permute_qk(rearrange(q, " o g n i -> o (g n i)").T, num_heads, num_heads) k = self._hf_permute_qk(rearrange(k, " o g n i -> o (g n i)").T, num_heads, num_kv_heads) @@ -2061,6 +2069,7 @@ def phantom(tok, typ): if tok.startswith(b"##"): return tok[2:] return b"\xe2\x96\x81" + tok + tokens = tuple(phantom(t, y) for t, y in zip(tokens, toktypes)) # set up bos and eos tokens (cls and sep) @@ -2153,6 +2162,38 @@ def get_tensors(self): yield name, data +@Model.register("JinaBertModel") +class JinaBertModel(BertModel): + model_arch = gguf.MODEL_ARCH.JINA_BERT + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + print(f'hparams {self.hparams}') + + assert self.hparams["position_embedding_type"] == "alibi" + + # def __init__(self, *args, **kwargs): + # super().__init__(*args, **kwargs) + # + # assert self.hparams["position_embedding_type"] == "alibi" + # + # # GeGLU activation + # assert self.hparams["feed_forward_type"] == "geglu" + # + # def get_tensors(self): + # assert self.vocab_size is not None + # for name, data in super().get_tensors(): + # print(f'get_tensors: {name} {data.shape}') + # # Nomic Embed's token embeddings tensor is padded, but llama.cpp wants tensor sizes to match exactly. + # if name == 'embeddings.word_embeddings.weight' and data.shape[1] != self.vocab_size: + # rounded_vocab_size = (self.vocab_size + 63) // 64 * 64 + # assert data.shape == (rounded_vocab_size, self.hparams["hidden_size"]) + # data = data[:self.vocab_size, :] + # yield name, data + + + @Model.register("GemmaForCausalLM") class GemmaModel(Model): model_arch = gguf.MODEL_ARCH.GEMMA @@ -2170,7 +2211,8 @@ def set_gguf_parameters(self): self.gguf_writer.add_block_count(block_count) self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"]) self.gguf_writer.add_head_count(hparams["num_attention_heads"]) - self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"]) + self.gguf_writer.add_head_count_kv( + self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"]) self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"]) self.gguf_writer.add_key_length(hparams["head_dim"]) self.gguf_writer.add_value_length(hparams["head_dim"]) @@ -2255,7 +2297,7 @@ def set_vocab(self): def set_gguf_parameters(self): d_model = self.find_hparam(["hidden_size", "d_model"]) - d_conv = self.find_hparam(["conv_kernel", "d_conv"], optional=True) or 4 + d_conv = self.find_hparam(["conv_kernel", "d_conv"], optional=True) or 4 d_inner = self.find_hparam(["intermediate_size", "d_inner"], optional=True) or 2 * d_model d_state = self.find_hparam(["state_size", "d_state"], optional=True) or 16 # ceiling division @@ -2268,10 +2310,10 @@ def set_gguf_parameters(self): assert d_inner == 2 * d_model self.gguf_writer.add_name(self.dir_model.name) - self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the default + self.gguf_writer.add_context_length(2 ** 20) # arbitrary value; for those who use the default self.gguf_writer.add_embedding_length(d_model) - self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loading - self.gguf_writer.add_head_count(0) # unused, but seemingly required when loading + self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loading + self.gguf_writer.add_head_count(0) # unused, but seemingly required when loading self.gguf_writer.add_block_count(self.hparams["n_layer"]) self.gguf_writer.add_ssm_conv_kernel(d_conv) self.gguf_writer.add_ssm_inner_size(d_inner) @@ -2286,7 +2328,7 @@ def write_tensors(self): tok_embd = None tok_embd_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.TOKEN_EMBD] + ".weight" - output_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.OUTPUT] + ".weight" + output_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.OUTPUT] + ".weight" for name, data_torch in self.get_tensors(): old_dtype = data_torch.dtype @@ -2327,7 +2369,8 @@ def write_tensors(self): data = data.astype(np.float32) # if f16 desired, convert big float32 2-dim weight tensors to float16 - if self.ftype == 1 and data_dtype == np.float32 and new_name.removesuffix(".weight").endswith((".ssm_in", ".ssm_out", "token_embd", "output")) and n_dims == 2: + if self.ftype == 1 and data_dtype == np.float32 and new_name.removesuffix(".weight").endswith( + (".ssm_in", ".ssm_out", "token_embd", "output")) and n_dims == 2: data = data.astype(np.float16) print(f"{new_name}, n_dims = {n_dims}, {old_dtype} --> {data.dtype}") @@ -2420,6 +2463,7 @@ def main() -> None: hparams = Model.load_hparams(dir_model) with torch.inference_mode(): + print(hparams["architectures"]) model_class = Model.from_model_architecture(hparams["architectures"][0]) model_instance = model_class(dir_model, ftype_map[args.outtype], fname_out, args.bigendian) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index a6454a10e20b9..cfb78327981a3 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -111,6 +111,7 @@ class MODEL_ARCH(IntEnum): REFACT = auto() BERT = auto() NOMIC_BERT = auto() + JINA_BERT = auto() BLOOM = auto() STABLELM = auto() QWEN = auto() @@ -180,6 +181,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.REFACT: "refact", MODEL_ARCH.BERT: "bert", MODEL_ARCH.NOMIC_BERT: "nomic-bert", + MODEL_ARCH.JINA_BERT: "jina-bert", MODEL_ARCH.BLOOM: "bloom", MODEL_ARCH.STABLELM: "stablelm", MODEL_ARCH.QWEN: "qwen", @@ -357,6 +359,20 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_UP, MODEL_TENSOR.LAYER_OUT_NORM, ], + MODEL_ARCH.JINA_BERT: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.TOKEN_EMBD_NORM, + MODEL_TENSOR.TOKEN_TYPES, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_OUT_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.LAYER_OUT_NORM, + ], MODEL_ARCH.MPT: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 4f02d298e13de..7c7abab08184d 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -217,6 +217,9 @@ class TensorNameMap: "model.layers.{bid}.mlp.up_proj", # llama-hf refact "layers.{bid}.feed_forward.w3", # llama-pth "encoder.layer.{bid}.intermediate.dense", # bert + "encoder.layer.{bid}.mlp.gated_layers", # jina-bert + "encoder.layer.{bid}.mlp.layernorm", # jina-bert + "encoder.layer.{bid}.mlp.wo", # jina-bert "transformer.h.{bid}.mlp.fc_in", # gpt-j "language_model.encoder.layers.{bid}.mlp.dense_h_to_4h", # persimmon "model.layers.{bid}.mlp.dense_h_to_4h", # persimmon diff --git a/llama.cpp b/llama.cpp index 6a090d1bbc24c..e9f430fa704a3 100644 --- a/llama.cpp +++ b/llama.cpp @@ -205,6 +205,7 @@ enum llm_arch { LLM_ARCH_REFACT, LLM_ARCH_BERT, LLM_ARCH_NOMIC_BERT, + LLM_ARCH_JINA_BERT, LLM_ARCH_BLOOM, LLM_ARCH_STABLELM, LLM_ARCH_QWEN, @@ -237,6 +238,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_REFACT, "refact" }, { LLM_ARCH_BERT, "bert" }, { LLM_ARCH_NOMIC_BERT, "nomic-bert" }, + { LLM_ARCH_JINA_BERT, "jina-bert" }, { LLM_ARCH_BLOOM, "bloom" }, { LLM_ARCH_STABLELM, "stablelm" }, { LLM_ARCH_QWEN, "qwen" }, @@ -665,6 +667,22 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, + { + LLM_ARCH_JINA_BERT, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" }, + { LLM_TENSOR_TOKEN_TYPES, "token_types" }, + { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + }, + }, { LLM_ARCH_BLOOM, { @@ -3770,6 +3788,18 @@ static void llm_load_hparams( model.type = e_model::MODEL_335M; break; // bge-large } } break; + case LLM_ARCH_JINA_BERT: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); + ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); + ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type); + ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type); + + switch (hparams.n_layer) { + case 4: model.type = e_model::MODEL_33M; break; // jina-embeddings-small + case 12: model.type = e_model::MODEL_137M; break; // jina-embeddings-base + } + } break; case LLM_ARCH_NOMIC_BERT: { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); @@ -4488,6 +4518,7 @@ static bool llm_load_tensors( model.layers.resize(n_layer); const auto tn = LLM_TN(model.arch); + //std::printf("JOAN HERE ARCH %i", model.arch); switch (model.arch) { case LLM_ARCH_LLAMA: case LLM_ARCH_REFACT: @@ -4782,6 +4813,7 @@ static bool llm_load_tensors( } } break; case LLM_ARCH_BERT: + case LLM_ARCH_JINA_BERT: case LLM_ARCH_NOMIC_BERT: { model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); @@ -4799,7 +4831,7 @@ static bool llm_load_tensors( auto & layer = model.layers[i]; - if (model.arch == LLM_ARCH_BERT) { + if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT) { layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}); @@ -4820,7 +4852,7 @@ static bool llm_load_tensors( layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); - if (model.arch == LLM_ARCH_BERT) { + if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT) { layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); @@ -14558,6 +14590,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_REFACT: case LLM_ARCH_BLOOM: case LLM_ARCH_MAMBA: + case LLM_ARCH_JINA_BERT: return LLAMA_ROPE_TYPE_NONE; // use what we call a normal RoPE, operating on pairs of consecutive head values From 747d17a62cf8567cc036d9add9c8d7be8a4a55b1 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Fri, 12 Apr 2024 12:47:48 +0200 Subject: [PATCH 02/36] feat: create tensors for Jina architecture --- convert-hf-to-gguf.py | 34 +++------------------ gguf-py/gguf/constants.py | 4 +-- gguf-py/gguf/tensor_mapping.py | 6 ++-- llama.cpp | 55 ++++++++++++++++++++++++++++++++-- 4 files changed, 61 insertions(+), 38 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 5f3b565be4b2c..3285a7ef82881 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -77,6 +77,7 @@ def get_tensors(self) -> Iterator[tuple[str, Tensor]]: for part_name in self.part_names: print(f"gguf: loading model part '{part_name}'") ctx: ContextManager[Any] + if self.is_safetensors: from safetensors import safe_open ctx = cast(ContextManager[Any], safe_open(self.dir_model / part_name, framework="pt", device="cpu")) @@ -91,6 +92,7 @@ def get_tensors(self) -> Iterator[tuple[str, Tensor]]: def set_gguf_parameters(self): self.gguf_writer.add_name(self.dir_model.name) + print(f'self.block_count {self.block_count}') self.gguf_writer.add_block_count(self.block_count) if (n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx"], optional=True)) is not None: @@ -136,6 +138,7 @@ def set_gguf_parameters(self): def write_tensors(self): block_count = self.hparams.get("n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer"))) tensor_map = gguf.get_tensor_name_map(self.model_arch, block_count) + print(f'Block_count {block_count} with tensor_map {tensor_map}') for name, data_torch in self.get_tensors(): # we don't need these if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq")): @@ -2096,6 +2099,7 @@ def write_tensors(self): # map tensor names new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias")) + if new_name is None: print(f"Can not map tensor {name!r}") sys.exit() @@ -2166,34 +2170,6 @@ def get_tensors(self): class JinaBertModel(BertModel): model_arch = gguf.MODEL_ARCH.JINA_BERT - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - print(f'hparams {self.hparams}') - - assert self.hparams["position_embedding_type"] == "alibi" - - # def __init__(self, *args, **kwargs): - # super().__init__(*args, **kwargs) - # - # assert self.hparams["position_embedding_type"] == "alibi" - # - # # GeGLU activation - # assert self.hparams["feed_forward_type"] == "geglu" - # - # def get_tensors(self): - # assert self.vocab_size is not None - # for name, data in super().get_tensors(): - # print(f'get_tensors: {name} {data.shape}') - # # Nomic Embed's token embeddings tensor is padded, but llama.cpp wants tensor sizes to match exactly. - # if name == 'embeddings.word_embeddings.weight' and data.shape[1] != self.vocab_size: - # rounded_vocab_size = (self.vocab_size + 63) // 64 * 64 - # assert data.shape == (rounded_vocab_size, self.hparams["hidden_size"]) - # data = data[:self.vocab_size, :] - # yield name, data - - - @Model.register("GemmaForCausalLM") class GemmaModel(Model): model_arch = gguf.MODEL_ARCH.GEMMA @@ -2461,9 +2437,7 @@ def main() -> None: print(f"Loading model: {dir_model.name}") hparams = Model.load_hparams(dir_model) - with torch.inference_mode(): - print(hparams["architectures"]) model_class = Model.from_model_architecture(hparams["architectures"][0]) model_instance = model_class(dir_model, ftype_map[args.outtype], fname_out, args.bigendian) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index cfb78327981a3..98a42c2037c9a 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -363,14 +363,14 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.TOKEN_EMBD_NORM, MODEL_TENSOR.TOKEN_TYPES, - MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.ATTN_OUT_NORM, MODEL_TENSOR.ATTN_Q, MODEL_TENSOR.ATTN_K, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, - MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.LAYER_OUT_NORM, ], MODEL_ARCH.MPT: [ diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 7c7abab08184d..b768a278c6479 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -217,9 +217,6 @@ class TensorNameMap: "model.layers.{bid}.mlp.up_proj", # llama-hf refact "layers.{bid}.feed_forward.w3", # llama-pth "encoder.layer.{bid}.intermediate.dense", # bert - "encoder.layer.{bid}.mlp.gated_layers", # jina-bert - "encoder.layer.{bid}.mlp.layernorm", # jina-bert - "encoder.layer.{bid}.mlp.wo", # jina-bert "transformer.h.{bid}.mlp.fc_in", # gpt-j "language_model.encoder.layers.{bid}.mlp.dense_h_to_4h", # persimmon "model.layers.{bid}.mlp.dense_h_to_4h", # persimmon @@ -251,6 +248,7 @@ class TensorNameMap: "model.layers.layers.{bid}.mlp.gate_proj", # plamo "model.layers.{bid}.feed_forward.w1", # internlm2 "encoder.layers.{bid}.mlp.fc12", # nomic-bert + "encoder.layer.{bid}.mlp.gated_layers", # jina-bert ), MODEL_TENSOR.FFN_GATE_EXP: ( @@ -278,6 +276,7 @@ class TensorNameMap: "model.layers.{bid}.feed_forward.w2", # internlm2 "encoder.layers.{bid}.mlp.fc2", # nomic-bert "model.layers.{bid}.mlp.c_proj", # starcoder2 + "encoder.layer.{bid}.mlp.wo", # jina-bert ), MODEL_TENSOR.FFN_DOWN_EXP: ( @@ -307,6 +306,7 @@ class TensorNameMap: "encoder.layer.{bid}.output.LayerNorm", # bert "encoder.layers.{bid}.norm2", # nomic-bert "transformer.decoder_layer.{bid}.rms_norm_3", # Grok + "encoder.layer.{bid}.mlp.layernorm", # jina-bert ), MODEL_TENSOR.SSM_IN: ( diff --git a/llama.cpp b/llama.cpp index e9f430fa704a3..5d28f6e456f61 100644 --- a/llama.cpp +++ b/llama.cpp @@ -680,6 +680,7 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" }, { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, @@ -1921,6 +1922,16 @@ struct llama_layer { // mamba bias struct ggml_tensor * ssm_conv1d_b; struct ggml_tensor * ssm_dt_b; + + //glu mlp (jina-bert) + struct ggml_tensor * mlp_gated_layer_w; + + struct ggml_tensor * mlp_wo_w; + struct ggml_tensor * mlp_wo_b; + + struct ggml_tensor * mlp_norm_w; + struct ggml_tensor * mlp_norm_b; + }; struct llama_kv_cell { @@ -4813,7 +4824,6 @@ static bool llm_load_tensors( } } break; case LLM_ARCH_BERT: - case LLM_ARCH_JINA_BERT: case LLM_ARCH_NOMIC_BERT: { model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); @@ -4831,7 +4841,7 @@ static bool llm_load_tensors( auto & layer = model.layers[i]; - if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT) { + if (model.arch == LLM_ARCH_BERT) { layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}); @@ -4852,7 +4862,7 @@ static bool llm_load_tensors( layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); - if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT) { + if (model.arch == LLM_ARCH_BERT) { layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); @@ -4865,6 +4875,44 @@ static bool llm_load_tensors( layer.layer_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}); } } break; + case LLM_ARCH_JINA_BERT: + { + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // word_embeddings + model.type_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_vocab_type}); //token_type_embeddings + model.tok_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}); // LayerNorm + model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}); //LayerNorm bias? Not sure needed + + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); + + auto & layer = model.layers[i]; // JinaBertLayer + + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}); + + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}); + + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}); + + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); //output_dens + layer.bo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); //output_dens + + layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}); //output_norm + layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}); + + // TODO: HANDLE ALL THE MLP + layer.mlp_gated_layer_w = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, 2 * n_ff}); + + layer.mlp_wo_w = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); + layer.mlp_wo_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); + + layer.mlp_norm_w = ml.create_tensor(ctx_split, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}); + layer.mlp_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}); + } + } break; case LLM_ARCH_BLOOM: { model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); @@ -9713,6 +9761,7 @@ static struct ggml_cgraph * llama_build_graph( result = llm.build_refact(); } break; case LLM_ARCH_BERT: + case LLM_ARCH_JINA_BERT: case LLM_ARCH_NOMIC_BERT: { result = llm.build_bert(); From a40156a077c9181156edf220bde2ee81444ada47 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Fri, 12 Apr 2024 16:09:18 +0200 Subject: [PATCH 03/36] fix: use other tensors --- gguf-py/gguf/constants.py | 1 - gguf-py/gguf/tensor_mapping.py | 2 +- llama.cpp | 25 ++++++++----------------- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 98a42c2037c9a..016aa07e523fa 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -370,7 +370,6 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_DOWN, - MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.LAYER_OUT_NORM, ], MODEL_ARCH.MPT: [ diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index b768a278c6479..4e4c775a7a031 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -228,6 +228,7 @@ class TensorNameMap: "model.layers.{bid}.feed_forward.w3", # internlm2 "encoder.layers.{bid}.mlp.fc11", # nomic-bert "model.layers.{bid}.mlp.c_fc", # starcoder2 + "encoder.layer.{bid}.mlp.gated_layers", # jina-bert ), MODEL_TENSOR.FFN_UP_EXP: ( @@ -248,7 +249,6 @@ class TensorNameMap: "model.layers.layers.{bid}.mlp.gate_proj", # plamo "model.layers.{bid}.feed_forward.w1", # internlm2 "encoder.layers.{bid}.mlp.fc12", # nomic-bert - "encoder.layer.{bid}.mlp.gated_layers", # jina-bert ), MODEL_TENSOR.FFN_GATE_EXP: ( diff --git a/llama.cpp b/llama.cpp index 5d28f6e456f61..23a9aa86e31eb 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1922,16 +1922,6 @@ struct llama_layer { // mamba bias struct ggml_tensor * ssm_conv1d_b; struct ggml_tensor * ssm_dt_b; - - //glu mlp (jina-bert) - struct ggml_tensor * mlp_gated_layer_w; - - struct ggml_tensor * mlp_wo_w; - struct ggml_tensor * mlp_wo_b; - - struct ggml_tensor * mlp_norm_w; - struct ggml_tensor * mlp_norm_b; - }; struct llama_kv_cell { @@ -4904,13 +4894,13 @@ static bool llm_load_tensors( layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}); // TODO: HANDLE ALL THE MLP - layer.mlp_gated_layer_w = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, 2 * n_ff}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, 2 * n_ff}); - layer.mlp_wo_w = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); - layer.mlp_wo_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); - layer.mlp_norm_w = ml.create_tensor(ctx_split, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}); - layer.mlp_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}); + layer.layer_out_norm = ml.create_tensor(ctx_split, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}); + layer.layer_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}); } } break; case LLM_ARCH_BLOOM: @@ -7564,7 +7554,7 @@ struct llm_build_context { struct ggml_tensor * Vcur; // self-attention - if (model.arch == LLM_ARCH_BERT) { + if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT) { Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), model.layers[il].bq); cb(Qcur, "Qcur", il); @@ -7654,7 +7644,7 @@ struct llm_build_context { cb(ffn_inp, "ffn_inp", il); // feed-forward network - if (model.arch == LLM_ARCH_BERT) { + if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT) { cur = llm_build_ffn(ctx0, cur, model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, NULL, @@ -7677,6 +7667,7 @@ struct llm_build_context { // output layer norm cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].layer_out_norm, model.layers[il].layer_out_norm_b, LLM_NORM, cb, il); + // input for next layer inpL = cur; } From b00d38b0b16bf8c055baaa40c33a53b29edf4075 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Tue, 16 Apr 2024 11:51:38 +0200 Subject: [PATCH 04/36] feat: embedding gets results --- convert-hf-to-gguf.py | 23 +++++++++++++++++++++++ gguf-py/gguf/constants.py | 1 + gguf-py/gguf/tensor_mapping.py | 3 ++- llama.cpp | 22 ++++++++++++++++------ 4 files changed, 42 insertions(+), 7 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 3285a7ef82881..218b136f90549 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -2170,6 +2170,29 @@ def get_tensors(self): class JinaBertModel(BertModel): model_arch = gguf.MODEL_ARCH.JINA_BERT + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.intermediate_size = self.hparams["intermediate_size"] + + def get_tensors(self): + import string + print(f'Intermediate SIZE: {self.intermediate_size}') + + for name, data in super().get_tensors(): + if 'gated_layers' in name: + print(f'name {name} => {data.shape}') + d1 = data[:self.intermediate_size, :] + name1 = name.replace('gated_layers', 'gated_layers_w') + d2 = data[self.intermediate_size:, :] + name2 = name.replace('gated_layers', 'gated_layers_v') + print(f'd1 {d1.shape}, d2 {d2.shape}') + yield name1, d1 + yield name2, d2 + continue + + yield name, data + + @Model.register("GemmaForCausalLM") class GemmaModel(Model): model_arch = gguf.MODEL_ARCH.GEMMA diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 016aa07e523fa..5eb0259591b65 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -369,6 +369,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.LAYER_OUT_NORM, ], diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 4e4c775a7a031..1d582676fb4db 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -228,7 +228,7 @@ class TensorNameMap: "model.layers.{bid}.feed_forward.w3", # internlm2 "encoder.layers.{bid}.mlp.fc11", # nomic-bert "model.layers.{bid}.mlp.c_fc", # starcoder2 - "encoder.layer.{bid}.mlp.gated_layers", # jina-bert + "encoder.layer.{bid}.mlp.gated_layers_v", # jina-bert ), MODEL_TENSOR.FFN_UP_EXP: ( @@ -249,6 +249,7 @@ class TensorNameMap: "model.layers.layers.{bid}.mlp.gate_proj", # plamo "model.layers.{bid}.feed_forward.w1", # internlm2 "encoder.layers.{bid}.mlp.fc12", # nomic-bert + "encoder.layer.{bid}.mlp.gated_layers_w", # jina-bert ), MODEL_TENSOR.FFN_GATE_EXP: ( diff --git a/llama.cpp b/llama.cpp index 23a9aa86e31eb..eafabd48de7f7 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4870,7 +4870,7 @@ static bool llm_load_tensors( model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // word_embeddings model.type_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_vocab_type}); //token_type_embeddings model.tok_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}); // LayerNorm - model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}); //LayerNorm bias? Not sure needed + model.tok_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}); //LayerNorm bias for (int i = 0; i < n_layer; ++i) { ggml_context * ctx_layer = ctx_for_layer(i); @@ -4893,8 +4893,8 @@ static bool llm_load_tensors( layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}); //output_norm layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}); - // TODO: HANDLE ALL THE MLP - layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, 2 * n_ff}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); @@ -5851,7 +5851,7 @@ static struct ggml_tensor * llm_build_ffn( llm_ffn_gate_type type_gate, const llm_build_cb & cb, int il) { - struct ggml_tensor * tmp = ggml_mul_mat(ctx, up, cur); + struct ggml_tensor * tmp = up ? ggml_mul_mat(ctx, up, cur): cur; cb(tmp, "ffn_up", il); if (up_b) { @@ -7522,8 +7522,11 @@ struct llm_build_context { struct ggml_tensor * cur; struct ggml_tensor * inpL; + struct ggml_tensor * inp_pos = nullptr; - struct ggml_tensor * inp_pos = build_inp_pos(); + if (model.arch != LLM_ARCH_JINA_BERT) { + inp_pos = build_inp_pos(); + } struct ggml_tensor * inp_mean = build_inp_mean(); struct ggml_tensor * inp_cls = build_inp_cls(); @@ -7644,13 +7647,20 @@ struct llm_build_context { cb(ffn_inp, "ffn_inp", il); // feed-forward network - if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT) { + if (model.arch == LLM_ARCH_BERT) { cur = llm_build_ffn(ctx0, cur, model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL, NULL, model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); + } else if (model.arch == LLM_ARCH_JINA_BERT) { + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, NULL, + model.layers[il].ffn_gate, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, + NULL, + LLM_FFN_GELU, LLM_FFN_PAR, cb, il); } else { cur = llm_build_ffn(ctx0, cur, model.layers[il].ffn_up, NULL, From cf1c1447e32a21d9ce5f0447eff9d8459e011a7f Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 22 Apr 2024 13:05:26 +0200 Subject: [PATCH 05/36] fix: fix usage of ALIBI --- convert-hf-to-gguf.py | 5 ----- ggml.c | 14 ++++++-------- llama.cpp | 6 +++--- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 218b136f90549..9c01c296e2984 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -2175,17 +2175,12 @@ def __init__(self, *args, **kwargs): self.intermediate_size = self.hparams["intermediate_size"] def get_tensors(self): - import string - print(f'Intermediate SIZE: {self.intermediate_size}') - for name, data in super().get_tensors(): if 'gated_layers' in name: - print(f'name {name} => {data.shape}') d1 = data[:self.intermediate_size, :] name1 = name.replace('gated_layers', 'gated_layers_w') d2 = data[self.intermediate_size:, :] name2 = name.replace('gated_layers', 'gated_layers_v') - print(f'd1 {d1.shape}, d2 {d2.shape}') yield name1, d1 yield name2, d2 continue diff --git a/ggml.c b/ggml.c index 793b67f4c7020..6ae51fd13e690 100644 --- a/ggml.c +++ b/ggml.c @@ -5406,10 +5406,6 @@ static struct ggml_tensor * ggml_soft_max_impl( GGML_ASSERT(pos->ne[0] == a->ne[0]); } - if (max_bias > 0.0f) { - GGML_ASSERT(pos); - } - bool is_node = false; if (a->grad) { @@ -12241,11 +12237,11 @@ static void ggml_compute_forward_soft_max_f32( float * wp = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith; // when max_bias <= 0.0f, src2 is not used and we default it to src0 to avoid branching - float * pos = src2 ? (float *) src2->data : src0->data; + float * pos = src2 ? (float *) src2->data : NULL; for (int i1 = ir0; i1 < ir1; i1++) { float * sp = (float *)((char *) src0->data + i1*src0->nb[1]); - float * dp = (float *)((char *) dst->data + i1*dst->nb[1]); + float * dp = (float *)((char *) dst->data + i1*dst->nb[1]); // broadcast the mask across rows float * mp = src1 ? (float *)((char *) src1->data + (i1%ne11)*src1->nb[1]) : NULL; @@ -12262,7 +12258,7 @@ static void ggml_compute_forward_soft_max_f32( const float slope = h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1); for (int i = 0; i < nc; i++) { - wp[i] = wp[i] + slope*pos[i]; + wp[i] = wp[i] - slope*abs(i1%nc - i); } } @@ -12478,7 +12474,7 @@ static void ggml_compute_forward_alibi_f32( for (int64_t j = 0; j < ne1; j++) { float * const src = (float *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2); float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2); - pdst[0] = i * m_k + src[0]; + pdst[0] = -1.0f * i * m_k; } } } @@ -16111,6 +16107,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm return; } + fprintf(stdout, "Computing forward (%s) for tensor %s\n", GGML_OP_NAME[tensor->op], tensor->name); switch (tensor->op) { case GGML_OP_DUP: { @@ -16447,6 +16444,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm GGML_ASSERT(false); } break; } + fprintf(stdout, "After FORWARD %s (%p): Shape:%li, %li, %li, %li tensor: %9.6f, %9.6f, %9.6f, %9.6f \n", tensor->name, tensor, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], ((float *)(tensor->data))[0], ((float *)(tensor->data))[1], ((float *)(tensor->data))[2], ((float *)(tensor->data))[3]); } //////////////////////////////////////////////////////////////////////////////// diff --git a/llama.cpp b/llama.cpp index eafabd48de7f7..e52b39d12d929 100644 --- a/llama.cpp +++ b/llama.cpp @@ -3795,6 +3795,7 @@ static void llm_load_hparams( ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, hparams.n_vocab_type); ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type); + hparams.f_max_alibi_bias = 8.0f; switch (hparams.n_layer) { case 4: model.type = e_model::MODEL_33M; break; // jina-embeddings-small @@ -4001,7 +4002,7 @@ static void llm_load_hparams( model.ftype = ml.ftype; - if (hparams.f_max_alibi_bias > 0.0f) { + if (hparams.f_max_alibi_bias > 0.0f && model.arch != LLM_ARCH_JINA_BERT) { hparams.need_kq_pos = true; } @@ -4519,7 +4520,6 @@ static bool llm_load_tensors( model.layers.resize(n_layer); const auto tn = LLM_TN(model.arch); - //std::printf("JOAN HERE ARCH %i", model.arch); switch (model.arch) { case LLM_ARCH_LLAMA: case LLM_ARCH_REFACT: @@ -7525,7 +7525,7 @@ struct llm_build_context { struct ggml_tensor * inp_pos = nullptr; if (model.arch != LLM_ARCH_JINA_BERT) { - inp_pos = build_inp_pos(); + inp_pos = build_inp_pos(); } struct ggml_tensor * inp_mean = build_inp_mean(); struct ggml_tensor * inp_cls = build_inp_cls(); From 63a1d7c0beb2654e8f387cc077cf6c968b357737 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 22 Apr 2024 13:06:05 +0200 Subject: [PATCH 06/36] fix: clean prints --- ggml.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/ggml.c b/ggml.c index 6ae51fd13e690..131cc8fd96a63 100644 --- a/ggml.c +++ b/ggml.c @@ -16107,7 +16107,6 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm return; } - fprintf(stdout, "Computing forward (%s) for tensor %s\n", GGML_OP_NAME[tensor->op], tensor->name); switch (tensor->op) { case GGML_OP_DUP: { @@ -16444,7 +16443,6 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm GGML_ASSERT(false); } break; } - fprintf(stdout, "After FORWARD %s (%p): Shape:%li, %li, %li, %li tensor: %9.6f, %9.6f, %9.6f, %9.6f \n", tensor->name, tensor, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], ((float *)(tensor->data))[0], ((float *)(tensor->data))[1], ((float *)(tensor->data))[2], ((float *)(tensor->data))[3]); } //////////////////////////////////////////////////////////////////////////////// From c229e48937e6696b2f332282f1ab698c4f110500 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 22 Apr 2024 13:12:14 +0200 Subject: [PATCH 07/36] fix: do some cleanup unused vars --- ggml.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ggml.c b/ggml.c index 131cc8fd96a63..622df3a5affd5 100644 --- a/ggml.c +++ b/ggml.c @@ -12258,7 +12258,11 @@ static void ggml_compute_forward_soft_max_f32( const float slope = h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1); for (int i = 0; i < nc; i++) { - wp[i] = wp[i] - slope*abs(i1%nc - i); + if (pos == NULL) { + wp[i] = wp[i] + pos[i]; + } else { + wp[i] = wp[i] - slope*abs(i1%nc - i); + } } } @@ -12472,7 +12476,6 @@ static void ggml_compute_forward_alibi_f32( for (int64_t i = 0; i < ne0; i++) { for (int64_t j = 0; j < ne1; j++) { - float * const src = (float *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2); float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2); pdst[0] = -1.0f * i * m_k; } From e2323706e42543f5230f660ad69100632542a30a Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 22 Apr 2024 13:15:34 +0200 Subject: [PATCH 08/36] fix: revert changes to Makefile and CMakeLists --- CMakeLists.txt | 8 +- Makefile | 2731 +++++++++++++++++------------------------------- 2 files changed, 984 insertions(+), 1755 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a144e2cf323a7..19fdfa46ca4f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,9 +59,9 @@ option(LLAMA_GPROF "llama: enable gprof" option(LLAMA_FATAL_WARNINGS "llama: enable -Werror flag" OFF) # sanitizers -option(LLAMA_SANITIZE_THREAD "llama: enable thread sanitizer" ON) -option(LLAMA_SANITIZE_ADDRESS "llama: enable address sanitizer" ON) -option(LLAMA_SANITIZE_UNDEFINED "llama: enable undefined sanitizer" ON) +option(LLAMA_SANITIZE_THREAD "llama: enable thread sanitizer" OFF) +option(LLAMA_SANITIZE_ADDRESS "llama: enable address sanitizer" OFF) +option(LLAMA_SANITIZE_UNDEFINED "llama: enable undefined sanitizer" OFF) # instruction set specific if (LLAMA_NATIVE) @@ -126,7 +126,7 @@ option(LLAMA_CPU_HBM "llama: use memkind for CPU HBM" set(LLAMA_SCHED_MAX_COPIES "4" CACHE STRING "llama: max input copies for pipeline parallelism") option(LLAMA_BUILD_TESTS "llama: build tests" ${LLAMA_STANDALONE}) -option(LLAMA_BUILD_EXAMPLES "llama: build examples" ON) +option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE}) option(LLAMA_BUILD_SERVER "llama: build server example" ON) # add perf arguments diff --git a/Makefile b/Makefile index ec7edd425de0b..11b31c5c84182 100644 --- a/Makefile +++ b/Makefile @@ -1,1761 +1,990 @@ -# CMAKE generated file: DO NOT EDIT! -# Generated by "Unix Makefiles" Generator, CMake Version 3.29 +# Define the default target now so that it is always the first target +BUILD_TARGETS = \ + main quantize quantize-stats perplexity imatrix embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \ + simple batched batched-bench save-load-state server gguf gguf-split llama-bench libllava.a llava-cli baby-llama beam-search \ + retrieval speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead lookup passkey gritlm tests/test-c.o + +# Binaries only useful for tests +TEST_TARGETS = \ + tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt \ + tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama \ + tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe tests/test-rope \ + tests/test-backend-ops tests/test-model-load-cancel tests/test-autorelease \ + tests/test-json-schema-to-grammar tests/test-grammar-integration + +# Code coverage output files +COV_TARGETS = *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report + +ifndef UNAME_S +UNAME_S := $(shell uname -s) +endif + +ifndef UNAME_P +UNAME_P := $(shell uname -p) +endif + +ifndef UNAME_M +UNAME_M := $(shell uname -m) +endif + +# Mac OS + Arm can report x86_64 +# ref: https://github.com/ggerganov/whisper.cpp/issues/66#issuecomment-1282546789 +ifeq ($(UNAME_S),Darwin) + ifndef LLAMA_NO_METAL + LLAMA_METAL := 1 + endif + + ifneq ($(UNAME_P),arm) + SYSCTL_M := $(shell sysctl -n hw.optional.arm64 2>/dev/null) + ifeq ($(SYSCTL_M),1) + # UNAME_P := arm + # UNAME_M := arm64 + warn := $(warning Your arch is announced as x86_64, but it seems to actually be ARM64. Not fixing that can lead to bad performance. For more info see: https://github.com/ggerganov/whisper.cpp/issues/66\#issuecomment-1282546789) + endif + endif +endif + +default: $(BUILD_TARGETS) + +test: $(TEST_TARGETS) + @failures=0; \ + for test_target in $(TEST_TARGETS); do \ + if [ "$$test_target" = "tests/test-tokenizer-0-llama" ]; then \ + ./$$test_target $(CURDIR)/models/ggml-vocab-llama.gguf; \ + elif [ "$$test_target" = "tests/test-tokenizer-0-falcon" ]; then \ + ./$$test_target $(CURDIR)/models/ggml-vocab-falcon.gguf; \ + elif [ "$$test_target" = "tests/test-tokenizer-1-llama" ]; then \ + continue; \ + elif [ "$$test_target" = "tests/test-tokenizer-1-bpe" ]; then \ + continue; \ + else \ + echo "Running test $$test_target..."; \ + ./$$test_target; \ + fi; \ + if [ $$? -ne 0 ]; then \ + printf 'Test %s FAILED!\n\n' $$test_target; \ + failures=$$(( failures + 1 )); \ + else \ + printf 'Test %s passed.\n\n' $$test_target; \ + fi; \ + done; \ + if [ $$failures -gt 0 ]; then \ + printf '\n%s tests failed.\n' $$failures; \ + exit 1; \ + fi + @echo 'All tests passed.' + +all: $(BUILD_TARGETS) $(TEST_TARGETS) + +coverage: ## Run code coverage + gcov -pb tests/*.cpp + +lcov-report: coverage ## Generate lcov report + mkdir -p lcov-report + lcov --capture --directory . --output-file lcov-report/coverage.info + genhtml lcov-report/coverage.info --output-directory lcov-report + +gcovr-report: coverage ## Generate gcovr report + mkdir -p gcovr-report + gcovr --root . --html --html-details --output gcovr-report/coverage.html + +ifdef RISCV_CROSS_COMPILE +CC := riscv64-unknown-linux-gnu-gcc +CXX := riscv64-unknown-linux-gnu-g++ +endif + +# +# Compile flags +# + +# keep standard at C11 and C++11 +MK_CPPFLAGS = -I. -Icommon +MK_CFLAGS = -std=c11 -fPIC +MK_CXXFLAGS = -std=c++11 -fPIC +MK_NVCCFLAGS = -std=c++11 + +# -Ofast tends to produce faster code, but may not be available for some compilers. +ifdef LLAMA_FAST +MK_CFLAGS += -Ofast +HOST_CXXFLAGS += -Ofast +MK_NVCCFLAGS += -O3 +else +MK_CFLAGS += -O3 +MK_CXXFLAGS += -O3 +MK_NVCCFLAGS += -O3 +endif + +ifndef LLAMA_NO_CCACHE +CCACHE := $(shell which ccache) +ifdef CCACHE +export CCACHE_SLOPPINESS = time_macros +$(info I ccache found, compilation results will be cached. Disable with LLAMA_NO_CCACHE.) +CC := $(CCACHE) $(CC) +CXX := $(CCACHE) $(CXX) +else +$(info I ccache not found. Consider installing it for faster compilation.) +endif # CCACHE +endif # LLAMA_NO_CCACHE + +# clock_gettime came in POSIX.1b (1993) +# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional +# posix_memalign came in POSIX.1-2001 / SUSv3 +# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985) +MK_CPPFLAGS += -D_XOPEN_SOURCE=600 + +# Somehow in OpenBSD whenever POSIX conformance is specified +# some string functions rely on locale_t availability, +# which was introduced in POSIX.1-2008, forcing us to go higher +ifeq ($(UNAME_S),OpenBSD) + MK_CPPFLAGS += -U_XOPEN_SOURCE -D_XOPEN_SOURCE=700 +endif + +# Data types, macros and functions related to controlling CPU affinity and +# some memory allocation are available on Linux through GNU extensions in libc +ifeq ($(UNAME_S),Linux) + MK_CPPFLAGS += -D_GNU_SOURCE +endif + +# RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1, +# and on macOS its availability depends on enabling Darwin extensions +# similarly on DragonFly, enabling BSD extensions is necessary +ifeq ($(UNAME_S),Darwin) + MK_CPPFLAGS += -D_DARWIN_C_SOURCE +endif +ifeq ($(UNAME_S),DragonFly) + MK_CPPFLAGS += -D__BSD_VISIBLE +endif + +# alloca is a non-standard interface that is not visible on BSDs when +# POSIX conformance is specified, but not all of them provide a clean way +# to enable it in such cases +ifeq ($(UNAME_S),FreeBSD) + MK_CPPFLAGS += -D__BSD_VISIBLE +endif +ifeq ($(UNAME_S),NetBSD) + MK_CPPFLAGS += -D_NETBSD_SOURCE +endif +ifeq ($(UNAME_S),OpenBSD) + MK_CPPFLAGS += -D_BSD_SOURCE +endif + +ifdef LLAMA_SCHED_MAX_COPIES + MK_CPPFLAGS += -DGGML_SCHED_MAX_COPIES=$(LLAMA_SCHED_MAX_COPIES) +endif + +ifdef LLAMA_DEBUG + MK_CFLAGS += -O0 -g + MK_CXXFLAGS += -O0 -g + MK_LDFLAGS += -g + + ifeq ($(UNAME_S),Linux) + MK_CPPFLAGS += -D_GLIBCXX_ASSERTIONS + endif +else + MK_CPPFLAGS += -DNDEBUG +endif + +ifdef LLAMA_SANITIZE_THREAD + MK_CFLAGS += -fsanitize=thread -g + MK_CXXFLAGS += -fsanitize=thread -g + MK_LDFLAGS += -fsanitize=thread -g +endif + +ifdef LLAMA_SANITIZE_ADDRESS + MK_CFLAGS += -fsanitize=address -fno-omit-frame-pointer -g + MK_CXXFLAGS += -fsanitize=address -fno-omit-frame-pointer -g + MK_LDFLAGS += -fsanitize=address -fno-omit-frame-pointer -g +endif + +ifdef LLAMA_SANITIZE_UNDEFINED + MK_CFLAGS += -fsanitize=undefined -g + MK_CXXFLAGS += -fsanitize=undefined -g + MK_LDFLAGS += -fsanitize=undefined -g +endif + +ifdef LLAMA_SERVER_VERBOSE + MK_CPPFLAGS += -DSERVER_VERBOSE=$(LLAMA_SERVER_VERBOSE) +endif + +ifdef LLAMA_SERVER_SSL + MK_CPPFLAGS += -DCPPHTTPLIB_OPENSSL_SUPPORT + MK_LDFLAGS += -lssl -lcrypto +endif + +ifdef LLAMA_CODE_COVERAGE + MK_CXXFLAGS += -fprofile-arcs -ftest-coverage -dumpbase '' +endif + +ifdef LLAMA_DISABLE_LOGS + MK_CPPFLAGS += -DLOG_DISABLE_LOGS +endif # LLAMA_DISABLE_LOGS + +# warnings +WARN_FLAGS = -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function +MK_CFLAGS += $(WARN_FLAGS) -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int \ + -Werror=implicit-function-declaration +MK_CXXFLAGS += $(WARN_FLAGS) -Wmissing-declarations -Wmissing-noreturn + +ifeq ($(LLAMA_FATAL_WARNINGS),1) + MK_CFLAGS += -Werror + MK_CXXFLAGS += -Werror +endif + +# this version of Apple ld64 is buggy +ifneq '' '$(findstring dyld-1015.7,$(shell $(CC) $(LDFLAGS) -Wl,-v 2>&1))' + MK_CPPFLAGS += -DHAVE_BUGGY_APPLE_LINKER +endif + +# OS specific +# TODO: support Windows +ifneq '' '$(filter $(UNAME_S),Linux Darwin FreeBSD NetBSD OpenBSD Haiku)' + MK_CFLAGS += -pthread + MK_CXXFLAGS += -pthread +endif + +# detect Windows +ifneq ($(findstring _NT,$(UNAME_S)),) + _WIN32 := 1 +endif + +# library name prefix +ifneq ($(_WIN32),1) + LIB_PRE := lib +endif + +# Dynamic Shared Object extension +ifneq ($(_WIN32),1) + DSO_EXT := .so +else + DSO_EXT := .dll +endif + +# Windows Sockets 2 (Winsock) for network-capable apps +ifeq ($(_WIN32),1) + LWINSOCK2 := -lws2_32 +endif + +ifdef LLAMA_GPROF + MK_CFLAGS += -pg + MK_CXXFLAGS += -pg +endif +ifdef LLAMA_PERF + MK_CPPFLAGS += -DGGML_PERF +endif + +# Architecture specific +# TODO: probably these flags need to be tweaked on some architectures +# feel free to update the Makefile for your architecture and send a pull request or issue + +ifndef RISCV + +ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) + # Use all CPU extensions that are available: + MK_CFLAGS += -march=native -mtune=native + HOST_CXXFLAGS += -march=native -mtune=native + + # Usage AVX-only + #MK_CFLAGS += -mfma -mf16c -mavx + #MK_CXXFLAGS += -mfma -mf16c -mavx + + # Usage SSSE3-only (Not is SSE3!) + #MK_CFLAGS += -mssse3 + #MK_CXXFLAGS += -mssse3 +endif + +ifneq '' '$(findstring mingw,$(shell $(CC) -dumpmachine))' + # The stack is only 16-byte aligned on Windows, so don't let gcc emit aligned moves. + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412 + # https://github.com/ggerganov/llama.cpp/issues/2922 + MK_CFLAGS += -Xassembler -muse-unaligned-vector-move + MK_CXXFLAGS += -Xassembler -muse-unaligned-vector-move + + # Target Windows 8 for PrefetchVirtualMemory + MK_CPPFLAGS += -D_WIN32_WINNT=0x602 +endif + +ifneq ($(filter aarch64%,$(UNAME_M)),) + # Apple M1, M2, etc. + # Raspberry Pi 3, 4, Zero 2 (64-bit) + # Nvidia Jetson + MK_CFLAGS += -mcpu=native + MK_CXXFLAGS += -mcpu=native + JETSON_RELEASE_INFO = $(shell jetson_release) + ifdef JETSON_RELEASE_INFO + ifneq ($(filter TX2%,$(JETSON_RELEASE_INFO)),) + JETSON_EOL_MODULE_DETECT = 1 + CC = aarch64-unknown-linux-gnu-gcc + cxx = aarch64-unknown-linux-gnu-g++ + endif + endif +endif + +ifneq ($(filter armv6%,$(UNAME_M)),) + # Raspberry Pi 1, Zero + MK_CFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access + MK_CXXFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access +endif + +ifneq ($(filter armv7%,$(UNAME_M)),) + # Raspberry Pi 2 + MK_CFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access -funsafe-math-optimizations + MK_CXXFLAGS += -mfpu=neon-fp-armv8 -mfp16-format=ieee -mno-unaligned-access -funsafe-math-optimizations +endif + +ifneq ($(filter armv8%,$(UNAME_M)),) + # Raspberry Pi 3, 4, Zero 2 (32-bit) + MK_CFLAGS += -mfp16-format=ieee -mno-unaligned-access + MK_CXXFLAGS += -mfp16-format=ieee -mno-unaligned-access +endif + +ifneq ($(filter ppc64%,$(UNAME_M)),) + POWER9_M := $(shell grep "POWER9" /proc/cpuinfo) + ifneq (,$(findstring POWER9,$(POWER9_M))) + MK_CFLAGS += -mcpu=power9 + MK_CXXFLAGS += -mcpu=power9 + endif +endif + +ifneq ($(filter ppc64le%,$(UNAME_M)),) + MK_CFLAGS += -mcpu=powerpc64le + MK_CXXFLAGS += -mcpu=powerpc64le + CUDA_POWER_ARCH = 1 +endif + +else + MK_CFLAGS += -march=rv64gcv -mabi=lp64d + MK_CXXFLAGS += -march=rv64gcv -mabi=lp64d +endif + +ifdef LLAMA_QKK_64 + MK_CPPFLAGS += -DGGML_QKK_64 +endif + +ifndef LLAMA_NO_ACCELERATE + # Mac OS - include Accelerate framework. + # `-framework Accelerate` works both with Apple Silicon and Mac Intel + ifeq ($(UNAME_S),Darwin) + MK_CPPFLAGS += -DGGML_USE_ACCELERATE + MK_CPPFLAGS += -DACCELERATE_NEW_LAPACK + MK_CPPFLAGS += -DACCELERATE_LAPACK_ILP64 + MK_LDFLAGS += -framework Accelerate + endif +endif # LLAMA_NO_ACCELERATE + +ifdef LLAMA_MPI + MK_CPPFLAGS += -DGGML_USE_MPI + MK_CFLAGS += -Wno-cast-qual + MK_CXXFLAGS += -Wno-cast-qual + OBJS += ggml-mpi.o +endif # LLAMA_MPI + +ifdef LLAMA_OPENBLAS + MK_CPPFLAGS += -DGGML_USE_OPENBLAS $(shell pkg-config --cflags-only-I openblas) + MK_CFLAGS += $(shell pkg-config --cflags-only-other openblas) + MK_LDFLAGS += $(shell pkg-config --libs openblas) +endif # LLAMA_OPENBLAS + +ifdef LLAMA_BLIS + MK_CPPFLAGS += -DGGML_USE_OPENBLAS -I/usr/local/include/blis -I/usr/include/blis + MK_LDFLAGS += -lblis -L/usr/local/lib +endif # LLAMA_BLIS + +ifdef LLAMA_CUBLAS +# LLAMA_CUBLAS is deprecated and will be removed in the future + LLAMA_CUDA := 1 +endif + +ifdef LLAMA_CUDA + ifneq ('', '$(wildcard /opt/cuda)') + CUDA_PATH ?= /opt/cuda + else + CUDA_PATH ?= /usr/local/cuda + endif + MK_CPPFLAGS += -DGGML_USE_CUDA -I$(CUDA_PATH)/include -I$(CUDA_PATH)/targets/$(UNAME_M)-linux/include + MK_LDFLAGS += -lcuda -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L$(CUDA_PATH)/lib64 -L/usr/lib64 -L$(CUDA_PATH)/targets/$(UNAME_M)-linux/lib -L/usr/lib/wsl/lib + OBJS += ggml-cuda.o + OBJS += $(patsubst %.cu,%.o,$(wildcard ggml-cuda/*.cu)) + MK_NVCCFLAGS += -use_fast_math +ifdef LLAMA_FATAL_WARNINGS + MK_NVCCFLAGS += -Werror all-warnings +endif # LLAMA_FATAL_WARNINGS +ifndef JETSON_EOL_MODULE_DETECT + MK_NVCCFLAGS += --forward-unknown-to-host-compiler +endif # JETSON_EOL_MODULE_DETECT +ifdef LLAMA_DEBUG + MK_NVCCFLAGS += -lineinfo +endif # LLAMA_DEBUG +ifdef LLAMA_CUDA_NVCC + NVCC = $(CCACHE) $(LLAMA_CUDA_NVCC) +else + NVCC = $(CCACHE) nvcc +endif #LLAMA_CUDA_NVCC +ifdef CUDA_DOCKER_ARCH + MK_NVCCFLAGS += -Wno-deprecated-gpu-targets -arch=$(CUDA_DOCKER_ARCH) +else ifndef CUDA_POWER_ARCH + MK_NVCCFLAGS += -arch=native +endif # CUDA_DOCKER_ARCH +ifdef LLAMA_CUDA_FORCE_DMMV + MK_NVCCFLAGS += -DGGML_CUDA_FORCE_DMMV +endif # LLAMA_CUDA_FORCE_DMMV +ifdef LLAMA_CUDA_FORCE_MMQ + MK_NVCCFLAGS += -DGGML_CUDA_FORCE_MMQ +endif # LLAMA_CUDA_FORCE_MMQ +ifdef LLAMA_CUDA_DMMV_X + MK_NVCCFLAGS += -DGGML_CUDA_DMMV_X=$(LLAMA_CUDA_DMMV_X) +else + MK_NVCCFLAGS += -DGGML_CUDA_DMMV_X=32 +endif # LLAMA_CUDA_DMMV_X +ifdef LLAMA_CUDA_MMV_Y + MK_NVCCFLAGS += -DGGML_CUDA_MMV_Y=$(LLAMA_CUDA_MMV_Y) +else ifdef LLAMA_CUDA_DMMV_Y + MK_NVCCFLAGS += -DGGML_CUDA_MMV_Y=$(LLAMA_CUDA_DMMV_Y) # for backwards compatibility +else + MK_NVCCFLAGS += -DGGML_CUDA_MMV_Y=1 +endif # LLAMA_CUDA_MMV_Y +ifdef LLAMA_CUDA_F16 + MK_NVCCFLAGS += -DGGML_CUDA_F16 +endif # LLAMA_CUDA_F16 +ifdef LLAMA_CUDA_DMMV_F16 + MK_NVCCFLAGS += -DGGML_CUDA_F16 +endif # LLAMA_CUDA_DMMV_F16 +ifdef LLAMA_CUDA_KQUANTS_ITER + MK_NVCCFLAGS += -DK_QUANTS_PER_ITERATION=$(LLAMA_CUDA_KQUANTS_ITER) +else + MK_NVCCFLAGS += -DK_QUANTS_PER_ITERATION=2 +endif +ifdef LLAMA_CUDA_PEER_MAX_BATCH_SIZE + MK_NVCCFLAGS += -DGGML_CUDA_PEER_MAX_BATCH_SIZE=$(LLAMA_CUDA_PEER_MAX_BATCH_SIZE) +else + MK_NVCCFLAGS += -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 +endif # LLAMA_CUDA_PEER_MAX_BATCH_SIZE +ifdef LLAMA_CUDA_NO_PEER_COPY + MK_NVCCFLAGS += -DGGML_CUDA_NO_PEER_COPY +endif # LLAMA_CUDA_NO_PEER_COPY +ifdef LLAMA_CUDA_CCBIN + MK_NVCCFLAGS += -ccbin $(LLAMA_CUDA_CCBIN) +endif + +ifdef JETSON_EOL_MODULE_DETECT +define NVCC_COMPILE + $(NVCC) -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUDA -I/usr/local/cuda/include -I/opt/cuda/include -I/usr/local/cuda/targets/aarch64-linux/include -std=c++11 -O3 $(NVCCFLAGS) $(CPPFLAGS) -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ +endef # NVCC_COMPILE +else +define NVCC_COMPILE + $(NVCC) $(NVCCFLAGS) $(CPPFLAGS) -Xcompiler "$(CUDA_CXXFLAGS)" -c $< -o $@ +endef # NVCC_COMPILE +endif # JETSON_EOL_MODULE_DETECT + +ggml-cuda/%.o: ggml-cuda/%.cu ggml-cuda/%.cuh ggml.h ggml-common.h ggml-cuda/common.cuh + $(NVCC_COMPILE) + +ggml-cuda.o: ggml-cuda.cu ggml-cuda.h ggml.h ggml-backend.h ggml-backend-impl.h ggml-common.h $(wildcard ggml-cuda/*.cuh) + $(NVCC_COMPILE) + +endif # LLAMA_CUDA + +ifdef LLAMA_CLBLAST + + MK_CPPFLAGS += -DGGML_USE_CLBLAST $(shell pkg-config --cflags-only-I clblast OpenCL) + MK_CFLAGS += $(shell pkg-config --cflags-only-other clblast OpenCL) + MK_CXXFLAGS += $(shell pkg-config --cflags-only-other clblast OpenCL) + + # Mac provides OpenCL as a framework + ifeq ($(UNAME_S),Darwin) + MK_LDFLAGS += -lclblast -framework OpenCL + else + MK_LDFLAGS += $(shell pkg-config --libs clblast OpenCL) + endif + OBJS += ggml-opencl.o + +ggml-opencl.o: ggml-opencl.cpp ggml-opencl.h + $(CXX) $(CXXFLAGS) -c $< -o $@ +endif # LLAMA_CLBLAST + +ifdef LLAMA_VULKAN + MK_CPPFLAGS += -DGGML_USE_VULKAN + MK_LDFLAGS += -lvulkan + OBJS += ggml-vulkan.o + +ifdef LLAMA_VULKAN_CHECK_RESULTS + MK_CPPFLAGS += -DGGML_VULKAN_CHECK_RESULTS +endif + +ifdef LLAMA_VULKAN_DEBUG + MK_CPPFLAGS += -DGGML_VULKAN_DEBUG +endif + +ifdef LLAMA_VULKAN_VALIDATE + MK_CPPFLAGS += -DGGML_VULKAN_VALIDATE +endif + +ifdef LLAMA_VULKAN_RUN_TESTS + MK_CPPFLAGS += -DGGML_VULKAN_RUN_TESTS +endif + +ggml-vulkan.o: ggml-vulkan.cpp ggml-vulkan.h + $(CXX) $(CXXFLAGS) -c $< -o $@ +endif # LLAMA_VULKAN + +ifdef LLAMA_HIPBLAS + ifeq ($(wildcard /opt/rocm),) + ROCM_PATH ?= /usr + GPU_TARGETS ?= $(shell $(shell which amdgpu-arch)) + else + ROCM_PATH ?= /opt/rocm + GPU_TARGETS ?= $(shell $(ROCM_PATH)/llvm/bin/amdgpu-arch) + endif + HIPCC ?= $(CCACHE) $(ROCM_PATH)/bin/hipcc + LLAMA_CUDA_DMMV_X ?= 32 + LLAMA_CUDA_MMV_Y ?= 1 + LLAMA_CUDA_KQUANTS_ITER ?= 2 + MK_CPPFLAGS += -DGGML_USE_HIPBLAS -DGGML_USE_CUDA +ifdef LLAMA_HIP_UMA + MK_CPPFLAGS += -DGGML_HIP_UMA +endif # LLAMA_HIP_UMA + MK_LDFLAGS += -L$(ROCM_PATH)/lib -Wl,-rpath=$(ROCM_PATH)/lib + MK_LDFLAGS += -lhipblas -lamdhip64 -lrocblas + HIPFLAGS += $(addprefix --offload-arch=,$(GPU_TARGETS)) + HIPFLAGS += -DGGML_CUDA_DMMV_X=$(LLAMA_CUDA_DMMV_X) + HIPFLAGS += -DGGML_CUDA_MMV_Y=$(LLAMA_CUDA_MMV_Y) + HIPFLAGS += -DK_QUANTS_PER_ITERATION=$(LLAMA_CUDA_KQUANTS_ITER) +ifdef LLAMA_CUDA_FORCE_DMMV + HIPFLAGS += -DGGML_CUDA_FORCE_DMMV +endif # LLAMA_CUDA_FORCE_DMMV +ifdef LLAMA_CUDA_NO_PEER_COPY + HIPFLAGS += -DGGML_CUDA_NO_PEER_COPY +endif # LLAMA_CUDA_NO_PEER_COPY + OBJS += ggml-cuda.o + OBJS += $(patsubst %.cu,%.o,$(wildcard ggml-cuda/*.cu)) + +ggml-cuda.o: ggml-cuda.cu ggml-cuda.h ggml.h ggml-backend.h ggml-backend-impl.h ggml-common.h $(wildcard ggml-cuda/*.cuh) + $(HIPCC) $(CXXFLAGS) $(HIPFLAGS) -x hip -c -o $@ $< + +ggml-cuda/%.o: ggml-cuda/%.cu ggml-cuda/%.cuh ggml.h ggml-common.h ggml-cuda/common.cuh + $(HIPCC) $(CXXFLAGS) $(HIPFLAGS) -x hip -c -o $@ $< + +endif # LLAMA_HIPBLAS + +ifdef LLAMA_METAL + MK_CPPFLAGS += -DGGML_USE_METAL + MK_LDFLAGS += -framework Foundation -framework Metal -framework MetalKit + OBJS += ggml-metal.o +ifdef LLAMA_METAL_NDEBUG + MK_CPPFLAGS += -DGGML_METAL_NDEBUG +endif +ifdef LLAMA_METAL_EMBED_LIBRARY + MK_CPPFLAGS += -DGGML_METAL_EMBED_LIBRARY + OBJS += ggml-metal-embed.o +endif +endif # LLAMA_METAL + +ifdef LLAMA_METAL +ggml-metal.o: ggml-metal.m ggml-metal.h ggml.h + $(CC) $(CFLAGS) -c $< -o $@ + +ifdef LLAMA_METAL_EMBED_LIBRARY +ggml-metal-embed.o: ggml-metal.metal ggml-common.h + @echo "Embedding Metal library" + @sed -e '/#include "ggml-common.h"/r ggml-common.h' -e '/#include "ggml-common.h"/d' < ggml-metal.metal > ggml-metal-embed.metal + $(eval TEMP_ASSEMBLY=$(shell mktemp)) + @echo ".section __DATA, __ggml_metallib" > $(TEMP_ASSEMBLY) + @echo ".globl _ggml_metallib_start" >> $(TEMP_ASSEMBLY) + @echo "_ggml_metallib_start:" >> $(TEMP_ASSEMBLY) + @echo ".incbin \"ggml-metal-embed.metal\"" >> $(TEMP_ASSEMBLY) + @echo ".globl _ggml_metallib_end" >> $(TEMP_ASSEMBLY) + @echo "_ggml_metallib_end:" >> $(TEMP_ASSEMBLY) + @$(AS) $(TEMP_ASSEMBLY) -o $@ + @rm -f ${TEMP_ASSEMBLY} +endif +endif # LLAMA_METAL + +ifdef LLAMA_MPI +ggml-mpi.o: ggml-mpi.c ggml-mpi.h + $(CC) $(CFLAGS) -c $< -o $@ +endif # LLAMA_MPI + +GF_CC := $(CC) +include scripts/get-flags.mk + +# combine build flags with cmdline overrides +override CPPFLAGS := $(MK_CPPFLAGS) $(CPPFLAGS) +override CFLAGS := $(CPPFLAGS) $(MK_CFLAGS) $(GF_CFLAGS) $(CFLAGS) +BASE_CXXFLAGS := $(MK_CXXFLAGS) $(CXXFLAGS) +override CXXFLAGS := $(BASE_CXXFLAGS) $(HOST_CXXFLAGS) $(GF_CXXFLAGS) $(CPPFLAGS) +override NVCCFLAGS := $(MK_NVCCFLAGS) $(NVCCFLAGS) +override LDFLAGS := $(MK_LDFLAGS) $(LDFLAGS) + +# identify CUDA host compiler +ifdef LLAMA_CUDA +GF_CC := $(NVCC) $(NVCCFLAGS) 2>/dev/null .c -Xcompiler +include scripts/get-flags.mk +CUDA_CXXFLAGS := $(BASE_CXXFLAGS) $(GF_CXXFLAGS) -Wno-pedantic +endif + +ifdef LLAMA_CURL +override CXXFLAGS := $(CXXFLAGS) -DLLAMA_USE_CURL +override LDFLAGS := $(LDFLAGS) -lcurl +endif + +# +# Print build information +# + +$(info I llama.cpp build info: ) +$(info I UNAME_S: $(UNAME_S)) +$(info I UNAME_P: $(UNAME_P)) +$(info I UNAME_M: $(UNAME_M)) +$(info I CFLAGS: $(CFLAGS)) +$(info I CXXFLAGS: $(CXXFLAGS)) +$(info I NVCCFLAGS: $(NVCCFLAGS)) +$(info I LDFLAGS: $(LDFLAGS)) +$(info I CC: $(shell $(CC) --version | head -n 1)) +$(info I CXX: $(shell $(CXX) --version | head -n 1)) +ifdef LLAMA_CUDA +$(info I NVCC: $(shell $(NVCC) --version | tail -n 1)) +CUDA_VERSION := $(shell $(NVCC) --version | grep -oP 'release (\K[0-9]+\.[0-9])') +ifeq ($(shell awk -v "v=$(CUDA_VERSION)" 'BEGIN { print (v < 11.7) }'),1) +ifndef CUDA_DOCKER_ARCH +ifndef CUDA_POWER_ARCH +$(error I ERROR: For CUDA versions < 11.7 a target CUDA architecture must be explicitly provided via CUDA_DOCKER_ARCH) +endif # CUDA_POWER_ARCH +endif # CUDA_DOCKER_ARCH +endif # eq ($(shell echo "$(CUDA_VERSION) < 11.7" | bc),1) +endif # LLAMA_CUDA +$(info ) + +ifdef LLAMA_CUBLAS +$(info !!!!) +$(info LLAMA_CUBLAS is deprecated and will be removed in the future. Use LLAMA_CUDA instead.) +$(info !!!!) +$(info ) +endif + +# +# Build library +# + +ggml.o: ggml.c ggml.h ggml-cuda.h + $(CC) $(CFLAGS) -c $< -o $@ + +ggml-alloc.o: ggml-alloc.c ggml.h ggml-alloc.h + $(CC) $(CFLAGS) -c $< -o $@ + +ggml-backend.o: ggml-backend.c ggml.h ggml-backend.h + $(CC) $(CFLAGS) -c $< -o $@ + +ggml-quants.o: ggml-quants.c ggml.h ggml-quants.h ggml-common.h + $(CC) $(CFLAGS) -c $< -o $@ + +unicode.o: unicode.cpp unicode.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + +unicode-data.o: unicode-data.cpp unicode-data.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + +OBJS += ggml-alloc.o ggml-backend.o ggml-quants.o unicode.o unicode-data.o + +llama.o: llama.cpp unicode.h ggml.h ggml-alloc.h ggml-backend.h ggml-cuda.h ggml-metal.h llama.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + +COMMON_H_DEPS = common/common.h common/sampling.h common/log.h +COMMON_DEPS = common.o sampling.o grammar-parser.o build-info.o + +common.o: common/common.cpp $(COMMON_H_DEPS) + $(CXX) $(CXXFLAGS) -c $< -o $@ + +sampling.o: common/sampling.cpp $(COMMON_H_DEPS) + $(CXX) $(CXXFLAGS) -c $< -o $@ + +console.o: common/console.cpp common/console.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + +grammar-parser.o: common/grammar-parser.cpp common/grammar-parser.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + +json-schema-to-grammar.o: common/json-schema-to-grammar.cpp common/json-schema-to-grammar.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + +train.o: common/train.cpp common/train.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + +ngram-cache.o: common/ngram-cache.cpp common/ngram-cache.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + +libllama.so: llama.o ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -shared -fPIC -o $@ $^ $(LDFLAGS) + +libllama.a: llama.o ggml.o $(OBJS) $(COMMON_DEPS) + ar rcs libllama.a llama.o ggml.o $(OBJS) $(COMMON_DEPS) -# Default target executed when no arguments are given to make. -default_target: all -.PHONY : default_target - -# Allow only one "make -f Makefile2" at a time, but pass parallelism. -.NOTPARALLEL: - -#============================================================================= -# Special targets provided by cmake. - -# Disable implicit rules so canonical targets will work. -.SUFFIXES: - -# Disable VCS-based implicit rules. -% : %,v - -# Disable VCS-based implicit rules. -% : RCS/% - -# Disable VCS-based implicit rules. -% : RCS/%,v - -# Disable VCS-based implicit rules. -% : SCCS/s.% - -# Disable VCS-based implicit rules. -% : s.% - -.SUFFIXES: .hpux_make_needs_suffix_list - -# Command-line flag to silence nested $(MAKE). -$(VERBOSE)MAKESILENT = -s - -#Suppress display of executed commands. -$(VERBOSE).SILENT: - -# A target that is always out of date. -cmake_force: -.PHONY : cmake_force - -#============================================================================= -# Set environment variables for the build. - -# The shell in which to execute make rules. -SHELL = /bin/sh - -# The CMake executable. -CMAKE_COMMAND = /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake - -# The command to remove a file. -RM = /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -E rm -f - -# Escaping for special characters. -EQUALS = = - -# The top-level source directory on which CMake was run. -CMAKE_SOURCE_DIR = /home/joan/workspace/llama.cpp - -# The top-level build directory on which CMake was run. -CMAKE_BINARY_DIR = /home/joan/workspace/llama.cpp - -#============================================================================= -# Targets provided globally by CMake. - -# Special rule for the target test -test: - @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Running tests..." - /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/ctest --force-new-ctest-process $(ARGS) -.PHONY : test - -# Special rule for the target test -test/fast: test -.PHONY : test/fast - -# Special rule for the target edit_cache -edit_cache: - @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "No interactive CMake dialog available..." - /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -E echo No\ interactive\ CMake\ dialog\ available. -.PHONY : edit_cache - -# Special rule for the target edit_cache -edit_cache/fast: edit_cache -.PHONY : edit_cache/fast - -# Special rule for the target rebuild_cache -rebuild_cache: - @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Running CMake to regenerate build system..." - /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake --regenerate-during-build -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) -.PHONY : rebuild_cache +clean: + rm -vrf *.o tests/*.o *.so *.a *.dll benchmark-matmult lookup-create lookup-merge lookup-stats common/build-info.cpp *.dot $(COV_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) + rm -vrf ggml-cuda/*.o + find examples pocs -type f -name "*.o" -delete + +# +# Examples +# + +# $< is the first prerequisite, i.e. the source file. +# Explicitly compile this to an object file so that it can be cached with ccache. +# The source file is then filtered out from $^ (the list of all prerequisites) and the object file is added instead. + +# Helper function that replaces .c, .cpp, and .cu file endings with .o: +GET_OBJ_FILE = $(patsubst %.c,%.o,$(patsubst %.cpp,%.o,$(patsubst %.cu,%.o,$(1)))) -# Special rule for the target rebuild_cache -rebuild_cache/fast: rebuild_cache -.PHONY : rebuild_cache/fast +main: examples/main/main.cpp ggml.o llama.o $(COMMON_DEPS) console.o grammar-parser.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + @echo + @echo '==== Run ./main -h for help. ====' + @echo + +infill: examples/infill/infill.cpp ggml.o llama.o $(COMMON_DEPS) console.o grammar-parser.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +simple: examples/simple/simple.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tokenize: examples/tokenize/tokenize.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +batched: examples/batched/batched.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +batched-bench: examples/batched-bench/batched-bench.cpp build-info.o ggml.o llama.o common.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +quantize: examples/quantize/quantize.cpp build-info.o ggml.o llama.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +quantize-stats: examples/quantize-stats/quantize-stats.cpp build-info.o ggml.o llama.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +perplexity: examples/perplexity/perplexity.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +imatrix: examples/imatrix/imatrix.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +embedding: examples/embedding/embedding.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +gritlm: examples/gritlm/gritlm.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +server: examples/server/server.cpp examples/server/utils.hpp examples/server/httplib.h common/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp json-schema-to-grammar.o common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h %.hpp $<,$^) -Iexamples/server $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) $(LWINSOCK2) + +gguf: examples/gguf/gguf.cpp ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +gguf-split: examples/gguf-split/gguf-split.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +convert-llama2c-to-ggml: examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +llama-bench: examples/llama-bench/llama-bench.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +libllava.a: examples/llava/llava.cpp examples/llava/llava.h examples/llava/clip.cpp examples/llava/clip.h common/stb_image.h common/base64.hpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -static -fPIC -c $< -o $@ -Wno-cast-qual + +llava-cli: examples/llava/llava-cli.cpp examples/llava/clip.h examples/llava/clip.cpp examples/llava/llava.h examples/llava/llava.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) -c examples/llava/clip.cpp -o $(call GET_OBJ_FILE, examples/llava/clip.cpp) -Wno-cast-qual + $(CXX) $(CXXFLAGS) -c examples/llava/llava.cpp -o $(call GET_OBJ_FILE, examples/llava/llava.cpp) + $(CXX) $(CXXFLAGS) $(filter-out %.h $< examples/llava/clip.cpp examples/llava/llava.cpp,$^) $(call GET_OBJ_FILE, $<) $(call GET_OBJ_FILE, examples/llava/clip.cpp) $(call GET_OBJ_FILE, examples/llava/llava.cpp) -o $@ $(LDFLAGS) + +baby-llama: examples/baby-llama/baby-llama.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +beam-search: examples/beam-search/beam-search.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +finetune: examples/finetune/finetune.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +export-lora: examples/export-lora/export-lora.cpp ggml.o common/common.h $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +retrieval: examples/retrieval/retrieval.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +speculative: examples/speculative/speculative.cpp ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +parallel: examples/parallel/parallel.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +lookahead: examples/lookahead/lookahead.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +lookup: examples/lookup/lookup.cpp ggml.o llama.o ngram-cache.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c examples/lookup/lookup-create.cpp -o $(call GET_OBJ_FILE, examples/lookup/lookup-create.cpp) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, examples/lookup/lookup-create.cpp) -o lookup-create $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c examples/lookup/lookup-merge.cpp -o $(call GET_OBJ_FILE, examples/lookup/lookup-merge.cpp) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, examples/lookup/lookup-merge.cpp) -o lookup-merge $(LDFLAGS) + $(CXX) $(CXXFLAGS) -c examples/lookup/lookup-stats.cpp -o $(call GET_OBJ_FILE, examples/lookup/lookup-stats.cpp) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, examples/lookup/lookup-stats.cpp) -o lookup-stats $(LDFLAGS) + +passkey: examples/passkey/passkey.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +gbnf-validator: examples/gbnf-validator/gbnf-validator.cpp ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +ifeq ($(UNAME_S),Darwin) +swift: examples/batched.swift + (cd examples/batched.swift; make build) +endif + +common/build-info.cpp: $(wildcard .git/index) scripts/build-info.sh + @sh scripts/build-info.sh "$(CC)" > $@.tmp + @if ! cmp -s $@.tmp $@; then \ + mv $@.tmp $@; \ + else \ + rm $@.tmp; \ + fi + +build-info.o: common/build-info.cpp + $(CXX) $(CXXFLAGS) -c $(filter-out %.h,$^) -o $@ + +# +# Tests +# + +tests: $(TEST_TARGETS) + +benchmark-matmult: examples/benchmark/benchmark-matmult.cpp build-info.o ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +run-benchmark-matmult: benchmark-matmult + ./$@ + +.PHONY: run-benchmark-matmult swift + +vdot: pocs/vdot/vdot.cpp ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +q8dot: pocs/vdot/q8dot.cpp ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-llama-grammar: tests/test-llama-grammar.cpp ggml.o grammar-parser.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-grammar-parser: tests/test-grammar-parser.cpp ggml.o llama.o grammar-parser.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-grammar-integration: tests/test-grammar-integration.cpp ggml.o llama.o grammar-parser.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-double-float: tests/test-double-float.cpp ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-json-schema-to-grammar: tests/test-json-schema-to-grammar.cpp json-schema-to-grammar.o ggml.o llama.o grammar-parser.o $(OBJS) + $(CXX) $(CXXFLAGS) -Iexamples/server -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-grad0: tests/test-grad0.cpp ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-opt: tests/test-opt.cpp ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-quantize-fns: tests/test-quantize-fns.cpp ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-quantize-perf: tests/test-quantize-perf.cpp ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-sampling: tests/test-sampling.cpp ggml.o llama.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-tokenizer-0-falcon: tests/test-tokenizer-0-falcon.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + +tests/test-tokenizer-0-llama: tests/test-tokenizer-0-llama.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -# Special rule for the target list_install_components -list_install_components: - @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Available install components are: \"Unspecified\"" -.PHONY : list_install_components +tests/test-tokenizer-1-bpe: tests/test-tokenizer-1-bpe.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -# Special rule for the target list_install_components -list_install_components/fast: list_install_components -.PHONY : list_install_components/fast +tests/test-tokenizer-1-llama: tests/test-tokenizer-1-llama.cpp ggml.o llama.o $(COMMON_DEPS) console.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -# Special rule for the target install -install: preinstall - @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Install the project..." - /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -P cmake_install.cmake -.PHONY : install +tests/test-rope: tests/test-rope.cpp ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -# Special rule for the target install -install/fast: preinstall/fast - @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Install the project..." - /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -P cmake_install.cmake -.PHONY : install/fast - -# Special rule for the target install/local -install/local: preinstall - @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing only the local directory..." - /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake -.PHONY : install/local - -# Special rule for the target install/local -install/local/fast: preinstall/fast - @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing only the local directory..." - /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake -.PHONY : install/local/fast - -# Special rule for the target install/strip -install/strip: preinstall - @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing the project stripped..." - /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake -.PHONY : install/strip - -# Special rule for the target install/strip -install/strip/fast: preinstall/fast - @$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing the project stripped..." - /home/joan/jina/gateway-api-server/venv/lib/python3.8/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake -.PHONY : install/strip/fast - -# The main all target -all: cmake_check_build_system - $(CMAKE_COMMAND) -E cmake_progress_start /home/joan/workspace/llama.cpp/CMakeFiles /home/joan/workspace/llama.cpp//CMakeFiles/progress.marks - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 all - $(CMAKE_COMMAND) -E cmake_progress_start /home/joan/workspace/llama.cpp/CMakeFiles 0 -.PHONY : all - -# The main clean target -clean: - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 clean -.PHONY : clean - -# The main clean target -clean/fast: clean -.PHONY : clean/fast - -# Prepare targets for installation. -preinstall: all - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 preinstall -.PHONY : preinstall - -# Prepare targets for installation. -preinstall/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 preinstall -.PHONY : preinstall/fast - -# clear depends -depend: - $(CMAKE_COMMAND) -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 1 -.PHONY : depend - -#============================================================================= -# Target rules for targets named ggml - -# Build rule for target. -ggml: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ggml -.PHONY : ggml - -# fast build rule for target. -ggml/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/build -.PHONY : ggml/fast - -#============================================================================= -# Target rules for targets named ggml_static - -# Build rule for target. -ggml_static: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ggml_static -.PHONY : ggml_static - -# fast build rule for target. -ggml_static/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml_static.dir/build.make CMakeFiles/ggml_static.dir/build -.PHONY : ggml_static/fast - -#============================================================================= -# Target rules for targets named llama - -# Build rule for target. -llama: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 llama -.PHONY : llama - -# fast build rule for target. -llama/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/build -.PHONY : llama/fast - -#============================================================================= -# Target rules for targets named Experimental - -# Build rule for target. -Experimental: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 Experimental -.PHONY : Experimental - -# fast build rule for target. -Experimental/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/Experimental.dir/build.make CMakeFiles/Experimental.dir/build -.PHONY : Experimental/fast - -#============================================================================= -# Target rules for targets named Nightly - -# Build rule for target. -Nightly: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 Nightly -.PHONY : Nightly - -# fast build rule for target. -Nightly/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/Nightly.dir/build.make CMakeFiles/Nightly.dir/build -.PHONY : Nightly/fast - -#============================================================================= -# Target rules for targets named Continuous - -# Build rule for target. -Continuous: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 Continuous -.PHONY : Continuous - -# fast build rule for target. -Continuous/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/Continuous.dir/build.make CMakeFiles/Continuous.dir/build -.PHONY : Continuous/fast - -#============================================================================= -# Target rules for targets named NightlyMemoryCheck - -# Build rule for target. -NightlyMemoryCheck: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyMemoryCheck -.PHONY : NightlyMemoryCheck - -# fast build rule for target. -NightlyMemoryCheck/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyMemoryCheck.dir/build.make CMakeFiles/NightlyMemoryCheck.dir/build -.PHONY : NightlyMemoryCheck/fast - -#============================================================================= -# Target rules for targets named NightlyStart - -# Build rule for target. -NightlyStart: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyStart -.PHONY : NightlyStart - -# fast build rule for target. -NightlyStart/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyStart.dir/build.make CMakeFiles/NightlyStart.dir/build -.PHONY : NightlyStart/fast - -#============================================================================= -# Target rules for targets named NightlyUpdate - -# Build rule for target. -NightlyUpdate: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyUpdate -.PHONY : NightlyUpdate - -# fast build rule for target. -NightlyUpdate/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyUpdate.dir/build.make CMakeFiles/NightlyUpdate.dir/build -.PHONY : NightlyUpdate/fast - -#============================================================================= -# Target rules for targets named NightlyConfigure - -# Build rule for target. -NightlyConfigure: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyConfigure -.PHONY : NightlyConfigure - -# fast build rule for target. -NightlyConfigure/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyConfigure.dir/build.make CMakeFiles/NightlyConfigure.dir/build -.PHONY : NightlyConfigure/fast - -#============================================================================= -# Target rules for targets named NightlyBuild - -# Build rule for target. -NightlyBuild: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyBuild -.PHONY : NightlyBuild - -# fast build rule for target. -NightlyBuild/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyBuild.dir/build.make CMakeFiles/NightlyBuild.dir/build -.PHONY : NightlyBuild/fast - -#============================================================================= -# Target rules for targets named NightlyTest - -# Build rule for target. -NightlyTest: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyTest -.PHONY : NightlyTest - -# fast build rule for target. -NightlyTest/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyTest.dir/build.make CMakeFiles/NightlyTest.dir/build -.PHONY : NightlyTest/fast - -#============================================================================= -# Target rules for targets named NightlyCoverage - -# Build rule for target. -NightlyCoverage: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyCoverage -.PHONY : NightlyCoverage - -# fast build rule for target. -NightlyCoverage/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyCoverage.dir/build.make CMakeFiles/NightlyCoverage.dir/build -.PHONY : NightlyCoverage/fast - -#============================================================================= -# Target rules for targets named NightlyMemCheck - -# Build rule for target. -NightlyMemCheck: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlyMemCheck -.PHONY : NightlyMemCheck - -# fast build rule for target. -NightlyMemCheck/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlyMemCheck.dir/build.make CMakeFiles/NightlyMemCheck.dir/build -.PHONY : NightlyMemCheck/fast - -#============================================================================= -# Target rules for targets named NightlySubmit - -# Build rule for target. -NightlySubmit: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 NightlySubmit -.PHONY : NightlySubmit - -# fast build rule for target. -NightlySubmit/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/NightlySubmit.dir/build.make CMakeFiles/NightlySubmit.dir/build -.PHONY : NightlySubmit/fast - -#============================================================================= -# Target rules for targets named ExperimentalStart - -# Build rule for target. -ExperimentalStart: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalStart -.PHONY : ExperimentalStart - -# fast build rule for target. -ExperimentalStart/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalStart.dir/build.make CMakeFiles/ExperimentalStart.dir/build -.PHONY : ExperimentalStart/fast - -#============================================================================= -# Target rules for targets named ExperimentalUpdate - -# Build rule for target. -ExperimentalUpdate: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalUpdate -.PHONY : ExperimentalUpdate - -# fast build rule for target. -ExperimentalUpdate/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalUpdate.dir/build.make CMakeFiles/ExperimentalUpdate.dir/build -.PHONY : ExperimentalUpdate/fast - -#============================================================================= -# Target rules for targets named ExperimentalConfigure - -# Build rule for target. -ExperimentalConfigure: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalConfigure -.PHONY : ExperimentalConfigure - -# fast build rule for target. -ExperimentalConfigure/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalConfigure.dir/build.make CMakeFiles/ExperimentalConfigure.dir/build -.PHONY : ExperimentalConfigure/fast - -#============================================================================= -# Target rules for targets named ExperimentalBuild - -# Build rule for target. -ExperimentalBuild: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalBuild -.PHONY : ExperimentalBuild - -# fast build rule for target. -ExperimentalBuild/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalBuild.dir/build.make CMakeFiles/ExperimentalBuild.dir/build -.PHONY : ExperimentalBuild/fast - -#============================================================================= -# Target rules for targets named ExperimentalTest - -# Build rule for target. -ExperimentalTest: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalTest -.PHONY : ExperimentalTest - -# fast build rule for target. -ExperimentalTest/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalTest.dir/build.make CMakeFiles/ExperimentalTest.dir/build -.PHONY : ExperimentalTest/fast - -#============================================================================= -# Target rules for targets named ExperimentalCoverage - -# Build rule for target. -ExperimentalCoverage: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalCoverage -.PHONY : ExperimentalCoverage - -# fast build rule for target. -ExperimentalCoverage/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalCoverage.dir/build.make CMakeFiles/ExperimentalCoverage.dir/build -.PHONY : ExperimentalCoverage/fast - -#============================================================================= -# Target rules for targets named ExperimentalMemCheck - -# Build rule for target. -ExperimentalMemCheck: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalMemCheck -.PHONY : ExperimentalMemCheck - -# fast build rule for target. -ExperimentalMemCheck/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalMemCheck.dir/build.make CMakeFiles/ExperimentalMemCheck.dir/build -.PHONY : ExperimentalMemCheck/fast - -#============================================================================= -# Target rules for targets named ExperimentalSubmit - -# Build rule for target. -ExperimentalSubmit: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ExperimentalSubmit -.PHONY : ExperimentalSubmit - -# fast build rule for target. -ExperimentalSubmit/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ExperimentalSubmit.dir/build.make CMakeFiles/ExperimentalSubmit.dir/build -.PHONY : ExperimentalSubmit/fast - -#============================================================================= -# Target rules for targets named ContinuousStart - -# Build rule for target. -ContinuousStart: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousStart -.PHONY : ContinuousStart - -# fast build rule for target. -ContinuousStart/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousStart.dir/build.make CMakeFiles/ContinuousStart.dir/build -.PHONY : ContinuousStart/fast - -#============================================================================= -# Target rules for targets named ContinuousUpdate - -# Build rule for target. -ContinuousUpdate: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousUpdate -.PHONY : ContinuousUpdate - -# fast build rule for target. -ContinuousUpdate/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousUpdate.dir/build.make CMakeFiles/ContinuousUpdate.dir/build -.PHONY : ContinuousUpdate/fast - -#============================================================================= -# Target rules for targets named ContinuousConfigure - -# Build rule for target. -ContinuousConfigure: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousConfigure -.PHONY : ContinuousConfigure - -# fast build rule for target. -ContinuousConfigure/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousConfigure.dir/build.make CMakeFiles/ContinuousConfigure.dir/build -.PHONY : ContinuousConfigure/fast - -#============================================================================= -# Target rules for targets named ContinuousBuild - -# Build rule for target. -ContinuousBuild: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousBuild -.PHONY : ContinuousBuild - -# fast build rule for target. -ContinuousBuild/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousBuild.dir/build.make CMakeFiles/ContinuousBuild.dir/build -.PHONY : ContinuousBuild/fast - -#============================================================================= -# Target rules for targets named ContinuousTest - -# Build rule for target. -ContinuousTest: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousTest -.PHONY : ContinuousTest - -# fast build rule for target. -ContinuousTest/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousTest.dir/build.make CMakeFiles/ContinuousTest.dir/build -.PHONY : ContinuousTest/fast - -#============================================================================= -# Target rules for targets named ContinuousCoverage - -# Build rule for target. -ContinuousCoverage: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousCoverage -.PHONY : ContinuousCoverage - -# fast build rule for target. -ContinuousCoverage/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousCoverage.dir/build.make CMakeFiles/ContinuousCoverage.dir/build -.PHONY : ContinuousCoverage/fast - -#============================================================================= -# Target rules for targets named ContinuousMemCheck - -# Build rule for target. -ContinuousMemCheck: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousMemCheck -.PHONY : ContinuousMemCheck - -# fast build rule for target. -ContinuousMemCheck/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousMemCheck.dir/build.make CMakeFiles/ContinuousMemCheck.dir/build -.PHONY : ContinuousMemCheck/fast - -#============================================================================= -# Target rules for targets named ContinuousSubmit - -# Build rule for target. -ContinuousSubmit: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 ContinuousSubmit -.PHONY : ContinuousSubmit - -# fast build rule for target. -ContinuousSubmit/fast: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ContinuousSubmit.dir/build.make CMakeFiles/ContinuousSubmit.dir/build -.PHONY : ContinuousSubmit/fast - -#============================================================================= -# Target rules for targets named build_info - -# Build rule for target. -build_info: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 build_info -.PHONY : build_info - -# fast build rule for target. -build_info/fast: - $(MAKE) $(MAKESILENT) -f common/CMakeFiles/build_info.dir/build.make common/CMakeFiles/build_info.dir/build -.PHONY : build_info/fast - -#============================================================================= -# Target rules for targets named json-schema-to-grammar - -# Build rule for target. -json-schema-to-grammar: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 json-schema-to-grammar -.PHONY : json-schema-to-grammar - -# fast build rule for target. -json-schema-to-grammar/fast: - $(MAKE) $(MAKESILENT) -f common/CMakeFiles/json-schema-to-grammar.dir/build.make common/CMakeFiles/json-schema-to-grammar.dir/build -.PHONY : json-schema-to-grammar/fast - -#============================================================================= -# Target rules for targets named common - -# Build rule for target. -common: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 common -.PHONY : common - -# fast build rule for target. -common/fast: - $(MAKE) $(MAKESILENT) -f common/CMakeFiles/common.dir/build.make common/CMakeFiles/common.dir/build -.PHONY : common/fast - -#============================================================================= -# Target rules for targets named test-quantize-fns - -# Build rule for target. -test-quantize-fns: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-quantize-fns -.PHONY : test-quantize-fns - -# fast build rule for target. -test-quantize-fns/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-quantize-fns.dir/build.make tests/CMakeFiles/test-quantize-fns.dir/build -.PHONY : test-quantize-fns/fast - -#============================================================================= -# Target rules for targets named test-quantize-perf - -# Build rule for target. -test-quantize-perf: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-quantize-perf -.PHONY : test-quantize-perf - -# fast build rule for target. -test-quantize-perf/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-quantize-perf.dir/build.make tests/CMakeFiles/test-quantize-perf.dir/build -.PHONY : test-quantize-perf/fast - -#============================================================================= -# Target rules for targets named test-sampling - -# Build rule for target. -test-sampling: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-sampling -.PHONY : test-sampling - -# fast build rule for target. -test-sampling/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-sampling.dir/build.make tests/CMakeFiles/test-sampling.dir/build -.PHONY : test-sampling/fast - -#============================================================================= -# Target rules for targets named test-chat-template - -# Build rule for target. -test-chat-template: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-chat-template -.PHONY : test-chat-template - -# fast build rule for target. -test-chat-template/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-chat-template.dir/build.make tests/CMakeFiles/test-chat-template.dir/build -.PHONY : test-chat-template/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-0-llama - -# Build rule for target. -test-tokenizer-0-llama: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-0-llama -.PHONY : test-tokenizer-0-llama - -# fast build rule for target. -test-tokenizer-0-llama/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-0-llama.dir/build.make tests/CMakeFiles/test-tokenizer-0-llama.dir/build -.PHONY : test-tokenizer-0-llama/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-0-falcon - -# Build rule for target. -test-tokenizer-0-falcon: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-0-falcon -.PHONY : test-tokenizer-0-falcon - -# fast build rule for target. -test-tokenizer-0-falcon/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-0-falcon.dir/build.make tests/CMakeFiles/test-tokenizer-0-falcon.dir/build -.PHONY : test-tokenizer-0-falcon/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-1-llama - -# Build rule for target. -test-tokenizer-1-llama: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-llama -.PHONY : test-tokenizer-1-llama - -# fast build rule for target. -test-tokenizer-1-llama/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-llama.dir/build.make tests/CMakeFiles/test-tokenizer-1-llama.dir/build -.PHONY : test-tokenizer-1-llama/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-1-baichuan - -# Build rule for target. -test-tokenizer-1-baichuan: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-baichuan -.PHONY : test-tokenizer-1-baichuan - -# fast build rule for target. -test-tokenizer-1-baichuan/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-baichuan.dir/build.make tests/CMakeFiles/test-tokenizer-1-baichuan.dir/build -.PHONY : test-tokenizer-1-baichuan/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-1-falcon - -# Build rule for target. -test-tokenizer-1-falcon: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-falcon -.PHONY : test-tokenizer-1-falcon - -# fast build rule for target. -test-tokenizer-1-falcon/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-falcon.dir/build.make tests/CMakeFiles/test-tokenizer-1-falcon.dir/build -.PHONY : test-tokenizer-1-falcon/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-1-aquila - -# Build rule for target. -test-tokenizer-1-aquila: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-aquila -.PHONY : test-tokenizer-1-aquila - -# fast build rule for target. -test-tokenizer-1-aquila/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-aquila.dir/build.make tests/CMakeFiles/test-tokenizer-1-aquila.dir/build -.PHONY : test-tokenizer-1-aquila/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-1-mpt - -# Build rule for target. -test-tokenizer-1-mpt: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-mpt -.PHONY : test-tokenizer-1-mpt - -# fast build rule for target. -test-tokenizer-1-mpt/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-mpt.dir/build.make tests/CMakeFiles/test-tokenizer-1-mpt.dir/build -.PHONY : test-tokenizer-1-mpt/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-1-stablelm-3b-4e1t - -# Build rule for target. -test-tokenizer-1-stablelm-3b-4e1t: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-stablelm-3b-4e1t -.PHONY : test-tokenizer-1-stablelm-3b-4e1t - -# fast build rule for target. -test-tokenizer-1-stablelm-3b-4e1t/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-stablelm-3b-4e1t.dir/build.make tests/CMakeFiles/test-tokenizer-1-stablelm-3b-4e1t.dir/build -.PHONY : test-tokenizer-1-stablelm-3b-4e1t/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-1-gpt-neox - -# Build rule for target. -test-tokenizer-1-gpt-neox: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-gpt-neox -.PHONY : test-tokenizer-1-gpt-neox - -# fast build rule for target. -test-tokenizer-1-gpt-neox/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-gpt-neox.dir/build.make tests/CMakeFiles/test-tokenizer-1-gpt-neox.dir/build -.PHONY : test-tokenizer-1-gpt-neox/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-1-refact - -# Build rule for target. -test-tokenizer-1-refact: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-refact -.PHONY : test-tokenizer-1-refact - -# fast build rule for target. -test-tokenizer-1-refact/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-refact.dir/build.make tests/CMakeFiles/test-tokenizer-1-refact.dir/build -.PHONY : test-tokenizer-1-refact/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-1-starcoder - -# Build rule for target. -test-tokenizer-1-starcoder: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-starcoder -.PHONY : test-tokenizer-1-starcoder - -# fast build rule for target. -test-tokenizer-1-starcoder/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-starcoder.dir/build.make tests/CMakeFiles/test-tokenizer-1-starcoder.dir/build -.PHONY : test-tokenizer-1-starcoder/fast - -#============================================================================= -# Target rules for targets named test-tokenizer-1-gpt2 - -# Build rule for target. -test-tokenizer-1-gpt2: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-tokenizer-1-gpt2 -.PHONY : test-tokenizer-1-gpt2 - -# fast build rule for target. -test-tokenizer-1-gpt2/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-tokenizer-1-gpt2.dir/build.make tests/CMakeFiles/test-tokenizer-1-gpt2.dir/build -.PHONY : test-tokenizer-1-gpt2/fast - -#============================================================================= -# Target rules for targets named test-grammar-parser - -# Build rule for target. -test-grammar-parser: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-grammar-parser -.PHONY : test-grammar-parser - -# fast build rule for target. -test-grammar-parser/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-grammar-parser.dir/build.make tests/CMakeFiles/test-grammar-parser.dir/build -.PHONY : test-grammar-parser/fast - -#============================================================================= -# Target rules for targets named test-llama-grammar - -# Build rule for target. -test-llama-grammar: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-llama-grammar -.PHONY : test-llama-grammar - -# fast build rule for target. -test-llama-grammar/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-llama-grammar.dir/build.make tests/CMakeFiles/test-llama-grammar.dir/build -.PHONY : test-llama-grammar/fast - -#============================================================================= -# Target rules for targets named test-grammar-integration - -# Build rule for target. -test-grammar-integration: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-grammar-integration -.PHONY : test-grammar-integration - -# fast build rule for target. -test-grammar-integration/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-grammar-integration.dir/build.make tests/CMakeFiles/test-grammar-integration.dir/build -.PHONY : test-grammar-integration/fast - -#============================================================================= -# Target rules for targets named test-grad0 - -# Build rule for target. -test-grad0: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-grad0 -.PHONY : test-grad0 - -# fast build rule for target. -test-grad0/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-grad0.dir/build.make tests/CMakeFiles/test-grad0.dir/build -.PHONY : test-grad0/fast - -#============================================================================= -# Target rules for targets named test-backend-ops - -# Build rule for target. -test-backend-ops: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-backend-ops -.PHONY : test-backend-ops - -# fast build rule for target. -test-backend-ops/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-backend-ops.dir/build.make tests/CMakeFiles/test-backend-ops.dir/build -.PHONY : test-backend-ops/fast - -#============================================================================= -# Target rules for targets named test-rope - -# Build rule for target. -test-rope: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-rope -.PHONY : test-rope - -# fast build rule for target. -test-rope/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-rope.dir/build.make tests/CMakeFiles/test-rope.dir/build -.PHONY : test-rope/fast - -#============================================================================= -# Target rules for targets named test-model-load-cancel - -# Build rule for target. -test-model-load-cancel: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-model-load-cancel -.PHONY : test-model-load-cancel - -# fast build rule for target. -test-model-load-cancel/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-model-load-cancel.dir/build.make tests/CMakeFiles/test-model-load-cancel.dir/build -.PHONY : test-model-load-cancel/fast - -#============================================================================= -# Target rules for targets named test-autorelease - -# Build rule for target. -test-autorelease: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-autorelease -.PHONY : test-autorelease - -# fast build rule for target. -test-autorelease/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-autorelease.dir/build.make tests/CMakeFiles/test-autorelease.dir/build -.PHONY : test-autorelease/fast - -#============================================================================= -# Target rules for targets named test-json-schema-to-grammar - -# Build rule for target. -test-json-schema-to-grammar: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-json-schema-to-grammar -.PHONY : test-json-schema-to-grammar - -# fast build rule for target. -test-json-schema-to-grammar/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-json-schema-to-grammar.dir/build.make tests/CMakeFiles/test-json-schema-to-grammar.dir/build -.PHONY : test-json-schema-to-grammar/fast - -#============================================================================= -# Target rules for targets named test-c - -# Build rule for target. -test-c: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 test-c -.PHONY : test-c - -# fast build rule for target. -test-c/fast: - $(MAKE) $(MAKESILENT) -f tests/CMakeFiles/test-c.dir/build.make tests/CMakeFiles/test-c.dir/build -.PHONY : test-c/fast - -#============================================================================= -# Target rules for targets named baby-llama - -# Build rule for target. -baby-llama: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 baby-llama -.PHONY : baby-llama - -# fast build rule for target. -baby-llama/fast: - $(MAKE) $(MAKESILENT) -f examples/baby-llama/CMakeFiles/baby-llama.dir/build.make examples/baby-llama/CMakeFiles/baby-llama.dir/build -.PHONY : baby-llama/fast - -#============================================================================= -# Target rules for targets named batched - -# Build rule for target. -batched: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 batched -.PHONY : batched - -# fast build rule for target. -batched/fast: - $(MAKE) $(MAKESILENT) -f examples/batched/CMakeFiles/batched.dir/build.make examples/batched/CMakeFiles/batched.dir/build -.PHONY : batched/fast - -#============================================================================= -# Target rules for targets named batched-bench - -# Build rule for target. -batched-bench: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 batched-bench -.PHONY : batched-bench - -# fast build rule for target. -batched-bench/fast: - $(MAKE) $(MAKESILENT) -f examples/batched-bench/CMakeFiles/batched-bench.dir/build.make examples/batched-bench/CMakeFiles/batched-bench.dir/build -.PHONY : batched-bench/fast - -#============================================================================= -# Target rules for targets named beam-search - -# Build rule for target. -beam-search: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 beam-search -.PHONY : beam-search - -# fast build rule for target. -beam-search/fast: - $(MAKE) $(MAKESILENT) -f examples/beam-search/CMakeFiles/beam-search.dir/build.make examples/beam-search/CMakeFiles/beam-search.dir/build -.PHONY : beam-search/fast - -#============================================================================= -# Target rules for targets named benchmark - -# Build rule for target. -benchmark: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 benchmark -.PHONY : benchmark - -# fast build rule for target. -benchmark/fast: - $(MAKE) $(MAKESILENT) -f examples/benchmark/CMakeFiles/benchmark.dir/build.make examples/benchmark/CMakeFiles/benchmark.dir/build -.PHONY : benchmark/fast - -#============================================================================= -# Target rules for targets named convert-llama2c-to-ggml - -# Build rule for target. -convert-llama2c-to-ggml: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 convert-llama2c-to-ggml -.PHONY : convert-llama2c-to-ggml - -# fast build rule for target. -convert-llama2c-to-ggml/fast: - $(MAKE) $(MAKESILENT) -f examples/convert-llama2c-to-ggml/CMakeFiles/convert-llama2c-to-ggml.dir/build.make examples/convert-llama2c-to-ggml/CMakeFiles/convert-llama2c-to-ggml.dir/build -.PHONY : convert-llama2c-to-ggml/fast - -#============================================================================= -# Target rules for targets named embedding - -# Build rule for target. -embedding: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 embedding -.PHONY : embedding - -# fast build rule for target. -embedding/fast: - $(MAKE) $(MAKESILENT) -f examples/embedding/CMakeFiles/embedding.dir/build.make examples/embedding/CMakeFiles/embedding.dir/build -.PHONY : embedding/fast - -#============================================================================= -# Target rules for targets named finetune - -# Build rule for target. -finetune: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 finetune -.PHONY : finetune - -# fast build rule for target. -finetune/fast: - $(MAKE) $(MAKESILENT) -f examples/finetune/CMakeFiles/finetune.dir/build.make examples/finetune/CMakeFiles/finetune.dir/build -.PHONY : finetune/fast - -#============================================================================= -# Target rules for targets named gritlm - -# Build rule for target. -gritlm: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 gritlm -.PHONY : gritlm - -# fast build rule for target. -gritlm/fast: - $(MAKE) $(MAKESILENT) -f examples/gritlm/CMakeFiles/gritlm.dir/build.make examples/gritlm/CMakeFiles/gritlm.dir/build -.PHONY : gritlm/fast - -#============================================================================= -# Target rules for targets named gguf-split - -# Build rule for target. -gguf-split: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 gguf-split -.PHONY : gguf-split - -# fast build rule for target. -gguf-split/fast: - $(MAKE) $(MAKESILENT) -f examples/gguf-split/CMakeFiles/gguf-split.dir/build.make examples/gguf-split/CMakeFiles/gguf-split.dir/build -.PHONY : gguf-split/fast - -#============================================================================= -# Target rules for targets named infill - -# Build rule for target. -infill: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 infill -.PHONY : infill - -# fast build rule for target. -infill/fast: - $(MAKE) $(MAKESILENT) -f examples/infill/CMakeFiles/infill.dir/build.make examples/infill/CMakeFiles/infill.dir/build -.PHONY : infill/fast - -#============================================================================= -# Target rules for targets named llama-bench - -# Build rule for target. -llama-bench: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 llama-bench -.PHONY : llama-bench - -# fast build rule for target. -llama-bench/fast: - $(MAKE) $(MAKESILENT) -f examples/llama-bench/CMakeFiles/llama-bench.dir/build.make examples/llama-bench/CMakeFiles/llama-bench.dir/build -.PHONY : llama-bench/fast - -#============================================================================= -# Target rules for targets named llava - -# Build rule for target. -llava: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 llava -.PHONY : llava - -# fast build rule for target. -llava/fast: - $(MAKE) $(MAKESILENT) -f examples/llava/CMakeFiles/llava.dir/build.make examples/llava/CMakeFiles/llava.dir/build -.PHONY : llava/fast - -#============================================================================= -# Target rules for targets named llava_static - -# Build rule for target. -llava_static: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 llava_static -.PHONY : llava_static - -# fast build rule for target. -llava_static/fast: - $(MAKE) $(MAKESILENT) -f examples/llava/CMakeFiles/llava_static.dir/build.make examples/llava/CMakeFiles/llava_static.dir/build -.PHONY : llava_static/fast - -#============================================================================= -# Target rules for targets named llava-cli - -# Build rule for target. -llava-cli: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 llava-cli -.PHONY : llava-cli - -# fast build rule for target. -llava-cli/fast: - $(MAKE) $(MAKESILENT) -f examples/llava/CMakeFiles/llava-cli.dir/build.make examples/llava/CMakeFiles/llava-cli.dir/build -.PHONY : llava-cli/fast - -#============================================================================= -# Target rules for targets named main - -# Build rule for target. -main: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 main -.PHONY : main - -# fast build rule for target. -main/fast: - $(MAKE) $(MAKESILENT) -f examples/main/CMakeFiles/main.dir/build.make examples/main/CMakeFiles/main.dir/build -.PHONY : main/fast - -#============================================================================= -# Target rules for targets named tokenize - -# Build rule for target. -tokenize: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 tokenize -.PHONY : tokenize - -# fast build rule for target. -tokenize/fast: - $(MAKE) $(MAKESILENT) -f examples/tokenize/CMakeFiles/tokenize.dir/build.make examples/tokenize/CMakeFiles/tokenize.dir/build -.PHONY : tokenize/fast - -#============================================================================= -# Target rules for targets named parallel - -# Build rule for target. -parallel: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 parallel -.PHONY : parallel - -# fast build rule for target. -parallel/fast: - $(MAKE) $(MAKESILENT) -f examples/parallel/CMakeFiles/parallel.dir/build.make examples/parallel/CMakeFiles/parallel.dir/build -.PHONY : parallel/fast - -#============================================================================= -# Target rules for targets named perplexity - -# Build rule for target. -perplexity: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 perplexity -.PHONY : perplexity - -# fast build rule for target. -perplexity/fast: - $(MAKE) $(MAKESILENT) -f examples/perplexity/CMakeFiles/perplexity.dir/build.make examples/perplexity/CMakeFiles/perplexity.dir/build -.PHONY : perplexity/fast - -#============================================================================= -# Target rules for targets named quantize - -# Build rule for target. -quantize: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 quantize -.PHONY : quantize - -# fast build rule for target. -quantize/fast: - $(MAKE) $(MAKESILENT) -f examples/quantize/CMakeFiles/quantize.dir/build.make examples/quantize/CMakeFiles/quantize.dir/build -.PHONY : quantize/fast - -#============================================================================= -# Target rules for targets named quantize-stats - -# Build rule for target. -quantize-stats: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 quantize-stats -.PHONY : quantize-stats - -# fast build rule for target. -quantize-stats/fast: - $(MAKE) $(MAKESILENT) -f examples/quantize-stats/CMakeFiles/quantize-stats.dir/build.make examples/quantize-stats/CMakeFiles/quantize-stats.dir/build -.PHONY : quantize-stats/fast - -#============================================================================= -# Target rules for targets named retrieval - -# Build rule for target. -retrieval: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 retrieval -.PHONY : retrieval - -# fast build rule for target. -retrieval/fast: - $(MAKE) $(MAKESILENT) -f examples/retrieval/CMakeFiles/retrieval.dir/build.make examples/retrieval/CMakeFiles/retrieval.dir/build -.PHONY : retrieval/fast - -#============================================================================= -# Target rules for targets named save-load-state - -# Build rule for target. -save-load-state: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 save-load-state -.PHONY : save-load-state - -# fast build rule for target. -save-load-state/fast: - $(MAKE) $(MAKESILENT) -f examples/save-load-state/CMakeFiles/save-load-state.dir/build.make examples/save-load-state/CMakeFiles/save-load-state.dir/build -.PHONY : save-load-state/fast - -#============================================================================= -# Target rules for targets named simple - -# Build rule for target. -simple: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 simple -.PHONY : simple - -# fast build rule for target. -simple/fast: - $(MAKE) $(MAKESILENT) -f examples/simple/CMakeFiles/simple.dir/build.make examples/simple/CMakeFiles/simple.dir/build -.PHONY : simple/fast - -#============================================================================= -# Target rules for targets named passkey - -# Build rule for target. -passkey: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 passkey -.PHONY : passkey - -# fast build rule for target. -passkey/fast: - $(MAKE) $(MAKESILENT) -f examples/passkey/CMakeFiles/passkey.dir/build.make examples/passkey/CMakeFiles/passkey.dir/build -.PHONY : passkey/fast - -#============================================================================= -# Target rules for targets named speculative - -# Build rule for target. -speculative: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 speculative -.PHONY : speculative - -# fast build rule for target. -speculative/fast: - $(MAKE) $(MAKESILENT) -f examples/speculative/CMakeFiles/speculative.dir/build.make examples/speculative/CMakeFiles/speculative.dir/build -.PHONY : speculative/fast - -#============================================================================= -# Target rules for targets named lookahead - -# Build rule for target. -lookahead: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 lookahead -.PHONY : lookahead - -# fast build rule for target. -lookahead/fast: - $(MAKE) $(MAKESILENT) -f examples/lookahead/CMakeFiles/lookahead.dir/build.make examples/lookahead/CMakeFiles/lookahead.dir/build -.PHONY : lookahead/fast - -#============================================================================= -# Target rules for targets named lookup - -# Build rule for target. -lookup: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 lookup -.PHONY : lookup - -# fast build rule for target. -lookup/fast: - $(MAKE) $(MAKESILENT) -f examples/lookup/CMakeFiles/lookup.dir/build.make examples/lookup/CMakeFiles/lookup.dir/build -.PHONY : lookup/fast - -#============================================================================= -# Target rules for targets named lookup-create - -# Build rule for target. -lookup-create: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 lookup-create -.PHONY : lookup-create - -# fast build rule for target. -lookup-create/fast: - $(MAKE) $(MAKESILENT) -f examples/lookup/CMakeFiles/lookup-create.dir/build.make examples/lookup/CMakeFiles/lookup-create.dir/build -.PHONY : lookup-create/fast - -#============================================================================= -# Target rules for targets named lookup-merge - -# Build rule for target. -lookup-merge: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 lookup-merge -.PHONY : lookup-merge - -# fast build rule for target. -lookup-merge/fast: - $(MAKE) $(MAKESILENT) -f examples/lookup/CMakeFiles/lookup-merge.dir/build.make examples/lookup/CMakeFiles/lookup-merge.dir/build -.PHONY : lookup-merge/fast - -#============================================================================= -# Target rules for targets named lookup-stats - -# Build rule for target. -lookup-stats: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 lookup-stats -.PHONY : lookup-stats - -# fast build rule for target. -lookup-stats/fast: - $(MAKE) $(MAKESILENT) -f examples/lookup/CMakeFiles/lookup-stats.dir/build.make examples/lookup/CMakeFiles/lookup-stats.dir/build -.PHONY : lookup-stats/fast - -#============================================================================= -# Target rules for targets named gguf - -# Build rule for target. -gguf: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 gguf -.PHONY : gguf - -# fast build rule for target. -gguf/fast: - $(MAKE) $(MAKESILENT) -f examples/gguf/CMakeFiles/gguf.dir/build.make examples/gguf/CMakeFiles/gguf.dir/build -.PHONY : gguf/fast - -#============================================================================= -# Target rules for targets named train-text-from-scratch - -# Build rule for target. -train-text-from-scratch: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 train-text-from-scratch -.PHONY : train-text-from-scratch - -# fast build rule for target. -train-text-from-scratch/fast: - $(MAKE) $(MAKESILENT) -f examples/train-text-from-scratch/CMakeFiles/train-text-from-scratch.dir/build.make examples/train-text-from-scratch/CMakeFiles/train-text-from-scratch.dir/build -.PHONY : train-text-from-scratch/fast - -#============================================================================= -# Target rules for targets named imatrix - -# Build rule for target. -imatrix: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 imatrix -.PHONY : imatrix - -# fast build rule for target. -imatrix/fast: - $(MAKE) $(MAKESILENT) -f examples/imatrix/CMakeFiles/imatrix.dir/build.make examples/imatrix/CMakeFiles/imatrix.dir/build -.PHONY : imatrix/fast - -#============================================================================= -# Target rules for targets named server - -# Build rule for target. -server: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 server -.PHONY : server - -# fast build rule for target. -server/fast: - $(MAKE) $(MAKESILENT) -f examples/server/CMakeFiles/server.dir/build.make examples/server/CMakeFiles/server.dir/build -.PHONY : server/fast - -#============================================================================= -# Target rules for targets named export-lora - -# Build rule for target. -export-lora: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 export-lora -.PHONY : export-lora - -# fast build rule for target. -export-lora/fast: - $(MAKE) $(MAKESILENT) -f examples/export-lora/CMakeFiles/export-lora.dir/build.make examples/export-lora/CMakeFiles/export-lora.dir/build -.PHONY : export-lora/fast - -#============================================================================= -# Target rules for targets named vdot - -# Build rule for target. -vdot: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 vdot -.PHONY : vdot - -# fast build rule for target. -vdot/fast: - $(MAKE) $(MAKESILENT) -f pocs/vdot/CMakeFiles/vdot.dir/build.make pocs/vdot/CMakeFiles/vdot.dir/build -.PHONY : vdot/fast - -#============================================================================= -# Target rules for targets named q8dot - -# Build rule for target. -q8dot: cmake_check_build_system - $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 q8dot -.PHONY : q8dot - -# fast build rule for target. -q8dot/fast: - $(MAKE) $(MAKESILENT) -f pocs/vdot/CMakeFiles/q8dot.dir/build.make pocs/vdot/CMakeFiles/q8dot.dir/build -.PHONY : q8dot/fast - -ggml-alloc.o: ggml-alloc.c.o -.PHONY : ggml-alloc.o - -# target to build an object file -ggml-alloc.c.o: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-alloc.c.o -.PHONY : ggml-alloc.c.o - -ggml-alloc.i: ggml-alloc.c.i -.PHONY : ggml-alloc.i - -# target to preprocess a source file -ggml-alloc.c.i: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-alloc.c.i -.PHONY : ggml-alloc.c.i - -ggml-alloc.s: ggml-alloc.c.s -.PHONY : ggml-alloc.s - -# target to generate assembly for a file -ggml-alloc.c.s: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-alloc.c.s -.PHONY : ggml-alloc.c.s - -ggml-backend.o: ggml-backend.c.o -.PHONY : ggml-backend.o - -# target to build an object file -ggml-backend.c.o: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-backend.c.o -.PHONY : ggml-backend.c.o - -ggml-backend.i: ggml-backend.c.i -.PHONY : ggml-backend.i - -# target to preprocess a source file -ggml-backend.c.i: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-backend.c.i -.PHONY : ggml-backend.c.i - -ggml-backend.s: ggml-backend.c.s -.PHONY : ggml-backend.s - -# target to generate assembly for a file -ggml-backend.c.s: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-backend.c.s -.PHONY : ggml-backend.c.s +tests/test-c.o: tests/test-c.c llama.h + $(CC) $(CFLAGS) -c $(filter-out %.h,$^) -o $@ -ggml-quants.o: ggml-quants.c.o -.PHONY : ggml-quants.o - -# target to build an object file -ggml-quants.c.o: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-quants.c.o -.PHONY : ggml-quants.c.o +tests/test-backend-ops: tests/test-backend-ops.cpp ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -ggml-quants.i: ggml-quants.c.i -.PHONY : ggml-quants.i - -# target to preprocess a source file -ggml-quants.c.i: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-quants.c.i -.PHONY : ggml-quants.c.i +tests/test-model-load-cancel: tests/test-model-load-cancel.cpp ggml.o llama.o tests/get-model.cpp $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -ggml-quants.s: ggml-quants.c.s -.PHONY : ggml-quants.s - -# target to generate assembly for a file -ggml-quants.c.s: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml-quants.c.s -.PHONY : ggml-quants.c.s - -ggml.o: ggml.c.o -.PHONY : ggml.o - -# target to build an object file -ggml.c.o: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml.c.o -.PHONY : ggml.c.o - -ggml.i: ggml.c.i -.PHONY : ggml.i - -# target to preprocess a source file -ggml.c.i: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml.c.i -.PHONY : ggml.c.i - -ggml.s: ggml.c.s -.PHONY : ggml.s - -# target to generate assembly for a file -ggml.c.s: - $(MAKE) $(MAKESILENT) -f CMakeFiles/ggml.dir/build.make CMakeFiles/ggml.dir/ggml.c.s -.PHONY : ggml.c.s - -llama.o: llama.cpp.o -.PHONY : llama.o - -# target to build an object file -llama.cpp.o: - $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/llama.cpp.o -.PHONY : llama.cpp.o - -llama.i: llama.cpp.i -.PHONY : llama.i - -# target to preprocess a source file -llama.cpp.i: - $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/llama.cpp.i -.PHONY : llama.cpp.i - -llama.s: llama.cpp.s -.PHONY : llama.s - -# target to generate assembly for a file -llama.cpp.s: - $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/llama.cpp.s -.PHONY : llama.cpp.s - -unicode-data.o: unicode-data.cpp.o -.PHONY : unicode-data.o - -# target to build an object file -unicode-data.cpp.o: - $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode-data.cpp.o -.PHONY : unicode-data.cpp.o - -unicode-data.i: unicode-data.cpp.i -.PHONY : unicode-data.i - -# target to preprocess a source file -unicode-data.cpp.i: - $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode-data.cpp.i -.PHONY : unicode-data.cpp.i - -unicode-data.s: unicode-data.cpp.s -.PHONY : unicode-data.s - -# target to generate assembly for a file -unicode-data.cpp.s: - $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode-data.cpp.s -.PHONY : unicode-data.cpp.s - -unicode.o: unicode.cpp.o -.PHONY : unicode.o - -# target to build an object file -unicode.cpp.o: - $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode.cpp.o -.PHONY : unicode.cpp.o - -unicode.i: unicode.cpp.i -.PHONY : unicode.i - -# target to preprocess a source file -unicode.cpp.i: - $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode.cpp.i -.PHONY : unicode.cpp.i - -unicode.s: unicode.cpp.s -.PHONY : unicode.s - -# target to generate assembly for a file -unicode.cpp.s: - $(MAKE) $(MAKESILENT) -f CMakeFiles/llama.dir/build.make CMakeFiles/llama.dir/unicode.cpp.s -.PHONY : unicode.cpp.s - -# Help Target -help: - @echo "The following are some of the valid targets for this Makefile:" - @echo "... all (the default if no target is provided)" - @echo "... clean" - @echo "... depend" - @echo "... edit_cache" - @echo "... install" - @echo "... install/local" - @echo "... install/strip" - @echo "... list_install_components" - @echo "... rebuild_cache" - @echo "... test" - @echo "... Continuous" - @echo "... ContinuousBuild" - @echo "... ContinuousConfigure" - @echo "... ContinuousCoverage" - @echo "... ContinuousMemCheck" - @echo "... ContinuousStart" - @echo "... ContinuousSubmit" - @echo "... ContinuousTest" - @echo "... ContinuousUpdate" - @echo "... Experimental" - @echo "... ExperimentalBuild" - @echo "... ExperimentalConfigure" - @echo "... ExperimentalCoverage" - @echo "... ExperimentalMemCheck" - @echo "... ExperimentalStart" - @echo "... ExperimentalSubmit" - @echo "... ExperimentalTest" - @echo "... ExperimentalUpdate" - @echo "... Nightly" - @echo "... NightlyBuild" - @echo "... NightlyConfigure" - @echo "... NightlyCoverage" - @echo "... NightlyMemCheck" - @echo "... NightlyMemoryCheck" - @echo "... NightlyStart" - @echo "... NightlySubmit" - @echo "... NightlyTest" - @echo "... NightlyUpdate" - @echo "... baby-llama" - @echo "... batched" - @echo "... batched-bench" - @echo "... beam-search" - @echo "... benchmark" - @echo "... build_info" - @echo "... common" - @echo "... convert-llama2c-to-ggml" - @echo "... embedding" - @echo "... export-lora" - @echo "... finetune" - @echo "... ggml" - @echo "... ggml_static" - @echo "... gguf" - @echo "... gguf-split" - @echo "... gritlm" - @echo "... imatrix" - @echo "... infill" - @echo "... json-schema-to-grammar" - @echo "... llama" - @echo "... llama-bench" - @echo "... llava" - @echo "... llava-cli" - @echo "... llava_static" - @echo "... lookahead" - @echo "... lookup" - @echo "... lookup-create" - @echo "... lookup-merge" - @echo "... lookup-stats" - @echo "... main" - @echo "... parallel" - @echo "... passkey" - @echo "... perplexity" - @echo "... q8dot" - @echo "... quantize" - @echo "... quantize-stats" - @echo "... retrieval" - @echo "... save-load-state" - @echo "... server" - @echo "... simple" - @echo "... speculative" - @echo "... test-autorelease" - @echo "... test-backend-ops" - @echo "... test-c" - @echo "... test-chat-template" - @echo "... test-grad0" - @echo "... test-grammar-integration" - @echo "... test-grammar-parser" - @echo "... test-json-schema-to-grammar" - @echo "... test-llama-grammar" - @echo "... test-model-load-cancel" - @echo "... test-quantize-fns" - @echo "... test-quantize-perf" - @echo "... test-rope" - @echo "... test-sampling" - @echo "... test-tokenizer-0-falcon" - @echo "... test-tokenizer-0-llama" - @echo "... test-tokenizer-1-aquila" - @echo "... test-tokenizer-1-baichuan" - @echo "... test-tokenizer-1-falcon" - @echo "... test-tokenizer-1-gpt-neox" - @echo "... test-tokenizer-1-gpt2" - @echo "... test-tokenizer-1-llama" - @echo "... test-tokenizer-1-mpt" - @echo "... test-tokenizer-1-refact" - @echo "... test-tokenizer-1-stablelm-3b-4e1t" - @echo "... test-tokenizer-1-starcoder" - @echo "... tokenize" - @echo "... train-text-from-scratch" - @echo "... vdot" - @echo "... ggml-alloc.o" - @echo "... ggml-alloc.i" - @echo "... ggml-alloc.s" - @echo "... ggml-backend.o" - @echo "... ggml-backend.i" - @echo "... ggml-backend.s" - @echo "... ggml-quants.o" - @echo "... ggml-quants.i" - @echo "... ggml-quants.s" - @echo "... ggml.o" - @echo "... ggml.i" - @echo "... ggml.s" - @echo "... llama.o" - @echo "... llama.i" - @echo "... llama.s" - @echo "... unicode-data.o" - @echo "... unicode-data.i" - @echo "... unicode-data.s" - @echo "... unicode.o" - @echo "... unicode.i" - @echo "... unicode.s" -.PHONY : help - - - -#============================================================================= -# Special targets to cleanup operation of make. - -# Special rule to run CMake to check the build system integrity. -# No rule that depends on this can have commands that come from listfiles -# because they might be regenerated. -cmake_check_build_system: - $(CMAKE_COMMAND) -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 0 -.PHONY : cmake_check_build_system +tests/test-autorelease: tests/test-autorelease.cpp ggml.o llama.o tests/get-model.cpp $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +tests/test-chat-template: tests/test-chat-template.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) From 795ff1d3d39e97e20e12805d067da05791419f54 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 22 Apr 2024 13:20:03 +0200 Subject: [PATCH 09/36] fix: revert some changes --- convert-hf-to-gguf.py | 5 +++-- ggml.c | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 9c01c296e2984..2cce4c2de2848 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -92,7 +92,6 @@ def get_tensors(self) -> Iterator[tuple[str, Tensor]]: def set_gguf_parameters(self): self.gguf_writer.add_name(self.dir_model.name) - print(f'self.block_count {self.block_count}') self.gguf_writer.add_block_count(self.block_count) if (n_ctx := self.find_hparam(["max_position_embeddings", "n_ctx"], optional=True)) is not None: @@ -138,7 +137,6 @@ def set_gguf_parameters(self): def write_tensors(self): block_count = self.hparams.get("n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer"))) tensor_map = gguf.get_tensor_name_map(self.model_arch, block_count) - print(f'Block_count {block_count} with tensor_map {tensor_map}') for name, data_torch in self.get_tensors(): # we don't need these if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq")): @@ -2188,6 +2186,9 @@ def get_tensors(self): yield name, data +JinaBertForMaskedML = JinaBertModel + + @Model.register("GemmaForCausalLM") class GemmaModel(Model): model_arch = gguf.MODEL_ARCH.GEMMA diff --git a/ggml.c b/ggml.c index 622df3a5affd5..b0af1512f38e6 100644 --- a/ggml.c +++ b/ggml.c @@ -12476,8 +12476,9 @@ static void ggml_compute_forward_alibi_f32( for (int64_t i = 0; i < ne0; i++) { for (int64_t j = 0; j < ne1; j++) { + float * const src = (float *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2); float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2); - pdst[0] = -1.0f * i * m_k; + pdst[0] = i * m_k + src[0]; } } } From d6ac931b7a7d2ca17653b9182f7f737b347ba2b2 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 22 Apr 2024 13:23:00 +0200 Subject: [PATCH 10/36] fix: fix small detail --- ggml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index b0af1512f38e6..b4e48a2628871 100644 --- a/ggml.c +++ b/ggml.c @@ -12258,7 +12258,7 @@ static void ggml_compute_forward_soft_max_f32( const float slope = h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1); for (int i = 0; i < nc; i++) { - if (pos == NULL) { + if (pos != NULL) { wp[i] = wp[i] + pos[i]; } else { wp[i] = wp[i] - slope*abs(i1%nc - i); From c1c0f4d883d9b234c49e26e30d02d928f1bd103f Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 22 Apr 2024 13:45:32 +0200 Subject: [PATCH 11/36] fix: fix convert formatting --- convert-hf-to-gguf.py | 94 +++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 52 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index f4a758aaa5eb9..c1b6888bcd4a6 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -77,13 +77,11 @@ def get_tensors(self) -> Iterator[tuple[str, Tensor]]: for part_name in self.part_names: print(f"gguf: loading model part '{part_name}'") ctx: ContextManager[Any] - if self.is_safetensors: from safetensors import safe_open ctx = cast(ContextManager[Any], safe_open(self.dir_model / part_name, framework="pt", device="cpu")) else: - ctx = contextlib.nullcontext( - torch.load(str(self.dir_model / part_name), map_location="cpu", mmap=True, weights_only=True)) + ctx = contextlib.nullcontext(torch.load(str(self.dir_model / part_name), map_location="cpu", mmap=True, weights_only=True)) with ctx as model_part: for name in model_part.keys(): @@ -120,8 +118,7 @@ def set_gguf_parameters(self): if (f_rms_eps := self.hparams.get("rms_norm_eps")) is not None: self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps) print(f"gguf: rms norm epsilon = {f_rms_eps}") - if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], - optional=True)) is not None: + if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], optional=True)) is not None: self.gguf_writer.add_layer_norm_eps(f_norm_eps) print(f"gguf: layer norm epsilon = {f_norm_eps}") if (n_experts := self.hparams.get("num_local_experts")) is not None: @@ -209,7 +206,6 @@ def func(modelcls: type[Model]): for name in names: cls._model_classes[name] = modelcls return modelcls - return func @classmethod @@ -294,7 +290,7 @@ def _set_vocab_qwen(self): # for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combined added_vocab = tokenizer.special_tokens - reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in (vocab | added_vocab).items()} + reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in (vocab | added_vocab).items()} for i in range(vocab_size): if i not in reverse_vocab: @@ -779,8 +775,8 @@ def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | Non return ( weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) - .swapaxes(1, 2) - .reshape(weights.shape) + .swapaxes(1, 2) + .reshape(weights.shape) ) def _reverse_hf_permute_part( @@ -931,8 +927,8 @@ def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | Non return ( weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) - .swapaxes(1, 2) - .reshape(weights.shape) + .swapaxes(1, 2) + .reshape(weights.shape) ) @@ -1209,8 +1205,7 @@ def set_gguf_parameters(self): self.gguf_writer.add_block_count(block_count) self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"]) rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"]) - self.gguf_writer.add_rope_dimension_count( - int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))) + self.gguf_writer.add_rope_dimension_count(int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))) self.gguf_writer.add_head_count(hparams["num_attention_heads"]) self.gguf_writer.add_head_count_kv(hparams["num_key_value_heads"]) self.gguf_writer.add_parallel_residual(hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True) @@ -1304,7 +1299,7 @@ class LlamaModel(Model): def set_vocab(self): try: - self._set_vocab_sentencepiece() + self. _set_vocab_sentencepiece() except FileNotFoundError: try: self._set_vocab_llama_hf() @@ -1653,8 +1648,8 @@ def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | Non return ( weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) - .swapaxes(1, 2) - .reshape(weights.shape) + .swapaxes(1, 2) + .reshape(weights.shape) ) def write_tensors(self): @@ -1914,8 +1909,7 @@ def write_tensors(self): for name, data_torch in self.get_tensors(): # we don't need these - if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq", - ".attn.bias", ".attn.masked_bias")): + if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq", ".attn.bias", ".attn.masked_bias")): continue if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight", ".c_proj.weight")): @@ -2300,8 +2294,7 @@ def write_tensors(self): bid = re.findall(qkv_pattern, name)[0] qkv = data_torch qkv = rearrange(qkv.T, " o (g n i) ->o g n i", g=num_groups, n=q_per_kv + 2, i=head_dim) - q, k, v = qkv[..., : q_per_kv, :], qkv[..., q_per_kv: q_per_kv + 1, :], qkv[..., - q_per_kv + 1: q_per_kv + 2, :] + q, k, v = qkv[..., : q_per_kv, :], qkv[..., q_per_kv: q_per_kv + 1, :], qkv[..., q_per_kv + 1: q_per_kv + 2, :] # The model weights of q and k equire additional reshape. q = self._hf_permute_qk(rearrange(q, " o g n i -> o (g n i)").T, num_heads, num_heads) k = self._hf_permute_qk(rearrange(k, " o g n i -> o (g n i)").T, num_heads, num_kv_heads) @@ -2384,7 +2377,6 @@ def write_tensors(self): # map tensor names new_name = tensor_map.get_name(name, try_suffixes=(".weight", ".bias")) - if new_name is None: print(f"Can not map tensor {name!r}") sys.exit() @@ -2441,31 +2433,6 @@ def set_gguf_parameters(self): self.gguf_writer.add_rope_freq_base(self.hparams["rotary_emb_base"]) -@Model.register("JinaBertModel") -class JinaBertModel(BertModel): - model_arch = gguf.MODEL_ARCH.JINA_BERT - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.intermediate_size = self.hparams["intermediate_size"] - - def get_tensors(self): - for name, data in super().get_tensors(): - if 'gated_layers' in name: - d1 = data[:self.intermediate_size, :] - name1 = name.replace('gated_layers', 'gated_layers_w') - d2 = data[self.intermediate_size:, :] - name2 = name.replace('gated_layers', 'gated_layers_v') - yield name1, d1 - yield name2, d2 - continue - - yield name, data - - -JinaBertForMaskedML = JinaBertModel - - @Model.register("GemmaForCausalLM") class GemmaModel(Model): model_arch = gguf.MODEL_ARCH.GEMMA @@ -2493,8 +2460,7 @@ def set_gguf_parameters(self): self.gguf_writer.add_block_count(block_count) self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"]) self.gguf_writer.add_head_count(hparams["num_attention_heads"]) - self.gguf_writer.add_head_count_kv( - self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"]) + self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"] if "num_key_value_heads" in hparams else hparams["num_attention_heads"]) self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"]) self.gguf_writer.add_key_length(hparams["head_dim"]) self.gguf_writer.add_value_length(hparams["head_dim"]) @@ -2604,10 +2570,10 @@ def set_gguf_parameters(self): assert d_inner == 2 * d_model self.gguf_writer.add_name(self.dir_model.name) - self.gguf_writer.add_context_length(2 ** 20) # arbitrary value; for those who use the default + self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the default self.gguf_writer.add_embedding_length(d_model) - self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loading - self.gguf_writer.add_head_count(0) # unused, but seemingly required when loading + self.gguf_writer.add_feed_forward_length(0) # unused, but seemingly required when loading + self.gguf_writer.add_head_count(0) # unused, but seemingly required when loading self.gguf_writer.add_block_count(self.hparams["n_layer"]) self.gguf_writer.add_ssm_conv_kernel(d_conv) self.gguf_writer.add_ssm_inner_size(d_inner) @@ -2622,7 +2588,7 @@ def write_tensors(self): tok_embd = None tok_embd_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.TOKEN_EMBD] + ".weight" - output_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.OUTPUT] + ".weight" + output_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.OUTPUT] + ".weight" for name, data_torch in self.get_tensors(): old_dtype = data_torch.dtype @@ -2748,6 +2714,29 @@ def write_tensors(self): self.gguf_writer.add_tensor(new_name, data) +@Model.register("JinaBertModel") +class JinaBertModel(BertModel): + model_arch = gguf.MODEL_ARCH.JINA_BERT + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.intermediate_size = self.hparams["intermediate_size"] + + def get_tensors(self): + for name, data in super().get_tensors(): + if 'gated_layers' in name: + d1 = data[:self.intermediate_size, :] + name1 = name.replace('gated_layers', 'gated_layers_w') + d2 = data[self.intermediate_size:, :] + name2 = name.replace('gated_layers', 'gated_layers_v') + yield name1, d1 + yield name2, d2 + continue + + yield name, data + + +JinaBertForMaskedML = JinaBertModel ###### CONVERSION LOGIC ###### @@ -2816,6 +2805,7 @@ def main() -> None: print(f"Loading model: {dir_model.name}") hparams = Model.load_hparams(dir_model) + with torch.inference_mode(): model_class = Model.from_model_architecture(hparams["architectures"][0]) model_instance = model_class(dir_model, ftype_map[args.outtype], fname_out, args.bigendian, args.use_temp_file) From 64cd4b133945f875a1ae0014fdfd86bc0d5ce881 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 22 Apr 2024 17:42:48 +0200 Subject: [PATCH 12/36] fix: fix linting and editor --- convert-hf-to-gguf.py | 1 + llama.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index c1b6888bcd4a6..23e1606ed8c97 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -2714,6 +2714,7 @@ def write_tensors(self): self.gguf_writer.add_tensor(new_name, data) + @Model.register("JinaBertModel") class JinaBertModel(BertModel): model_arch = gguf.MODEL_ARCH.JINA_BERT diff --git a/llama.cpp b/llama.cpp index a257e46e3d818..309f4eeccd72e 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5124,7 +5124,7 @@ static bool llm_load_tensors( auto & layer = model.layers[i]; // JinaBertLayer - layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}); layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); From 71ff763e0e4d9134f2eb7e3f5846e1c3d28de382 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 22 Apr 2024 18:02:48 +0200 Subject: [PATCH 13/36] feat: set proper vocab settings --- convert-hf-to-gguf.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 23e1606ed8c97..4619c80959608 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -2736,6 +2736,11 @@ def get_tensors(self): yield name, data + def set_vocab(self, *args, **kwargs): + super().set_vocab() + self.gguf_writer.add_add_bos_token(True) + self.gguf_writer.add_add_eos_token(True) + JinaBertForMaskedML = JinaBertModel From d7d6a4ed466211019366b2b3bcc5efb3edb3c287 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Tue, 23 Apr 2024 09:48:40 +0200 Subject: [PATCH 14/36] fix: JinaBertForMaskedLM registration --- convert-hf-to-gguf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 4619c80959608..a132f9c5b3ad6 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -2715,7 +2715,7 @@ def write_tensors(self): self.gguf_writer.add_tensor(new_name, data) -@Model.register("JinaBertModel") +@Model.register("JinaBertModel", "JinaBertForMaskedLM") class JinaBertModel(BertModel): model_arch = gguf.MODEL_ARCH.JINA_BERT @@ -2742,8 +2742,6 @@ def set_vocab(self, *args, **kwargs): self.gguf_writer.add_add_eos_token(True) -JinaBertForMaskedML = JinaBertModel - ###### CONVERSION LOGIC ###### From cde49b7448170a000b12946f78f9fb09b2c9c143 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Tue, 23 Apr 2024 16:10:38 +0200 Subject: [PATCH 15/36] feat: support q_normalization and k_normalization in Jina arch --- gguf-py/gguf/constants.py | 2 ++ gguf-py/gguf/tensor_mapping.py | 4 +++- llama.cpp | 21 +++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 6217bef9b4877..702842ffec542 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -386,7 +386,9 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.TOKEN_TYPES, MODEL_TENSOR.ATTN_OUT_NORM, MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_NORM, MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.FFN_UP, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 350e649f931c1..45a68fc06de9c 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -318,6 +318,7 @@ class TensorNameMap: "model.layers.{bid}.self_attn.q_layernorm", # persimmon "model.layers.{bid}.self_attn.q_norm", # cohere "transformer.blocks.{bid}.attn.q_ln", # sea-lion + "encoder.layer.{bid}.attention.self.layer_norm_q" # jina-bert ), MODEL_TENSOR.ATTN_K_NORM: ( @@ -325,6 +326,7 @@ class TensorNameMap: "model.layers.{bid}.self_attn.k_layernorm", # persimmon "model.layers.{bid}.self_attn.k_norm", # cohere "transformer.blocks.{bid}.attn.k_ln", # sea-lion + "encoder.layer.{bid}.attention.self.layer_norm_k" # jina-bert ), MODEL_TENSOR.ROPE_FREQS: ( @@ -335,7 +337,7 @@ class TensorNameMap: "encoder.layer.{bid}.output.LayerNorm", # bert "encoder.layers.{bid}.norm2", # nomic-bert "transformer.decoder_layer.{bid}.rms_norm_3", # Grok - "encoder.layer.{bid}.mlp.layernorm", # jina-bert + "encoder.layer.{bid}.mlp.layernorm", # jina-bert ), MODEL_TENSOR.SSM_IN: ( diff --git a/llama.cpp b/llama.cpp index 309f4eeccd72e..47f6b9267b486 100644 --- a/llama.cpp +++ b/llama.cpp @@ -697,7 +697,9 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_TOKEN_TYPES, "token_types" }, { LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" }, { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" }, { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" }, { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, { LLM_TENSOR_LAYER_OUT_NORM, "blk.%d.layer_output_norm" }, @@ -5127,9 +5129,15 @@ static bool llm_load_tensors( layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}); + layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, false); + layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {n_embd}, false); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}); + layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, false); + layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {n_embd}, false); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}); @@ -8023,9 +8031,22 @@ struct llm_build_context { Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), model.layers[il].bq); cb(Qcur, "Qcur", il); + if (model.layers[il].attn_q_norm) { + Qcur = llm_build_norm(ctx0, Qcur, hparams, + model.layers[il].attn_q_norm, + model.layers[il].attn_q_norm_b, + LLM_NORM, cb, il); + } + Kcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wk, cur), model.layers[il].bk); cb(Kcur, "Kcur", il); + if (model.layers[il].attn_k_norm) { + Kcur = llm_build_norm(ctx0, Kcur, hparams, + model.layers[il].attn_k_norm, + model.layers[il].attn_k_norm_b, + LLM_NORM, cb, il); + } Vcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wv, cur), model.layers[il].bv); cb(Vcur, "Vcur", il); From dd060a2a4ec844ed16647afac4305f29aaab97ce Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Wed, 24 Apr 2024 10:05:34 +0200 Subject: [PATCH 16/36] feat: handle gpt2 tokenizer with Jina architecture --- convert-hf-to-gguf.py | 14 +++++++++++++- llama.cpp | 7 ++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index a132f9c5b3ad6..2ba675ef0ab57 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -134,6 +134,7 @@ def set_gguf_parameters(self): def write_tensors(self): block_count = self.hparams.get("n_layers", self.hparams.get("num_hidden_layers", self.hparams.get("n_layer"))) tensor_map = gguf.get_tensor_name_map(self.model_arch, block_count) + for name, data_torch in self.get_tensors(): # we don't need these if name.endswith((".attention.masked_bias", ".attention.bias", ".attention.rotary_emb.inv_freq")): @@ -2370,6 +2371,7 @@ def phantom(tok): def write_tensors(self): tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) tensors = dict(self.get_tensors()) + for name, data_torch in tensors.items(): # we are only using BERT for embeddings so we don't need the pooling layer if name in ("embeddings.position_ids", "pooler.dense.weight", "pooler.dense.bias"): @@ -2737,7 +2739,17 @@ def get_tensors(self): yield name, data def set_vocab(self, *args, **kwargs): - super().set_vocab() + tokenizer_class = 'BertTokenizer' + with open(self.dir_model / "tokenizer_config.json", "r", encoding="utf-8") as f: + tokenizer_class = json.load(f)['tokenizer_class'] + + if tokenizer_class == 'BertTokenizer': + super().set_vocab() + elif tokenizer_class == 'RobertaTokenizer': + self._set_vocab_gpt2() + self.gguf_writer.add_token_type_count(2) + else: + raise NotImplementedError(f'Tokenizer {tokenizer_class} is not supported for JinaBertModel') self.gguf_writer.add_add_bos_token(True) self.gguf_writer.add_add_eos_token(True) diff --git a/llama.cpp b/llama.cpp index 47f6b9267b486..7460e3531db11 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12512,7 +12512,12 @@ static std::vector llama_tokenize_internal(const llama_vocab & } } - GGML_ASSERT(vocab.special_add_eos != 1); + //GGML_ASSERT(vocab.special_add_eos != 1); + //TODO: Check this, why this tokenizer does not add at the end, why not leaving up to the `gguf` exporter? + if (add_special && vocab.special_add_eos == 1) { + GGML_ASSERT(vocab.special_add_eos != -1); + output.push_back(vocab.special_eos_id); + } } break; case LLAMA_VOCAB_TYPE_WPM: { From dfa067631c6f7e5f5c153794150d47bea4f5e439 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Wed, 24 Apr 2024 10:14:02 +0200 Subject: [PATCH 17/36] feat: example comments in embedding --- examples/embedding/embedding.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index 6a93147d70e88..fe357c44be008 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -49,6 +49,12 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu } float * out = output + batch.seq_id[i][0] * n_embd; + //TODO: I would also add a parameter here to enable normalization or not. + /*fprintf(stdout, "unnormalized_embedding:"); + for (int hh = 0; hh < n_embd; hh++) { + fprintf(stdout, "%9.6f ", embd[hh]); + } + fprintf(stdout, "\n");*/ llama_embd_normalize(embd, out, n_embd); } } @@ -124,6 +130,8 @@ int main(int argc, char ** argv) { } // add SEP if not present + // JoanFM: I propose to remove this line so that user can make sure that their model is properly configured to tokenize as expected. + // We could also add a parameter, but I think that adding parameters specific for the examples can become messy and unmantaibable easy for (auto & inp : inputs) { if (inp.empty() || inp.back() != llama_token_sep(model)) { inp.push_back(llama_token_sep(model)); From c3f4b1f2d297bdcaf1c424aefec4a9f756b1a885 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Wed, 24 Apr 2024 15:46:18 +0200 Subject: [PATCH 18/36] feat: rename Jina Bert to Jina Bert V2 --- convert-hf-to-gguf.py | 4 ++-- gguf-py/gguf/constants.py | 6 +++--- gguf-py/gguf/tensor_mapping.py | 12 ++++++------ llama.cpp | 22 +++++++++++----------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 2ba675ef0ab57..1f7515e7c4582 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -2718,8 +2718,8 @@ def write_tensors(self): @Model.register("JinaBertModel", "JinaBertForMaskedLM") -class JinaBertModel(BertModel): - model_arch = gguf.MODEL_ARCH.JINA_BERT +class JinaBertV2Model(BertModel): + model_arch = gguf.MODEL_ARCH.JINA_BERT_V2 def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 702842ffec542..71039fabbd9b2 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -118,7 +118,7 @@ class MODEL_ARCH(IntEnum): REFACT = auto() BERT = auto() NOMIC_BERT = auto() - JINA_BERT = auto() + JINA_BERT_V2 = auto() BLOOM = auto() STABLELM = auto() QWEN = auto() @@ -195,7 +195,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.REFACT: "refact", MODEL_ARCH.BERT: "bert", MODEL_ARCH.NOMIC_BERT: "nomic-bert", - MODEL_ARCH.JINA_BERT: "jina-bert", + MODEL_ARCH.JINA_BERT_V2: "jina-bert-v2", MODEL_ARCH.BLOOM: "bloom", MODEL_ARCH.STABLELM: "stablelm", MODEL_ARCH.QWEN: "qwen", @@ -380,7 +380,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_UP, MODEL_TENSOR.LAYER_OUT_NORM, ], - MODEL_ARCH.JINA_BERT: [ + MODEL_ARCH.JINA_BERT_V2: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.TOKEN_EMBD_NORM, MODEL_TENSOR.TOKEN_TYPES, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 45a68fc06de9c..8531b2f730d2f 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -238,7 +238,7 @@ class TensorNameMap: "model.layers.{bid}.feed_forward.w3", # internlm2 "encoder.layers.{bid}.mlp.fc11", # nomic-bert "model.layers.{bid}.mlp.c_fc", # starcoder2 - "encoder.layer.{bid}.mlp.gated_layers_v", # jina-bert + "encoder.layer.{bid}.mlp.gated_layers_v", # jina-bert-v2 ), MODEL_TENSOR.FFN_UP_EXP: ( @@ -265,7 +265,7 @@ class TensorNameMap: "model.layers.layers.{bid}.mlp.gate_proj", # plamo "model.layers.{bid}.feed_forward.w1", # internlm2 "encoder.layers.{bid}.mlp.fc12", # nomic-bert - "encoder.layer.{bid}.mlp.gated_layers_w", # jina-bert + "encoder.layer.{bid}.mlp.gated_layers_w", # jina-bert-v2 ), MODEL_TENSOR.FFN_GATE_EXP: ( @@ -299,7 +299,7 @@ class TensorNameMap: "model.layers.{bid}.feed_forward.w2", # internlm2 "encoder.layers.{bid}.mlp.fc2", # nomic-bert "model.layers.{bid}.mlp.c_proj", # starcoder2 - "encoder.layer.{bid}.mlp.wo", # jina-bert + "encoder.layer.{bid}.mlp.wo", # jina-bert-v2 ), MODEL_TENSOR.FFN_DOWN_EXP: ( @@ -318,7 +318,7 @@ class TensorNameMap: "model.layers.{bid}.self_attn.q_layernorm", # persimmon "model.layers.{bid}.self_attn.q_norm", # cohere "transformer.blocks.{bid}.attn.q_ln", # sea-lion - "encoder.layer.{bid}.attention.self.layer_norm_q" # jina-bert + "encoder.layer.{bid}.attention.self.layer_norm_q" # jina-bert-v2 ), MODEL_TENSOR.ATTN_K_NORM: ( @@ -326,7 +326,7 @@ class TensorNameMap: "model.layers.{bid}.self_attn.k_layernorm", # persimmon "model.layers.{bid}.self_attn.k_norm", # cohere "transformer.blocks.{bid}.attn.k_ln", # sea-lion - "encoder.layer.{bid}.attention.self.layer_norm_k" # jina-bert + "encoder.layer.{bid}.attention.self.layer_norm_k" # jina-bert-v2 ), MODEL_TENSOR.ROPE_FREQS: ( @@ -337,7 +337,7 @@ class TensorNameMap: "encoder.layer.{bid}.output.LayerNorm", # bert "encoder.layers.{bid}.norm2", # nomic-bert "transformer.decoder_layer.{bid}.rms_norm_3", # Grok - "encoder.layer.{bid}.mlp.layernorm", # jina-bert + "encoder.layer.{bid}.mlp.layernorm", # jina-bert-v2 ), MODEL_TENSOR.SSM_IN: ( diff --git a/llama.cpp b/llama.cpp index 7460e3531db11..330df9de5fffc 100644 --- a/llama.cpp +++ b/llama.cpp @@ -205,7 +205,7 @@ enum llm_arch { LLM_ARCH_REFACT, LLM_ARCH_BERT, LLM_ARCH_NOMIC_BERT, - LLM_ARCH_JINA_BERT, + LLM_ARCH_JINA_BERT_V2, LLM_ARCH_BLOOM, LLM_ARCH_STABLELM, LLM_ARCH_QWEN, @@ -241,7 +241,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_REFACT, "refact" }, { LLM_ARCH_BERT, "bert" }, { LLM_ARCH_NOMIC_BERT, "nomic-bert" }, - { LLM_ARCH_JINA_BERT, "jina-bert" }, + { LLM_ARCH_JINA_BERT_V2, "jina-bert-v2"}, { LLM_ARCH_BLOOM, "bloom" }, { LLM_ARCH_STABLELM, "stablelm" }, { LLM_ARCH_QWEN, "qwen" }, @@ -690,7 +690,7 @@ static const std::map> LLM_TENSOR_NA }, }, { - LLM_ARCH_JINA_BERT, + LLM_ARCH_JINA_BERT_V2, { { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, { LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" }, @@ -3893,7 +3893,7 @@ static void llm_load_hparams( model.type = e_model::MODEL_335M; break; // bge-large } } break; - case LLM_ARCH_JINA_BERT: + case LLM_ARCH_JINA_BERT_V2: { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn); @@ -4137,7 +4137,7 @@ static void llm_load_hparams( model.ftype = ml.ftype; - if (hparams.f_max_alibi_bias > 0.0f && model.arch != LLM_ARCH_JINA_BERT) { + if (hparams.f_max_alibi_bias > 0.0f && model.arch != LLM_ARCH_JINA_BERT_V2) { hparams.need_kq_pos = true; } @@ -5113,7 +5113,7 @@ static bool llm_load_tensors( layer.layer_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}); } } break; - case LLM_ARCH_JINA_BERT: + case LLM_ARCH_JINA_BERT_V2: { model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // word_embeddings model.type_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd, n_vocab_type}); //token_type_embeddings @@ -7994,7 +7994,7 @@ struct llm_build_context { struct ggml_tensor * inpL; struct ggml_tensor * inp_pos = nullptr; - if (model.arch != LLM_ARCH_JINA_BERT) { + if (model.arch != LLM_ARCH_JINA_BERT_V2) { inp_pos = build_inp_pos(); } struct ggml_tensor * inp_mean = build_inp_mean(); @@ -8027,7 +8027,7 @@ struct llm_build_context { struct ggml_tensor * Vcur; // self-attention - if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT) { + if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_JINA_BERT_V2) { Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].wq, cur), model.layers[il].bq); cb(Qcur, "Qcur", il); @@ -8137,7 +8137,7 @@ struct llm_build_context { model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); - } else if (model.arch == LLM_ARCH_JINA_BERT) { + } else if (model.arch == LLM_ARCH_JINA_BERT_V2) { cur = llm_build_ffn(ctx0, cur, model.layers[il].ffn_up, NULL, model.layers[il].ffn_gate, NULL, @@ -10544,7 +10544,7 @@ static struct ggml_cgraph * llama_build_graph( result = llm.build_refact(); } break; case LLM_ARCH_BERT: - case LLM_ARCH_JINA_BERT: + case LLM_ARCH_JINA_BERT_V2: case LLM_ARCH_NOMIC_BERT: { result = llm.build_bert(); @@ -15473,7 +15473,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_REFACT: case LLM_ARCH_BLOOM: case LLM_ARCH_MAMBA: - case LLM_ARCH_JINA_BERT: + case LLM_ARCH_JINA_BERT_V2: return LLAMA_ROPE_TYPE_NONE; // use what we call a normal RoPE, operating on pairs of consecutive head values From 603f18bc4678ca78ecaf10c0f71b0c9ff82b4cb3 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 29 Apr 2024 12:23:20 +0200 Subject: [PATCH 19/36] feat: small changes to allow jina embeddings ZH model --- llama.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/llama.cpp b/llama.cpp index 330df9de5fffc..caff4c7671dd3 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4303,8 +4303,7 @@ static void llm_load_vocab( for (uint32_t i = 0; i < n_vocab; i++) { std::string word = gguf_get_arr_str(ctx, token_idx, i); - GGML_ASSERT(unicode_cpts_from_utf8(word).size() > 0); - + //GGML_ASSERT(unicode_cpts_from_utf8(word).size() > 0); Remove check, some vocabs contain by mistake the NULL in vocab, (not ideal if it happens more than once) (jinaai-embeddings-v2-base-zh) vocab.token_to_id[word] = i; auto & token_data = vocab.id_to_token[i]; @@ -4325,9 +4324,18 @@ static void llm_load_vocab( } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) { vocab.linefeed_id = vocab.special_pad_id; } else { - const std::vector ids = llama_tokenize_internal(vocab, "\xC4\x8A", false); // U+010A - GGML_ASSERT(!ids.empty() && "model vocab missing newline token"); - vocab.linefeed_id = ids[0]; + try { + const std::vector ids = llama_tokenize_internal(vocab, "\xC4\x8A", false); // U+010A + if (ids.empty()) { + LLAMA_LOG_WARN("%s: %s vocabulary, but newline token not found: %s! Using special_pad_id instead.", __func__, llama_model_vocab_type_name(vocab.type), "\xC4\x8A"); + vocab.linefeed_id = -1; + } else { + vocab.linefeed_id = ids[0]; + } + } catch (const std::exception & e) { + LLAMA_LOG_WARN("%s: %s vocabulary, but newline token not found: %s! Using special_pad_id instead.", __func__, llama_model_vocab_type_name(vocab.type), e.what()); + vocab.linefeed_id = vocab.special_pad_id; + } } // special tokens From da963685356155055a8eae649a553dfefa87b4cb Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Tue, 30 Apr 2024 14:15:50 +0200 Subject: [PATCH 20/36] fix: add some changes as per review --- ggml.c | 8 ++++---- llama.cpp | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ggml.c b/ggml.c index be25a5220dfe7..bc531661467d0 100644 --- a/ggml.c +++ b/ggml.c @@ -12436,13 +12436,13 @@ static void ggml_compute_forward_soft_max_f32( if (use_f16) { for (int i = 0; i < nc; ++i) { - wp[i] += slope*GGML_FP16_TO_FP32(pos_f16[i]); - //wp[i] = wp[i] - slope*abs(i1%nc - i); + //wp[i] -= slope*GGML_FP16_TO_FP32(pos_f16[i]); + wp[i] -= slope*abs(i1%nc - i); } } else { for (int i = 0; i < nc; ++i) { - wp[i] += slope*pos_f32[i]; - //wp[i] = wp[i] - slope*abs(i1%nc - i); + //wp[i] -= slope*pos_f32[i]; + wp[i] -= slope*abs(i1%nc - i); } } } diff --git a/llama.cpp b/llama.cpp index e212fecb0a163..9ee5be17c02f0 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8254,6 +8254,9 @@ struct llm_build_context { // KQ_mask (mask for 1 head, it will be broadcasted to all heads) struct ggml_tensor * KQ_mask = build_inp_KQ_mask(false); + // positions of the tokens in the KV cache + struct ggml_tensor * KQ_pos = build_inp_KQ_pos(false); + // iterate layers for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * cur = inpL; @@ -8322,7 +8325,7 @@ struct llm_build_context { struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); cb(kq, "kq", il); - kq = ggml_soft_max_ext(ctx0, kq, KQ_mask, nullptr, 1.0f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias); + kq = ggml_soft_max_ext(ctx0, kq, KQ_mask, KQ_pos, 1.0f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias); cb(kq, "kq_soft_max_ext", il); struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens))); @@ -11523,7 +11526,7 @@ static int llama_decode_internal( } // non-causal masks do not use the KV cache - if (hparams.causal_attn) { + if (hparams.causal_attn || model.arch == LLM_ARCH_JINA_BERT_V2) { llama_kv_cache_update(&lctx); // if we have enough unused cells before the current head -> From d9b8dd667dfb6fb8f643f4fc3e1c245c06522f34 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Tue, 30 Apr 2024 14:15:50 +0200 Subject: [PATCH 21/36] fix: add some changes as per review --- ggml.c | 13 ++++++------- llama.cpp | 7 +++++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ggml.c b/ggml.c index be25a5220dfe7..7924717254da1 100644 --- a/ggml.c +++ b/ggml.c @@ -5478,9 +5478,9 @@ static struct ggml_tensor * ggml_soft_max_impl( GGML_ASSERT(pos->type == mask->type); } - /*if (max_bias > 0.0f) { + if (max_bias > 0.0f) { GGML_ASSERT(pos); - }*/ + } bool is_node = false; @@ -12401,7 +12401,6 @@ static void ggml_compute_forward_soft_max_f32( float * wp = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith; // when max_bias <= 0.0f, src2 is not used and we default it to src0 to avoid branching - //float * pos = src2 ? (float *) src2->data : NULL; ggml_fp16_t * pos_f16 = src2 ? (ggml_fp16_t *) src2->data : src0->data; float * pos_f32 = src2 ? (float *) src2->data : src0->data; @@ -12436,13 +12435,13 @@ static void ggml_compute_forward_soft_max_f32( if (use_f16) { for (int i = 0; i < nc; ++i) { - wp[i] += slope*GGML_FP16_TO_FP32(pos_f16[i]); - //wp[i] = wp[i] - slope*abs(i1%nc - i); + //wp[i] -= slope*GGML_FP16_TO_FP32(pos_f16[i]); + wp[i] -= slope*abs(i1%nc - i); } } else { for (int i = 0; i < nc; ++i) { - wp[i] += slope*pos_f32[i]; - //wp[i] = wp[i] - slope*abs(i1%nc - i); + //wp[i] -= slope*pos_f32[i]; + wp[i] -= slope*abs(i1%nc - i); } } } diff --git a/llama.cpp b/llama.cpp index e212fecb0a163..9ee5be17c02f0 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8254,6 +8254,9 @@ struct llm_build_context { // KQ_mask (mask for 1 head, it will be broadcasted to all heads) struct ggml_tensor * KQ_mask = build_inp_KQ_mask(false); + // positions of the tokens in the KV cache + struct ggml_tensor * KQ_pos = build_inp_KQ_pos(false); + // iterate layers for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * cur = inpL; @@ -8322,7 +8325,7 @@ struct llm_build_context { struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); cb(kq, "kq", il); - kq = ggml_soft_max_ext(ctx0, kq, KQ_mask, nullptr, 1.0f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias); + kq = ggml_soft_max_ext(ctx0, kq, KQ_mask, KQ_pos, 1.0f/sqrtf(float(n_embd_head)), hparams.f_max_alibi_bias); cb(kq, "kq_soft_max_ext", il); struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens))); @@ -11523,7 +11526,7 @@ static int llama_decode_internal( } // non-causal masks do not use the KV cache - if (hparams.causal_attn) { + if (hparams.causal_attn || model.arch == LLM_ARCH_JINA_BERT_V2) { llama_kv_cache_update(&lctx); // if we have enough unused cells before the current head -> From 14073a2cafcae959bebe507aef19652b16f60c02 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Tue, 30 Apr 2024 16:22:35 +0200 Subject: [PATCH 22/36] feat: proper KQ_pos for Jina embeddings --- ggml.c | 9 ++++----- llama.cpp | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/ggml.c b/ggml.c index 7924717254da1..b535b1bc322eb 100644 --- a/ggml.c +++ b/ggml.c @@ -5469,7 +5469,7 @@ static struct ggml_tensor * ggml_soft_max_impl( } if (pos) { - GGML_ASSERT(ggml_is_vector(pos)); + GGML_ASSERT(ggml_is_vector(pos) || ggml_is_matrix(pos)); GGML_ASSERT(pos->type == GGML_TYPE_F16 || pos->type == GGML_TYPE_F32); GGML_ASSERT(pos->ne[0] == a->ne[0]); } @@ -12401,6 +12401,7 @@ static void ggml_compute_forward_soft_max_f32( float * wp = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith; // when max_bias <= 0.0f, src2 is not used and we default it to src0 to avoid branching + const bool is_pos_matrix = src2 ? ggml_is_matrix(src2): false; ggml_fp16_t * pos_f16 = src2 ? (ggml_fp16_t *) src2->data : src0->data; float * pos_f32 = src2 ? (float *) src2->data : src0->data; @@ -12435,13 +12436,11 @@ static void ggml_compute_forward_soft_max_f32( if (use_f16) { for (int i = 0; i < nc; ++i) { - //wp[i] -= slope*GGML_FP16_TO_FP32(pos_f16[i]); - wp[i] -= slope*abs(i1%nc - i); + wp[i] += slope*GGML_FP16_TO_FP32(pos_f16[is_pos_matrix ? i1%nc * nc + i: i]); } } else { for (int i = 0; i < nc; ++i) { - //wp[i] -= slope*pos_f32[i]; - wp[i] -= slope*abs(i1%nc - i); + wp[i] += slope*pos_f32[is_pos_matrix ? i1%nc * nc + i: i]; } } } diff --git a/llama.cpp b/llama.cpp index 9ee5be17c02f0..634f26b109370 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6998,7 +6998,7 @@ struct llm_build_context { } else { // TODO: this will be needed for ALiBi-based BERT models // https://github.com/ggerganov/llama.cpp/pull/6826 - lctx.inp_KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_tokens); + lctx.inp_KQ_pos = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens); } cb(lctx.inp_KQ_pos, "KQ_pos", -1); ggml_set_input(lctx.inp_KQ_pos); @@ -11166,11 +11166,22 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { GGML_ASSERT(lctx.inp_KQ_pos); GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_pos->buffer)); + GGML_ASSERT(ggml_is_vector(lctx.inp_KQ_pos) || ggml_is_matrix(lctx.inp_KQ_pos)); + if (ggml_is_vector(lctx.inp_KQ_pos)) { + float * data = (float *) lctx.inp_KQ_pos->data; - float * data = (float *) lctx.inp_KQ_pos->data; + for (int i = 0; i < n_kv; ++i) { + data[i] = float(lctx.kv_self.cells[i].pos); + } + } else if(ggml_is_matrix(lctx.inp_KQ_pos)) { + const int64_t n_tokens = batch.n_tokens; + float * data = (float *) lctx.inp_KQ_pos->data; - for (int i = 0; i < n_kv; ++i) { - data[i] = float(lctx.kv_self.cells[i].pos); + for (int i = 0; i < n_tokens; ++i) { + for (int j = 0; j < n_tokens; ++j) { + data[i * n_tokens + j] = -1.0 * abs(i - j); + } + } } } From 14cd69a87d8aadde3f8caee53eefae34acaecc3e Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Thu, 2 May 2024 11:59:03 +0200 Subject: [PATCH 23/36] feat: add pre tokenization --- convert-hf-to-gguf-update.py | 3 +++ convert-hf-to-gguf.py | 9 +++++++++ llama.cpp | 9 +++++++++ llama.h | 19 +++++++++++-------- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index b019c1e3dc59f..d2c6db16b3c31 100644 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -56,6 +56,9 @@ class TOKENIZER_TYPE(IntEnum): { "name": "mpt", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", }, { "name": "starcoder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", }, { "name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", }, + { "name": "jina-embeddings-v2-base-es", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-es", }, + { "name": "jina-embeddings-v2-base-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", }, + { "name": "jina-embeddings-v2-base-zh", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-zh", }, ] # make directory "models/tokenizers" if it doesn't exist diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 0f40850724a37..67839f341e612 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -307,6 +307,15 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "3ce83efda5659b07b1ad37ca97ca5797ea4285d9b9ab0dc679e4a720c9da7454": # ref: https://huggingface.co/openai-community/gpt2 res = "gpt-2" + if chkhsh == "171aeeedd6fb548d418a7461d053f11b6f1f1fc9b387bd66640d28a4b9f5c643": + # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-es + res = "jina-embeddings-v2-base-es" + if chkhsh == "27949a2493fc4a9f53f5b9b029c82689cfbe5d3a1929bb25e043089e28466de6": + # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-de + res = "jina-embeddings-v2-base-de" + if chkhsh == "c7699093ba4255a91e702aa38a596aa81669f3525dae06c2953267dde580f448": + # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-zh + res = "jina-embeddings-v2-base-zh" if res is None: print("\n") diff --git a/llama.cpp b/llama.cpp index e76f58820b6f8..23013ddcab97e 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4417,6 +4417,15 @@ static void llm_load_vocab( } else if ( tokenizer_pre == "gpt-2") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2; + } else if ( + tokenizer_pre == "jina-embeddings-v2-base-es") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_JINA_EMBEDDINGS_V2_ES; + } else if ( + tokenizer_pre == "jina-embeddings-v2-base-de") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_JINA_EMBEDDINGS_V2_DE; + } else if ( + tokenizer_pre == "jina-embeddings-v2-base-zh") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_JINA_EMBEDDINGS_V2_ZH; } else { throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); } diff --git a/llama.h b/llama.h index 059d78f115c6d..d9966315506a2 100644 --- a/llama.h +++ b/llama.h @@ -71,14 +71,17 @@ extern "C" { // pre-tokenization types enum llama_vocab_pre_type { - LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, - LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, - LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, - LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3, - LLAMA_VOCAB_PRE_TYPE_FALCON = 4, - LLAMA_VOCAB_PRE_TYPE_MPT = 5, - LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, - LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, + LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, + LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, + LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, + LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3, + LLAMA_VOCAB_PRE_TYPE_FALCON = 4, + LLAMA_VOCAB_PRE_TYPE_MPT = 5, + LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, + LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, + LLAMA_VOCAB_PRE_TYPE_JINA_EMBEDDINGS_V2_ES = 8, + LLAMA_VOCAB_PRE_TYPE_JINA_EMBEDDINGS_V2_DE = 9, + LLAMA_VOCAB_PRE_TYPE_JINA_EMBEDDINGS_V2_ZH = 10, }; // note: these values should be synchronized with ggml_rope From d5c3525bffac964041932a3dba6390fc3156d3bb Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 6 May 2024 16:04:01 +0200 Subject: [PATCH 24/36] feat: first iteration NFC --- unicode-data.cpp | 13 +++++++++ unicode-data.h | 3 ++ unicode.cpp | 72 ++++++++++++++++++++++++++++++++++++++++++++++++ unicode.h | 3 ++ 4 files changed, 91 insertions(+) diff --git a/unicode-data.cpp b/unicode-data.cpp index e6bafb3a9add7..c3df7a1d54b37 100644 --- a/unicode-data.cpp +++ b/unicode-data.cpp @@ -1649,3 +1649,16 @@ const std::map unicode_map_lowercase = { {0x1E917, 0x1E939}, {0x1E918, 0x1E93A}, {0x1E919, 0x1E93B}, {0x1E91A, 0x1E93C}, {0x1E91B, 0x1E93D}, {0x1E91C, 0x1E93E}, {0x1E91D, 0x1E93F}, {0x1E91E, 0x1E940}, {0x1E91F, 0x1E941}, {0x1E920, 0x1E942}, {0x1E921, 0x1E943}, }; + + +const std::unordered_map> unicode_decompose_map = { + {65, {65, 769}}, // Example: Unicode point A decomposes into A + combining acute accent + {231, {99, 807}} // Example: Unicode point รง decomposes into c + combining cedilla +}; + +const std::unordered_map unicode_canonical_class = { + {65, 0}, // Example: Unicode point A has canonical class 0 + {769, 1}, // Example: Combining acute accent has canonical class 1 + {99, 0}, // Example: Unicode point c has canonical class 0 + {807, 1} // Example: Combining cedilla has canonical class 1 +}; diff --git a/unicode-data.h b/unicode-data.h index cb9dd8aa5403c..0878ac15e3afa 100644 --- a/unicode-data.h +++ b/unicode-data.h @@ -4,6 +4,7 @@ #include #include #include +#include extern const std::vector> unicode_ranges_digit; extern const std::vector> unicode_ranges_letter; @@ -14,3 +15,5 @@ extern const std::vector> unicode_ranges_symbol; extern const std::vector> unicode_ranges_control; extern const std::multimap unicode_map_nfd; extern const std::map unicode_map_lowercase; +extern const std::unordered_map> unicode_decompose_map; +extern const std::unordered_map unicode_canonical_class; diff --git a/unicode.cpp b/unicode.cpp index f2ccda05ff8c3..1d2c9f2f3621f 100644 --- a/unicode.cpp +++ b/unicode.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include static std::string unicode_cpts_to_utf8(const std::vector & cps) { std::string result; @@ -469,6 +471,68 @@ std::string unicode_cpt_to_utf8(uint32_t cp) { throw std::invalid_argument("invalid codepoint"); } +// Function to recursively decompose a string +std::vector decompose_cpts(const std::vector & cpts) { + std::vector result; + for (const auto& cpt : cpts) { + auto it = unicode_decompose_map.find(cpt); + if (it != unicode_decompose_map.end()) { + for (const auto& decomp: it->second) { + const auto & inner_result = decompose_cpts({decomp}); + result.insert(result.end(), inner_result.begin(), inner_result.end()); + } + } else { + result.push_back(cpt); + } + } + return result; +} + +// Function to sort subsequences based on canonical class +std::vector sort_by_canonical_class(const std::vector & cpts) { + std::vector subsequence; + std::vector result; + auto compareByCanonicalClass = [&](const uint32_t& a, const uint32_t& b) { + auto cc_a_it = unicode_canonical_class.find(a); + if (cc_a_it != unicode_canonical_class.end()) { + auto cc_b_it = unicode_canonical_class.find(b); + if (cc_b_it != unicode_canonical_class.end()) { + return cc_a_it->second < cc_b_it->second; + } + + } + return false; + }; + + for (const auto& cpt : cpts) { + auto it = unicode_canonical_class.find(cpt); + if (it != unicode_canonical_class.end()) { + if (it->second > 0) { + subsequence.push_back(cpt); + } else { + if (!subsequence.empty()) { + sort(subsequence.begin(), subsequence.end(), compareByCanonicalClass); + for (const auto& codepoint : subsequence) { + result.push_back(codepoint); + } + subsequence.clear(); + } + + result.push_back(cpt); + } + } + } + + if (!subsequence.empty()) { + sort(subsequence.begin(), subsequence.end(), compareByCanonicalClass); + for (const auto& codepoint : subsequence) { + result.push_back(codepoint); + } + } + + return result; +} + std::vector unicode_cpts_normalize_nfd(const std::vector & cpts) { std::vector result; result.reserve(cpts.size()); @@ -483,6 +547,14 @@ std::vector unicode_cpts_normalize_nfd(const std::vector & c return result; } + +std::vector unicode_cpts_normalize_nfc(const std::vector & cpts) { + const auto &decomposed_cpts = decompose_cpts(cpts); + const auto &sorted_sequence = sort_by_canonical_class(decomposed_cpts); + //TODO: Do canonical composition + return sorted_sequence; +} + std::vector unicode_cpts_from_utf8(const std::string & utf8) { std::vector result; size_t offset = 0; diff --git a/unicode.h b/unicode.h index ce2bcef5a24b1..ee8411d192cc2 100644 --- a/unicode.h +++ b/unicode.h @@ -17,6 +17,9 @@ std::string unicode_cpt_to_utf8(uint32_t cp); std::vector unicode_cpts_from_utf8(const std::string & utf8); std::vector unicode_cpts_normalize_nfd(const std::vector & cpts); +std::vector unicode_cpts_normalize_nfc(const std::vector & cpts); +std::vector decompose_cpts(const std::vector & cpts); +std::vector sort_by_canonical_class(const std::vector & cpts); int unicode_cpt_type(uint32_t cp); int unicode_cpt_type(const std::string & utf8); From 8957cacd982eea13c2ffac7df5d6a6f07f288b14 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 13 May 2024 09:40:46 +0200 Subject: [PATCH 25/36] refactor: rename jina tokenizers to v2 --- convert-hf-to-gguf-update.py | 6 +++--- convert-hf-to-gguf.py | 6 +++--- llama.cpp | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index cd2674a0ea97d..14aa0c45a6a87 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -74,9 +74,9 @@ class TOKENIZER_TYPE(IntEnum): {"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", }, {"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", }, {"name": "dbrx", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", }, - {"name": "jina-en", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-en", }, # WPM! - {"name": "jina-es", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-es", }, - {"name": "jina-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", }, + {"name": "jina-v2-en", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-en", }, # WPM! + {"name": "jina-v2-es", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-es", }, + {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", }, ] # make directory "models/tokenizers" if it doesn't exist diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index ec7f4dd758c72..d6e5dece0a2c3 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -475,13 +475,13 @@ def get_vocab_base_pre(self, tokenizer) -> str: res = "dbrx" if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f": # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-en - res = "jina-en" + res = "jina-v2-en" if chkhsh == "171aeeedd6fb548d418a7461d053f11b6f1f1fc9b387bd66640d28a4b9f5c643": # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-es - res = "jina-es" + res = "jina-v2-es" if chkhsh == "27949a2493fc4a9f53f5b9b029c82689cfbe5d3a1929bb25e043089e28466de6": # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-de - res = "jina-de" + res = "jina-v2-de" if res is None: logger.warning("\n") diff --git a/llama.cpp b/llama.cpp index e91ad7285da99..5e597ea44b489 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4423,8 +4423,8 @@ static void llm_load_vocab( vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_STARCODER; } else if ( tokenizer_pre == "gpt-2" || - tokenizer_pre == "jina-es" || - tokenizer_pre == "jina-de") { + tokenizer_pre == "jina-v2-es" || + tokenizer_pre == "jina-v2-de") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2; } else if ( tokenizer_pre == "refact") { From 22a011329963834e1a948b53143cf18e5d8aca57 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 13 May 2024 10:27:23 +0200 Subject: [PATCH 26/36] fix: fix alignment --- llama.cpp | 18 ++++++------- llama.h | 1 - unicode-data.h | 5 +--- unicode.cpp | 72 -------------------------------------------------- unicode.h | 3 --- 5 files changed, 10 insertions(+), 89 deletions(-) diff --git a/llama.cpp b/llama.cpp index 92d1eeeca7bb0..eff22bb967579 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8241,9 +8241,6 @@ struct llm_build_context { // KQ_mask (mask for 1 head, it will be broadcasted to all heads) struct ggml_tensor * KQ_mask = build_inp_KQ_mask(false); - // positions of the tokens in the KV cache - struct ggml_tensor * KQ_pos = build_inp_KQ_pos(false); - // iterate layers for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * cur = inpL; @@ -8386,7 +8383,6 @@ struct llm_build_context { // output layer norm cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].layer_out_norm, model.layers[il].layer_out_norm_b, LLM_NORM, cb, il); - // input for next layer inpL = cur; } @@ -11506,7 +11502,7 @@ static int llama_decode_internal( } // non-causal masks do not use the KV cache - if (hparams.causal_attn || model.arch == LLM_ARCH_JINA_BERT_V2) { + if (hparams.causal_attn) { llama_kv_cache_update(&lctx); // if we have enough unused cells before the current head -> @@ -12350,10 +12346,14 @@ struct llm_tokenizer_bpe { break; case LLAMA_VOCAB_PRE_TYPE_JINA_V2_ZH: //TODO: Apply GPT2 + lowercasing - word_collection = unicode_regex_split(text, { - "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)", - }); - //TODO: Apply lowercasing + { + std::string lowercase_text = text; + std::transform(lowercase_text.begin(), lowercase_text.end(), lowercase_text.begin(), [](unsigned char c){ return std::tolower(c); }); + word_collection = unicode_regex_split(lowercase_text, { + "", + }); + } + break; default: // default regex for BPE tokenization pre-processing word_collection = unicode_regex_split(text, { diff --git a/llama.h b/llama.h index d24e3cd965495..5c9fc9a2faa4e 100644 --- a/llama.h +++ b/llama.h @@ -71,7 +71,6 @@ extern "C" { // pre-tokenization types enum llama_vocab_pre_type { - LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, diff --git a/unicode-data.h b/unicode-data.h index fc2ea944f59c6..a9c2fd2588076 100644 --- a/unicode-data.h +++ b/unicode-data.h @@ -4,7 +4,6 @@ #include #include #include -#include extern const std::vector> unicode_ranges_number; extern const std::vector> unicode_ranges_letter; @@ -15,6 +14,4 @@ extern const std::vector> unicode_ranges_punctuati extern const std::vector> unicode_ranges_symbol; extern const std::vector> unicode_ranges_control; extern const std::multimap unicode_map_nfd; -extern const std::map unicode_map_lowercase; -extern const std::unordered_map> unicode_decompose_map; -extern const std::unordered_map unicode_canonical_class; +extern const std::map unicode_map_lowercase; \ No newline at end of file diff --git a/unicode.cpp b/unicode.cpp index c2b9ba9c652ba..ca03c49d39c7c 100644 --- a/unicode.cpp +++ b/unicode.cpp @@ -14,8 +14,6 @@ #include #include #include -#include -#include static std::string unicode_cpts_to_utf8(const std::vector & cps) { std::string result; @@ -590,68 +588,6 @@ std::string unicode_cpt_to_utf8(uint32_t cp) { throw std::invalid_argument("invalid codepoint"); } -// Function to recursively decompose a string -std::vector decompose_cpts(const std::vector & cpts) { - std::vector result; - for (const auto& cpt : cpts) { - auto it = unicode_decompose_map.find(cpt); - if (it != unicode_decompose_map.end()) { - for (const auto& decomp: it->second) { - const auto & inner_result = decompose_cpts({decomp}); - result.insert(result.end(), inner_result.begin(), inner_result.end()); - } - } else { - result.push_back(cpt); - } - } - return result; -} - -// Function to sort subsequences based on canonical class -std::vector sort_by_canonical_class(const std::vector & cpts) { - std::vector subsequence; - std::vector result; - auto compareByCanonicalClass = [&](const uint32_t& a, const uint32_t& b) { - auto cc_a_it = unicode_canonical_class.find(a); - if (cc_a_it != unicode_canonical_class.end()) { - auto cc_b_it = unicode_canonical_class.find(b); - if (cc_b_it != unicode_canonical_class.end()) { - return cc_a_it->second < cc_b_it->second; - } - - } - return false; - }; - - for (const auto& cpt : cpts) { - auto it = unicode_canonical_class.find(cpt); - if (it != unicode_canonical_class.end()) { - if (it->second > 0) { - subsequence.push_back(cpt); - } else { - if (!subsequence.empty()) { - sort(subsequence.begin(), subsequence.end(), compareByCanonicalClass); - for (const auto& codepoint : subsequence) { - result.push_back(codepoint); - } - subsequence.clear(); - } - - result.push_back(cpt); - } - } - } - - if (!subsequence.empty()) { - sort(subsequence.begin(), subsequence.end(), compareByCanonicalClass); - for (const auto& codepoint : subsequence) { - result.push_back(codepoint); - } - } - - return result; -} - std::vector unicode_cpts_normalize_nfd(const std::vector & cpts) { std::vector result; result.reserve(cpts.size()); @@ -666,14 +602,6 @@ std::vector unicode_cpts_normalize_nfd(const std::vector & c return result; } - -std::vector unicode_cpts_normalize_nfc(const std::vector & cpts) { - const auto &decomposed_cpts = decompose_cpts(cpts); - const auto &sorted_sequence = sort_by_canonical_class(decomposed_cpts); - //TODO: Do canonical composition - return sorted_sequence; -} - std::vector unicode_cpts_from_utf8(const std::string & utf8) { std::vector result; size_t offset = 0; diff --git a/unicode.h b/unicode.h index 3f4938d4d855d..d6a14d470bfc3 100644 --- a/unicode.h +++ b/unicode.h @@ -17,9 +17,6 @@ std::string unicode_cpt_to_utf8(uint32_t cp); std::vector unicode_cpts_from_utf8(const std::string & utf8); std::vector unicode_cpts_normalize_nfd(const std::vector & cpts); -std::vector unicode_cpts_normalize_nfc(const std::vector & cpts); -std::vector decompose_cpts(const std::vector & cpts); -std::vector sort_by_canonical_class(const std::vector & cpts); int unicode_cpt_type(uint32_t cp); int unicode_cpt_type(const std::string & utf8); From fb83012096463c27c89a828036cee2c957a3a8e7 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 13 May 2024 10:28:26 +0200 Subject: [PATCH 27/36] refactor: keep refactoring non-breaking --- llama.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llama.cpp b/llama.cpp index 5e597ea44b489..adbcc07e20fc5 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4423,6 +4423,8 @@ static void llm_load_vocab( vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_STARCODER; } else if ( tokenizer_pre == "gpt-2" || + tokenizer_pre == "jina-es" || + tokenizer_pre == "jina-de" || tokenizer_pre == "jina-v2-es" || tokenizer_pre == "jina-v2-de") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2; From cc0ac09712c2ba745431f57bb7da02170c02a335 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Tue, 28 May 2024 20:45:04 +0200 Subject: [PATCH 28/36] feat: add changes to handle jina v2 base code --- convert-hf-to-gguf-update.py | 1 + convert-hf-to-gguf.py | 13 ++++++------- gguf-py/gguf/constants.py | 6 ++++++ gguf-py/gguf/tensor_mapping.py | 13 +++++++++++++ llama.cpp | 24 ++++++++++++++++++++++++ 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index 84b72348dc579..ee1eeaca007b1 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -82,6 +82,7 @@ class TOKENIZER_TYPE(IntEnum): {"name": "jina-v2-es", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-es", }, {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", }, {"name": "smaug-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/abacusai/Smaug-Llama-3-70B-Instruct", }, + {"name": "jina-v2-code", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-code", }, ] diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 1b060e4e6eef0..2ece07f8138c1 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -422,9 +422,6 @@ def get_vocab_base_pre(self, tokenizer) -> str: # NOTE: if you get an error here, you need to update the convert-hf-to-gguf-update.py script # or pull the latest version of the model from Huggingface # don't edit the hashes manually! - if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5": - # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B - res = "llama-bpe" if chkhsh == "049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754": # ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-base res = "deepseek-llm" @@ -461,9 +458,6 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166": # ref: https://huggingface.co/allenai/OLMo-1.7-7B-hf res = "olmo" - if chkhsh == "a8594e3edff7c29c003940395316294b2c623e09894deebbc65f33f1515df79e": - # ref: https://huggingface.co/databricks/dbrx-base - res = "dbrx" if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f": # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-en res = "jina-v2-en" @@ -476,6 +470,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "c136ed14d01c2745d4f60a9596ae66800e2b61fa45643e72436041855ad4089d": # ref: https://huggingface.co/abacusai/Smaug-Llama-3-70B-Instruct res = "smaug-bpe" + if chkhsh == "7967bfa498ade6b757b064f31e964dddbb80f8f9a4d68d4ba7998fcf281c531a": + # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-code + res = "jina-v2-code" if res is None: logger.warning("\n") @@ -2442,11 +2439,13 @@ def __init__(self, *args, **kwargs): def get_tensors(self): for name, data in super().get_tensors(): - if 'gated_layers' in name: + if 'gated_layer' in name: d1 = data[:self.intermediate_size, :] name1 = name.replace('gated_layers', 'gated_layers_w') + name1 = name.replace('up_gated_layer', 'gated_layers_w') d2 = data[self.intermediate_size:, :] name2 = name.replace('gated_layers', 'gated_layers_v') + name2 = name.replace('up_gated_layer', 'gated_layers_v') yield name1, d1 yield name2, d2 continue diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 55ec2cb5c848a..47a108779bab7 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -186,6 +186,8 @@ class MODEL_TENSOR(IntEnum): ATTN_Q_NORM = auto() ATTN_K_NORM = auto() LAYER_OUT_NORM = auto() + LAYER_NORM_1 = auto() + LAYER_NORM_2 = auto() SSM_IN = auto() SSM_CONV1D = auto() SSM_X = auto() @@ -274,6 +276,8 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN_EXP: "blk.{bid}.ffn_down_exps", MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up_exps", MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm", + MODEL_TENSOR.LAYER_NORM_1: "blk.{bid}.layer_norm_1", + MODEL_TENSOR.LAYER_NORM_2: "blk.{bid}.layer_norm_2", MODEL_TENSOR.SSM_IN: "blk.{bid}.ssm_in", MODEL_TENSOR.SSM_CONV1D: "blk.{bid}.ssm_conv1d", MODEL_TENSOR.SSM_X: "blk.{bid}.ssm_x", @@ -426,6 +430,8 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.LAYER_OUT_NORM, + MODEL_TENSOR.LAYER_NORM_1, + MODEL_TENSOR.LAYER_NORM_2, ], MODEL_ARCH.MPT: [ MODEL_TENSOR.TOKEN_EMBD, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 83e3c4c3381a0..ea139339c5511 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -311,6 +311,7 @@ class TensorNameMap: "model.layers.{bid}.mlp.c_proj", # starcoder2 "encoder.layer.{bid}.mlp.wo", # jina-bert-v2 "model.layers.{bid}.residual_mlp.w2", # arctic + "encoder.layer.{bid}.mlp.down_layer", # jina-bert-v2 ), MODEL_TENSOR.FFN_DOWN_EXP: ( @@ -350,6 +351,18 @@ class TensorNameMap: "encoder.layers.{bid}.norm2", # nomic-bert "transformer.decoder_layer.{bid}.rms_norm_3", # Grok "encoder.layer.{bid}.mlp.layernorm", # jina-bert-v2 + "encoder.layer.{bid}.layer_norm_1", # jina-v2-code + "encoder.layer.{bid}.layer_norm_2" # jina-v2-code + ), + + + MODEL_TENSOR.LAYER_NORM_1: ( + "encoder.layer.{bid}.layer_norm_1", # jina-v2-code + ), + + + MODEL_TENSOR.LAYER_NORM_2: ( + "encoder.layer.{bid}.layer_norm_2", # jina-v2-code ), MODEL_TENSOR.SSM_IN: ( diff --git a/llama.cpp b/llama.cpp index 10c9e47dd62ef..229b63a299ba6 100644 --- a/llama.cpp +++ b/llama.cpp @@ -496,6 +496,8 @@ enum llm_tensor { LLM_TENSOR_ATTN_KV_B, LLM_TENSOR_ATTN_Q_A_NORM, LLM_TENSOR_ATTN_KV_A_NORM, + LLM_TENSOR_LAYER_NORM_1, + LLM_TENSOR_LAYER_NORM_2, }; static const std::map> LLM_TENSOR_NAMES = { @@ -717,6 +719,8 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + { LLM_TENSOR_LAYER_NORM_1, "blk.%d.layer_norm_1" }, + { LLM_TENSOR_LAYER_NORM_2, "blk.%d.layer_norm_2" }, }, }, { @@ -2010,6 +2014,12 @@ struct llama_layer { struct ggml_tensor * layer_out_norm_b; struct ggml_tensor * ffn_norm_exps; + // extra normalization layers needed by `jina-embeddings-v2-base-code` + struct ggml_tensor * layer_norm_1; + struct ggml_tensor * layer_norm_1_b; + struct ggml_tensor * layer_norm_2; + struct ggml_tensor * layer_norm_2_b; + // ff struct ggml_tensor * ffn_gate; // w1 struct ggml_tensor * ffn_down; // w2 @@ -5537,6 +5547,12 @@ static bool llm_load_tensors( layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}); //output_norm layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}); + layer.layer_norm_1 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_NORM_1, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.layer_norm_1_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_NORM_1, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + layer.layer_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_NORM_2, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.layer_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_NORM_2, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); @@ -8500,6 +8516,14 @@ struct llm_build_context { // attention layer norm cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_out_norm, model.layers[il].attn_out_norm_b, LLM_NORM, cb, il); + if (model.layers[il].layer_norm_1 != nullptr) { + cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].layer_norm_1, model.layers[il].layer_norm_1_b, LLM_NORM, cb, il); + } + + if (model.layers[il].layer_norm_2 != nullptr) { + cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].layer_norm_2, model.layers[il].layer_norm_2_b, LLM_NORM, cb, il); + } + struct ggml_tensor * ffn_inp = cur; cb(ffn_inp, "ffn_inp", il); From 21936ddb5d2ee0d9345479330556198564c72fc7 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Tue, 28 May 2024 21:06:12 +0200 Subject: [PATCH 29/36] fix: do not complicate things --- gguf-py/gguf/constants.py | 7 +------ gguf-py/gguf/tensor_mapping.py | 12 +----------- llama.cpp | 29 +++++++---------------------- 3 files changed, 9 insertions(+), 39 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 47a108779bab7..539808bcc8bb1 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -186,8 +186,6 @@ class MODEL_TENSOR(IntEnum): ATTN_Q_NORM = auto() ATTN_K_NORM = auto() LAYER_OUT_NORM = auto() - LAYER_NORM_1 = auto() - LAYER_NORM_2 = auto() SSM_IN = auto() SSM_CONV1D = auto() SSM_X = auto() @@ -276,8 +274,6 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN_EXP: "blk.{bid}.ffn_down_exps", MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up_exps", MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm", - MODEL_TENSOR.LAYER_NORM_1: "blk.{bid}.layer_norm_1", - MODEL_TENSOR.LAYER_NORM_2: "blk.{bid}.layer_norm_2", MODEL_TENSOR.SSM_IN: "blk.{bid}.ssm_in", MODEL_TENSOR.SSM_CONV1D: "blk.{bid}.ssm_conv1d", MODEL_TENSOR.SSM_X: "blk.{bid}.ssm_x", @@ -430,8 +426,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_GATE, MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.LAYER_OUT_NORM, - MODEL_TENSOR.LAYER_NORM_1, - MODEL_TENSOR.LAYER_NORM_2, + MODEL_TENSOR.ATTN_NORM_2, ], MODEL_ARCH.MPT: [ MODEL_TENSOR.TOKEN_EMBD, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index ea139339c5511..81b4992a51eed 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -102,6 +102,7 @@ class TensorNameMap: # Attention norm 2 MODEL_TENSOR.ATTN_NORM_2: ( "transformer.h.{bid}.ln_attn", # falcon40b + "encoder.layer.{bid}.layer_norm_1", # jina-v2-code ), # Attention query-key-value @@ -351,20 +352,9 @@ class TensorNameMap: "encoder.layers.{bid}.norm2", # nomic-bert "transformer.decoder_layer.{bid}.rms_norm_3", # Grok "encoder.layer.{bid}.mlp.layernorm", # jina-bert-v2 - "encoder.layer.{bid}.layer_norm_1", # jina-v2-code "encoder.layer.{bid}.layer_norm_2" # jina-v2-code ), - - MODEL_TENSOR.LAYER_NORM_1: ( - "encoder.layer.{bid}.layer_norm_1", # jina-v2-code - ), - - - MODEL_TENSOR.LAYER_NORM_2: ( - "encoder.layer.{bid}.layer_norm_2", # jina-v2-code - ), - MODEL_TENSOR.SSM_IN: ( "model.layers.{bid}.in_proj", "backbone.layers.{bid}.mixer.in_proj", diff --git a/llama.cpp b/llama.cpp index 229b63a299ba6..4662f1fdd8483 100644 --- a/llama.cpp +++ b/llama.cpp @@ -496,8 +496,6 @@ enum llm_tensor { LLM_TENSOR_ATTN_KV_B, LLM_TENSOR_ATTN_Q_A_NORM, LLM_TENSOR_ATTN_KV_A_NORM, - LLM_TENSOR_LAYER_NORM_1, - LLM_TENSOR_LAYER_NORM_2, }; static const std::map> LLM_TENSOR_NAMES = { @@ -719,8 +717,7 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, - { LLM_TENSOR_LAYER_NORM_1, "blk.%d.layer_norm_1" }, - { LLM_TENSOR_LAYER_NORM_2, "blk.%d.layer_norm_2" }, + { LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" }, }, }, { @@ -2014,12 +2011,6 @@ struct llama_layer { struct ggml_tensor * layer_out_norm_b; struct ggml_tensor * ffn_norm_exps; - // extra normalization layers needed by `jina-embeddings-v2-base-code` - struct ggml_tensor * layer_norm_1; - struct ggml_tensor * layer_norm_1_b; - struct ggml_tensor * layer_norm_2; - struct ggml_tensor * layer_norm_2_b; - // ff struct ggml_tensor * ffn_gate; // w1 struct ggml_tensor * ffn_down; // w2 @@ -4680,7 +4671,8 @@ static void llm_load_vocab( tokenizer_pre == "jina-es" || tokenizer_pre == "jina-de" || tokenizer_pre == "jina-v2-es" || - tokenizer_pre == "jina-v2-de") { + tokenizer_pre == "jina-v2-de" || + tokenizer_pre == "jina-v2-code") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2; } else if ( tokenizer_pre == "refact") { @@ -5547,12 +5539,9 @@ static bool llm_load_tensors( layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}); //output_norm layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}); - layer.layer_norm_1 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_NORM_1, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); - layer.layer_norm_1_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_NORM_1, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); - layer.layer_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_NORM_2, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); - layer.layer_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_NORM_2, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); - layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); @@ -8516,12 +8505,8 @@ struct llm_build_context { // attention layer norm cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_out_norm, model.layers[il].attn_out_norm_b, LLM_NORM, cb, il); - if (model.layers[il].layer_norm_1 != nullptr) { - cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].layer_norm_1, model.layers[il].layer_norm_1_b, LLM_NORM, cb, il); - } - - if (model.layers[il].layer_norm_2 != nullptr) { - cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].layer_norm_2, model.layers[il].layer_norm_2_b, LLM_NORM, cb, il); + if (model.layers[il].attn_norm_2 != nullptr) { + cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_norm_2, model.layers[il].attn_norm_2_b, LLM_NORM, cb, il); } struct ggml_tensor * ffn_inp = cur; From 9a65c7a2732bc90e0e69839a8e238fb5749c7ac3 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Fri, 31 May 2024 15:10:43 +0200 Subject: [PATCH 30/36] fix: fix the usage of the code model --- convert-hf-to-gguf.py | 4 ++-- llama.cpp | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 2ece07f8138c1..c04cd8dffd363 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -2442,10 +2442,10 @@ def get_tensors(self): if 'gated_layer' in name: d1 = data[:self.intermediate_size, :] name1 = name.replace('gated_layers', 'gated_layers_w') - name1 = name.replace('up_gated_layer', 'gated_layers_w') + name1 = name.replace('up_gated_layer', 'gated_layers_v') d2 = data[self.intermediate_size:, :] name2 = name.replace('gated_layers', 'gated_layers_v') - name2 = name.replace('up_gated_layer', 'gated_layers_v') + name2 = name.replace('up_gated_layer', 'gated_layers_w') yield name1, d1 yield name2, d2 continue diff --git a/llama.cpp b/llama.cpp index 4662f1fdd8483..bc81963d0980c 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5498,7 +5498,7 @@ static bool llm_load_tensors( layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); } else { - layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); } layer.layer_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}); @@ -8506,6 +8506,8 @@ struct llm_build_context { cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_out_norm, model.layers[il].attn_out_norm_b, LLM_NORM, cb, il); if (model.layers[il].attn_norm_2 != nullptr) { + // re-add the layer input + cur = ggml_add(ctx0, cur, inpL); cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_norm_2, model.layers[il].attn_norm_2_b, LLM_NORM, cb, il); } From 4bce30cc0e87706e3eb9a9e17430635f8c7d3245 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Tue, 4 Jun 2024 17:08:47 +0200 Subject: [PATCH 31/36] fix: fix comments --- convert-hf-to-gguf.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 050f98e3a6fff..0669addb3edab 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -420,6 +420,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: # NOTE: if you get an error here, you need to update the convert-hf-to-gguf-update.py script # or pull the latest version of the model from Huggingface # don't edit the hashes manually! + if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5": + # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B + res = "llama-bpe" if chkhsh == "049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754": # ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-base res = "deepseek-llm" @@ -456,6 +459,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166": # ref: https://huggingface.co/allenai/OLMo-1.7-7B-hf res = "olmo" + if chkhsh == "a8594e3edff7c29c003940395316294b2c623e09894deebbc65f33f1515df79e": + # ref: https://huggingface.co/databricks/dbrx-base + res = "dbrx" if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f": # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-en res = "jina-v2-en" @@ -2451,10 +2457,10 @@ def get_tensors(self): if 'gated_layer' in name: d1 = data[:self.intermediate_size, :] name1 = name.replace('gated_layers', 'gated_layers_w') - name1 = name.replace('up_gated_layer', 'gated_layers_v') + name1 = name1.replace('up_gated_layer', 'gated_layers_v') d2 = data[self.intermediate_size:, :] name2 = name.replace('gated_layers', 'gated_layers_v') - name2 = name.replace('up_gated_layer', 'gated_layers_w') + name2 = name2.replace('up_gated_layer', 'gated_layers_w') yield name1, d1 yield name2, d2 continue From 3b44f8f658601933778e7b43bb9fd84aa6ac95fc Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Wed, 5 Jun 2024 08:53:26 +0200 Subject: [PATCH 32/36] fix: fix linting issues --- convert-hf-to-gguf.py | 2 +- llama.cpp | 41 ++++++++++++++++++++--------------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 0669addb3edab..6632bd95a23f4 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -422,7 +422,7 @@ def get_vocab_base_pre(self, tokenizer) -> str: # don't edit the hashes manually! if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5": # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B - res = "llama-bpe" + res = "llama-bpe" if chkhsh == "049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754": # ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-base res = "deepseek-llm" diff --git a/llama.cpp b/llama.cpp index b6d4662ed7839..8621591e0b3d6 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4653,16 +4653,7 @@ static void llm_load_vocab( // for now, only BPE models have pre-tokenizers if (vocab.type == LLAMA_VOCAB_TYPE_BPE) { - if (tokenizer_pre.empty()) { - LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__); - LLAMA_LOG_WARN("%s: \n", __func__); - LLAMA_LOG_WARN("%s: ************************************ \n", __func__); - LLAMA_LOG_WARN("%s: GENERATION QUALITY WILL BE DEGRADED! \n", __func__); - LLAMA_LOG_WARN("%s: CONSIDER REGENERATING THE MODEL \n", __func__); - LLAMA_LOG_WARN("%s: ************************************ \n", __func__); - LLAMA_LOG_WARN("%s: \n", __func__); - vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; - } else if ( + if ( tokenizer_pre == "default") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; } else if ( @@ -4715,7 +4706,8 @@ static void llm_load_vocab( tokenizer_pre == "smaug-bpe") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMAUG; } else { - throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); + LLAMA_LOG_WARN("%s: missing or unrecognized pre-tokenizer type, using: 'default'\n", __func__); + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; } } else { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; @@ -5569,7 +5561,7 @@ static bool llm_load_tensors( layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); - layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); @@ -6631,7 +6623,7 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam } } catch (const std::exception & err) { LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what()); - return -1; + throw; } return 0; @@ -16254,16 +16246,23 @@ struct llama_model * llama_load_model_from_file( } model->rpc_servers.push_back(servers); } - int status = llama_model_load(path_model, *model, params); - GGML_ASSERT(status <= 0); - if (status < 0) { - if (status == -1) { - LLAMA_LOG_ERROR("%s: failed to load model\n", __func__); - } else if (status == -2) { - LLAMA_LOG_INFO("%s: cancelled model load\n", __func__); + + try { + int status = llama_model_load(path_model, *model, params); + GGML_ASSERT(status <= 0); + if (status < 0) { + if (status == -1) { + LLAMA_LOG_ERROR("%s: failed to load model\n", __func__); + } else if (status == -2) { + LLAMA_LOG_INFO("%s: cancelled model load\n", __func__); + } + delete model; + return nullptr; } + } catch (...) { + LLAMA_LOG_ERROR("%s: exception loading model\n", __func__); delete model; - return nullptr; + throw; } return model; From 05659d3c7b160f804297b57ea26870929a5a1155 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Wed, 5 Jun 2024 09:15:36 +0200 Subject: [PATCH 33/36] fix: remove ollama patches --- llama.cpp | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/llama.cpp b/llama.cpp index 8621591e0b3d6..e76da869cef68 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4653,8 +4653,16 @@ static void llm_load_vocab( // for now, only BPE models have pre-tokenizers if (vocab.type == LLAMA_VOCAB_TYPE_BPE) { - if ( - tokenizer_pre == "default") { + if (tokenizer_pre.empty()) { + LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__); + LLAMA_LOG_WARN("%s: \n", __func__); + LLAMA_LOG_WARN("%s: ************************************ \n", __func__); + LLAMA_LOG_WARN("%s: GENERATION QUALITY WILL BE DEGRADED! \n", __func__); + LLAMA_LOG_WARN("%s: CONSIDER REGENERATING THE MODEL \n", __func__); + LLAMA_LOG_WARN("%s: ************************************ \n", __func__); + LLAMA_LOG_WARN("%s: \n", __func__); + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; + } else if (tokenizer_pre == "default") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; } else if ( tokenizer_pre == "llama3" || @@ -4706,8 +4714,7 @@ static void llm_load_vocab( tokenizer_pre == "smaug-bpe") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMAUG; } else { - LLAMA_LOG_WARN("%s: missing or unrecognized pre-tokenizer type, using: 'default'\n", __func__); - vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; + throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); } } else { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; @@ -6623,7 +6630,7 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam } } catch (const std::exception & err) { LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what()); - throw; + return -1; } return 0; @@ -16246,23 +16253,16 @@ struct llama_model * llama_load_model_from_file( } model->rpc_servers.push_back(servers); } - - try { - int status = llama_model_load(path_model, *model, params); - GGML_ASSERT(status <= 0); - if (status < 0) { - if (status == -1) { - LLAMA_LOG_ERROR("%s: failed to load model\n", __func__); - } else if (status == -2) { - LLAMA_LOG_INFO("%s: cancelled model load\n", __func__); - } - delete model; - return nullptr; + int status = llama_model_load(path_model, *model, params); + GGML_ASSERT(status <= 0); + if (status < 0) { + if (status == -1) { + LLAMA_LOG_ERROR("%s: failed to load model\n", __func__); + } else if (status == -2) { + LLAMA_LOG_INFO("%s: cancelled model load\n", __func__); } - } catch (...) { - LLAMA_LOG_ERROR("%s: exception loading model\n", __func__); delete model; - throw; + return nullptr; } return model; From a8a64fd0733ff0c4ec6c52348bc5292b72000f6a Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Thu, 6 Jun 2024 10:15:07 +0200 Subject: [PATCH 34/36] fix: fix preprocessing jina v2 zh --- llama.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/llama.cpp b/llama.cpp index 20ac0f9168674..aaf22944c1854 100644 --- a/llama.cpp +++ b/llama.cpp @@ -13038,13 +13038,17 @@ struct llm_tokenizer_bpe { }); break; case LLAMA_VOCAB_PRE_TYPE_JINA_V2_ZH: - //TODO: Apply GPT2 + lowercasing + //TODO: Apply lowercase + whitespace pretokenization { std::string lowercase_text = text; std::transform(lowercase_text.begin(), lowercase_text.end(), lowercase_text.begin(), [](unsigned char c){ return std::tolower(c); }); - word_collection = unicode_regex_split(lowercase_text, { - "", - }); + std::regex regexPattern("\\w+|[^\\w\\s]+"); + std::sregex_token_iterator it(lowercase_text.begin(), lowercase_text.end(), regexPattern); + std::sregex_token_iterator end; + + while (it != end) { + word_collection.push_back(*it++); + } } break; default: @@ -13153,10 +13157,9 @@ struct llm_tokenizer_bpe { for (auto j = str.begin(); j != str.end(); ++j) { std::string byte_str(1, *j); auto token_multibyte = vocab.token_to_id.find(byte_str); - if (token_multibyte == vocab.token_to_id.end()) { - throw std::runtime_error("ERROR: byte not found in vocab"); + if (token_multibyte != vocab.token_to_id.end()) { + output.push_back((*token_multibyte).second); } - output.push_back((*token_multibyte).second); } } else { output.push_back((*token).second); From 728e1b4da0cbed99b817016115ec1a30f7281d61 Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Fri, 7 Jun 2024 09:55:21 +0200 Subject: [PATCH 35/36] fix: lowercase unicode pt by unicode pt --- llama.cpp | 3 +-- unicode.cpp | 14 ++++++++++++++ unicode.h | 2 ++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index d060da871b211..f95ecd39ad07f 100644 --- a/llama.cpp +++ b/llama.cpp @@ -13039,8 +13039,7 @@ struct llm_tokenizer_bpe { case LLAMA_VOCAB_PRE_TYPE_JINA_V2_ZH: //TODO: Apply lowercase + whitespace pretokenization { - std::string lowercase_text = text; - std::transform(lowercase_text.begin(), lowercase_text.end(), lowercase_text.begin(), [](unsigned char c){ return std::tolower(c); }); + std::string lowercase_text = lowercase(text); std::regex regexPattern("\\w+|[^\\w\\s]+"); std::sregex_token_iterator it(lowercase_text.begin(), lowercase_text.end(), regexPattern); std::sregex_token_iterator end; diff --git a/unicode.cpp b/unicode.cpp index 056a4c74172c7..695eb6f3e226f 100644 --- a/unicode.cpp +++ b/unicode.cpp @@ -794,3 +794,17 @@ std::vector unicode_regex_split(const std::string & text, const std return unicode_byte_encoding_process(bpe_words); } + + + +std::string lowercase(const std::string & text) { + std::string lowercase(""); + const std::vector cpts = unicode_cpts_from_utf8(text); + + for (const char32_t cpt : cpts) { + const std::string s = unicode_cpt_to_utf8(unicode_tolower(cpt)); + lowercase += unicode_cpt_to_utf8(unicode_tolower(cpt)); // append char to word + } + + return lowercase; +} diff --git a/unicode.h b/unicode.h index 7513be4ad0d4f..9b6317c6045d9 100644 --- a/unicode.h +++ b/unicode.h @@ -61,3 +61,5 @@ uint8_t unicode_utf8_to_byte(const std::string & utf8); char32_t unicode_tolower(char32_t cp); std::vector unicode_regex_split(const std::string & text, const std::vector & regex_exprs); + +std::string lowercase(const std::string & text); From afd76e62547001d7d2bff9227d6d7a2875f301df Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Mon, 8 Jul 2024 15:40:27 +0200 Subject: [PATCH 36/36] fix: handle default --- src/llama.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index 8ab94d9ef926b..2879a5348bc4c 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -15468,6 +15468,15 @@ struct llm_tokenizer_bpe { case LLAMA_VOCAB_PRE_TYPE_JINA_V2_ZH: regex_exprs = {"\\w+|[^\\w\\s]+"}; break; + default: + // default regex for BPE tokenization pre-processing + regex_exprs = { + "[\\p{P}\\$\\+<=>\\^~\\|]+", + "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)", + "\\p{N}+", + "[0-9][0-9][0-9]", + }; + break; } }