From 36c71abcfb7c796bd54398fb4bc19f925ac7817c Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Wed, 25 Oct 2023 13:57:00 +0100 Subject: [PATCH 1/7] Fixed `LLama.StreamingTokenDecoderLLamaLLama.StreamingTokenDecoderLLamaLLama.StreamingTokenDecoderLLama` spam in all executors except Stateless. --- LLama/LLamaContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LLama/LLamaContext.cs b/LLama/LLamaContext.cs index 46b0ae3f8..47240dc71 100644 --- a/LLama/LLamaContext.cs +++ b/LLama/LLamaContext.cs @@ -110,7 +110,7 @@ public string DeTokenize(IReadOnlyList tokens) var decoder = new StreamingTokenDecoder(this); decoder.AddRange(tokens); - return decoder.ToString(); + return decoder.Read(); } /// From b4e57f7966407077312eeda1a6fabb92b8023b3e Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Wed, 25 Oct 2023 16:22:40 +0100 Subject: [PATCH 2/7] Added extra diagnostics to test runner --- .github/workflows/main.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 977603075..dfb49b79e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,4 +52,9 @@ jobs: - name: Build run: dotnet build LLamaSharp.sln -c ${{ matrix.config }} --no-restore - name: Test - run: dotnet test LLamaSharp.sln -c ${{ matrix.config }} -l "console;verbosity=detailed" + run: dotnet test LLamaSharp.sln -c ${{ matrix.config }} -l "console;verbosity=detailed" --diag:logs/log.txt + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: logs/ + name: logs From 8b582a4cc8ce4e98e5d4db640d8cc0b5de9064cf Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Wed, 25 Oct 2023 16:30:22 +0100 Subject: [PATCH 3/7] Added `if: always()` to force logs to be written even if earlier steps fail --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dfb49b79e..ce22cd046 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,6 +54,7 @@ jobs: - name: Test run: dotnet test LLamaSharp.sln -c ${{ matrix.config }} -l "console;verbosity=detailed" --diag:logs/log.txt - name: Upload artifacts + if: always() uses: actions/upload-artifact@v3 with: path: logs/ From d5874a279ca4b4bbd7e9463de201171da188b320 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Wed, 25 Oct 2023 16:57:19 +0100 Subject: [PATCH 4/7] Updated test runner to latest version --- LLama.Unittest/LLama.Unittest.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LLama.Unittest/LLama.Unittest.csproj b/LLama.Unittest/LLama.Unittest.csproj index b6b3f0b0f..596e5da90 100644 --- a/LLama.Unittest/LLama.Unittest.csproj +++ b/LLama.Unittest/LLama.Unittest.csproj @@ -11,7 +11,7 @@ - + From a769ab3a405c98ab8e58875795074e1abcbcc2b2 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Wed, 25 Oct 2023 17:59:33 +0100 Subject: [PATCH 5/7] Logging out events from the OOM killer --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ce22cd046..d4bedf37b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,6 +53,9 @@ jobs: run: dotnet build LLamaSharp.sln -c ${{ matrix.config }} --no-restore - name: Test run: dotnet test LLamaSharp.sln -c ${{ matrix.config }} -l "console;verbosity=detailed" --diag:logs/log.txt + - name: OOM Killer Log + if: runner.os == 'Linux' + run: dmesg -T | egrep -i 'killed process' > logs/dmesg.log - name: Upload artifacts if: always() uses: actions/upload-artifact@v3 From b72b2535495ae7fd7b8e64aea1cb572df77750e2 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Wed, 25 Oct 2023 18:05:04 +0100 Subject: [PATCH 6/7] removed OOM killer step (not allowed on GitHub actions) --- .github/workflows/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d4bedf37b..ce22cd046 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,9 +53,6 @@ jobs: run: dotnet build LLamaSharp.sln -c ${{ matrix.config }} --no-restore - name: Test run: dotnet test LLamaSharp.sln -c ${{ matrix.config }} -l "console;verbosity=detailed" --diag:logs/log.txt - - name: OOM Killer Log - if: runner.os == 'Linux' - run: dmesg -T | egrep -i 'killed process' > logs/dmesg.log - name: Upload artifacts if: always() uses: actions/upload-artifact@v3 From bb5dedf5eb9d3a1214849e6680a19aa87f78db2d Mon Sep 17 00:00:00 2001 From: Rinne Date: Thu, 26 Oct 2023 02:22:03 +0800 Subject: [PATCH 7/7] docs: create CONTRIBUTING.md --- CONTRIBUTING.md | 65 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..c8acd2da3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,65 @@ +# LLamaSharp Contributing Guide + +Hi, welcome to develop LLamaSharp with us together! We are always open for every contributor and any format of contributions! If you want to maintain this library actively together, please contact us to get the write access after some PRs. (Email: AsakusaRinne@gmail.com) + +In this page, we'd like to introduce how to make contributions here easily. 😊 + +## Compile the native library from source + +Firstly, please clone the [llama.cpp](https://github.com/ggerganov/llama.cpp) repository and following the instructions in [llama.cpp readme](https://github.com/ggerganov/llama.cpp#build) to configure your local environment. + +If you want to support cublas in the compilation, please make sure that you've installed the cuda. + +When building from source, please add `-DBUILD_SHARED_LIBS=ON` to the cmake instruction. For example, when building with cublas but without openblas, use the following instruction: + +```bash +cmake .. -DLLAMA_CUBLAS=ON -DBUILD_SHARED_LIBS=ON +``` + +After running `cmake --build . --config Release`, you could find the `llama.dll`, `llama.so` or `llama.dylib` in your build directory. After pasting it to `LLamaSharp/LLama/runtimes` and renaming it to `libllama.dll`, `libllama.so` or `libllama.dylib`, you can use it as the native library in LLamaSharp. + + +## Add a new feature to LLamaSharp + +After refactoring the framework in `v0.4.0`, LLamaSharp will try to maintain the backward compatibility. However, in the following cases a breaking change will be required: + +1. Due to some break changes in [llama.cpp](https://github.com/ggerganov/llama.cpp), making a breaking change will help to maintain the good abstraction and friendly user APIs. +2. An important feature cannot be implemented unless refactoring some parts. +3. After some discussions, an agreement was reached that making the break change is reasonable. + +If a new feature could be added without introducing any break change, please **open a PR** rather than open an issue first. We will never refuse the PR but help to improve it, unless it's malicious. + +When adding the feature, please take care of the namespace and the naming convention. For example, if you are adding an integration for WPF, please put the code under namespace `LLama.WPF` or `LLama.Integration.WPF` instead of putting it under the root namespace. The naming convention of LLamaSharp follows the pascal naming convention, but in some parts that are invisible to users, you can do whatever you want. + +## Find the problem and fix the BUG + +If the issue is related to the LLM internal behaviour, such as endless generating the response, the best way to find the problem is to do comparison test between llama.cpp and LLamaSharp. + +You could use exactly the same prompt, the same model and the same parameters to run the inference in llama.cpp and LLamaSharp respectively to see if it's really a problem caused by the implementation in LLamaSharp. + +If the experiment showed that it worked well in llama.cpp but didn't in LLamaSharp, a search for the problem could be started. While the reason of the problem could be various, the best way I think is to add log-print in the code of llama.cpp and use it in LLamaSharp after compilation. Thus, when running LLamaSharp, you could see what happened in the native library. + +After finding out the reason, a painful but happy process comes. When working on the BUG fix, there's only one rule to follow, that is keeping the examples working well. If the modification fixed the BUG but impact on other functions, it would not be a good fix. + +During the BUG fix process, please don't hesitate to discuss together when you stuck on something. + +## Add integrations + +All kinds of integration are welcomed here! Currently the following integrations are under work or on our schedule: + +1. BotSharp +2. semantic-kernel +3. Unity + +Besides, for some other integrations, like `ASP.NET core`, `SQL`, `Blazor` and so on, we'll appreciate it if you could help with that. If the time is limited for you, providing an example for it also means a lot! + +## Add examples + +There're mainly two ways to add an example: + +1. Add the example to `LLama.Examples` of the repository. +2. Put the example in another repository and add the link to the readme or docs of LLamaSharp. + +## Add documents + +LLamaSharp uses [mkdocs](https://github.com/mkdocs/mkdocs) to build the documentation, please follow the tutorial of mkdocs to add or modify documents in LLamaSharp.