Skip to content

Commit

Permalink
Enhance model handling with improved logic and performance optimizati…
Browse files Browse the repository at this point in the history 
…ons (#90)
  • Loading branch information
@lcarrere
lcarrere authored Dec 26, 2024
1 parent 9407192 commit d25df8b
Show file tree
Hide file tree
Showing 16 changed files with 108 additions and 93 deletions.
2 changes: 1 addition & 1 deletion LM-Kit-Maestro/Converters/LevelToColorConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ public object Convert(object value, Type targetType, object parameter, CultureIn
{
if (value is float floatValue)
{
int h = (int)(120 * Math.Pow(floatValue, 3.5));
int h = (int)(120 * Math.Pow(floatValue, 2));
int s = 100;
int v = 100;

Expand Down
6 changes: 3 additions & 3 deletions LM-Kit-Maestro/LM-Kit-Maestro.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,11 @@
</ItemGroup>

<ItemGroup Condition="'$(OS)' == 'Windows_NT'">
<PackageReference Include="LM-Kit.NET.Backend.Cuda12.Windows" Version="2024.12.11" />
<PackageReference Include="LM-Kit.NET.Backend.Cuda12.Windows" Version="2024.12.12" />
</ItemGroup>

<ItemGroup>
<PackageReference Include="LM-Kit.NET" Version="2024.12.11" />
<PackageReference Include="LM-Kit.NET" Version="2024.12.12" />
<PackageReference Include="Majorsoft.Blazor.Components.Common.JsInterop" Version="1.5.0" />
<PackageReference Include="Markdig" Version="0.39.1" />
<PackageReference Include="Microsoft.Maui.Controls" Version="9.0.21" />
Expand All @@ -86,7 +86,7 @@
<PackageReference Include="CommunityToolkit.Mvvm" Version="8.4.0" />
<PackageReference Include="MetroLog.Maui" Version="2.1.0" />
<PackageReference Include="Mopups" Version="1.3.2" />
<PackageReference Include="SimpleToolkit.SimpleShell" Version="4.1.3" />
<PackageReference Include="SimpleToolkit.SimpleShell" Version="5.0.0" />
<PackageReference Include="sqlite-net-pcl" Version="1.9.172" />
<PackageReference Include="System.Text.Json" Version="9.0.0" />
</ItemGroup>
Expand Down
10 changes: 2 additions & 8 deletions LM-Kit-Maestro/LM-Kit-Maestro.dev.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,7 @@
</ItemGroup>-->

<ItemGroup>
<MauiCss Include="wwwroot\css\font-awesome.css" />
<MauiCss Include="wwwroot\css\font-awesome.min.css" />
<None Include="wwwroot\css\bootstrap.min.css" />
</ItemGroup>

<ItemGroup>
Expand All @@ -91,13 +90,8 @@
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>

<ItemGroup>
<None Include="wwwroot\css\bootstrap.min.css" />
</ItemGroup>

<ItemGroup>
<PackageReference Include="LM-Kit.NET" Version="2024.12.8" />

<PackageReference Include="Majorsoft.Blazor.Components.Common.JsInterop" Version="1.5.0" />
<PackageReference Include="Markdig" Version="0.39.1" />
<PackageReference Include="Microsoft.Maui.Controls" Version="9.0.21" />
Expand Down
6 changes: 3 additions & 3 deletions LM-Kit-Maestro/Services/AppSettingsService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,15 @@ public string ModelStorageDirectory
}
}

public bool EnableSlowModels
public bool EnableLowPerformanceModels
{
get
{
return Settings.Get(nameof(EnableSlowModels), LMKitDefaultSettings.DefaultEnableSlowModels);
return Settings.Get(nameof(EnableLowPerformanceModels), LMKitDefaultSettings.DefaultEnableLowPerformanceModels);
}
set
{
Settings.Set(nameof(EnableSlowModels), value);
Settings.Set(nameof(EnableLowPerformanceModels), value);
OnPropertyChanged();
}
}
Expand Down
2 changes: 1 addition & 1 deletion LM-Kit-Maestro/Services/Interfaces/IAppSettingsService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,5 @@ public interface IAppSettingsService
public SamplingMode SamplingMode { get; set; }
public RandomSamplingConfig RandomSamplingConfig { get; set; }
public Mirostat2SamplingConfig Mirostat2SamplingConfig { get; set; }
public bool EnableSlowModels { get; set; }
public bool EnableLowPerformanceModels { get; set; }
}
3 changes: 1 addition & 2 deletions LM-Kit-Maestro/Services/Interfaces/ILLMFileManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,10 @@ public interface ILLMFileManager
string ModelStorageDirectory { get; set; }
long TotalModelSize { get; }
int DownloadedCount { get; }
public bool EnableSlowModels { get; set; }
public bool EnableLowPerformanceModels { get; set; }
event EventHandler? FileCollectingCompleted;
void Initialize();
void DeleteModel(ModelCard modelCard);
bool IsPredefinedModel(ModelCard modelCard);
void OnModelDownloaded(ModelCard modelInfo);

public event PropertyChangedEventHandler PropertyChanged;
Expand Down
100 changes: 55 additions & 45 deletions LM-Kit-Maestro/Services/LLMFileManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ namespace LMKit.Maestro.Services;


/// <summary>
/// This service is intended to be used as a singleton via Dependency Injection.
/// This service is intended to be used as a singleton via Dependency Injection.
/// Please register with <c>services.AddSingleton&lt;LLMFileManager&gt;()</c>.
/// </summary>
public partial class LLMFileManager : ObservableObject, ILLMFileManager
Expand All @@ -26,14 +26,18 @@ public partial class LLMFileManager : ObservableObject, ILLMFileManager
private readonly FileSystemEntryRecorder _fileSystemEntryRecorder;
private readonly IAppSettingsService _appSettingsService;
private readonly HttpClient _httpClient;
private bool _enablePredefinedModels = true; //todo: Implement this as a configurable option in the configuration panel
private bool _enableCustomModels = true; //todo: Implement this as a configurable option in the configuration panel
private bool _enablePredefinedModels = true; //todo: Implement this as a configurable option in the configuration panel
//todo: make this user-configurable is some way.
private List<ModelCapabilities> _filteredCapabilities = new List<ModelCapabilities>() { ModelCapabilities.Chat,
ModelCapabilities.Math,
ModelCapabilities.CodeCompletion };
private bool _enableCustomModels = true;
private bool _isLoaded = false;

private readonly Dictionary<Uri, FileDownloader> _fileDownloads = new Dictionary<Uri, FileDownloader>();

private delegate bool ModelDownloadingProgressCallback(string path, long? contentLength, long bytesRead);
public event NotifyCollectionChangedEventHandler? SortedModelCollectionChanged;
public event NotifyCollectionChangedEventHandler? SortedModelCollectionChanged;

private CancellationTokenSource? _cancellationTokenSource;
private Task? _collectModelFilesTask;
Expand All @@ -54,10 +58,7 @@ public partial class LLMFileManager : ObservableObject, ILLMFileManager
private bool _fileCollectingInProgress;

[ObservableProperty]
private bool _enableSlowModels;


private List<ModelCard> _predefinedModelCards = ModelCard.GetPredefinedModelCards();
private bool _enableLowPerformanceModels;


private string _modelStorageDirectory = string.Empty;
Expand Down Expand Up @@ -96,7 +97,7 @@ public LLMFileManager(IAppSettingsService appSettingsService, HttpClient httpCli
Models = new ReadOnlyObservableCollection<ModelCard>(_models);
UnsortedModels = new ReadOnlyObservableCollection<ModelCard>(_unsortedModels);
_appSettingsService = appSettingsService;
_enableSlowModels = _appSettingsService.EnableSlowModels;
_enableLowPerformanceModels = _appSettingsService.EnableLowPerformanceModels;
_httpClient = httpClient;
_models.CollectionChanged += OnModelCollectionChanged;
_unsortedModels.CollectionChanged += OnUnsortedModelCollectionChanged;
Expand Down Expand Up @@ -240,7 +241,7 @@ public void DeleteModel(ModelCard modelCard)
TotalModelSize -= modelCard.FileSize;

#if !WINDOWS
if (!IsPredefinedModel(modelCard))
if (!modelCard.IsPredefined)
{
if (_unsortedModels.Contains(modelCard))
{
Expand All @@ -250,7 +251,7 @@ public void DeleteModel(ModelCard modelCard)
if (_models.Contains(modelCard))
{
_models.Remove(modelCard);
}
}
}
#endif
}
Expand All @@ -260,21 +261,6 @@ public void DeleteModel(ModelCard modelCard)
}
}

public bool IsPredefinedModel(ModelCard modelCard)
{
if (_enablePredefinedModels)
{
foreach (var predefinedModel in ModelCard.GetPredefinedModelCards())
{
if (modelCard.ModelUri == predefinedModel.ModelUri)
{
return true;
}
}
}

return false;
}

private void EnsureModelDirectoryExists()
{
Expand Down Expand Up @@ -353,10 +339,23 @@ private void TerminateFileCollectingOperation()

private void CollectModels()
{

if (_enablePredefinedModels)
{
foreach (var modelCard in _predefinedModelCards)
var predefinedModels = ModelCard.GetPredefinedModelCards(dropSmallerModels: !EnableLowPerformanceModels);

if (_models.Count > 0)
{
for (int index = 0; index < _models.Count; index++)
{
if (_models[index].IsPredefined && !predefinedModels.Contains(_models[index]))
{
_models.RemoveAt(index);
index--;
}
}
}

foreach (var modelCard in ModelCard.GetPredefinedModelCards(dropSmallerModels: !EnableLowPerformanceModels))
{
TryRegisterChatModel(modelCard, isSorted: true);

Expand All @@ -371,7 +370,8 @@ private void CollectModels()
foreach (var filePath in files)
{
bool processed = false;
foreach (var predefinedModel in _predefinedModelCards)

foreach (var predefinedModel in ModelCard.GetPredefinedModelCards(dropSmallerModels: false))
{
if (predefinedModel.LocalPath == filePath)
{//Skip this model because it has already been processed
Expand Down Expand Up @@ -399,18 +399,27 @@ private bool TryRegisterChatModel(ModelCard? modelCard, bool isSorted)
{
if (modelCard != null)
{
if (!modelCard.Capabilities.HasFlag(ModelCapabilities.Chat) &&
!modelCard.Capabilities.HasFlag(ModelCapabilities.CodeCompletion) &&
!modelCard.Capabilities.HasFlag(ModelCapabilities.Math))
bool hasAnyFilteredCap = false;

foreach (var cap in _filteredCapabilities)
{
if (modelCard.Capabilities.HasFlag(cap))
{
hasAnyFilteredCap = true;
break;
}
}

if (!hasAnyFilteredCap)
{
return false;
}

bool isSlowModel = Graphics.DeviceConfiguration.GetPerformanceScore(modelCard) < 0.3;

if (!ContainsModel(_models, modelCard, out _))
if (!ContainsModel(_models, modelCard))
{
if (isSlowModel && !EnableSlowModels)
if (isSlowModel && !EnableLowPerformanceModels)
{
return false;
}
Expand All @@ -424,7 +433,7 @@ private bool TryRegisterChatModel(ModelCard? modelCard, bool isSorted)

return true;
}
else if (isSlowModel && !EnableSlowModels)
else if (isSlowModel && !EnableLowPerformanceModels)
{
_models.Remove(modelCard);
}
Expand Down Expand Up @@ -499,7 +508,7 @@ private void OnFileDeleted(object sender, FileSystemEventArgs e)

if (ContainsModel(_models, fileUri, out int index))
{
if (!IsPredefinedModel(_models[index]))
if (!_models[index].IsPredefined)
{
var model = _models[index];
_models.Remove(model);
Expand Down Expand Up @@ -582,9 +591,9 @@ private void OnModelDownloadingProgressed(string path, long? contentLength, long
}
#endif

partial void OnEnableSlowModelsChanged(bool value)
partial void OnEnableLowPerformanceModelsChanged(bool value)
{
_appSettingsService.EnableSlowModels = value;
_appSettingsService.EnableLowPerformanceModels = value;
_ = CollectModelsAsync();
}

Expand Down Expand Up @@ -676,10 +685,14 @@ private void OnFileRecordPathChanged(object? sender, EventArgs e)

#region Static methods

private static bool ContainsModel(IList<ModelCard> models, ModelCard modelCard, out int index)
private static bool ContainsModel(IList<ModelCard> models, ModelCard modelCard)
{
index = 0;
if (models.Contains(modelCard))
{
return true;
}

//In this scope, we are essentially searching for duplicate model files..
foreach (var model in models)
{
/*if (model.SHA256 == modelCard.SHA256) //Loïc: commented. This is too slow.
Expand All @@ -696,15 +709,12 @@ private static bool ContainsModel(IList<ModelCard> models, ModelCard modelCard,
}
else if (model.SHA256 == modelCard.SHA256)
{
//todo: propagate feedback indicating that a duplicate file exists.
return true;
}
}

index++;
}

index = -1;

return false;
}

Expand Down Expand Up @@ -732,7 +742,7 @@ private static bool TryValidateModelFile(string filePath, string modelFolderPath
isSorted = false;
modelCard = null;

if (LLM.ValidateFormat(filePath))
if (LM.ValidateFormat(filePath))
{
try
{
Expand Down
4 changes: 2 additions & 2 deletions LM-Kit-Maestro/Services/LMKitDefaultSettings.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ public static class LMKitDefaultSettings
{
public static readonly string DefaultModelStorageDirectory = Global.Configuration.ModelStorageDirectory;

public const string DefaultSystemPrompt = "You are a chatbot that always responds promptly and helpfully to user requests.";
public const string DefaultSystemPrompt = "You are Maestro, a chatbot designed to provide prompt, helpful, and accurate responses to user requests in a friendly and professional manner.";
public const int DefaultMaximumCompletionTokens = 2048; // TODO: Evan, consider setting this to -1 to indicate no limitation. Ensure the option to configure the chat with a predefined limit remains available.
public static readonly int DefaultContextSize = Graphics.DeviceConfiguration.GetOptimalContextSize();
public const int DefaultRequestTimeout = 120;
public const SamplingMode DefaultSamplingMode = SamplingMode.Random;
public const bool DefaultEnableSlowModels = false;
public const bool DefaultEnableLowPerformanceModels = false;

public static SamplingMode[] AvailableSamplingModes { get; } = (SamplingMode[])Enum.GetValues(typeof(SamplingMode));

Expand Down
13 changes: 7 additions & 6 deletions LM-Kit-Maestro/Services/LMKitService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public partial class LMKitService : INotifyPropertyChanged

private static Uri? _currentlyLoadingModelUri;
private static Conversation? _lastConversationUsed = null;
private static LLM? _model;
private static LM? _model;
private static MultiTurnConversation? _multiTurnConversation;


Expand Down Expand Up @@ -63,7 +63,7 @@ public void LoadModel(Uri fileUri, string? localFilePath = null)

try
{
_model = new LLM(fileUri, downloadingProgress: OnModelDownloadingProgressed, loadingProgress: OnModelLoadingProgressed);
_model = new LM(fileUri, downloadingProgress: OnModelDownloadingProgressed, loadingProgress: OnModelLoadingProgressed);

modelLoadingSuccess = true;
}
Expand Down Expand Up @@ -323,7 +323,7 @@ private async Task<LMKitResult> SubmitRequest(LMKitRequest request)
result.Status == LMKitTextGenerationStatus.Undefined &&
!string.IsNullOrEmpty(((TextGenerationResult)result.Result!).Completion))
{
GenerateConversationSummaryTitle(conversation, promptParameters!.Prompt);
GenerateConversationSummaryTitle(conversation);
}
}

Expand All @@ -348,10 +348,11 @@ private async Task<LMKitResult> SubmitRequest(LMKitRequest request)
}
}

private void GenerateConversationSummaryTitle(Conversation conversation, string prompt)
private void GenerateConversationSummaryTitle(Conversation conversation)
{
string firstMessage = conversation.ChatHistory.Messages.First(message => message.AuthorRole == AuthorRole.User).Content;
LMKitRequest titleGenerationRequest = new LMKitRequest(LMKitRequest.LMKitRequestType.GenerateTitle,
new LMKitRequest.PromptRequestParameters(conversation, prompt), 60);
new LMKitRequest.PromptRequestParameters(conversation, firstMessage), 60);

_titleGenerationSchedule.Schedule(titleGenerationRequest);

Expand All @@ -378,7 +379,7 @@ private void GenerateConversationSummaryTitle(Conversation conversation, string

try
{
promptResult.Result = await summarizer.SummarizeAsync(prompt, titleGenerationRequest.CancellationTokenSource.Token);
promptResult.Result = await summarizer.SummarizeAsync(firstMessage, titleGenerationRequest.CancellationTokenSource.Token);
}
catch (Exception exception)
{
Expand Down
Loading

0 comments on commit d25df8b

Please sign in to comment.