diff --git a/applications/lokqlDx/Copilot.cs b/applications/lokqlDx/Copilot.cs new file mode 100644 index 00000000..f7feeeda --- /dev/null +++ b/applications/lokqlDx/Copilot.cs @@ -0,0 +1,136 @@ +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using KustoLoco.Core.Console; +using NotNullStrings; + +namespace lokqlDx; + +public class Copilot +{ + public static class Roles + { + public const string User = "user"; + public const string Assistant = "assistant"; + public const string System = "system"; + public const string Kql = "kql"; + } + public void RenderResponses(IKustoConsole console,params string [] roles) + { + foreach(var message in context) + { + if (roles.Length > 0 && !roles.Contains(message.role)) + { + continue; + } + var color = message.role switch + { + Roles.User => ConsoleColor.Green, + Roles.Assistant => ConsoleColor.White, + Roles.System => ConsoleColor.Red, + Roles.Kql => ConsoleColor.Yellow, + _ => ConsoleColor.White + }; + console.ForegroundColor = color; + console.WriteLine(message.content); + console.WriteLine(); + } + } + + private HttpClient _client; + + List context = []; + public Copilot(string apiToken) + { + Initialised = apiToken.IsNotBlank(); + _client = new HttpClient(); + _client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Bearer", apiToken); + AddSystemInstructions(@" +You are an assistant to the user. You will be asked questions about tables of data and +will be expected to provide answers in the form of KQL queries. + +This KQL engine does not support the top operator. Please avoid it in your responses. + +This KQL engine does not support joining on more than a single column. Please avoid doing this in your responses and +instead use the extend operator to generate a new column that can be used for joining. + +This KQL implementation does not support arg_min or dynamic property types. + +You should provide only a brief explanation of the query and ensure that the KQL query is enclosed within the markdown code block syntax. +Prefer to place each pipeline stage on a new line to improve readability. + +When asked to render a chart, use the render operator to specify the type of chart to render. The render operator should be the final operator in the query. +The only 'with' property that the render operator supports is 'title'. Please use this to specify a suitable name for the chart. + +When asked to render multi-series charts use the project operator to order columns such that the column that defines the series +name is the final one in the list. Charts are rendered such that the first column is the x-axis, the second is the y-axis and +the last one is the series name. + +In general, prefer to place any time-based column as the x-axis and quantity-based columns as the y-axis but this is not a strict requirement. + +Here are some common mistakes I want you to avoid: +- Using the top operator +- Using the arg_min operator +- Using the dynamic property types +- Joining on more than a single column +- Using the render operator in the middle of the query +- using 'm' or 'mon' to specify months in a timespan. Remember that 'm' is minutes and that 'mon' is not a valid timespan. Instead you need to convert months to a number of days. + +I will now give some some information about the schema of the tables that you will be asked to query. + +"); + } + + + public async Task Issue(string question) + { + AddUserMessage(question); + var requestData = new + { + model = "gpt-4", + messages = context + .Where(m=> m.role!= Roles.Kql) + .Select(x => new { x.role, x.content }).ToArray() + }; + + var requestContent = new StringContent(JsonSerializer.Serialize(requestData), Encoding.UTF8, + "application/json"); + var response = await _client.PostAsync("https://api.openai.com/v1/chat/completions", requestContent); + var responseString = await response.Content.ReadAsStringAsync(); + var responseDto = JsonSerializer.Deserialize(responseString); + await response.Content.ReadFromJsonAsync(); + var assistanceResponse = responseDto?.choices?.FirstOrDefault()?.message.content ?? string.Empty; + AddCopilotResponse(assistanceResponse); + return assistanceResponse; + } + + private void AddMessage(string role, string content) => context.Add(new ChatMessage(role, content)); + + private void AddUserMessage(string content) => AddMessage(Roles.User, content); + + private void AddCopilotResponse(string content) => AddMessage(Roles.Assistant, content); + public void AddSystemInstructions(string content) => AddMessage(Roles.System, content); + + + public readonly record struct ChatMessage(string role, string content); + + public class ChatResponse + { + public ChatChoice[] choices { get; set; } = []; + } + + public class ChatChoice + { + public ChatMessage message { get; set; } = new ChatMessage(); + } + + public bool Initialised { get; private set; } + + public void AddResponse(string response) + { + AddMessage(Roles.Kql,response); + } +} diff --git a/applications/lokqlDx/MainWindow.xaml b/applications/lokqlDx/MainWindow.xaml index 1b4dcaa8..a05bfa89 100644 --- a/applications/lokqlDx/MainWindow.xaml +++ b/applications/lokqlDx/MainWindow.xaml @@ -50,10 +50,50 @@ - + + + + + + + Terse + + + + + + + + + + + + + + + + - + + + + + + + + 100 && Height > 100 && Left > 0 && Top > 0) { - Width = _preferenceManager.Preferences.WindowWidth; - Height = _preferenceManager.Preferences.WindowHeight; + Width = _preferenceManager.Preferences.WindowWidth < _minWindowSize.Width + ? _minWindowSize.Width + : _preferenceManager.Preferences.WindowWidth; + Height = _preferenceManager.Preferences.WindowHeight < _minWindowSize.Height + ? _minWindowSize.Height + : _preferenceManager.Preferences.WindowHeight; Left = _preferenceManager.Preferences.WindowLeft; Top = _preferenceManager.Preferences.WindowTop; } @@ -230,6 +240,8 @@ private void UpdateFontSize() Editor.SetFontSize(_preferenceManager.Preferences.FontSize); OutputText.FontSize = _preferenceManager.Preferences.FontSize; dataGrid.FontSize = _preferenceManager.Preferences.FontSize; + UserChat.FontSize = _preferenceManager.Preferences.FontSize; + ChatHistory.FontSize = _preferenceManager.Preferences.FontSize; } protected override void OnKeyDown(KeyEventArgs e) @@ -266,4 +278,81 @@ private void EnableJumpList(object sender, RoutedEventArgs e) { RegistryOperations.AssociateFileType(); } + + private async void SubmitToCopilot(object sender, RoutedEventArgs e) + { + SubmitButton.IsEnabled = false; + if (!_copilot.Initialised) + { + _copilot = new Copilot(_explorer.Settings.GetOr("copilot", string.Empty)); + foreach (var table in _explorer.GetCurrentContext().Tables()) + { + var sb = new StringBuilder(); + sb.AppendLine($"The table named '{table.Name}' has the following columns"); + var cols = table.ColumnNames.Zip(table.Type.Columns) + .Select(z => $" {z.First} is of type {z.Second.Type.Name}").ToArray(); + foreach (var column in cols) sb.AppendLine(column); + _copilot.AddSystemInstructions(sb.ToString()); + } + } + + var userchat = UserChat.Text; + UserChat.Text = string.Empty; + const int maxResubmissions = 3; + for (var i = 0; i < maxResubmissions; i++) + { + var response = await _copilot.Issue(userchat); + + + var console = new WpfConsole(ChatHistory); + + //now try to extract kql... + var lines = response.Split('\n'); + var kql = new StringBuilder(); + var getting = false; + foreach (var line in lines) + { + if (line.StartsWith("```kql") || line.StartsWith("```kusto")) + { + kql.Clear(); + getting = true; + continue; + } + + if (line.StartsWith("```")) + { + getting = false; + continue; + } + + if (getting) + kql.AppendLine(line.Trim()); + } + + _copilot.AddResponse(kql.ToString()); + console.PrepareForOutput(); + var options = new List { Copilot.Roles.System, Copilot.Roles.User, Copilot.Roles.Kql }; + if (TerseMode.IsChecked != true) + options.Add(Copilot.Roles.Assistant); + _copilot.RenderResponses(console, options.ToArray()); + + if (kql.ToString().IsBlank()) + break; + await RunQuery(kql.ToString()); + var lastResult = _explorer._prevResultIncludingError; + + if (lastResult.Error.IsBlank()) + break; + userchat = $"That query gave an error: {lastResult.Error}"; + } + + + SubmitButton.IsEnabled = true; + } + + private void ResetCopilot(object sender, RoutedEventArgs e) + { + _copilot = new Copilot(string.Empty); + ChatHistory.Document.Blocks.Clear(); + } } diff --git a/applications/lokqlDx/Preferences.cs b/applications/lokqlDx/Preferences.cs index b69e5526..ce10ae46 100644 --- a/applications/lokqlDx/Preferences.cs +++ b/applications/lokqlDx/Preferences.cs @@ -3,7 +3,7 @@ public class Preferences { public string LastWorkspacePath { get; set; } = string.Empty; - public double FontSize { get; set; } + public double FontSize { get; set; } = 20; public string FontFamily { get; set; } = string.Empty; public double WindowWidth { get; set; } public double WindowHeight { get; set; } diff --git a/applications/lokqlDx/WorkspaceManager.cs b/applications/lokqlDx/WorkspaceManager.cs index 4b5697db..27ec2fd3 100644 --- a/applications/lokqlDx/WorkspaceManager.cs +++ b/applications/lokqlDx/WorkspaceManager.cs @@ -71,6 +71,18 @@ public void Save(string path,string userText) public void Load(string path) { + if (!File.Exists(path)) + { + var rootSettingFolderPath = + System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), + "kustoloco"); + if (!Directory.Exists(rootSettingFolderPath)) + Directory.CreateDirectory(rootSettingFolderPath); + + path = System.IO.Path.Combine(rootSettingFolderPath, "settings"); + File.WriteAllText(path, JsonSerializer.Serialize(new Workspace())); + } + Path = path; try { diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MaxAggregate.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MaxAggregate.cs index 11c8eb5f..b70137af 100644 --- a/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MaxAggregate.cs +++ b/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MaxAggregate.cs @@ -47,7 +47,7 @@ internal static TimeSpan TsImpl(NumericAggregate context, TimeSpan n) internal static TimeSpan? TsImplFinish(NumericAggregate context) => context.Count == 0 ? null : new TimeSpan((long)context.Total); - internal static DateTime DtImpl(NumericAggregate context, TimeSpan n) + internal static DateTime DtImpl(NumericAggregate context, DateTime n) { context.Total = context.Count == 0 ? n.Ticks : Math.Max(context.Total, n.Ticks); context.Count++; @@ -56,4 +56,4 @@ internal static DateTime DtImpl(NumericAggregate context, TimeSpan n) internal static DateTime? DtImplFinish(NumericAggregate context) => context.Count == 0 ? null : new DateTime((long)context.Total); -} \ No newline at end of file +} diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MinAggregate.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MinAggregate.cs index be52ae1b..4f355d3f 100644 --- a/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MinAggregate.cs +++ b/libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MinAggregate.cs @@ -47,7 +47,7 @@ internal static TimeSpan TsImpl(NumericAggregate context, TimeSpan n) internal static TimeSpan? TsImplFinish(NumericAggregate context) => context.Count == 0 ? null : new TimeSpan((long)context.Total); - internal static DateTime DtImpl(NumericAggregate context, TimeSpan n) + internal static DateTime DtImpl(NumericAggregate context, DateTime n) { context.Total = context.Count == 0 ? n.Ticks : Math.Min(context.Total, n.Ticks); context.Count++; @@ -56,4 +56,4 @@ internal static DateTime DtImpl(NumericAggregate context, TimeSpan n) internal static DateTime? DtImplFinish(NumericAggregate context) => context.Count == 0 ? null : new DateTime((long)context.Total); -} \ No newline at end of file +} diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/BuiltInScalarFunctions.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/BuiltInScalarFunctions.cs index aa37954f..95e2676e 100644 --- a/libraries/KustoLoco.Core/Evaluation/BuiltIns/BuiltInScalarFunctions.cs +++ b/libraries/KustoLoco.Core/Evaluation/BuiltIns/BuiltInScalarFunctions.cs @@ -148,6 +148,7 @@ static void AddCoalesce(List overloads, Func a.Year - b.Year, "month" => (a.Year - b.Year) * 12 + a.Month - b.Month, diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/DatetimePartFunction.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/DatetimePartFunction.cs new file mode 100644 index 00000000..e9a50f01 --- /dev/null +++ b/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/DatetimePartFunction.cs @@ -0,0 +1,28 @@ +using System; + +namespace KustoLoco.Core.Evaluation.BuiltIns.Impl; + +[KustoImplementation(Keyword = "Functions.DatetimePart")] +internal partial class DatetimePartFunction +{ + + private static int Impl(string period, DateTime a) + { + return period.ToLowerInvariant() switch + { + "year" => a.Year, + "month" => a.Month, + "quarter" => 1+((a.Month-1)/3), + "week" => 1+((a.DayOfYear-1)/7), + "day" => a.DayOfYear, + "hour" => a.Hour, + "minute" => a.Minute, + "second" => a.Second, + "millisecond" => a.Millisecond, + "microsecond" => a.Microsecond, + "nanosecond" => (int) a.Ticks, + _ => 0 + }; + } + +} diff --git a/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/RoundFunction.cs b/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/RoundFunction.cs index 58f4fa36..06efc00c 100644 --- a/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/RoundFunction.cs +++ b/libraries/KustoLoco.Core/Evaluation/BuiltIns/ScalarFunctions/RoundFunction.cs @@ -7,5 +7,6 @@ namespace KustoLoco.Core.Evaluation.BuiltIns.Impl; [KustoImplementation(Keyword = "Functions.Round")] internal partial class RoundFunction { - private static double Impl(double input, long precision) => Math.Round(input, (int)precision); -} \ No newline at end of file + private static double ImplP(double input, long precision) => Math.Round(input, (int)precision); + private static double Impl(double input) => Math.Round(input, 0); +} diff --git a/libraries/lokql-engine/InteractiveTableExplorer.cs b/libraries/lokql-engine/InteractiveTableExplorer.cs index 9b171037..10dd6095 100644 --- a/libraries/lokql-engine/InteractiveTableExplorer.cs +++ b/libraries/lokql-engine/InteractiveTableExplorer.cs @@ -31,6 +31,7 @@ public class InteractiveTableExplorer public readonly KustoSettingsProvider Settings; public DisplayOptions _currentDisplayOptions = new(10); + public KustoQueryResult _prevResultIncludingError =KustoQueryResult.Empty; public InteractiveTableExplorer(IKustoConsole outputConsole, ITableAdaptor loader, KustoSettingsProvider settings, CommandProcessor commandProcessor) @@ -129,6 +130,7 @@ public async Task RunNextBlock(BlockSequence blocks) var result = await GetCurrentContext().RunQuery(query); if (result.Error.Length == 0) _prevResult = result; + _prevResultIncludingError = result; DisplayResults(result); }