Merge remote-tracking branch 'origin/main' into editkql

applications/lokqlDx/Copilot.cs

@@ -0,0 +1,136 @@
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using System.Text;
+using System.Text.Json;
+using KustoLoco.Core.Console;
+using NotNullStrings;
+
+namespace lokqlDx;
+
+public class Copilot
+{
+    public static class Roles
+    {
+        public const string User = "user";
+        public const string Assistant = "assistant";
+        public const string System = "system";
+        public const string Kql = "kql";
+    }
+    public void RenderResponses(IKustoConsole console,params string [] roles)
+    {
+        foreach(var message in context)
+        {
+            if (roles.Length > 0 && !roles.Contains(message.role))
+            {
+                continue;
+            }
+            var color = message.role switch
+            {
+                Roles.User => ConsoleColor.Green,
+                Roles.Assistant => ConsoleColor.White,
+                Roles.System => ConsoleColor.Red,
+                Roles.Kql => ConsoleColor.Yellow,
+                _ => ConsoleColor.White
+            };
+            console.ForegroundColor = color;
+            console.WriteLine(message.content);
+            console.WriteLine();
+        }
+    }
+
+    private HttpClient _client;
+
+    List<ChatMessage> context = [];
+    public Copilot(string apiToken)
+    {
+        Initialised = apiToken.IsNotBlank();
+        _client = new HttpClient();
+        _client.DefaultRequestHeaders.Authorization =
+            new AuthenticationHeaderValue("Bearer", apiToken);
+        AddSystemInstructions(@"
+You are an assistant to the user. You will be asked questions about tables of data and
+will be expected to provide answers in the form of KQL queries.
+
+This KQL engine does not support the top operator. Please avoid it in your responses.
+
+This KQL engine does not support joining on more than a single column. Please avoid doing this in your responses and
+instead use the extend operator to generate a new column that can be used for joining.
+
+This KQL implementation does not support arg_min or dynamic property types.
+
+You should provide only a brief explanation of the query and ensure that the KQL query is enclosed within the markdown code block syntax.
+Prefer to place each pipeline stage on a new line to improve readability.
+
+When asked to render a chart, use the render operator to specify the type of chart to render.  The render operator should be the final operator in the query.
+The only 'with' property that the render operator supports is 'title'.  Please use this to specify a suitable name for the chart.   
+
+When asked to render multi-series charts use the project operator to order columns such that the column that defines the series
+name is the final one in the list.  Charts are rendered such that the first column is the x-axis, the second is the y-axis and
+the last one is the series name.
+
+In general, prefer to place any time-based column as the x-axis and quantity-based columns as the y-axis but this is not a strict requirement.
+
+Here are some common mistakes I want you to avoid:
+- Using the top operator
+- Using the arg_min operator
+- Using the dynamic property types
+- Joining on more than a single column
+- Using the render operator in the middle of the query
+- using 'm' or 'mon' to specify months in a timespan.  Remember that 'm' is minutes and that 'mon' is not a valid timespan.  Instead you need to convert months to a number of days.
+
+I will now give some some information about the schema of the tables that you will be asked to query.
+
+");
+    }
+
+
+    public async Task<string> Issue(string question)
+    {
+        AddUserMessage(question);
+        var requestData = new
+        {
+            model = "gpt-4",
+            messages = context
+                .Where(m=> m.role!= Roles.Kql)
+                .Select(x => new { x.role, x.content }).ToArray()
+        };
+
+        var requestContent = new StringContent(JsonSerializer.Serialize(requestData), Encoding.UTF8,
+            "application/json");
+        var response = await _client.PostAsync("https://api.openai.com/v1/chat/completions", requestContent);
+        var responseString = await response.Content.ReadAsStringAsync();
+        var responseDto = JsonSerializer.Deserialize<ChatResponse>(responseString);
+        await response.Content.ReadFromJsonAsync<ChatResponse>();
+        var assistanceResponse = responseDto?.choices?.FirstOrDefault()?.message.content ?? string.Empty;
+        AddCopilotResponse(assistanceResponse);
+        return assistanceResponse;
+    }
+
+    private void AddMessage(string role, string content) => context.Add(new ChatMessage(role, content));
+
+    private void AddUserMessage(string content) => AddMessage(Roles.User, content);
+
+    private void AddCopilotResponse(string content) => AddMessage(Roles.Assistant, content);
+    public void AddSystemInstructions(string content) => AddMessage(Roles.System, content);
+
+
+    public readonly record struct ChatMessage(string role, string content);
+
+    public class ChatResponse
+    {
+        public ChatChoice[] choices { get; set; } = [];
+    }
+
+    public class ChatChoice
+    {
+        public ChatMessage message { get; set; } = new ChatMessage();
+    }
+
+    public bool Initialised { get; private set; }
+
+    public void AddResponse(string response)
+    {
+       AddMessage(Roles.Kql,response);
+    }
+}

applications/lokqlDx/MainWindow.xaml

@@ -50,10 +50,50 @@
                 <RowDefinition Height="Auto" />
                 <RowDefinition Height="*" />
             </Grid.RowDefinitions>
-            <local:QueryEditor x:Name="Editor" Grid.Row="0" 
+        <TabControl Grid.Row="0" >
+          <TabItem Header="Edit">
+            <local:QueryEditor x:Name="Editor" 
                                RunEvent="OnQueryEditorRunTextBlock" />
+          </TabItem>
+          <TabItem Header="Copilot">
+            <DockPanel>
+              <StackPanel Orientation="Horizontal" DockPanel.Dock="Top">
+                <CheckBox Name ="TerseMode" VerticalAlignment="Center">Terse</CheckBox>
+               </StackPanel>
+              <Grid>
+              <Grid.RowDefinitions>
+                <RowDefinition Height="*" />
+                <RowDefinition Height="Auto" />
+                <RowDefinition Height="*" />
+              </Grid.RowDefinitions>
+              <RichTextBox Grid.Row="0" Name="ChatHistory" FontFamily="Consolas"
+                           Background="DarkBlue"
+                           FontSize="10"/>
+              <GridSplitter Grid.Row="1"
+                            VerticalAlignment="Stretch"
+                            Height="5"
+                            HorizontalAlignment="Stretch"
+                            Background="DarkGray"/>
+              <DockPanel Grid.Row="2">
+                <DockPanel DockPanel.Dock="Bottom" Margin="5" >
+                  <Button Name="SubmitButton" DockPanel.Dock="Right"  Width="100" HorizontalAlignment="Right" Click="SubmitToCopilot">Submit</Button>
+                  <Button Name="ClearChatButton" DockPanel.Dock="Left"  Width="100" HorizontalAlignment="Left" Click="ResetCopilot">Restart</Button>
+                </DockPanel>
+
+                <TextBox  Name="UserChat" FontFamily="Consolas"
+                              Background="LightBlue"
+                              FontSize="10"
+                              AcceptsReturn="True"/>
 
-            <GridSplitter Grid.Row="1"
+              </DockPanel>
+
+            </Grid>
+
+            </DockPanel>
+
+          </TabItem>
+        </TabControl>
+        <GridSplitter Grid.Row="1"
                           VerticalAlignment="Stretch"
                           Height="5"
                           HorizontalAlignment="Stretch"

applications/lokqlDx/MainWindow.xaml.cs

@@ -1,5 +1,6 @@
 using System.ComponentModel;
 using System.IO;
+using System.Text;
 using System.Windows;
 using System.Windows.Input;
 using System.Windows.Shell;
@@ -8,19 +9,21 @@
 using KustoLoco.Core.Settings;
 using KustoLoco.Rendering;
 using Lokql.Engine;
-using Lokql.Engine.Commands;
 using Microsoft.Win32;
+using NotNullStrings;
 
 namespace lokqlDx;
 
 public partial class MainWindow : Window
 {
     private readonly string[] _args;
     private readonly WpfConsole _console;
+    private readonly Size _minWindowSize = new(600, 400);
     private readonly PreferencesManager _preferenceManager = new();
     private readonly WorkspaceManager _workspaceManager;
-    private InteractiveTableExplorer _explorer;
 
+    private Copilot _copilot = new(string.Empty);
+    private InteractiveTableExplorer _explorer;
     private bool isBusy;
 
 
@@ -100,20 +103,27 @@ private void UpdateUIFromWorkspace()
         Editor.SetText(_workspaceManager.UserText);
         var settings = _workspaceManager.Settings;
         var loader = new StandardFormatAdaptor(settings, _console);
-        _explorer = new InteractiveTableExplorer(_console, loader, settings, CommandProcessorProvider.GetCommandProcessor());
+        _explorer = new InteractiveTableExplorer(_console, loader, settings,
+            CommandProcessorProvider.GetCommandProcessor());
         UpdateFontSize();
         Title = $"LokqlDX - {_workspaceManager.Path}";
     }
 
     private async void MainWindow_OnLoaded(object sender, RoutedEventArgs e)
     {
         _preferenceManager.Load();
-        var pathToLoad = _args.Any() ? _args[0] : _preferenceManager.Preferences.LastWorkspacePath;
+        var pathToLoad = _args.Any()
+                             ? _args[0]
+                             : _preferenceManager.Preferences.LastWorkspacePath;
         _workspaceManager.Load(pathToLoad);
         if (Width > 100 && Height > 100 && Left > 0 && Top > 0)
         {
-            Width = _preferenceManager.Preferences.WindowWidth;
-            Height = _preferenceManager.Preferences.WindowHeight;
+            Width = _preferenceManager.Preferences.WindowWidth < _minWindowSize.Width
+                        ? _minWindowSize.Width
+                        : _preferenceManager.Preferences.WindowWidth;
+            Height = _preferenceManager.Preferences.WindowHeight < _minWindowSize.Height
+                         ? _minWindowSize.Height
+                         : _preferenceManager.Preferences.WindowHeight;
             Left = _preferenceManager.Preferences.WindowLeft;
             Top = _preferenceManager.Preferences.WindowTop;
         }
@@ -230,6 +240,8 @@ private void UpdateFontSize()
         Editor.SetFontSize(_preferenceManager.Preferences.FontSize);
         OutputText.FontSize = _preferenceManager.Preferences.FontSize;
         dataGrid.FontSize = _preferenceManager.Preferences.FontSize;
+        UserChat.FontSize = _preferenceManager.Preferences.FontSize;
+        ChatHistory.FontSize = _preferenceManager.Preferences.FontSize;
     }
 
     protected override void OnKeyDown(KeyEventArgs e)
@@ -266,4 +278,81 @@ private void EnableJumpList(object sender, RoutedEventArgs e)
     {
         RegistryOperations.AssociateFileType();
     }
+
+    private async void SubmitToCopilot(object sender, RoutedEventArgs e)
+    {
+        SubmitButton.IsEnabled = false;
+        if (!_copilot.Initialised)
+        {
+            _copilot = new Copilot(_explorer.Settings.GetOr("copilot", string.Empty));
+            foreach (var table in _explorer.GetCurrentContext().Tables())
+            {
+                var sb = new StringBuilder();
+                sb.AppendLine($"The table named '{table.Name}' has the following columns");
+                var cols = table.ColumnNames.Zip(table.Type.Columns)
+                    .Select(z => $"  {z.First} is of type {z.Second.Type.Name}").ToArray();
+                foreach (var column in cols) sb.AppendLine(column);
+                _copilot.AddSystemInstructions(sb.ToString());
+            }
+        }
+
+        var userchat = UserChat.Text;
+        UserChat.Text = string.Empty;
+        const int maxResubmissions = 3;
+        for (var i = 0; i < maxResubmissions; i++)
+        {
+            var response = await _copilot.Issue(userchat);
+
+
+            var console = new WpfConsole(ChatHistory);
+
+            //now try to extract kql...
+            var lines = response.Split('\n');
+            var kql = new StringBuilder();
+            var getting = false;
+            foreach (var line in lines)
+            {
+                if (line.StartsWith("```kql") || line.StartsWith("```kusto"))
+                {
+                    kql.Clear();
+                    getting = true;
+                    continue;
+                }
+
+                if (line.StartsWith("```"))
+                {
+                    getting = false;
+                    continue;
+                }
+
+                if (getting)
+                    kql.AppendLine(line.Trim());
+            }
+
+            _copilot.AddResponse(kql.ToString());
+            console.PrepareForOutput();
+            var options = new List<string> { Copilot.Roles.System, Copilot.Roles.User, Copilot.Roles.Kql };
+            if (TerseMode.IsChecked != true)
+                options.Add(Copilot.Roles.Assistant);
+            _copilot.RenderResponses(console, options.ToArray());
+
+            if (kql.ToString().IsBlank())
+                break;
+            await RunQuery(kql.ToString());
+            var lastResult = _explorer._prevResultIncludingError;
+
+            if (lastResult.Error.IsBlank())
+                break;
+            userchat = $"That query gave an error: {lastResult.Error}";
+        }
+
+
+        SubmitButton.IsEnabled = true;
+    }
+
+    private void ResetCopilot(object sender, RoutedEventArgs e)
+    {
+        _copilot = new Copilot(string.Empty);
+        ChatHistory.Document.Blocks.Clear();
+    }
 }

applications/lokqlDx/Preferences.cs

@@ -3,7 +3,7 @@
 public class Preferences
 {
     public string LastWorkspacePath { get; set; } = string.Empty;
-    public double FontSize { get; set; }
+    public double FontSize { get; set; } = 20;
     public string FontFamily { get; set; } = string.Empty;
     public double WindowWidth { get; set; }
     public double WindowHeight { get; set; }

applications/lokqlDx/WorkspaceManager.cs

@@ -71,6 +71,18 @@ public void Save(string path,string userText)
 
     public void Load(string path)
     {
+        if (!File.Exists(path))
+        {
+            var rootSettingFolderPath =
+                System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
+                                       "kustoloco");
+            if (!Directory.Exists(rootSettingFolderPath))
+                Directory.CreateDirectory(rootSettingFolderPath);
+
+            path = System.IO.Path.Combine(rootSettingFolderPath, "settings");
+            File.WriteAllText(path, JsonSerializer.Serialize(new Workspace()));
+        }
+
         Path = path;
         try
         {

libraries/KustoLoco.Core/Evaluation/BuiltIns/Aggregates/MaxAggregate.cs

@@ -47,7 +47,7 @@ internal static TimeSpan TsImpl(NumericAggregate context, TimeSpan n)
     internal static TimeSpan? TsImplFinish(NumericAggregate context)
         => context.Count == 0 ? null : new TimeSpan((long)context.Total);
 
-    internal static DateTime DtImpl(NumericAggregate context, TimeSpan n)
+    internal static DateTime DtImpl(NumericAggregate context, DateTime n)
     {
         context.Total = context.Count == 0 ? n.Ticks : Math.Max(context.Total, n.Ticks);
         context.Count++;
@@ -56,4 +56,4 @@ internal static DateTime DtImpl(NumericAggregate context, TimeSpan n)
 
     internal static DateTime? DtImplFinish(NumericAggregate context)
         => context.Count == 0 ? null : new DateTime((long)context.Total);
-}
+}