Infer specific data kernel package version in discoverability kernels (…

…#3788)

* publish PostreSQL package

* make repack.ps1 version number match informational version number

* data kernel cleanup, suggest specific package version in discoverability

eng/publish/PublishPolyglotNotebooksHelper.psm1

@@ -70,6 +70,7 @@ function PublishStableExtensionAndNuGetPackages {
         "Microsoft.DotNet.Interactive.Mermaid",
         "Microsoft.DotNet.Interactive.NamedPipeConnector",
         "Microsoft.DotNet.Interactive.PackageManagement",
+        "Microsoft.DotNet.Interactive.PostgreSQL",
         "Microsoft.DotNet.Interactive.PowerShell",
         "Microsoft.DotNet.Interactive.SQLite",
         "Microsoft.DotNet.Interactive.SqlServer",

repack.ps1

@@ -8,8 +8,8 @@ try
 
     # build and pack dotnet-interactive 
     dotnet clean -c debug
-    dotnet build -c debug
-    dotnet pack -c debug /p:PackageVersion=2.0.0
+    dotnet build -c debug /p:Version=2.0.0
+    dotnet pack --no-build -c debug /p:PackageVersion=2.0.0 
 
     # copy the dotnet-interactive packages to the temp directory
     $destinationPath = "C:\temp\packages"

src/Microsoft.DotNet.Interactive.ApiCompatibility.Tests/ApiCompatibilityTests.Interactive_api_is_not_changed.approved.txt

@@ -253,8 +253,6 @@ Microsoft.DotNet.Interactive
     .ctor(System.String name = value, System.Net.Http.HttpClient httpClient = null)
     public Microsoft.DotNet.Interactive.Directives.KernelSpecifierDirective KernelSpecifierDirective { get;}
     public System.Collections.Generic.IReadOnlyDictionary<System.String,FormattedValue> Values { get;}
-  public class KqlDiscoverabilityKernel : Kernel, IKernelCommandHandler<Microsoft.DotNet.Interactive.Commands.RequestKernelInfo>, IKernelCommandHandler<Microsoft.DotNet.Interactive.Commands.SubmitCode>, System.IDisposable
-    .ctor()
   public class LinePosition, System.IEquatable<LinePosition>
     public static LinePosition FromCodeAnalysisLinePosition(Microsoft.CodeAnalysis.Text.LinePosition linePosition)
     public static System.Boolean op_Equality(LinePosition left, LinePosition right)
@@ -338,8 +336,6 @@ Microsoft.DotNet.Interactive
     public FormattedValue Documentation { get;}
     public System.String Label { get;}
     public System.Collections.Generic.IReadOnlyList<ParameterInformation> Parameters { get;}
-  public class SqlDiscoverabilityKernel : Kernel, IKernelCommandHandler<Microsoft.DotNet.Interactive.Commands.RequestKernelInfo>, IKernelCommandHandler<Microsoft.DotNet.Interactive.Commands.SubmitCode>, System.IDisposable
-    .ctor()
   public class TabularDataResourceSummaryExplorer : DataExplorer<Microsoft.DotNet.Interactive.Formatting.TabularData.TabularDataResource>
     .ctor(Microsoft.DotNet.Interactive.Formatting.TabularData.TabularDataResource data)
     protected Microsoft.AspNetCore.Html.IHtmlContent ToHtml()

src/Microsoft.DotNet.Interactive.DuckDB.Tests/KernelExtensionTests.cs

@@ -16,9 +16,9 @@
 
 namespace Microsoft.DotNet.Interactive.DuckDB.Tests;
 
+[Trait("Databases", "Data query tests")]
 public class DuckDbConnectionTests
 {
-
     [Fact]
     public async Task It_can_connect_and_query_data()
     {

src/Microsoft.DotNet.Interactive.Formatting/TabularData/TabularDataFormatterSource.cs

@@ -14,6 +14,8 @@ internal class TabularDataFormatterSource : ITypeFormatterSource
 {
     public IEnumerable<ITypeFormatter> CreateTypeFormatters()
     {
+        Formatter.SetPreferredMimeTypesFor(typeof(TabularDataResource), HtmlFormatter.MimeType, CsvFormatter.MimeType);
+
         yield return new HtmlFormatter<TabularDataResource>((value, context) => FormatHtml(context, value));
 
         yield return new JsonFormatter<TabularDataResource>((value, context) =>

src/Microsoft.DotNet.Interactive.Jupyter.Tests/JupyterKernelTestBase.cs

@@ -3,17 +3,13 @@
 
 using FluentAssertions;
 using Microsoft.DotNet.Interactive.CSharp;
-using Microsoft.DotNet.Interactive.Formatting;
-using Microsoft.DotNet.Interactive.Formatting.Csv;
-using Microsoft.DotNet.Interactive.Formatting.TabularData;
 using Microsoft.DotNet.Interactive.Jupyter.Connection;
 using Microsoft.DotNet.Interactive.Jupyter.Protocol;
 using Microsoft.DotNet.Interactive.Tests.Utility;
 using System;
 using System.Collections.Generic;
 using System.Reactive.Disposables;
 using System.Threading.Tasks;
-using Formatter = Microsoft.DotNet.Interactive.Formatting.Formatter;
 using Message = Microsoft.DotNet.Interactive.Jupyter.Messaging.Message;
 
 namespace Microsoft.DotNet.Interactive.Jupyter.Tests;
@@ -34,8 +30,6 @@ public virtual void Dispose()
 
     protected CompositeKernel CreateCompositeKernelAsync(params IJupyterKernelConnectionOptions[] optionsList)
     {
-        Formatter.SetPreferredMimeTypesFor(typeof(TabularDataResource), HtmlFormatter.MimeType, CsvFormatter.MimeType);
-
         var csharpKernel = new CSharpKernel()
                                 .UseKernelHelpers()
                                 .UseValueSharing();

src/Microsoft.DotNet.Interactive.Kql.Tests/KqlConnectionTests.cs

@@ -1,13 +1,13 @@
 // Copyright (c) .NET Foundation and contributors. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 
 using FluentAssertions;
 using Microsoft.DotNet.Interactive.App;
+using Microsoft.DotNet.Interactive.App.Connection;
 using Microsoft.DotNet.Interactive.Commands;
 using Microsoft.DotNet.Interactive.CSharp;
 using Microsoft.DotNet.Interactive.Events;
@@ -21,11 +21,11 @@
 
 namespace Microsoft.DotNet.Interactive.Kql.Tests;
 
-public class KqlConnectionTests : IDisposable
+[Trait("Databases", "Data query tests")]
+public class KqlConnectionTests
 {
     private static async Task<CompositeKernel> CreateKernelAsync()
     {
-        Formatter.SetPreferredMimeTypesFor(typeof(TabularDataResource), HtmlFormatter.MimeType, CsvFormatter.MimeType);
         var csharpKernel = new CSharpKernel().UseNugetDirective();
 
         var kernel = new CompositeKernel
@@ -448,10 +448,4 @@ await kernel.SubmitCodeAsync(
 
         resultSet.Should().NotBeNull().And.HaveCount(1);
     }
-
-    public void Dispose()
-    {
-        Formatter.ResetToDefault();
-        DataExplorer.ResetToDefault();
-    }
 }

src/Microsoft.DotNet.Interactive.Kql.Tests/KqlFactAttribute.cs

@@ -26,8 +26,10 @@ internal static string TestConnectionAndReturnSkipReason()
         string clusterName = GetClusterForTests();
         if (string.IsNullOrWhiteSpace(clusterName))
         {
-            return $"Environment variable {TEST_KQL_CONNECTION_STRING} is not set. To run tests that require "
-                   + "KQL Cluster, this environment variable must be set to a valid connection string value.";
+            return
+                $"""
+                 Environment variable {TEST_KQL_CONNECTION_STRING} is not set. To run tests that require a KQL Cluster, this environment variable must be set to a valid connection string value.
+                 """;
         }
 
         return null;

src/Microsoft.DotNet.Interactive.PostgreSql.Tests/PostgreSqlConnectionTests.cs

@@ -12,14 +12,16 @@
 using Microsoft.DotNet.Interactive.CSharp;
 using Microsoft.DotNet.Interactive.App;
 using FluentAssertions;
+using Microsoft.DotNet.Interactive.App.Connection;
+using Xunit;
 
 namespace Microsoft.DotNet.Interactive.PostgreSql.Tests;
 
+[Trait("Databases", "Data query tests")]
 public class PostgreSqlConnectionTests : IDisposable
 {
     private static CompositeKernel CreateKernel()
     {
-        Formatter.SetPreferredMimeTypesFor(typeof(TabularDataResource), HtmlFormatter.MimeType, CsvFormatter.MimeType);
         var csharpKernel = new CSharpKernel().UseNugetDirective().UseValueSharing();
 
         var kernel = new CompositeKernel
@@ -48,7 +50,7 @@ public async Task It_can_connect_and_query_data()
 
         result = await kernel.SubmitCodeAsync("""
             #!sql-adventureworks
-            SELECT * FROM Person.Person LIMIT 100;
+            SELECT * FROM customers LIMIT 100;
             """);
 
         result.Events.Should().NotContainErrors();
@@ -71,7 +73,7 @@ public async Task It_returns_error_if_query_is_not_valid()
 
         result = await kernel.SubmitCodeAsync("""
             #!sql-adventureworks
-            SELECT not_known_column FROM Person.Person LIMIT 100;
+            SELECT not_known_column FROM customers LIMIT 100;
             """);
 
         result.Events.Should()
@@ -85,6 +87,5 @@ public async Task It_returns_error_if_query_is_not_valid()
     public void Dispose()
     {
         DataExplorer.ResetToDefault();
-        Formatter.ResetToDefault();
     }
 }

src/Microsoft.DotNet.Interactive.PostgreSql.Tests/PostgreSqlFactAttribute.cs

@@ -30,8 +30,10 @@ internal static string TestConnectionAndReturnSkipReason()
         string connectionString = GetConnectionStringForTests();
         if (string.IsNullOrWhiteSpace(connectionString))
         {
-            return $"Environment variable {TEST_POSTGRESQL_CONNECTION_STRING} is not set. To run tests that require "
-                   + "SQL Server, this environment variable must be set to a valid connection string value.";
+            return
+                $"""
+                 Environment variable {TEST_POSTGRESQL_CONNECTION_STRING} is not set. To run tests that require PostgreSQL, this environment variable must be set to a valid connection string value.
+                 """;
         }
 
         try
@@ -41,17 +43,18 @@ internal static string TestConnectionAndReturnSkipReason()
         }
         catch (Exception e)
         {
-            return $"A connection could not be established to SQL Server. Verify the connection string value used " +
-                   $"for environment variable {TEST_POSTGRESQL_CONNECTION_STRING} targets a running SQL Server instance. " +
-                   $"Connection failed failed with error: {e}";
+            return
+                $"""
+                 A connection could not be established to a PostgreSQL server. Verify the connection string value used for environment variable {TEST_POSTGRESQL_CONNECTION_STRING} targets a running PostgreSQL instance. Connection failed failed with error: {e}
+                 """;
         }
 
         return null;
     }
 
     public static string GetConnectionStringForTests()
     {
-        const string fallbackConnectionString = "Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=Adventureworks";
+        const string fallbackConnectionString = "Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=northwind";
 
         return Environment.GetEnvironmentVariable(TEST_POSTGRESQL_CONNECTION_STRING) ?? fallbackConnectionString;
     }

src/Microsoft.DotNet.Interactive.PostgreSql.Tests/README.md

@@ -2,25 +2,10 @@
 
 ## Setup Test Database
 
-Take a look at [AdventureWorks-for-Postgres](https://github.com/lorint/AdventureWorks-for-Postgres) repository. It contains a script to create the AdventureWorks database on a PostgreSQL server.
+The tests use the sample [Northwind database](https://github.com/pthom/northwind_psql). You can create a blank database and install it using [this script](https://github.com/pthom/northwind_psql/blob/master/northwind.sql).
 
-You can use Docker to run a PostgreSQL server with the AdventureWorks database:
-
-Install repository:
-
-```bash
-git clone https://github.com/lorint/AdventureWorks-for-Postgres
-```
-
-Run PostgreSQL server:
-
-```bash
-podman build -t adventure-postgres ./
-podman run -d -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -p 5432:5432 adventure-postgres
-```
-
-Setup connection string as environment variable:
+The tests will only run when the environment variable `TEST_POSTGRESQL_CONNECTION_STRING` has set to a valid connection string. Here's an example using a default Postgres installation:
 
 ```bash
-export TEST_POSTGRESQL_CONNECTION_STRING='Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=Adventureworks'
+export TEST_POSTGRESQL_CONNECTION_STRING='Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=northwind'
 ```

src/Microsoft.DotNet.Interactive.PostgreSql/Microsoft.DotNet.Interactive.PostgreSql.csproj

@@ -3,15 +3,15 @@
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
     <IsPackable>true</IsPackable>
-    <PackageDescription>Microsoft PostgreSQL support for .NET Interactive</PackageDescription>
+    <PackageDescription>PostgreSQL support for .NET Interactive</PackageDescription>
     <PackageTags>polyglot notebook dotnet interactive SQL PostgreSQL Data</PackageTags>
     <IncludeBuildOutput>true</IncludeBuildOutput>
     <NoWarn>$(NoWarn);NU5100;VSTHRD002</NoWarn>
     <!-- dll outside of lib/ folder -->
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Npgsql" Version="8.0.4" />
+    <PackageReference Include="Npgsql" Version="8.0.6" />
     <PackageReference Include="Pocket.Disposable" Version="1.2.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

src/Microsoft.DotNet.Interactive.PostgreSql/PostgreSqlKernel.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System.Collections.Generic;
@@ -109,23 +109,20 @@ void ResolveColumnNameClashes(string[] names)
         }
     }
 
-    public static void AddPostgreSqlKernelConnectorTo(CompositeKernel kernel)
-    {
-        kernel.AddConnectDirective(new ConnectPostgreSqlDirective());
-
-        KernelInvocationContext.Current?.Display(
-            new HtmlString(@"<details><summary>Query PostgreSql databases.</summary>
-    <p>This extension adds support for connecting to PostgreSql databases using the <code>#!connect postgres</code> magic command. For more information, run a cell using the <code>#!sql</code> magic command.</p>
-    </details>"),
-            "text/html");
-    }
-
     public static void AddPostgreSqlKernelConnectorToCurrentRoot()
     {
         if (KernelInvocationContext.Current is { } context &&
             context.HandlingKernel.RootKernel is CompositeKernel root)
         {
-            AddPostgreSqlKernelConnectorTo(root);
+            root.AddConnectDirective(new ConnectPostgreSqlDirective());
+
+            context.Display(
+                new HtmlString("""
+                               <details><summary>Query PostgreSQL databases.</summary>
+                                   <p>This extension adds support for connecting to PostgreSql databases using the <code>#!connect postgres</code> magic command.</p>
+                                   </details>
+                               """),
+                "text/html");
         }
     }
 }

src/Microsoft.DotNet.Interactive.SQLite.Tests/SQLiteConnectionTests.cs

@@ -6,17 +6,20 @@
 using FluentAssertions;
 using Microsoft.Data.Sqlite;
 using Microsoft.DotNet.Interactive.App;
+using Microsoft.DotNet.Interactive.App.Connection;
 using Microsoft.DotNet.Interactive.CSharp;
 using Microsoft.DotNet.Interactive.Events;
 using Microsoft.DotNet.Interactive.Formatting;
 using Microsoft.DotNet.Interactive.Tests.Utility;
+using Xunit;
 
 namespace Microsoft.DotNet.Interactive.SQLite.Tests;
 
+[Trait("Databases", "Data query tests")]
 public class SQLiteConnectionTests
 {
     [WindowsFact]
-    public async Task SQLKernel_suggests_SQLite_connection_when_statements_are_submitted_to_it()
+    public async Task SQLDiscoverabilityKernel_suggests_SQLite_connection_when_statements_are_submitted_to_it()
     {
         using var kernel = new CompositeKernel
         {

src/Microsoft.DotNet.Interactive.SqlServer.Tests/MsSqlConnectionTests.cs

@@ -9,6 +9,7 @@
 using FluentAssertions;
 using FluentAssertions.Execution;
 using Microsoft.DotNet.Interactive.App;
+using Microsoft.DotNet.Interactive.App.Connection;
 using Microsoft.DotNet.Interactive.Commands;
 using Microsoft.DotNet.Interactive.CSharp;
 using Microsoft.DotNet.Interactive.Events;
@@ -20,14 +21,13 @@
 
 namespace Microsoft.DotNet.Interactive.SqlServer.Tests;
 
+[Trait("Databases", "Data query tests")]
 public class MsSqlConnectionTests : IDisposable
 {
     private async Task<CompositeKernel> CreateKernelAsync()
     {
-        Formatter.SetPreferredMimeTypesFor(typeof(TabularDataResource), HtmlFormatter.MimeType, CsvFormatter.MimeType);
         var csharpKernel = new CSharpKernel().UseNugetDirective().UseValueSharing();
 
-        // TODO: remove SQLKernel it is used to test current patch
         var kernel = new CompositeKernel
         {
             new SqlDiscoverabilityKernel(),
@@ -584,6 +584,5 @@ public async Task An_input_type_hint_is_set_for_connection_strings()
     public void Dispose()
     {
         DataExplorer.ResetToDefault();
-        Formatter.ResetToDefault();
     }
 }

src/Microsoft.DotNet.Interactive.Tests/KqlDiscoverabilityKernelTests.cs

@@ -3,6 +3,7 @@
 
 using System.Threading.Tasks;
 using FluentAssertions;
+using Microsoft.DotNet.Interactive.App.Connection;
 using Microsoft.DotNet.Interactive.Commands;
 using Microsoft.DotNet.Interactive.Events;
 using Microsoft.DotNet.Interactive.Tests.Utility;
@@ -78,9 +79,10 @@ public async Task kql_kernel_emits_help_message_without_sql_server_extension_ins
 
         var message = displayValue.Value.ToString();
 
-
         // Should contain instructions for how to install SqlServer extension package
-        message.Should().Contain(@"#r ""nuget:Microsoft.DotNet.Interactive.Kql,*-*""");
+        message.Should().Contain("""
+                                 #r "nuget:Microsoft.DotNet.Interactive.Kql,1.0.0
+                                 """);
 
         // Should contain instructions for how to get help message for MSSQL kernel
         message.Should().Contain("#!connect kql --kernel-name mydatabase --cluster \"https://help.kusto.windows.net\" --database \"Samples\"");

src/Microsoft.DotNet.Interactive.Tests/LanguageKernelFormattingTests.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
@@ -173,7 +173,7 @@ public async Task DisplayTable_produces_tabular_HTML_output_for_IEnumerable_T(La
         result.Events
               .Should().ContainSingle<DisplayedValueProduced>()
               .Which
-              .FormattedValues.Should().ContainSingle()
+              .FormattedValues.Should().ContainSingle(v => v.MimeType == "text/html")
               .Which
               .Value.Should().ContainEquivalentHtmlFragments("""
                 <table>