WIP load .net system.dll shim into python engine to enable compatibil…

…ity with .Net8 (#8)

* add test
add shim load
update some versions

* comments

* dont reload if engine is the same engine

* fix test

* use assembly load context to isolate deps
use config to default to loading the shims for now.
add test

* move to extra

* review comment

.github/workflows/build.yml

@@ -27,7 +27,7 @@ jobs:
         if: ${{ true }}
         run: |
           Write-Output "***Locating iron python package!***"
-          if (Test-Path -Path "${{ github.workspace }}\DSIronPython3\package_output\DSIronPython3\bin\python3eval.dll") {
+          if (Test-Path -Path "${{ github.workspace }}\DSIronPython3\package_output\DSIronPython3\extra\python3eval.dll") {
             Write-Output "python node dll exists!"
           } else {
             Write-Error "python node dll was not found!"

DSIronPython3Empty/Class1.cs

@@ -0,0 +1,11 @@
+namespace DSIronPython3Empty
+{
+    /// <summary>
+    /// This class/dll exists so that Dynamo knows this package is loaded,
+    /// even though it really only contains an extension.
+    /// </summary>
+    internal class Class1
+    {
+
+    }
+}

DSIronPython3Empty/DSIronPython3Empty.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+<PropertyGroup>
+	<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
+	<AppendRuntimeIdentifierToOutputPath>false</AppendRuntimeIdentifierToOutputPath>
+	<OutputPath>$(SolutionDir)\package_output\DSIronPython3\bin\</OutputPath>
+	<TargetFramework>net8.0</TargetFramework>
+	<ImplicitUsings>enable</ImplicitUsings>
+	<Nullable>enable</Nullable>
+</PropertyGroup>
+
+</Project>

IronPython3Extension.sln

@@ -7,7 +7,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IronPython3Extension", "Iro
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "python3eval", "python3eval\python3eval.csproj", "{2B256D7E-4864-4086-A481-6E5B24CBA95E}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IronPython3Tests", "IronPython3Tests\IronPython3Tests.csproj", "{CD76A408-05CF-45E6-A508-6A6BA6A95E75}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IronPython3Tests", "IronPython3Tests\IronPython3Tests.csproj", "{CD76A408-05CF-45E6-A508-6A6BA6A95E75}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DSIronPython3Empty", "DSIronPython3Empty\DSIronPython3Empty.csproj", "{AC582B53-98DA-48D3-97AC-DCDA17EAAFAA}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -27,6 +29,10 @@ Global
 		{CD76A408-05CF-45E6-A508-6A6BA6A95E75}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CD76A408-05CF-45E6-A508-6A6BA6A95E75}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{CD76A408-05CF-45E6-A508-6A6BA6A95E75}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AC582B53-98DA-48D3-97AC-DCDA17EAAFAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AC582B53-98DA-48D3-97AC-DCDA17EAAFAA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AC582B53-98DA-48D3-97AC-DCDA17EAAFAA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AC582B53-98DA-48D3-97AC-DCDA17EAAFAA}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

IronPython3Extension/Extension.cs

@@ -1,13 +1,19 @@
 using System;
 using System.IO;
+using System.Linq;
 using System.Reflection;
+using System.Runtime.Loader;
 using Dynamo.Extensions;
+using Dynamo.Graph.Workspaces;
 using Dynamo.Logging;
+using Dynamo.PythonServices;
 
 namespace IronPython3Extension
 {
     public class IronPython3Extension : IExtension, ILogSource
     {
+        private const string PythonEvaluatorAssembly = "python3eval";
+
         #region ILogSource
 
         public event Action<ILogMessage> MessageLogged;
@@ -29,9 +35,78 @@ public void Dispose()
 
         }
 
-        public void Ready(ReadyParams sp)
+        /// <summary>
+        /// Action to be invoked when the Dynamo has started up and is ready
+        /// for user interaction. 
+        /// </summary>
+        /// <param name="rp"></param>
+        public void Ready(ReadyParams rp)
+        {
+            var extraPath = Path.Combine(new FileInfo(Assembly.GetExecutingAssembly().Location).Directory.Parent.FullName, "extra");
+            var alc = new IsolatedPythonContext(Path.Combine(extraPath, $"{PythonEvaluatorAssembly}.dll"));
+            var dsIronAssem = alc.LoadFromAssemblyName(new AssemblyName(PythonEvaluatorAssembly));
+
+            //load the engine into Dynamo ourselves.
+            LoadPythonEngine(dsIronAssem);
+
+            //we used to do this:
+            //but it's not neccesary to load anything into the VM.
+            //instead we skip all the extra work and trigger the side effect we want
+            //which is re executing the graph after the dsIronPython evaluator is loaded into the PythonEngineManager.
+            //rp.StartupParams.LibraryLoader.LoadNodeLibrary(dsIronAssem);
+
+            if (rp.CurrentWorkspaceModel is HomeWorkspaceModel hwm)
+            {
+                foreach (var n in hwm.Nodes)
+                {
+                    n.MarkNodeAsModified(true);
+                }
+                hwm.Run();
+            }
+        }
+
+        private static void LoadPythonEngine(Assembly assembly)
         {
+            if (assembly == null)
+            {
+                throw new ArgumentNullException($"Error while loading python engine - assembly {PythonEvaluatorAssembly}.dll was not loaded successfully.");
+            }
+
+            // Currently we are using try-catch to validate loaded assembly and Singleton Instance method exist
+            // but we can optimize by checking all loaded types against evaluators interface later
+            try
+            {
+                Type eType = null;
+                PropertyInfo instanceProp = null;
+                try
+                {
+                    eType = assembly.GetTypes().FirstOrDefault(x => typeof(PythonEngine).IsAssignableFrom(x) && !x.IsInterface && !x.IsAbstract);
+                    if (eType == null) return;
+
+                    instanceProp = eType?.GetProperty("Instance", BindingFlags.NonPublic | BindingFlags.Static);
+                    if (instanceProp == null) return;
+                }
+                catch
+                {
+                    // Ignore exceptions from iterating assembly types.
+                    return;
+                }
 
+                PythonEngine engine = (PythonEngine)instanceProp.GetValue(null);
+                if (engine == null)
+                {
+                    throw new Exception($"Could not get a valid PythonEngine instance by calling the {eType.Name}.Instance method");
+                }
+
+                if (PythonEngineManager.Instance.AvailableEngines.All(x => x.Name != engine.Name))
+                {
+                    PythonEngineManager.Instance.AvailableEngines.Add(engine);
+                }
+            }
+            catch (Exception ex)
+            {
+                throw new Exception($"Failed to add a Python engine from assembly {assembly.GetName().Name}.dll with error: {ex.Message}");
+            }
         }
 
         public void Shutdown()
@@ -44,4 +119,35 @@ public void Startup(StartupParams sp)
 
         }
     }
+    internal class IsolatedPythonContext : AssemblyLoadContext
+    {
+        private AssemblyDependencyResolver resolver;
+
+        public IsolatedPythonContext(string libPath)
+        {
+            resolver = new AssemblyDependencyResolver(libPath);
+        }
+
+        protected override Assembly Load(AssemblyName assemblyName)
+        {
+            string assemblyPath = resolver.ResolveAssemblyToPath(assemblyName);
+            if (assemblyPath != null)
+            {
+                return LoadFromAssemblyPath(assemblyPath);
+            }
+
+            return null;
+        }
+
+        protected override IntPtr LoadUnmanagedDll(string unmanagedDllName)
+        {
+            string libraryPath = resolver.ResolveUnmanagedDllToPath(unmanagedDllName);
+            if (libraryPath != null)
+            {
+                return LoadUnmanagedDllFromPath(libraryPath);
+            }
+
+            return IntPtr.Zero;
+        }
+    }
 }

IronPython3Extension/IronPython3Extension.csproj

@@ -4,8 +4,8 @@
     <TargetFramework>net8.0</TargetFramework>
     <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
     <AppendRuntimeIdentifierToOutputPath>false</AppendRuntimeIdentifierToOutputPath>
-    <OutputPath>$(SolutionDir)\package_output\DSIronPython3\bin\</OutputPath>
-	<AssemblyVersion>1.0.2</AssemblyVersion>
+    <OutputPath>$(SolutionDir)\package_output\DSIronPython3\extra\</OutputPath>
+	<AssemblyVersion>1.4.0</AssemblyVersion>
   </PropertyGroup>
 
   <ItemGroup>
@@ -21,4 +21,10 @@
     <PackageReference Include="IronPython.StdLib" Version="3.4.1" />
   </ItemGroup>
 
+  <ItemGroup>
+    <None Update="IronPython3Extension_ExtensionDefinition.xml">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 </Project>

IronPython3Extension/IronPython3Extension_ExtensionDefinition.xml

@@ -0,0 +1,4 @@
+<ExtensionDefinition>
+	<AssemblyPath>IronPython3Extension.dll</AssemblyPath>
+	<TypeName>IronPython3Extension.IronPython3Extension</TypeName>
+</ExtensionDefinition>

IronPython3Tests/pythonEvalTests.cs

@@ -1,4 +1,5 @@
 using Dynamo;
+using Microsoft.Extensions.Configuration;
 using NUnit.Framework;
 using System.Collections;
 
@@ -46,6 +47,63 @@ public void BindingsWork()
             }
         }
 
+        [Test]
+        [Category("UnitTests")]
+        public void SysDiagProccess_AndOtherShimmedCLRTypesWork()
+        {
+
+
+            foreach (var pythonEvaluator in Evaluators)
+            {
+                var output = pythonEvaluator(
+                    @"
+import clr
+from System.Reflection import Assembly
+from System.Diagnostics import Process
+dynamoCore = Assembly.Load(""DynamoCore"")
+version_long = dynamoCore.GetName().Version.Major.ToString()
+proc = Process.GetCurrentProcess().ProcessName
+OUT = (version_long,proc)
+",
+                    new ArrayList(),
+                    new ArrayList()
+                );
+                Assert.AreEqual("3", (output as IList)[0]);
+                //we do this because the process name can change depending
+                //on where tests are running.
+                Assert.IsNotEmpty((string?)(output as IList)[1]);
+            }
+        }
+        [Test]
+        [Category("UnitTests")]
+        public void ConfigLoadShimsCanBeDisabled()
+        {
+
+            var configb = new ConfigurationBuilder();
+            var config = configb.AddInMemoryCollection(new Dictionary<string, string>()
+            {
+                {"config:CONFIG_ENABLE_NET48SHIMCOMPAT","false" }
+            }).Build();
+
+            var evaluator = new IronPython3.Evaluator.IronPython3Evaluator(config);
+
+            var e = Assert.Throws<Exception>(() =>
+            {
+                var output = evaluator.Evaluate(
+                    @"
+import clr
+from System.Reflection import Assembly
+from System.Diagnostics import Process
+",
+                    new ArrayList(),
+                    new ArrayList()
+                );
+
+            });
+            StringAssert.Contains("ImportError", e.Message);
+        }
+
+
         [Test]
         [Category("UnitTests")]
         public void DataMarshaling_Output()

package_output/DSIronPython3/pkg.json

@@ -2,8 +2,8 @@
   "license": "",
   "file_hash": null,
   "name": "DynamoIronPython3",
-  "version": "1.0.0",
-  "description": "unofficial python package for the IronPython 3 engine + stdlib ",
+  "version": "1.4.0",
+  "description": "unofficial python package for the IronPython 3 engine + stdlib. If you are using Dynamo < 3.x use package version < 1.2 ",
   "group": "",
   "keywords": [ "python", "ironpython", "ironpython3" ],
   "dependencies": [],
@@ -15,5 +15,5 @@
   "site_url": "https://dynamobim.org/",
   "repository_url": "https://github.com/DynamoDS/Dynamo",
   "contains_binaries": true,
-  "node_libraries": ["python3eval"]
+  "node_libraries": [ "DSIronPython3Empty, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null" ]
 }

python3eval/appsettings.json

@@ -0,0 +1,5 @@
+{
+  "config": {
+    "CONFIG_ENABLE_NET48SHIMCOMPAT": "true"
+  }
+}

python3eval/python3eval.csproj

@@ -4,9 +4,9 @@
     <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
     <AppendRuntimeIdentifierToOutputPath>false</AppendRuntimeIdentifierToOutputPath>
     <TargetFramework>net8.0</TargetFramework>
-    <OutputPath>$(SolutionDir)\package_output\DSIronPython3\bin\</OutputPath>
+    <OutputPath>$(SolutionDir)\package_output\DSIronPython3\extra\</OutputPath>
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
-	  <AssemblyVersion>1.0.2</AssemblyVersion>
+	  <AssemblyVersion>1.4.0</AssemblyVersion>
   </PropertyGroup>
 
   <ItemGroup>
@@ -25,18 +25,14 @@
     </PackageReference>
     <PackageReference Include="IronPython.StdLib" Version="3.4.1" />
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
   </ItemGroup>
-
-  <Target Name="Move python libs to extra" AfterTargets="Build">
-    <ItemGroup> 
-      <MySourceFiles Include="$(OutputPath)\*lib\**;" />
-    </ItemGroup>
-
-    <Move SourceFiles="@(MySourceFiles)" DestinationFiles="$(OutputPath)..\extra\%(RecursiveDir)%(Filename)%(Extension)" />
-  </Target>
 
-  <Target Name="Remove Lib" AfterTargets="Move python libs to extra">
-    <RemoveDir Directories="$(OutputPath)\lib" />
-  </Target>
+  <ItemGroup>
+    <None Update="appsettings.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 
 </Project>