Final improvements to Dynamo ML data pipeline. (#15116)

* updates

* more updates

* Update DynamoViewModel.cs

* Update DynamoMLDataPipeline.cs

* Remove IExtensionSource from DynamoMLDataPipelineExtension.

* Update PublicAPI.Unshipped.txt

* Update IDSDKManager.cs

* updates

* Update DynamoMLDataPipeline.cs

* Update DynamoMLDataPipeline.cs

* Update App.config

* update log message.

* change parameter type.

* Update DynamoMLDataPipeline.cs

* update greg nuget version.

* Use IOAuth2UserIDProvider interface

* update conflicts

* Update PublicAPI.Unshipped.txt

* rename to RequestAttribute

* change class name.

* update analytics info

* Update DynamoViewModel.cs

* add checks for authprovider

* Update DynamoMLDataPipeline.cs

* Remove PII data from workspace

* Update PIIDetector.cs

src/DynamoCore/Configuration/GraphChecksumItem.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 
 namespace Dynamo.Configuration
 {
@@ -13,14 +12,4 @@ public class GraphChecksumItem
 
         public string Checksum { get; set; }   
     }
-
-    /// <summary>
-    /// Represents the stringified version of the nodes connections from a graph
-    /// </summary>
-    public class GraphChecksumPair
-    {
-        public string GraphId { get; set; }
-
-        public List<string> Checksum { get; set; }
-    }
 }

src/DynamoCore/Core/IDSDKManager.cs

@@ -12,7 +12,7 @@ namespace Dynamo.Core
     /// <summary>
     /// The class to provide auth APIs for IDSDK related methods.
     /// </summary>
-    public class IDSDKManager : IOAuth2AuthProvider, IOAuth2AccessTokenProvider, IDisposable
+    public class IDSDKManager : IOAuth2AuthProvider, IOAuth2AccessTokenProvider, IOAuth2UserIDProvider, IDisposable
     {
         /// <summary>
         /// Used by the auth provider to request authentication.
@@ -78,6 +78,18 @@ public string Username
             }
         }
 
+        /// <summary>
+        /// Gets the userid of the logged in user.
+        /// </summary>
+        public string UserId
+        {
+            get
+            {
+                var result = IDSDK_GetUserInfo();
+                return result != null ? result.UserId : String.Empty;
+            }
+        }
+
         /// <summary>
         /// Used by the auth provider to sign request with the authorized token.
         /// </summary>

src/DynamoCore/DynamoCore.csproj

@@ -31,7 +31,7 @@
   </ItemGroup>
   <ItemGroup Label="Common dependencies">
     <PackageReference Include="Autodesk.IDSDK" Version="1.1.8" />
-    <PackageReference Include="Greg" Version="3.0.0.3175" />
+    <PackageReference Include="Greg" Version="3.0.1.4707" />
     <PackageReference Include="DynamoVisualProgramming.LibG_230_0_0" Version="3.0.0.4655" GeneratePathProperty="true" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" CopyXML="true" />
     <PackageReference Include="RestSharp" Version="108.0.1" CopyXML="true" />

src/DynamoCore/Models/DynamoModel.cs

@@ -155,11 +155,6 @@ internal LuceneSearchUtility LuceneUtility
         /// </summary>
         internal Dictionary<string, List<string>> GraphChecksumDictionary { get; set; }
 
-        /// <summary>
-        /// Return a list of GraphChecksumItems
-        /// </summary>
-        public List<GraphChecksumPair> GraphChecksumList { get; set; }
-
         #endregion
 
         #region static properties
@@ -992,7 +987,6 @@ protected DynamoModel(IStartConfiguration config)
                 LuceneUtility.DisposeWriter();
             }
 
-            GraphChecksumList = new List<GraphChecksumPair>();
             GraphChecksumDictionary = new Dictionary<string, List<string>>();
 
             // This event should only be raised at the end of this method.

src/DynamoCore/PublicAPI.Unshipped.txt

@@ -107,12 +107,6 @@ Dynamo.Configuration.GraphChecksumItem.Checksum.set -> void
 Dynamo.Configuration.GraphChecksumItem.GraphChecksumItem() -> void
 Dynamo.Configuration.GraphChecksumItem.GraphId.get -> string
 Dynamo.Configuration.GraphChecksumItem.GraphId.set -> void
-Dynamo.Configuration.GraphChecksumPair
-Dynamo.Configuration.GraphChecksumPair.Checksum.get -> System.Collections.Generic.List<string>
-Dynamo.Configuration.GraphChecksumPair.Checksum.set -> void
-Dynamo.Configuration.GraphChecksumPair.GraphChecksumPair() -> void
-Dynamo.Configuration.GraphChecksumPair.GraphId.get -> string
-Dynamo.Configuration.GraphChecksumPair.GraphId.set -> void
 Dynamo.Configuration.GroupStyleItem
 Dynamo.Configuration.GroupStyleItem.GroupStyleItem() -> void
 Dynamo.Configuration.PreferenceSettings
@@ -374,6 +368,7 @@ Dynamo.Core.IDSDKManager.LoginStateChanged -> System.Action<Greg.AuthProviders.L
 Dynamo.Core.IDSDKManager.Logout() -> void
 Dynamo.Core.IDSDKManager.RequestLogin -> System.Func<object, bool>
 Dynamo.Core.IDSDKManager.SignRequest(ref RestSharp.RestRequest m, RestSharp.RestClient client) -> void
+Dynamo.Core.IDSDKManager.UserId.get -> string
 Dynamo.Core.IDSDKManager.Username.get -> string
 Dynamo.Core.NotificationObject
 Dynamo.Core.NotificationObject.NotificationObject() -> void
@@ -1876,8 +1871,6 @@ Dynamo.Models.DynamoModel.ExtensionManager.get -> Dynamo.Extensions.IExtensionMa
 Dynamo.Models.DynamoModel.ForceRun() -> void
 Dynamo.Models.DynamoModel.ForceRunCancelCommand
 Dynamo.Models.DynamoModel.ForceRunCancelCommand.ForceRunCancelCommand(bool showErrors, bool cancelRun) -> void
-Dynamo.Models.DynamoModel.GraphChecksumList.get -> System.Collections.Generic.List<Dynamo.Configuration.GraphChecksumPair>
-Dynamo.Models.DynamoModel.GraphChecksumList.set -> void
 Dynamo.Models.DynamoModel.HostVersion.get -> string
 Dynamo.Models.DynamoModel.HostVersion.set -> void
 Dynamo.Models.DynamoModel.InsertFileCommand

src/DynamoCoreWpf/DynamoCoreWpf.csproj

@@ -170,7 +170,7 @@
       <PackageReference Include="DynamoVisualProgramming.LibG_230_0_0" Version="3.0.0.4655" />
     <PackageReference Include="FontAwesome5" Version="2.1.11" />
     <PackageReference Include="AvalonEdit" Version="6.3.0.90" CopyXML="true" />
-    <PackageReference Include="Greg" Version="3.0.0.3175" />
+    <PackageReference Include="Greg" Version="3.0.1.4707" />
     <PackageReference Include="Microsoft.Web.WebView2" Version="1.0.2045.28" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="RestSharp" Version="108.0.1" />

src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs

@@ -14,8 +14,6 @@
 using System.Windows.Forms;
 using System.Windows.Media;
 using System.Windows.Threading;
-using System.Xml;
-using System.Xml.Serialization;
 using Dynamo.Configuration;
 using Dynamo.Core;
 using Dynamo.Engine;
@@ -50,6 +48,7 @@
 using DynamoMLDataPipeline;
 using DynamoUtilities;
 using ICSharpCode.AvalonEdit;
+using Newtonsoft.Json;
 using PythonNodeModels;
 using ISelectable = Dynamo.Selection.ISelectable;
 using WpfResources = Dynamo.Wpf.Properties.Resources;
@@ -70,6 +69,7 @@ public partial class DynamoViewModel : ViewModelBase, IDynamoViewModel
         private bool showStartPage = false;
         private PreferencesViewModel preferencesViewModel;
         private string dynamoMLDataPath = string.Empty;
+        private const string dynamoMLDataFileName = "DynamoMLDataPipeline.json"; 
 
         // Can the user run the graph
         private bool CanRunGraph => HomeSpace.RunSettings.RunEnabled && !HomeSpace.GraphRunInProgress;
@@ -209,7 +209,7 @@ internal bool IsMLDataIngestionPipelineinBeta
         {
             get
             {
-                return DynamoModel.FeatureFlags?.CheckFeatureFlag("IsMLDataIngestionPipelineinBeta", false) ?? false;
+                return DynamoModel.FeatureFlags?.CheckFeatureFlag("IsMLDataIngestionPipelineinBeta", true) ?? true;
             }
         }
 
@@ -776,23 +776,23 @@ protected DynamoViewModel(StartConfiguration startConfiguration)
 
             preferencesViewModel = new PreferencesViewModel(this);
 
-            dynamoMLDataPath = Path.Combine(Model.PathManager.UserDataDirectory, "DynamoMLDataPipeline.xml");
+            dynamoMLDataPath = Path.Combine(Model.PathManager.UserDataDirectory, dynamoMLDataFileName);
 
             if (!DynamoModel.IsTestMode && !DynamoModel.IsHeadless)
             {
                 model.State = DynamoModel.DynamoModelState.StartedUI;
 
                 // deserialize workspace checksum hashes that is used for Dynamo ML data pipeline.
-                var checksums = new List<GraphChecksumPair>();
-                var serializer = new XmlSerializer(Model.GraphChecksumList.GetType());
-
                 if (File.Exists(dynamoMLDataPath))
                 {
-                    using (var reader = XmlReader.Create(dynamoMLDataPath))
+                    try
+                    {
+                        Model.GraphChecksumDictionary = JsonConvert.DeserializeObject<Dictionary<string, List<string>>>(File.ReadAllText(dynamoMLDataPath));
+                    }
+                    catch (Exception ex)
                     {
-                        checksums = (List<GraphChecksumPair>)serializer.Deserialize(reader);
+                        Model.Logger.Log($"Failed to deserialize {dynamoMLDataFileName} : {ex.Message}", LogLevel.File);
                     }
-                    Model.GraphChecksumDictionary = checksums.ToDictionary(x => x.GraphId, x => x.Checksum);
                 }
             }
 
@@ -2273,7 +2273,6 @@ private bool HasDifferentialCheckSum()
             bool differentialChecksum = false;
             string graphId = Model.CurrentWorkspace.Guid.ToString();
 
-
             Model.GraphChecksumDictionary.TryGetValue(graphId, out List<string> checksums);
 
             // compare the current checksum with previous hash values.
@@ -2296,23 +2295,17 @@ private bool HasDifferentialCheckSum()
             // if the checksum is different from previous hashes, serialize this new info. 
             if (differentialChecksum)
             {
-                var graphChecksums = new List<GraphChecksumPair>();
-                foreach (KeyValuePair<string, List<string>> entry in Model.GraphChecksumDictionary)
+                try
                 {
-                    var item = new GraphChecksumPair
+                    using (StreamWriter file = File.CreateText(dynamoMLDataPath))
                     {
-                        GraphId = entry.Key,
-                        Checksum = entry.Value
-                    };
-
-                    graphChecksums.Add(item);
+                        JsonSerializer serializer = new JsonSerializer();
+                        serializer.Serialize(file, Model.GraphChecksumDictionary);
+                    }
                 }
-
-                var serializer = new XmlSerializer(Model.GraphChecksumList.GetType());
-                using (var writer = XmlWriter.Create(dynamoMLDataPath))
+                catch (Exception ex)
                 {
-                    Model.GraphChecksumList = graphChecksums;
-                    serializer.Serialize(writer, Model.GraphChecksumList);
+                    Model.Logger.Log("Failed to serialize " + dynamoMLDataFileName + "::" + ex.Message, LogLevel.File);
                 }
             }
 
@@ -2339,16 +2332,6 @@ private void InternalSaveAs(string path, SaveContext saveContext, bool isBackup
                 if (!isBackup && hasSaved)
                 {
                     AddToRecentFiles(path);
-
-                    if ((currentWorkspaceViewModel?.IsHomeSpace ?? true) && HomeSpace.HasRunWithoutCrash &&
-                         Model.CurrentWorkspace.IsValidForFDX && !IsMLDataIngestionPipelineinBeta && currentWorkspaceViewModel.Checksum != string.Empty)
-                    {
-                        if (HasDifferentialCheckSum())
-                        {
-                            Model.Logger.Log("This Workspace is shared to train the Dynamo Machine Learning model.");
-                            MLDataPipelineExtension.DynamoMLDataPipeline.DataExchange(path);
-                        }
-                    }
                 }                           
             }
             catch (Exception ex)
@@ -2936,6 +2919,32 @@ public void MakeNewHomeWorkspace(object parameter)
 
         private void CloseHomeWorkspace(object parameter)
         {
+            // Upon closing a workspace, validate if the workspace is valid to be sent to the ML datapipeline and then send it.
+            if (!DynamoModel.IsTestMode && !HomeSpace.HasUnsavedChanges && (currentWorkspaceViewModel?.IsHomeSpace ?? true) && HomeSpace.HasRunWithoutCrash)
+            {
+                if (!IsMLDataIngestionPipelineinBeta && Model.CurrentWorkspace.IsValidForFDX && currentWorkspaceViewModel.Checksum != string.Empty)
+                {
+                    if (HasDifferentialCheckSum())
+                    {
+                        Model.Logger.Log("This Workspace is being shared to train the Dynamo Machine Learning model.", LogLevel.File);
+                        var workspacePath = model.CurrentWorkspace.FileName;
+
+                        Task.Run(() =>
+                        {
+                            try
+                            {
+                                MLDataPipelineExtension.DynamoMLDataPipeline.SendWorkspaceLog(workspacePath);
+                            }
+                            catch (Exception ex)
+                            {
+                                Model.Logger.Log("Failed to share this workspace with ML pipeline.", LogLevel.File);
+                                Model.Logger.Log(ex.StackTrace, LogLevel.File);
+                            }
+                        });
+                    }
+                }
+            }
+
             if (ClearHomeWorkspaceInternal())
             {
                 // If after closing the HOME workspace, and there are no other custom 

src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs

@@ -374,7 +374,6 @@ public string Checksum
             get
             {
                 List<string> nodeInfoConnections = new List<string>();
-                var connectors = Connectors;
 
                 foreach (var connector in Connectors)
                 {

src/DynamoMLDataPipeline/App.config

@@ -10,11 +10,5 @@
         <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.8" />
     </startup>
   <runtime>
-    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
-      <dependentAssembly>
-        <assemblyIdentity name="System.Runtime.CompilerServices.Unsafe" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
-      </dependentAssembly>
-    </assemblyBinding>
   </runtime>
 </configuration>

src/DynamoMLDataPipeline/BaseComponent.cs

@@ -3,7 +3,7 @@
 
 namespace DynamoMLDataPipeline
 {
-    class BaseComponent : Dictionary<string, Dictionary<string, ObjectInfo>>
+    internal class BaseComponent : Dictionary<string, Dictionary<string, ObjectInfo>>
     {
         private string objectId = "autodesk.design:components.base-1.0.0";
         public BaseComponent(string name)
@@ -19,7 +19,7 @@ public BaseComponent(string name)
         public string ObjectId { get { return objectId; } }
     }
 
-    class ObjectInfo
+    internal class ObjectInfo
     {
         [JsonProperty("name")]
         public string Name { get; set; }

src/DynamoMLDataPipeline/BinaryAsset.cs

@@ -5,21 +5,21 @@
 
 namespace DynamoMLDataPipeline
 {
-    class BaseBinaryAsset
+    internal class BaseBinaryAsset
     {
         [JsonProperty("id")]
         public string Id { get; set; }
     }
 
-    class UploadedBinaryAsset : BaseBinaryAsset
+    internal class UploadedBinaryAsset : BaseBinaryAsset
     {
         public UploadedBinaryAsset(string guid)
         {
             Id = guid;
         }
     }
 
-    class BinaryAsset : BaseBinaryAsset
+    internal class BinaryAsset : BaseBinaryAsset
     {
         public BinaryAsset()
         {
@@ -37,7 +37,7 @@ public BinaryAsset()
         public string Type { get; set; }
     }
 
-    class BinaryAssets
+    internal class BinaryAssets
     {
         [JsonProperty("binaries")]
         public List<BaseBinaryAsset> Binaries { get; set; }

src/DynamoMLDataPipeline/BinaryReferenceComponent.cs

@@ -3,7 +3,7 @@
 
 namespace DynamoMLDataPipeline
 {
-    class BinaryReferenceComponent : Dictionary<string, Dictionary<string, IPropertySet>>
+    internal class BinaryReferenceComponent : Dictionary<string, Dictionary<string, IPropertySet>>
     {
         private string objectId = "autodesk.data:binary.reference.component-1.0.0";
         public string ObjectId { get { return objectId; } }
@@ -17,7 +17,7 @@ public BinaryReferenceComponent(string binaryId)
         }
     }
 
-    class StringPropertySet : IPropertySet
+    internal class StringPropertySet : IPropertySet
     {
         [JsonProperty("id")]
         public string Id { get; set; }
@@ -31,7 +31,7 @@ public StringPropertySet(string binaryId, string revision = "v0")
         }
     }
 
-    class IntPropertySet : IPropertySet
+    internal class IntPropertySet : IPropertySet
     {
         [JsonProperty("end")]
         public int End { get; set; }
@@ -44,6 +44,6 @@ public IntPropertySet(int start = 0, int end = 8710)
             Start = start;
         }
     }
-    
-    interface IPropertySet { }
+
+    internal interface IPropertySet { }
 }