[DYN-6666] Improvements to workspace checksum that is needed for Dynamo ML data pipeline.

src/DynamoCore/Configuration/GraphChecksumItem.cs

@@ -1,17 +1,26 @@
 using System;
-using System.Collections.ObjectModel;
-using Dynamo.Core;
-using Dynamo.Properties;
+using System.Collections.Generic;
 
 namespace Dynamo.Configuration
 {
     /// <summary>
     /// Represents the stringified version of the nodes connections from a graph
     /// </summary>
+    [Obsolete("This property is not needed anymore in the preference settings and can be removed in a future version of Dynamo.")]
     public class GraphChecksumItem
     {
         public string GraphId { get; set; }
 
         public string Checksum { get; set; }   
     }
+
+    /// <summary>
+    /// Represents the stringified version of the nodes connections from a graph
+    /// </summary>
+    public class GraphChecksumPair
+    {
+        public string GraphId { get; set; }
+
+        public List<string> Checksum { get; set; }
+    }
 }

src/DynamoCore/Configuration/IPreferences.cs

@@ -148,6 +148,7 @@ public interface IPreferences
         /// <param name="value">Active state to set</param>
         void SetIsBackgroundPreviewActive(string name, bool value);
 
+        [Obsolete("This property is not needed anymore in the preference settings and can be removed in a future version of Dynamo.")]
         /// <summary>
         /// Return a list of GraphChecksumItems
         /// </summary>

src/DynamoCore/Configuration/PreferenceSettings.cs

@@ -472,6 +472,7 @@ public bool DisableTrustWarnings
         /// <summary>
         /// Return a list of GraphChecksumItems
         /// </summary>
+        [Obsolete("This property is not needed anymore in the preference settings and can be removed in a future version of Dynamo.")]
         public List<GraphChecksumItem> GraphChecksumItemsList { get; set; }
 
         // This function is used to deserialize the trusted locations manually

src/DynamoCore/Models/DynamoModel.cs

@@ -148,6 +148,17 @@ internal LuceneSearchUtility LuceneUtility
             }
         }
 
+        /// <summary>
+        /// Return a dictionary of GraphChecksumItems.
+        /// Key will be the workspace guid and its value will be a list of saved checksums(sha256 hash) for that workspace.
+        /// </summary>
+        internal Dictionary<string, List<string>> GraphChecksumDictionary { get; set; }
+
+        /// <summary>
+        /// Return a list of GraphChecksumItems
+        /// </summary>
+        public List<GraphChecksumPair> GraphChecksumList { get; set; }
+
         #endregion
 
         #region static properties
@@ -979,6 +990,10 @@ protected DynamoModel(IStartConfiguration config)
             {
                 LuceneUtility.DisposeWriter();
             }
+
+            GraphChecksumList = new List<GraphChecksumPair>();
+            GraphChecksumDictionary = new Dictionary<string, List<string>>();
+
             // This event should only be raised at the end of this method.
             DynamoReady(new ReadyParams(this));
         }

src/DynamoCore/PublicAPI.Unshipped.txt

@@ -107,6 +107,12 @@ Dynamo.Configuration.GraphChecksumItem.Checksum.set -> void
 Dynamo.Configuration.GraphChecksumItem.GraphChecksumItem() -> void
 Dynamo.Configuration.GraphChecksumItem.GraphId.get -> string
 Dynamo.Configuration.GraphChecksumItem.GraphId.set -> void
+Dynamo.Configuration.GraphChecksumPair
+Dynamo.Configuration.GraphChecksumPair.Checksum.get -> System.Collections.Generic.List<string>
+Dynamo.Configuration.GraphChecksumPair.Checksum.set -> void
+Dynamo.Configuration.GraphChecksumPair.GraphChecksumPair() -> void
+Dynamo.Configuration.GraphChecksumPair.GraphId.get -> string
+Dynamo.Configuration.GraphChecksumPair.GraphId.set -> void
 Dynamo.Configuration.GroupStyleItem
 Dynamo.Configuration.GroupStyleItem.GroupStyleItem() -> void
 Dynamo.Configuration.PreferenceSettings
@@ -1864,6 +1870,8 @@ Dynamo.Models.DynamoModel.ExtensionManager.get -> Dynamo.Extensions.IExtensionMa
 Dynamo.Models.DynamoModel.ForceRun() -> void
 Dynamo.Models.DynamoModel.ForceRunCancelCommand
 Dynamo.Models.DynamoModel.ForceRunCancelCommand.ForceRunCancelCommand(bool showErrors, bool cancelRun) -> void
+Dynamo.Models.DynamoModel.GraphChecksumList.get -> System.Collections.Generic.List<Dynamo.Configuration.GraphChecksumPair>
+Dynamo.Models.DynamoModel.GraphChecksumList.set -> void
 Dynamo.Models.DynamoModel.HostVersion.get -> string
 Dynamo.Models.DynamoModel.HostVersion.set -> void
 Dynamo.Models.DynamoModel.InsertFileCommand

src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs

@@ -14,6 +14,8 @@
 using System.Windows.Forms;
 using System.Windows.Media;
 using System.Windows.Threading;
+using System.Xml;
+using System.Xml.Serialization;
 using Dynamo.Configuration;
 using Dynamo.Core;
 using Dynamo.Engine;
@@ -67,6 +69,7 @@ public partial class DynamoViewModel : ViewModelBase, IDynamoViewModel
         private Point transformOrigin;
         private bool showStartPage = false;
         private PreferencesViewModel preferencesViewModel;
+        private string dynamoMLDataPath = string.Empty;
 
         // Can the user run the graph
         private bool CanRunGraph => HomeSpace.RunSettings.RunEnabled && !HomeSpace.GraphRunInProgress;
@@ -768,11 +771,26 @@ protected DynamoViewModel(StartConfiguration startConfiguration)
             model.ComputeModelDeserialized += model_ComputeModelDeserialized;
             model.RequestNotification += model_RequestNotification;
 
-            preferencesViewModel = new PreferencesViewModel(this);            
+            preferencesViewModel = new PreferencesViewModel(this);
+
+            dynamoMLDataPath = Path.Combine(Model.PathManager.UserDataDirectory, "DynamoMLDataPipeline.xml");
 
             if (!DynamoModel.IsTestMode && !DynamoModel.IsHeadless)
             {
                 model.State = DynamoModel.DynamoModelState.StartedUI;
+
+                // deserialize workspace checksum hashes that is used for Dynamo ML data pipeline.
+                var checksums = new List<GraphChecksumPair>();
+                var serializer = new XmlSerializer(Model.GraphChecksumList.GetType());
+
+                if (File.Exists(dynamoMLDataPath))
+                {
+                    using (var reader = XmlReader.Create(dynamoMLDataPath))
+                    {
+                        checksums = (List<GraphChecksumPair>)serializer.Deserialize(reader);
+                    }
+                    Model.GraphChecksumDictionary = checksums.ToDictionary(x => x.GraphId, x => x.Checksum);
+                }
             }
 
             FileTrustViewModel = new FileTrustWarningViewModel();
@@ -2192,30 +2210,58 @@ internal bool CanSaveAs(object parameters)
         }
 
         /// <summary>
-        /// Indicates if the graph has been changed substantially bearing in mind the connections of its nodes and store the checksum value of the graph in the preferences to later comparison
+        /// Indicates if the workspace has been changed based on node connections and store the checksum value of the graph.
         /// </summary>
         /// <returns></returns>
-        private bool HasSubstantialCheckSum()
+        private bool HasDifferentialCheckSum()
         {
-            bool substantialChecksum = false;
+            bool differentialChecksum = false;
             string graphId = Model.CurrentWorkspace.Guid.ToString();
 
-            GraphChecksumItem checksumItem = PreferenceSettings.GraphChecksumItemsList.Where(i => i.GraphId == graphId).FirstOrDefault();
-            if (checksumItem != null)
+
+            Model.GraphChecksumDictionary.TryGetValue(graphId, out List<string> checksums);
+
+            // compare the current checksum with previous hash values.
+            if (checksums != null)
             {
-                if (checksumItem.Checksum != currentWorkspaceViewModel.Checksum)
+                if (!checksums.Contains(currentWorkspaceViewModel.CurrentCheckSum))
                 {
-                    PreferenceSettings.GraphChecksumItemsList.Remove(checksumItem);
-                    PreferenceSettings.GraphChecksumItemsList.Add(new GraphChecksumItem() { GraphId = graphId, Checksum = currentWorkspaceViewModel.Checksum });
-                    substantialChecksum = true;
+                    checksums.Add(currentWorkspaceViewModel.CurrentCheckSum);
+                    Model.GraphChecksumDictionary.Remove(graphId);
+                    Model.GraphChecksumDictionary.Add(graphId, checksums);
+                    differentialChecksum = true;
                 }
             }
             else
             {
-                PreferenceSettings.GraphChecksumItemsList.Add(new GraphChecksumItem() { GraphId = graphId, Checksum = currentWorkspaceViewModel.Checksum });
-                substantialChecksum = true;
+                Model.GraphChecksumDictionary.Add(graphId, new List<string>() { currentWorkspaceViewModel.CurrentCheckSum });
+                differentialChecksum = true;
+            }
+
+            // if the checksum is different from previous hashes, serialize this new info. 
+            if (differentialChecksum)
+            {
+                var graphChecksums = new List<GraphChecksumPair>();
+                foreach (KeyValuePair<string, List<string>> entry in Model.GraphChecksumDictionary)
+                {
+                    var item = new GraphChecksumPair
+                    {
+                        GraphId = entry.Key,
+                        Checksum = entry.Value
+                    };
+
+                    graphChecksums.Add(item);
+                }
+
+                var serializer = new XmlSerializer(Model.GraphChecksumList.GetType());
+                using (var writer = XmlWriter.Create(dynamoMLDataPath))
+                {
+                    Model.GraphChecksumList = graphChecksums;
+                    serializer.Serialize(writer, Model.GraphChecksumList);
+                }
             }
-            return substantialChecksum;
+
+            return differentialChecksum;
         }
 
         private void InternalSaveAs(string path, SaveContext saveContext, bool isBackup = false)
@@ -2239,13 +2285,14 @@ private void InternalSaveAs(string path, SaveContext saveContext, bool isBackup
                 {
                     AddToRecentFiles(path);
 
-                    if ((currentWorkspaceViewModel?.IsHomeSpace ?? true) && HomeSpace.HasRunWithoutCrash && Model.CurrentWorkspace.IsValidForFDX && IsMLDataIngestionPipelineinBeta && currentWorkspaceViewModel.Checksum != string.Empty)
+                    if ((currentWorkspaceViewModel?.IsHomeSpace ?? true) && HomeSpace.HasRunWithoutCrash &&
+                         Model.CurrentWorkspace.IsValidForFDX && !IsMLDataIngestionPipelineinBeta && currentWorkspaceViewModel.Checksum != string.Empty)
                     {
-                        Model.Logger.Log("The Workspace is valid for FDX");
-                        Model.Logger.Log("The Workspace id is : " + currentWorkspaceViewModel.Model.Guid.ToString());
-                        Model.Logger.Log("The Workspace checksum is : " + currentWorkspaceViewModel.Checksum);
-                        Model.Logger.Log("The Workspace has Substantial checksum, so is ready to send to FDX : " + HasSubstantialCheckSum().ToString());
-                        MLDataPipelineExtension.DynamoMLDataPipeline.DataExchange(path);
+                        if (HasDifferentialCheckSum())
+                        {
+                            Model.Logger.Log("This Workspace is shared to train the Dynamo Machine Learning model.");
+                            MLDataPipelineExtension.DynamoMLDataPipeline.DataExchange(path);
+                        }
                     }
                 }                           
             }

src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs

@@ -4,7 +4,6 @@
 using System.Collections.Specialized;
 using System.ComponentModel;
 using System.Diagnostics;
-using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Windows;
@@ -363,57 +362,42 @@ public bool IsHomeSpace
         [JsonIgnore]
         internal JObject JsonRepresentation { get; set; }
 
+        [JsonIgnore]
+        internal string CurrentCheckSum { get; set; }
+
         /// <summary>
-        /// Returns the stringified representation of the connected nodes
+        /// Returns the stringified representation of the node connections in the workspace.
         /// </summary>
         [JsonIgnore]
         public string Checksum
         {
             get
             {
                 List<string> nodeInfoConnections = new List<string>();
-                JObject jsonWorkspace = JsonRepresentation;
-                var nodes = jsonWorkspace["Nodes"];                
+                var connectors = Connectors;
 
-                List<string> nodeIds = new List<string>();
-                foreach (JObject node in nodes)
+                foreach (var connector in Connectors)
                 {
-                    var nodeProperties = node.Children<JProperty>();
-                    JProperty id = nodeProperties.FirstOrDefault(x => x.Name == "Id");
-                    nodeIds.Add(id.Value.ToString());
-                }
-
-                nodeIds.Sort();
-
-                foreach (string nodeId in nodeIds)
-                {
-                    List<string> outputIds = new List<string>();
-                    var node = jsonWorkspace["Nodes"].Where(t => t.Value<string>("Id") == nodeId).Select(t => t).FirstOrDefault();
-                    var outputsProperty = node.Children<JProperty>().FirstOrDefault(x => x.Name == "Outputs");
-                    var outputs = (JArray)outputsProperty.Value;
-                    int outputIndex = 1;
-
-                    foreach (JObject output in outputs)
-                    {
-                        var outputProperties = output.Children<JProperty>();
-                        JProperty outputId = outputProperties.FirstOrDefault(x => x.Name == "Id");
-                        outputIds.Add(outputId.Value.ToString());
+                    var connectorModel = connector.ConnectorModel;
 
-                        var connectorsProperty = jsonWorkspace["Connectors"].Where(t => t.Value<string>("Start") == outputId.Value.ToString());
+                    var startingPort= connectorModel.Start;
+                    var endingPort = connectorModel.End;
 
-                        foreach (var connector in connectorsProperty)
-                        {
-                            var connectorProperties = connector.Children<JProperty>();
-                            JProperty endProperty = connectorProperties.FirstOrDefault(x => x.Name == "End");
-                            string inputId = (String)endProperty.Value;
+                    // node info connections has a unique id in the format: startnodeid[outputindex]endnodeid[outputindex].
+                    nodeInfoConnections.Add(startingPort.Owner.AstIdentifierGuid + "[" + startingPort.Index.ToString() + "]" + endingPort.Owner.AstIdentifierGuid + "[" + endingPort.Index.ToString() + "]");
+                }
 
-                            var outputConnectedNode = GetNodeByInputId(inputId, jsonWorkspace);
-                            nodeInfoConnections.Add(nodeId + "|[" + outputIndex.ToString() + "|" + outputConnectedNode.Item1 + "|" + outputConnectedNode.Item2.ToString() + "]");
-                        }
-                        outputIndex++;
-                    }
+                if (nodeInfoConnections.Count > 0)
+                {
+                    var checksumhash = Hash.ToSha256String(String.Join(",", nodeInfoConnections));
+                    CurrentCheckSum = checksumhash;
+                    return checksumhash;
+                }
+                else
+                {
+                    CurrentCheckSum = string.Empty;
+                    return string.Empty;
                 }
-                return nodeInfoConnections.Count > 0 ? string.Join(",", nodeInfoConnections) : string.Empty;
             }            
         }
 

src/DynamoUtilities/Hash.cs

@@ -91,6 +91,22 @@ internal static string ToBase32String(byte[] input, bool addPadding = false)
 
             return result;
         }
+
+
+        // converts the string into a sha 256 hash.
+        internal static string ToSha256String(string s)
+        {
+            using var mySHA256 = SHA256.Create();
+
+            byte[] bytes = mySHA256.ComputeHash(Encoding.UTF8.GetBytes(s));
+            var sb = new StringBuilder();
+
+            for (int i = 0; i < bytes.Length; i++)
+            {
+                sb.Append(bytes[i].ToString("x2"));
+            }
+            return sb.ToString();
+        }
     }
 }
 

test/DynamoCoreTests/Configuration/PreferenceSettingsTests.cs

@@ -1,13 +1,13 @@
+using System;
 using System.Collections.Generic;
 using System.IO;
-using Dynamo.Configuration;
-using Dynamo.Models;
-using NUnit.Framework;
 using System.Linq;
-using System;
-using Dynamo.Interfaces;
 using System.Reflection;
+using Dynamo.Configuration;
+using Dynamo.Interfaces;
+using Dynamo.Models;
 using Dynamo.Utilities;
+using NUnit.Framework;
 
 namespace Dynamo.Tests.Configuration
 {
@@ -358,18 +358,6 @@ PreferencesComparison comparePrefenceSettings(PreferenceSettings defaultSettings
                             propertiesWithDifferentValue.Add(destinationPi.Name);
                         }
                     }
-                    else if (destinationPi.PropertyType == typeof(List<GraphChecksumItem>))
-                    {
-                        if (((List<GraphChecksumItem>)sourcePi.GetValue(newGeneralSettings, null)).Count ==
-                            ((List<GraphChecksumItem>)destinationPi.GetValue(defaultSettings, null)).Count)
-                        {
-                            propertiesWithSameValue.Add(destinationPi.Name);
-                        }
-                        else
-                        {
-                            propertiesWithDifferentValue.Add(destinationPi.Name);
-                        }
-                    }
                     else
                     {
                         if (newValue?.ToString() == oldValue?.ToString())