diff --git a/src/DynamoCore/Configuration/GraphChecksumItem.cs b/src/DynamoCore/Configuration/GraphChecksumItem.cs index a00b4538800..de6bfdd2fe8 100644 --- a/src/DynamoCore/Configuration/GraphChecksumItem.cs +++ b/src/DynamoCore/Configuration/GraphChecksumItem.cs @@ -1,17 +1,26 @@ using System; -using System.Collections.ObjectModel; -using Dynamo.Core; -using Dynamo.Properties; +using System.Collections.Generic; namespace Dynamo.Configuration { /// /// Represents the stringified version of the nodes connections from a graph /// + [Obsolete("This property is not needed anymore in the preference settings and can be removed in a future version of Dynamo.")] public class GraphChecksumItem { public string GraphId { get; set; } public string Checksum { get; set; } } + + /// + /// Represents the stringified version of the nodes connections from a graph + /// + public class GraphChecksumPair + { + public string GraphId { get; set; } + + public List Checksum { get; set; } + } } diff --git a/src/DynamoCore/Configuration/IPreferences.cs b/src/DynamoCore/Configuration/IPreferences.cs index 4134b2705c2..8c391feb35f 100644 --- a/src/DynamoCore/Configuration/IPreferences.cs +++ b/src/DynamoCore/Configuration/IPreferences.cs @@ -148,6 +148,7 @@ public interface IPreferences /// Active state to set void SetIsBackgroundPreviewActive(string name, bool value); + [Obsolete("This property is not needed anymore in the preference settings and can be removed in a future version of Dynamo.")] /// /// Return a list of GraphChecksumItems /// diff --git a/src/DynamoCore/Configuration/PreferenceSettings.cs b/src/DynamoCore/Configuration/PreferenceSettings.cs index bc8da91ffbb..5848ce6c0bf 100644 --- a/src/DynamoCore/Configuration/PreferenceSettings.cs +++ b/src/DynamoCore/Configuration/PreferenceSettings.cs @@ -472,6 +472,7 @@ public bool DisableTrustWarnings /// /// Return a list of GraphChecksumItems /// + [Obsolete("This property is not needed anymore in the preference settings and can be removed in a future version of Dynamo.")] public List GraphChecksumItemsList { get; set; } // This function is used to deserialize the trusted locations manually diff --git a/src/DynamoCore/Models/DynamoModel.cs b/src/DynamoCore/Models/DynamoModel.cs index 55ce632758d..309cbe98984 100644 --- a/src/DynamoCore/Models/DynamoModel.cs +++ b/src/DynamoCore/Models/DynamoModel.cs @@ -148,6 +148,17 @@ internal LuceneSearchUtility LuceneUtility } } + /// + /// Return a dictionary of GraphChecksumItems. + /// Key will be the workspace guid and its value will be a list of saved checksums(sha256 hash) for that workspace. + /// + internal Dictionary> GraphChecksumDictionary { get; set; } + + /// + /// Return a list of GraphChecksumItems + /// + public List GraphChecksumList { get; set; } + #endregion #region static properties @@ -979,6 +990,10 @@ protected DynamoModel(IStartConfiguration config) { LuceneUtility.DisposeWriter(); } + + GraphChecksumList = new List(); + GraphChecksumDictionary = new Dictionary>(); + // This event should only be raised at the end of this method. DynamoReady(new ReadyParams(this)); } diff --git a/src/DynamoCore/PublicAPI.Unshipped.txt b/src/DynamoCore/PublicAPI.Unshipped.txt index 6b064216cef..fe7729eb432 100644 --- a/src/DynamoCore/PublicAPI.Unshipped.txt +++ b/src/DynamoCore/PublicAPI.Unshipped.txt @@ -107,6 +107,12 @@ Dynamo.Configuration.GraphChecksumItem.Checksum.set -> void Dynamo.Configuration.GraphChecksumItem.GraphChecksumItem() -> void Dynamo.Configuration.GraphChecksumItem.GraphId.get -> string Dynamo.Configuration.GraphChecksumItem.GraphId.set -> void +Dynamo.Configuration.GraphChecksumPair +Dynamo.Configuration.GraphChecksumPair.Checksum.get -> System.Collections.Generic.List +Dynamo.Configuration.GraphChecksumPair.Checksum.set -> void +Dynamo.Configuration.GraphChecksumPair.GraphChecksumPair() -> void +Dynamo.Configuration.GraphChecksumPair.GraphId.get -> string +Dynamo.Configuration.GraphChecksumPair.GraphId.set -> void Dynamo.Configuration.GroupStyleItem Dynamo.Configuration.GroupStyleItem.GroupStyleItem() -> void Dynamo.Configuration.PreferenceSettings @@ -1864,6 +1870,8 @@ Dynamo.Models.DynamoModel.ExtensionManager.get -> Dynamo.Extensions.IExtensionMa Dynamo.Models.DynamoModel.ForceRun() -> void Dynamo.Models.DynamoModel.ForceRunCancelCommand Dynamo.Models.DynamoModel.ForceRunCancelCommand.ForceRunCancelCommand(bool showErrors, bool cancelRun) -> void +Dynamo.Models.DynamoModel.GraphChecksumList.get -> System.Collections.Generic.List +Dynamo.Models.DynamoModel.GraphChecksumList.set -> void Dynamo.Models.DynamoModel.HostVersion.get -> string Dynamo.Models.DynamoModel.HostVersion.set -> void Dynamo.Models.DynamoModel.InsertFileCommand diff --git a/src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs b/src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs index 0becc9ac897..87fbdd06dc7 100644 --- a/src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs @@ -14,6 +14,8 @@ using System.Windows.Forms; using System.Windows.Media; using System.Windows.Threading; +using System.Xml; +using System.Xml.Serialization; using Dynamo.Configuration; using Dynamo.Core; using Dynamo.Engine; @@ -67,6 +69,7 @@ public partial class DynamoViewModel : ViewModelBase, IDynamoViewModel private Point transformOrigin; private bool showStartPage = false; private PreferencesViewModel preferencesViewModel; + private string dynamoMLDataPath = string.Empty; // Can the user run the graph private bool CanRunGraph => HomeSpace.RunSettings.RunEnabled && !HomeSpace.GraphRunInProgress; @@ -768,11 +771,26 @@ protected DynamoViewModel(StartConfiguration startConfiguration) model.ComputeModelDeserialized += model_ComputeModelDeserialized; model.RequestNotification += model_RequestNotification; - preferencesViewModel = new PreferencesViewModel(this); + preferencesViewModel = new PreferencesViewModel(this); + + dynamoMLDataPath = Path.Combine(Model.PathManager.UserDataDirectory, "DynamoMLDataPipeline.xml"); if (!DynamoModel.IsTestMode && !DynamoModel.IsHeadless) { model.State = DynamoModel.DynamoModelState.StartedUI; + + // deserialize workspace checksum hashes that is used for Dynamo ML data pipeline. + var checksums = new List(); + var serializer = new XmlSerializer(Model.GraphChecksumList.GetType()); + + if (File.Exists(dynamoMLDataPath)) + { + using (var reader = XmlReader.Create(dynamoMLDataPath)) + { + checksums = (List)serializer.Deserialize(reader); + } + Model.GraphChecksumDictionary = checksums.ToDictionary(x => x.GraphId, x => x.Checksum); + } } FileTrustViewModel = new FileTrustWarningViewModel(); @@ -2192,30 +2210,58 @@ internal bool CanSaveAs(object parameters) } /// - /// Indicates if the graph has been changed substantially bearing in mind the connections of its nodes and store the checksum value of the graph in the preferences to later comparison + /// Indicates if the workspace has been changed based on node connections and store the checksum value of the graph. /// /// - private bool HasSubstantialCheckSum() + private bool HasDifferentialCheckSum() { - bool substantialChecksum = false; + bool differentialChecksum = false; string graphId = Model.CurrentWorkspace.Guid.ToString(); - GraphChecksumItem checksumItem = PreferenceSettings.GraphChecksumItemsList.Where(i => i.GraphId == graphId).FirstOrDefault(); - if (checksumItem != null) + + Model.GraphChecksumDictionary.TryGetValue(graphId, out List checksums); + + // compare the current checksum with previous hash values. + if (checksums != null) { - if (checksumItem.Checksum != currentWorkspaceViewModel.Checksum) + if (!checksums.Contains(currentWorkspaceViewModel.CurrentCheckSum)) { - PreferenceSettings.GraphChecksumItemsList.Remove(checksumItem); - PreferenceSettings.GraphChecksumItemsList.Add(new GraphChecksumItem() { GraphId = graphId, Checksum = currentWorkspaceViewModel.Checksum }); - substantialChecksum = true; + checksums.Add(currentWorkspaceViewModel.CurrentCheckSum); + Model.GraphChecksumDictionary.Remove(graphId); + Model.GraphChecksumDictionary.Add(graphId, checksums); + differentialChecksum = true; } } else { - PreferenceSettings.GraphChecksumItemsList.Add(new GraphChecksumItem() { GraphId = graphId, Checksum = currentWorkspaceViewModel.Checksum }); - substantialChecksum = true; + Model.GraphChecksumDictionary.Add(graphId, new List() { currentWorkspaceViewModel.CurrentCheckSum }); + differentialChecksum = true; + } + + // if the checksum is different from previous hashes, serialize this new info. + if (differentialChecksum) + { + var graphChecksums = new List(); + foreach (KeyValuePair> entry in Model.GraphChecksumDictionary) + { + var item = new GraphChecksumPair + { + GraphId = entry.Key, + Checksum = entry.Value + }; + + graphChecksums.Add(item); + } + + var serializer = new XmlSerializer(Model.GraphChecksumList.GetType()); + using (var writer = XmlWriter.Create(dynamoMLDataPath)) + { + Model.GraphChecksumList = graphChecksums; + serializer.Serialize(writer, Model.GraphChecksumList); + } } - return substantialChecksum; + + return differentialChecksum; } private void InternalSaveAs(string path, SaveContext saveContext, bool isBackup = false) @@ -2239,13 +2285,14 @@ private void InternalSaveAs(string path, SaveContext saveContext, bool isBackup { AddToRecentFiles(path); - if ((currentWorkspaceViewModel?.IsHomeSpace ?? true) && HomeSpace.HasRunWithoutCrash && Model.CurrentWorkspace.IsValidForFDX && IsMLDataIngestionPipelineinBeta && currentWorkspaceViewModel.Checksum != string.Empty) + if ((currentWorkspaceViewModel?.IsHomeSpace ?? true) && HomeSpace.HasRunWithoutCrash && + Model.CurrentWorkspace.IsValidForFDX && !IsMLDataIngestionPipelineinBeta && currentWorkspaceViewModel.Checksum != string.Empty) { - Model.Logger.Log("The Workspace is valid for FDX"); - Model.Logger.Log("The Workspace id is : " + currentWorkspaceViewModel.Model.Guid.ToString()); - Model.Logger.Log("The Workspace checksum is : " + currentWorkspaceViewModel.Checksum); - Model.Logger.Log("The Workspace has Substantial checksum, so is ready to send to FDX : " + HasSubstantialCheckSum().ToString()); - MLDataPipelineExtension.DynamoMLDataPipeline.DataExchange(path); + if (HasDifferentialCheckSum()) + { + Model.Logger.Log("This Workspace is shared to train the Dynamo Machine Learning model."); + MLDataPipelineExtension.DynamoMLDataPipeline.DataExchange(path); + } } } } diff --git a/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs b/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs index 4f1dcc9fa1c..271ad7a7065 100644 --- a/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs @@ -4,7 +4,6 @@ using System.Collections.Specialized; using System.ComponentModel; using System.Diagnostics; -using System.Globalization; using System.IO; using System.Linq; using System.Windows; @@ -363,8 +362,11 @@ public bool IsHomeSpace [JsonIgnore] internal JObject JsonRepresentation { get; set; } + [JsonIgnore] + internal string CurrentCheckSum { get; set; } + /// - /// Returns the stringified representation of the connected nodes + /// Returns the stringified representation of the node connections in the workspace. /// [JsonIgnore] public string Checksum @@ -372,48 +374,30 @@ public string Checksum get { List nodeInfoConnections = new List(); - JObject jsonWorkspace = JsonRepresentation; - var nodes = jsonWorkspace["Nodes"]; + var connectors = Connectors; - List nodeIds = new List(); - foreach (JObject node in nodes) + foreach (var connector in Connectors) { - var nodeProperties = node.Children(); - JProperty id = nodeProperties.FirstOrDefault(x => x.Name == "Id"); - nodeIds.Add(id.Value.ToString()); - } - - nodeIds.Sort(); - - foreach (string nodeId in nodeIds) - { - List outputIds = new List(); - var node = jsonWorkspace["Nodes"].Where(t => t.Value("Id") == nodeId).Select(t => t).FirstOrDefault(); - var outputsProperty = node.Children().FirstOrDefault(x => x.Name == "Outputs"); - var outputs = (JArray)outputsProperty.Value; - int outputIndex = 1; - - foreach (JObject output in outputs) - { - var outputProperties = output.Children(); - JProperty outputId = outputProperties.FirstOrDefault(x => x.Name == "Id"); - outputIds.Add(outputId.Value.ToString()); + var connectorModel = connector.ConnectorModel; - var connectorsProperty = jsonWorkspace["Connectors"].Where(t => t.Value("Start") == outputId.Value.ToString()); + var startingPort= connectorModel.Start; + var endingPort = connectorModel.End; - foreach (var connector in connectorsProperty) - { - var connectorProperties = connector.Children(); - JProperty endProperty = connectorProperties.FirstOrDefault(x => x.Name == "End"); - string inputId = (String)endProperty.Value; + // node info connections has a unique id in the format: startnodeid[outputindex]endnodeid[outputindex]. + nodeInfoConnections.Add(startingPort.Owner.AstIdentifierGuid + "[" + startingPort.Index.ToString() + "]" + endingPort.Owner.AstIdentifierGuid + "[" + endingPort.Index.ToString() + "]"); + } - var outputConnectedNode = GetNodeByInputId(inputId, jsonWorkspace); - nodeInfoConnections.Add(nodeId + "|[" + outputIndex.ToString() + "|" + outputConnectedNode.Item1 + "|" + outputConnectedNode.Item2.ToString() + "]"); - } - outputIndex++; - } + if (nodeInfoConnections.Count > 0) + { + var checksumhash = Hash.ToSha256String(String.Join(",", nodeInfoConnections)); + CurrentCheckSum = checksumhash; + return checksumhash; + } + else + { + CurrentCheckSum = string.Empty; + return string.Empty; } - return nodeInfoConnections.Count > 0 ? string.Join(",", nodeInfoConnections) : string.Empty; } } diff --git a/src/DynamoUtilities/Hash.cs b/src/DynamoUtilities/Hash.cs index 075c421e92b..08bf0257f5c 100644 --- a/src/DynamoUtilities/Hash.cs +++ b/src/DynamoUtilities/Hash.cs @@ -91,6 +91,22 @@ internal static string ToBase32String(byte[] input, bool addPadding = false) return result; } + + + // converts the string into a sha 256 hash. + internal static string ToSha256String(string s) + { + using var mySHA256 = SHA256.Create(); + + byte[] bytes = mySHA256.ComputeHash(Encoding.UTF8.GetBytes(s)); + var sb = new StringBuilder(); + + for (int i = 0; i < bytes.Length; i++) + { + sb.Append(bytes[i].ToString("x2")); + } + return sb.ToString(); + } } } diff --git a/test/DynamoCoreTests/Configuration/PreferenceSettingsTests.cs b/test/DynamoCoreTests/Configuration/PreferenceSettingsTests.cs index 7c23143c70a..9d0ae696e33 100644 --- a/test/DynamoCoreTests/Configuration/PreferenceSettingsTests.cs +++ b/test/DynamoCoreTests/Configuration/PreferenceSettingsTests.cs @@ -1,13 +1,13 @@ +using System; using System.Collections.Generic; using System.IO; -using Dynamo.Configuration; -using Dynamo.Models; -using NUnit.Framework; using System.Linq; -using System; -using Dynamo.Interfaces; using System.Reflection; +using Dynamo.Configuration; +using Dynamo.Interfaces; +using Dynamo.Models; using Dynamo.Utilities; +using NUnit.Framework; namespace Dynamo.Tests.Configuration { @@ -358,18 +358,6 @@ PreferencesComparison comparePrefenceSettings(PreferenceSettings defaultSettings propertiesWithDifferentValue.Add(destinationPi.Name); } } - else if (destinationPi.PropertyType == typeof(List)) - { - if (((List)sourcePi.GetValue(newGeneralSettings, null)).Count == - ((List)destinationPi.GetValue(defaultSettings, null)).Count) - { - propertiesWithSameValue.Add(destinationPi.Name); - } - else - { - propertiesWithDifferentValue.Add(destinationPi.Name); - } - } else { if (newValue?.ToString() == oldValue?.ToString()) diff --git a/test/DynamoCoreWpfTests/WorkspaceSaving.cs b/test/DynamoCoreWpfTests/WorkspaceSaving.cs index cb524c089d6..79bea7c0096 100644 --- a/test/DynamoCoreWpfTests/WorkspaceSaving.cs +++ b/test/DynamoCoreWpfTests/WorkspaceSaving.cs @@ -1541,6 +1541,21 @@ public void CustomNodeWorkspaceSaveBackupKeepNodeName() Assert.True(initialNodeName == customNodeInstance.Name); Assert.False(Path.GetFileNameWithoutExtension(savePath) == customNodeInstance.Name); } + + /// + /// Workspace checksum test. + /// + [Test] + public void WorkapceChecksumTest() + { + var model = ViewModel.Model; + var examplePath = Path.Combine(TestDirectory, @"core\math", "Add.dyn"); + ViewModel.OpenCommand.Execute(examplePath); + + var checksumString = ViewModel.CurrentSpaceViewModel.Checksum; + + Assert.AreEqual("65b395b9874b9d82e088093f30234c496704006030ecf35471404f62b62a6442", checksumString); + } #endregion } }