From 0984ed997cd1b8be296d1e2b4248ce22ff507e03 Mon Sep 17 00:00:00 2001 From: reddyashish <43763136+reddyashish@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:42:20 -0700 Subject: [PATCH] Final improvements to Dynamo ML data pipeline. (#15116) (#15145) * updates * more updates * Update DynamoViewModel.cs * Update DynamoMLDataPipeline.cs * Remove IExtensionSource from DynamoMLDataPipelineExtension. * Update PublicAPI.Unshipped.txt * Update IDSDKManager.cs * updates * Update DynamoMLDataPipeline.cs * Update DynamoMLDataPipeline.cs * Update App.config * update log message. * change parameter type. * Update DynamoMLDataPipeline.cs * update greg nuget version. * Use IOAuth2UserIDProvider interface * update conflicts * Update PublicAPI.Unshipped.txt * rename to RequestAttribute * change class name. * update analytics info * Update DynamoViewModel.cs * add checks for authprovider * Update DynamoMLDataPipeline.cs * Remove PII data from workspace * Update PIIDetector.cs (cherry picked from commit 727fe42bed006c77dcac55d0fcdbc22afba38863) --- .../Configuration/GraphChecksumItem.cs | 11 -- src/DynamoCore/Core/IDSDKManager.cs | 14 +- src/DynamoCore/DynamoCore.csproj | 2 +- src/DynamoCore/Models/DynamoModel.cs | 6 - src/DynamoCore/PublicAPI.Unshipped.txt | 9 +- src/DynamoCoreWpf/DynamoCoreWpf.csproj | 2 +- .../ViewModels/Core/DynamoViewModel.cs | 77 ++++---- .../ViewModels/Core/WorkspaceViewModel.cs | 1 - src/DynamoMLDataPipeline/App.config | 6 - src/DynamoMLDataPipeline/BaseComponent.cs | 4 +- src/DynamoMLDataPipeline/BinaryAsset.cs | 8 +- .../BinaryReferenceComponent.cs | 10 +- .../DynamoMLDataPipeline.cs | 165 +++++++----------- .../DynamoMLDataPipeline.csproj | 26 +-- .../DynamoMLDataPipelineExtension.cs | 27 +-- src/DynamoMLDataPipeline/ExchangeComponent.cs | 8 +- src/DynamoMLDataPipeline/InstanceAsset.cs | 2 +- .../ParameterComponent.cs | 6 +- .../{Attribute.cs => RequestAttribute.cs} | 4 +- src/DynamoMLDataPipeline/Schema.cs | 6 +- .../UploadAssetsRequestBody.cs | 2 +- src/DynamoPackages/DynamoPackages.csproj | 2 +- src/DynamoUtilities/Hash.cs | 3 + src/DynamoUtilities/PIIDetector.cs | 12 +- .../Properties/AssemblyInfo.cs | 1 + .../PackageDetailsViewExtension.csproj | 2 +- .../NodeDocumentationMarkdownGenerator.csproj | 2 +- test/DynamoCoreTests/DynamoCoreTests.csproj | 2 +- .../DynamoCoreWpfTests.csproj | 2 +- .../PackageManagerTests.csproj | 2 +- .../DynamoPackagesAnalyzer.csproj | 2 +- 31 files changed, 187 insertions(+), 239 deletions(-) rename src/DynamoMLDataPipeline/{Attribute.cs => RequestAttribute.cs} (78%) diff --git a/src/DynamoCore/Configuration/GraphChecksumItem.cs b/src/DynamoCore/Configuration/GraphChecksumItem.cs index de6bfdd2fe8..8f5963c52a8 100644 --- a/src/DynamoCore/Configuration/GraphChecksumItem.cs +++ b/src/DynamoCore/Configuration/GraphChecksumItem.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; namespace Dynamo.Configuration { @@ -13,14 +12,4 @@ public class GraphChecksumItem public string Checksum { get; set; } } - - /// - /// Represents the stringified version of the nodes connections from a graph - /// - public class GraphChecksumPair - { - public string GraphId { get; set; } - - public List Checksum { get; set; } - } } diff --git a/src/DynamoCore/Core/IDSDKManager.cs b/src/DynamoCore/Core/IDSDKManager.cs index c3aff23ea2a..533b0768cc2 100644 --- a/src/DynamoCore/Core/IDSDKManager.cs +++ b/src/DynamoCore/Core/IDSDKManager.cs @@ -12,7 +12,7 @@ namespace Dynamo.Core /// /// The class to provide auth APIs for IDSDK related methods. /// - public class IDSDKManager : IOAuth2AuthProvider, IOAuth2AccessTokenProvider, IDisposable + public class IDSDKManager : IOAuth2AuthProvider, IOAuth2AccessTokenProvider, IOAuth2UserIDProvider, IDisposable { /// /// Used by the auth provider to request authentication. @@ -78,6 +78,18 @@ public string Username } } + /// + /// Gets the userid of the logged in user. + /// + public string UserId + { + get + { + var result = IDSDK_GetUserInfo(); + return result != null ? result.UserId : String.Empty; + } + } + /// /// Used by the auth provider to sign request with the authorized token. /// diff --git a/src/DynamoCore/DynamoCore.csproj b/src/DynamoCore/DynamoCore.csproj index 4f3cdcc70bd..930e9db7436 100644 --- a/src/DynamoCore/DynamoCore.csproj +++ b/src/DynamoCore/DynamoCore.csproj @@ -31,7 +31,7 @@ - + diff --git a/src/DynamoCore/Models/DynamoModel.cs b/src/DynamoCore/Models/DynamoModel.cs index 2a41a547573..c13014e3ccf 100644 --- a/src/DynamoCore/Models/DynamoModel.cs +++ b/src/DynamoCore/Models/DynamoModel.cs @@ -155,11 +155,6 @@ internal LuceneSearchUtility LuceneUtility /// internal Dictionary> GraphChecksumDictionary { get; set; } - /// - /// Return a list of GraphChecksumItems - /// - public List GraphChecksumList { get; set; } - #endregion #region static properties @@ -992,7 +987,6 @@ protected DynamoModel(IStartConfiguration config) LuceneUtility.DisposeWriter(); } - GraphChecksumList = new List(); GraphChecksumDictionary = new Dictionary>(); // This event should only be raised at the end of this method. diff --git a/src/DynamoCore/PublicAPI.Unshipped.txt b/src/DynamoCore/PublicAPI.Unshipped.txt index b2d6700c3d2..f1d04bf7b45 100644 --- a/src/DynamoCore/PublicAPI.Unshipped.txt +++ b/src/DynamoCore/PublicAPI.Unshipped.txt @@ -107,12 +107,6 @@ Dynamo.Configuration.GraphChecksumItem.Checksum.set -> void Dynamo.Configuration.GraphChecksumItem.GraphChecksumItem() -> void Dynamo.Configuration.GraphChecksumItem.GraphId.get -> string Dynamo.Configuration.GraphChecksumItem.GraphId.set -> void -Dynamo.Configuration.GraphChecksumPair -Dynamo.Configuration.GraphChecksumPair.Checksum.get -> System.Collections.Generic.List -Dynamo.Configuration.GraphChecksumPair.Checksum.set -> void -Dynamo.Configuration.GraphChecksumPair.GraphChecksumPair() -> void -Dynamo.Configuration.GraphChecksumPair.GraphId.get -> string -Dynamo.Configuration.GraphChecksumPair.GraphId.set -> void Dynamo.Configuration.GroupStyleItem Dynamo.Configuration.GroupStyleItem.GroupStyleItem() -> void Dynamo.Configuration.PreferenceSettings @@ -374,6 +368,7 @@ Dynamo.Core.IDSDKManager.LoginStateChanged -> System.Action void Dynamo.Core.IDSDKManager.RequestLogin -> System.Func Dynamo.Core.IDSDKManager.SignRequest(ref RestSharp.RestRequest m, RestSharp.RestClient client) -> void +Dynamo.Core.IDSDKManager.UserId.get -> string Dynamo.Core.IDSDKManager.Username.get -> string Dynamo.Core.NotificationObject Dynamo.Core.NotificationObject.NotificationObject() -> void @@ -1876,8 +1871,6 @@ Dynamo.Models.DynamoModel.ExtensionManager.get -> Dynamo.Extensions.IExtensionMa Dynamo.Models.DynamoModel.ForceRun() -> void Dynamo.Models.DynamoModel.ForceRunCancelCommand Dynamo.Models.DynamoModel.ForceRunCancelCommand.ForceRunCancelCommand(bool showErrors, bool cancelRun) -> void -Dynamo.Models.DynamoModel.GraphChecksumList.get -> System.Collections.Generic.List -Dynamo.Models.DynamoModel.GraphChecksumList.set -> void Dynamo.Models.DynamoModel.HostVersion.get -> string Dynamo.Models.DynamoModel.HostVersion.set -> void Dynamo.Models.DynamoModel.InsertFileCommand diff --git a/src/DynamoCoreWpf/DynamoCoreWpf.csproj b/src/DynamoCoreWpf/DynamoCoreWpf.csproj index a7c55e397ac..97fac04cd36 100644 --- a/src/DynamoCoreWpf/DynamoCoreWpf.csproj +++ b/src/DynamoCoreWpf/DynamoCoreWpf.csproj @@ -170,7 +170,7 @@ - + diff --git a/src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs b/src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs index b4a08922a4a..0c79ea9c016 100644 --- a/src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs @@ -14,8 +14,6 @@ using System.Windows.Forms; using System.Windows.Media; using System.Windows.Threading; -using System.Xml; -using System.Xml.Serialization; using Dynamo.Configuration; using Dynamo.Core; using Dynamo.Engine; @@ -50,6 +48,7 @@ using DynamoMLDataPipeline; using DynamoUtilities; using ICSharpCode.AvalonEdit; +using Newtonsoft.Json; using PythonNodeModels; using ISelectable = Dynamo.Selection.ISelectable; using WpfResources = Dynamo.Wpf.Properties.Resources; @@ -70,6 +69,7 @@ public partial class DynamoViewModel : ViewModelBase, IDynamoViewModel private bool showStartPage = false; private PreferencesViewModel preferencesViewModel; private string dynamoMLDataPath = string.Empty; + private const string dynamoMLDataFileName = "DynamoMLDataPipeline.json"; // Can the user run the graph private bool CanRunGraph => HomeSpace.RunSettings.RunEnabled && !HomeSpace.GraphRunInProgress; @@ -209,7 +209,7 @@ internal bool IsMLDataIngestionPipelineinBeta { get { - return DynamoModel.FeatureFlags?.CheckFeatureFlag("IsMLDataIngestionPipelineinBeta", false) ?? false; + return DynamoModel.FeatureFlags?.CheckFeatureFlag("IsMLDataIngestionPipelineinBeta", true) ?? true; } } @@ -776,23 +776,23 @@ protected DynamoViewModel(StartConfiguration startConfiguration) preferencesViewModel = new PreferencesViewModel(this); - dynamoMLDataPath = Path.Combine(Model.PathManager.UserDataDirectory, "DynamoMLDataPipeline.xml"); + dynamoMLDataPath = Path.Combine(Model.PathManager.UserDataDirectory, dynamoMLDataFileName); if (!DynamoModel.IsTestMode && !DynamoModel.IsHeadless) { model.State = DynamoModel.DynamoModelState.StartedUI; // deserialize workspace checksum hashes that is used for Dynamo ML data pipeline. - var checksums = new List(); - var serializer = new XmlSerializer(Model.GraphChecksumList.GetType()); - if (File.Exists(dynamoMLDataPath)) { - using (var reader = XmlReader.Create(dynamoMLDataPath)) + try + { + Model.GraphChecksumDictionary = JsonConvert.DeserializeObject>>(File.ReadAllText(dynamoMLDataPath)); + } + catch (Exception ex) { - checksums = (List)serializer.Deserialize(reader); + Model.Logger.Log($"Failed to deserialize {dynamoMLDataFileName} : {ex.Message}", LogLevel.File); } - Model.GraphChecksumDictionary = checksums.ToDictionary(x => x.GraphId, x => x.Checksum); } } @@ -2273,7 +2273,6 @@ private bool HasDifferentialCheckSum() bool differentialChecksum = false; string graphId = Model.CurrentWorkspace.Guid.ToString(); - Model.GraphChecksumDictionary.TryGetValue(graphId, out List checksums); // compare the current checksum with previous hash values. @@ -2296,23 +2295,17 @@ private bool HasDifferentialCheckSum() // if the checksum is different from previous hashes, serialize this new info. if (differentialChecksum) { - var graphChecksums = new List(); - foreach (KeyValuePair> entry in Model.GraphChecksumDictionary) + try { - var item = new GraphChecksumPair + using (StreamWriter file = File.CreateText(dynamoMLDataPath)) { - GraphId = entry.Key, - Checksum = entry.Value - }; - - graphChecksums.Add(item); + JsonSerializer serializer = new JsonSerializer(); + serializer.Serialize(file, Model.GraphChecksumDictionary); + } } - - var serializer = new XmlSerializer(Model.GraphChecksumList.GetType()); - using (var writer = XmlWriter.Create(dynamoMLDataPath)) + catch (Exception ex) { - Model.GraphChecksumList = graphChecksums; - serializer.Serialize(writer, Model.GraphChecksumList); + Model.Logger.Log("Failed to serialize " + dynamoMLDataFileName + "::" + ex.Message, LogLevel.File); } } @@ -2339,16 +2332,6 @@ private void InternalSaveAs(string path, SaveContext saveContext, bool isBackup if (!isBackup && hasSaved) { AddToRecentFiles(path); - - if ((currentWorkspaceViewModel?.IsHomeSpace ?? true) && HomeSpace.HasRunWithoutCrash && - Model.CurrentWorkspace.IsValidForFDX && !IsMLDataIngestionPipelineinBeta && currentWorkspaceViewModel.Checksum != string.Empty) - { - if (HasDifferentialCheckSum()) - { - Model.Logger.Log("This Workspace is shared to train the Dynamo Machine Learning model."); - MLDataPipelineExtension.DynamoMLDataPipeline.DataExchange(path); - } - } } } catch (Exception ex) @@ -2936,6 +2919,32 @@ public void MakeNewHomeWorkspace(object parameter) private void CloseHomeWorkspace(object parameter) { + // Upon closing a workspace, validate if the workspace is valid to be sent to the ML datapipeline and then send it. + if (!DynamoModel.IsTestMode && !HomeSpace.HasUnsavedChanges && (currentWorkspaceViewModel?.IsHomeSpace ?? true) && HomeSpace.HasRunWithoutCrash) + { + if (!IsMLDataIngestionPipelineinBeta && Model.CurrentWorkspace.IsValidForFDX && currentWorkspaceViewModel.Checksum != string.Empty) + { + if (HasDifferentialCheckSum()) + { + Model.Logger.Log("This Workspace is being shared to train the Dynamo Machine Learning model.", LogLevel.File); + var workspacePath = model.CurrentWorkspace.FileName; + + Task.Run(() => + { + try + { + MLDataPipelineExtension.DynamoMLDataPipeline.SendWorkspaceLog(workspacePath); + } + catch (Exception ex) + { + Model.Logger.Log("Failed to share this workspace with ML pipeline.", LogLevel.File); + Model.Logger.Log(ex.StackTrace, LogLevel.File); + } + }); + } + } + } + if (ClearHomeWorkspaceInternal()) { // If after closing the HOME workspace, and there are no other custom diff --git a/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs b/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs index 271ad7a7065..89c7fce8982 100644 --- a/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs +++ b/src/DynamoCoreWpf/ViewModels/Core/WorkspaceViewModel.cs @@ -374,7 +374,6 @@ public string Checksum get { List nodeInfoConnections = new List(); - var connectors = Connectors; foreach (var connector in Connectors) { diff --git a/src/DynamoMLDataPipeline/App.config b/src/DynamoMLDataPipeline/App.config index 6ea8f9b3715..4ed7efb8c06 100644 --- a/src/DynamoMLDataPipeline/App.config +++ b/src/DynamoMLDataPipeline/App.config @@ -10,11 +10,5 @@ - - - - - - diff --git a/src/DynamoMLDataPipeline/BaseComponent.cs b/src/DynamoMLDataPipeline/BaseComponent.cs index 2dda71998e2..490609a6a67 100644 --- a/src/DynamoMLDataPipeline/BaseComponent.cs +++ b/src/DynamoMLDataPipeline/BaseComponent.cs @@ -3,7 +3,7 @@ namespace DynamoMLDataPipeline { - class BaseComponent : Dictionary> + internal class BaseComponent : Dictionary> { private string objectId = "autodesk.design:components.base-1.0.0"; public BaseComponent(string name) @@ -19,7 +19,7 @@ public BaseComponent(string name) public string ObjectId { get { return objectId; } } } - class ObjectInfo + internal class ObjectInfo { [JsonProperty("name")] public string Name { get; set; } diff --git a/src/DynamoMLDataPipeline/BinaryAsset.cs b/src/DynamoMLDataPipeline/BinaryAsset.cs index 513da24c417..c06c188f1dc 100644 --- a/src/DynamoMLDataPipeline/BinaryAsset.cs +++ b/src/DynamoMLDataPipeline/BinaryAsset.cs @@ -5,13 +5,13 @@ namespace DynamoMLDataPipeline { - class BaseBinaryAsset + internal class BaseBinaryAsset { [JsonProperty("id")] public string Id { get; set; } } - class UploadedBinaryAsset : BaseBinaryAsset + internal class UploadedBinaryAsset : BaseBinaryAsset { public UploadedBinaryAsset(string guid) { @@ -19,7 +19,7 @@ public UploadedBinaryAsset(string guid) } } - class BinaryAsset : BaseBinaryAsset + internal class BinaryAsset : BaseBinaryAsset { public BinaryAsset() { @@ -37,7 +37,7 @@ public BinaryAsset() public string Type { get; set; } } - class BinaryAssets + internal class BinaryAssets { [JsonProperty("binaries")] public List Binaries { get; set; } diff --git a/src/DynamoMLDataPipeline/BinaryReferenceComponent.cs b/src/DynamoMLDataPipeline/BinaryReferenceComponent.cs index f3dd7d1d19e..b4479cf45a1 100644 --- a/src/DynamoMLDataPipeline/BinaryReferenceComponent.cs +++ b/src/DynamoMLDataPipeline/BinaryReferenceComponent.cs @@ -3,7 +3,7 @@ namespace DynamoMLDataPipeline { - class BinaryReferenceComponent : Dictionary> + internal class BinaryReferenceComponent : Dictionary> { private string objectId = "autodesk.data:binary.reference.component-1.0.0"; public string ObjectId { get { return objectId; } } @@ -17,7 +17,7 @@ public BinaryReferenceComponent(string binaryId) } } - class StringPropertySet : IPropertySet + internal class StringPropertySet : IPropertySet { [JsonProperty("id")] public string Id { get; set; } @@ -31,7 +31,7 @@ public StringPropertySet(string binaryId, string revision = "v0") } } - class IntPropertySet : IPropertySet + internal class IntPropertySet : IPropertySet { [JsonProperty("end")] public int End { get; set; } @@ -44,6 +44,6 @@ public IntPropertySet(int start = 0, int end = 8710) Start = start; } } - - interface IPropertySet { } + + internal interface IPropertySet { } } diff --git a/src/DynamoMLDataPipeline/DynamoMLDataPipeline.cs b/src/DynamoMLDataPipeline/DynamoMLDataPipeline.cs index e60c998af1c..f3c81f2b2c2 100644 --- a/src/DynamoMLDataPipeline/DynamoMLDataPipeline.cs +++ b/src/DynamoMLDataPipeline/DynamoMLDataPipeline.cs @@ -3,6 +3,8 @@ using System.IO; using System.Text; using Dynamo.Logging; +using Dynamo.Models; +using Dynamo.Utilities; using Greg; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -11,7 +13,6 @@ namespace DynamoMLDataPipeline { - internal class DynamoMLDataPipeline { internal static string CollectionId { get; set; } @@ -49,24 +50,45 @@ internal string ProductionCollectionID } } - internal static IOAuth2AccessTokenProvider AuthTokenProvider { get; set; } + internal DynamoModel DynamoModel { get; set; } + + // Authprovider to get the token. + internal IOAuth2AccessTokenProvider AuthTokenProvider { get; set; } + + // Authprovider to get user info. + internal IOAuth2UserIDProvider AuthUserInfoProvider { get; set; } + + // Points to the Dynamo version. + internal Version DynamoVersion { get; set; } + + // Id of the user sending the post request. + private string GetUserId() + { + return AuthUserInfoProvider?.UserId ?? string.Empty; + } + + // Authorization token needed for the restsharp post request in this pipeline. + private string GetAuthorizationToken() + { + return AuthTokenProvider.GetAccessToken(); + } - public static void AddHeadersToPostRequest(RestRequest request, string token) + internal static void AddHeadersToPostRequest(RestRequest request, string token) { request.AddHeader("Accept", "*/*"); request.AddHeader("Authorization", $"Bearer {token}"); request.AddHeader("Content-Type", "application/json"); } - public static string ConstructCreateAssetRequestBody(string schemaNamespaceId, string binaryId, string operation) + internal string ConstructCreateAssetRequestBody(string schemaNamespaceId, string binaryId, string operation) { // Define the custom parameter schemas var schemas = new List(); - var userIdSchema = new StringParameterSchema("UserId", schemaNamespaceId, "DynamoUserIdParam"); //here we need to pass the oxygenId + var userIdSchema = new StringParameterSchema("UserId", schemaNamespaceId, "DynamoUserIdParam"); schemas.Add(userIdSchema); - var hostSchema = new StringParameterSchema("Host", schemaNamespaceId, "DynamoHostParam"); //here we need to pass Dynamo's host similar to what we pass to ADP logs + var hostSchema = new StringParameterSchema("Host", schemaNamespaceId, "DynamoHostParam"); schemas.Add(hostSchema); - var dynamoVersionSchema = new StringParameterSchema("DynamoVersion", schemaNamespaceId, "DynamoVersionParam"); //here we need to pass Dynamo's version similar to what we pass to ADP logs + var dynamoVersionSchema = new StringParameterSchema("DynamoVersion", schemaNamespaceId, "DynamoVersionParam"); schemas.Add(dynamoVersionSchema); // Define the assets @@ -74,9 +96,10 @@ public static string ConstructCreateAssetRequestBody(string schemaNamespaceId, s // Construct parameter component var parameterComponent = new ParameterComponent(); - parameterComponent.AddParameterFromSchema("", userIdSchema); - parameterComponent.AddParameterFromSchema("", hostSchema); - parameterComponent.AddParameterFromSchema("", dynamoVersionSchema); + + parameterComponent.AddParameterFromSchema(GetUserId(), userIdSchema); + parameterComponent.AddParameterFromSchema(DynamoModel.HostAnalyticsInfo.HostName, hostSchema); + parameterComponent.AddParameterFromSchema(DynamoVersion.ToString(), dynamoVersionSchema); // Construct the base component var baseComponent = new BaseComponent("DynamoGraphLog"); @@ -92,15 +115,16 @@ public static string ConstructCreateAssetRequestBody(string schemaNamespaceId, s return bodyJSON; } - public static string ConstructCreateExchangeRequestBody() + public string ConstructCreateExchangeRequestBody() { // Instantiate attributes we want to pass to the data exchange // Note: we want to pass attributes to be able to query the data once collected // filtering on these attributes (query based on a custom attribute not currently // supported by the FDX API, but it may become available in the future) - var clientIdAttribute = new Attribute("clientId", "Dynamo"); - var clientVersionAttribute = new Attribute("clientVersion", ""); - var attributes = new List + var clientIdAttribute = new RequestAttribute("clientId", "Dynamo"); + var clientVersionAttribute = new RequestAttribute("clientVersion", DynamoVersion.ToString()); + + var attributes = new List { clientIdAttribute, clientVersionAttribute @@ -131,44 +155,26 @@ static public void EndFullFillment(RestClient client, string collectionId, strin RestRequest endFulfillmentRequest = new RestRequest(endFulfillmentUrl); AddHeadersToPostRequest(endFulfillmentRequest, token); var endFulfillmentResponse = client.ExecutePost(endFulfillmentRequest); - dynamic endFulfillmentResponseBody = JObject.Parse(endFulfillmentResponse.Content); - var fulfillmentStatus = endFulfillmentResponseBody.status.Value; - while (fulfillmentStatus == "IN_PROGRESS") - { - var fulfillmentStatusUrl = $"/v1/collections/{collectionId}/exchanges/{exchangeContainerId}/fulfillments/{fulfillmentId}"; - RestRequest fulfillementStatusRequest = new RestRequest(fulfillmentStatusUrl); - fulfillementStatusRequest.AddHeader("Authorization", $"Bearer {token}"); - var fulfillmentStatusResponse = client.ExecuteGet(fulfillementStatusRequest); - dynamic fulfillmentStatusResponseBody = JObject.Parse(fulfillmentStatusResponse.Content); - fulfillmentStatus = fulfillmentStatusResponseBody.status.Value; - } - LogMessage.Info($"The data exchange is {fulfillmentStatus}"); } static public string ConvertDynToBase64(string filePath) { - string parentDir = Directory.GetParent(filePath).FullName; - string savePath = Path.Combine(parentDir, "data.txt"); - // Read .dyn file as a string string sourceFileContent = File.ReadAllText(filePath); - LogMessage.Info($"Read file '{filePath}' with {sourceFileContent.Length} bytes"); // Should be 337,787 bytes + string formattedSourceContent = PIIDetector.RemovePIIData(sourceFileContent); // Convert the string to a byte array (buffer) - byte[] stringBuffer = Encoding.UTF8.GetBytes(sourceFileContent); + byte[] stringBuffer = Encoding.UTF8.GetBytes(formattedSourceContent); // Compress to gzip and then convert to base64 to optimize size byte[] compressedBuffer = DataUtilities.Compress(stringBuffer); string base64CompressedBuffer = Convert.ToBase64String(compressedBuffer); - LogMessage.Info($"BASE64 string buffer has {base64CompressedBuffer.Length} bytes"); - // Write to file for testing purposes - File.WriteAllText(savePath, base64CompressedBuffer); return base64CompressedBuffer; } - static void DataExchangeToForge(string filePath, RestClient client, string token) + void SendToMLDataPipeline(string filePath, RestClient client, string token) { // STEP 1: CREATE A DATA EXCHANGE CONTAINER --------------------- string exchangeBody = ConstructCreateExchangeRequestBody(); @@ -184,8 +190,6 @@ static void DataExchangeToForge(string filePath, RestClient client, string token // from the response - these will be consumed by the following API calls. ExchangeContainerId = exchangeRequestResponseBody["id"].Value; - Analytics.TrackEvent(Actions.Export, Categories.DynamoMLDataPipelineOperations, "ExchangeContainerID", Convert.ToInt32(ExchangeContainerId)); - var schemaNamespaceId = exchangeRequestResponseBody["components"]["data"]["insert"]["autodesk.data:exchange.source.default-1.0.0"]["source"]["String"]["id"].Value; // STEP 2: START A FULLFILLMENT --------------------- @@ -219,20 +223,19 @@ static void DataExchangeToForge(string filePath, RestClient client, string token uploadBinaryRequest.AddHeader("Content-Type", ContentType.Binary); - uploadBinaryRequest.AddOrUpdateParameter(ContentType.Binary.ToString(), base64CompressedBuffer); + uploadBinaryRequest.AddOrUpdateParameter("text/txt", base64CompressedBuffer, ParameterType.RequestBody); + var uploadBinaryResponse = fileUploadClient.ExecutePut(uploadBinaryRequest); if (uploadBinaryResponse.StatusCode != System.Net.HttpStatusCode.OK) { LogMessage.Info("Binary upload failed!"); } - LogMessage.Info("Binary upload started!"); - - Analytics.TrackEvent(Actions.Export, Categories.DynamoMLDataPipelineOperations, "BinarySize", base64CompressedBuffer.Length); // STEP 4b: FINISH BINARY UPLOAD ------------------- // Finish uploading binary assets: Let the system know that the binary assets have been uploaded and are ready for processing. // This call can be made for a single binary or a batch of 25 binaries. var finishBinaryUploadUrl = $"/v1/collections/{CollectionId}/exchanges/{ExchangeContainerId}/fulfillments/{fulfillmentId}/binaries:finish"; + // Construct request body var uploadedBinaryAsset = new UploadedBinaryAsset(BinaryAssetGuid); var uploadedBinaries = new BinaryAssets(); @@ -259,7 +262,6 @@ static void DataExchangeToForge(string filePath, RestClient client, string token // Construct request body for an insert operation, i.e. fulfilling an initial excahnge string syncAssetRequestBody = ConstructCreateAssetRequestBody(schemaNamespaceId, BinaryAssetGuid, "insert"); - //HttpClient client = new HttpClient(); RestRequest syncAssetRequest = new RestRequest(syncAssetUrl); syncAssetRequest.AddStringBody(syncAssetRequestBody, ContentType.Json); AddHeadersToPostRequest(syncAssetRequest, token); @@ -267,72 +269,37 @@ static void DataExchangeToForge(string filePath, RestClient client, string token // STEP 6: END FULFILLMENT --------------------- EndFullFillment(client, CollectionId, ExchangeContainerId, fulfillmentId, token); - } - static void TestPipeline(string filePath, string token, RestClient client) - { - // TEST THE PIPELINE ----------------------------------------------------------- - // RETRIEVE BACK THE UPLOADED BINARY ASSET AND CONVERT TO DYNAMO GRAPH - // STEP 1: Download the binary asset from a specified exchange - this call will return an AWS download URL - var downloadBinaryUrl = $"/v1/collections/{CollectionId}/exchanges/{ExchangeContainerId}/assets/binaries:download"; - RestRequest downloadBinaryRequest = new RestRequest(downloadBinaryUrl); - - var downloadBinaryAsset = new UploadedBinaryAsset(BinaryAssetGuid); - var downloadBinaries = new BinaryAssets(); - downloadBinaries.AddBinary(downloadBinaryAsset); - string binaryDownloadBody = JsonConvert.SerializeObject(downloadBinaries); - - downloadBinaryRequest.AddStringBody(binaryDownloadBody, ContentType.Json); - AddHeadersToPostRequest(downloadBinaryRequest, token); - var downloadBinaryResponse = client.ExecutePost(downloadBinaryRequest); - dynamic downloadBinaryResponseBody = JObject.Parse(downloadBinaryResponse.Content); - var downloadURL = downloadBinaryResponseBody["binaries"][0]["downloadUrl"].Value; - - // STEP 2: Retrieve the binary data using the returned AWS download URL - var fileDownloadClient = new RestClient(); - RestRequest downloadDataRequest = new RestRequest(downloadURL); - var downloadDataResponse = fileDownloadClient.ExecuteGet(downloadDataRequest); // Data returned in base 64 compressed - var base64Data = downloadDataResponse.Content; - - // STEP 3: Convert base64 compressed data back to .dyn file - byte[] compressedData = Convert.FromBase64String(base64Data); - byte[] uncompressedData = DataUtilities.Decompress(compressedData); - string uncompressedString = Encoding.UTF8.GetString(uncompressedData); - string parentDir = Directory.GetParent(filePath).FullName; - var dynamoParsedPath = Path.Combine(parentDir, "MaximizeWindowViews-deserialized.dyn"); - File.WriteAllText(dynamoParsedPath, uncompressedString); + // Send Analytics information. + var analyticsInfo = new Dictionary + { + { "CollectionId", CollectionId }, + { "ExchangeContainerId", ExchangeContainerId }, + { "CompressedDataSize", base64CompressedBuffer.Length } + }; + + Analytics.TrackTaskCommandEvent(Actions.Export.ToString(), Categories.DynamoMLDataPipelineOperations.ToString(), null, analyticsInfo); } - public void DataExchange(string filePath) + public void SendWorkspaceLog(string filePath) { - // The Forge team has recommended to use the Stage environment for testing. - // Depending on whether we are using Stage or Prod, a Forge token needs to be retrieved - // using client and secret id from a Forge app created in the respective environment. - // Assuming there is a way to retrieve the 3-leg Forge token in Dynamo, I will use a hardcoded one here. + // Depending on whether we are using Stage or Prod, a token needs to be retrieved + // using client and secret id from the app created in the respective environment. var token = GetAuthorizationToken(); - // Stage collectionId created for Dynamo + // CollectionId created for Dynamo CollectionId = ProductionCollectionID; - Analytics.TrackEvent(Actions.Export, Categories.DynamoMLDataPipelineOperations, "CollectionID", Convert.ToInt32(CollectionId)); - //ExchangeContainerId = ""; - - var forgeClient = new RestClient(ProductionClientUrl); - - DataExchangeToForge(filePath, forgeClient, token); + var client = new RestClient(ProductionClientUrl); - // This is needed here to allow for the testing workflow to run after the upload has been completed. - // It looks like even though the fullfillment ran by teh SendNewLog function is being reported - // as COMPLETED the upload is not done. - - // Test pipeline is being commented out. - //Thread.Sleep(5000); - //TestPipeline(filePath, token, forgeClient); - } - - private static string GetAuthorizationToken() - { - return AuthTokenProvider.GetAccessToken(); + try + { + SendToMLDataPipeline(filePath, client, token); + } + catch (Exception ex) + { + LogMessage.Error("Failed to share the workspace with ML data pipeline: " + ex.StackTrace); + } } } } diff --git a/src/DynamoMLDataPipeline/DynamoMLDataPipeline.csproj b/src/DynamoMLDataPipeline/DynamoMLDataPipeline.csproj index 1f2430e73b8..ee94e15110a 100644 --- a/src/DynamoMLDataPipeline/DynamoMLDataPipeline.csproj +++ b/src/DynamoMLDataPipeline/DynamoMLDataPipeline.csproj @@ -3,41 +3,25 @@ - Debug - AnyCPU {5DF79F45-5F2C-41C1-BACC-890AE514CDA8} Library DynamoMLDataPipeline DynamoMLDataPipeline - 512 - true - true - false + {7858fa8c-475f-4b8e-b468-1f8200778cf8} DynamoCore - - - - - - - - - - - - - - - + + {B5F435CB-0D8A-40B1-A4F7-5ECB3CE792A9} + DynamoUtilities + diff --git a/src/DynamoMLDataPipeline/DynamoMLDataPipelineExtension.cs b/src/DynamoMLDataPipeline/DynamoMLDataPipelineExtension.cs index 0a6b258d4c4..adecd22ee8a 100644 --- a/src/DynamoMLDataPipeline/DynamoMLDataPipelineExtension.cs +++ b/src/DynamoMLDataPipeline/DynamoMLDataPipelineExtension.cs @@ -1,16 +1,12 @@ -using System; -using System.Collections.Generic; using Dynamo.Extensions; -using Dynamo.Logging; using Greg; namespace DynamoMLDataPipeline { - internal class DynamoMLDataPipelineExtension : IExtension, IExtensionSource + internal class DynamoMLDataPipelineExtension : IExtension { private ReadyParams ReadyParams; - private DynamoLogger logger; /// /// Dynamo Package Manager Instance. @@ -24,15 +20,20 @@ public string UniqueId public string Name { get { return "DynamoMLDataPipelineExtension"; } } - public IEnumerable RequestedExtensions => throw new NotImplementedException(); - - public event Func RequestLoadExtension; - public event Action RequestAddExtension; - public void Startup(StartupParams sp) { DynamoMLDataPipeline = new DynamoMLDataPipeline(); - DynamoMLDataPipeline.AuthTokenProvider = (IOAuth2AccessTokenProvider)sp.AuthProvider; + DynamoMLDataPipeline.DynamoVersion = sp.DynamoVersion; + + if (sp.AuthProvider is IOAuth2AccessTokenProvider tokenProvider) + { + DynamoMLDataPipeline.AuthTokenProvider = tokenProvider; + } + + if (sp.AuthProvider is IOAuth2UserIDProvider userIdProvider) + { + DynamoMLDataPipeline.AuthUserInfoProvider = userIdProvider; + } } public void Ready(ReadyParams sp) @@ -42,12 +43,12 @@ public void Ready(ReadyParams sp) public void Shutdown() { - throw new NotImplementedException(); + // do nothing. } public void Dispose() { - throw new NotImplementedException(); + // do nothing. } } } diff --git a/src/DynamoMLDataPipeline/ExchangeComponent.cs b/src/DynamoMLDataPipeline/ExchangeComponent.cs index 6e193fc0097..d9b031c83d7 100644 --- a/src/DynamoMLDataPipeline/ExchangeComponent.cs +++ b/src/DynamoMLDataPipeline/ExchangeComponent.cs @@ -4,11 +4,11 @@ namespace DynamoMLDataPipeline { - class ExchangeComponent + internal class ExchangeComponent { private readonly string type = "autodesk.data:exchange.space-1.0.0"; - public ExchangeComponent(List attributes = null) + public ExchangeComponent(List attributes = null) { Attributes = attributes; Components = new Dictionary @@ -23,10 +23,10 @@ public ExchangeComponent(List attributes = null) public Dictionary Components { get; set; } [JsonProperty("attributes")] - public List Attributes { get; set; } + public List Attributes { get; set; } } - class Contract : Dictionary>> + internal class Contract : Dictionary>> { private readonly string type = "autodesk.data:exchange.contract.dynamo-1.0.0"; diff --git a/src/DynamoMLDataPipeline/InstanceAsset.cs b/src/DynamoMLDataPipeline/InstanceAsset.cs index b4b4b571251..4e5b83a54b5 100644 --- a/src/DynamoMLDataPipeline/InstanceAsset.cs +++ b/src/DynamoMLDataPipeline/InstanceAsset.cs @@ -6,7 +6,7 @@ namespace DynamoMLDataPipeline { - class InstanceAsset + internal class InstanceAsset { public InstanceAsset(ParameterComponent parameterComponent, BaseComponent baseComponent, BinaryReferenceComponent binaryRefComponent, string operation) { diff --git a/src/DynamoMLDataPipeline/ParameterComponent.cs b/src/DynamoMLDataPipeline/ParameterComponent.cs index d2bcc1ebd53..319ab2c5af2 100644 --- a/src/DynamoMLDataPipeline/ParameterComponent.cs +++ b/src/DynamoMLDataPipeline/ParameterComponent.cs @@ -6,7 +6,7 @@ namespace DynamoMLDataPipeline { // Parameters like host and user info for the request schema - class ParameterComponent: Dictionary> + internal class ParameterComponent: Dictionary> { private string objectId = "autodesk.design:components.parameter-1.0.0"; public string ObjectId { get { return objectId; } } @@ -24,7 +24,7 @@ public void AddParameterFromSchema(dynamic value, Schema schema) } } - class Parameter + internal class Parameter { [JsonProperty("parameterValue")] public Dictionary ValueEntry { get; set; } @@ -37,7 +37,7 @@ public Parameter(dynamic value, Schema schema) } } - class Value + internal class Value { [JsonProperty("value")] public dynamic ValueEntry { get; set; } diff --git a/src/DynamoMLDataPipeline/Attribute.cs b/src/DynamoMLDataPipeline/RequestAttribute.cs similarity index 78% rename from src/DynamoMLDataPipeline/Attribute.cs rename to src/DynamoMLDataPipeline/RequestAttribute.cs index 9c02c8654f8..edaf4220a5c 100644 --- a/src/DynamoMLDataPipeline/Attribute.cs +++ b/src/DynamoMLDataPipeline/RequestAttribute.cs @@ -3,7 +3,7 @@ namespace DynamoMLDataPipeline { // Attributes for the data request object. - class Attribute + internal class RequestAttribute { [JsonProperty("category")] public string Category { get; set; } @@ -14,7 +14,7 @@ class Attribute [JsonProperty("type")] public string Type { get; set; } - public Attribute(string name, string value, string category = "application", string type = "String") + public RequestAttribute(string name, string value, string category = "application", string type = "String") { Category = category; Name = name; diff --git a/src/DynamoMLDataPipeline/Schema.cs b/src/DynamoMLDataPipeline/Schema.cs index aebc560bcdd..7394b485f04 100644 --- a/src/DynamoMLDataPipeline/Schema.cs +++ b/src/DynamoMLDataPipeline/Schema.cs @@ -4,7 +4,7 @@ namespace DynamoMLDataPipeline { // Schema for the parameters used in the request. - class StringParameterSchema : Schema + internal class StringParameterSchema : Schema { public StringParameterSchema(string value, string schemaNamespaceId, string schemaId) { @@ -25,7 +25,7 @@ public StringParameterSchema(string value, string schemaNamespaceId, string sche } - class Schema + internal class Schema { [JsonProperty("constants")] public List Constants { get; set; } @@ -41,7 +41,7 @@ class Schema } - class Constant + internal class Constant { public Constant(string value) { diff --git a/src/DynamoMLDataPipeline/UploadAssetsRequestBody.cs b/src/DynamoMLDataPipeline/UploadAssetsRequestBody.cs index ba5f44aca53..cad404ae85d 100644 --- a/src/DynamoMLDataPipeline/UploadAssetsRequestBody.cs +++ b/src/DynamoMLDataPipeline/UploadAssetsRequestBody.cs @@ -5,7 +5,7 @@ namespace DynamoMLDataPipeline { // Schema and the assets for the data request body. - class UploadAssetsRequestBody + internal class UploadAssetsRequestBody { public UploadAssetsRequestBody(List schemas, List assets, string operation) { diff --git a/src/DynamoPackages/DynamoPackages.csproj b/src/DynamoPackages/DynamoPackages.csproj index e5bbbd4e177..1fc32772e9e 100644 --- a/src/DynamoPackages/DynamoPackages.csproj +++ b/src/DynamoPackages/DynamoPackages.csproj @@ -30,7 +30,7 @@ - + diff --git a/src/DynamoUtilities/Hash.cs b/src/DynamoUtilities/Hash.cs index 08bf0257f5c..bef347c9cae 100644 --- a/src/DynamoUtilities/Hash.cs +++ b/src/DynamoUtilities/Hash.cs @@ -99,6 +99,9 @@ internal static string ToSha256String(string s) using var mySHA256 = SHA256.Create(); byte[] bytes = mySHA256.ComputeHash(Encoding.UTF8.GetBytes(s)); + + mySHA256.Dispose(); + var sb = new StringBuilder(); for (int i = 0; i < bytes.Length; i++) diff --git a/src/DynamoUtilities/PIIDetector.cs b/src/DynamoUtilities/PIIDetector.cs index 43ef96c0398..2ec7036acd3 100644 --- a/src/DynamoUtilities/PIIDetector.cs +++ b/src/DynamoUtilities/PIIDetector.cs @@ -1,9 +1,8 @@ -using Newtonsoft.Json.Linq; using System; -using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; +using Newtonsoft.Json.Linq; namespace Dynamo.Utilities { @@ -119,13 +118,16 @@ public static JToken GetNoteValue(JObject jsonWorkspace, string nodeId) internal static string RemovePIIData(string data) { string result; + + // ToDo: some of the regex patterns are interfering with other data in the json, + // so commenting them for now and will address those patterns later. result = Regex.Replace(data, emailPattern, ""); - result = Regex.Replace(result, websitePattern, ""); result = Regex.Replace(result, directoryPattern, ""); - result = Regex.Replace(result, creditCardPattern, ""); result = Regex.Replace(result, ssnPattern, ""); - result = Regex.Replace(result, ipPattern, ""); result = Regex.Replace(result, datePattern, ""); + //result = Regex.Replace(result, websitePattern, ""); + //result = Regex.Replace(result, creditCardPattern, ""); + //result = Regex.Replace(result, ipPattern, ""); return result; } diff --git a/src/DynamoUtilities/Properties/AssemblyInfo.cs b/src/DynamoUtilities/Properties/AssemblyInfo.cs index 53a438070a1..181c011d33e 100644 --- a/src/DynamoUtilities/Properties/AssemblyInfo.cs +++ b/src/DynamoUtilities/Properties/AssemblyInfo.cs @@ -22,6 +22,7 @@ [assembly: InternalsVisibleTo("ProtoScript")] [assembly: InternalsVisibleTo("ProtoCore")] [assembly: InternalsVisibleTo("DynamoApplications")] +[assembly: InternalsVisibleTo("DynamoMLDataPipeline")] [assembly: InternalsVisibleTo("DynamoCLI")] [assembly: InternalsVisibleTo("NodeDocumentationMarkdownGenerator")] [assembly: InternalsVisibleTo("DynamoUtilitiesTests")] diff --git a/src/PackageDetailsViewExtension/PackageDetailsViewExtension.csproj b/src/PackageDetailsViewExtension/PackageDetailsViewExtension.csproj index a31568f7fd5..e5bc78299dd 100644 --- a/src/PackageDetailsViewExtension/PackageDetailsViewExtension.csproj +++ b/src/PackageDetailsViewExtension/PackageDetailsViewExtension.csproj @@ -14,7 +14,7 @@ false - + diff --git a/src/Tools/NodeDocumentationMarkdownGenerator/NodeDocumentationMarkdownGenerator.csproj b/src/Tools/NodeDocumentationMarkdownGenerator/NodeDocumentationMarkdownGenerator.csproj index 824eb40b899..e1095953c5f 100644 --- a/src/Tools/NodeDocumentationMarkdownGenerator/NodeDocumentationMarkdownGenerator.csproj +++ b/src/Tools/NodeDocumentationMarkdownGenerator/NodeDocumentationMarkdownGenerator.csproj @@ -12,7 +12,7 @@ - + diff --git a/test/DynamoCoreTests/DynamoCoreTests.csproj b/test/DynamoCoreTests/DynamoCoreTests.csproj index 150bca5f8ef..8de071c28e9 100644 --- a/test/DynamoCoreTests/DynamoCoreTests.csproj +++ b/test/DynamoCoreTests/DynamoCoreTests.csproj @@ -34,7 +34,7 @@ - + all compile; build; native; contentfiles; analyzers; buildtransitive diff --git a/test/DynamoCoreWpfTests/DynamoCoreWpfTests.csproj b/test/DynamoCoreWpfTests/DynamoCoreWpfTests.csproj index 338b264496e..2b345480239 100644 --- a/test/DynamoCoreWpfTests/DynamoCoreWpfTests.csproj +++ b/test/DynamoCoreWpfTests/DynamoCoreWpfTests.csproj @@ -37,7 +37,7 @@ - + all compile; build; native; contentfiles; analyzers; buildtransitive diff --git a/test/Libraries/PackageManagerTests/PackageManagerTests.csproj b/test/Libraries/PackageManagerTests/PackageManagerTests.csproj index 0834509d899..a556d4e4a89 100644 --- a/test/Libraries/PackageManagerTests/PackageManagerTests.csproj +++ b/test/Libraries/PackageManagerTests/PackageManagerTests.csproj @@ -20,7 +20,7 @@ - + all compile; build; native; contentfiles; analyzers; buildtransitive diff --git a/tools/DynamoPackagesAnalyzer/DynamoPackagesAnalyzer.csproj b/tools/DynamoPackagesAnalyzer/DynamoPackagesAnalyzer.csproj index b5dc76d7f53..386e1603d1b 100644 --- a/tools/DynamoPackagesAnalyzer/DynamoPackagesAnalyzer.csproj +++ b/tools/DynamoPackagesAnalyzer/DynamoPackagesAnalyzer.csproj @@ -10,7 +10,7 @@ - +