Skip to content

Commit

Permalink
Migrating formula node deserialization to code block nodes (#14196)
Browse files Browse the repository at this point in the history
* attempt at migrating formula node deserialization to code block nodes

* use DS parser to migrate formula node code, remove ncalc.

* delete formula nodemodel and view customization files

* review comments

* add migration for more builtin methods

* fix net6 build by removing unused usings

* review comments, add ncalc to remove_me

* review comment

* cleanup, add migration test

* add xml migration for formula node

* cleanup

* cleanup, merge master

* update tests, warning message

* remove unused usings
  • Loading branch information
aparajit-pratap authored Aug 7, 2023
1 parent 863d891 commit 130d3c3
Show file tree
Hide file tree
Showing 20 changed files with 602 additions and 410 deletions.
Binary file added extern/legacy_remove_me/nodes/NCalc.dll
Binary file not shown.
94 changes: 94 additions & 0 deletions src/DynamoCore/Engine/MigrateFormulaToDS.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using ProtoCore;
using ProtoCore.AST.AssociativeAST;
using ProtoCore.SyntaxAnalysis;
using ProtoCore.Utils;

namespace Dynamo.Engine
{
internal class MigrateFormulaToDS : AstReplacer
{
private static readonly TextInfo textInfo = CultureInfo.InvariantCulture.TextInfo;

/// <summary>
/// Regex to match "if(<condition>, <true>, <false>)" pattern in Formula node and extract the parts of the
/// "if" statement to construct the corresponding conditional AST in DS. This is because the DS parser does not
/// recognize this "if" syntax.
/// </summary>
private const string ifCond = @"(if\s*)\(\s*(.*?)\s*\)$";
private static readonly Regex ifRgx = new Regex(ifCond, RegexOptions.IgnoreCase);


internal string ConvertFormulaToDS(string formula)
{
CodeBlockNode cbn = null;
var match = ifRgx.Match(formula);

List<AssociativeNode> dsAst = new List<AssociativeNode>();
if (match.Success)
{
var cond = match.Groups[2].Value;
var expArray = $"[{cond}];";

cbn = ParserUtils.Parse(expArray);
var asts = MigrateFormulaToCodeBlockNode(cbn.Body).ToList();
var exprList = (asts[0] as BinaryExpressionNode).RightNode as ExprListNode;
var condAst = exprList.Exprs[0];
var trueAst = exprList.Exprs[1];
var falseAst = exprList.Exprs[2];

dsAst.Add(AstFactory.BuildConditionalNode(condAst, trueAst, falseAst));
}
else
{
cbn = ParserUtils.Parse(formula + ";");
dsAst.AddRange(MigrateFormulaToCodeBlockNode(cbn.Body));
}
var codegen = new CodeGenDS(dsAst);
return codegen.GenerateCode();
}

private IEnumerable<AssociativeNode> MigrateFormulaToCodeBlockNode(IEnumerable<AssociativeNode> nodes)
{
return nodes.Select(node => node.Accept(new MigrateFormulaToDS()));
}

public override AssociativeNode VisitFunctionCallNode(FunctionCallNode node)
{
node = base.VisitFunctionCallNode(node) as FunctionCallNode;
var funcName = textInfo.ToTitleCase(node.Function.Name);

if (funcName == "Sin" || funcName == "Cos" || funcName == "Tan")
{
funcName = $"DSCore.Math.{funcName}";
node.Function.Name = funcName;
(node.Function as IdentifierNode).Value = funcName;

var arg = node.FormalArguments.FirstOrDefault();

var newArg = AstFactory.BuildFunctionCall("DSCore.Math", "RadiansToDegrees", new List<AssociativeNode> { arg });
node.FormalArguments = new List<AssociativeNode> { newArg };
}
else if (funcName == "Asin" || funcName == "Acos" || funcName == "Atan")
{
funcName = $"DSCore.Math.{funcName}";
node.Function.Name = funcName;
(node.Function as IdentifierNode).Value = funcName;

node = AstFactory.BuildFunctionCall(
"DSCore.Math", "DegreesToRadians", new List<AssociativeNode> { node }) as FunctionCallNode;
}
else if(funcName == "Exp" || funcName == "Log10" || funcName == "Pow" || funcName == "Sqrt" || funcName == "Abs"
|| funcName == "Floor" || funcName == "Ceiling" || funcName == "Round")
{
funcName = $"DSCore.Math.{funcName}";
node.Function.Name = funcName;
(node.Function as IdentifierNode).Value = funcName;
}
return node;
}
}
}
15 changes: 15 additions & 0 deletions src/DynamoCore/Graph/Nodes/CodeBlockNode.cs
Original file line number Diff line number Diff line change
Expand Up @@ -658,6 +658,21 @@ internal void ProcessCodeDirect(ProcessCodeDelegate handler, bool isCodeBlockNod
OnNodeModified();
}

internal void FormulaMigrationWarning(string p)
{
State = ElementState.MigratedFormula;
if (!Infos.Any(x => x.Message.Equals(p)))
{
var texts = p.Split(new[] { Environment.NewLine }, StringSplitOptions.None);
var infoList = new List<Info>();
foreach (var text in texts)
{
infoList.Add(new Info(text, State));
}
Infos.AddRange(infoList);
}
}

/// <summary>
/// Undefine a function definition in a code block node if it has any.
/// </summary>
Expand Down
2 changes: 1 addition & 1 deletion src/DynamoCore/Graph/Nodes/NodeLoaders/NodeFactory.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Xml;
using Dynamo.Graph.Nodes.CustomNodes;
Expand Down
5 changes: 3 additions & 2 deletions src/DynamoCore/Graph/Nodes/NodeModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1841,7 +1841,7 @@ protected virtual void SetNodeStateBasedOnConnectionAndDefaults()

if (State == ElementState.PersistentWarning) return;

if (Infos.Any(x => x.State == ElementState.PersistentWarning))
if (Infos.Any(x => x.State == ElementState.PersistentWarning || x.State == ElementState.MigratedFormula))
{
State = ElementState.PersistentWarning;
return;
Expand Down Expand Up @@ -2921,7 +2921,8 @@ public enum ElementState
PersistentWarning,
Error,
AstBuildBroken,
Info
Info,
MigratedFormula
};

/// <summary>
Expand Down
88 changes: 56 additions & 32 deletions src/DynamoCore/Graph/Workspaces/SerializationConverters.cs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,39 @@ public class NodeReadConverter : JsonConverter
// Map of all loaded assemblies including LoadFrom context assemblies
private Dictionary<string, List<Assembly>> loadedAssemblies;

private CodeBlockNodeModel DeserializeAsCBN(string code, JObject obj, Guid guid)
{
var codeBlockNode = new CodeBlockNodeModel(code, guid, 0.0, 0.0, libraryServices, ElementResolver);

// If the code block node is in an error state read the extra port data
// and initialize the input and output ports
if (codeBlockNode.IsInErrorState)
{
List<string> inPortNames = new List<string>();
var inputs = obj["Inputs"];
foreach (var input in inputs)
{
inPortNames.Add(input["Name"].ToString());
}

// NOTE: This could be done in a simpler way, but is being implemented
// in this manner to allow for possible future port line number
// information being available in the file
List<int> outPortLineIndexes = new List<int>();
var outputs = obj["Outputs"];
int outputLineIndex = 0;
foreach (var output in outputs)
{
outPortLineIndexes.Add(outputLineIndex++);
}

codeBlockNode.SetErrorStatePortData(inPortNames, outPortLineIndexes);
}
return codeBlockNode;
}



[Obsolete("This constructor will be removed in Dynamo 3.0, please use new NodeReadConverter constructor with additional parameters to support node migration.")]
public NodeReadConverter(CustomNodeManager manager, LibraryServices libraryServices, bool isTestMode = false)
{
Expand Down Expand Up @@ -111,7 +144,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
{
type = Type.GetType(obj["$type"].Value<string>());
typeName = obj["$type"].Value<string>().Split(',').FirstOrDefault();

if (typeName.Equals("Dynamo.Graph.Nodes.ZeroTouch.DSFunction"))
{
// If it is a zero touch node, then get the whole function name including the namespace.
Expand Down Expand Up @@ -165,7 +198,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist

// If the id is not a guid, makes a guid based on the id of the node
var guid = GuidUtility.tryParseOrCreateGuid(obj["Id"].Value<string>());

var replication = obj["Replication"].Value<string>();

var inPorts = obj["Inputs"].ToArray().Select(t => t.ToObject<PortModel>()).ToArray();
Expand All @@ -176,9 +209,9 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist

bool remapPorts = true;

// If type is still null at this point return a dummy node
if (type == null)
{
// If type is still null at this point return a dummy node
node = CreateDummyNode(obj, typeName, assemblyName, functionName, inPorts, outPorts);
}
// Attempt to create a valid node using the type
Expand All @@ -197,37 +230,10 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
if (isUnresolved)
function.UpdatePortsForUnresolved(inPorts, outPorts);
}

else if (type == typeof(CodeBlockNodeModel))
{
var code = obj["Code"].Value<string>();
CodeBlockNodeModel codeBlockNode = new CodeBlockNodeModel(code, guid, 0.0, 0.0, libraryServices, ElementResolver);
node = codeBlockNode;

// If the code block node is in an error state read the extra port data
// and initialize the input and output ports
if (node.IsInErrorState)
{
List<string> inPortNames = new List<string>();
var inputs = obj["Inputs"];
foreach (var input in inputs)
{
inPortNames.Add(input["Name"].ToString());
}

// NOTE: This could be done in a simpler way, but is being implemented
// in this manner to allow for possible future port line number
// information being available in the file
List<int> outPortLineIndexes = new List<int>();
var outputs = obj["Outputs"];
int outputLineIndex = 0;
foreach (var output in outputs)
{
outPortLineIndexes.Add(outputLineIndex++);
}

codeBlockNode.SetErrorStatePortData(inPortNames, outPortLineIndexes);
}
node = DeserializeAsCBN(code, obj, guid);
}
else if (typeof(DSFunctionBase).IsAssignableFrom(type))
{
Expand Down Expand Up @@ -263,7 +269,25 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
}
else if (type.ToString() == "CoreNodeModels.Formula")
{
node = (NodeModel)obj.ToObject(type);
var code = obj["Formula"].Value<string>();
var formulaConverter = new MigrateFormulaToDS();
string convertedCode = string.Empty;
bool conversionFailed = false;
try
{
convertedCode = formulaConverter.ConvertFormulaToDS(code);
}
catch (BuildHaltException)
{
node = DeserializeAsCBN(code + ";", obj, guid);
(node as CodeBlockNodeModel).FormulaMigrationWarning(Resources.FormulaDSConversionFailure);
conversionFailed = true;
}
if (!conversionFailed)
{
node = DeserializeAsCBN(convertedCode + ";", obj, guid);
(node as CodeBlockNodeModel).FormulaMigrationWarning(Resources.FormulaMigrated);
}
}
else
{
Expand Down
22 changes: 20 additions & 2 deletions src/DynamoCore/Properties/Resources.Designer.cs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions src/DynamoCore/Properties/Resources.en-US.resx
Original file line number Diff line number Diff line change
Expand Up @@ -902,4 +902,10 @@ This package likely contains an assembly that is blocked. You will need to load
<data name="DynamoLanguages_noxlate" xml:space="preserve">
<value>English,Čeština,Deutsch,Español,Français,Italiano,日本語,한국어,Polski,Português (Brasil),Русский,简体中文,繁體中文</value>
</data>
<data name="FormulaDSConversionFailure" xml:space="preserve">
<value>Formula failed to convert to DesignScript code. Please edit the formula manually to use it in a CodeBlock node.</value>
</data>
<data name="FormulaMigrated" xml:space="preserve">
<value>Formula node has been deprecated. It has been automatically migrated to a CodeBlock node. Note that results may vary after the migration depending on lacing options selected on the original Formula node. Appropriate replication guides might need to be applied to the CodeBlock node script.</value>
</data>
</root>
6 changes: 6 additions & 0 deletions src/DynamoCore/Properties/Resources.resx
Original file line number Diff line number Diff line change
Expand Up @@ -905,4 +905,10 @@ This package likely contains an assembly that is blocked. You will need to load
<data name="DynamoLanguages_noxlate" xml:space="preserve">
<value>English,Čeština,Deutsch,Español,Français,Italiano,日本語,한국어,Polski,Português (Brasil),Русский,简体中文,繁體中文</value>
</data>
<data name="FormulaDSConversionFailure" xml:space="preserve">
<value>Formula failed to convert to DesignScript code. Please edit the formula manually to use it in a CodeBlock node.</value>
</data>
<data name="FormulaMigrated" xml:space="preserve">
<value>Formula node has been deprecated. It has been automatically migrated to a CodeBlock node. Note that results may vary after the migration depending on lacing options selected on the original Formula node. Appropriate replication guides might need to be applied to the CodeBlock node script.</value>
</data>
</root>
8 changes: 0 additions & 8 deletions src/Libraries/CoreNodeModels/CoreNodeModels.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,6 @@
<None Remove="CoreNodeModelsImages.resources" />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'net48' ">
<PackageReference Include="ncalc" Version="1.3.8">
<GeneratePathProperty>true</GeneratePathProperty>
<!--Exclude copying the runtime dlls because we will handle it in the BinaryCompatibilityOps target -->
<ExcludeAssets>runtime</ExcludeAssets>
</PackageReference>
<Reference Include="System.Web" />
<Reference Include="System.Windows" />
<Reference Include="System.Xaml" />
Expand Down Expand Up @@ -96,9 +91,6 @@
<LastGenOutput>Resources.en-US.Designer.cs</LastGenOutput>
</EmbeddedResource>
</ItemGroup>
<Target Name="BinaryCompatibilityOps" BeforeTargets="Build" Condition=" '$(TargetFramework)' == 'net48' ">
<Copy SourceFiles="$(PkgNCalc)\lib\NCalc.dll" DestinationFolder="$(OutputPath)" />
</Target>
<Target Name="GenerateFiles" AfterTargets="ResolveSateliteResDeps" Condition=" '$(OS)' != 'Unix' ">
<!-- Generate customization dll -->
<GenerateResource SdkToolsPath="$(TargetFrameworkSDKToolsDirectory)" UseSourcePath="true" Sources="$(ProjectDir)CoreNodeModelsImages.resx" OutputResources="$(ProjectDir)CoreNodeModelsImages.resources" References="$(SystemDrawingDllPath)" />
Expand Down
Loading

0 comments on commit 130d3c3

Please sign in to comment.