feat: scoped pipeline activities and added support for more functions

microsoft · Sep 20, 2023 · 2d53cfd · 2d53cfd
1 parent 02bf906
commit 2d53cfd
Show file tree

Hide file tree

Showing 14 changed files with 66 additions and 33 deletions.
diff --git a/src/AzureDataFactory.TestingFramework.sln b/src/AzureDataFactory.TestingFramework.sln
@@ -1,10 +1,18 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureDataFactory.TestingFramework", "AzureDataFactory.TestingFramework\AzureDataFactory.TestingFramework.csproj", "{F5C4C98C-B717-4ED8-95EA-5DD51CB87C92}"
+# Visual Studio Version 17
+VisualStudioVersion = 17.7.34031.279
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureDataFactory.TestingFramework", "AzureDataFactory.TestingFramework\AzureDataFactory.TestingFramework.csproj", "{F5C4C98C-B717-4ED8-95EA-5DD51CB87C92}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureDataFactory.TestingFramework.Tests", "AzureDataFactory.TestingFramework.Tests\AzureDataFactory.TestingFramework.Tests.csproj", "{76089DF4-314B-4556-9ED7-D2B2F3BBA87A}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureDataFactory.TestingFramework.Tests", "AzureDataFactory.TestingFramework.Tests\AzureDataFactory.TestingFramework.Tests.csproj", "{76089DF4-314B-4556-9ED7-D2B2F3BBA87A}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureDataFactory.TestingFramework.Example", "AzureDataFactory.TestingFramework.Example\AzureDataFactory.TestingFramework.Example.csproj", "{F044EB13-DF36-43FB-90E8-9ADDE0E03821}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureDataFactory.TestingFramework.Example", "AzureDataFactory.TestingFramework.Example\AzureDataFactory.TestingFramework.Example.csproj", "{F044EB13-DF36-43FB-90E8-9ADDE0E03821}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{2EC1EFEA-98DC-47FC-9EAD-69541C65ACA1}"
+	ProjectSection(SolutionItems) = preProject
+		..\README.md = ..\README.md
+	EndProjectSection
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -25,6 +33,7 @@ Global
 		{F044EB13-DF36-43FB-90E8-9ADDE0E03821}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F044EB13-DF36-43FB-90E8-9ADDE0E03821}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
-	GlobalSection(NestedProjects) = preSolution
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
 	EndGlobalSection
 EndGlobal
diff --git a/src/AzureDataFactory.TestingFramework/AzureDataFactory.TestingFramework.csproj b/src/AzureDataFactory.TestingFramework/AzureDataFactory.TestingFramework.csproj
@@ -16,7 +16,7 @@
 
     <PropertyGroup>
         <PackageId>AzureDataFactory.TestingFramework</PackageId>
-        <PackageVersion>0.1.3-alpha</PackageVersion>
+        <PackageVersion>0.1.4-alpha</PackageVersion>
         <Authors>arjendev</Authors>
         <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
     </PropertyGroup>

diff --git a/src/AzureDataFactory.TestingFramework/Expressions/ActivityExpression.cs b/src/AzureDataFactory.TestingFramework/Expressions/ActivityExpression.cs
@@ -19,7 +19,7 @@ public ActivityExpression(string expression) : base(expression)
     public TType Evaluate<TType>(PipelineRunState state)
     {
         var (activityName, fields) = GetActivityNameAndFields();
-        var activity = state.PipelineActivityResults.SingleOrDefault(x => string.Equals(x.Name, activityName, StringComparison.CurrentCultureIgnoreCase)) ??
+        var activity = state.PipelineActivityResults.LastOrDefault(x => string.Equals(x.Name, activityName, StringComparison.CurrentCultureIgnoreCase)) ??
                        throw new ActivityNotFoundException(activityName);
         if (activity.Status == null)
             throw new ActivityNotEvaluatedException(activity.Name);

diff --git a/src/AzureDataFactory.TestingFramework/Functions/FunctionArgument.cs b/src/AzureDataFactory.TestingFramework/Functions/FunctionArgument.cs
@@ -48,6 +48,7 @@ public TType Evaluate<TType>(RunState state)
         {
             { } type when type == typeof(bool) && bool.TryParse(evalExpression, out var boolValue) => (TType)(object) boolValue,
             { } type when type == typeof(int) && int.TryParse(evalExpression, out var intValue) => (TType)(object) intValue,
+            { } type when type == typeof(long) && long.TryParse(evalExpression, out var longValue) => (TType)(object) longValue,
             { } type when type == typeof(string) => (TType)(object) evalExpression.TrimOneChar('\''),
             { } type => throw new ArgumentException($"The result {evalExpression} with DataType: {type} could not be parsed accordingly.")
         };

diff --git a/src/AzureDataFactory.TestingFramework/Functions/FunctionCall.cs b/src/AzureDataFactory.TestingFramework/Functions/FunctionCall.cs
@@ -41,6 +41,7 @@ public TType Evaluate<TType>(RunState state)
                 { } when parameterType == typeof(float) => argument.Evaluate<float>(state),
                 { } when parameterType == typeof(double) => argument.Evaluate<double>(state),
                 { } when parameterType == typeof(object) => argument.Evaluate<object>(state),
+                { IsArray: true } => argument.Evaluate<object[]>(state),
                 _ => throw new Exception($"Unsupported parameter type: {parameterType}")
             };
         }).ToList();

diff --git a/src/AzureDataFactory.TestingFramework/Functions/FunctionsRepository.cs b/src/AzureDataFactory.TestingFramework/Functions/FunctionsRepository.cs
@@ -15,8 +15,16 @@ public static class FunctionsRepository
         { "replace",  (string input, string pattern, string replacement) => input.Replace(pattern, replacement) },
         { "string", (string input) => input },
         { "union" , (string arg0, string arg1) => JsonSerializer.Serialize(JsonSerializer.Deserialize<JsonArray>(arg0).Union(JsonSerializer.Deserialize<JsonArray>((arg1)))) },
-        { "coalesce", (IEnumerable<string> args) => args.FirstOrDefault(arg => !string.IsNullOrEmpty(arg)) }
-
+        { "coalesce", (IEnumerable<string> args) => args.FirstOrDefault(arg => !string.IsNullOrEmpty(arg)) },
+        { "or", (bool a, bool b) => a || b },
+        { "utcnow", () => DateTime.UtcNow.ToString("o") },
+        { "utcNow", () => DateTime.UtcNow.ToString("o") },
+        { "ticks", (string dateTime) => DateTime.Parse(dateTime).Ticks },
+        { "sub", (long a, long b) => a - b },
+        { "div", (long a, long b) => a / b },
+        { "greaterOrEquals", (long a, long b) => a >= b },
+        { "not", (bool value) => !value },
+        { "empty", (object[] array) => array.Length == 0 },
     };
 
     public static void Register(string functionName, Delegate function)

diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/Base/ActivitiesEvaluator.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/Base/ActivitiesEvaluator.cs
@@ -6,26 +6,31 @@ public static class ActivitiesEvaluator
 {
     public static IEnumerable<PipelineActivity> Evaluate(List<PipelineActivity> activities, PipelineRunState state)
     {
-        while (state.PipelineActivityResults.Count != activities.Count)
+        while (state.ScopedPipelineActivityResults.Count != activities.Count)
         {
             var anyActivityEvaluated = false;
             foreach (var activity in activities
-                         .Where(activity => !state.PipelineActivityResults.Contains(activity))
+                         .Where(activity => !state.ScopedPipelineActivityResults.Contains(activity))
                          .Where(activity => activity.AreDependencyConditionMet(state)))
             {
-                yield return (PipelineActivity) activity.Evaluate(state);
+                var evaluatedActivity = (PipelineActivity) activity.Evaluate(state);
+                if (evaluatedActivity is not IIterationActivity)
+                    yield return evaluatedActivity;
+
                 anyActivityEvaluated = true;
                 state.AddActivityResult(activity);
 
-                if (activity is ControlActivity controlActivity)
+                if (activity is IIterationActivity)
                 {
-                    if (controlActivity is UntilActivity untilActivity)
+                    if (activity is UntilActivity untilActivity)
                     {
-                        while (untilActivity.Expression.Evaluate<bool>(state))
+                        do
+                        {
                             foreach (var child in untilActivity.EvaluateChildActivities(state))
                                 yield return child;
+                        } while (!untilActivity.Expression.Evaluate<bool>(state));
                     }
-                    else
+                    else if (activity is ControlActivity controlActivity)
                     {
                         foreach (var child in controlActivity.EvaluateChildActivities(state))
                             yield return child;

diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/Base/PipelineActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/Base/PipelineActivity.cs
@@ -28,7 +28,7 @@ public bool AreDependencyConditionMet(PipelineRunState state)
     {
         foreach (var dependency in DependsOn)
         {
-            var dependencyActivity = state.PipelineActivityResults.SingleOrDefault(a => a.Name == dependency.Activity);
+            var dependencyActivity = state.ScopedPipelineActivityResults.SingleOrDefault(a => a.Name == dependency.Activity);
 
             // If dependency is not yet evaluated, conditions are not met
             if (dependencyActivity == null)

diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ControlActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ControlActivity.cs
@@ -19,5 +19,7 @@ public virtual IEnumerable<PipelineActivity> EvaluateChildActivities(PipelineRun
         {
             yield return activity;
         }
+
+        state.AddScopedActivityResultsFromScopedState(scopedState);
     }
 }
diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ForEachActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ForEachActivity.cs
@@ -4,7 +4,7 @@
 
 namespace AzureDataFactory.TestingFramework.Models;
 
-public partial class ForEachActivity
+public partial class ForEachActivity : IIterationActivity
 {
     protected override List<PipelineActivity> GetNextActivities()
     {

diff --git a/...AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/IIterationActivity.cs b/...AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/IIterationActivity.cs
@@ -0,0 +1,6 @@
+namespace AzureDataFactory.TestingFramework.Models;
+
+public interface IIterationActivity
+{
+
+}
diff --git a/...zureDataFactory.TestingFramework/Models/Activities/ControlActivity/IfConditionActivity.cs b/...zureDataFactory.TestingFramework/Models/Activities/ControlActivity/IfConditionActivity.cs
@@ -4,14 +4,14 @@
 
 namespace AzureDataFactory.TestingFramework.Models;
 
-public partial class IfConditionActivity
+public partial class IfConditionActivity : IIterationActivity
 {
     protected override List<PipelineActivity> GetNextActivities()
     {
-        return _evaluatedExpression.Value ? IfTrueActivities.ToList() : IfFalseActivities.ToList();
+        return EvaluatedExpression ? IfTrueActivities.ToList() : IfFalseActivities.ToList();
     }
 
-    public bool? _evaluatedExpression;
+    private bool? _evaluatedExpression;
     public bool EvaluatedExpression => _evaluatedExpression ?? throw new InvalidOperationException("Expression has not been evaluated yet.");
     public override DataFactoryEntity Evaluate(PipelineRunState state)
     {

diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/UntilActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/UntilActivity.cs
@@ -3,20 +3,10 @@
 
 namespace AzureDataFactory.TestingFramework.Models;
 
-public partial class UntilActivity
+public partial class UntilActivity : IIterationActivity
 {
     protected override List<PipelineActivity> GetNextActivities()
     {
         return Activities.ToList();
     }
-
-    private bool? _evaluatedExpression;
-    public override PipelineActivity Evaluate(PipelineRunState state)
-    {
-        base.Evaluate(state);
-
-        _evaluatedExpression = Expression.Evaluate<bool>(state);
-
-        return this;
-    }
 }
diff --git a/src/AzureDataFactory.TestingFramework/Models/Pipelines/PipelineRunState.cs b/src/AzureDataFactory.TestingFramework/Models/Pipelines/PipelineRunState.cs
@@ -9,6 +9,7 @@ public class PipelineRunState : RunState
     public List<IPipelineRunVariable> Variables { get; }
     public string? IterationItem { get; set; }
     public List<IPipelineActivityResult> PipelineActivityResults { get; }
+    public List<IPipelineActivityResult> ScopedPipelineActivityResults { get; }
 
     public PipelineRunState(List<IRunParameter> parameters, IDictionary<string, PipelineVariableSpecification> variables) : base(parameters)
     {
@@ -27,30 +28,40 @@ public PipelineRunState(List<IRunParameter> parameters, IDictionary<string, Pipe
             throw new Exception($"Unknown variable type: {variable.Value.VariableType}");
         }).ToList();;
         PipelineActivityResults = new List<IPipelineActivityResult>();
+        ScopedPipelineActivityResults = new List<IPipelineActivityResult>();
         IterationItem = null;
     }
 
-    public PipelineRunState(List<IRunParameter> parameters, List<IPipelineRunVariable> variables, string? iterationItem) : base(parameters)
+    public PipelineRunState(List<IRunParameter> parameters, List<IPipelineRunVariable> variables, List<IPipelineActivityResult> activityResults, string? iterationItem) : base(parameters)
     {
         Variables = variables;
         PipelineActivityResults = new List<IPipelineActivityResult>();
+        PipelineActivityResults.AddRange(activityResults);
+        ScopedPipelineActivityResults = new List<IPipelineActivityResult>();
         IterationItem = iterationItem;
     }
 
     public PipelineRunState(): base(new List<IRunParameter>())
     {
         Variables = new List<IPipelineRunVariable>();
         PipelineActivityResults = new List<IPipelineActivityResult>();
+        ScopedPipelineActivityResults = new List<IPipelineActivityResult>();
         IterationItem = null;
     }
 
     public void AddActivityResult(IPipelineActivityResult pipelineActivityResult)
     {
+        ScopedPipelineActivityResults.Add(pipelineActivityResult);
         PipelineActivityResults.Add(pipelineActivityResult);
     }
 
+    public void AddScopedActivityResultsFromScopedState(PipelineRunState scopedState)
+    {
+        PipelineActivityResults.AddRange(scopedState.ScopedPipelineActivityResults);
+    }
+
     public PipelineRunState CreateIterationScope(string? iterationItem)
     {
-        return new PipelineRunState(Parameters, Variables, iterationItem);
+        return new PipelineRunState(Parameters, Variables, PipelineActivityResults, iterationItem);
     }
 }