diff --git a/src/AzureDataFactory.TestingFramework.Example/AzureDataFactory.TestingFramework.Example.csproj b/src/AzureDataFactory.TestingFramework.Example/AzureDataFactory.TestingFramework.Example.csproj index bd9fc46d..60205238 100644 --- a/src/AzureDataFactory.TestingFramework.Example/AzureDataFactory.TestingFramework.Example.csproj +++ b/src/AzureDataFactory.TestingFramework.Example/AzureDataFactory.TestingFramework.Example.csproj @@ -23,13 +23,13 @@ - - PreserveNewest - + - + + PreserveNewest + diff --git a/src/AzureDataFactory.TestingFramework.Example/BatchJob/BatchJobFunctionalTests.cs b/src/AzureDataFactory.TestingFramework.Example/BatchJob/BatchJobFunctionalTests.cs index 29f0a8c3..275b1022 100644 --- a/src/AzureDataFactory.TestingFramework.Example/BatchJob/BatchJobFunctionalTests.cs +++ b/src/AzureDataFactory.TestingFramework.Example/BatchJob/BatchJobFunctionalTests.cs @@ -4,21 +4,20 @@ using AzureDataFactory.TestingFramework.Exceptions; using AzureDataFactory.TestingFramework.Models; using AzureDataFactory.TestingFramework.Models.Base; -using AzureDataFactory.TestingFramework.Models.Pipelines; namespace AzureDataFactory.TestingFramework.Example.BatchJob; public class BatchJobFunctionalTests { - [Fact] public void BatchJobTest() { - var pipeline = PipelineFactory.ParseFromFile("BatchJob/pipeline.json"); + var testFramework = new TestFramework(dataFactoryFolderPath: "BatchJob"); + var pipeline = testFramework.Repository.GetPipelineByName("batch_job"); Assert.Equal("batch_job", pipeline.Name); Assert.Equal(11, pipeline.Activities.Count); - var activities = pipeline.EvaluateWithActivityEnumerator(new List + var activities = testFramework.Evaluate(pipeline, new List { new RunParameter(ParameterType.Parameter, "BatchPoolId", "batch-pool-id"), new RunParameter(ParameterType.Parameter, "WorkloadApplicationPackageName", "test-application"), diff --git a/src/AzureDataFactory.TestingFramework.Example/BatchJob/BatchJobUnitTests.cs b/src/AzureDataFactory.TestingFramework.Example/BatchJob/BatchJobUnitTests.cs index 9772c166..3382febe 100644 --- a/src/AzureDataFactory.TestingFramework.Example/BatchJob/BatchJobUnitTests.cs +++ b/src/AzureDataFactory.TestingFramework.Example/BatchJob/BatchJobUnitTests.cs @@ -16,7 +16,7 @@ public class BatchJobUnitTests public BatchJobUnitTests() { - _pipeline = PipelineFactory.ParseFromFile("BatchJob/pipeline.json"); + _pipeline = PipelineFactory.ParseFromFile("BatchJob/pipeline/batch_job.json"); _state = new PipelineRunState(); } diff --git a/src/AzureDataFactory.TestingFramework.Example/BatchJob/pipeline.json b/src/AzureDataFactory.TestingFramework.Example/BatchJob/pipeline/batch_job.json similarity index 100% rename from src/AzureDataFactory.TestingFramework.Example/BatchJob/pipeline.json rename to src/AzureDataFactory.TestingFramework.Example/BatchJob/pipeline/batch_job.json diff --git a/src/AzureDataFactory.TestingFramework.Tests/AzureDataFactory.TestingFramework.Tests.csproj b/src/AzureDataFactory.TestingFramework.Tests/AzureDataFactory.TestingFramework.Tests.csproj index 454414e2..5c47e705 100644 --- a/src/AzureDataFactory.TestingFramework.Tests/AzureDataFactory.TestingFramework.Tests.csproj +++ b/src/AzureDataFactory.TestingFramework.Tests/AzureDataFactory.TestingFramework.Tests.csproj @@ -27,6 +27,12 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + diff --git a/src/AzureDataFactory.TestingFramework.Tests/Functional/Pipelines/Child/ChildPipelineTests.cs b/src/AzureDataFactory.TestingFramework.Tests/Functional/Pipelines/Child/ChildPipelineTests.cs new file mode 100644 index 00000000..ef215498 --- /dev/null +++ b/src/AzureDataFactory.TestingFramework.Tests/Functional/Pipelines/Child/ChildPipelineTests.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using AzureDataFactory.TestingFramework.Exceptions; +using AzureDataFactory.TestingFramework.Models; +using AzureDataFactory.TestingFramework.Models.Base; +using AzureDataFactory.TestingFramework.Models.Pipelines; + +namespace AzureDataFactory.TestingFramework.Tests.Functional.Pipelines.Child; + +public class ChildPipelineTests +{ + [Fact] + public void WhenExecutePipelineActivityIsCalled_ThenChildPipelineActivitiesAreExecuted() + { + // Arrange + var testFramework = new TestFramework(dataFactoryFolderPath: "Functional/Pipelines/Child", shouldEvaluateChildPipelines: true); + var pipeline = testFramework.Repository.GetPipelineByName("main"); + + // Act + var activities = testFramework.Evaluate(pipeline, new List() + { + new RunParameter(ParameterType.Parameter, "Url", "https://example.com"), + new RunParameter(ParameterType.Parameter, "Body", "{ \"key\": \"value\" }") + }); + + // Assert + var childWebActivity = activities.GetNext(); + Assert.Equal("API Call", childWebActivity.Name); + Assert.Equal("https://example.com", childWebActivity.Uri); + Assert.Equal("{ \"key\": \"value\" }", childWebActivity.Body); + } + + [Fact] + public void WhenExecutePipelineActivityIsCalledAndExecutionIsEnforcedAndPipelineIsNotLoaded_ThenExceptionShouldBeThrown() + { + // Arrange + var testFramework = new TestFramework(shouldEvaluateChildPipelines: true); + var pipeline = PipelineFactory.ParseFromFile("Functional/Pipelines/Child/pipeline/main.json"); + + // Act + var exception = Assert.Throws(() => testFramework.EvaluateAll(pipeline, new List() + { + new RunParameter(ParameterType.Parameter, "Url", "https://example.com"), + new RunParameter(ParameterType.Parameter, "Body", "{ \"key\": \"value\" }") + })); + + // Assert + Assert.Equal("Pipeline with name 'child' was not found in the repository. Make sure to load the repository before evaluating pipelines.", exception.Message); + } +} diff --git a/src/AzureDataFactory.TestingFramework.Tests/Functional/Pipelines/Child/pipeline/child.json b/src/AzureDataFactory.TestingFramework.Tests/Functional/Pipelines/Child/pipeline/child.json new file mode 100644 index 00000000..9ad9931c --- /dev/null +++ b/src/AzureDataFactory.TestingFramework.Tests/Functional/Pipelines/Child/pipeline/child.json @@ -0,0 +1,43 @@ +{ + "name": "child", + "properties": { + "activities": [ + { + "name": "API Call", + "type": "WebActivity", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "url": { + "value": "@pipeline().parameters.Url", + "type": "Expression" + }, + "method": "POST", + "body": { + "value": "@pipeline().parameters.Body", + "type": "Expression" + } + } + } + ], + "parameters": { + "Url": { + "type": "string" + }, + "Body": { + "type": "string" + } + }, + "folder": { + "name": "tests" + }, + "annotations": [] + } +} \ No newline at end of file diff --git a/src/AzureDataFactory.TestingFramework.Tests/Functional/Pipelines/Child/pipeline/main.json b/src/AzureDataFactory.TestingFramework.Tests/Functional/Pipelines/Child/pipeline/main.json new file mode 100644 index 00000000..ef4f6d3a --- /dev/null +++ b/src/AzureDataFactory.TestingFramework.Tests/Functional/Pipelines/Child/pipeline/main.json @@ -0,0 +1,42 @@ +{ + "name": "main", + "properties": { + "activities": [ + { + "name": "Execute Child pipeline", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "child", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "Url": { + "value": "@pipeline().parameters.Url", + "type": "Expression" + }, + "Body": { + "value": "@pipeline().parameters.Body", + "type": "Expression" + } + } + } + } + ], + "parameters": { + "Url": { + "type": "string" + }, + "Body": { + "type": "string" + } + }, + "folder": { + "name": "tests" + }, + "annotations": [] + } +} \ No newline at end of file diff --git a/src/AzureDataFactory.TestingFramework.Tests/Models/Activities/ControlActivities/ForEachActivityTests.cs b/src/AzureDataFactory.TestingFramework.Tests/Models/Activities/ControlActivities/ForEachActivityTests.cs index d45b1ad4..bdc17719 100644 --- a/src/AzureDataFactory.TestingFramework.Tests/Models/Activities/ControlActivities/ForEachActivityTests.cs +++ b/src/AzureDataFactory.TestingFramework.Tests/Models/Activities/ControlActivities/ForEachActivityTests.cs @@ -28,7 +28,7 @@ public void WhenEvaluateChildActivities_ThenShouldReturnTheActivityWithItemExpre // Act forEachActivity.Evaluate(state); - var childActivities = forEachActivity.EvaluateChildActivities(state); + var childActivities = forEachActivity.EvaluateChildActivities(state, new TestFramework()); // Assert using var enumarator = childActivities.GetEnumerator(); diff --git a/src/AzureDataFactory.TestingFramework.Tests/Models/Activities/Base/ActivitiesEvaluatorTests.cs b/src/AzureDataFactory.TestingFramework.Tests/Models/TestFrameworkTests.cs similarity index 88% rename from src/AzureDataFactory.TestingFramework.Tests/Models/Activities/Base/ActivitiesEvaluatorTests.cs rename to src/AzureDataFactory.TestingFramework.Tests/Models/TestFrameworkTests.cs index 944087bf..822d881d 100644 --- a/src/AzureDataFactory.TestingFramework.Tests/Models/Activities/Base/ActivitiesEvaluatorTests.cs +++ b/src/AzureDataFactory.TestingFramework.Tests/Models/TestFrameworkTests.cs @@ -10,14 +10,16 @@ namespace AzureDataFactory.TestingFramework.Tests.Models.Activities.Base; -public class ActivitiesEvaluatorTests +public class TestFrameworkTests { private readonly List _activities; private readonly WebActivity _webActivity; private readonly SetVariableActivity _setVariableActivity; + private readonly TestFramework _testFramework; - public ActivitiesEvaluatorTests() + public TestFrameworkTests() { + _testFramework = new TestFramework(); _activities = new List(); _webActivity = new WebActivity("webActivity", WebActivityMethod.Get, "https://www.example.com") { @@ -41,7 +43,7 @@ public void EvaluateWithoutIterationActivities_ShouldEvaluateAccordingToDependen // Act var state = new PipelineRunState(); state.Variables.Add(new PipelineRunVariable("variable1", string.Empty)); - var evaluatedActivities = ActivitiesEvaluator.Evaluate(_activities, state).ToList(); + var evaluatedActivities = _testFramework.EvaluateActivities(_activities, state).ToList(); // Assert Assert.NotNull(evaluatedActivities); @@ -57,7 +59,7 @@ public void EvaluateWithCircularDependencies_ShouldThrowActivitiesEvaluatorInval _setVariableActivity.DependsOn.Add(new PipelineActivityDependency("webActivity", new[] { DependencyCondition.Succeeded })); // Assert - Assert.Throws(() => ActivitiesEvaluator.Evaluate(_activities, new PipelineRunState()).ToList()); + Assert.Throws(() => _testFramework.EvaluateActivities(_activities, new PipelineRunState()).ToList()); } [Fact] @@ -73,7 +75,7 @@ public void EvaluateWithForeachActivities_ShouldEvaluateAccordingToDependencies( _webActivity.Uri = new DataFactoryElement("@concat('https://www.example.com/', item())", DataFactoryElementKind.Expression); // Act - var evaluatedActivities = ActivitiesEvaluator.Evaluate(new List { foreachActivity }, state); + var evaluatedActivities = _testFramework.EvaluateActivities(new List { foreachActivity }, state); // Assert using var enumerator = evaluatedActivities.GetEnumerator(); @@ -106,7 +108,7 @@ public void EvaluateWithUntilActivities_ShouldEvaluateAccordingToDependencies() _activities); // Act - var evaluatedActivities = ActivitiesEvaluator.Evaluate(new List { untilActivity }, state); + var evaluatedActivities = _testFramework.EvaluateActivities(new List { untilActivity }, state); // Assert using var enumerator = evaluatedActivities.GetEnumerator(); diff --git a/src/AzureDataFactory.TestingFramework/AzureDataFactory.TestingFramework.csproj b/src/AzureDataFactory.TestingFramework/AzureDataFactory.TestingFramework.csproj index 9b055f0d..3bcf7e3d 100644 --- a/src/AzureDataFactory.TestingFramework/AzureDataFactory.TestingFramework.csproj +++ b/src/AzureDataFactory.TestingFramework/AzureDataFactory.TestingFramework.csproj @@ -11,9 +11,14 @@ - + + + <_Parameter1>AzureDataFactory.TestingFramework.Tests + + + AzureDataFactory.TestingFramework 0.1.6-alpha diff --git a/src/AzureDataFactory.TestingFramework/Exceptions/PipelineNotFoundException.cs b/src/AzureDataFactory.TestingFramework/Exceptions/PipelineNotFoundException.cs new file mode 100644 index 00000000..5a408690 --- /dev/null +++ b/src/AzureDataFactory.TestingFramework/Exceptions/PipelineNotFoundException.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace AzureDataFactory.TestingFramework.Exceptions; + +public class PipelineNotFoundException : Exception +{ + public PipelineNotFoundException(string name) : base($"Pipeline with name '{name}' was not found in the repository. Make sure to load the repository before evaluating pipelines.") + { + } +} diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/Base/ActivitiesEvaluator.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/Base/ActivitiesEvaluator.cs deleted file mode 100644 index ea637dad..00000000 --- a/src/AzureDataFactory.TestingFramework/Models/Activities/Base/ActivitiesEvaluator.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using AzureDataFactory.TestingFramework.Exceptions; -using AzureDataFactory.TestingFramework.Models.Pipelines; - -namespace AzureDataFactory.TestingFramework.Models.Activities.Base; - -public static class ActivitiesEvaluator -{ - public static IEnumerable Evaluate(List activities, PipelineRunState state) - { - while (state.ScopedPipelineActivityResults.Count != activities.Count) - { - var anyActivityEvaluated = false; - foreach (var activity in activities - .Where(activity => !state.ScopedPipelineActivityResults.Contains(activity)) - .Where(activity => activity.AreDependencyConditionMet(state))) - { - var evaluatedActivity = (PipelineActivity)activity.Evaluate(state); - if (evaluatedActivity is not IIterationActivity) - yield return evaluatedActivity; - - anyActivityEvaluated = true; - state.AddActivityResult(activity); - - if (activity is IIterationActivity) - { - if (activity is UntilActivity untilActivity) - { - do - { - foreach (var child in untilActivity.EvaluateChildActivities(state)) - yield return child; - } while (!untilActivity.Expression.Evaluate(state)); - } - else if (activity is ControlActivity controlActivity) - { - foreach (var child in controlActivity.EvaluateChildActivities(state)) - yield return child; - } - } - } - - if (!anyActivityEvaluated) - { - throw new ActivitiesEvaluatorInvalidDependencyException("Validate that there are no circular dependencies or whether activity results were not set correctly."); - } - } - } -} \ No newline at end of file diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ControlActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ControlActivity.cs index 0b18e3ac..1c895de0 100644 --- a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ControlActivity.cs +++ b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ControlActivity.cs @@ -13,15 +13,20 @@ protected virtual List GetNextActivities() return new List(); } - public virtual IEnumerable EvaluateChildActivities(PipelineRunState state) + internal virtual IEnumerable EvaluateChildActivities(PipelineRunState state, TestFramework testFramework) { var scopedState = state.CreateIterationScope(null); var activities = GetNextActivities(); - foreach (var activity in ActivitiesEvaluator.Evaluate(activities, scopedState)) + foreach (var activity in testFramework.EvaluateActivities(activities, scopedState)) { yield return activity; } state.AddScopedActivityResultsFromScopedState(scopedState); } + + internal virtual IEnumerable EvaluateChildActivities(PipelineRunState state) + { + return EvaluateChildActivities(state, new TestFramework()); + } } \ No newline at end of file diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ExecutePipelineActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ExecutePipelineActivity.cs new file mode 100644 index 00000000..50c3a224 --- /dev/null +++ b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ExecutePipelineActivity.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using AzureDataFactory.TestingFramework.Models.Base; +using AzureDataFactory.TestingFramework.Models.Pipelines; + +namespace AzureDataFactory.TestingFramework.Models; + +public partial class ExecutePipelineActivity : IIterationActivity +{ + internal List GetChildRunParameters(PipelineRunState state) + { + var parameters = state.Parameters.Where(p => p.Type == ParameterType.Global).ToList(); + parameters.AddRange(Parameters.Select(p => new RunParameter(ParameterType.Parameter, p.Key, p.Value))); + return parameters; + } +} diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ForEachActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ForEachActivity.cs index bf76658c..1598a58d 100644 --- a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ForEachActivity.cs +++ b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ForEachActivity.cs @@ -23,14 +23,14 @@ public override DataFactoryEntity Evaluate(PipelineRunState state) return base.Evaluate(state); } - public override IEnumerable EvaluateChildActivities(PipelineRunState state) + internal override IEnumerable EvaluateChildActivities(PipelineRunState state, TestFramework testFramework) { var activities = GetNextActivities(); return IterationItems.SelectMany(item => { var scopedState = state.CreateIterationScope(item); - return ActivitiesEvaluator.Evaluate(activities, scopedState); + return testFramework.EvaluateActivities(activities, scopedState); }); } } \ No newline at end of file diff --git a/src/AzureDataFactory.TestingFramework/Models/Pipelines/Pipeline.cs b/src/AzureDataFactory.TestingFramework/Models/Pipelines/Pipeline.cs index e2ec783a..98bf9aca 100644 --- a/src/AzureDataFactory.TestingFramework/Models/Pipelines/Pipeline.cs +++ b/src/AzureDataFactory.TestingFramework/Models/Pipelines/Pipeline.cs @@ -26,10 +26,11 @@ public PipelineActivity GetActivityByName(string activityName) /// Evaluates the pipeline with the provided parameters. The order of activity execution is simulated based on the dependencies. Any expression part of the activity is evaluated based on the state of the pipeline. /// /// The global and regular parameters to be used for evaluating expressions. + /// Reference to the test framework containing the repository and configuration /// /// Thrown if a required pipeline parameter is not required /// Thrown if a pipeline parameter is provided more than once - public IEnumerable Evaluate(List parameters) + internal IEnumerable Evaluate(List parameters, TestFramework testFramework) { //Check if all parameters are provided foreach (var parameter in Parameters.Where(parameter => parameters.All(p => p.Name != parameter.Key))) @@ -41,20 +42,33 @@ public IEnumerable Evaluate(List parameters) throw new PipelineDuplicateParameterProvidedException($"Duplicate parameters provided: {string.Join(", ", duplicateParameters.Select(x => $"{x.Name} ({x.Type})"))}"); var state = new PipelineRunState(parameters, Variables); - foreach (var activity in ActivitiesEvaluator.Evaluate(Activities.ToList(), state)) + foreach (var activity in testFramework.EvaluateActivities(Activities.ToList(), state)) yield return activity; } + /// + /// Evaluates the pipeline with the provided parameters. The order of activity execution is simulated based on the dependencies. Any expression part of the activity is evaluated based on the state of the pipeline. + /// + /// The global and regular parameters to be used for evaluating expressions. + /// + /// Thrown if a required pipeline parameter is not required + /// Thrown if a pipeline parameter is provided more than once + internal IEnumerable Evaluate(List parameters) + { + return Evaluate(parameters, new TestFramework()); + } + /// /// Evaluates the pipeline with the provider parameters and returns an enumerator to easily iterate over the activities. The order of activity execution is simulated based on the dependencies. Any expression part of the activity is evaluated based on the state of the pipeline. /// /// The global and regular parameters to be used for evaluating expressions. + /// Reference to the test framework containing the repository and configuration /// /// Thrown if a required pipeline parameter is not required /// Thrown if a pipeline parameter is provided more than once - public ActivityEnumerator EvaluateWithActivityEnumerator(List parameters) + internal ActivityEnumerator EvaluateWithActivityEnumerator(List parameters, TestFramework testFramework) { - var activities = Evaluate(parameters); + var activities = Evaluate(parameters, testFramework); return new ActivityEnumerator(activities); } } \ No newline at end of file diff --git a/src/AzureDataFactory.TestingFramework/Models/Repositories/DataFactoryRepository.cs b/src/AzureDataFactory.TestingFramework/Models/Repositories/DataFactoryRepository.cs new file mode 100644 index 00000000..86605054 --- /dev/null +++ b/src/AzureDataFactory.TestingFramework/Models/Repositories/DataFactoryRepository.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.ResourceManager.DataFactory; +using AzureDataFactory.TestingFramework.Exceptions; + +namespace AzureDataFactory.TestingFramework.Models.Repositories; + +public class DataFactoryRepository +{ + /// + /// Loads pipelines, linkedServices, datasets and triggers into the static repository which will be used to evaluate child entities upon pipeline evaluation + /// + /// List of deserialized pipelines + /// List of deserialized linkedServices + /// List of deserialized datasets + /// List of deserialized triggers + public DataFactoryRepository(List pipelines, List linkedServices, List datasets, List triggers) + { + Pipelines = pipelines; + LinkedServices = linkedServices; + Datasets = datasets; + Triggers = triggers; + } + + public DataFactoryRepository() + { + Pipelines = new List(); + LinkedServices = new List(); + Datasets = new List(); + Triggers = new List(); + } + + public List Pipelines { get; private set; } + public List LinkedServices { get; private set; } + public List Datasets { get; private set; } + public List Triggers { get; private set; } + + public Pipeline GetPipelineByName(string name) + { + var pipeline = Pipelines.SingleOrDefault(pipeline => pipeline.Name == name); + if (pipeline == null) + throw new PipelineNotFoundException(name); + + return pipeline; + } +} diff --git a/src/AzureDataFactory.TestingFramework/Models/Repositories/DataFactoryRepositoryFactory.cs b/src/AzureDataFactory.TestingFramework/Models/Repositories/DataFactoryRepositoryFactory.cs new file mode 100644 index 00000000..5462f8d6 --- /dev/null +++ b/src/AzureDataFactory.TestingFramework/Models/Repositories/DataFactoryRepositoryFactory.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text.Json; +using Azure.ResourceManager.DataFactory; + +namespace AzureDataFactory.TestingFramework.Models.Repositories; + +public static class DataFactoryRepositoryFactory +{ + public static DataFactoryRepository ParseFromFolder(string folderPath) + { + var pipelines = GetDataFactoryEntitiesByFolderPath(Path.Combine(folderPath, "pipeline"), Pipeline.DeserializeDataFactoryPipelineData); + var linkedServices = GetDataFactoryEntitiesByFolderPath(Path.Combine(folderPath, "linkedService"), DataFactoryLinkedServiceProperties.DeserializeDataFactoryLinkedServiceProperties); + var datasets = GetDataFactoryEntitiesByFolderPath(Path.Combine(folderPath, "dataset"), DataFactoryDatasetProperties.DeserializeDataFactoryDatasetProperties); + var triggers = GetDataFactoryEntitiesByFolderPath(Path.Combine(folderPath, "trigger"), DataFactoryTriggerProperties.DeserializeDataFactoryTriggerProperties); + + return new DataFactoryRepository(pipelines, linkedServices, datasets, triggers); + } + + private static List GetDataFactoryEntitiesByFolderPath(string folderPath, Func deserialize) where TType : class + { + if (!Directory.Exists(folderPath)) + return new List(); + + return Directory.GetFiles(folderPath, "*.json") + .Select(File.ReadAllText) + .Select(json => JsonSerializer.Deserialize(json)) + .Select(deserialize) + .ToList(); + } +} diff --git a/src/AzureDataFactory.TestingFramework/Models/TestFramework.cs b/src/AzureDataFactory.TestingFramework/Models/TestFramework.cs new file mode 100644 index 00000000..2836d969 --- /dev/null +++ b/src/AzureDataFactory.TestingFramework/Models/TestFramework.cs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.ResourceManager.DataFactory; +using AzureDataFactory.TestingFramework.Exceptions; +using AzureDataFactory.TestingFramework.Models.Activities.Base; +using AzureDataFactory.TestingFramework.Models.Base; +using AzureDataFactory.TestingFramework.Models.Pipelines; +using AzureDataFactory.TestingFramework.Models.Repositories; + +namespace AzureDataFactory.TestingFramework.Models; + +public class TestFramework +{ + private readonly bool _shouldEvaluateChildPipelines; + public DataFactoryRepository Repository { get; } + + public TestFramework(string? dataFactoryFolderPath = null, bool shouldEvaluateChildPipelines = false) + { + Repository = dataFactoryFolderPath != null ? DataFactoryRepositoryFactory.ParseFromFolder(dataFactoryFolderPath) : new DataFactoryRepository(); + _shouldEvaluateChildPipelines = shouldEvaluateChildPipelines; + } + + /// + /// Get an enumerator to yield the evaluation of the pipeline activities one by one using the provided parameters. The order of activity execution is simulated based on the dependencies. Any expression part of the activity is evaluated based on the state of the pipeline. + /// + /// The pipeline to evaluate. + /// The global and regular parameters to be used for evaluating expressions. + /// + /// Thrown if a required pipeline parameter is not required + /// Thrown if a pipeline parameter is provided more than once + public ActivityEnumerator Evaluate(Pipeline pipeline, List parameters) + { + return pipeline.EvaluateWithActivityEnumerator(parameters, this); + } + + /// + /// Evaluates all pipeline activities using the provided parameters. The order of activity execution is simulated based on the dependencies. Any expression part of the activity is evaluated based on the state of the pipeline. + /// + /// The pipeline to evaluate. + /// The global and regular parameters to be used for evaluating expressions. + /// + /// Thrown if a required pipeline parameter is not required + /// Thrown if a pipeline parameter is provided more than once + public List EvaluateAll(Pipeline pipeline, List parameters) + { + return pipeline.Evaluate(parameters, this).ToList(); + } + + internal IEnumerable EvaluateActivities(List activities, PipelineRunState state) + { + while (state.ScopedPipelineActivityResults.Count != activities.Count) + { + var anyActivityEvaluated = false; + foreach (var activity in activities + .Where(activity => !state.ScopedPipelineActivityResults.Contains(activity)) + .Where(activity => activity.AreDependencyConditionMet(state))) + { + var evaluatedActivity = (PipelineActivity)activity.Evaluate(state); + if (evaluatedActivity is not IIterationActivity || (evaluatedActivity is ExecutePipelineActivity && !_shouldEvaluateChildPipelines)) + yield return evaluatedActivity; + + anyActivityEvaluated = true; + state.AddActivityResult(activity); + + if (activity is IIterationActivity) + { + if (activity is ExecutePipelineActivity executePipelineActivity && _shouldEvaluateChildPipelines) + { + var pipeline = Repository.GetPipelineByName(executePipelineActivity.Pipeline.ReferenceName); + + // Evaluate the pipeline with its own scope + foreach (var childActivity in pipeline.Evaluate(executePipelineActivity.GetChildRunParameters(state))) + yield return childActivity; + } + else if (activity is UntilActivity untilActivity) + { + do + { + foreach (var child in untilActivity.EvaluateChildActivities(state, this)) + yield return child; + } while (!untilActivity.Expression.Evaluate(state)); + } + else if (activity is ControlActivity controlActivity) + { + foreach (var child in controlActivity.EvaluateChildActivities(state, this)) + yield return child; + } + } + } + + if (!anyActivityEvaluated) + { + throw new ActivitiesEvaluatorInvalidDependencyException("Validate that there are no circular dependencies or whether activity results were not set correctly."); + } + } + } +}