Skip to content

Commit

Permalink
docs: add initial documentation on public members (#15)
Browse files Browse the repository at this point in the history
  • Loading branch information
arjendev authored Sep 21, 2023
1 parent 7b8a521 commit 6d97c7e
Show file tree
Hide file tree
Showing 23 changed files with 207 additions and 50 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -43,26 +43,22 @@ public void BatchJobTest()
new RunParameter<string>(ParameterType.Global, "ADFName", "adf-name"),
});

var setUserAssignedIdentityActivity = activities.GetNext() as SetVariableActivity;
Assert.NotNull(setUserAssignedIdentityActivity);
var setUserAssignedIdentityActivity = activities.GetNext<SetVariableActivity>();
Assert.Equal("Set UserAssignedIdentityReference", setUserAssignedIdentityActivity.Name);
Assert.Equal("UserAssignedIdentityReference", setUserAssignedIdentityActivity.VariableName);
Assert.Equal("/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name", setUserAssignedIdentityActivity.Value);

var setManagerApplicationPackagePathActivity = activities.GetNext() as SetVariableActivity;
Assert.NotNull(setManagerApplicationPackagePathActivity);
var setManagerApplicationPackagePathActivity = activities.GetNext<SetVariableActivity>();
Assert.Equal("Set ManagerApplicationPackagePath", setManagerApplicationPackagePathActivity.Name);
Assert.Equal("ManagerApplicationPackagePath", setManagerApplicationPackagePathActivity.VariableName);
Assert.Equal("$AZ_BATCH_APP_PACKAGE_batchmanager_2_0_0/batchmanager.tar.gz", setManagerApplicationPackagePathActivity.Value);

var setWorkloadApplicationPackagePathActivity = activities.GetNext() as SetVariableActivity;
Assert.NotNull(setWorkloadApplicationPackagePathActivity);
var setWorkloadApplicationPackagePathActivity = activities.GetNext<SetVariableActivity>();
Assert.Equal("Set WorkloadApplicationPackagePath", setWorkloadApplicationPackagePathActivity.Name);
Assert.Equal("WorkloadApplicationPackagePath", setWorkloadApplicationPackagePathActivity.VariableName);
Assert.Equal("$AZ_BATCH_APP_PACKAGE_test-application_1_5_0/test-application.tar.gz", setWorkloadApplicationPackagePathActivity.Value);

var setCommonEnvironmentSettingsActivity = activities.GetNext() as SetVariableActivity;
Assert.NotNull(setCommonEnvironmentSettingsActivity);
var setCommonEnvironmentSettingsActivity = activities.GetNext<SetVariableActivity>();
Assert.Equal("Set CommonEnvironmentSettings", setCommonEnvironmentSettingsActivity.Name);
Assert.Equal("CommonEnvironmentSettings", setCommonEnvironmentSettingsActivity.VariableName);
Assert.Equal(@"[
Expand Down Expand Up @@ -100,27 +96,23 @@ public void BatchJobTest()
}
]", setCommonEnvironmentSettingsActivity.Value, ignoreLineEndingDifferences: true, ignoreWhiteSpaceDifferences: true);

var setJobContainerNameActivity = activities.GetNext() as SetVariableActivity;
Assert.NotNull(setJobContainerNameActivity);
var setJobContainerNameActivity = activities.GetNext<SetVariableActivity>();
Assert.Equal("Set JobContainerName", setJobContainerNameActivity.Name);
Assert.Equal("JobContainerName", setJobContainerNameActivity.VariableName);
Assert.Equal("job-802100a5-ec79-4a52-be62-8d6109f3ff9a", setJobContainerNameActivity.Value);

var setJobContainerUrlActivity = activities.GetNext() as SetVariableActivity;
Assert.NotNull(setJobContainerUrlActivity);
var setJobContainerUrlActivity = activities.GetNext<SetVariableActivity>();
Assert.Equal("Set Job Container URL", setJobContainerUrlActivity.Name);
Assert.Equal("JobContainerURL", setJobContainerUrlActivity.VariableName);
Assert.Equal("https://batch-account-name.blob.core.windows.net/job-802100a5-ec79-4a52-be62-8d6109f3ff9a", setJobContainerUrlActivity.Value);

var createJobContainer = activities.GetNext() as WebActivity;
Assert.NotNull(createJobContainer);
var createJobContainer = activities.GetNext<WebActivity>();
Assert.Equal("Create Job Storage Container", createJobContainer.Name);
Assert.Equal("https://batch-account-name.blob.core.windows.net/job-802100a5-ec79-4a52-be62-8d6109f3ff9a?restype=container", createJobContainer.Uri);
Assert.Equal("PUT", createJobContainer.Method);
Assert.Equal("{}", createJobContainer.Body);

var startJobActivity = activities.GetNext() as WebActivity;
Assert.NotNull(startJobActivity);
var startJobActivity = activities.GetNext<WebActivity>();
Assert.Equal("Start Job", startJobActivity.Name);
Assert.Equal("https://batch-account-name.westeurope.batch.azure.com/jobs?api-version=2022-10-01.16.0", startJobActivity.Uri);
Assert.Equal("POST", startJobActivity.Method);
Expand Down Expand Up @@ -188,15 +180,13 @@ public void BatchJobTest()
""usesTaskDependencies"": true,
""commonEnvironmentSettings"": [{""name"":""WORKLOAD_APP_PACKAGE"",""value"":""test-application""},{""name"":""WORKLOAD_APP_PACKAGE_VERSION"",""value"":""1.5.0""},{""name"":""MANAGER_APP_PACKAGE"",""value"":""batchmanager""},{""name"":""MANAGER_APP_PACKAGE_VERSION"",""value"":""2.0.0""},{""name"":""BATCH_JOB_TIMEOUT"",""value"":""PT4H""},{""name"":""WORKLOAD_AUTO_STORAGE_ACCOUNT_NAME"",""value"":""batch-account-name""},{""name"":""WORKLOAD_USER_ASSIGNED_IDENTITY_RESOURCE_ID"",""value"":""/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name""},{""name"":""WORKLOAD_USER_ASSIGNED_IDENTITY_CLIENT_ID"",""value"":""/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name""}]}", startJobActivity.Body, ignoreLineEndingDifferences: true, ignoreWhiteSpaceDifferences: true);

var monitorActivity = activities.GetNext() as ExecutePipelineActivity;
Assert.NotNull(monitorActivity);
var monitorActivity = activities.GetNext<ExecutePipelineActivity>();
Assert.Equal("Monitor Batch Job", monitorActivity.Name);
Assert.Equal("monitor_batch_job", monitorActivity.Pipeline.ReferenceName);
Assert.Equal(1, monitorActivity.Parameters.Count);
Assert.Equal("802100a5-ec79-4a52-be62-8d6109f3ff9a", monitorActivity.Parameters["JobId"]);

var copyOutputFiles = activities.GetNext() as ExecutePipelineActivity;
Assert.NotNull(copyOutputFiles);
var copyOutputFiles = activities.GetNext<ExecutePipelineActivity>();
Assert.Equal("Copy Output Files", copyOutputFiles.Name);
Assert.Equal("copy_output_files", copyOutputFiles.Pipeline.ReferenceName);
Assert.Equal(5, copyOutputFiles.Parameters.Count);
Expand All @@ -206,8 +196,7 @@ public void BatchJobTest()
Assert.Equal("test-application-output-container-name", copyOutputFiles.Parameters["OutputContainerName"]);
Assert.Equal("TEMP", copyOutputFiles.Parameters["OutputFolderName"]);

var deleteJobContainer = activities.GetNext() as WebActivity;
Assert.NotNull(deleteJobContainer);
var deleteJobContainer = activities.GetNext<WebActivity>();
Assert.Equal("Delete Job Storage Container", deleteJobContainer.Name);
Assert.Equal("https://batch-account-name.blob.core.windows.net/job-802100a5-ec79-4a52-be62-8d6109f3ff9a?restype=container", deleteJobContainer.Uri);
Assert.Equal("DELETE", deleteJobContainer.Method);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// Copyright (c) Microsoft Corporation.

namespace AzureDataFactory.TestingFramework.Exceptions;

public class ActivityEnumeratorTypeMismatchException : Exception
{
public ActivityEnumeratorTypeMismatchException(string message) : base(message)
{
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ namespace AzureDataFactory.TestingFramework.Expressions;
/// <summary>
/// Used to evaluate any activity('activityName').status or activity('activityName').output.field1.field2
/// </summary>
public class ActivityExpression : BaseExpression, IPipelineExpression
internal class ActivityExpression : BaseExpression, IPipelineExpression
{
public const string ExtractActivityRegex = @"(@?{?activity\('(\w+( +\w+)*)'\)(\.(\w+))*}?)";
private const string ExtractActivityRegex = @"(@?{?activity\('(\w+( +\w+)*)'\)(\.(\w+))*}?)";

public ActivityExpression(string expression) : base(expression)
private ActivityExpression(string expression) : base(expression)
{
}

Expand Down Expand Up @@ -67,7 +67,7 @@ public TType Evaluate<TType>(PipelineRunState state)
return (activityName, fields);
}

public static IEnumerable<ActivityExpression> Find(string text)
internal static IEnumerable<ActivityExpression> Find(string text)
{
return ExpressionFinder.FindByRegex(text, ExtractActivityRegex, e => new ActivityExpression(e));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@

namespace AzureDataFactory.TestingFramework.Expressions;

public abstract class BaseExpression
internal abstract class BaseExpression
{
public string ExpressionValue { get; }
internal string ExpressionValue { get; }

protected BaseExpression(string expressionValue)
{
Expand All @@ -28,12 +28,12 @@ protected static TType GetParameterValue<TType>(RunState state, string parameter
}
}

public interface IPipelineExpression
internal interface IPipelineExpression
{
TType Evaluate<TType>(PipelineRunState state);
}

public interface IRunExpression
internal interface IRunExpression
{
TType Evaluate<TType>(RunState state);
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ namespace AzureDataFactory.TestingFramework.Expressions;
/// <summary>
/// Used to evaluate any dataset('datasetName') expression.
/// </summary>
public class DatasetExpression : BaseExpression, IRunExpression
internal class DatasetExpression : BaseExpression, IRunExpression
{
public const string ExtractDatasetRegex = @"(@?{?dataset\(\)\.(\w+)}?)";
private const string ExtractDatasetRegex = @"(@?{?dataset\(\)\.(\w+)}?)";

public DatasetExpression(string expression) : base(expression)
private DatasetExpression(string expression) : base(expression)
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ namespace AzureDataFactory.TestingFramework.Expressions;

public static class ExpressionFinder
{
public static IReadOnlyList<TExpression> FindByRegex<TExpression>(string text, string regex, Func<string, TExpression> expressionFactory) where TExpression : BaseExpression
internal static IReadOnlyList<TExpression> FindByRegex<TExpression>(string text, string regex, Func<string, TExpression> expressionFactory) where TExpression : BaseExpression
{
var matches = Regex.Matches(text, regex, RegexOptions.Singleline);
return matches.Select(x => x.Value).Select(expressionFactory).ToList();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ namespace AzureDataFactory.TestingFramework.Expressions;
/// <summary>
/// Used to evaluate any item() expression.
/// </summary>
public class IterationItemExpression : BaseExpression, IPipelineExpression
internal class IterationItemExpression : BaseExpression, IPipelineExpression
{
public const string ExtractIterationItemRegex = "(@?{?item\\(\\)\\}?)";
private const string ExtractIterationItemRegex = "(@?{?item\\(\\)\\}?)";

public IterationItemExpression(string expression) : base(expression)
private IterationItemExpression(string expression) : base(expression)
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ namespace AzureDataFactory.TestingFramework.Expressions;
/// <summary>
/// Used to evaluate any linkedService('linkedServiceName') expression.
/// </summary>
public class LinkedServiceExpression : BaseExpression, IRunExpression
internal class LinkedServiceExpression : BaseExpression, IRunExpression
{
public const string ExtractLinkedServiceRegex = @"(@?{?linkedService\(\)\.(\w+)}?)";
private const string ExtractLinkedServiceRegex = @"(@?{?linkedService\(\)\.(\w+)}?)";

public LinkedServiceExpression(string expression) : base(expression)
private LinkedServiceExpression(string expression) : base(expression)
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ namespace AzureDataFactory.TestingFramework.Expressions;
/// <summary>
/// Used to evaluate any pipeline() expression, like pipeline().parameters.name and pipeline().globalParameters.name.
/// </summary>
public class ParameterExpression : BaseExpression, IRunExpression
internal class ParameterExpression : BaseExpression, IRunExpression
{
private readonly ParameterType _type;

public ParameterExpression(string expression, ParameterType type) : base(expression)
private ParameterExpression(string expression, ParameterType type) : base(expression)
{
_type = type;
}
Expand Down Expand Up @@ -50,5 +50,4 @@ public static IEnumerable<ParameterExpression> Find(string text,
var regex = "(@?{?pipeline\\(\\)\\." + GetParameterStringTemplate(parameterType) + @"\.(\w+)}?)";
return ExpressionFinder.FindByRegex(text, regex, e => new ParameterExpression(e, parameterType));
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ namespace AzureDataFactory.TestingFramework.Expressions;
/// <summary>
/// Used to evaluate any variables('variableName') expression.
/// </summary>
public class VariableExpression : BaseExpression, IPipelineExpression
internal class VariableExpression : BaseExpression, IPipelineExpression
{
public const string ExtractVariablesRegex = @"(@?{?variables\('(\w+)'\)}?)";
private const string ExtractVariablesRegex = @"(@?{?variables\('(\w+)'\)}?)";

public VariableExpression(string expression) : base(expression)
private VariableExpression(string expression) : base(expression)
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@

namespace AzureDataFactory.TestingFramework.Extensions;

public static class StringExtensions
internal static class StringExtensions
{
public static string TrimOneChar(this string text, char character)
internal static string TrimOneChar(this string text, char character)
{
text = text.StartsWith(character) ? text[1..] : text;
text = text.EndsWith(character) ? text[..^1] : text;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
// Copyright (c) Microsoft Corporation.

using System.Collections;
using AzureDataFactory.TestingFramework.Expressions;
using AzureDataFactory.TestingFramework.Extensions;
using AzureDataFactory.TestingFramework.Models.Base;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ public static class FunctionsRepository
{ "split", (string input, string delimiter) => input.Split(delimiter).ToList() }
};

/// <summary>
/// Registers a function to be used to evaluate ADF expressions.
/// </summary>
/// <param name="functionName">The name of the function</param>
/// <param name="function">The delegate that takes the expected arguments and evaluates them in C#.</param>
public static void Register(string functionName, Delegate function)
{
Functions[functionName] = function;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,39 @@ public class ActivityEnumerator
{
private readonly IEnumerator<PipelineActivity> _enumerator;

/// <summary>
/// Initializes a new instance of the <see cref="ActivityEnumerator"/> class. Can be used to easily iterate over a list of activities one by one.
/// </summary>
/// <param name="activities"></param>
public ActivityEnumerator(IEnumerable<PipelineActivity> activities)
{
_enumerator = activities.GetEnumerator();
}

/// <summary>
/// Moves the enumerator to the next activity and returns it.
/// </summary>
/// <returns>The next activity</returns>
/// <exception cref="ActivityEnumeratorException">If there are no more activities available for evaluation</exception>
public PipelineActivity GetNext()
{
if (_enumerator.MoveNext())
return _enumerator.Current!;

throw new ActivityEnumeratorException("No more activities to evaluate");
}

/// <summary>
/// Moves the enumerator to the next activity and returns it.
/// </summary>
/// <returns>The next activity</returns>
/// <exception cref="ActivityEnumeratorException">If there are no more activities available for evaluation</exception>
/// <exception cref="ActivityEnumeratorTypeMismatchException">If the returned activity is of a different type than requested</exception>
public TActivityType GetNext<TActivityType>() where TActivityType : PipelineActivity
{
if (_enumerator.MoveNext())
return _enumerator.Current as TActivityType ?? throw new ActivityEnumeratorTypeMismatchException($"Expected activity of type {typeof(TActivityType).Name} but found activity of type {_enumerator.Current?.GetType().Name}");

throw new ActivityEnumeratorException("No more activities to evaluate");
}
}
17 changes: 17 additions & 0 deletions src/AzureDataFactory.TestingFramework/Models/Base/RunParameter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,31 @@ public interface IRunParameter
}
public class RunParameter<TValueType> : IRunParameter
{
/// <summary>
/// Initializes a new instance of the <see cref="RunParameter{TValueType}"/> class. Can be passed to a pipeline evaluation run.
/// </summary>
/// <param name="type">The type of the parameter (e.g. parameter, global, dataset, LinkedService).</param>
/// <param name="name">The name of the parameter. Must match the name of the parameter in the pipeline being targeted.</param>
/// <param name="value">The value of the parameter</param>
public RunParameter(ParameterType type, string name, TValueType value)
{
Name = name;
Type = type;
Value = value;
}

/// <summary>
/// The name of the parameter. Must match the name of the parameter in the pipeline being targeted.
/// </summary>
public string Name { get; }

/// <summary>
/// The type of the parameter (e.g. parameter, global, dataset, LinkedService).
/// </summary>
public ParameterType Type { get; }

/// <summary>
/// The value of the parameter
/// </summary>
public TValueType Value { get; }
}
7 changes: 7 additions & 0 deletions src/AzureDataFactory.TestingFramework/Models/Base/RunState.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,17 @@ namespace AzureDataFactory.TestingFramework.Models.Base;

public class RunState
{
/// <summary>
/// Initializes a new instance of the <see cref="RunState"/> class. Can be used to purely evaluate an activity in a non-pipeline context.
/// </summary>
/// <param name="parameters">The name of the parameter. Must match the name of the parameter in the pipeline being targeted.</param>
public RunState(List<IRunParameter> parameters)
{
Parameters = parameters;
}

/// <summary>
/// The name of the parameter. Must match the name of the parameter in the pipeline being targeted.
/// </summary>
public List<IRunParameter> Parameters { get; }
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ namespace AzureDataFactory.TestingFramework.Models.Base;

public abstract class DataFactoryEntity
{
/// <summary>
/// Used to evaluate all the DataFactoryElements in the object graph.
/// </summary>
/// <param name="state">The pipeline state to be used for evaluating the expressions.</param>
/// <returns>Reference to itself.</returns>
public virtual DataFactoryEntity Evaluate(PipelineRunState state)
{
EvaluateProperties(this, state, new List<object>());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ namespace AzureDataFactory.TestingFramework.Models;

public partial class DataFactoryExpression
{
/// <summary>
/// Evaluates the expression by replacing all parameters and variables with their values and then evaluating the expression.
/// </summary>
/// <param name="state">The pipeline state to be used for evaluating the expressions.</param>
/// <typeparam name="TType">The type of the expression result. Can be string, bool, int and long.</typeparam>
/// <returns>The evaluated result of the expression.</returns>
public TType Evaluate<TType>(PipelineRunState state)
{
return FunctionPart.Parse(Value).Evaluate<TType>(state);
Expand Down
Loading

0 comments on commit 6d97c7e

Please sign in to comment.