diff --git a/ARM Templates/Data Factory/v1.8.6 Export.json b/ARM Templates/Data Factory/v1.8.6 Export.json new file mode 100644 index 00000000..fd56fdf2 --- /dev/null +++ b/ARM Templates/Data Factory/v1.8.6 Export.json @@ -0,0 +1,2788 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "factoryName": { + "type": "string", + "metadata": "Data Factory name", + "defaultValue": "" + }, + "FrameworkFunctions_properties_typeProperties_functionAppUrl": { + "type": "string", + "defaultValue": "" + }, + "Keys_properties_typeProperties_baseUrl": { + "type": "string", + "defaultValue": "" + }, + "SupportDatabase_properties_typeProperties_connectionString_secretName": { + "type": "string", + "defaultValue": "" + } + }, + "variables": { + "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" + }, + "resources": [ + { + "name": "[concat(parameters('factoryName'), '/01-Grandparent')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "ADF.procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", + "activities": [ + { + "name": "Framework Processing", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "02-Parent", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": {} + } + } + ], + "folder": { + "name": "_ProcFwk" + }, + "annotations": [ + "ADF.procfwk", + "Grandparent" + ] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/pipelines/02-Parent')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/02-Parent')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "ADF.procfwk parent pipeline used to bootstrap the orchestration framework in perform the first level ForEach calls in sequence for the metadata stages.", + "activities": [ + { + "name": "Get Stages", + "description": "Returns a distinct list of execution stages within the framework metadata.", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Set Execution Id", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[GetStages]", + "storedProcedureParameters": { + "ExecutionId": { + "type": "Guid", + "value": { + "value": "@variables('ExecutionId')", + "type": "Expression" + } + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + }, + { + "name": "Execute Stages", + "description": "Top level ForEach to sequentially call all processing stages within the framework metadata. Items for iteration passed from the Get Stages lookup activity.", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Stages", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Get Tenant Id", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Get Subscription Id", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Stages').output.value", + "type": "Expression" + }, + "isSequential": true, + "activities": [ + { + "name": "Stage Executor", + "description": "Call to the framework generic child pipeline for a given execution stage.", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Log Stage Preparing", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "03-Child", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "StageId": { + "value": "@item().StageId", + "type": "Expression" + }, + "ExecutionId": { + "value": "@variables('ExecutionId')", + "type": "Expression" + }, + "TenantId": { + "value": "@activity('Get Tenant Id').output.firstRow.PropertyValue", + "type": "Expression" + }, + "SubscriptionId": { + "value": "@activity('Get Subscription Id').output.firstRow.PropertyValue", + "type": "Expression" + } + } + } + }, + { + "name": "Log Stage Preparing", + "description": "Update the current execution table flagging all pipelines within the stage as preparing.", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Check and Update Blockers", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogStagePreparing]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@variables('ExecutionId')", + "type": "Expression" + }, + "type": "Guid" + }, + "StageId": { + "value": { + "value": "@item().StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + }, + { + "name": "Check and Update Blockers", + "description": "Used to double check and stop the next execution stage if failures and blockers have be incurred. This also depends on the failure handling property value which defines the stored procedure behaviour.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[CheckForBlockedPipelines]", + "storedProcedureParameters": { + "StageId": { + "value": { + "value": "@item().StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + } + }, + { + "name": "Execution Wrapper", + "description": "Wrapper to reset and restart processing or create a completely new execution instance of the framework metadata.", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Clean Up Previous Run", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[ExecutionWrapper]", + "storedProcedureParameters": { + "CallingDataFactory": { + "type": "String", + "value": { + "value": "@pipeline().DataFactory", + "type": "Expression" + } + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + } + } + }, + { + "name": "Check Outcome and Update Logs", + "description": "After a successful execution run the current execution metadata is moved to the long term logging table by this stored procedure call. Otherwise an error will be raised.", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Execute Stages", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[UpdateExecutionLog]", + "storedProcedureParameters": { + "PerformErrorCheck": { + "value": { + "value": "@bool(1)", + "type": "Expression" + }, + "type": "Boolean" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + }, + { + "name": "Get Tenant Id", + "description": "Returning the Azure Tenant Id from the metadata properties table.", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Metadata Integrity Checks", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", + "storedProcedureParameters": { + "PropertyName": { + "type": "String", + "value": "TenantId" + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": true + } + }, + { + "name": "Get Subscription Id", + "description": "Returning the Azure Subscription Id from the metadata properties table.", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Metadata Integrity Checks", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": true + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", + "storedProcedureParameters": { + "PropertyName": { + "type": "String", + "value": "SubscriptionId" + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + } + } + }, + { + "name": "Metadata Integrity Checks", + "description": "Performs a series of checks on all metadata held in the framework SQLDB. This is intended to raise errors before an execution run even starts.", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Execute Precursor", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[CheckMetadataIntegrity]", + "storedProcedureParameters": { + "DebugMode": { + "type": "Boolean", + "value": "false" + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + }, + { + "name": "Clean Up Previous Run", + "description": "Handle Worker pipelines that are reported as Running when the parent pipeline is called again. Get what the actual status of those pipelines is.", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Metadata Integrity Checks", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Metadata Integrity Checks').output.value", + "type": "Expression" + }, + "isSequential": false, + "batchCount": 20, + "activities": [ + { + "name": "Get SPN Details", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", + "storedProcedureParameters": { + "DataFactory": { + "type": "String", + "value": { + "value": "@item().DataFactoryName", + "type": "Expression" + } + }, + "PipelineName": { + "type": "String", + "value": { + "value": "@item().PipelineName", + "type": "Expression" + } + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + } + } + }, + { + "name": "Log Pipeline Checking", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineChecking]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@item().LocalExecutionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@item().PipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@item().StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + }, + { + "name": "Get Pipeline Status", + "type": "AzureFunctionActivity", + "dependsOn": [ + { + "activity": "Get SPN Details", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Log Pipeline Checking", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "CheckPipelineStatus", + "method": "POST", + "headers": {}, + "body": { + "value": "@concat('\n{\n \"tenantId\": \"',item().TenantId,'\",\n \"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n \"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n \"subscriptionId\": \"',item().SubscriptionId,'\",\n \"resourceGroup\": \"',item().ResourceGroupName,'\",\n \"factoryName\": \"',item().DataFactoryName,'\",\n \"pipelineName\": \"',item().PipelineName,'\",\n \"runId\": \"',item().AdfPipelineRunId,'\"\n}')", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "FrameworkFunctions", + "type": "LinkedServiceReference" + } + }, + { + "name": "Set Pipeline Status", + "description": "Update the metadata depending on the actual pipeline outcome. Using the status as the case.", + "type": "Switch", + "dependsOn": [ + { + "activity": "Get Pipeline Status", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "on": { + "value": "@activity('Get Pipeline Status').output.Status", + "type": "Expression" + }, + "cases": [ + { + "value": "Failed", + "activities": [ + { + "name": "Pipeline Status Failed", + "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@item().LocalExecutionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@item().PipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "RunId": { + "value": null, + "type": "Guid" + }, + "StageId": { + "value": { + "value": "@item().StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + }, + { + "value": "Succeeded", + "activities": [ + { + "name": "Pipeline Status Succeeded", + "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@item().LocalExecutionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@item().PipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@item().StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + }, + { + "value": "Queued", + "activities": [ + { + "name": "Pipeline Status Queued - Running", + "description": "Updates the current execution table with a pipeline status of running if the function outcome is queued.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@item().LocalExecutionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@item().PipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@item().StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + }, + { + "value": "InProgress", + "activities": [ + { + "name": "Pipeline Status InProgress - Running", + "description": "Updates the current execution table with a pipeline status of running if the function outcome is in progress.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@item().LocalExecutionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@item().PipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@item().StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + }, + { + "value": "Cancelled", + "activities": [ + { + "name": "Pipeline Status Cancelled", + "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@item().LocalExecutionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@item().PipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@item().StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + } + ], + "defaultActivities": [ + { + "name": "Pipeline Status Unknown", + "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@item().LocalExecutionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@item().PipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@item().StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + } + }, + { + "name": "Set Last Check DateTime", + "description": "Update the current execution table with a date time from when the function last checked the pipeline status.", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Get Pipeline Status", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@item().LocalExecutionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@item().PipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@item().StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + } + }, + { + "name": "Execute Precursor", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[ExecutePrecursorProcedure]" + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + }, + { + "name": "Set Execution Id", + "description": "Set the local execution Id to a pipeline variable for each in several downstream activities.", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Execution Wrapper", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "ExecutionId", + "value": { + "value": "@activity('Execution Wrapper').output.firstRow.ExecutionId", + "type": "Expression" + } + } + } + ], + "variables": { + "ExecutionId": { + "type": "String" + } + }, + "folder": { + "name": "_ProcFwk" + }, + "annotations": [ + "ADF.procfwk", + "Parent" + ] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", + "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", + "[concat(variables('factoryId'), '/pipelines/03-Child')]", + "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/03-Child')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "ADF.procfwk child pipeline used to execute Worker pipelines within a given execution stage. This pipeline will be called once for each stage, then execute all Workers in parallel.", + "activities": [ + { + "name": "Get Pipelines", + "description": "Returns all pipelines from the metadata to be executed within a given processing stage.", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelinesInStage]", + "storedProcedureParameters": { + "StageId": { + "type": "Int32", + "value": { + "value": "@pipeline().parameters.StageId", + "type": "Expression" + } + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": false + } + }, + { + "name": "Execute Pipelines", + "description": "Second level ForEach to run in parallel all pipelines within the stage. Items for iteration passed from the Get Pipelines lookup activity.", + "type": "ForEach", + "dependsOn": [ + { + "activity": "Get Pipelines", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "items": { + "value": "@activity('Get Pipelines').output.value", + "type": "Expression" + }, + "isSequential": false, + "batchCount": 50, + "activities": [ + { + "name": "Execute Pipeline", + "description": "The lowest level executor with the metadata framework to call existing processing pipelines within Data Factory. The function called will block processing and wait for an outcome.", + "type": "AzureFunctionActivity", + "dependsOn": [ + { + "activity": "Log Pipeline Running", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Get Pipeline Params", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Get SPN Details", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "ExecutePipeline", + "method": "POST", + "headers": {}, + "body": { + "value": "@concat('\n{\n\t\"tenantId\": \"',pipeline().parameters.TenantId,'\",\n\t\"applicationId\": \"',activity('Get SPN Details').output.firstRow.Id,'\",\n\t\"authenticationKey\": \"',activity('Get SPN Details').output.firstRow.Secret,'\",\n\t\"subscriptionId\": \"',pipeline().parameters.SubscriptionId,'\",\n\t\"resourceGroup\": \"',item().ResourceGroupName,'\",\n\t\"factoryName\": \"',item().DataFactoryName,'\",\n\t\"pipelineName\": \"',item().PipelineName,'\"',activity('Get Pipeline Params').output.firstRow.Params,'\n}')", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "FrameworkFunctions", + "type": "LinkedServiceReference" + } + }, + { + "name": "Get Pipeline Params", + "description": "Returns any parameters from metadata required for the processing pipeline being called. The output can be an empty string if no parameters are required.", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[GetPipelineParameters]", + "storedProcedureParameters": { + "PipelineId": { + "type": "Int32", + "value": { + "value": "@item().PipelineId", + "type": "Expression" + } + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + } + } + }, + { + "name": "Log Pipeline Running", + "description": "Sets the current pipeline with a status of running within the current execution database table.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineRunning]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@pipeline().parameters.ExecutionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@item().PipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@pipeline().parameters.StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + }, + { + "name": "Get SPN Details", + "description": "Return the SPN ID and Secret for the processing pipeline being executed. Called at this level as each pipeline can have a different SPN.", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[GetServicePrincipal]", + "storedProcedureParameters": { + "DataFactory": { + "type": "String", + "value": { + "value": "@item().DataFactoryName", + "type": "Expression" + } + }, + "PipelineName": { + "type": "String", + "value": { + "value": "@item().PipelineName", + "type": "Expression" + } + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + } + } + }, + { + "name": "Log Activity Failure", + "description": "Handle true failures from calling out to the Azure Function and update the current execution table accordingly so a restart can occur.", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Execute Pipeline", + "dependencyConditions": [ + "Failed" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@pipeline().parameters.ExecutionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@item().PipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@pipeline().parameters.StageId", + "type": "Expression" + }, + "type": "Int32" + }, + "CallingActivity": { + "value": "ExecutePipeline", + "type": "String" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + }, + { + "name": "Running Pipeline Handler", + "type": "ExecutePipeline", + "dependsOn": [ + { + "activity": "Execute Pipeline", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "04-Infant", + "type": "PipelineReference" + }, + "waitOnCompletion": true, + "parameters": { + "tenantId": { + "value": "@pipeline().parameters.TenantId", + "type": "Expression" + }, + "applicationId": { + "value": "@activity('Get SPN Details').output.firstRow.Id", + "type": "Expression" + }, + "authenticationKey": { + "value": "@activity('Get SPN Details').output.firstRow.Secret", + "type": "Expression" + }, + "subscriptionId": { + "value": "@pipeline().parameters.SubscriptionId", + "type": "Expression" + }, + "resourceGroup": { + "value": "@item().ResourceGroupName", + "type": "Expression" + }, + "factoryName": { + "value": "@item().DataFactoryName", + "type": "Expression" + }, + "pipelineName": { + "value": "@item().PipelineName", + "type": "Expression" + }, + "runId": { + "value": "@activity('Execute Pipeline').output.RunId", + "type": "Expression" + }, + "executionId": { + "value": "@pipeline().parameters.ExecutionId", + "type": "Expression" + }, + "stageId": { + "value": "@pipeline().parameters.StageId", + "type": "Expression" + }, + "pipelineId": { + "value": "@item().PipelineId", + "type": "Expression" + } + } + } + }, + { + "name": "Set Run Id", + "description": "Provide the actual ADF run ID back to the current execution table for long term logging and alignment between the metadata other Azure monitoring tools.", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Execute Pipeline", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineRunId]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@pipeline().parameters.ExecutionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@item().PipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "RunId": { + "value": { + "value": "@activity('Execute Pipeline').output.RunId", + "type": "Expression" + }, + "type": "Guid" + }, + "StageId": { + "value": { + "value": "@pipeline().parameters.StageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + }, + { + "name": "Check For Alerts", + "description": "Checks the properties tables and if any recipients in the database require alerts sending for the current pipeline ID.", + "type": "Lookup", + "dependsOn": [ + { + "activity": "Set Run Id", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Running Pipeline Handler", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[CheckForEmailAlerts]", + "storedProcedureParameters": { + "PipelineId": { + "type": "Int32", + "value": { + "value": "@item().PipelineId", + "type": "Expression" + } + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": true + } + }, + { + "name": "Send Alerts", + "description": "True = alerts need sending.\nFalse = do nothing.", + "type": "IfCondition", + "dependsOn": [ + { + "activity": "Check For Alerts", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "expression": { + "value": "@activity('Check For Alerts').output.firstRow.SendAlerts", + "type": "Expression" + }, + "ifTrueActivities": [ + { + "name": "Get Email Parts", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[GetEmailAlertParts]", + "storedProcedureParameters": { + "PipelineId": { + "type": "Int32", + "value": { + "value": "@item().PipelineId", + "type": "Expression" + } + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + }, + "firstRowOnly": true + } + }, + { + "name": "Send Email", + "type": "AzureFunctionActivity", + "dependsOn": [ + { + "activity": "Get Email Parts", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "SendEmail", + "method": "POST", + "headers": {}, + "body": { + "value": "@activity('Get Email Parts').output.firstRow", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "FrameworkFunctions", + "type": "LinkedServiceReference" + } + } + ] + } + } + ] + } + } + ], + "parameters": { + "StageId": { + "type": "int" + }, + "ExecutionId": { + "type": "string" + }, + "TenantId": { + "type": "string" + }, + "SubscriptionId": { + "type": "string" + } + }, + "folder": { + "name": "_ProcFwk" + }, + "annotations": [ + "ADF.procfwk", + "Child" + ] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", + "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", + "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]", + "[concat(variables('factoryId'), '/pipelines/04-Infant')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/04-Infant')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "ADF.procfwk infant pipeline used to check when the processing pipeline called by the Child completes and passes the resulting status back to the metadata database.", + "activities": [ + { + "name": "Wait Until Pipeline Completes", + "description": "Loops until the Worker pipeline called completes.\n\nSimple status:\n- Running = new iteration.\n- Done = break.", + "type": "Until", + "dependsOn": [ + { + "activity": "Get Wait Duration", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "expression": { + "value": "@variables('WorkerPipelineState')", + "type": "Expression" + }, + "activities": [ + { + "name": "Get Pipeline Status", + "description": "Checks the status of a given processing pipeline and provides the value for the downstream framework activities to act upon.", + "type": "AzureFunctionActivity", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "CheckPipelineStatus", + "method": "POST", + "headers": {}, + "body": { + "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "FrameworkFunctions", + "type": "LinkedServiceReference" + } + }, + { + "name": "Wait If Running", + "description": "True = Do nothing.\nFalse = Wait, before the next iteration.", + "type": "IfCondition", + "dependsOn": [ + { + "activity": "Set Worker State", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "expression": { + "value": "@variables('WorkerPipelineState')", + "type": "Expression" + }, + "ifFalseActivities": [ + { + "name": "Wait for Pipeline", + "description": "The processing pipeline is still running so Wait before checking its status again.", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@activity('Get Wait Duration').output.firstRow.PropertyValue", + "type": "Expression" + } + } + } + ] + } + }, + { + "name": "Set Last Check DateTime", + "description": "Update the current execution table with a date time from when the Worker pipeline status was last checked as part of the Until iterations.", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Get Pipeline Status", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineLastStatusCheck]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@pipeline().parameters.executionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@pipeline().parameters.pipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@pipeline().parameters.stageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + }, + { + "name": "Log Activity Failure", + "description": "Report to the current execution table that the framework pipeline activity has failed. This failure is outside of the scope of the framework and is probably related to a wider platform problem.", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Get Pipeline Status", + "dependencyConditions": [ + "Failed" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogActivityFailed]", + "storedProcedureParameters": { + "CallingActivity": { + "value": "GetPipelineStatus", + "type": "String" + }, + "ExecutionId": { + "value": { + "value": "@pipeline().parameters.executionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@pipeline().parameters.pipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@pipeline().parameters.stageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + }, + { + "name": "Set Worker State", + "description": "Set the bool state of the Worker pipeline to be used by the Until and If expressions. True = Complete, False = Running.", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Get Pipeline Status", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "WorkerPipelineState", + "value": { + "value": "@equals('Done',activity('Get Pipeline Status').output.SimpleStatus)", + "type": "Expression" + } + } + } + ], + "timeout": "7.00:00:00" + } + }, + { + "name": "Set Pipeline Result", + "description": "Receives the outcome from the function execution for a given processing pipeline and updates the current execution table with different pipelines status values depending on the result (case).", + "type": "Switch", + "dependsOn": [ + { + "activity": "Wait Until Pipeline Completes", + "dependencyConditions": [ + "Completed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "on": { + "value": "@activity('Get Pipeline Status').output.Status", + "type": "Expression" + }, + "cases": [ + { + "value": "Succeeded", + "activities": [ + { + "name": "Pipeline Status Succeeded", + "description": "Updates the current execution table with a pipeline status of success if the function outcome is succeeded.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineSuccess]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@pipeline().parameters.executionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@pipeline().parameters.pipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@pipeline().parameters.stageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + }, + { + "value": "Failed", + "activities": [ + { + "name": "Pipeline Status Failed", + "description": "Updates the current execution table with a pipeline status of failed if the function outcome is failed. Also blocks pipelines in the downstream execution stage.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineFailed]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@pipeline().parameters.executionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@pipeline().parameters.pipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "RunId": { + "value": { + "value": "@activity('Get Pipeline Status').output.RunId", + "type": "Expression" + }, + "type": "Guid" + }, + "StageId": { + "value": { + "value": "@pipeline().parameters.stageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + }, + { + "name": "Get Error Details", + "description": "Get the activity error details for the run ID of the worker pipeline called. Returns an array of all errors.", + "type": "AzureFunctionActivity", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "functionName": "GetActivityErrors", + "method": "POST", + "headers": {}, + "body": { + "value": "@concat('\n{\n \"tenantId\": \"',pipeline().parameters.TenantId,'\",\n \"applicationId\": \"',pipeline().parameters.applicationId,'\",\n \"authenticationKey\": \"',pipeline().parameters.authenticationKey,'\",\n \"subscriptionId\": \"',pipeline().parameters.subscriptionId,'\",\n \"resourceGroup\": \"',pipeline().parameters.resourceGroup,'\",\n \"factoryName\": \"',pipeline().parameters.factoryName,'\",\n \"pipelineName\": \"',pipeline().parameters.pipelineName,'\",\n \"runId\": \"',pipeline().parameters.runId,'\"\n}')", + "type": "Expression" + } + }, + "linkedServiceName": { + "referenceName": "FrameworkFunctions", + "type": "LinkedServiceReference" + } + }, + { + "name": "Log Error Details", + "description": "Parses pipeline error details and persists them to the metadata database error log table.", + "type": "SqlServerStoredProcedure", + "dependsOn": [ + { + "activity": "Get Error Details", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetErrorLogDetails]", + "storedProcedureParameters": { + "JsonErrorDetails": { + "value": { + "value": "@string(activity('Get Error Details').output)", + "type": "Expression" + }, + "type": "String" + }, + "LocalExecutionId": { + "value": { + "value": "@pipeline().parameters.executionId", + "type": "Expression" + }, + "type": "Guid" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + }, + { + "value": "Cancelled", + "activities": [ + { + "name": "Pipeline Status Cancelled", + "description": "Updates the current execution table with a pipeline status of cancelled if the function outcome is cancelled.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineCancelled]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@pipeline().parameters.executionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@pipeline().parameters.pipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@pipeline().parameters.stageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + } + ], + "defaultActivities": [ + { + "name": "Pipeline Status Unknown", + "description": "Updates the current execution table with a pipeline status of unknown if the function returns an unexpected outcome.", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[procfwk].[SetLogPipelineUnknown]", + "storedProcedureParameters": { + "ExecutionId": { + "value": { + "value": "@pipeline().parameters.executionId", + "type": "Expression" + }, + "type": "Guid" + }, + "PipelineId": { + "value": { + "value": "@pipeline().parameters.pipelineId", + "type": "Expression" + }, + "type": "Int32" + }, + "StageId": { + "value": { + "value": "@pipeline().parameters.stageId", + "type": "Expression" + }, + "type": "Int32" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + } + }, + { + "name": "Get Wait Duration", + "description": "Return wait duration in seconds from database properties table to be used during each Until iteration when the Worker pipeline is still running.", + "type": "Lookup", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "AzureSqlSource", + "sqlReaderStoredProcedureName": "[[procfwk].[GetPropertyValue]", + "storedProcedureParameters": { + "PropertyName": { + "type": "String", + "value": "PipelineStatusCheckDuration" + } + }, + "queryTimeout": "02:00:00", + "partitionOption": "None" + }, + "dataset": { + "referenceName": "GetSetMetadata", + "type": "DatasetReference", + "parameters": {} + } + } + } + ], + "parameters": { + "tenantId": { + "type": "string" + }, + "applicationId": { + "type": "string" + }, + "authenticationKey": { + "type": "string" + }, + "subscriptionId": { + "type": "string" + }, + "resourceGroup": { + "type": "string" + }, + "factoryName": { + "type": "string" + }, + "pipelineName": { + "type": "string" + }, + "runId": { + "type": "string" + }, + "executionId": { + "type": "string" + }, + "stageId": { + "type": "int" + }, + "pipelineId": { + "type": "int" + } + }, + "variables": { + "WorkerPipelineState": { + "type": "Boolean" + } + }, + "folder": { + "name": "_ProcFwk" + }, + "annotations": [ + "ADF.procfwk", + "Infant" + ] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/datasets/GetSetMetadata')]", + "[concat(variables('factoryId'), '/linkedServices/FrameworkFunctions')]", + "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/Intentional Error')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used just so the ADF.procfwk has something to call during development.", + "activities": [ + { + "name": "Wait1", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@pipeline().parameters.WaitTime", + "type": "Expression" + } + } + }, + { + "name": "Raise Errors or Not", + "type": "IfCondition", + "dependsOn": [ + { + "activity": "Wait1", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "userProperties": [], + "typeProperties": { + "expression": { + "value": "@equals(pipeline().parameters.RaiseErrors,'true')", + "type": "Expression" + }, + "ifTrueActivities": [ + { + "name": "Call Fail Procedure", + "type": "SqlServerStoredProcedure", + "dependsOn": [], + "policy": { + "timeout": "7.00:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "storedProcedureName": "[[dbo].[FailProcedure]", + "storedProcedureParameters": { + "RaiseError": { + "value": { + "value": "@pipeline().parameters.RaiseErrors", + "type": "Expression" + }, + "type": "String" + } + } + }, + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + } + } + ] + } + } + ], + "parameters": { + "RaiseErrors": { + "type": "string", + "defaultValue": "false" + }, + "WaitTime": { + "type": "int", + "defaultValue": 5 + } + }, + "folder": { + "name": "_Workers" + }, + "annotations": [ + "Worker" + ] + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 1')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used just so the ADF.procfwk has something to call during development.", + "activities": [ + { + "name": "Wait1", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@pipeline().parameters.WaitTime", + "type": "Expression" + } + } + } + ], + "parameters": { + "WaitTime": { + "type": "int", + "defaultValue": 5 + } + }, + "folder": { + "name": "_Workers" + }, + "annotations": [ + "Worker" + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 10')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used just so the ADF.procfwk has something to call during development.", + "activities": [ + { + "name": "Wait10", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@pipeline().parameters.WaitTime", + "type": "Expression" + } + } + } + ], + "parameters": { + "WaitTime": { + "type": "int", + "defaultValue": 5 + } + }, + "folder": { + "name": "_Workers" + }, + "annotations": [ + "Worker" + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 2')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used just so the ADF.procfwk has something to call during development.", + "activities": [ + { + "name": "Wait2", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@pipeline().parameters.WaitTime", + "type": "Expression" + } + } + } + ], + "parameters": { + "WaitTime": { + "type": "int", + "defaultValue": 5 + } + }, + "folder": { + "name": "_Workers" + }, + "annotations": [ + "Worker" + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 3')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used just so the ADF.procfwk has something to call during development.", + "activities": [ + { + "name": "Wait3", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@pipeline().parameters.WaitTime", + "type": "Expression" + } + } + } + ], + "parameters": { + "WaitTime": { + "type": "int", + "defaultValue": 5 + } + }, + "folder": { + "name": "_Workers" + }, + "annotations": [ + "Worker" + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 4')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used just so the ADF.procfwk has something to call during development.", + "activities": [ + { + "name": "Wait4", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@pipeline().parameters.WaitTime", + "type": "Expression" + } + } + } + ], + "parameters": { + "WaitTime": { + "type": "int", + "defaultValue": 5 + } + }, + "folder": { + "name": "_Workers" + }, + "annotations": [ + "Worker" + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 5')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used just so the ADF.procfwk has something to call during development.", + "activities": [ + { + "name": "Wait5", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@pipeline().parameters.WaitTime", + "type": "Expression" + } + } + } + ], + "parameters": { + "WaitTime": { + "type": "int", + "defaultValue": 5 + } + }, + "folder": { + "name": "_Workers" + }, + "annotations": [ + "Worker" + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 6')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used just so the ADF.procfwk has something to call during development.", + "activities": [ + { + "name": "Wait6", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@pipeline().parameters.WaitTime", + "type": "Expression" + } + } + } + ], + "parameters": { + "WaitTime": { + "type": "int", + "defaultValue": 5 + } + }, + "folder": { + "name": "_Workers" + }, + "annotations": [ + "Worker" + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 7')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used just so the ADF.procfwk has something to call during development.", + "activities": [ + { + "name": "Wait7", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@pipeline().parameters.WaitTime", + "type": "Expression" + } + } + } + ], + "parameters": { + "WaitTime": { + "type": "int", + "defaultValue": 5 + } + }, + "folder": { + "name": "_Workers" + }, + "annotations": [ + "Worker" + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 8')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used just so the ADF.procfwk has something to call during development.", + "activities": [ + { + "name": "Wait8", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@pipeline().parameters.WaitTime", + "type": "Expression" + } + } + } + ], + "parameters": { + "WaitTime": { + "type": "int", + "defaultValue": 5 + } + }, + "folder": { + "name": "_Workers" + }, + "annotations": [ + "Worker" + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/Wait 9')]", + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used just so the ADF.procfwk has something to call during development.", + "activities": [ + { + "name": "Wait9", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": { + "value": "@pipeline().parameters.WaitTime", + "type": "Expression" + } + } + } + ], + "parameters": { + "WaitTime": { + "type": "int", + "defaultValue": 15 + } + }, + "folder": { + "name": "_Workers" + }, + "annotations": [ + "Worker" + ] + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/GetSetMetadata')]", + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "properties": { + "linkedServiceName": { + "referenceName": "SupportDatabase", + "type": "LinkedServiceReference" + }, + "annotations": [ + "ADF.procfwk" + ], + "type": "AzureSqlTable", + "schema": [], + "typeProperties": {} + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/SupportDatabase')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/FrameworkFunctions')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "annotations": [ + "ADF.procfwk" + ], + "type": "AzureFunction", + "typeProperties": { + "functionAppUrl": "[parameters('FrameworkFunctions_properties_typeProperties_functionAppUrl')]", + "functionKey": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "Keys", + "type": "LinkedServiceReference" + }, + "secretName": "FrameworkFunctionsKey" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/Keys')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/Keys')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework.", + "annotations": [ + "ADF.procfwk" + ], + "type": "AzureKeyVault", + "typeProperties": { + "baseUrl": "[parameters('Keys_properties_typeProperties_baseUrl')]" + } + }, + "dependsOn": [] + }, + { + "name": "[concat(parameters('factoryName'), '/SupportDatabase')]", + "type": "Microsoft.DataFactory/factories/linkedServices", + "apiVersion": "2018-06-01", + "properties": { + "description": "Connection between ADF and processing framework metadata SQLDB.", + "annotations": [ + "ADF.procfwk" + ], + "type": "AzureSqlDatabase", + "typeProperties": { + "connectionString": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "Keys", + "type": "LinkedServiceReference" + }, + "secretName": "[parameters('SupportDatabase_properties_typeProperties_connectionString_secretName')]" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/linkedServices/Keys')]" + ] + }, + { + "name": "[concat(parameters('factoryName'), '/FunctionalTestingTrigger')]", + "type": "Microsoft.DataFactory/factories/triggers", + "apiVersion": "2018-06-01", + "properties": { + "description": "Used for functional testing of the framework in a dedicated environment.", + "annotations": [ + "ADF.procfwk" + ], + "runtimeState": "Stopped", + "pipelines": [ + { + "pipelineReference": { + "referenceName": "01-Grandparent", + "type": "PipelineReference" + }, + "parameters": {} + } + ], + "type": "ScheduleTrigger", + "typeProperties": { + "recurrence": { + "frequency": "Hour", + "interval": 2, + "startTime": "2020-04-06T15:00:00.000Z", + "timeZone": "UTC" + } + } + }, + "dependsOn": [ + "[concat(variables('factoryId'), '/pipelines/01-Grandparent')]" + ] + } + ] +} \ No newline at end of file diff --git a/Images/ADFprocfwk Designs.vsdx b/Images/ADFprocfwk Designs.vsdx index b50bcdcf..3f2c4afd 100644 Binary files a/Images/ADFprocfwk Designs.vsdx and b/Images/ADFprocfwk Designs.vsdx differ diff --git a/Images/All ADF Activities.pptx b/Images/All ADF Activities.pptx index 1d5a1b21..776f8632 100644 Binary files a/Images/All ADF Activities.pptx and b/Images/All ADF Activities.pptx differ diff --git a/ProcessingFramework.sln b/ProcessingFramework.sln index 83ab707d..8505da84 100644 --- a/ProcessingFramework.sln +++ b/ProcessingFramework.sln @@ -62,6 +62,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Workers", "Workers", "{0414 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Images", "Images", "{16E32CF7-361C-4A58-B0E1-5C8C46181ADE}" ProjectSection(SolutionItems) = preProject + Images\Activity Chain.png = Images\Activity Chain.png Images\ADFprocfwk Designs.vsdx = Images\ADFprocfwk Designs.vsdx Images\ADFprocfwk Icon - No Background.png = Images\ADFprocfwk Icon - No Background.png Images\ADFprocfwk Icon.png = Images\ADFprocfwk Icon.png @@ -93,6 +94,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataFactory", "DataFactory" ARM Templates\Data Factory\v1.8 Export.json = ARM Templates\Data Factory\v1.8 Export.json ARM Templates\Data Factory\v1.8.3 Export.json = ARM Templates\Data Factory\v1.8.3 Export.json ARM Templates\Data Factory\v1.8.5 Export.json = ARM Templates\Data Factory\v1.8.5 Export.json + ARM Templates\Data Factory\v1.8.6 Export.json = ARM Templates\Data Factory\v1.8.6 Export.json EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "trigger", "trigger", "{FC7877EF-110F-4AD7-B203-EEECE6F08A86}" diff --git a/README.md b/README.md index 138a6d2e..8349095f 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The framework is designed to integrate with any existing Data Factory solution b This framework can also be used in any Azure Tenant and allow the creation of complex control flows across multiple Data Factory resources by connecting Service Principal details through metadata to targeted Subscriptions > Resource Groups > Data Factory's and Pipelines, this offers very granular administration over data processing components in a given environment. - ## Framework Features +## Framework Features * Granular metadata control. * Metadata integrity checking. @@ -32,7 +32,11 @@ This framework can also be used in any Azure Tenant and allow the creation of co [ADFprocfwk.com](http://ADFprocfwk.com/) - ## Contributors +## Complete Data Factory Activity Chain + +![alt text](https://mrpaulandrew.files.wordpress.com/2020/09/activity-chain.png "ADF.procfwk Icon") + +## Contributors | Who | Details | |------------|-------------| @@ -68,6 +72,7 @@ Go to... [Glossary](https://github.com/mrpaulandrew/ADF.procfwk/blob/master/Glos | Version | Overview | Related Blog(s) & Version Details | |:----:|--------------|--------| +| 1.8.6 |Pipeline Expressions Refactored to Use Variables added, plus: |Issues:
[ADF.procfwk #51](https://github.com/mrpaulandrew/ADF.procfwk/issues/51)
[ADF.procfwk #52](https://github.com/mrpaulandrew/ADF.procfwk/issues/52) | | 1.8.5 |Execution Precursor added, plus: |[ADF.procfwk v1.8.5 - Execution Precursor](https://mrpaulandrew.com/2020/08/17/adf-procfwk-v1-8-5-execution-precursor/) | | 1.8.4 |Database Schema Reorganise and Restructuring |[ADF.procfwk v1.8.4 - Database Schema Reorganise and Restructuring](https://mrpaulandrew.com/2020/07/23/adf-procfwk-v1-8-4-database-schema-reorganise-and-restructuring/) | | 1.8.3 |Bug Fixes from the Community, including: |Issues:
[ADF.procfwk #38](https://github.com/mrpaulandrew/ADF.procfwk/issues/38)
[ADF.procfwk #37](https://github.com/mrpaulandrew/ADF.procfwk/issues/37) |