Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pv2 3401 archiving job check status in adf pipeline to prevent double triggering #1225

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Pv2 3401 archiving job check status in adf pipeline to prevent double triggering #1225

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8659aa4
Added new column to record when the archiving job starts
abroodie Sep 18, 2024
fb008ab
Added new json file
PravPattisapu Sep 18, 2024
6b0801f
Merge branch 'master' into PV2-3401_Archiving-Job---Check-Status-in-A…
abroodie Nov 27, 2024
d45fbe0
New Json file for 3401
PravPattisapu Nov 28, 2024
55f079d
added new table to record history of adf pipeline
abroodie Dec 10, 2024
afd7bce
Added Failed column to the AuditMigrationJob table
abroodie Dec 11, 2024
7f80249
Added new json file for 3401
PravPattisapu Dec 12, 2024
bad895f
Merge branch 'PV2-3401_Archiving-Job---Check-Status-in-ADF-pipeline-t…
PravPattisapu Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 153 additions & 42 deletions Infrastructure/SFA.DAS.Payments.ADF/DCOL-DAS-DataFactoryDAS-WEU/ARMTemplateForFactory.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,42 @@
"[concat(variables('factoryId'), '/linkedServices/datamart_stagingDB_linkedservices')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/default')]",
"type": "Microsoft.DataFactory/factories/managedVirtualNetworks",
"apiVersion": "2018-06-01",
"properties": {},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/AzureVNETIntegrationRuntime')]",
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
"apiVersion": "2018-06-01",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "West Europe",
"dataFlowProperties": {
"computeType": "General",
"coreCount": 8,
"timeToLive": 10,
"cleanup": false
},
"pipelineExternalComputeScaleProperties": {
"timeToLive": 60
}
}
},
"managedVirtualNetwork": {
"type": "ManagedVirtualNetworkReference",
"referenceName": "default"
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/managedVirtualNetworks/default')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/AzureKeyVault1')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
Expand Down Expand Up @@ -249,42 +285,6 @@
"[concat(variables('factoryId'), '/integrationRuntimes/AzureVNETIntegrationRuntime')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/AzureVNETIntegrationRuntime')]",
"type": "Microsoft.DataFactory/factories/integrationRuntimes",
"apiVersion": "2018-06-01",
"properties": {
"type": "Managed",
"typeProperties": {
"computeProperties": {
"location": "West Europe",
"dataFlowProperties": {
"computeType": "General",
"coreCount": 8,
"timeToLive": 10,
"cleanup": false
},
"pipelineExternalComputeScaleProperties": {
"timeToLive": 60
}
}
},
"managedVirtualNetwork": {
"type": "ManagedVirtualNetworkReference",
"referenceName": "default"
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/managedVirtualNetworks/default')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/default')]",
"type": "Microsoft.DataFactory/factories/managedVirtualNetworks",
"apiVersion": "2018-06-01",
"properties": {},
"dependsOn": []
},
{
"name": "[concat(parameters('factoryName'), '/CopyPaymentsToArchive')]",
"type": "Microsoft.DataFactory/factories/pipelines",
Expand All @@ -294,7 +294,14 @@
{
"name": "GetAcYrPeriodToCopy",
"type": "Lookup",
"dependsOn": [],
"dependsOn": [
{
"activity": "Set-AuditMigrationStartDate",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
Expand All @@ -307,7 +314,7 @@
"source": {
"type": "AzureSqlSource",
"sqlReaderQuery": {
"value": "IF (CAST(@{pipeline().parameters.IsManualRun} as bit) != CAST(1 as bit))\nBEGIN\n SELECT AcademicYear,Period\n FROM payments2.CollectionPeriod\n WHERE AuditMigrationCompletionDate is null;\nEnd\nElse\nBEGIN\n SELECT @{pipeline().parameters.AcademicYear} as AcademicYear, @{pipeline().parameters.CollectionPeriod} AS Period\nEND",
"value": "SELECT @{pipeline().parameters.AcademicYear} as AcademicYear, @{pipeline().parameters.CollectionPeriod} AS Period\n",
"type": "Expression"
},
"queryTimeout": "02:00:00",
Expand All @@ -317,7 +324,7 @@
"referenceName": "SrcPayments",
"type": "DatasetReference",
"parameters": {
"Table_Name": "collectionperiod"
"Table_Name": "AuditMigrationJob"
}
},
"firstRowOnly": false
Expand Down Expand Up @@ -4714,7 +4721,7 @@
{
"type": "NonQuery",
"text": {
"value": "IF (CAST(@{pipeline().parameters.IsManualRun} as bit) != CAST(1 as bit))\nBEGIN\n UPDATE [payments2].[collectionperiod]\n SET [AuditMigrationCompletionDate]=getdate()\n WHERE [AcademicYear]='@{item().AcademicYear}'\n AND [Period]='@{item().Period}'\nEND",
"value": "UPDATE [payments2].[AuditMigrationJob]\n SET [CompletionDate]=sysutcdatetime()\n WHERE [AcademicYear]='@{item().AcademicYear}'\n AND [Period]='@{item().Period}'\n AND [CompletionDate] is null\n",
"type": "Expression"
}
}
Expand Down Expand Up @@ -6243,8 +6250,112 @@
}
]
}
},
{
"name": "GetPreviousRun",
"type": "Lookup",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "AzureSqlSource",
"sqlReaderQuery": {
"value": "SELECT \n @{pipeline().parameters.AcademicYear} as AcademicYear, \n @{pipeline().parameters.CollectionPeriod} AS Period\n Where not exists (\n select \n 1 \n FROM [Payments2].[AuditMigrationJob] \n Where AcademicYear = @{pipeline().parameters.AcademicYear} \n and [Period] = @{pipeline().parameters.CollectionPeriod}\n and Failed = 0\n )\n\n",
"type": "Expression"
},
"queryTimeout": "02:00:00",
"partitionOption": "None"
},
"dataset": {
"referenceName": "SrcPayments",
"type": "DatasetReference",
"parameters": {
"Table_Name": "AuditMigrationJob"
}
},
"firstRowOnly": true
}
},
{
"name": "ForEachPreviousAuditJob",
"type": "ForEach",
"dependsOn": [
{
"activity": "GetPreviousRun",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"typeProperties": {
"items": {
"value": "@activity('GetPreviousRun').output.value",
"type": "Expression"
},
"isSequential": true,
"activities": [
{
"name": "JobAlreadyRunFailure",
"type": "Fail",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"message": {
"value": "The Audit Migration Job for '@{pipeline().parameters.AcademicYear}-@{pipeline().parameters.CollectionPeriod}' is currently running or has already completed.",
"type": "Expression"
},
"errorCode": "500"
}
}
]
}
},
{
"name": "Set-AuditMigrationStartDate",
"type": "Script",
"dependsOn": [
{
"activity": "ForEachPreviousAuditJob",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"linkedServiceName": {
"referenceName": "DASPayments",
"type": "LinkedServiceReference"
},
"typeProperties": {
"scripts": [
{
"type": "NonQuery",
"text": {
"value": "INSERT INTO [Payments2].[AuditMigrationJob]\n (AcademicYear,[Period],StartDate)\n VALUES (@{pipeline().parameters.AcademicYear},@{pipeline().parameters.CollectionPeriod},sysutcdatetime())",
"type": "Expression"
}
}
],
"scriptBlockExecutionTimeout": "02:00:00"
}
}
],
"concurrency": 1,
"policy": {
"elapsedTimeMetric": {}
},
Expand All @@ -6263,7 +6374,7 @@
}
},
"annotations": [],
"lastPublishTime": "2024-10-29T12:51:54Z"
"lastPublishTime": "2024-12-12T09:03:25Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/datasets/SrcPayments')]",
Expand Down Expand Up @@ -7521,7 +7632,7 @@
}
},
"annotations": [],
"lastPublishTime": "2024-10-29T12:51:53Z"
"lastPublishTime": "2024-12-10T16:54:37Z"
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/datamart_stagingDB_linkedservices')]",
Expand Down
19 changes: 19 additions & 0 deletions src/SFA.DAS.Payments.Database/Payments2/Tables/Payments2.AuditMigrationJob.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
CREATE TABLE [Payments2].[AuditMigrationJob]
(
[Id] INT NOT NULL IDENTITY(1,1) CONSTRAINT PK_AuditArchiveJob PRIMARY KEY CLUSTERED,
[AcademicYear] SMALLINT NOT NULL,
[Period] TINYINT NOT NULL,
[StartDate] DATETIME2 NULL,
[CompletionDate] DATETIME2 NULL,
[Failed] BIT NOT NULL CONSTRAINT DF_AuditMigrationJob__Failed DEFAULT (0),
[CreationDate] DATETIME2 NOT NULL CONSTRAINT DF_AuditMigrationJob__CreationDate DEFAULT sysutcdatetime()
)
GO

CREATE NONCLUSTERED INDEX [IX_AuditMigrationJob__AcademicYear_Period] on [Payments2].[AuditMigrationJob]
(
[AcademicYear],
[Period]
)
WITH (ONLINE = ON)
GO
1 change: 1 addition & 0 deletions src/SFA.DAS.Payments.Database/Payments2/Tables/Payments2.CollectionPeriod.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
[Period] TINYINT NOT NULL,
[ReferenceDataValidationDate] DATETIME2 NULL,
[CompletionDate] DATETIME2 NOT NULL,
[AuditMigrationStartDate] DATETIME2 NULL,
[AuditMigrationCompletionDate] DATETIME2 NULL,
[CreationDate] DATETIME2 NOT NULL DEFAULT sysutcdatetime()
)
1 change: 1 addition & 0 deletions src/SFA.DAS.Payments.Database/SFA.DAS.Payments.Database.sqlproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@
<Build Include="dbo\Stored Procedures\dbo.usp_DisplayDeploymentProperties.sql" />
<Build Include="Payments2\Tables\Payments2.LevyAccountAudit.sql" />
<Build Include="Payments2\Tables\Payments2.FundingPlatformType.sql" />
<Build Include="Payments2\Tables\Payments2.AuditMigrationJob.sql" />
</ItemGroup>
<ItemGroup>
<PostDeploy Include="zzz.Script.PostDeployment.sql" />
Expand Down
4 changes: 4 additions & 0 deletions src/SFA.DAS.Payments.sln.DotSettings
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateStaticFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PublicFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/UserRules/=53eecf85_002Dd821_002D40e8_002Dac97_002Dfdb734542b84/@EntryIndexedValue">&lt;Policy&gt;&lt;Descriptor Staticness="Instance" AccessRightKinds="Protected, ProtectedInternal, Internal, Public, PrivateProtected" Description="Instance fields (not private)"&gt;&lt;ElementKinds&gt;&lt;Kind Name="FIELD" /&gt;&lt;Kind Name="READONLY_FIELD" /&gt;&lt;/ElementKinds&gt;&lt;/Descriptor&gt;&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;&lt;/Policy&gt;</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/UserRules/=70345118_002D4b40_002D4ece_002D937c_002Dbbeb7a0b2e70/@EntryIndexedValue">&lt;Policy&gt;&lt;Descriptor Staticness="Static" AccessRightKinds="Protected, ProtectedInternal, Internal, Public, PrivateProtected" Description="Static fields (not private)"&gt;&lt;ElementKinds&gt;&lt;Kind Name="FIELD" /&gt;&lt;/ElementKinds&gt;&lt;/Descriptor&gt;&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;&lt;/Policy&gt;</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/UserRules/=f9fce829_002De6f4_002D4cb2_002D80f1_002D5497c44f51df/@EntryIndexedValue">&lt;Policy&gt;&lt;Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"&gt;&lt;ElementKinds&gt;&lt;Kind Name="FIELD" /&gt;&lt;/ElementKinds&gt;&lt;/Descriptor&gt;&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;&lt;/Policy&gt;</s:String>
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EPredefinedNamingRulesToUserRulesUpgrade/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=automapper/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=clawback/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=deserialise/@EntryIndexedValue">True</s:Boolean>
Expand Down