[CES-157] - Implement tooling for storage account migration from WEU …

…to ITN (#1238) Co-authored-by: ccuffari <[email protected]> Co-authored-by: christian-calabrese <[email protected]> Co-authored-by: Mario Mupo <[email protected]>
pagopa · Nov 6, 2024 · 911e597 · 911e597
1 parent 55cf892
commit 911e597
Show file tree

Hide file tree

Showing 19 changed files with 638 additions and 0 deletions.
diff --git a/src/_modules/data_factory_storage_account/data.tf b/src/_modules/data_factory_storage_account/data.tf
@@ -0,0 +1,20 @@
+data "azurerm_storage_account" "source" {
+  name                = var.storage_accounts.source.name
+  resource_group_name = var.storage_accounts.source.resource_group_name
+}
+
+data "azurerm_storage_account" "target" {
+  name                = var.storage_accounts.target.name
+  resource_group_name = var.storage_accounts.target.resource_group_name
+}
+
+data "azurerm_storage_containers" "this" {
+  count              = var.what_to_migrate.blob.enabled && length(var.what_to_migrate.blob.containers) == 0 ? 1 : 0
+  storage_account_id = data.azurerm_storage_account.source.id
+}
+
+data "azapi_resource_list" "tables" {
+  type                   = "Microsoft.Storage/storageAccounts/tableServices/tables@2021-09-01"
+  parent_id              = "${data.azurerm_storage_account.source.id}/tableServices/default"
+  response_export_values = ["*"]
+}
diff --git a/src/_modules/data_factory_storage_account/datasets_containers.tf b/src/_modules/data_factory_storage_account/datasets_containers.tf
@@ -0,0 +1,41 @@
+resource "azurerm_data_factory_custom_dataset" "source_dataset_container" {
+  for_each        = toset(local.containers)
+  name            = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}", "/[$-]/", "_")
+  data_factory_id = var.data_factory_id
+  type            = "AzureBlob"
+  folder          = "${var.storage_accounts.source.name}/source/blob"
+
+  linked_service {
+    name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob[0].name
+  }
+
+  type_properties_json = jsonencode({
+    linkedServiceName = {
+      referenceName = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob[0].name
+      type          = "LinkedServiceReference"
+    }
+    type       = "AzureBlob"
+    folderPath = each.value
+  })
+}
+
+resource "azurerm_data_factory_custom_dataset" "target_dataset_container" {
+  for_each        = toset(local.containers)
+  name            = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-blob-${module.naming_convention.suffix}", "/[$-]/", "_")
+  data_factory_id = var.data_factory_id
+  type            = "AzureBlob"
+  folder          = "${var.storage_accounts.source.name}/target/blob"
+
+  linked_service {
+    name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob[0].name
+  }
+
+  type_properties_json = jsonencode({
+    linkedServiceName = {
+      referenceName = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob[0].name
+      type          = "LinkedServiceReference"
+    }
+    type       = "AzureBlob"
+    folderPath = each.value
+  })
+}
diff --git a/src/_modules/data_factory_storage_account/datasets_tables.tf b/src/_modules/data_factory_storage_account/datasets_tables.tf
@@ -0,0 +1,30 @@
+resource "azurerm_data_factory_custom_dataset" "source_dataset_table" {
+  for_each        = toset(local.tables)
+  name            = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}", "/[$-]/", "_")
+  data_factory_id = var.data_factory_id
+  type            = "AzureTable"
+  folder          = "${var.storage_accounts.source.name}/source/table"
+
+  linked_service {
+    name = azurerm_data_factory_linked_service_azure_table_storage.source_linked_service_table[0].name
+  }
+
+  type_properties_json = jsonencode({
+    tableName = each.value
+  })
+}
+
+resource "azurerm_data_factory_custom_dataset" "target_dataset_table" {
+  for_each        = toset(local.tables)
+  name            = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value}-table-${module.naming_convention.suffix}", "/[$-]/", "_")
+  data_factory_id = var.data_factory_id
+  type            = "AzureTable"
+  folder          = "${var.storage_accounts.source.name}/target/table"
+  linked_service {
+    name = azurerm_data_factory_linked_service_azure_table_storage.target_linked_service_table[0].name
+  }
+
+  type_properties_json = jsonencode({
+    tableName = each.value
+  })
+}
diff --git a/src/_modules/data_factory_storage_account/iam.tf b/src/_modules/data_factory_storage_account/iam.tf
@@ -0,0 +1,40 @@
+module "roles" {
+  source       = "github.com/pagopa/dx//infra/modules/azure_role_assignments?ref=main"
+  principal_id = var.data_factory_principal_id
+
+  storage_blob = var.what_to_migrate.blob.enabled ? [
+    {
+      storage_account_name = var.storage_accounts.source.name
+      resource_group_name  = var.storage_accounts.source.resource_group_name
+      role                 = "reader"
+    },
+    {
+      storage_account_name = var.storage_accounts.target.name
+      resource_group_name  = var.storage_accounts.target.resource_group_name
+      role                 = "writer"
+    }
+  ] : []
+
+  # ADF terraform resources still force to use connection strings for tables
+  # but it's possible to switch to managed identities from the portal
+  storage_table = var.what_to_migrate.table.enabled ? [
+    {
+      storage_account_name = var.storage_accounts.source.name
+      resource_group_name  = var.storage_accounts.source.resource_group_name
+      role                 = "reader"
+    },
+    {
+      storage_account_name = var.storage_accounts.target.name
+      resource_group_name  = var.storage_accounts.target.resource_group_name
+      role                 = "writer"
+    }
+  ] : []
+}
+
+# Permission needed to allow Data Factory to create tables in the target storage account
+resource "azurerm_role_assignment" "storage_account_contributor" {
+  count                = var.what_to_migrate.table.enabled ? 1 : 0
+  scope                = data.azurerm_storage_account.target.id
+  role_definition_name = "Storage Account Contributor"
+  principal_id         = var.data_factory_principal_id
+}
diff --git a/src/_modules/data_factory_storage_account/linked_services_containers.tf b/src/_modules/data_factory_storage_account/linked_services_containers.tf
@@ -0,0 +1,19 @@
+resource "azurerm_data_factory_linked_service_azure_blob_storage" "source_linked_service_blob" {
+  count           = var.what_to_migrate.blob.enabled ? 1 : 0
+  name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-blob-${module.naming_convention.suffix}"
+  data_factory_id = var.data_factory_id
+
+  service_endpoint = "https://${data.azurerm_storage_account.source.name}.blob.core.windows.net"
+
+  use_managed_identity = true
+}
+
+resource "azurerm_data_factory_linked_service_azure_blob_storage" "target_linked_service_blob" {
+  count           = var.what_to_migrate.blob.enabled ? 1 : 0
+  name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-blob-${module.naming_convention.suffix}"
+  data_factory_id = var.data_factory_id
+
+  service_endpoint = "https://${data.azurerm_storage_account.target.name}.blob.core.windows.net"
+
+  use_managed_identity = true
+}
diff --git a/src/_modules/data_factory_storage_account/linked_services_tables.tf b/src/_modules/data_factory_storage_account/linked_services_tables.tf
@@ -0,0 +1,15 @@
+resource "azurerm_data_factory_linked_service_azure_table_storage" "source_linked_service_table" {
+  count           = var.what_to_migrate.table.enabled ? 1 : 0
+  name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-table-${module.naming_convention.suffix}"
+  data_factory_id = var.data_factory_id
+
+  connection_string = data.azurerm_storage_account.source.primary_connection_string
+}
+
+resource "azurerm_data_factory_linked_service_azure_table_storage" "target_linked_service_table" {
+  count           = var.what_to_migrate.table.enabled ? 1 : 0
+  name            = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-table-${module.naming_convention.suffix}"
+  data_factory_id = var.data_factory_id
+
+  connection_string = data.azurerm_storage_account.target.primary_connection_string
+}
diff --git a/src/_modules/data_factory_storage_account/locals.tf b/src/_modules/data_factory_storage_account/locals.tf
@@ -0,0 +1,6 @@
+locals {
+  containers = var.what_to_migrate.blob.enabled ? (length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name]) : []
+
+  azapi_tables = jsondecode(data.azapi_resource_list.tables.output)
+  tables       = var.what_to_migrate.table.enabled ? (length(var.what_to_migrate.table.tables) > 0 ? var.what_to_migrate.table.tables : [for table in local.azapi_tables.value : table.name]) : []
+}
diff --git a/src/_modules/data_factory_storage_account/main.tf b/src/_modules/data_factory_storage_account/main.tf
@@ -0,0 +1,21 @@
+terraform {
+  required_providers {
+    azapi = {
+      source  = "Azure/azapi"
+      version = "<= 1.15.0"
+    }
+  }
+}
+
+module "naming_convention" {
+  source = "github.com/pagopa/dx//infra/modules/azure_naming_convention/?ref=main"
+
+  environment = {
+    prefix          = var.environment.prefix
+    env_short       = var.environment.env_short
+    location        = var.environment.location
+    domain          = var.environment.domain
+    app_name        = var.environment.app_name
+    instance_number = var.environment.instance_number
+  }
+}
diff --git a/src/_modules/data_factory_storage_account/network.tf b/src/_modules/data_factory_storage_account/network.tf
@@ -0,0 +1,31 @@
+resource "azurerm_data_factory_managed_private_endpoint" "blob_source" {
+  count              = var.what_to_migrate.blob.enabled ? 1 : 0
+  name               = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-blob-${module.naming_convention.suffix}"
+  data_factory_id    = var.data_factory_id
+  target_resource_id = data.azurerm_storage_account.source.id
+  subresource_name   = "blob"
+}
+
+resource "azurerm_data_factory_managed_private_endpoint" "blob_target" {
+  count              = var.what_to_migrate.blob.enabled ? 1 : 0
+  name               = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-blob-${module.naming_convention.suffix}"
+  data_factory_id    = var.data_factory_id
+  target_resource_id = data.azurerm_storage_account.target.id
+  subresource_name   = "blob"
+}
+
+resource "azurerm_data_factory_managed_private_endpoint" "table_source" {
+  count              = var.what_to_migrate.table.enabled ? 1 : 0
+  name               = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-table-${module.naming_convention.suffix}"
+  data_factory_id    = var.data_factory_id
+  target_resource_id = data.azurerm_storage_account.source.id
+  subresource_name   = "table"
+}
+
+resource "azurerm_data_factory_managed_private_endpoint" "table_target" {
+  count              = var.what_to_migrate.table.enabled ? 1 : 0
+  name               = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-table-${module.naming_convention.suffix}"
+  data_factory_id    = var.data_factory_id
+  target_resource_id = data.azurerm_storage_account.target.id
+  subresource_name   = "table"
+}
diff --git a/src/_modules/data_factory_storage_account/outputs.tf b/src/_modules/data_factory_storage_account/outputs.tf
@@ -0,0 +1,10 @@
+output "pipelines" {
+  value = {
+    for pipeline in merge(azurerm_data_factory_pipeline.pipeline_container, azurerm_data_factory_pipeline.pipeline_table)
+    : pipeline.name => {
+      id   = pipeline.id
+      name = pipeline.name
+      url  = "https://adf.azure.com/en/authoring/pipeline/${pipeline.name}?factory=${pipeline.data_factory_id}"
+    }
+  }
+}
diff --git a/src/_modules/data_factory_storage_account/pipeline_containers.tf b/src/_modules/data_factory_storage_account/pipeline_containers.tf
@@ -0,0 +1,60 @@
+resource "azurerm_data_factory_pipeline" "pipeline_container" {
+  for_each        = toset(local.containers)
+  name            = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-blob-${module.naming_convention.suffix}", "/[$-]/", "_")
+  data_factory_id = var.data_factory_id
+  folder          = "${var.storage_accounts.source.name}/blob"
+
+  activities_json = jsonencode(
+    [
+      {
+        name      = "CopyActivity"
+        type      = "Copy"
+        dependsOn = []
+        policy = {
+          timeout                = "0.12:00:00"
+          retry                  = 0
+          retryIntervalInSeconds = 30
+          secureOutput           = false
+          secureInput            = false
+        }
+        userProperties = []
+        typeProperties = {
+          source = {
+            type = "BinarySource"
+            storeSettings = {
+              type                     = "AzureBlobStorageReadSettings"
+              recursive                = true
+              enablePartitionDiscovery = false
+              wildcardFileName         = "*" # Copy all files
+            }
+            formatSettings = {
+              type = ""
+            }
+          }
+          sink = {
+            type = "BinarySink"
+            storeSettings = {
+              type = "AzureBlobStorageWriteSettings"
+            }
+            formatSettings = {
+              type = ""
+            }
+          }
+          enableStaging = false
+        }
+        inputs = [
+          {
+            referenceName = azurerm_data_factory_custom_dataset.source_dataset_container[each.value].name
+            type          = "DatasetReference"
+          }
+        ]
+        outputs = [
+          {
+            referenceName = azurerm_data_factory_custom_dataset.target_dataset_container[each.value].name
+            type          = "DatasetReference"
+          }
+        ]
+      }
+    ]
+  )
+}
diff --git a/src/_modules/data_factory_storage_account/pipeline_tables.tf b/src/_modules/data_factory_storage_account/pipeline_tables.tf
@@ -0,0 +1,58 @@
+resource "azurerm_data_factory_pipeline" "pipeline_table" {
+  for_each        = toset(local.tables)
+  name            = replace("${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value}-table-${module.naming_convention.suffix}", "/[$-]/", "_")
+  data_factory_id = var.data_factory_id
+  folder          = "${var.storage_accounts.source.name}/table"
+
+  activities_json = jsonencode(
+    [
+      {
+        name      = "CopyActivity"
+        type      = "Copy"
+        dependsOn = []
+        policy = {
+          timeout                = "0.12:00:00"
+          retry                  = 0
+          retryIntervalInSeconds = 30
+          secureOutput           = false
+          secureInput            = false
+        }
+        userProperties = []
+        typeProperties = {
+          source = {
+            type                                = "AzureTableSource"
+            azureTableSourceIgnoreTableNotFound = false
+          }
+          sink = {
+            type                 = "AzureTableSink"
+            writeBatchSize       = 10000
+            writeBatchTimeout    = "00:02:00"
+            azureTableInsertType = "merge",
+            azureTablePartitionKeyName = {
+              value = "PartitionKey",
+              type  = "Expression"
+            },
+            azureTableRowKeyName = {
+              value = "RowKey",
+              type  = "Expression"
+            },
+          }
+          enableStaging = false
+        }
+        inputs = [
+          {
+            referenceName = azurerm_data_factory_custom_dataset.source_dataset_table[each.value].name
+            type          = "DatasetReference"
+          }
+        ]
+        outputs = [
+          {
+            referenceName = azurerm_data_factory_custom_dataset.target_dataset_table[each.value].name
+            type          = "DatasetReference"
+          }
+        ]
+      }
+    ]
+  )
+
+}