Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CES-157] - Implement tooling for storage account migration from WEU to ITN #1238

Merged
merged 32 commits into from
Nov 6, 2024
Merged
Changes from 1 commit
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
822e19d
Aggiunta codice di test del modulo per Azure data factory
ccuffari Oct 10, 2024
2e1af1d
File folder structure check
ccuffari Oct 11, 2024
5bb5809
Update italynorth.tf
ccuffari Oct 15, 2024
dab8db1
feat: started refactoring
christian-calabrese Oct 15, 2024
4e11cf8
chore: what_to_migrate variable added
christian-calabrese Oct 15, 2024
7760c8f
feat: added iam and fixed pipelines
christian-calabrese Oct 16, 2024
f9aa68d
feat: added missing datasets and checks
christian-calabrese Oct 16, 2024
9d2c460
fix: changed folder structure and ran pre-commit
christian-calabrese Oct 16, 2024
0c4c0a8
feat: added the example list
christian-calabrese Oct 16, 2024
2e0da2f
fix: errors and tested
christian-calabrese Oct 16, 2024
0a92fb2
Merge branch 'main' into bip-dev
christian-calabrese Oct 16, 2024
a28f762
chore: ran pre-commit
christian-calabrese Oct 16, 2024
7f81909
feat: using instance number from local
christian-calabrese Oct 16, 2024
09d17e6
feat: adf public_network_enabled set to false
christian-calabrese Oct 16, 2024
7200a5a
feat: added managed virtual network support to adf
christian-calabrese Oct 16, 2024
59cac9f
fix: changed from jsonsource to binarysource
christian-calabrese Oct 16, 2024
5ea38bf
Merge branch 'main' into bip-dev
christian-calabrese Oct 16, 2024
0e714ca
fix: references to resources
christian-calabrese Oct 17, 2024
041e94b
feat: subdivide datafactory resources in folders
christian-calabrese Oct 17, 2024
088d468
feat: substitute dollar sign with underscore in adf resource names
christian-calabrese Oct 17, 2024
1190c2b
fix: re added state configuration
christian-calabrese Oct 17, 2024
f43ca19
fix: comment
christian-calabrese Oct 17, 2024
3f68237
fix: deleted test storage accounts
christian-calabrese Oct 17, 2024
02bc83b
fix: updated lock with platforms
mamu0 Oct 17, 2024
2521718
Merge branch 'main' into bip-dev
christian-calabrese Oct 21, 2024
f5edf0c
Merge branch 'main' into bip-dev
christian-calabrese Oct 22, 2024
ec80614
Merge branch 'main' into bip-dev
christian-calabrese Oct 28, 2024
f0e366a
Merge branch 'main' into bip-dev
christian-calabrese Oct 29, 2024
c9b8318
Merge branch 'main' into bip-dev
christian-calabrese Oct 29, 2024
d241f7d
Merge branch 'main' into bip-dev
christian-calabrese Nov 4, 2024
0f1374b
Merge branch 'main' into bip-dev
christian-calabrese Nov 5, 2024
8347b39
Merge branch 'main' into bip-dev
christian-calabrese Nov 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat: added the example list
christian-calabrese committed Oct 16, 2024

Unverified

No user is associated with the committer email.
commit 0c4c0a8226175b509932c5c96c8593d813f42910
8 changes: 7 additions & 1 deletion src/_modules/data_factory_storage_account/data.tf
Original file line number Diff line number Diff line change
@@ -9,6 +9,12 @@ data "azurerm_storage_account" "target" {
}

data "azurerm_storage_containers" "this" {
for_each = var.what_to_migrate.blob.enabled && length(var.what_to_migrate.blob.containers) == 0 ? [1] : []
count = var.what_to_migrate.blob.enabled && length(var.what_to_migrate.blob.containers) == 0 ? 1 : 0
storage_account_id = data.azurerm_storage_account.source.id
}

data "azapi_resource_list" "tables" {
Krusty93 marked this conversation as resolved.
Show resolved Hide resolved
type = "Microsoft.Storage/storageAccounts/tableServices/tables@2021-09-01"
parent_id = "${data.azurerm_storage_account.source.id}/tableServices/default"
response_export_values = ["*"]
}
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
resource "azurerm_data_factory_custom_dataset" "source_dataset_container" {
for_each = local.containers
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-blob-${module.naming_convention.suffix}"
data_factory_id = var.data_factory.id
data_factory_id = var.data_factory_id
type = "AzureBlob"

linked_service {
name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob.name
name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob[0].name
}

type_properties_json = jsonencode({
linkedServiceName = {
referenceName = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob.name
referenceName = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_blob[0].name
type = "LinkedServiceReference"
}
type = "AzureBlob"
@@ -21,16 +21,16 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_container" {
resource "azurerm_data_factory_custom_dataset" "target_dataset_container" {
for_each = local.containers
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value.name}-blob-${module.naming_convention.suffix}"
data_factory_id = var.data_factory.id
data_factory_id = var.data_factory_id
type = "AzureBlob"

linked_service {
name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob.name
name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob[0].name
}

type_properties_json = jsonencode({
linkedServiceName = {
referenceName = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob.name
referenceName = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_blob[0].name
type = "LinkedServiceReference"
}
type = "AzureBlob"
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
resource "azurerm_data_factory_custom_dataset" "source_dataset_table" {
for_each = local.tables
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-table-${module.naming_convention.suffix}"
data_factory_id = var.data_factory.id
data_factory_id = var.data_factory_id
type = "AzureTable"

linked_service {
name = azurerm_data_factory_linked_service_azure_blob_storage.source_linked_service_table.name
name = azurerm_data_factory_linked_service_azure_table_storage.source_linked_service_table[0].name
}

type_properties_json = jsonencode({
@@ -16,11 +16,11 @@ resource "azurerm_data_factory_custom_dataset" "source_dataset_table" {
resource "azurerm_data_factory_custom_dataset" "target_dataset_table" {
for_each = local.tables
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-${each.value.name}-table-${module.naming_convention.suffix}"
data_factory_id = var.data_factory.id
data_factory_id = var.data_factory_id
type = "AzureTable"

linked_service {
name = azurerm_data_factory_linked_service_azure_blob_storage.target_linked_service_table.name
name = azurerm_data_factory_linked_service_azure_table_storage.target_linked_service_table[0].name
}

type_properties_json = jsonencode({
2 changes: 1 addition & 1 deletion src/_modules/data_factory_storage_account/iam.tf
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module "roles" {
source = "github.com/pagopa/dx//infra/modules/azure_role_assignments?ref=main"
principal_id = var.function_app.user_func_02.principal_id
principal_id = var.data_factory_principal_id

storage_blob = var.what_to_migrate.blob.enabled ? [
{
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
resource "azurerm_data_factory_linked_service_azure_blob_storage" "source_linked_service_blob" {
for_each = var.what_to_migrate.blob.enabled ? [1] : []
count = var.what_to_migrate.blob.enabled ? 1 : 0
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-blob-${module.naming_convention.suffix}"
data_factory_id = var.data_factory.id
data_factory_id = var.data_factory_id

service_endpoint = "https://${data.azurerm_storage_account.source.id}.blob.core.windows.net"

use_managed_identity = true
}

resource "azurerm_data_factory_linked_service_azure_blob_storage" "target_linked_service_blob" {
for_each = var.what_to_migrate.blob.enabled ? [1] : []
count = var.what_to_migrate.blob.enabled ? 1 : 0
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-blob-${module.naming_convention.suffix}"
data_factory_id = var.data_factory.id
data_factory_id = var.data_factory_id

service_endpoint = "https://${data.azurerm_storage_account.target.id}.blob.core.windows.net"

Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
resource "azurerm_data_factory_linked_service_azure_table_storage" "source_linked_service_table" {
for_each = var.what_to_migrate.table.enabled ? [1] : []
count = var.what_to_migrate.table.enabled ? 1 : 0
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-table-${module.naming_convention.suffix}"
data_factory_id = var.data_factory.id
data_factory_id = var.data_factory_id

connection_string = data.azurerm_storage_account.source.primary_table_endpoint
}

resource "azurerm_data_factory_linked_service_azure_table_storage" "target_linked_service_table" {
for_each = var.what_to_migrate.table.enabled ? [1] : []
count = var.what_to_migrate.table.enabled ? 1 : 0
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-table-${module.naming_convention.suffix}"
data_factory_id = var.data_factory.id
data_factory_id = var.data_factory_id

connection_string = data.azurerm_storage_account.target.primary_table_endpoint
}
4 changes: 2 additions & 2 deletions src/_modules/data_factory_storage_account/locals.tf
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
locals {
containers = var.what_to_migrate.blob.enabled ? length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name] : []
tables = var.what_to_migrate.table.enabled ? var.what_to_migrate.table.tables : []
containers = var.what_to_migrate.blob.enabled ? (length(var.what_to_migrate.blob.containers) > 0 ? var.what_to_migrate.blob.containers : [for container in data.azurerm_storage_containers.this[0].containers : container.name]) : []
tables = var.what_to_migrate.table.enabled ? (length(var.what_to_migrate.table.tables) > 0 ? var.what_to_migrate.table.tables : [for table in jsondecode(data.azapi_resource_list.tables.output).value : table.TableName]) : []
}
9 changes: 9 additions & 0 deletions src/_modules/data_factory_storage_account/main.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,12 @@
terraform {
required_providers {
azapi = {
source = "Azure/azapi"
version = "<= 1.15.0"
}
}
}

module "naming_convention" {
source = "github.com/pagopa/dx//infra/modules/azure_naming_convention/?ref=main"

27 changes: 27 additions & 0 deletions src/_modules/data_factory_storage_account/network.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
resource "azurerm_data_factory_managed_private_endpoint" "blob_source" {
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-blob-${module.naming_convention.suffix}"
data_factory_id = var.data_factory_id
target_resource_id = data.azurerm_storage_account.source.id
subresource_name = "blob"
}

resource "azurerm_data_factory_managed_private_endpoint" "blob_target" {
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-blob-${module.naming_convention.suffix}"
data_factory_id = var.data_factory_id
target_resource_id = data.azurerm_storage_account.target.id
subresource_name = "blob"
}

resource "azurerm_data_factory_managed_private_endpoint" "table_source" {
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-table-${module.naming_convention.suffix}"
data_factory_id = var.data_factory_id
target_resource_id = data.azurerm_storage_account.source.id
subresource_name = "table"
}

resource "azurerm_data_factory_managed_private_endpoint" "table_target" {
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.target.name}-table-${module.naming_convention.suffix}"
data_factory_id = var.data_factory_id
target_resource_id = data.azurerm_storage_account.target.id
subresource_name = "table"
}
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
resource "azurerm_data_factory_pipeline" "pipeline_container" {
for_each = local.containers
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-blob-${module.naming_convention.suffix}"
data_factory_id = var.data_factory.id

depends_on = [
azurerm_data_factory_custom_dataset.dataset_container
]
data_factory_id = var.data_factory_id

activities_json = jsonencode(
[
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
resource "azurerm_data_factory_pipeline" "pipeline_table" {
for_each = local.tables
name = "${module.naming_convention.prefix}-adf-${var.storage_accounts.source.name}-${each.value.name}-table-${module.naming_convention.suffix}"
data_factory_id = var.data_factory.id

depends_on = [
azurerm_data_factory_custom_dataset.dataset_table
]
data_factory_id = var.data_factory_id

activities_json = jsonencode(
[
15 changes: 7 additions & 8 deletions src/_modules/data_factory_storage_account/variables.tf
Original file line number Diff line number Diff line change
@@ -16,6 +16,11 @@ variable "data_factory_id" {
type = string
}

variable "data_factory_principal_id" {
description = "Data Factory principal id to grant access to."
type = string
}

variable "storage_accounts" {
type = object({
source = object({
@@ -42,7 +47,7 @@ variable "what_to_migrate" {
table = optional(object(
{
enabled = bool
tables = list(string)
tables = optional(list(string), [])
}),
{ enabled = false }
)
@@ -54,11 +59,5 @@ variable "what_to_migrate" {
error_message = "At least one between blob and table should be enabled."
}

# validate that if table is enabled, at least one table is specified
validation {
condition = !(var.what_to_migrate.table.enabled && length(var.what_to_migrate.table.tables) == 0)
error_message = "If table is enabled, at least one table should be specified."
}

description = "List of databases, file shares, containers and tables to migrate."
description = "List of storage account containers and tables to migrate."
}
42 changes: 42 additions & 0 deletions src/migration/prod/.terraform.lock.hcl

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion src/migration/prod/README.md
Original file line number Diff line number Diff line change
@@ -5,13 +5,14 @@

| Name | Version |
|------|---------|
| <a name="requirement_azapi"></a> [azapi](#requirement\_azapi) | <= 1.15.0 |
| <a name="requirement_azurerm"></a> [azurerm](#requirement\_azurerm) | <= 3.112.0 |

## Providers

| Name | Version |
|------|---------|
| <a name="provider_azurerm"></a> [azurerm](#provider\_azurerm) | <= 3.112.0 |
| <a name="provider_azurerm"></a> [azurerm](#provider\_azurerm) | 3.112.0 |

## Modules

24 changes: 9 additions & 15 deletions src/migration/prod/italynorth.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
resource "azurerm_resource_group" "migration" {
name = "${local.project_itn}-migration-rg-01"
name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-rg-01"
location = "italynorth"

tags = local.tags
@@ -8,7 +8,7 @@ resource "azurerm_resource_group" "migration" {
# Create Azure Data Factory instances
# Enables system-assigned managed identity for secure access to resources
resource "azurerm_data_factory" "this" {
name = "${local.project_itn}-migration-adf-01"
name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adf-01"
location = "italynorth"
resource_group_name = azurerm_resource_group.migration.name

mamu0 marked this conversation as resolved.
Show resolved Hide resolved
@@ -20,25 +20,19 @@ resource "azurerm_data_factory" "this" {
}

resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" {
name = "${local.project_itn}-migration-adfir-01"
name = "${local.project_itn}-${local.environment.domain}-${local.environment.app_name}-adfir-01"
location = "italynorth"
data_factory_id = azurerm_data_factory.this.id
}

module "migrate_storage_accounts" {
for_each = local.storage_accounts
for_each = { for migration in local.storage_accounts : "${migration.source.name}|${migration.target.name}" => migration }
source = "../../_modules/data_factory_storage_account"

environment = {
prefix = local.prefix
env_short = local.env_short
location = "italynorth"
domain = "eng"
app_name = "mig"
instance_number = "01"
}
environment = local.environment

data_factory_id = azurerm_data_factory.this.id
data_factory_id = azurerm_data_factory.this.id
data_factory_principal_id = azurerm_data_factory.this.identity[0].principal_id

storage_accounts = {
source = each.value.source
@@ -48,12 +42,12 @@ module "migrate_storage_accounts" {
what_to_migrate = {
blob = {
enabled = each.value.blob.enabled
containers = each.value.blob.containers
containers = try(each.value.blob.containers, [])
}

table = {
enabled = each.value.table.enabled
tables = each.value.table.tables
tables = try(each.value.table.tables, [])
}
}
}
18 changes: 18 additions & 0 deletions src/migration/prod/locals.tf
Original file line number Diff line number Diff line change
@@ -7,11 +7,29 @@ locals {
project_weu_legacy = "${local.prefix}-${local.env_short}"
secondary_project = "${local.prefix}-${local.env_short}-${local.location_short.germanywestcentral}"

environment = {
prefix = local.prefix
env_short = local.env_short
location = "italynorth"
domain = "eng"
app_name = "migitn"
instance_number = "01"
}

tags = {
CostCenter = "TS310 - PAGAMENTI & SERVIZI"
CreatedBy = "Terraform"
Environment = "Prod"
Owner = "IO"
Source = "https://github.com/pagopa/io-infra/blob/main/src/migration/prod"
}

storage_accounts = [
mamu0 marked this conversation as resolved.
Show resolved Hide resolved
{
source = { name = "iopweuabc", resource_group_name = "abc" }
target = { name = "iopitnabc", resource_group_name = "abc" }
blob = { enabled = true }
table = { enabled = true }
}
]
}
Loading