Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CES-157] - Implement tooling for storage account migration from WEU to ITN #1238

Merged
merged 32 commits into from
Nov 6, 2024
Merged
Changes from 1 commit
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
822e19d
Aggiunta codice di test del modulo per Azure data factory
ccuffari Oct 10, 2024
2e1af1d
File folder structure check
ccuffari Oct 11, 2024
5bb5809
Update italynorth.tf
ccuffari Oct 15, 2024
dab8db1
feat: started refactoring
christian-calabrese Oct 15, 2024
4e11cf8
chore: what_to_migrate variable added
christian-calabrese Oct 15, 2024
7760c8f
feat: added iam and fixed pipelines
christian-calabrese Oct 16, 2024
f9aa68d
feat: added missing datasets and checks
christian-calabrese Oct 16, 2024
9d2c460
fix: changed folder structure and ran pre-commit
christian-calabrese Oct 16, 2024
0c4c0a8
feat: added the example list
christian-calabrese Oct 16, 2024
2e0da2f
fix: errors and tested
christian-calabrese Oct 16, 2024
0a92fb2
Merge branch 'main' into bip-dev
christian-calabrese Oct 16, 2024
a28f762
chore: ran pre-commit
christian-calabrese Oct 16, 2024
7f81909
feat: using instance number from local
christian-calabrese Oct 16, 2024
09d17e6
feat: adf public_network_enabled set to false
christian-calabrese Oct 16, 2024
7200a5a
feat: added managed virtual network support to adf
christian-calabrese Oct 16, 2024
59cac9f
fix: changed from jsonsource to binarysource
christian-calabrese Oct 16, 2024
5ea38bf
Merge branch 'main' into bip-dev
christian-calabrese Oct 16, 2024
0e714ca
fix: references to resources
christian-calabrese Oct 17, 2024
041e94b
feat: subdivide datafactory resources in folders
christian-calabrese Oct 17, 2024
088d468
feat: substitute dollar sign with underscore in adf resource names
christian-calabrese Oct 17, 2024
1190c2b
fix: re added state configuration
christian-calabrese Oct 17, 2024
f43ca19
fix: comment
christian-calabrese Oct 17, 2024
3f68237
fix: deleted test storage accounts
christian-calabrese Oct 17, 2024
02bc83b
fix: updated lock with platforms
mamu0 Oct 17, 2024
2521718
Merge branch 'main' into bip-dev
christian-calabrese Oct 21, 2024
f5edf0c
Merge branch 'main' into bip-dev
christian-calabrese Oct 22, 2024
ec80614
Merge branch 'main' into bip-dev
christian-calabrese Oct 28, 2024
f0e366a
Merge branch 'main' into bip-dev
christian-calabrese Oct 29, 2024
c9b8318
Merge branch 'main' into bip-dev
christian-calabrese Oct 29, 2024
d241f7d
Merge branch 'main' into bip-dev
christian-calabrese Nov 4, 2024
0f1374b
Merge branch 'main' into bip-dev
christian-calabrese Nov 5, 2024
8347b39
Merge branch 'main' into bip-dev
christian-calabrese Nov 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Aggiunta codice di test del modulo per Azure data factory
ccuffari committed Oct 10, 2024
commit 822e19dfdfbe45c21ee48650d3a55909b9c78689
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
terraform {
backend "azurerm" {
resource_group_name = "dev-fasanorg"
storage_account_name = "stbipdevtest"
container_name = "bc-tf-bip-dev-test"
key = "terraform.tfstate"
}
}
21 changes: 21 additions & 0 deletions src/_modules/test_users/DataFactory/environments/dev/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Main.tf per l'ambiente DEV
provider "azurerm" {
features {}
subscription_id = var.subscription_id
}

data "azurerm_client_config" "current" {}

module "data_factory" {
source = "../../modules/data_factory"

data_factories = var.data_factories
pipelines = var.pipelines
azure_runtimes = var.azure_runtimes
self_hosted_runtimes = var.self_hosted_runtimes
linked_services = var.linked_services
datasets = var.datasets
storage_account_resource_groups = var.storage_account_resource_groups # Aggiungi questa riga
christian-calabrese marked this conversation as resolved.
Show resolved Hide resolved
}


Empty file.
165 changes: 165 additions & 0 deletions src/_modules/test_users/DataFactory/environments/dev/terraform.tfvars
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
data_factories = {
dev_adf = {
name = "adfdevbiptest1"
location = "Italy North"
resource_group_name = "dev-fasanorg"
tags = {
Provider = "BIP"
Environment = "DEV"
Application = "Test"
Owners = "Cristian Felice Cuffari, Domenico Fasano"
}
}
}

azure_runtimes = {
dev_runtime = {
name = "runtime-dev"
location = "West Europe"
data_factory = "dev_adf"
}
}

self_hosted_runtimes = {
dev_self_hosted = {
name = "self-hosted-dev"
data_factory = "dev_adf"
}
}

linked_services = {
dev_blob_service = {
name = "blob-storage-dev"
data_factory = "dev_adf"
type = "AzureBlobStorage"
type_properties_json = <<JSON
{
"connectionString": ""
}
JSON
storage_account = "stbipdevtest1"
},
prod_blob_service = {
name = "blob-storage-dev2"
data_factory = "dev_adf"
type = "AzureBlobStorage"
type_properties_json = <<JSON
{
"connectionString": ""
}
JSON
storage_account = "stbipdevtest"
}
}

# Resource group associato agli storage account
storage_account_resource_groups = {
dev_blob_service = "dev-fasanorg"
prod_blob_service = "dev-fasanorg"
}


datasets = {
dev_json_dataset = {
name = "json_dataset_dev"
data_factory = "dev_adf"
type = "Json"
linked_service = "dev_blob_service"
type_properties_json = <<JSON
{
"location": {
"container": "source",
"fileName": "",
"folderPath": "",
"type": "AzureBlobStorageLocation"
},
"encodingName": "UTF-8"
}
JSON
},
dev2_json_dataset = {
name = "json2_dataset_dev"
data_factory = "dev_adf"
type = "Json"
linked_service = "prod_blob_service"
type_properties_json = <<JSON
{
"location": {
"container": "destination",
"fileName": "",
"folderPath": "",
"type": "AzureBlobStorageLocation"
},
"encodingName": "UTF-8"
}
JSON
}

}

# Definizione della pipeline
pipelines = {
pipeline_1 = {
name = "Pipeline1"
data_factory = "dev_adf"
variables = {
"bob" = "item1"
}
activities_json = <<JSON
[
{
"name": "CopyActivity",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "0.12:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true,
"enablePartitionDiscovery": false,
"wildcardFileName": "*"
},
"formatSettings": {
"type": "JsonReadSettings"
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings"
},
"formatSettings": {
"type": "JsonWriteSettings"
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "json_dataset_dev",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "json2_dataset_dev",
"type": "DatasetReference"
}
]
}
]
JSON
}
}


75 changes: 75 additions & 0 deletions src/_modules/test_users/DataFactory/environments/dev/variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# Variabile opzionale per PostgreSQL
variable "subscription_id" {
description = "nome della subscription"
type = string
default = "d7de83e0-0571-40ad-b63a-64c942385eae"
}

variable "data_factories" {
type = map(object({
name = string
location = string
resource_group_name = string
tags = map(string)
}))
}

variable "pipelines" {
type = map(object({
name = string
data_factory = string
variables = map(string) # Per le variabili della pipeline
activities_json = string # Il contenuto JSON dell'attività
}))
}


variable "azure_runtimes" {
type = map(object({
name = string
location = string
data_factory = string
}))
}

variable "self_hosted_runtimes" {
type = map(object({
name = string
data_factory = string
}))
}

variable "linked_services" {
description = "Elenco dei linked services da creare per Data Factory"
type = map(object({
name = string
data_factory = string
type = string
type_properties_json = string
storage_account = string
}))
}

variable "storage_account_resource_groups" {
description = "Resource groups for each storage account used in linked services"
type = map(string)
}


variable "datasets" {
type = map(object({
name = string
data_factory = string
type = string
linked_service = string
type_properties_json = string
description = optional(string)
annotations = optional(list(string))
folder = optional(string)
parameters = optional(map(string))
additional_properties = optional(map(string))
schema_json = optional(string)
}))
}


10 changes: 10 additions & 0 deletions src/_modules/test_users/DataFactory/environments/dev/versions.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
terraform {
required_version = ">= 1.0.0"

required_providers {
azurerm = {
source = "hashicorp/azurerm"
version = ">= 3.0"
}
}
}
81 changes: 81 additions & 0 deletions src/_modules/test_users/DataFactory/modules/data_factory/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
resource "azurerm_data_factory" "this" {
for_each = var.data_factories
name = "adf${each.value.name}"
location = each.value.location
resource_group_name = each.value.resource_group_name
tags = each.value.tags

identity {
type = "SystemAssigned"
}
}

resource "azurerm_data_factory_pipeline" "this" {
for_each = var.pipelines
name = each.value.name
data_factory_id = azurerm_data_factory.this[each.value.data_factory].id

variables = each.value.variables

activities_json = each.value.activities_json
# Dipendenza sui dataset
depends_on = [
azurerm_data_factory_custom_dataset.this
]
}


resource "azurerm_data_factory_integration_runtime_azure" "azure_runtime" {
for_each = var.azure_runtimes
name = "azure-runtime-${each.value.name}"
data_factory_id = azurerm_data_factory.this[each.value.data_factory].id
location = each.value.location
}

resource "azurerm_data_factory_integration_runtime_self_hosted" "self_hosted_runtime" {
for_each = var.self_hosted_runtimes
name = "self-hosted-runtime-${each.value.name}"
data_factory_id = azurerm_data_factory.this[each.value.data_factory].id
}

data "azurerm_storage_account" "existing" {
for_each = { for linked_service, details in var.linked_services : linked_service => details.storage_account if details.storage_account != "" }
name = each.value
resource_group_name = var.storage_account_resource_groups[each.key]
}

# Linked service per Azure Data Factory
resource "azurerm_data_factory_linked_custom_service" "this" {
for_each = var.linked_services
name = each.value.name
data_factory_id = azurerm_data_factory.this[each.value.data_factory].id
type = each.value.type

type_properties_json = <<JSON
{
"connectionString": "${data.azurerm_storage_account.existing[each.key].primary_connection_string}"
}
JSON
}


# Dataset JSON
resource "azurerm_data_factory_custom_dataset" "this" {
for_each = var.datasets
name = each.value.name
data_factory_id = azurerm_data_factory.this[each.value.data_factory].id
type = each.value.type

linked_service {
name = azurerm_data_factory_linked_custom_service.this[each.value.linked_service].name
parameters = each.value.parameters
}

type_properties_json = each.value.type_properties_json
description = each.value.description
annotations = each.value.annotations
folder = each.value.folder
parameters = each.value.parameters
additional_properties = each.value.additional_properties
schema_json = each.value.schema_json
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
output "storage_account_connection_string" {
value = { for key, account in data.azurerm_storage_account.existing : key => account.primary_connection_string }
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
variable "data_factories" {
description = "Elenco delle Data Factory da creare"
type = map(object({
name = string
location = string
resource_group_name = string
tags = map(string)
}))
}

variable "pipelines" {
type = map(object({
name = string
data_factory = string
variables = map(string) # Per le variabili della pipeline
activities_json = string # Il contenuto JSON dell'attività
}))
}




variable "azure_runtimes" {
description = "Elenco degli Azure Integration Runtimes per Data Factory"
type = map(object({
name = string
location = string
data_factory = string
}))
}

variable "self_hosted_runtimes" {
description = "Elenco dei Self-Hosted Integration Runtimes per Data Factory"
type = map(object({
name = string
data_factory = string
}))
}

variable "linked_services" {
description = "Elenco dei linked services da creare per Data Factory"
type = map(object({
name = string
data_factory = string
type = string
type_properties_json = string
storage_account = string
}))
}

variable "storage_account_resource_groups" {
description = "Resource groups for each storage account used in linked services"
type = map(string)
}


variable "datasets" {
description = "Elenco dei dataset da creare per Data Factory"
type = map(object({
name = string
data_factory = string
type = string
linked_service = string
type_properties_json = string
description = optional(string)
annotations = optional(list(string))
folder = optional(string)
parameters = optional(map(string))
additional_properties = optional(map(string))
schema_json = optional(string)
}))
}