From 34737a9c61b903226228a499db8738172ee0ddfd Mon Sep 17 00:00:00 2001 From: Anthony Howe Date: Mon, 19 Jul 2021 17:16:07 -0400 Subject: [PATCH] update cachewarmer modules to support proxy (#1309) --- .../centos-e2e/3.cache/config.auto.tfvars | 7 +- .../examples/centos-e2e/3.cache/main.tf | 165 +++++++++++++++++- .../cachewarmer_manager_install/main.tf | 3 +- .../cachewarmer_manager_install/variables.tf | 5 + .../cachewarmer_prepare_bootstrapdir/main.tf | 2 +- .../modules/cachewarmer_submitjobs/main.tf | 3 +- .../cachewarmer_submitjobs/variables.tf | 10 +- .../cachewarmer_worker_install/main.tf | 3 +- .../cachewarmer_worker_install/variables.tf | 5 + 9 files changed, 190 insertions(+), 13 deletions(-) diff --git a/src/terraform/examples/centos-e2e/3.cache/config.auto.tfvars b/src/terraform/examples/centos-e2e/3.cache/config.auto.tfvars index dc6129713..a28ad5c25 100644 --- a/src/terraform/examples/centos-e2e/3.cache/config.auto.tfvars +++ b/src/terraform/examples/centos-e2e/3.cache/config.auto.tfvars @@ -7,8 +7,11 @@ vm_admin_username = "azureuser" ssh_public_key = "" // controller settings, used for vfxt and cachewarmer -controller_private_ip = "10.0.1.254" // at end of range to not interfere with cache -controller_add_public_ip = true +controller_private_ip = "10.0.1.254" // at end of range to not interfere with cache +controller_add_public_ip = true +install_cachewarmer = true +cachewarmer_storage_account_name = "cachewarmerstg" +queue_prefix_name = "cachewarmer" // There are 2 Cache choices // "HPCCache" - deploys HPC Cache diff --git a/src/terraform/examples/centos-e2e/3.cache/main.tf b/src/terraform/examples/centos-e2e/3.cache/main.tf index 372e17189..5daaf6dad 100644 --- a/src/terraform/examples/centos-e2e/3.cache/main.tf +++ b/src/terraform/examples/centos-e2e/3.cache/main.tf @@ -44,6 +44,18 @@ variable "controller_add_public_ip" { type = bool } +variable "install_cachewarmer" { + type = bool +} + +variable "cachewarmer_storage_account_name" { + type = string +} + +variable "queue_prefix_name" { + type = string +} + variable "cache_type" { type = string } @@ -291,10 +303,10 @@ resource "azurerm_hpc_cache_nfs_target" "nfs_targets" { } //////////////////////////////////////////////////////////////// -// Avere vFXT related resources +// Controller - used for cachewarmer and vFXT install //////////////////////////////////////////////////////////////// resource "azurerm_network_security_rule" "controllersshin" { - count = local.deployAverevFXT && var.controller_add_public_ip ? 1 : 0 + count = (local.deployAverevFXT || var.install_cachewarmer) && var.controller_add_public_ip ? 1 : 0 name = "controllersshin" priority = 120 direction = "Inbound" @@ -310,7 +322,7 @@ resource "azurerm_network_security_rule" "controllersshin" { } module "vfxtcontroller" { - count = local.deployAverevFXT ? 1 : 0 + count = local.deployAverevFXT || var.install_cachewarmer ? 1 : 0 source = "github.com/Azure/Avere/src/terraform/modules/controller3" create_resource_group = false resource_group_name = var.cache_rg @@ -335,6 +347,9 @@ module "vfxtcontroller" { ] } +//////////////////////////////////////////////////////////////// +// Avere vFXT related resources +//////////////////////////////////////////////////////////////// resource "azurerm_network_security_rule" "avere" { count = local.deployAverevFXT && !data.terraform_remote_state.network.outputs.use_proxy_server ? 1 : 0 name = "avere" @@ -405,6 +420,148 @@ resource "avere_vfxt" "vfxt" { } } +//////////////////////////////////////////////////////////////// +// Cachewarmer +//////////////////////////////////////////////////////////////// +resource "azurerm_storage_account" "storage" { + count = var.install_cachewarmer ? 1 : 0 + name = var.cachewarmer_storage_account_name + resource_group_name = var.cache_rg // must be in same rg as controller for access by controller + location = var.location + account_kind = "Storage" // set to storage v1 for cheapest cost on queue transactions + account_tier = "Standard" + account_replication_type = "LRS" + + depends_on = [ + azurerm_resource_group.cache_rg, + ] +} + +module "cachewarmer_prepare_bootstrapdir" { + count = var.install_cachewarmer ? 1 : 0 + source = "github.com/Azure/Avere/src/terraform/modules/cachewarmer_prepare_bootstrapdir" + + // authentication with controller + jumpbox_address = module.vfxtcontroller[0].controller_address + jumpbox_username = module.vfxtcontroller[0].controller_username + jumpbox_password = data.azurerm_key_vault_secret.virtualmachine.value + jumpbox_ssh_key_data = var.ssh_public_key + proxy = local.proxy_uri + + // bootstrap directory to store the cache manager binaries and install scripts + bootstrap_mount_address = data.terraform_remote_state.onprem.outputs.nfsfiler_address + bootstrap_export_path = data.terraform_remote_state.onprem.outputs.nfsfiler_export + bootstrap_subdir = "/tools/bootstrap" + + # use the release binaries by setting build_cachewarmer to false + build_cachewarmer = false + + depends_on = [ + module.vfxtcontroller, + ] +} + +module "cachewarmer_manager_install" { + count = var.install_cachewarmer ? 1 : 0 + source = "github.com/Azure/Avere/src/terraform/modules/cachewarmer_manager_install" + + // authentication with controller + jumpbox_address = module.vfxtcontroller[0].controller_address + jumpbox_username = module.vfxtcontroller[0].controller_username + jumpbox_password = data.azurerm_key_vault_secret.virtualmachine.value + jumpbox_ssh_key_data = var.ssh_public_key + proxy = local.proxy_uri + + // bootstrap directory to install the cache manager service + bootstrap_mount_address = module.cachewarmer_prepare_bootstrapdir[0].bootstrap_mount_address + bootstrap_export_path = module.cachewarmer_prepare_bootstrapdir[0].bootstrap_export_path + bootstrap_manager_script_path = module.cachewarmer_prepare_bootstrapdir[0].cachewarmer_manager_bootstrap_script_path + + // the job path + storage_account = azurerm_storage_account.storage[0].name + storage_account_rg = azurerm_storage_account.storage[0].resource_group_name + queue_name_prefix = var.queue_prefix_name + + // the cachewarmer VMSS auth details + vmss_user_name = module.vfxtcontroller[0].controller_username + vmss_password = data.azurerm_key_vault_secret.virtualmachine.value + vmss_ssh_public_key = var.ssh_public_key + vmss_subnet_name = data.terraform_remote_state.network.outputs.render_subnet_name + vmss_worker_count = length(local.deployAverevFXT ? avere_vfxt.vfxt[0].node_names : azurerm_hpc_cache.hpc_cache[0].mount_addresses) * 4 // 4 D2sv3 nodes per cache node + + // the cachewarmer install the work script + bootstrap_worker_script_path = module.cachewarmer_prepare_bootstrapdir[0].cachewarmer_worker_bootstrap_script_path + + depends_on = [ + module.cachewarmer_prepare_bootstrapdir, + avere_vfxt.vfxt, + azurerm_hpc_cache.hpc_cache, + azurerm_storage_account.storage, + ] +} + +module "cachewarmer_worker_install" { + count = var.install_cachewarmer ? 1 : 0 + source = "github.com/Azure/Avere/src/terraform/modules/cachewarmer_worker_install" + + // authentication with controller + jumpbox_address = module.vfxtcontroller[0].controller_address + jumpbox_username = module.vfxtcontroller[0].controller_username + jumpbox_password = data.azurerm_key_vault_secret.virtualmachine.value + jumpbox_ssh_key_data = var.ssh_public_key + proxy = local.proxy_uri + + // bootstrap directory to install the cache manager service + bootstrap_mount_address = module.cachewarmer_prepare_bootstrapdir[0].bootstrap_mount_address + bootstrap_export_path = module.cachewarmer_prepare_bootstrapdir[0].bootstrap_export_path + bootstrap_worker_script_path = module.cachewarmer_prepare_bootstrapdir[0].cachewarmer_worker_bootstrap_script_path + + // the job path + storage_account = azurerm_storage_account.storage[0].name + storage_account_rg = azurerm_storage_account.storage[0].resource_group_name + queue_name_prefix = var.queue_prefix_name + + depends_on = [ + module.cachewarmer_manager_install, + ] +} + +module "cachewarmer_submitjobs" { + count = var.install_cachewarmer ? 1 : 0 + source = "github.com/Azure/Avere/src/terraform/modules/cachewarmer_submitjobs" + + // authentication with controller + jumpbox_address = module.vfxtcontroller[0].controller_address + jumpbox_username = module.vfxtcontroller[0].controller_username + jumpbox_password = data.azurerm_key_vault_secret.virtualmachine.value + jumpbox_ssh_key_data = var.ssh_public_key + proxy = local.proxy_uri + + // the job path + storage_account = azurerm_storage_account.storage[0].name + storage_account_rg = azurerm_storage_account.storage[0].resource_group_name + queue_name_prefix = var.queue_prefix_name + + // the path to warm + warm_mount_addresses = join(",", tolist(local.deployAverevFXT ? avere_vfxt.vfxt[0].vserver_ip_addresses : azurerm_hpc_cache.hpc_cache[0].mount_addresses)) + warm_paths = { + "${data.terraform_remote_state.onprem.outputs.nfsfiler_export}" : ["/tools", "/island"], + } + + inclusion_csv = "" // example "*.jpg,*.png" + exclusion_csv = "" // example "*.tgz,*.tmp" + maxFileSizeBytes = 0 + + block_until_warm = true + + depends_on = [ + module.cachewarmer_worker_install, + avere_vfxt.vfxt, + azurerm_hpc_cache.hpc_cache, + azurerm_storage_account.storage, + ] +} + ### Outputs output "controller_username" { value = length(module.vfxtcontroller) == 0 ? "" : module.vfxtcontroller[0].controller_username @@ -421,5 +578,3 @@ output "mount_addresses" { output "management_ip" { value = local.deployAverevFXT ? avere_vfxt.vfxt[0].vfxt_management_ip : "" } - - diff --git a/src/terraform/modules/cachewarmer_manager_install/main.tf b/src/terraform/modules/cachewarmer_manager_install/main.tf index 14ba785f3..93fa0e7d4 100644 --- a/src/terraform/modules/cachewarmer_manager_install/main.tf +++ b/src/terraform/modules/cachewarmer_manager_install/main.tf @@ -3,7 +3,8 @@ locals { vmss_password_str = var.vmss_password == null ? "" : var.vmss_password vmss_ssh_public_key_str = var.vmss_ssh_public_key == null ? "" : var.vmss_ssh_public_key vmss_subnet_name_str = var.vmss_subnet_name == null ? "" : var.vmss_subnet_name - env_vars = "export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix} && export BOOTSTRAP_EXPORT_PATH=${var.bootstrap_export_path} && export BOOTSTRAP_MOUNT_ADDRESS=${var.bootstrap_mount_address} && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path} && export VMSS_USERNAME=${var.vmss_user_name} && export VMSS_SSHPUBLICKEY='${local.vmss_ssh_public_key_str}' && export VMSS_PASSWORD='${local.vmss_password_str}' && export VMSS_SUBNET=${local.vmss_subnet_name_str} && export VMSS_WORKER_COUNT=${var.vmss_worker_count}" + proxy_env = (var.proxy == null || var.proxy == "") ? "" : "export http_proxy=${var.proxy} && export https_proxy=${var.proxy} && export no_proxy=169.254.169.254 &&" + env_vars = "${local.proxy_env} export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix} && export BOOTSTRAP_EXPORT_PATH=${var.bootstrap_export_path} && export BOOTSTRAP_MOUNT_ADDRESS=${var.bootstrap_mount_address} && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path} && export VMSS_USERNAME=${var.vmss_user_name} && export VMSS_SSHPUBLICKEY='${local.vmss_ssh_public_key_str}' && export VMSS_PASSWORD='${local.vmss_password_str}' && export VMSS_SUBNET=${local.vmss_subnet_name_str} && export VMSS_WORKER_COUNT=${var.vmss_worker_count}" } resource "null_resource" "install_cachewarmer_manager" { diff --git a/src/terraform/modules/cachewarmer_manager_install/variables.tf b/src/terraform/modules/cachewarmer_manager_install/variables.tf index 4ce9b0dea..99c3e2447 100644 --- a/src/terraform/modules/cachewarmer_manager_install/variables.tf +++ b/src/terraform/modules/cachewarmer_manager_install/variables.tf @@ -75,3 +75,8 @@ variable "vmss_worker_count" { default = 12 type = number } + +variable "proxy" { + description = "specify a proxy address if one exists in the format of http://PROXY_SERVER:PORT" + default = null +} diff --git a/src/terraform/modules/cachewarmer_prepare_bootstrapdir/main.tf b/src/terraform/modules/cachewarmer_prepare_bootstrapdir/main.tf index 51392e8e7..40682a909 100644 --- a/src/terraform/modules/cachewarmer_prepare_bootstrapdir/main.tf +++ b/src/terraform/modules/cachewarmer_prepare_bootstrapdir/main.tf @@ -1,7 +1,7 @@ locals { mount_dir = "/bcwpb" - proxy_env = (var.proxy == null || var.proxy == "") ? "" : "http_proxy=${var.proxy} https_proxy=${var.proxy} no_proxy=169.254.169.254" + proxy_env = (var.proxy == null || var.proxy == "") ? "" : "export http_proxy=${var.proxy} && export https_proxy=${var.proxy} && export no_proxy=169.254.169.254 && " build_cachewarmer_lines = [ "set -x", diff --git a/src/terraform/modules/cachewarmer_submitjobs/main.tf b/src/terraform/modules/cachewarmer_submitjobs/main.tf index 618167b09..0bd16d62d 100644 --- a/src/terraform/modules/cachewarmer_submitjobs/main.tf +++ b/src/terraform/modules/cachewarmer_submitjobs/main.tf @@ -7,6 +7,7 @@ locals { maxFileSizeBytes = var.maxFileSizeBytes == 0 ? "" : " -maxFileSizeBytes ${var.maxFileSizeBytes} " inclusion_csv = var.inclusion_csv == null || length(var.inclusion_csv) == 0 ? "" : " -inclusionCsv \"${var.inclusion_csv}\" " exclusion_csv = var.exclusion_csv == null || length(var.exclusion_csv) == 0 ? "" : " -exclusionCsv \"${var.exclusion_csv}\" " + proxy_env = (var.proxy == null || var.proxy == "") ? "" : "export http_proxy=${var.proxy} && export https_proxy=${var.proxy} && export no_proxy=169.254.169.254 &&" } resource "null_resource" "cachewarmer_submitmultiplejobs" { @@ -24,7 +25,7 @@ resource "null_resource" "cachewarmer_submitmultiplejobs" { provisioner "remote-exec" { inline = [ "set -x", - "sudo /usr/local/bin/cachewarmer-jobsubmitter -storageAccountName ${var.storage_account} -storageAccountResourceGroup ${var.storage_account_rg} -queueNamePrefix ${var.queue_name_prefix} -warmTargetExportPath \"${local.warm_paths_sets[count.index].export}\" -warmTargetMountAddresses \"${var.warm_mount_addresses}\" -warmTargetPath \"${local.warm_paths_sets[count.index].path}\" ${local.inclusion_csv} ${local.exclusion_csv} ${local.maxFileSizeBytes} ${var.block_until_warm && count.index == 0 ? local.block_flag : local.no_block_flag}", + "${local.proxy_env} sudo -E /usr/local/bin/cachewarmer-jobsubmitter -storageAccountName ${var.storage_account} -storageAccountResourceGroup ${var.storage_account_rg} -queueNamePrefix ${var.queue_name_prefix} -warmTargetExportPath \"${local.warm_paths_sets[count.index].export}\" -warmTargetMountAddresses \"${var.warm_mount_addresses}\" -warmTargetPath \"${local.warm_paths_sets[count.index].path}\" ${local.inclusion_csv} ${local.exclusion_csv} ${local.maxFileSizeBytes} ${var.block_until_warm && count.index == 0 ? local.block_flag : local.no_block_flag}", ] } } diff --git a/src/terraform/modules/cachewarmer_submitjobs/variables.tf b/src/terraform/modules/cachewarmer_submitjobs/variables.tf index b8ed45f51..6172c1e3e 100644 --- a/src/terraform/modules/cachewarmer_submitjobs/variables.tf +++ b/src/terraform/modules/cachewarmer_submitjobs/variables.tf @@ -45,12 +45,12 @@ variable "warm_paths" { variable "inclusion_csv" { description = "the inclusion list of file match strings per https://golang.org/pkg/path/filepath/#Match. Leave blank to include everything." - default = "" + default = "" } variable "exclusion_csv" { description = "the exclusion list of file match strings per https://golang.org/pkg/path/filepath/#Match. Leave blank to not exlude anything." - default = "" + default = "" } variable "maxFileSizeBytes" { @@ -63,3 +63,9 @@ variable "block_until_warm" { description = "block the operation until the cache warming has finished" default = true } + + +variable "proxy" { + description = "specify a proxy address if one exists in the format of http://PROXY_SERVER:PORT" + default = null +} diff --git a/src/terraform/modules/cachewarmer_worker_install/main.tf b/src/terraform/modules/cachewarmer_worker_install/main.tf index 326a9f8ff..b9c795b2d 100644 --- a/src/terraform/modules/cachewarmer_worker_install/main.tf +++ b/src/terraform/modules/cachewarmer_worker_install/main.tf @@ -1,6 +1,7 @@ locals { mount_dir = "/b" - env_vars = "export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix} && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path}" + proxy_env = (var.proxy == null || var.proxy == "") ? "" : "export http_proxy=${var.proxy} && export https_proxy=${var.proxy} && export no_proxy=169.254.169.254 &&" + env_vars = "${local.proxy_env} export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix} && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path}" } resource "null_resource" "install_cachewarmer_worker" { diff --git a/src/terraform/modules/cachewarmer_worker_install/variables.tf b/src/terraform/modules/cachewarmer_worker_install/variables.tf index a5dc15496..ae999febf 100644 --- a/src/terraform/modules/cachewarmer_worker_install/variables.tf +++ b/src/terraform/modules/cachewarmer_worker_install/variables.tf @@ -45,3 +45,8 @@ variable "storage_account" { variable "queue_name_prefix" { description = "the queue name prefix for the job management" } + +variable "proxy" { + description = "specify a proxy address if one exists in the format of http://PROXY_SERVER:PORT" + default = null +}