update cachewarmer modules to support proxy (#1309)

src/terraform/examples/centos-e2e/3.cache/config.auto.tfvars

@@ -7,8 +7,11 @@ vm_admin_username = "azureuser"
 ssh_public_key = ""
 
 // controller settings, used for vfxt and cachewarmer
-controller_private_ip    = "10.0.1.254" // at end of range to not interfere with cache
-controller_add_public_ip = true
+controller_private_ip            = "10.0.1.254" // at end of range to not interfere with cache
+controller_add_public_ip         = true
+install_cachewarmer              = true
+cachewarmer_storage_account_name = "cachewarmerstg"
+queue_prefix_name                = "cachewarmer"
 
 // There are 2 Cache choices
 // "HPCCache" - deploys HPC Cache

src/terraform/examples/centos-e2e/3.cache/main.tf

@@ -44,6 +44,18 @@ variable "controller_add_public_ip" {
   type = bool
 }
 
+variable "install_cachewarmer" {
+  type = bool
+}
+
+variable "cachewarmer_storage_account_name" {
+  type = string
+}
+
+variable "queue_prefix_name" {
+  type = string
+}
+
 variable "cache_type" {
   type = string
 }
@@ -291,10 +303,10 @@ resource "azurerm_hpc_cache_nfs_target" "nfs_targets" {
 }
 
 ////////////////////////////////////////////////////////////////
-// Avere vFXT related resources
+// Controller - used for cachewarmer and vFXT install
 ////////////////////////////////////////////////////////////////
 resource "azurerm_network_security_rule" "controllersshin" {
-  count                  = local.deployAverevFXT && var.controller_add_public_ip ? 1 : 0
+  count                  = (local.deployAverevFXT || var.install_cachewarmer) && var.controller_add_public_ip ? 1 : 0
   name                   = "controllersshin"
   priority               = 120
   direction              = "Inbound"
@@ -310,7 +322,7 @@ resource "azurerm_network_security_rule" "controllersshin" {
 }
 
 module "vfxtcontroller" {
-  count                       = local.deployAverevFXT ? 1 : 0
+  count                       = local.deployAverevFXT || var.install_cachewarmer ? 1 : 0
   source                      = "github.com/Azure/Avere/src/terraform/modules/controller3"
   create_resource_group       = false
   resource_group_name         = var.cache_rg
@@ -335,6 +347,9 @@ module "vfxtcontroller" {
   ]
 }
 
+////////////////////////////////////////////////////////////////
+// Avere vFXT related resources
+////////////////////////////////////////////////////////////////
 resource "azurerm_network_security_rule" "avere" {
   count                  = local.deployAverevFXT && !data.terraform_remote_state.network.outputs.use_proxy_server ? 1 : 0
   name                   = "avere"
@@ -405,6 +420,148 @@ resource "avere_vfxt" "vfxt" {
   }
 }
 
+////////////////////////////////////////////////////////////////
+// Cachewarmer
+////////////////////////////////////////////////////////////////
+resource "azurerm_storage_account" "storage" {
+  count                    = var.install_cachewarmer ? 1 : 0
+  name                     = var.cachewarmer_storage_account_name
+  resource_group_name      = var.cache_rg // must be in same rg as controller for access by controller
+  location                 = var.location
+  account_kind             = "Storage" // set to storage v1 for cheapest cost on queue transactions
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+
+  depends_on = [
+    azurerm_resource_group.cache_rg,
+  ]
+}
+
+module "cachewarmer_prepare_bootstrapdir" {
+  count  = var.install_cachewarmer ? 1 : 0
+  source = "github.com/Azure/Avere/src/terraform/modules/cachewarmer_prepare_bootstrapdir"
+
+  // authentication with controller
+  jumpbox_address      = module.vfxtcontroller[0].controller_address
+  jumpbox_username     = module.vfxtcontroller[0].controller_username
+  jumpbox_password     = data.azurerm_key_vault_secret.virtualmachine.value
+  jumpbox_ssh_key_data = var.ssh_public_key
+  proxy                = local.proxy_uri
+
+  // bootstrap directory to store the cache manager binaries and install scripts
+  bootstrap_mount_address = data.terraform_remote_state.onprem.outputs.nfsfiler_address
+  bootstrap_export_path   = data.terraform_remote_state.onprem.outputs.nfsfiler_export
+  bootstrap_subdir        = "/tools/bootstrap"
+
+  # use the release binaries by setting build_cachewarmer to false
+  build_cachewarmer = false
+
+  depends_on = [
+    module.vfxtcontroller,
+  ]
+}
+
+module "cachewarmer_manager_install" {
+  count  = var.install_cachewarmer ? 1 : 0
+  source = "github.com/Azure/Avere/src/terraform/modules/cachewarmer_manager_install"
+
+  // authentication with controller
+  jumpbox_address      = module.vfxtcontroller[0].controller_address
+  jumpbox_username     = module.vfxtcontroller[0].controller_username
+  jumpbox_password     = data.azurerm_key_vault_secret.virtualmachine.value
+  jumpbox_ssh_key_data = var.ssh_public_key
+  proxy                = local.proxy_uri
+
+  // bootstrap directory to install the cache manager service
+  bootstrap_mount_address       = module.cachewarmer_prepare_bootstrapdir[0].bootstrap_mount_address
+  bootstrap_export_path         = module.cachewarmer_prepare_bootstrapdir[0].bootstrap_export_path
+  bootstrap_manager_script_path = module.cachewarmer_prepare_bootstrapdir[0].cachewarmer_manager_bootstrap_script_path
+
+  // the job path
+  storage_account    = azurerm_storage_account.storage[0].name
+  storage_account_rg = azurerm_storage_account.storage[0].resource_group_name
+  queue_name_prefix  = var.queue_prefix_name
+
+  // the cachewarmer VMSS auth details
+  vmss_user_name      = module.vfxtcontroller[0].controller_username
+  vmss_password       = data.azurerm_key_vault_secret.virtualmachine.value
+  vmss_ssh_public_key = var.ssh_public_key
+  vmss_subnet_name    = data.terraform_remote_state.network.outputs.render_subnet_name
+  vmss_worker_count   = length(local.deployAverevFXT ? avere_vfxt.vfxt[0].node_names : azurerm_hpc_cache.hpc_cache[0].mount_addresses) * 4 // 4 D2sv3 nodes per cache node
+
+  // the cachewarmer install the work script
+  bootstrap_worker_script_path = module.cachewarmer_prepare_bootstrapdir[0].cachewarmer_worker_bootstrap_script_path
+
+  depends_on = [
+    module.cachewarmer_prepare_bootstrapdir,
+    avere_vfxt.vfxt,
+    azurerm_hpc_cache.hpc_cache,
+    azurerm_storage_account.storage,
+  ]
+}
+
+module "cachewarmer_worker_install" {
+  count  = var.install_cachewarmer ? 1 : 0
+  source = "github.com/Azure/Avere/src/terraform/modules/cachewarmer_worker_install"
+
+  // authentication with controller
+  jumpbox_address      = module.vfxtcontroller[0].controller_address
+  jumpbox_username     = module.vfxtcontroller[0].controller_username
+  jumpbox_password     = data.azurerm_key_vault_secret.virtualmachine.value
+  jumpbox_ssh_key_data = var.ssh_public_key
+  proxy                = local.proxy_uri
+
+  // bootstrap directory to install the cache manager service
+  bootstrap_mount_address      = module.cachewarmer_prepare_bootstrapdir[0].bootstrap_mount_address
+  bootstrap_export_path        = module.cachewarmer_prepare_bootstrapdir[0].bootstrap_export_path
+  bootstrap_worker_script_path = module.cachewarmer_prepare_bootstrapdir[0].cachewarmer_worker_bootstrap_script_path
+
+  // the job path
+  storage_account    = azurerm_storage_account.storage[0].name
+  storage_account_rg = azurerm_storage_account.storage[0].resource_group_name
+  queue_name_prefix  = var.queue_prefix_name
+
+  depends_on = [
+    module.cachewarmer_manager_install,
+  ]
+}
+
+module "cachewarmer_submitjobs" {
+  count  = var.install_cachewarmer ? 1 : 0
+  source = "github.com/Azure/Avere/src/terraform/modules/cachewarmer_submitjobs"
+
+  // authentication with controller
+  jumpbox_address      = module.vfxtcontroller[0].controller_address
+  jumpbox_username     = module.vfxtcontroller[0].controller_username
+  jumpbox_password     = data.azurerm_key_vault_secret.virtualmachine.value
+  jumpbox_ssh_key_data = var.ssh_public_key
+  proxy                = local.proxy_uri
+
+  // the job path
+  storage_account    = azurerm_storage_account.storage[0].name
+  storage_account_rg = azurerm_storage_account.storage[0].resource_group_name
+  queue_name_prefix  = var.queue_prefix_name
+
+  // the path to warm
+  warm_mount_addresses = join(",", tolist(local.deployAverevFXT ? avere_vfxt.vfxt[0].vserver_ip_addresses : azurerm_hpc_cache.hpc_cache[0].mount_addresses))
+  warm_paths = {
+    "${data.terraform_remote_state.onprem.outputs.nfsfiler_export}" : ["/tools", "/island"],
+  }
+
+  inclusion_csv    = "" // example "*.jpg,*.png"
+  exclusion_csv    = "" // example "*.tgz,*.tmp"
+  maxFileSizeBytes = 0
+
+  block_until_warm = true
+
+  depends_on = [
+    module.cachewarmer_worker_install,
+    avere_vfxt.vfxt,
+    azurerm_hpc_cache.hpc_cache,
+    azurerm_storage_account.storage,
+  ]
+}
+
 ### Outputs
 output "controller_username" {
   value = length(module.vfxtcontroller) == 0 ? "" : module.vfxtcontroller[0].controller_username
@@ -421,5 +578,3 @@ output "mount_addresses" {
 output "management_ip" {
   value = local.deployAverevFXT ? avere_vfxt.vfxt[0].vfxt_management_ip : ""
 }
-
-

src/terraform/modules/cachewarmer_manager_install/main.tf

@@ -3,7 +3,8 @@ locals {
   vmss_password_str       = var.vmss_password == null ? "" : var.vmss_password
   vmss_ssh_public_key_str = var.vmss_ssh_public_key == null ? "" : var.vmss_ssh_public_key
   vmss_subnet_name_str    = var.vmss_subnet_name == null ? "" : var.vmss_subnet_name
-  env_vars                = "export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix} && export BOOTSTRAP_EXPORT_PATH=${var.bootstrap_export_path} && export BOOTSTRAP_MOUNT_ADDRESS=${var.bootstrap_mount_address} && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path} && export VMSS_USERNAME=${var.vmss_user_name} && export VMSS_SSHPUBLICKEY='${local.vmss_ssh_public_key_str}' && export VMSS_PASSWORD='${local.vmss_password_str}' && export VMSS_SUBNET=${local.vmss_subnet_name_str} && export VMSS_WORKER_COUNT=${var.vmss_worker_count}"
+  proxy_env               = (var.proxy == null || var.proxy == "") ? "" : "export http_proxy=${var.proxy} && export https_proxy=${var.proxy} && export no_proxy=169.254.169.254 &&"
+  env_vars                = "${local.proxy_env} export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix} && export BOOTSTRAP_EXPORT_PATH=${var.bootstrap_export_path} && export BOOTSTRAP_MOUNT_ADDRESS=${var.bootstrap_mount_address} && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path} && export VMSS_USERNAME=${var.vmss_user_name} && export VMSS_SSHPUBLICKEY='${local.vmss_ssh_public_key_str}' && export VMSS_PASSWORD='${local.vmss_password_str}' && export VMSS_SUBNET=${local.vmss_subnet_name_str} && export VMSS_WORKER_COUNT=${var.vmss_worker_count}"
 }
 
 resource "null_resource" "install_cachewarmer_manager" {

src/terraform/modules/cachewarmer_manager_install/variables.tf

@@ -75,3 +75,8 @@ variable "vmss_worker_count" {
   default     = 12
   type        = number
 }
+
+variable "proxy" {
+  description = "specify a proxy address if one exists in the format of http://PROXY_SERVER:PORT"
+  default     = null
+}

src/terraform/modules/cachewarmer_prepare_bootstrapdir/main.tf

@@ -1,7 +1,7 @@
 locals {
   mount_dir = "/bcwpb"
 
-  proxy_env = (var.proxy == null || var.proxy == "") ? "" : "http_proxy=${var.proxy} https_proxy=${var.proxy} no_proxy=169.254.169.254"
+  proxy_env = (var.proxy == null || var.proxy == "") ? "" : "export http_proxy=${var.proxy} && export https_proxy=${var.proxy} && export no_proxy=169.254.169.254 && "
 
   build_cachewarmer_lines = [
     "set -x",

src/terraform/modules/cachewarmer_submitjobs/main.tf

@@ -7,6 +7,7 @@ locals {
   maxFileSizeBytes = var.maxFileSizeBytes == 0 ? "" : " -maxFileSizeBytes ${var.maxFileSizeBytes} "
   inclusion_csv    = var.inclusion_csv == null || length(var.inclusion_csv) == 0 ? "" : " -inclusionCsv \"${var.inclusion_csv}\" "
   exclusion_csv    = var.exclusion_csv == null || length(var.exclusion_csv) == 0 ? "" : " -exclusionCsv \"${var.exclusion_csv}\" "
+  proxy_env        = (var.proxy == null || var.proxy == "") ? "" : "export http_proxy=${var.proxy} && export https_proxy=${var.proxy} && export no_proxy=169.254.169.254 &&"
 }
 
 resource "null_resource" "cachewarmer_submitmultiplejobs" {
@@ -24,7 +25,7 @@ resource "null_resource" "cachewarmer_submitmultiplejobs" {
   provisioner "remote-exec" {
     inline = [
       "set -x",
-      "sudo /usr/local/bin/cachewarmer-jobsubmitter -storageAccountName ${var.storage_account} -storageAccountResourceGroup ${var.storage_account_rg} -queueNamePrefix ${var.queue_name_prefix} -warmTargetExportPath \"${local.warm_paths_sets[count.index].export}\" -warmTargetMountAddresses \"${var.warm_mount_addresses}\" -warmTargetPath \"${local.warm_paths_sets[count.index].path}\" ${local.inclusion_csv} ${local.exclusion_csv} ${local.maxFileSizeBytes} ${var.block_until_warm && count.index == 0 ? local.block_flag : local.no_block_flag}",
+      "${local.proxy_env} sudo -E /usr/local/bin/cachewarmer-jobsubmitter -storageAccountName ${var.storage_account} -storageAccountResourceGroup ${var.storage_account_rg} -queueNamePrefix ${var.queue_name_prefix} -warmTargetExportPath \"${local.warm_paths_sets[count.index].export}\" -warmTargetMountAddresses \"${var.warm_mount_addresses}\" -warmTargetPath \"${local.warm_paths_sets[count.index].path}\" ${local.inclusion_csv} ${local.exclusion_csv} ${local.maxFileSizeBytes} ${var.block_until_warm && count.index == 0 ? local.block_flag : local.no_block_flag}",
     ]
   }
 }

src/terraform/modules/cachewarmer_submitjobs/variables.tf

@@ -45,12 +45,12 @@ variable "warm_paths" {
 
 variable "inclusion_csv" {
   description = "the inclusion list of file match strings per https://golang.org/pkg/path/filepath/#Match.  Leave blank to include everything."
-  default = ""
+  default     = ""
 }
 
 variable "exclusion_csv" {
   description = "the exclusion list of file match strings per https://golang.org/pkg/path/filepath/#Match.  Leave blank to not exlude anything."
-  default = ""
+  default     = ""
 }
 
 variable "maxFileSizeBytes" {
@@ -63,3 +63,9 @@ variable "block_until_warm" {
   description = "block the operation until the cache warming has finished"
   default     = true
 }
+
+
+variable "proxy" {
+  description = "specify a proxy address if one exists in the format of http://PROXY_SERVER:PORT"
+  default     = null
+}

src/terraform/modules/cachewarmer_worker_install/main.tf

@@ -1,6 +1,7 @@
 locals {
   mount_dir = "/b"
-  env_vars  = "export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix}  && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path}"
+  proxy_env = (var.proxy == null || var.proxy == "") ? "" : "export http_proxy=${var.proxy} && export https_proxy=${var.proxy} && export no_proxy=169.254.169.254 &&"
+  env_vars  = "${local.proxy_env} export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix}  && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path}"
 }
 
 resource "null_resource" "install_cachewarmer_worker" {

src/terraform/modules/cachewarmer_worker_install/variables.tf

@@ -45,3 +45,8 @@ variable "storage_account" {
 variable "queue_name_prefix" {
   description = "the queue name prefix for the job management"
 }
+
+variable "proxy" {
+  description = "specify a proxy address if one exists in the format of http://PROXY_SERVER:PORT"
+  default     = null
+}