From c00410c37db17ee661c8fe957ff9c945084bab4d Mon Sep 17 00:00:00 2001 From: David Moore Date: Thu, 6 Apr 2023 09:53:03 +0100 Subject: [PATCH 1/7] build Dockerfile with Nexus config --- .../workspace_services/azureml/porter.yaml | 123 +++++++++++++++++- .../azureml/terraform/acr.tf | 6 +- .../azureml/terraform/data.tf | 4 + .../azureml/terraform/outputs.tf | 8 ++ .../terraform/private_image/Dockerfile | 22 ++++ .../azureml/terraform/variables.tf | 4 + 6 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 templates/workspace_services/azureml/terraform/private_image/Dockerfile diff --git a/templates/workspace_services/azureml/porter.yaml b/templates/workspace_services/azureml/porter.yaml index 8a90132e1b..e69a61463a 100644 --- a/templates/workspace_services/azureml/porter.yaml +++ b/templates/workspace_services/azureml/porter.yaml @@ -71,6 +71,11 @@ outputs: applyTo: - install - upgrade + - name: azureml_acr_name + type: string + applyTo: + - install + - upgrade - name: azureml_storage_account_id type: string applyTo: @@ -113,6 +118,7 @@ outputs: - upgrade mixins: + - exec - terraform: clientVersion: 1.3.6 - az: @@ -133,6 +139,7 @@ install: auth_client_id: ${ bundle.credentials.auth_client_id } auth_client_secret: ${ bundle.credentials.auth_client_secret } auth_tenant_id: ${ bundle.credentials.auth_tenant_id } + public_access_enabled: true backendConfig: resource_group_name: ${ bundle.parameters.tfstate_resource_group_name } storage_account_name: ${ bundle.parameters.tfstate_storage_account_name } @@ -141,6 +148,60 @@ install: outputs: - name: azureml_workspace_name - name: azureml_acr_id + - name: azureml_acr_name + - name: core_location + - name: azureml_storage_account_id + - name: connection_uri + - name: internal_connection_uri + - name: workspace_address_spaces + - name: aml_subnet_address_prefixes + - name: storage_tag + - name: batch_tag + - name: mcr_tag + + - az: + description: "Login to Azure" + arguments: + - login + flags: + identity: + username: ${ bundle.credentials.azure_client_id } + + - az: + description: "Build and push private AML image via ACR Tasks" + arguments: + - acr + - build + - ./terraform/private_image/ + flags: + registry: ${ bundle.outputs.azureml_acr_name } + image: aml-tre-nexus-image + build-arg: "TRE_LOCATION=${ bundle.outputs.core_location } --build-arg TRE_ID=${ bundle.parameters.tre_id }" + + - terraform: + description: "Update ACR to close network" + vars: + workspace_id: ${ bundle.parameters.workspace_id } + tre_id: ${ bundle.parameters.tre_id } + tre_resource_id: ${ bundle.parameters.id } + display_name: ${ bundle.parameters.display_name } + description: ${ bundle.parameters.description } + address_space: ${ bundle.parameters.address_space } + is_exposed_externally: ${ bundle.parameters.is_exposed_externally } + arm_tenant_id: ${ bundle.credentials.azure_tenant_id } + auth_client_id: ${ bundle.credentials.auth_client_id } + auth_client_secret: ${ bundle.credentials.auth_client_secret } + auth_tenant_id: ${ bundle.credentials.auth_tenant_id } + public_access_enabled: false + backendConfig: + resource_group_name: ${ bundle.parameters.tfstate_resource_group_name } + storage_account_name: ${ bundle.parameters.tfstate_storage_account_name } + container_name: ${ bundle.parameters.tfstate_container_name } + key: tre-service-azureml-${ bundle.parameters.id } + outputs: + - name: azureml_workspace_name + - name: azureml_acr_id + - name: azureml_acr_name - name: azureml_storage_account_id - name: connection_uri - name: internal_connection_uri @@ -152,7 +213,65 @@ install: upgrade: - terraform: - description: "Upgrade Azure ML Service" + description: "Upgrade Azure ML Service, with open network" + vars: + workspace_id: ${ bundle.parameters.workspace_id } + tre_id: ${ bundle.parameters.tre_id } + tre_resource_id: ${ bundle.parameters.id } + display_name: ${ bundle.parameters.display_name } + description: ${ bundle.parameters.description } + address_space: ${ bundle.parameters.address_space } + is_exposed_externally: ${ bundle.parameters.is_exposed_externally } + arm_tenant_id: ${ bundle.credentials.azure_tenant_id } + auth_client_id: ${ bundle.credentials.auth_client_id } + auth_client_secret: ${ bundle.credentials.auth_client_secret } + auth_tenant_id: ${ bundle.credentials.auth_tenant_id } + public_access_enabled: true + backendConfig: + resource_group_name: ${ bundle.parameters.tfstate_resource_group_name } + storage_account_name: ${ bundle.parameters.tfstate_storage_account_name } + container_name: ${ bundle.parameters.tfstate_container_name } + key: tre-service-azureml-${ bundle.parameters.id } + outputs: + - name: azureml_workspace_name + - name: azureml_acr_id + - name: azureml_acr_name + - name: core_location + - name: azureml_storage_account_id + - name: connection_uri + - name: internal_connection_uri + - name: workspace_address_spaces + - name: aml_subnet_address_prefixes + - name: storage_tag + - name: batch_tag + - name: mcr_tag + + - exec: + command: sleep #30 # wait for public network access to _actually_ take effect + arguments: + - "30" + + - az: + description: "Login to Azure" + arguments: + - login + flags: + identity: + username: ${ bundle.credentials.azure_client_id } + + - az: + description: "Build and push private AML image via ACR Tasks" + arguments: + - acr + - build + - ./terraform/private_image/ + flags: + registry: ${ bundle.outputs.azureml_acr_name } + image: aml-tre-nexus-image + build-arg: "TRE_LOCATION=${ bundle.outputs.core_location } --build-arg TRE_ID=${ bundle.parameters.tre_id }" + + - terraform: + description: "Update ACR to close network" vars: workspace_id: ${ bundle.parameters.workspace_id } tre_id: ${ bundle.parameters.tre_id } @@ -165,6 +284,7 @@ upgrade: auth_client_id: ${ bundle.credentials.auth_client_id } auth_client_secret: ${ bundle.credentials.auth_client_secret } auth_tenant_id: ${ bundle.credentials.auth_tenant_id } + public_access_enabled: false backendConfig: resource_group_name: ${ bundle.parameters.tfstate_resource_group_name } storage_account_name: ${ bundle.parameters.tfstate_storage_account_name } @@ -173,6 +293,7 @@ upgrade: outputs: - name: azureml_workspace_name - name: azureml_acr_id + - name: azureml_acr_name - name: azureml_storage_account_id - name: connection_uri - name: internal_connection_uri diff --git a/templates/workspace_services/azureml/terraform/acr.tf b/templates/workspace_services/azureml/terraform/acr.tf index 33b4ae315f..346bf0eaaf 100644 --- a/templates/workspace_services/azureml/terraform/acr.tf +++ b/templates/workspace_services/azureml/terraform/acr.tf @@ -1,12 +1,10 @@ - - resource "azurerm_container_registry" "acr" { name = local.acr_name location = data.azurerm_resource_group.ws.location resource_group_name = data.azurerm_resource_group.ws.name sku = "Premium" admin_enabled = false - public_network_access_enabled = false + public_network_access_enabled = var.public_access_enabled tags = local.tre_workspace_service_tags lifecycle { ignore_changes = [tags] } @@ -37,6 +35,4 @@ resource "azurerm_private_endpoint" "acrpe" { is_manual_connection = false subresource_names = ["registry"] } - } - diff --git a/templates/workspace_services/azureml/terraform/data.tf b/templates/workspace_services/azureml/terraform/data.tf index 63cf294ad7..9b84a0be11 100644 --- a/templates/workspace_services/azureml/terraform/data.tf +++ b/templates/workspace_services/azureml/terraform/data.tf @@ -2,6 +2,10 @@ data "azurerm_resource_group" "ws" { name = "rg-${var.tre_id}-ws-${local.short_workspace_id}" } +data "azurerm_resource_group" "core" { + name = "rg-${var.tre_id}" +} + data "azurerm_virtual_network" "ws" { name = "vnet-${var.tre_id}-ws-${local.short_workspace_id}" resource_group_name = data.azurerm_resource_group.ws.name diff --git a/templates/workspace_services/azureml/terraform/outputs.tf b/templates/workspace_services/azureml/terraform/outputs.tf index 427828e4ab..8c69f1871e 100644 --- a/templates/workspace_services/azureml/terraform/outputs.tf +++ b/templates/workspace_services/azureml/terraform/outputs.tf @@ -6,6 +6,14 @@ output "azureml_acr_id" { value = azurerm_container_registry.acr.id } +output "azureml_acr_name" { + value = azurerm_container_registry.acr.name +} + +output "core_location" { + value = data.azurerm_resource_group.core.location +} + output "azureml_storage_account_id" { value = azurerm_storage_account.aml.id } diff --git a/templates/workspace_services/azureml/terraform/private_image/Dockerfile b/templates/workspace_services/azureml/terraform/private_image/Dockerfile new file mode 100644 index 0000000000..89b2bf285c --- /dev/null +++ b/templates/workspace_services/azureml/terraform/private_image/Dockerfile @@ -0,0 +1,22 @@ +# https://github.com/Azure/AzureML-Containers/blob/master/base/cpu/openmpi4.1.0-ubuntu22.04/Dockerfile + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04 + +ARG TRE_ID +ARG TRE_LOCATION +ENV NEXUS_PROXY_URL="https://nexus-${TRE_ID}.${TRE_LOCATION}.cloudapp.azure.com" + +# Set Nexus Conda +ENV PATH="/anaconda/condabin":$PATH +ENV PATH="/anaconda/bin":$PATH +ENV PATH="/anaconda/envs/py38_default/bin":$PATH + +RUN conda config --add channels "${NEXUS_PROXY_URL}/repository/conda-mirror/main/" --system && \ + conda config --add channels "${NEXUS_PROXY_URL}/repository/conda-repo/main/" --system && \ + conda config --remove channels defaults --system && \ + conda config --set channel_alias "${NEXUS_PROXY_URL}/repository/conda-mirror/" --system + +# Set PyPi sources +RUN printf "[global]\nindex = ${NEXUS_PROXY_URL}/repository/pypi/pypi\nindex-url = ${NEXUS_PROXY_URL}/repository/pypi/simple\ntrusted-host = ${NEXUS_PROXY_URL}\n" >> /etc/pip.conf diff --git a/templates/workspace_services/azureml/terraform/variables.tf b/templates/workspace_services/azureml/terraform/variables.tf index ca61e4c792..b3e0706b47 100644 --- a/templates/workspace_services/azureml/terraform/variables.tf +++ b/templates/workspace_services/azureml/terraform/variables.tf @@ -35,3 +35,7 @@ variable "auth_client_secret" { sensitive = true description = "Used to authenticate into the AAD Tenant to get app role members" } +variable "public_access_enabled" { + type = bool + default = false +} From 7f70b902d678aaa4e6cfbd2b0e147ced2b29d6e5 Mon Sep 17 00:00:00 2001 From: David Moore Date: Thu, 6 Apr 2023 08:58:58 +0000 Subject: [PATCH 2/7] fmt --- templates/workspace_services/azureml/terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/workspace_services/azureml/terraform/variables.tf b/templates/workspace_services/azureml/terraform/variables.tf index 758b0a2712..d80066a89c 100644 --- a/templates/workspace_services/azureml/terraform/variables.tf +++ b/templates/workspace_services/azureml/terraform/variables.tf @@ -36,7 +36,7 @@ variable "auth_client_secret" { description = "Used to authenticate into the AAD Tenant to get app role members" } variable "public_access_enabled" { - type = bool + type = bool default = false } From 289276c8560ed00ac3db38ee23ebcb187131710b Mon Sep 17 00:00:00 2001 From: David Moore Date: Thu, 6 Apr 2023 10:15:15 +0100 Subject: [PATCH 3/7] lint + version bump --- templates/workspace_services/azureml/porter.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/workspace_services/azureml/porter.yaml b/templates/workspace_services/azureml/porter.yaml index 490604d7c4..c0e250e109 100644 --- a/templates/workspace_services/azureml/porter.yaml +++ b/templates/workspace_services/azureml/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-azureml -version: 0.8.0 +version: 0.9.0 description: "An Azure TRE service for Azure Machine Learning" registry: azuretre dockerfile: Dockerfile.tmpl @@ -257,7 +257,7 @@ upgrade: - name: mcr_tag - exec: - command: sleep #30 # wait for public network access to _actually_ take effect + command: sleep # wait for public network access to _actually_ take effect arguments: - "30" From 3871048de10d69345787b189ddcd714c6d48f933 Mon Sep 17 00:00:00 2001 From: David Moore Date: Thu, 6 Apr 2023 10:36:05 +0100 Subject: [PATCH 4/7] tagging FROM with latest --- .../azureml/terraform/private_image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/workspace_services/azureml/terraform/private_image/Dockerfile b/templates/workspace_services/azureml/terraform/private_image/Dockerfile index 89b2bf285c..827cd38718 100644 --- a/templates/workspace_services/azureml/terraform/private_image/Dockerfile +++ b/templates/workspace_services/azureml/terraform/private_image/Dockerfile @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04 +FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:latest ARG TRE_ID ARG TRE_LOCATION From 7c1630ef4e8cf18abf05fd4346c8583926aa6895 Mon Sep 17 00:00:00 2001 From: David Moore Date: Thu, 6 Apr 2023 10:54:29 +0100 Subject: [PATCH 5/7] tagging openmpi image with sha --- .../azureml/terraform/private_image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/workspace_services/azureml/terraform/private_image/Dockerfile b/templates/workspace_services/azureml/terraform/private_image/Dockerfile index 827cd38718..ea213f6a22 100644 --- a/templates/workspace_services/azureml/terraform/private_image/Dockerfile +++ b/templates/workspace_services/azureml/terraform/private_image/Dockerfile @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:latest +FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04@sha256:2c217d40498433d578cd4ec6cc519b49b324da34aca1e0df1930e1552c89af7d ARG TRE_ID ARG TRE_LOCATION From bd07f521cacb88249881b0eda3526a9081529a83 Mon Sep 17 00:00:00 2001 From: David Moore Date: Thu, 6 Apr 2023 11:28:49 +0100 Subject: [PATCH 6/7] printf --- .../azureml/terraform/private_image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/workspace_services/azureml/terraform/private_image/Dockerfile b/templates/workspace_services/azureml/terraform/private_image/Dockerfile index ea213f6a22..8a2590bf25 100644 --- a/templates/workspace_services/azureml/terraform/private_image/Dockerfile +++ b/templates/workspace_services/azureml/terraform/private_image/Dockerfile @@ -19,4 +19,4 @@ RUN conda config --add channels "${NEXUS_PROXY_URL}/repository/conda-mirror/main conda config --set channel_alias "${NEXUS_PROXY_URL}/repository/conda-mirror/" --system # Set PyPi sources -RUN printf "[global]\nindex = ${NEXUS_PROXY_URL}/repository/pypi/pypi\nindex-url = ${NEXUS_PROXY_URL}/repository/pypi/simple\ntrusted-host = ${NEXUS_PROXY_URL}\n" >> /etc/pip.conf +RUN printf "[global]\nindex = %s/repository/pypi/pypi\nindex-url = %s/repository/pypi/simple\ntrusted-host = %s\n" "${NEXUS_PROXY_URL}" "${NEXUS_PROXY_URL}" "${NEXUS_PROXY_URL}" >> /etc/pip.conf From 2dceb8048a4ec48b304e06dda72b97cbb745a0f9 Mon Sep 17 00:00:00 2001 From: David Moore Date: Thu, 6 Apr 2023 14:02:07 +0100 Subject: [PATCH 7/7] configurable azure_endpoint for gov cloud --- templates/workspace_services/azureml/porter.yaml | 10 ++++++++-- .../workspace_services/azureml/terraform/acr.tf | 15 +++++++++------ .../azureml/terraform/outputs.tf | 4 ++++ .../azureml/terraform/private_image/Dockerfile | 3 ++- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/templates/workspace_services/azureml/porter.yaml b/templates/workspace_services/azureml/porter.yaml index c0e250e109..76ace889b2 100644 --- a/templates/workspace_services/azureml/porter.yaml +++ b/templates/workspace_services/azureml/porter.yaml @@ -164,6 +164,7 @@ install: - name: storage_tag - name: batch_tag - name: mcr_tag + - name: azure_endpoint - az: description: "Login to Azure" @@ -182,7 +183,7 @@ install: flags: registry: ${ bundle.outputs.azureml_acr_name } image: aml-tre-nexus-image - build-arg: "TRE_LOCATION=${ bundle.outputs.core_location } --build-arg TRE_ID=${ bundle.parameters.tre_id }" + build-arg: "TRE_LOCATION=${ bundle.outputs.core_location } --build-arg TRE_ID=${ bundle.parameters.tre_id } --build-arg AZURE_ENDPOINT=${ bundle.outputs.azure_endpoint }" - terraform: description: "Update ACR to close network" @@ -218,6 +219,7 @@ install: - name: storage_tag - name: batch_tag - name: mcr_tag + - name: azure_endpoint upgrade: - terraform: @@ -255,6 +257,7 @@ upgrade: - name: storage_tag - name: batch_tag - name: mcr_tag + - name: azure_endpoint - exec: command: sleep # wait for public network access to _actually_ take effect @@ -278,7 +281,7 @@ upgrade: flags: registry: ${ bundle.outputs.azureml_acr_name } image: aml-tre-nexus-image - build-arg: "TRE_LOCATION=${ bundle.outputs.core_location } --build-arg TRE_ID=${ bundle.parameters.tre_id }" + build-arg: "TRE_LOCATION=${ bundle.outputs.core_location } --build-arg TRE_ID=${ bundle.parameters.tre_id } --build-arg AZURE_ENDPOINT=${ bundle.outputs.azure_endpoint }" - terraform: description: "Update ACR to close network" @@ -314,6 +317,7 @@ upgrade: - name: storage_tag - name: batch_tag - name: mcr_tag + - name: azure_endpoint uninstall: - terraform: @@ -330,7 +334,9 @@ uninstall: auth_client_id: ${ bundle.credentials.auth_client_id } auth_client_secret: ${ bundle.credentials.auth_client_secret } auth_tenant_id: ${ bundle.credentials.auth_tenant_id } + public_access_enabled: false arm_environment: ${ bundle.parameters.arm_environment } + azure_environment: ${ bundle.parameters.azure_environment } backendConfig: resource_group_name: ${ bundle.parameters.tfstate_resource_group_name } storage_account_name: ${ bundle.parameters.tfstate_storage_account_name } diff --git a/templates/workspace_services/azureml/terraform/acr.tf b/templates/workspace_services/azureml/terraform/acr.tf index 35058a92e2..984d57820f 100644 --- a/templates/workspace_services/azureml/terraform/acr.tf +++ b/templates/workspace_services/azureml/terraform/acr.tf @@ -1,11 +1,14 @@ resource "azurerm_container_registry" "acr" { - name = local.acr_name - location = data.azurerm_resource_group.ws.location - resource_group_name = data.azurerm_resource_group.ws.name - sku = "Premium" - admin_enabled = false + name = local.acr_name + location = data.azurerm_resource_group.ws.location + resource_group_name = data.azurerm_resource_group.ws.name + sku = "Premium" + admin_enabled = false + + # Allows porter to run once with network open, upload an image, then run again to close the network public_network_access_enabled = var.public_access_enabled - tags = local.tre_workspace_service_tags + + tags = local.tre_workspace_service_tags lifecycle { ignore_changes = [tags] } } diff --git a/templates/workspace_services/azureml/terraform/outputs.tf b/templates/workspace_services/azureml/terraform/outputs.tf index 8c69f1871e..c1f1fa26f4 100644 --- a/templates/workspace_services/azureml/terraform/outputs.tf +++ b/templates/workspace_services/azureml/terraform/outputs.tf @@ -63,3 +63,7 @@ output "mcr_tag" { output "batch_tag" { value = data.azurerm_network_service_tags.batch_tag.id } + +output "azure_endpoint" { + value = var.azure_environment == "AzureGovCloud" ? "cloudapp.usgovcloudapi.net" : "cloudapp.azure.com" +} diff --git a/templates/workspace_services/azureml/terraform/private_image/Dockerfile b/templates/workspace_services/azureml/terraform/private_image/Dockerfile index 8a2590bf25..f38b0bd1c8 100644 --- a/templates/workspace_services/azureml/terraform/private_image/Dockerfile +++ b/templates/workspace_services/azureml/terraform/private_image/Dockerfile @@ -6,7 +6,8 @@ FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04@sha256:2c217d40498433d57 ARG TRE_ID ARG TRE_LOCATION -ENV NEXUS_PROXY_URL="https://nexus-${TRE_ID}.${TRE_LOCATION}.cloudapp.azure.com" +ARG AZURE_ENDPOINT +ENV NEXUS_PROXY_URL="https://nexus-${TRE_ID}.${TRE_LOCATION}.${AZURE_ENDPOINT}" # Set Nexus Conda ENV PATH="/anaconda/condabin":$PATH