Merge pull request #17 from anyscale/brent/iam-minimalrole

feat: Shared VPCs support, Minimal scoped IAM Role

.gitignore

@@ -12,6 +12,7 @@
 crash.log
 crash.*.log
 
+buildkite-*
 # Exclude all .tfvars files, which are likely to contain sensitive data, such as
 # password, private keys, and other secrets. These should not be part of version
 # control as they are data points which are potentially sensitive and subject

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 default_stages: [commit]
 repos:
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.83.3
+    rev: v1.84.0
     hooks:
       - id: terraform_fmt
       - id: terraform_validate
@@ -22,7 +22,7 @@ repos:
         args:
           - --args=provider google -v "~> 4.0"
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer

.tflint.hcl

@@ -1,6 +1,6 @@
 plugin "google" {
   enabled = true
-  version = "0.25.0"
+  version = "0.26.0"
   source  = "github.com/terraform-linters/tflint-ruleset-google"
 }
 

CHANGELOG.md

@@ -1,3 +1,27 @@
+## 0.10.0 (Released)
+FEATURES:
+- Support for shared VPC
+  - Example `anyscale-v2-vpc-shared` created to demonstrate this. Please make sure to read the README in the example for additional requirements.
+  - Requires Anyscale CLI v0.5.163 or greater.
+  - Supports creating a Firewall in the Shared VPC Project if an existing firewall is not provided
+- Update VPC Firewall to allow GCP Load Balancer Health Check CIDR ranges.
+  - This rule is required for Anyscale Services
+- Minimal IAM Service Account Roles
+  - Project/Owner or Project/Editor are no longer required for the Anyscale Service Account.
+
+BUG FIXES:
+
+BREAKING CHANGES:
+- Updates to output names for Service Accounts.
+- Changes to IAM Service Account Terraform names
+  - This will replace existing IAM Service Accounts if upgrading.
+  - You will need to create new Anyscale Clouds.
+
+OTHER:
+- TFLint Rules Update
+- Cleanup of unused variables
+- pre-commit update
+
 ## 0.9.0 (Released)
 FEATURES:
 - Support for existing Workload Identify Federation pool and provider

data.tf

@@ -6,3 +6,10 @@ data "google_compute_subnetwork" "existing_vpc_subnet" {
 
   name = var.existing_vpc_subnet_name
 }
+
+data "google_compute_subnetwork" "shared_vpc_subnet" {
+  count   = local.execute_vpc_firewall_sub_module && var.existing_vpc_subnet_name != null && var.existing_vpc_name != null && var.shared_vpc_project_id != null ? 1 : 0
+  project = var.shared_vpc_project_id
+
+  name = var.existing_vpc_subnet_name
+}

examples/anyscale-v2-commonname/README.md

@@ -53,17 +53,5 @@ No resources.
 
 | Name | Description |
 |------|-------------|
-| <a name="output_anyscale_iam_cluster_node_role_email"></a> [anyscale\_iam\_cluster\_node\_role\_email](#output\_anyscale\_iam\_cluster\_node\_role\_email) | The Anyscale cluster service account email. |
-| <a name="output_anyscale_iam_service_account_email"></a> [anyscale\_iam\_service\_account\_email](#output\_anyscale\_iam\_service\_account\_email) | The Anyscale service account email. |
-| <a name="output_anyscale_iam_workload_identity_provider_id"></a> [anyscale\_iam\_workload\_identity\_provider\_id](#output\_anyscale\_iam\_workload\_identity\_provider\_id) | The Anyscale workload identity provider id. |
-| <a name="output_anyscale_iam_workload_identity_provider_name"></a> [anyscale\_iam\_workload\_identity\_provider\_name](#output\_anyscale\_iam\_workload\_identity\_provider\_name) | The Anyscale workload identity provider name. |
-| <a name="output_cloudstorage_bucket_name"></a> [cloudstorage\_bucket\_name](#output\_cloudstorage\_bucket\_name) | The Google Cloud Storage bucket name. |
-| <a name="output_filestore_instance_id"></a> [filestore\_instance\_id](#output\_filestore\_instance\_id) | The Google Filestore instance id. |
-| <a name="output_filestore_location"></a> [filestore\_location](#output\_filestore\_location) | The Google Filestore location. |
-| <a name="output_firewall_policy_name"></a> [firewall\_policy\_name](#output\_firewall\_policy\_name) | The Google VPC firewall policy name. |
-| <a name="output_project_name"></a> [project\_name](#output\_project\_name) | The Google Project name. |
 | <a name="output_registration_command"></a> [registration\_command](#output\_registration\_command) | The Anyscale registration command. |
-| <a name="output_subnet_name"></a> [subnet\_name](#output\_subnet\_name) | The Google VPC public subnet name. |
-| <a name="output_subnet_region"></a> [subnet\_region](#output\_subnet\_region) | The Google VPC public subnet region. |
-| <a name="output_vpc_name"></a> [vpc\_name](#output\_vpc\_name) | The Google VPC network name. |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/anyscale-v2-commonname/outputs.tf

@@ -1,58 +1,3 @@
-output "vpc_name" {
-  description = "The Google VPC network name."
-  value       = module.google_anyscale_v2_commonname.vpc_name
-}
-output "subnet_name" {
-  description = "The Google VPC public subnet name."
-  value       = module.google_anyscale_v2_commonname.public_subnet_name
-}
-output "subnet_region" {
-  description = "The Google VPC public subnet region."
-  value       = module.google_anyscale_v2_commonname.public_subnet_region
-}
-
-output "firewall_policy_name" {
-  description = "The Google VPC firewall policy name."
-  value       = module.google_anyscale_v2_commonname.vpc_firewall_policy_name
-}
-output "cloudstorage_bucket_name" {
-  description = "The Google Cloud Storage bucket name."
-  value       = module.google_anyscale_v2_commonname.cloudstorage_bucket_name
-}
-
-output "project_name" {
-  description = "The Google Project name."
-  value       = module.google_anyscale_v2_commonname.project_name
-}
-
-output "filestore_instance_id" {
-  description = "The Google Filestore instance id."
-  value       = module.google_anyscale_v2_commonname.filestore_name
-}
-output "filestore_location" {
-  description = "The Google Filestore location."
-  value       = module.google_anyscale_v2_commonname.filestore_location
-}
-
-output "anyscale_iam_service_account_email" {
-  description = "The Anyscale service account email."
-  value       = module.google_anyscale_v2_commonname.iam_anyscale_access_role_email
-}
-
-output "anyscale_iam_cluster_node_role_email" {
-  description = "The Anyscale cluster service account email."
-  value       = module.google_anyscale_v2_commonname.iam_anyscale_cluster_node_role_email
-}
-
-output "anyscale_iam_workload_identity_provider_id" {
-  description = "The Anyscale workload identity provider id."
-  value       = module.google_anyscale_v2_commonname.iam_workload_identity_provider_id
-}
-output "anyscale_iam_workload_identity_provider_name" {
-  description = "The Anyscale workload identity provider name."
-  value       = module.google_anyscale_v2_commonname.iam_workload_identity_provider_name
-}
-
 output "registration_command" {
   description = "The Anyscale registration command."
   value       = <<-EOT
@@ -66,8 +11,8 @@ output "registration_command" {
     --cloud-storage-bucket-name ${module.google_anyscale_v2_commonname.cloudstorage_bucket_name} \
     --filestore-instance-id ${module.google_anyscale_v2_commonname.filestore_name} \
     --filestore-location ${module.google_anyscale_v2_commonname.filestore_location} \
-    --anyscale-service-account-email ${module.google_anyscale_v2_commonname.iam_anyscale_access_role_email} \
-    --instance-service-account-email ${module.google_anyscale_v2_commonname.iam_anyscale_cluster_node_role_email} \
+    --anyscale-service-account-email ${module.google_anyscale_v2_commonname.iam_anyscale_access_service_acct_email} \
+    --instance-service-account-email ${module.google_anyscale_v2_commonname.iam_anyscale_cluster_node_service_acct_email} \
     --provider-name ${module.google_anyscale_v2_commonname.iam_workload_identity_provider_name}
   EOT
 }

examples/anyscale-v2-existingidentityfederation/README.md

@@ -43,11 +43,10 @@ No resources.
 | <a name="input_anyscale_google_region"></a> [anyscale\_google\_region](#input\_anyscale\_google\_region) | (Required) Google region to deploy Anyscale resources. | `string` | n/a | yes |
 | <a name="input_anyscale_google_zone"></a> [anyscale\_google\_zone](#input\_anyscale\_google\_zone) | (Required) Google zone to deploy Anyscale resources. | `string` | n/a | yes |
 | <a name="input_anyscale_org_id"></a> [anyscale\_org\_id](#input\_anyscale\_org\_id) | (Required) Anyscale Organization ID | `string` | n/a | yes |
-| <a name="input_billing_account_id"></a> [billing\_account\_id](#input\_billing\_account\_id) | (Required) Google billing account ID to deploy Anyscale resources. | `string` | n/a | yes |
 | <a name="input_customer_ingress_cidr_ranges"></a> [customer\_ingress\_cidr\_ranges](#input\_customer\_ingress\_cidr\_ranges) | The IPv4 CIDR blocks that allows access Anyscale clusters.<br>These are added to the firewall and allows port 443 (https) and 22 (ssh) access.<br>ex: `52.1.1.23/32,10.1.0.0/16'<br>` | `string` | n/a | yes |
+| <a name="input_existing_project_id"></a> [existing\_project\_id](#input\_existing\_project\_id) | (Required) Google project ID to deploy Anyscale resources. | `string` | n/a | yes |
 | <a name="input_existing_workload_identity_provider_name"></a> [existing\_workload\_identity\_provider\_name](#input\_existing\_workload\_identity\_provider\_name) | (Optional) Existing Workload Identity Provider Name.<br><br>The name of an existing Workload Identity Provider that you'd like to use. This can be in a different project.<br><br>You can retrieve the name of an existing Workload Identity Provider by running the following command:<pre>gcloud iam workload-identity-pools providers list --location global --workload-identity-pool anyscale-access-pool</pre>ex:<pre>existing_workload_identity_provider = "projects/1234567890/locations/global/workloadIdentityPools/anyscale-access-pool/providers/anyscale-access-provider"</pre> | `string` | n/a | yes |
 | <a name="input_labels"></a> [labels](#input\_labels) | (Optional) A map of labels to all resources that accept labels. | `map(string)` | <pre>{<br>  "environment": "test",<br>  "test": true<br>}</pre> | no |
-| <a name="input_root_folder_number"></a> [root\_folder\_number](#input\_root\_folder\_number) | (Required) Google Folder number to deploy Anyscale resources. Will create a new sub-project by default. | `string` | n/a | yes |
 
 ## Outputs
 

examples/anyscale-v2-existingidentityfederation/main.tf

@@ -31,8 +31,9 @@ module "google_anyscale_v2_existingidentityfederation" {
   use_common_name = true
 
   # Project Related
-  anyscale_project_billing_account = var.billing_account_id
-  anyscale_project_folder_id       = var.root_folder_number
+  # anyscale_project_billing_account = var.billing_account_id
+  # anyscale_project_folder_id       = var.root_folder_number
+  existing_project_id = var.existing_project_id
 
   # IAM Related
   existing_workload_identity_provider_name = var.existing_workload_identity_provider_name

examples/anyscale-v2-existingidentityfederation/outputs.tf

@@ -3,17 +3,16 @@ output "registration_command" {
   value       = <<-EOT
     anyscale cloud register --provider gcp \
     --name <anyscale_cloud_name> \
-    --project-id ${module.google_anyscale_v2_existingidentityfederation.project_name} \
+    --project-id ${var.existing_project_id} \
     --vpc-name ${module.google_anyscale_v2_existingidentityfederation.vpc_name} \
     --subnet-names ${module.google_anyscale_v2_existingidentityfederation.public_subnet_name} \
     --region ${module.google_anyscale_v2_existingidentityfederation.public_subnet_region} \
     --firewall-policy-names ${module.google_anyscale_v2_existingidentityfederation.vpc_firewall_policy_name} \
     --cloud-storage-bucket-name ${module.google_anyscale_v2_existingidentityfederation.cloudstorage_bucket_name} \
     --filestore-instance-id ${module.google_anyscale_v2_existingidentityfederation.filestore_name} \
     --filestore-location ${module.google_anyscale_v2_existingidentityfederation.filestore_location} \
-    --anyscale-service-account-email ${module.google_anyscale_v2_existingidentityfederation.iam_anyscale_access_role_email} \
-    --instance-service-account-email ${module.google_anyscale_v2_existingidentityfederation.iam_anyscale_cluster_node_role_email} \
+    --anyscale-service-account-email ${module.google_anyscale_v2_existingidentityfederation.iam_anyscale_access_service_acct_email} \
+    --instance-service-account-email ${module.google_anyscale_v2_existingidentityfederation.iam_anyscale_cluster_node_service_acct_email} \
     --provider-name ${var.existing_workload_identity_provider_name}
   EOT
 }
-#

examples/anyscale-v2-existingidentityfederation/variables.tf

@@ -30,12 +30,8 @@ variable "anyscale_org_id" {
 }
 
 # Project Related Required Variables
-variable "root_folder_number" {
-  description = "(Required) Google Folder number to deploy Anyscale resources. Will create a new sub-project by default."
-  type        = string
-}
-variable "billing_account_id" {
-  description = "(Required) Google billing account ID to deploy Anyscale resources."
+variable "existing_project_id" {
+  description = "(Required) Google project ID to deploy Anyscale resources."
   type        = string
 }
 

examples/anyscale-v2-existingproject/README.md

@@ -51,17 +51,5 @@ No resources.
 
 | Name | Description |
 |------|-------------|
-| <a name="output_anyscale_iam_cluster_node_role_email"></a> [anyscale\_iam\_cluster\_node\_role\_email](#output\_anyscale\_iam\_cluster\_node\_role\_email) | The Anyscale cluster service account email. |
-| <a name="output_anyscale_iam_service_account_email"></a> [anyscale\_iam\_service\_account\_email](#output\_anyscale\_iam\_service\_account\_email) | The Anyscale service account email. |
-| <a name="output_anyscale_iam_workload_identity_provider_id"></a> [anyscale\_iam\_workload\_identity\_provider\_id](#output\_anyscale\_iam\_workload\_identity\_provider\_id) | The Anyscale workload identity provider id. |
-| <a name="output_anyscale_iam_workload_identity_provider_name"></a> [anyscale\_iam\_workload\_identity\_provider\_name](#output\_anyscale\_iam\_workload\_identity\_provider\_name) | The Anyscale workload identity provider name. |
-| <a name="output_cloudstorage_bucket_name"></a> [cloudstorage\_bucket\_name](#output\_cloudstorage\_bucket\_name) | The Google Cloud Storage bucket name. |
-| <a name="output_filestore_instance_id"></a> [filestore\_instance\_id](#output\_filestore\_instance\_id) | The Google Filestore instance id. |
-| <a name="output_filestore_location"></a> [filestore\_location](#output\_filestore\_location) | The Google Filestore location. |
-| <a name="output_firewall_policy_name"></a> [firewall\_policy\_name](#output\_firewall\_policy\_name) | The Google VPC firewall policy name. |
-| <a name="output_project_name"></a> [project\_name](#output\_project\_name) | The Google Project name. |
 | <a name="output_registration_command"></a> [registration\_command](#output\_registration\_command) | The Anyscale registration command. |
-| <a name="output_subnet_name"></a> [subnet\_name](#output\_subnet\_name) | The Google VPC public subnet name. |
-| <a name="output_subnet_region"></a> [subnet\_region](#output\_subnet\_region) | The Google VPC public subnet region. |
-| <a name="output_vpc_name"></a> [vpc\_name](#output\_vpc\_name) | The Google VPC network name. |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/anyscale-v2-existingproject/outputs.tf

@@ -1,58 +1,3 @@
-output "vpc_name" {
-  description = "The Google VPC network name."
-  value       = module.google_anyscale_v2_existingproject.vpc_name
-}
-output "subnet_name" {
-  description = "The Google VPC public subnet name."
-  value       = module.google_anyscale_v2_existingproject.public_subnet_name
-}
-output "subnet_region" {
-  description = "The Google VPC public subnet region."
-  value       = module.google_anyscale_v2_existingproject.public_subnet_region
-}
-
-output "firewall_policy_name" {
-  description = "The Google VPC firewall policy name."
-  value       = module.google_anyscale_v2_existingproject.vpc_firewall_policy_name
-}
-output "cloudstorage_bucket_name" {
-  description = "The Google Cloud Storage bucket name."
-  value       = module.google_anyscale_v2_existingproject.cloudstorage_bucket_name
-}
-
-output "project_name" {
-  description = "The Google Project name."
-  value       = module.google_anyscale_v2_existingproject.project_name
-}
-
-output "filestore_instance_id" {
-  description = "The Google Filestore instance id."
-  value       = module.google_anyscale_v2_existingproject.filestore_name
-}
-output "filestore_location" {
-  description = "The Google Filestore location."
-  value       = module.google_anyscale_v2_existingproject.filestore_location
-}
-
-output "anyscale_iam_service_account_email" {
-  description = "The Anyscale service account email."
-  value       = module.google_anyscale_v2_existingproject.iam_anyscale_access_role_email
-}
-
-output "anyscale_iam_cluster_node_role_email" {
-  description = "The Anyscale cluster service account email."
-  value       = module.google_anyscale_v2_existingproject.iam_anyscale_cluster_node_role_email
-}
-
-output "anyscale_iam_workload_identity_provider_id" {
-  description = "The Anyscale workload identity provider id."
-  value       = module.google_anyscale_v2_existingproject.iam_workload_identity_provider_id
-}
-output "anyscale_iam_workload_identity_provider_name" {
-  description = "The Anyscale workload identity provider name."
-  value       = module.google_anyscale_v2_existingproject.iam_workload_identity_provider_name
-}
-
 output "registration_command" {
   description = "The Anyscale registration command."
   value       = <<-EOT
@@ -66,8 +11,8 @@ output "registration_command" {
     --cloud-storage-bucket-name ${module.google_anyscale_v2_existingproject.cloudstorage_bucket_name} \
     --filestore-instance-id ${module.google_anyscale_v2_existingproject.filestore_name} \
     --filestore-location ${module.google_anyscale_v2_existingproject.filestore_location} \
-    --anyscale-service-account-email ${module.google_anyscale_v2_existingproject.iam_anyscale_access_role_email} \
-    --instance-service-account-email ${module.google_anyscale_v2_existingproject.iam_anyscale_cluster_node_role_email} \
+    --anyscale-service-account-email ${module.google_anyscale_v2_existingproject.iam_anyscale_access_service_acct_email} \
+    --instance-service-account-email ${module.google_anyscale_v2_existingproject.iam_anyscale_cluster_node_service_acct_email} \
     --provider-name ${module.google_anyscale_v2_existingproject.iam_workload_identity_provider_name}
   EOT
 }

examples/anyscale-v2-kitchensink/main.tf

@@ -90,14 +90,14 @@ module "google_anyscale_v2_kitchensink" {
 
   # IAM Related
   enable_anyscale_iam                          = true
-  anyscale_iam_access_role_name_prefix         = "anyscale-tf-ks-acct-"
-  anyscale_iam_access_role_description         = "Anyscale Terraform KitchenSink IAM Access Role"
+  anyscale_iam_access_service_acct_name_prefix = "anyscale-tf-ks-acct-"
+  anyscale_iam_access_service_acct_description = "Anyscale Terraform KitchenSink IAM Access Role"
   anyscale_workload_identity_pool_name         = "anyscale-tf-ks-workload-id-pool"
   anyscale_workload_identity_pool_display_name = "Anyscale TF KS Identity Pool"
   anyscale_workload_identity_pool_description  = "Anyscale Terraform KitchenSink Workload Identity Pool"
 
-  anyscale_cluster_node_role_name        = "anyscale-tf-ks-cluster"
-  anyscale_cluster_node_role_description = "Anyscale Terraform KitchenSink IAM Cluster Node Role"
+  anyscale_cluster_node_service_acct_name        = "anyscale-tf-ks-cluster"
+  anyscale_cluster_node_service_acct_description = "Anyscale Terraform KitchenSink IAM Cluster Node Role"
 
   # Memorystore Related
   enable_anyscale_memorystore       = true

examples/anyscale-v2-kitchensink/outputs.tf

@@ -11,8 +11,8 @@ output "registration_command" {
     --cloud-storage-bucket-name ${module.google_anyscale_v2_kitchensink.cloudstorage_bucket_name} \
     --filestore-instance-id ${module.google_anyscale_v2_kitchensink.filestore_name} \
     --filestore-location ${module.google_anyscale_v2_kitchensink.filestore_location} \
-    --anyscale-service-account-email ${module.google_anyscale_v2_kitchensink.iam_anyscale_access_role_email} \
-    --instance-service-account-email ${module.google_anyscale_v2_kitchensink.iam_anyscale_cluster_node_role_email} \
+    --anyscale-service-account-email ${module.google_anyscale_v2_kitchensink.iam_anyscale_access_service_acct_email} \
+    --instance-service-account-email ${module.google_anyscale_v2_kitchensink.iam_anyscale_cluster_node_service_acct_email} \
     --provider-name ${module.google_anyscale_v2_kitchensink.iam_workload_identity_provider_name} \
     --memorystore-instance-name ${module.google_anyscale_v2_kitchensink.memorystore_id}
   EOT