\
+ --region ${var.aws_region} \
+ --compute-stack k8s \
+ --anyscale-iam-role-id ${module.anyscale_iam_roles.iam_anyscale_access_role_arn} \
+ --s3-bucket-id ${module.anyscale_s3.s3_bucket_id} \
+ --efs-id ${module.anyscale_efs.efs_id} \
+ --kubernetes-namespaces ${module.anyscale_k8s_namespace.anyscale_kubernetes_namespace_name} \
+ --kubernetes-ingress-external-address ${module.anyscale_k8s_helm.nginx_ingress_lb_hostname[0]} \
+ --kubernetes-zones ${local.kubernetes_zones} \
+ --kubernetes-dataplane-identity ${module.anyscale_iam_roles.iam_anyscale_eks_node_role_arn}
+ EOT
+}
diff --git a/examples/aws/eks-public/variables.tf b/examples/aws/eks-public/variables.tf
new file mode 100644
index 0000000..a16a9c0
--- /dev/null
+++ b/examples/aws/eks-public/variables.tf
@@ -0,0 +1,85 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# ENVIRONMENT VARIABLES
+# Define these secrets as environment variables
+# ---------------------------------------------------------------------------------------------------------------------
+
+# AWS_ACCESS_KEY_ID
+# AWS_SECRET_ACCESS_KEY
+
+# ---------------------------------------------------------------------------------------------------------------------
+# REQUIRED VARIABLES
+# These variables must be set when using this module.
+# ---------------------------------------------------------------------------------------------------------------------
+
+variable "aws_region" {
+ description = "The AWS region in which all resources will be created."
+ type = string
+ default = "us-east-2"
+}
+
+variable "anyscale_cloud_id" {
+ description = "(Optional) Anyscale Cloud ID. Default is `null`."
+ type = string
+ default = null
+ validation {
+ condition = (
+ var.anyscale_cloud_id == null ? true : (
+ length(var.anyscale_cloud_id) > 4 &&
+ substr(var.anyscale_cloud_id, 0, 4) == "cld_"
+ )
+ )
+ error_message = "The anyscale_cloud_id value must start with \"cld_\"."
+ }
+}
+
+# ------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These variables have defaults, but may be overridden.
+# ------------------------------------------------------------------------------
+variable "anyscale_deploy_env" {
+ description = "(Optional) Anyscale deploy environment. Used in resource names and tags."
+ type = string
+ default = "production"
+ validation {
+ condition = (
+ var.anyscale_deploy_env == "production" || var.anyscale_deploy_env == "development" || var.anyscale_deploy_env == "test"
+ )
+ error_message = "The anyscale_deploy_env only allows `production`, `test`, or `development`"
+ }
+}
+
+variable "tags" {
+ description = "(Optional) A map of tags to all resources that accept tags."
+ type = map(string)
+ default = {
+ "test" : true,
+ "environment" : "example"
+ "repo" : "terraform-kubernetes-anyscale-foundation-modules",
+ "example" : "aws/eks-public"
+ }
+}
+
+variable "anyscale_trusted_role_arns" {
+ description = <<-EOT
+ (Optional) A list of ARNs of IAM roles that are trusted by the Anyscale IAM role.
+
+ Including here to override for Anyscale Staging.
+ EOT
+ type = list(string)
+ default = []
+}
+
+variable "anyscale_s3_cors_rule" {
+ description = <<-EOT
+ (Optional) A map of CORS rules for the S3 bucket.
+
+ Including here to override for Anyscale Staging.
+ EOT
+ type = map(any)
+ default = {
+ allowed_headers = ["*"]
+ allowed_methods = ["GET", "POST", "PUT", "HEAD", "DELETE"]
+ allowed_origins = ["https://*.anyscale.com"]
+ expose_headers = []
+ }
+}
diff --git a/examples/aws/eks-public/versions.tf b/examples/aws/eks-public/versions.tf
new file mode 100644
index 0000000..e937434
--- /dev/null
+++ b/examples/aws/eks-public/versions.tf
@@ -0,0 +1,49 @@
+terraform {
+ required_version = ">= 1.0"
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "~> 5.0"
+ }
+
+ helm = {
+ source = "hashicorp/helm"
+ version = "~> 2.0"
+ }
+
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = "~> 2.0"
+ }
+ }
+}
+
+
+provider "helm" {
+ kubernetes {
+ host = module.anyscale_eks_cluster.eks_kubeconfig.endpoint
+ cluster_ca_certificate = base64decode(module.anyscale_eks_cluster.eks_kubeconfig.cluster_ca_certificate)
+
+ # https://registry.terraform.io/providers/hashicorp/helm/latest/docs#exec-plugins
+ exec {
+ api_version = "client.authentication.k8s.io/v1beta1"
+ args = ["eks", "get-token", "--cluster-name", module.anyscale_eks_cluster.eks_cluster_name]
+ command = "aws"
+ }
+ }
+}
+
+provider "kubernetes" {
+ host = module.anyscale_eks_cluster.eks_kubeconfig.endpoint
+ cluster_ca_certificate = base64decode(module.anyscale_eks_cluster.eks_kubeconfig.cluster_ca_certificate)
+
+ exec {
+ api_version = "client.authentication.k8s.io/v1beta1"
+ args = ["eks", "get-token", "--cluster-name", module.anyscale_eks_cluster.eks_cluster_name]
+ command = "aws"
+ }
+}
+
+provider "aws" {
+ region = var.aws_region
+}
diff --git a/main.tf b/main.tf
new file mode 100644
index 0000000..e69de29
diff --git a/modules/anyscale-k8s-configmap/README.md b/modules/anyscale-k8s-configmap/README.md
new file mode 100644
index 0000000..6fab950
--- /dev/null
+++ b/modules/anyscale-k8s-configmap/README.md
@@ -0,0 +1,63 @@
+[![Build Status][badge-build]][build-status]
+[![Terraform Version][badge-terraform]](https://github.com/hashicorp/terraform/releases)
+[![OpenTofu Version][badge-opentofu]](https://github.com/opentofu/opentofu/releases)
+[![Kubernetes Provider Version][badge-tf-kubernetes]](https://github.com/terraform-providers/terraform-provider-kubernetes/releases)
+[![AWS Provider Version][badge-tf-aws]](https://github.com/terraform-providers/terraform-provider-aws/releases)
+[![Google Provider Version][badge-tf-google]](https://github.com/terraform-providers/terraform-provider-google/releases)
+
+# anyscale-k8s-configmap
+This module creates Kubernetes Configmaps for Anyscale applications and workloads.
+
+The `instance-types` ConfigMap defines the instance types that you wish to run on Anyscale. This ConfigMap can also be created
+via the Anyscale Helm Chart.
+
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| [terraform](#requirement\_terraform) | >= 1.0 |
+| [kubernetes](#requirement\_kubernetes) | ~> 2.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| [kubernetes](#provider\_kubernetes) | 2.32.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [kubernetes_config_map.instance_type](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| [anyscale\_kubernetes\_namespace](#input\_anyscale\_kubernetes\_namespace) | (Optional) The namespace to install the Anyscale resources.
ex:anyscale_kubernetes_namespace = "anyscale-k8s"
| `string` | n/a | yes |
+| [cloud\_provider](#input\_cloud\_provider) | (Required) The cloud provider (aws or gcp)
ex:cloud_provider = "aws"
| `string` | n/a | yes |
+| [anyscale\_instance\_types](#input\_anyscale\_instance\_types) | (Optional) A list of instance types to create in the instance-types configmap.
ex:anyscale_instance_types = [
{
instanceType = "8CPU-32GB"
CPU = 8
memory = 32Gi # 32gb
},
{
instanceType = "4CPU-16GB-1xA10"
CPU = 4
GPU = 1
memory = 17179869184 # 16gb converted to bytes
accelerator_type = {"A10G" = 1}
},
{
instanceType = "8CPU-32GB-1xA10"
CPU = 8
GPU = 1
memory = 32Gi # 32gb
accelerator_type = {"A10G" = 1}
},
{
instanceType = "8CPU-32GB-1xT4"
CPU = 8
GPU = 1
memory = 32Gi # 32gb
accelerator_type = {"T4" = 1}
}
]
| list(object({
instanceType = string
CPU = number
GPU = optional(number)
memory = string
accelerator_type = optional(map(number)) # accelerator_type should be a map of key-value pairs
})) | [
{
"CPU": 8,
"instanceType": "8CPU-32GB",
"memory": "32Gi"
}
]
| no |
+| [anyscale\_instance\_types\_version](#input\_anyscale\_instance\_types\_version) | (Optional) The version of the instance-types configmap.
ex:anyscale_instance_types_version = "v1"
| `string` | `"v1"` | no |
+| [create\_anyscale\_instance\_types\_map](#input\_create\_anyscale\_instance\_types\_map) | (Optional) Determines if the instance-types configmap should be created.
ex:create_anyscale_instance_types_map = true
| `bool` | `true` | no |
+| [module\_enabled](#input\_module\_enabled) | (Optional) Determines if this module should create resources.
If set to true, `eks_role_arn`, `anyscale_subnet_ids`, and `anyscale_security_group_id` must be provided.
ex:module_enabled = true
| `bool` | `false` | no |
+
+## Outputs
+
+No outputs.
+
+
+
+[Terraform]: https://www.terraform.io
+[Issues]: https://github.com/anyscale/sa-sandbox-terraform/issues
+[badge-build]: https://github.com/anyscale/sa-sandbox-terraform/workflows/CI/CD%20Pipeline/badge.svg
+[badge-terraform]: https://img.shields.io/badge/terraform-1.x%20-623CE4.svg?logo=terraform
+[badge-tf-aws]: https://img.shields.io/badge/AWS-5.+-F8991D.svg?logo=terraform
+[build-status]: https://github.com/anyscale/sa-sandbox-terraform/actions
+[badge-opentofu]: https://img.shields.io/badge/opentofu-1.x%20-623CE4.svg?logo=terraform
+[badge-tf-google]: https://img.shields.io/badge/Google-5.+-F8991D.svg?logo=terraform
+[badge-tf-kubernetes]: https://img.shields.io/badge/KUBERNETES-2.+-F8991D.svg?logo=terraform
diff --git a/modules/anyscale-k8s-configmap/main.tf b/modules/anyscale-k8s-configmap/main.tf
new file mode 100644
index 0000000..69ffb75
--- /dev/null
+++ b/modules/anyscale-k8s-configmap/main.tf
@@ -0,0 +1,33 @@
+locals {
+ module_enabled = var.module_enabled
+
+ aws_enabled = local.module_enabled && var.cloud_provider == "aws"
+ gcp_enabled = local.module_enabled && var.cloud_provider == "gcp"
+
+ create_anyscale_instance_types = local.module_enabled && var.create_anyscale_instance_types_map
+
+}
+
+resource "kubernetes_config_map" "instance_type" {
+ count = local.module_enabled && var.create_anyscale_instance_types_map ? 1 : 0
+ metadata {
+ name = "instance-types"
+ namespace = var.anyscale_kubernetes_namespace
+ }
+
+ data = {
+ version = var.anyscale_instance_types_version
+ "instance_types.yaml" = yamlencode({
+ for instance in var.anyscale_instance_types : instance.instanceType => {
+ resources = merge(
+ {
+ CPU = instance.CPU
+ memory = instance.memory
+ },
+ instance.GPU != null ? { GPU = instance.GPU } : {},
+ instance.accelerator_type != null ? { for key, value in instance.accelerator_type : "accelerator_type:${key}" => value } : {}
+ )
+ }
+ })
+ }
+}
diff --git a/modules/anyscale-k8s-configmap/outputs.tf b/modules/anyscale-k8s-configmap/outputs.tf
new file mode 100644
index 0000000..e69de29
diff --git a/modules/anyscale-k8s-configmap/test/anyscale-aws-test/main.tf b/modules/anyscale-k8s-configmap/test/anyscale-aws-test/main.tf
new file mode 100644
index 0000000..ad4a79a
--- /dev/null
+++ b/modules/anyscale-k8s-configmap/test/anyscale-aws-test/main.tf
@@ -0,0 +1,197 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# CREATE Anyscale K8s ConfigMap Resources
+# This template creates EKS resources for Anyscale
+# Requires:
+# - VPC
+# - Security Group
+# - IAM Roles
+# - EKS Cluster
+# ---------------------------------------------------------------------------------------------------------------------
+locals {
+ # azs = slice(data.aws_availability_zones.available.names, 0, 3)
+
+ full_tags = merge(tomap({
+ anyscale-cloud-id = var.anyscale_cloud_id,
+ anyscale-deploy-environment = var.anyscale_deploy_env
+ }),
+ var.tags
+ )
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Create resources for EKS TF Module
+# Creates a VPC
+# Creates a Security Group
+# Creates IAM Roles
+# ---------------------------------------------------------------------------------------------------------------------
+locals {
+ public_subnets = ["172.24.101.0/24", "172.24.102.0/24", "172.24.103.0/24"]
+ private_subnets = ["172.24.20.0/24", "172.24.21.0/24", "172.24.22.0/24"]
+}
+module "eks_vpc" {
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-vpc"
+
+ anyscale_vpc_name = "tftest-k8s-configmap"
+ cidr_block = "172.24.0.0/16"
+
+ public_subnets = local.public_subnets
+ private_subnets = local.private_subnets
+}
+locals {
+ # Because subnet ID may not be known at plan time, we cannot use it as a key
+ anyscale_subnet_count = length(local.private_subnets)
+}
+
+module "eks_securitygroup" {
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-securitygroups"
+
+ vpc_id = module.eks_vpc.vpc_id
+
+ security_group_name_prefix = "tftest-k8s-configmap-"
+
+ ingress_with_self = [
+ { rule = "all-all" }
+ ]
+}
+
+module "eks_iam_roles" {
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-iam"
+
+ module_enabled = true
+ create_anyscale_access_role = true
+ anyscale_access_role_name = "tftest-k8s-configmap-controlplane-role"
+ create_cluster_node_instance_profile = false
+ create_iam_s3_policy = false
+
+ create_anyscale_eks_cluster_role = true
+ anyscale_eks_cluster_role_name = "tftest-k8s-configmap-cluster-role"
+ create_anyscale_eks_node_role = true
+ anyscale_eks_node_role_name = "tftest-k8s-configmap-node-role"
+
+ tags = local.full_tags
+}
+
+locals {
+ coredns_config = jsonencode({
+ affinity = {
+ nodeAffinity = {
+ requiredDuringSchedulingIgnoredDuringExecution = {
+ nodeSelectorTerms = [
+ {
+ matchExpressions = [
+ {
+ key = "node-type"
+ operator = "In"
+ values = ["management"]
+ }
+ ]
+ }
+ ]
+ }
+ }
+ },
+ nodeSelector = {
+ "node-type" = "management"
+ },
+ tolerations = [
+ {
+ key = "CriticalAddonsOnly"
+ operator = "Exists"
+ },
+ {
+ effect = "NoSchedule"
+ key = "node-role.kubernetes.io/control-plane"
+ }
+ ],
+ replicaCount = 2
+ })
+}
+
+module "eks_cluster" {
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-eks-cluster"
+
+ module_enabled = true
+
+ anyscale_subnet_ids = module.eks_vpc.public_subnet_ids
+ anyscale_subnet_count = local.anyscale_subnet_count
+ anyscale_security_group_id = module.eks_securitygroup.security_group_id
+ eks_role_arn = module.eks_iam_roles.iam_anyscale_eks_cluster_role_arn
+ anyscale_eks_name = "tftest-k8s-configmap"
+
+ tags = local.full_tags
+
+ depends_on = [module.eks_iam_roles, module.eks_vpc, module.eks_securitygroup]
+}
+
+module "k8s_default_namespace" {
+ source = "../../../anyscale-k8s-namespace"
+
+ module_enabled = true
+ cloud_provider = "aws"
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Create Resources with no optional parameters
+# ---------------------------------------------------------------------------------------------------------------------
+module "all_defaults" {
+ source = "../../"
+
+ module_enabled = true
+ cloud_provider = "aws"
+
+ anyscale_kubernetes_namespace = module.k8s_default_namespace.anyscale_kubernetes_namespace_name
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Create Resources with as many optional parameters as possible
+# ---------------------------------------------------------------------------------------------------------------------
+module "k8s_kitchensink_namespace" {
+ source = "../../../anyscale-k8s-namespace"
+
+ module_enabled = true
+ cloud_provider = "aws"
+
+ anyscale_kubernetes_namespace = "kitchensink"
+}
+
+module "kitchen_sink" {
+ source = "../../"
+
+ module_enabled = true
+ cloud_provider = "aws"
+
+ anyscale_kubernetes_namespace = module.k8s_kitchensink_namespace.anyscale_kubernetes_namespace_name
+
+ create_anyscale_instance_types_map = true
+ anyscale_instance_types_version = "v1"
+ anyscale_instance_types = [
+ {
+ instanceType = "t3.small"
+ CPU = 2
+ memory = "4Gi"
+ },
+ {
+ instanceType = "4CPU-16GB-1xA10"
+ CPU = 4
+ GPU = 1
+ memory = "16Gi" # 16gb converted to bytes
+ accelerator_type = { "A10G" = 1 }
+ },
+ ]
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Do not create any resources
+# ---------------------------------------------------------------------------------------------------------------------
+module "test_no_resources" {
+ source = "../.."
+
+ module_enabled = false
+
+ #Required variables
+ cloud_provider = "aws"
+ anyscale_kubernetes_namespace = module.k8s_default_namespace.anyscale_kubernetes_namespace_name
+}
diff --git a/modules/anyscale-k8s-configmap/test/anyscale-aws-test/outputs.tf b/modules/anyscale-k8s-configmap/test/anyscale-aws-test/outputs.tf
new file mode 100644
index 0000000..e69de29
diff --git a/modules/anyscale-k8s-configmap/test/anyscale-aws-test/variables.tf b/modules/anyscale-k8s-configmap/test/anyscale-aws-test/variables.tf
new file mode 100644
index 0000000..9393991
--- /dev/null
+++ b/modules/anyscale-k8s-configmap/test/anyscale-aws-test/variables.tf
@@ -0,0 +1,58 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# ENVIRONMENT VARIABLES
+# Define these secrets as environment variables
+# ---------------------------------------------------------------------------------------------------------------------
+
+# AWS_ACCESS_KEY_ID
+# AWS_SECRET_ACCESS_KEY
+
+# ---------------------------------------------------------------------------------------------------------------------
+# REQUIRED VARIABLES
+# These variables must be set when using this module.
+# ---------------------------------------------------------------------------------------------------------------------
+
+variable "aws_region" {
+ description = "The AWS region in which all resources will be created."
+ type = string
+ default = "us-east-2"
+}
+
+variable "anyscale_cloud_id" {
+ description = "(Optional) Anyscale Cloud ID. Default is `null`."
+ type = string
+ default = null
+ validation {
+ condition = (
+ var.anyscale_cloud_id == null ? true : (
+ length(var.anyscale_cloud_id) > 4 &&
+ substr(var.anyscale_cloud_id, 0, 4) == "cld_"
+ )
+ )
+ error_message = "The anyscale_cloud_id value must start with \"cld_\"."
+ }
+}
+
+# ------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These variables have defaults, but may be overridden.
+# ------------------------------------------------------------------------------
+variable "anyscale_deploy_env" {
+ description = "(Optional) Anyscale deploy environment. Used in resource names and tags."
+ type = string
+ default = "production"
+ validation {
+ condition = (
+ var.anyscale_deploy_env == "production" || var.anyscale_deploy_env == "development" || var.anyscale_deploy_env == "test"
+ )
+ error_message = "The anyscale_deploy_env only allows `production`, `test`, or `development`"
+ }
+}
+
+variable "tags" {
+ description = "(Optional) A map of tags to all resources that accept tags."
+ type = map(string)
+ default = {
+ "test" : true,
+ "environment" : "test"
+ }
+}
diff --git a/modules/anyscale-k8s-configmap/test/anyscale-aws-test/versions.tf b/modules/anyscale-k8s-configmap/test/anyscale-aws-test/versions.tf
new file mode 100644
index 0000000..c46128a
--- /dev/null
+++ b/modules/anyscale-k8s-configmap/test/anyscale-aws-test/versions.tf
@@ -0,0 +1,31 @@
+terraform {
+ required_version = ">= 1.0"
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "~> 5.0"
+ }
+
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = "~> 2.0"
+ }
+ }
+}
+
+
+
+provider "kubernetes" {
+ host = module.eks_cluster.eks_kubeconfig.endpoint
+ cluster_ca_certificate = base64decode(module.eks_cluster.eks_kubeconfig.cluster_ca_certificate)
+
+ exec {
+ api_version = "client.authentication.k8s.io/v1beta1"
+ args = ["eks", "get-token", "--cluster-name", module.eks_cluster.eks_cluster_name]
+ command = "aws"
+ }
+}
+
+provider "aws" {
+ region = var.aws_region
+}
diff --git a/modules/anyscale-k8s-configmap/test/anyscale-gcp-test/main.tf b/modules/anyscale-k8s-configmap/test/anyscale-gcp-test/main.tf
new file mode 100644
index 0000000..e69de29
diff --git a/modules/anyscale-k8s-configmap/test/anyscale-gcp-test/outputs.tf b/modules/anyscale-k8s-configmap/test/anyscale-gcp-test/outputs.tf
new file mode 100644
index 0000000..e69de29
diff --git a/modules/anyscale-k8s-configmap/test/anyscale-gcp-test/variables.tf b/modules/anyscale-k8s-configmap/test/anyscale-gcp-test/variables.tf
new file mode 100644
index 0000000..e69de29
diff --git a/modules/anyscale-k8s-configmap/test/anyscale-gcp-test/versions.tf b/modules/anyscale-k8s-configmap/test/anyscale-gcp-test/versions.tf
new file mode 100644
index 0000000..fb398ef
--- /dev/null
+++ b/modules/anyscale-k8s-configmap/test/anyscale-gcp-test/versions.tf
@@ -0,0 +1,23 @@
+terraform {
+ required_version = ">= 1.0"
+ required_providers {
+
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = "~> 2.0"
+ }
+ }
+}
+
+
+
+provider "kubernetes" {
+ host = module.eks_cluster.eks_kubeconfig.endpoint
+ cluster_ca_certificate = base64decode(module.eks_cluster.eks_kubeconfig.cluster_ca_certificate)
+
+ exec {
+ api_version = "client.authentication.k8s.io/v1beta1"
+ args = ["eks", "get-token", "--cluster-name", module.eks_cluster.eks_cluster_name]
+ command = "aws"
+ }
+}
diff --git a/modules/anyscale-k8s-configmap/variables.tf b/modules/anyscale-k8s-configmap/variables.tf
new file mode 100644
index 0000000..2ed565b
--- /dev/null
+++ b/modules/anyscale-k8s-configmap/variables.tf
@@ -0,0 +1,133 @@
+# ------------------------------------------------------------------------------
+# REQUIRED PARAMETERS
+# These variables must be set when using this module.
+# ------------------------------------------------------------------------------
+variable "cloud_provider" {
+ description = <<-EOT
+ (Required) The cloud provider (aws or gcp)
+
+ ex:
+ ```
+ cloud_provider = "aws"
+ ```
+ EOT
+ type = string
+ validation {
+ condition = (
+ var.cloud_provider == "aws" || var.cloud_provider == "gcp"
+ )
+ error_message = "The cloud_provider only allows `aws` or `gcp`"
+ }
+}
+
+variable "anyscale_kubernetes_namespace" {
+ description = <<-EOT
+ (Optional) The namespace to install the Anyscale resources.
+
+ ex:
+ ```
+ anyscale_kubernetes_namespace = "anyscale-k8s"
+ ```
+ EOT
+ type = string
+}
+
+
+# ------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These variables have defaults, but may be overridden.
+# ------------------------------------------------------------------------------
+variable "module_enabled" {
+ description = <<-EOT
+ (Optional) Determines if this module should create resources.
+
+ If set to true, `eks_role_arn`, `anyscale_subnet_ids`, and `anyscale_security_group_id` must be provided.
+ ex:
+ ```
+ module_enabled = true
+ ```
+ EOT
+ type = bool
+ default = false
+}
+
+# ------------------
+# Instance Types
+# ------------------
+variable "create_anyscale_instance_types_map" {
+ description = <<-EOT
+ (Optional) Determines if the instance-types configmap should be created.
+
+ ex:
+ ```
+ create_anyscale_instance_types_map = true
+ ```
+ EOT
+ type = bool
+ default = true
+}
+
+variable "anyscale_instance_types_version" {
+ description = <<-EOT
+ (Optional) The version of the instance-types configmap.
+
+ ex:
+ ```
+ anyscale_instance_types_version = "v1"
+ ```
+ EOT
+ type = string
+ default = "v1"
+}
+
+variable "anyscale_instance_types" {
+ description = <<-EOT
+ (Optional) A list of instance types to create in the instance-types configmap.
+
+ ex:
+ ```
+ anyscale_instance_types = [
+ {
+ instanceType = "8CPU-32GB"
+ CPU = 8
+ memory = 32Gi # 32gb
+ },
+ {
+ instanceType = "4CPU-16GB-1xA10"
+ CPU = 4
+ GPU = 1
+ memory = 17179869184 # 16gb converted to bytes
+ accelerator_type = {"A10G" = 1}
+ },
+ {
+ instanceType = "8CPU-32GB-1xA10"
+ CPU = 8
+ GPU = 1
+ memory = 32Gi # 32gb
+ accelerator_type = {"A10G" = 1}
+ },
+ {
+ instanceType = "8CPU-32GB-1xT4"
+ CPU = 8
+ GPU = 1
+ memory = 32Gi # 32gb
+ accelerator_type = {"T4" = 1}
+ }
+ ]
+ ```
+ EOT
+ type = list(object({
+ instanceType = string
+ CPU = number
+ GPU = optional(number)
+ memory = string
+ accelerator_type = optional(map(number)) # accelerator_type should be a map of key-value pairs
+ }))
+ default = [
+ {
+ instanceType = "8CPU-32GB"
+ CPU = 8
+ memory = "32Gi"
+ }
+ ]
+}
diff --git a/modules/anyscale-k8s-configmap/versions.tf b/modules/anyscale-k8s-configmap/versions.tf
new file mode 100644
index 0000000..94019cd
--- /dev/null
+++ b/modules/anyscale-k8s-configmap/versions.tf
@@ -0,0 +1,10 @@
+terraform {
+ required_version = ">= 1.0"
+
+ required_providers {
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = "~> 2.0"
+ }
+ }
+}
diff --git a/modules/anyscale-k8s-helm/README.md b/modules/anyscale-k8s-helm/README.md
new file mode 100644
index 0000000..38f33a3
--- /dev/null
+++ b/modules/anyscale-k8s-helm/README.md
@@ -0,0 +1,83 @@
+[![Build Status][badge-build]][build-status]
+[![Terraform Version][badge-terraform]](https://github.com/hashicorp/terraform/releases)
+[![OpenTofu Version][badge-opentofu]](https://github.com/opentofu/opentofu/releases)
+[![Kubernetes Provider Version][badge-tf-kubernetes]](https://github.com/terraform-providers/terraform-provider-kubernetes/releases)
+[![AWS Provider Version][badge-tf-aws]](https://github.com/terraform-providers/terraform-provider-aws/releases)
+[![Google Provider Version][badge-tf-google]](https://github.com/terraform-providers/terraform-provider-google/releases)
+
+# anyscale-k8s-helm
+This module creates Kubernetes helm charts for Anyscale applications and workloads.
+
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| [terraform](#requirement\_terraform) | >= 1.0 |
+| [helm](#requirement\_helm) | ~> 2.0 |
+| [kubernetes](#requirement\_kubernetes) | ~> 2.0 |
+| [time](#requirement\_time) | >= 0.12 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| [aws](#provider\_aws) | 5.63.0 |
+| [helm](#provider\_helm) | 2.15.0 |
+| [kubernetes](#provider\_kubernetes) | 2.32.0 |
+| [time](#provider\_time) | 0.12.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [helm_release.anyscale_cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.feature_metrics_server](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.nginx_ingress](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.nvidia](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.prometheus](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [kubernetes_namespace.ingress_nginx](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource |
+| [time_sleep.wait_helm_termination](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
+| [kubernetes_service.nginx_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/data-sources/service) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| [cloud\_provider](#input\_cloud\_provider) | (Required) The cloud provider (aws or gcp)
ex:cloud_provider = "aws"
| `string` | n/a | yes |
+| [anyscale\_cluster\_autoscaler\_chart](#input\_anyscale\_cluster\_autoscaler\_chart) | (Optional) The Helm chart to install the Cluster Autoscaler.
ex:anyscale_cluster_autoscaler_chart = {
enabled = true
name = "cluster-autoscaler"
respository = "https://kubernetes.github.io/autoscaler"
chart = "cluster-autoscaler"
chart_version = "9.37.0"
namespace = "kube-system"
values = {
"some.other.config" = "value"
}
} | object({
enabled = bool
name = optional(string)
repository = optional(string)
chart = optional(string)
chart_version = optional(string)
namespace = optional(string)
values = optional(map(string))
}) | {
"chart": "cluster-autoscaler",
"chart_version": "9.37.0",
"enabled": true,
"name": "cluster-autoscaler",
"namespace": "kube-system",
"repository": "https://kubernetes.github.io/autoscaler",
"values": {}
} | no |
+| [anyscale\_ingress\_aws\_nlb\_internal](#input\_anyscale\_ingress\_aws\_nlb\_internal) | (Optioanl) Determines if the AWS NLB should be internal.
Requires `cloud_provider` to be set to `aws`.
Requires `anyscale_ingress_chart` to be enabled.
ex:anyscale_ingress_aws_nlb_internal = true
| `bool` | `false` | no |
+| [anyscale\_ingress\_chart](#input\_anyscale\_ingress\_chart) | (Optional) The Helm chart to install the Cluster Ingress.
ex:anyscale_ingress_chart = {
enabled = true
name = "anyscale-ingress"
respository = "https://kubernetes.github.io/ingress-nginx"
chart = "ingress-nginx"
chart_version = "4.11.1"
namespace = "ingress-nginx"
values = {
"some.other.config" = "value"
}
} | object({
enabled = bool
name = optional(string)
repository = optional(string)
chart = optional(string)
chart_version = optional(string)
namespace = optional(string)
values = optional(map(string))
}) | {
"chart": "ingress-nginx",
"chart_version": "4.11.1",
"enabled": true,
"name": "anyscale-ingress",
"namespace": "ingress-nginx",
"repository": "https://kubernetes.github.io/ingress-nginx",
"values": {
"controller.allowSnippetAnnotations": "true",
"controller.autoscaling.enabled": "true",
"controller.service.type": "LoadBalancer"
}
} | no |
+| [anyscale\_metrics\_server\_chart](#input\_anyscale\_metrics\_server\_chart) | (Optional) The Helm chart to install the Metrics Server.
Required for the Anyscale Autoscaler to function.
ex:anyscale_metrics_server_chart = {
enabled = true
name = "metrics-server"
respository = "https://kubernetes-sigs.github.io/metrics-server/"
chart = "metrics-server"
chart_version = "3.12.1"
namespace = "metrics-server"
values = {
"some.other.config" = "value"
}
} | object({
enabled = bool
name = optional(string)
repository = optional(string)
chart = optional(string)
chart_version = optional(string)
namespace = optional(string)
values = optional(map(string))
}) | {
"chart": "metrics-server",
"chart_version": "3.12.1",
"enabled": true,
"name": "metrics-server",
"namespace": "metrics-server",
"repository": "https://kubernetes-sigs.github.io/metrics-server/",
"values": {}
} | no |
+| [anyscale\_nvidia\_device\_plugin\_chart](#input\_anyscale\_nvidia\_device\_plugin\_chart) | (Optional) The Helm chart to install the NVIDIA Device Plugin.
Valid settings can be found in the [nvidia documentation](https://github.com/NVIDIA/k8s-device-plugin?tab=readme-ov-file#deploying-with-gpu-feature-discovery-for-automatic-node-labels)
ex:anyscale_nvidia_device_plugin_chart = {
enabled = true
name = "nvidia-device-plugin"
respository = "https://nvidia.github.io/k8s-device-plugin"
chart = "nvidia-device-plugin"
chart_version = "0.16.2"
namespace = "nvidia-device-plugin"
values = {
"some.other.config" = "value"
}
} | object({
enabled = bool
name = optional(string)
repository = optional(string)
chart = optional(string)
chart_version = optional(string)
namespace = optional(string)
values = optional(map(string))
}) | {
"chart": "nvidia-device-plugin",
"chart_version": "0.16.2",
"enabled": true,
"name": "anyscale-nvidia-device-plugin",
"namespace": "nvidia-device-plugin",
"repository": "https://nvidia.github.io/k8s-device-plugin",
"values": {
"gfd.enabled": "true",
"nfd.worker.tolerations[0].effect": "NoSchedule",
"nfd.worker.tolerations[0].key": "node-role.kubernetes.io/master",
"nfd.worker.tolerations[0].operator": "Equal",
"nfd.worker.tolerations[0].value": "",
"nfd.worker.tolerations[1].effect": "NoSchedule",
"nfd.worker.tolerations[1].key": "nvidia.com/gpu",
"nfd.worker.tolerations[1].operator": "Equal",
"nfd.worker.tolerations[1].value": "present",
"nfd.worker.tolerations[2].effect": "NoSchedule",
"nfd.worker.tolerations[2].key": "node.anyscale.com/accelerator-type",
"nfd.worker.tolerations[2].operator": "Equal",
"nfd.worker.tolerations[2].value": "GPU",
"nfd.worker.tolerations[3].effect": "NoSchedule",
"nfd.worker.tolerations[3].key": "node.anyscale.com/capacity-type",
"nfd.worker.tolerations[3].operator": "Equal",
"nfd.worker.tolerations[3].value": "ANY",
"priorityClassName": "system-node-critical",
"tolerations[0].effect": "NoSchedule",
"tolerations[0].key": "nvidia.com/gpu",
"tolerations[0].operator": "Equal",
"tolerations[0].value": "present",
"tolerations[1].effect": "NoSchedule",
"tolerations[1].key": "node.anyscale.com/accelerator-type",
"tolerations[1].operator": "Equal",
"tolerations[1].value": "GPU",
"tolerations[2].effect": "NoSchedule",
"tolerations[2].key": "node.anyscale.com/capacity-type",
"tolerations[2].operator": "Equal",
"tolerations[2].value": "ANY"
}
} | no |
+| [anyscale\_prometheus\_chart](#input\_anyscale\_prometheus\_chart) | (Optional) The Helm chart to install Prometheus.
ex:anyscale_prometheus_chart = {
enabled = true
name = "prometheus"
respository = "https://prometheus-community.github.io/helm-charts"
chart = "prometheus"
chart_version = "16.0.0"
namespace = "prometheus"
values = {
"some.other.config" = "value"
}
} | object({
enabled = bool
name = optional(string)
repository = optional(string)
chart = optional(string)
chart_version = optional(string)
namespace = optional(string)
values = optional(map(string))
}) | {
"chart": "prometheus",
"chart_version": "25.26.0",
"enabled": false,
"name": "prometheus",
"namespace": "prometheus",
"repository": "https://prometheus-community.github.io/helm-charts",
"values": {}
} | no |
+| [kubernetes\_cluster\_name](#input\_kubernetes\_cluster\_name) | (Optional) The name of the Kubernetes cluster.
ex:kubernetes_cluster_name = "my-cluster"
| `string` | `null` | no |
+| [module\_enabled](#input\_module\_enabled) | (Optional) Determines if this module should create resources.
If set to true, `eks_role_arn`, `anyscale_subnet_ids`, and `anyscale_security_group_id` must be provided.
ex:module_enabled = true
| `bool` | `false` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| [helm\_autoscaler\_status](#output\_helm\_autoscaler\_status) | Status of the Cluster Autoscaler Helm release |
+| [helm\_nginx\_ingress\_status](#output\_helm\_nginx\_ingress\_status) | Status of the Ingress Helm release |
+| [helm\_nvidia\_status](#output\_helm\_nvidia\_status) | Status of the Nvidia Helm release |
+| [nginx\_ingress\_lb\_hostname](#output\_nginx\_ingress\_lb\_hostname) | Hostname of the nginx load balancer |
+| [nginx\_ingress\_lb\_ips](#output\_nginx\_ingress\_lb\_ips) | IPs of the nginx load balancer |
+
+
+
+[Terraform]: https://www.terraform.io
+[Issues]: https://github.com/anyscale/sa-sandbox-terraform/issues
+[badge-build]: https://github.com/anyscale/sa-sandbox-terraform/workflows/CI/CD%20Pipeline/badge.svg
+[badge-terraform]: https://img.shields.io/badge/terraform-1.x%20-623CE4.svg?logo=terraform
+[badge-tf-aws]: https://img.shields.io/badge/AWS-5.+-F8991D.svg?logo=terraform
+[build-status]: https://github.com/anyscale/sa-sandbox-terraform/actions
+[badge-opentofu]: https://img.shields.io/badge/opentofu-1.x%20-623CE4.svg?logo=terraform
+[badge-tf-google]: https://img.shields.io/badge/Google-5.+-F8991D.svg?logo=terraform
+[badge-tf-kubernetes]: https://img.shields.io/badge/KUBERNETES-2.+-F8991D.svg?logo=terraform
diff --git a/modules/anyscale-k8s-helm/data.tf b/modules/anyscale-k8s-helm/data.tf
new file mode 100644
index 0000000..fcd549b
--- /dev/null
+++ b/modules/anyscale-k8s-helm/data.tf
@@ -0,0 +1,12 @@
+# AWS Data Sources
+data "aws_caller_identity" "current" {
+ count = var.cloud_provider == "aws" ? 1 : 0
+}
+data "aws_region" "current" {
+ count = var.cloud_provider == "aws" ? 1 : 0
+}
+
+# GCP Data Sources
+# data "google_client_config" "current" {
+# count = var.cloud_provider == "gcp" ? 1 : 0
+# }
diff --git a/anyscale-k8s-helm/helm-autoscaler.tf b/modules/anyscale-k8s-helm/helm-autoscaler.tf
similarity index 69%
rename from anyscale-k8s-helm/helm-autoscaler.tf
rename to modules/anyscale-k8s-helm/helm-autoscaler.tf
index 5dcf4d5..e845577 100644
--- a/anyscale-k8s-helm/helm-autoscaler.tf
+++ b/modules/anyscale-k8s-helm/helm-autoscaler.tf
@@ -2,7 +2,7 @@
# https://github.com/kubernetes/autoscaler
resource "helm_release" "anyscale_cluster_autoscaler" {
- count = local.module_enabled && var.cloud_provider == "aws" ? 1 : 0
+ count = local.module_enabled && var.cloud_provider == "aws" && var.anyscale_cluster_autoscaler_chart.enabled ? 1 : 0
name = var.anyscale_cluster_autoscaler_chart.name
repository = var.anyscale_cluster_autoscaler_chart.repository
@@ -17,9 +17,17 @@ resource "helm_release" "anyscale_cluster_autoscaler" {
value = var.kubernetes_cluster_name
}
- set {
- name = "awsRegion"
- value = data.aws_region.current[0].name
+ dynamic "set" {
+ for_each = var.cloud_provider == "aws" ? [
+ {
+ name = "awsRegion"
+ value = data.aws_region.current[0].name
+ }
+ ] : []
+ content {
+ name = set.value.name
+ value = set.value.value
+ }
}
dynamic "set" {
diff --git a/modules/anyscale-k8s-helm/helm-ingress.tf b/modules/anyscale-k8s-helm/helm-ingress.tf
new file mode 100644
index 0000000..5fe6a8f
--- /dev/null
+++ b/modules/anyscale-k8s-helm/helm-ingress.tf
@@ -0,0 +1,78 @@
+# --------------------------------------------------------------------------------
+# Description: This file contains the terraform configuration to deploy the ingress controller using helm.
+# --------------------------------------------------------------------------------
+
+resource "kubernetes_namespace" "ingress_nginx" {
+ count = local.module_enabled && var.anyscale_ingress_chart.enabled ? 1 : 0
+
+ metadata {
+ name = try(var.anyscale_ingress_chart.namespace, "ingress-nginx")
+ }
+
+}
+
+resource "helm_release" "nginx_ingress" {
+ count = local.module_enabled && var.anyscale_ingress_chart.enabled ? 1 : 0
+
+ name = var.anyscale_ingress_chart.name
+ repository = var.anyscale_ingress_chart.repository
+ chart = var.anyscale_ingress_chart.chart
+ namespace = kubernetes_namespace.ingress_nginx[0].metadata[0].name
+ version = var.anyscale_ingress_chart.chart_version
+ create_namespace = false
+ wait = false
+
+ dynamic "set" {
+ for_each = var.anyscale_ingress_chart.values
+ content {
+ name = set.key
+ value = set.value
+ }
+ }
+
+ dynamic "set" {
+ for_each = var.cloud_provider == "aws" ? [
+ {
+ name = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-type"
+ value = "nlb"
+ },
+ {
+ name = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-cross-zone-load-balancing-enabled"
+ value = "true"
+ }
+ ] : []
+ content {
+ name = set.value["name"]
+ value = set.value["value"]
+ }
+ }
+
+ dynamic "set" {
+ for_each = var.cloud_provider == "aws" && var.anyscale_ingress_aws_nlb_internal ? [
+ {
+ name = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-internal"
+ value = "true"
+ }
+ ] : []
+ content {
+ name = set.value.name
+ value = set.value.value
+ }
+ }
+
+ depends_on = [
+ kubernetes_namespace.ingress_nginx,
+ time_sleep.wait_helm_termination[0]
+ ]
+
+ timeout = 600
+
+}
+
+data "kubernetes_service" "nginx_ingress" {
+ count = local.module_enabled ? 1 : 0
+ metadata {
+ name = "${helm_release.nginx_ingress[0].name}-${helm_release.nginx_ingress[0].chart}-controller"
+ namespace = var.anyscale_ingress_chart.namespace
+ }
+}
diff --git a/modules/anyscale-k8s-helm/helm-metricsserver.tf b/modules/anyscale-k8s-helm/helm-metricsserver.tf
new file mode 100644
index 0000000..56ef63e
--- /dev/null
+++ b/modules/anyscale-k8s-helm/helm-metricsserver.tf
@@ -0,0 +1,20 @@
+# Description: This file contains the terraform configuration to deploy the metrics server helm chart.
+resource "helm_release" "feature_metrics_server" {
+ count = local.module_enabled && var.anyscale_metrics_server_chart.enabled ? 1 : 0
+
+ name = var.anyscale_metrics_server_chart.name
+ repository = var.anyscale_metrics_server_chart.repository
+ chart = var.anyscale_metrics_server_chart.chart
+ namespace = var.anyscale_metrics_server_chart.namespace
+ version = var.anyscale_metrics_server_chart.chart_version
+
+ create_namespace = true
+
+ dynamic "set" {
+ for_each = var.anyscale_metrics_server_chart.values
+ content {
+ name = set.key
+ value = set.value
+ }
+ }
+}
diff --git a/modules/anyscale-k8s-helm/helm-nvidia.tf b/modules/anyscale-k8s-helm/helm-nvidia.tf
new file mode 100644
index 0000000..a31bd6d
--- /dev/null
+++ b/modules/anyscale-k8s-helm/helm-nvidia.tf
@@ -0,0 +1,20 @@
+# Description: This file contains the terraform configuration to deploy the NVIDIA device plugin helm chart.
+resource "helm_release" "nvidia" {
+ count = local.module_enabled && var.anyscale_nvidia_device_plugin_chart.enabled ? 1 : 0
+ name = var.anyscale_nvidia_device_plugin_chart.name
+ repository = var.anyscale_nvidia_device_plugin_chart.repository
+ chart = var.anyscale_nvidia_device_plugin_chart.chart
+ namespace = var.anyscale_nvidia_device_plugin_chart.namespace
+ version = var.anyscale_nvidia_device_plugin_chart.chart_version
+
+ create_namespace = true
+
+ dynamic "set" {
+ for_each = var.anyscale_nvidia_device_plugin_chart.values
+ content {
+ name = set.key
+ value = set.value
+ }
+ }
+
+}
diff --git a/modules/anyscale-k8s-helm/helm-prometheus.tf b/modules/anyscale-k8s-helm/helm-prometheus.tf
new file mode 100644
index 0000000..57d7404
--- /dev/null
+++ b/modules/anyscale-k8s-helm/helm-prometheus.tf
@@ -0,0 +1,22 @@
+# Description: This file contains the terraform configuration to deploy the prometheus helm chart.
+resource "helm_release" "prometheus" {
+ count = local.module_enabled && var.anyscale_prometheus_chart.enabled ? 1 : 0
+
+ name = var.anyscale_prometheus_chart.name
+ repository = var.anyscale_prometheus_chart.repository
+ chart = var.anyscale_prometheus_chart.chart
+ namespace = var.anyscale_prometheus_chart.namespace
+ version = var.anyscale_prometheus_chart.chart_version
+
+ create_namespace = true
+
+ dynamic "set" {
+ for_each = var.anyscale_prometheus_chart.values
+ content {
+ name = set.key
+ value = set.value
+ }
+ }
+
+ timeout = 900
+}
diff --git a/modules/anyscale-k8s-helm/main.tf b/modules/anyscale-k8s-helm/main.tf
new file mode 100644
index 0000000..a333d50
--- /dev/null
+++ b/modules/anyscale-k8s-helm/main.tf
@@ -0,0 +1,12 @@
+locals {
+ module_enabled = var.module_enabled
+ helm_termination_grace_period_seconds = 300 # 5 minutes to allow connection draining
+}
+
+# Helm chart destruction will return immediately, we need to wait until the pods are fully evicted
+# https://github.com/hashicorp/terraform-provider-helm/issues/593
+resource "time_sleep" "wait_helm_termination" {
+ count = local.module_enabled ? 1 : 0
+
+ destroy_duration = "${local.helm_termination_grace_period_seconds}s"
+}
diff --git a/modules/anyscale-k8s-helm/outputs.tf b/modules/anyscale-k8s-helm/outputs.tf
new file mode 100644
index 0000000..7055536
--- /dev/null
+++ b/modules/anyscale-k8s-helm/outputs.tf
@@ -0,0 +1,24 @@
+output "nginx_ingress_lb_hostname" {
+ description = "Hostname of the nginx load balancer"
+ value = try(data.kubernetes_service.nginx_ingress[0].status[0].load_balancer[0].ingress[*].hostname, [])
+}
+
+output "nginx_ingress_lb_ips" {
+ description = "IPs of the nginx load balancer"
+ value = try(data.kubernetes_service.nginx_ingress[0].status[0].load_balancer[0].ingress[*].ip, [])
+}
+
+output "helm_nginx_ingress_status" {
+ description = "Status of the Ingress Helm release"
+ value = try(helm_release.nginx_ingress[0].status, "")
+}
+
+output "helm_nvidia_status" {
+ description = "Status of the Nvidia Helm release"
+ value = try(helm_release.nvidia[0].status, "")
+}
+
+output "helm_autoscaler_status" {
+ description = "Status of the Cluster Autoscaler Helm release"
+ value = try(helm_release.anyscale_cluster_autoscaler[0].status, "")
+}
diff --git a/anyscale-k8s-helm/test/README.md b/modules/anyscale-k8s-helm/test/README.md
similarity index 99%
rename from anyscale-k8s-helm/test/README.md
rename to modules/anyscale-k8s-helm/test/README.md
index 83ed867..9912858 100644
--- a/anyscale-k8s-helm/test/README.md
+++ b/modules/anyscale-k8s-helm/test/README.md
@@ -2,4 +2,3 @@
The `anyscale-k8s-helm` module is cloud agnostic.
There are tests for both AWS and GCP as subfolders.
-
diff --git a/modules/anyscale-k8s-helm/test/anyscale-aws-test/main.tf b/modules/anyscale-k8s-helm/test/anyscale-aws-test/main.tf
new file mode 100644
index 0000000..650999e
--- /dev/null
+++ b/modules/anyscale-k8s-helm/test/anyscale-aws-test/main.tf
@@ -0,0 +1,212 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# CREATE Anyscale K8s Helm Resources
+# This template creates EKS resources for Anyscale
+# Requires:
+# - VPC
+# - Security Group
+# - IAM Roles
+# - EKS Cluster
+# - EKS Nodegroups
+# ---------------------------------------------------------------------------------------------------------------------
+locals {
+ # azs = slice(data.aws_availability_zones.available.names, 0, 3)
+
+ full_tags = merge(tomap({
+ anyscale-cloud-id = var.anyscale_cloud_id,
+ anyscale-deploy-environment = var.anyscale_deploy_env
+ }),
+ var.tags
+ )
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Create resources for EKS TF Module
+# Creates a VPC
+# Creates a Security Group
+# Creates IAM Roles
+# ---------------------------------------------------------------------------------------------------------------------
+locals {
+ public_subnets = ["172.24.101.0/24", "172.24.102.0/24", "172.24.103.0/24"]
+ private_subnets = ["172.24.20.0/24", "172.24.21.0/24", "172.24.22.0/24"]
+}
+module "eks_vpc" {
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-vpc"
+
+ anyscale_vpc_name = "tftest-k8s-helm"
+ cidr_block = "172.24.0.0/16"
+
+ public_subnets = local.public_subnets
+ private_subnets = local.private_subnets
+}
+locals {
+ # Because subnet ID may not be known at plan time, we cannot use it as a key
+ anyscale_subnet_count = length(local.private_subnets)
+}
+
+module "eks_securitygroup" {
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-securitygroups"
+
+ vpc_id = module.eks_vpc.vpc_id
+
+ security_group_name_prefix = "tftest-k8s-helm-"
+
+ ingress_with_self = [
+ { rule = "all-all" }
+ ]
+}
+
+module "eks_iam_roles" {
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-iam"
+
+ module_enabled = true
+ create_anyscale_access_role = false
+ create_cluster_node_instance_profile = false
+ create_iam_s3_policy = false
+
+ create_anyscale_eks_cluster_role = true
+ anyscale_eks_cluster_role_name = "tftest-k8s-helm-cluster"
+ create_anyscale_eks_node_role = true
+ anyscale_eks_node_role_name = "tftest-k8s-helm-node-role"
+
+ tags = local.full_tags
+}
+
+locals {
+ coredns_config = jsonencode({
+ affinity = {
+ nodeAffinity = {
+ requiredDuringSchedulingIgnoredDuringExecution = {
+ nodeSelectorTerms = [
+ {
+ matchExpressions = [
+ {
+ key = "node-type"
+ operator = "In"
+ values = ["management"]
+ }
+ ]
+ }
+ ]
+ }
+ }
+ },
+ nodeSelector = {
+ "node-type" = "management"
+ },
+ tolerations = [
+ {
+ key = "CriticalAddonsOnly"
+ operator = "Exists"
+ },
+ {
+ effect = "NoSchedule"
+ key = "node-role.kubernetes.io/control-plane"
+ }
+ ],
+ replicaCount = 2
+ })
+}
+
+module "eks_cluster" {
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-eks-cluster"
+
+ module_enabled = true
+
+ anyscale_subnet_ids = module.eks_vpc.public_subnet_ids
+ anyscale_subnet_count = local.anyscale_subnet_count
+ anyscale_security_group_id = module.eks_securitygroup.security_group_id
+ eks_role_arn = module.eks_iam_roles.iam_anyscale_eks_cluster_role_arn
+ anyscale_eks_name = "tftest-k8s-helm"
+
+ eks_addons = [
+ {
+ addon_name = "coredns"
+ addon_version = "v1.11.1-eksbuild.8"
+ configuration_values = local.coredns_config
+ }
+ ]
+ eks_addons_depends_on = module.eks_nodegroups
+
+ tags = local.full_tags
+
+ depends_on = [module.eks_iam_roles, module.eks_vpc, module.eks_securitygroup]
+}
+
+module "eks_nodegroups" {
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-eks-nodegroups"
+
+ module_enabled = true
+
+ eks_node_role_arn = module.eks_iam_roles.iam_anyscale_eks_node_role_arn
+ eks_cluster_name = module.eks_cluster.eks_cluster_name
+ subnet_ids = module.eks_vpc.public_subnet_ids
+
+ tags = local.full_tags
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Create Helm Resources with no optional parameters
+# ---------------------------------------------------------------------------------------------------------------------
+module "all_defaults" {
+ source = "../../"
+
+ module_enabled = true
+ cloud_provider = "aws"
+
+ kubernetes_cluster_name = module.eks_cluster.eks_cluster_name
+
+ depends_on = [module.eks_nodegroups]
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Create Helm Resources with as many optional parameters as possible - not currently working.
+# ---------------------------------------------------------------------------------------------------------------------
+# module "kitchen_sink" {
+# source = "../../"
+
+# module_enabled = true
+# cloud_provider = "aws"
+
+# kubernetes_cluster_name = module.eks_cluster.eks_cluster_name
+
+# anyscale_cluster_autoscaler_chart = {
+# enabled = false
+# }
+# anyscale_nvidia_device_plugin_chart = {
+# enabled = false
+# }
+
+# anyscale_metrics_server_chart = {
+# enabled = false
+# }
+
+# anyscale_ingress_chart = {
+# enabled = true
+# name = "kitchensink-ingress"
+# repository = "https://kubernetes.github.io/ingress-nginx"
+# chart = "ingress-nginx"
+# chart_version = "4.11.1"
+# namespace = "kitchensink-ingress-nginx"
+# values = {
+# "controller.service.type" = "LoadBalancer"
+# "controller.allowSnippetAnnotations" = "true"
+# "controller.autoscaling.enabled" = "true"
+# }
+# }
+# anyscale_ingress_aws_nlb_internal = true
+
+# depends_on = [module.eks_nodegroups]
+# }
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Do not create any resources
+# ---------------------------------------------------------------------------------------------------------------------
+module "test_no_resources" {
+ source = "../.."
+
+ module_enabled = false
+ cloud_provider = "aws"
+}
diff --git a/anyscale-k8s-helm/test/anyscale-aws-test/outputs.tf b/modules/anyscale-k8s-helm/test/anyscale-aws-test/outputs.tf
similarity index 100%
rename from anyscale-k8s-helm/test/anyscale-aws-test/outputs.tf
rename to modules/anyscale-k8s-helm/test/anyscale-aws-test/outputs.tf
diff --git a/modules/anyscale-k8s-helm/test/anyscale-aws-test/variables.tf b/modules/anyscale-k8s-helm/test/anyscale-aws-test/variables.tf
new file mode 100644
index 0000000..9393991
--- /dev/null
+++ b/modules/anyscale-k8s-helm/test/anyscale-aws-test/variables.tf
@@ -0,0 +1,58 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# ENVIRONMENT VARIABLES
+# Define these secrets as environment variables
+# ---------------------------------------------------------------------------------------------------------------------
+
+# AWS_ACCESS_KEY_ID
+# AWS_SECRET_ACCESS_KEY
+
+# ---------------------------------------------------------------------------------------------------------------------
+# REQUIRED VARIABLES
+# These variables must be set when using this module.
+# ---------------------------------------------------------------------------------------------------------------------
+
+variable "aws_region" {
+ description = "The AWS region in which all resources will be created."
+ type = string
+ default = "us-east-2"
+}
+
+variable "anyscale_cloud_id" {
+ description = "(Optional) Anyscale Cloud ID. Default is `null`."
+ type = string
+ default = null
+ validation {
+ condition = (
+ var.anyscale_cloud_id == null ? true : (
+ length(var.anyscale_cloud_id) > 4 &&
+ substr(var.anyscale_cloud_id, 0, 4) == "cld_"
+ )
+ )
+ error_message = "The anyscale_cloud_id value must start with \"cld_\"."
+ }
+}
+
+# ------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These variables have defaults, but may be overridden.
+# ------------------------------------------------------------------------------
+variable "anyscale_deploy_env" {
+ description = "(Optional) Anyscale deploy environment. Used in resource names and tags."
+ type = string
+ default = "production"
+ validation {
+ condition = (
+ var.anyscale_deploy_env == "production" || var.anyscale_deploy_env == "development" || var.anyscale_deploy_env == "test"
+ )
+ error_message = "The anyscale_deploy_env only allows `production`, `test`, or `development`"
+ }
+}
+
+variable "tags" {
+ description = "(Optional) A map of tags to all resources that accept tags."
+ type = map(string)
+ default = {
+ "test" : true,
+ "environment" : "test"
+ }
+}
diff --git a/anyscale-k8s-helm/test/anyscale-aws-test/versions.tf b/modules/anyscale-k8s-helm/test/anyscale-aws-test/versions.tf
similarity index 69%
rename from anyscale-k8s-helm/test/anyscale-aws-test/versions.tf
rename to modules/anyscale-k8s-helm/test/anyscale-aws-test/versions.tf
index 47701d0..f7f58d0 100644
--- a/anyscale-k8s-helm/test/anyscale-aws-test/versions.tf
+++ b/modules/anyscale-k8s-helm/test/anyscale-aws-test/versions.tf
@@ -33,6 +33,17 @@ provider "helm" {
}
}
+provider "kubernetes" {
+ host = module.eks_cluster.eks_kubeconfig.endpoint
+ cluster_ca_certificate = base64decode(module.eks_cluster.eks_kubeconfig.cluster_ca_certificate)
+
+ exec {
+ api_version = "client.authentication.k8s.io/v1beta1"
+ args = ["eks", "get-token", "--cluster-name", module.eks_cluster.eks_cluster_name]
+ command = "aws"
+ }
+}
+
provider "aws" {
region = var.aws_region
}
diff --git a/anyscale-k8s-helm/test/anyscale-gcp-test/versions.tf b/modules/anyscale-k8s-helm/test/anyscale-gcp-test/versions.tf
similarity index 100%
rename from anyscale-k8s-helm/test/anyscale-gcp-test/versions.tf
rename to modules/anyscale-k8s-helm/test/anyscale-gcp-test/versions.tf
diff --git a/modules/anyscale-k8s-helm/variables.tf b/modules/anyscale-k8s-helm/variables.tf
new file mode 100644
index 0000000..62ab5cd
--- /dev/null
+++ b/modules/anyscale-k8s-helm/variables.tf
@@ -0,0 +1,359 @@
+# ------------------------------------------------------------------------------
+# REQUIRED PARAMETERS
+# These variables must be set when using this module.
+# ------------------------------------------------------------------------------
+variable "cloud_provider" {
+ description = <<-EOT
+ (Required) The cloud provider (aws or gcp)
+
+ ex:
+ ```
+ cloud_provider = "aws"
+ ```
+ EOT
+ type = string
+ validation {
+ condition = (
+ var.cloud_provider == "aws" || var.cloud_provider == "gcp"
+ )
+ error_message = "The cloud_provider only allows `aws` or `gcp`"
+ }
+}
+
+variable "kubernetes_cluster_name" {
+ type = string
+ description = <<-EOT
+ (Optional) The name of the Kubernetes cluster.
+
+ ex:
+ ```
+ kubernetes_cluster_name = "my-cluster"
+ ```
+ EOT
+ default = null
+}
+
+# ------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These variables have defaults, but may be overridden.
+# ------------------------------------------------------------------------------
+variable "module_enabled" {
+ description = <<-EOT
+ (Optional) Determines if this module should create resources.
+
+ If set to true, `eks_role_arn`, `anyscale_subnet_ids`, and `anyscale_security_group_id` must be provided.
+ ex:
+ ```
+ module_enabled = true
+ ```
+ EOT
+ type = bool
+ default = false
+}
+
+
+# variable "anyscale_node_tolerations" {
+# description = <<-EOT
+# (Optional) List of tolerations to apply to helm charts that need to run on Anyscale Nodes.
+
+# ex:
+# ```
+# anyscale_node_tolerations = [
+# {
+# key = "node.anyscale.com/capacity-type"
+# operator = "Equal"
+# value = "ANY"
+# effect = "NoSchedule"
+# },
+# {
+# key = "node.anyscale.com/accelerator-type"
+# operator = "Equal"
+# value = "GPU"
+# effect = "NoSchedule"
+# }
+# ]
+# ```
+# EOT
+# type = list(
+# object({
+# key = string
+# operator = string
+# value = string
+# effect = string
+# })
+# )
+# default = [
+# {
+# key = "node.anyscale.com/capacity-type"
+# operator = "Equal"
+# value = "ANY"
+# effect = "NoSchedule"
+# },
+# {
+# key = "node.anyscale.com/accelerator-type"
+# operator = "Equal"
+# value = "GPU"
+# effect = "NoSchedule"
+# }
+# ]
+# }
+
+# ------------------------------------------------------------------------------
+# Helm Chart Variables
+# ------------------------------------------------------------------------------
+variable "anyscale_cluster_autoscaler_chart" {
+ description = <<-EOT
+ (Optional) The Helm chart to install the Cluster Autoscaler.
+
+ ex:
+ ```
+ anyscale_cluster_autoscaler_chart = {
+ enabled = true
+ name = "cluster-autoscaler"
+ respository = "https://kubernetes.github.io/autoscaler"
+ chart = "cluster-autoscaler"
+ chart_version = "9.37.0"
+ namespace = "kube-system"
+ values = {
+ "some.other.config" = "value"
+ }
+ }
+ ```
+ EOT
+ type = object({
+ enabled = bool
+ name = optional(string)
+ repository = optional(string)
+ chart = optional(string)
+ chart_version = optional(string)
+ namespace = optional(string)
+ values = optional(map(string))
+ })
+ default = {
+ enabled = true
+ name = "cluster-autoscaler"
+ repository = "https://kubernetes.github.io/autoscaler"
+ chart = "cluster-autoscaler"
+ chart_version = "9.37.0"
+ namespace = "kube-system"
+ values = {}
+ }
+}
+
+variable "anyscale_ingress_chart" {
+ description = <<-EOT
+ (Optional) The Helm chart to install the Cluster Ingress.
+
+ ex:
+ ```
+ anyscale_ingress_chart = {
+ enabled = true
+ name = "anyscale-ingress"
+ respository = "https://kubernetes.github.io/ingress-nginx"
+ chart = "ingress-nginx"
+ chart_version = "4.11.1"
+ namespace = "ingress-nginx"
+ values = {
+ "some.other.config" = "value"
+ }
+ }
+ ```
+ EOT
+ type = object({
+ enabled = bool
+ name = optional(string)
+ repository = optional(string)
+ chart = optional(string)
+ chart_version = optional(string)
+ namespace = optional(string)
+ values = optional(map(string))
+ })
+ default = {
+ enabled = true
+ name = "anyscale-ingress"
+ repository = "https://kubernetes.github.io/ingress-nginx"
+ chart = "ingress-nginx"
+ chart_version = "4.11.1"
+ namespace = "ingress-nginx"
+ values = {
+ "controller.service.type" = "LoadBalancer"
+ "controller.allowSnippetAnnotations" = "true"
+ "controller.autoscaling.enabled" = "true"
+ }
+ }
+}
+
+variable "anyscale_ingress_aws_nlb_internal" {
+ description = <<-EOT
+ (Optioanl) Determines if the AWS NLB should be internal.
+
+ Requires `cloud_provider` to be set to `aws`.
+ Requires `anyscale_ingress_chart` to be enabled.
+
+ ex:
+ ```
+ anyscale_ingress_aws_nlb_internal = true
+ ```
+ EOT
+ type = bool
+ default = false
+}
+
+variable "anyscale_nvidia_device_plugin_chart" {
+ description = <<-EOT
+ (Optional) The Helm chart to install the NVIDIA Device Plugin.
+
+ Valid settings can be found in the [nvidia documentation](https://github.com/NVIDIA/k8s-device-plugin?tab=readme-ov-file#deploying-with-gpu-feature-discovery-for-automatic-node-labels)
+
+ ex:
+ ```
+ anyscale_nvidia_device_plugin_chart = {
+ enabled = true
+ name = "nvidia-device-plugin"
+ respository = "https://nvidia.github.io/k8s-device-plugin"
+ chart = "nvidia-device-plugin"
+ chart_version = "0.16.2"
+ namespace = "nvidia-device-plugin"
+ values = {
+ "some.other.config" = "value"
+ }
+ }
+ ```
+ EOT
+ type = object({
+ enabled = bool
+ name = optional(string)
+ repository = optional(string)
+ chart = optional(string)
+ chart_version = optional(string)
+ namespace = optional(string)
+ values = optional(map(string))
+ })
+ default = {
+ enabled = true
+ name = "anyscale-nvidia-device-plugin"
+ repository = "https://nvidia.github.io/k8s-device-plugin"
+ chart = "nvidia-device-plugin"
+ chart_version = "0.16.2"
+ namespace = "nvidia-device-plugin"
+ values = {
+ "gfd.enabled" = "true",
+ "priorityClassName" = "system-node-critical"
+
+ "nfd.worker.tolerations[0].key" = "node-role.kubernetes.io/master"
+ "nfd.worker.tolerations[0].operator" = "Equal"
+ "nfd.worker.tolerations[0].value" = ""
+ "nfd.worker.tolerations[0].effect" = "NoSchedule"
+
+ "nfd.worker.tolerations[1].key" = "nvidia.com/gpu"
+ "nfd.worker.tolerations[1].operator" = "Equal"
+ "nfd.worker.tolerations[1].value" = "present"
+ "nfd.worker.tolerations[1].effect" = "NoSchedule"
+
+ "nfd.worker.tolerations[2].key" = "node.anyscale.com/accelerator-type"
+ "nfd.worker.tolerations[2].operator" = "Equal"
+ "nfd.worker.tolerations[2].value" = "GPU"
+ "nfd.worker.tolerations[2].effect" = "NoSchedule"
+
+ "nfd.worker.tolerations[3].key" = "node.anyscale.com/capacity-type"
+ "nfd.worker.tolerations[3].operator" = "Equal"
+ "nfd.worker.tolerations[3].value" = "ANY"
+ "nfd.worker.tolerations[3].effect" = "NoSchedule"
+
+ "tolerations[0].key" = "nvidia.com/gpu"
+ "tolerations[0].operator" = "Equal"
+ "tolerations[0].value" = "present"
+ "tolerations[0].effect" = "NoSchedule"
+
+ "tolerations[1].key" = "node.anyscale.com/accelerator-type"
+ "tolerations[1].operator" = "Equal"
+ "tolerations[1].value" = "GPU"
+ "tolerations[1].effect" = "NoSchedule"
+
+ "tolerations[2].key" = "node.anyscale.com/capacity-type"
+ "tolerations[2].operator" = "Equal"
+ "tolerations[2].value" = "ANY"
+ "tolerations[2].effect" = "NoSchedule"
+ }
+ }
+}
+
+variable "anyscale_metrics_server_chart" {
+ description = <<-EOT
+ (Optional) The Helm chart to install the Metrics Server.
+
+ Required for the Anyscale Autoscaler to function.
+
+ ex:
+ ```
+ anyscale_metrics_server_chart = {
+ enabled = true
+ name = "metrics-server"
+ respository = "https://kubernetes-sigs.github.io/metrics-server/"
+ chart = "metrics-server"
+ chart_version = "3.12.1"
+ namespace = "metrics-server"
+ values = {
+ "some.other.config" = "value"
+ }
+ }
+ ```
+ EOT
+ type = object({
+ enabled = bool
+ name = optional(string)
+ repository = optional(string)
+ chart = optional(string)
+ chart_version = optional(string)
+ namespace = optional(string)
+ values = optional(map(string))
+ })
+ default = {
+ enabled = true
+ name = "metrics-server"
+ repository = "https://kubernetes-sigs.github.io/metrics-server/"
+ chart = "metrics-server"
+ chart_version = "3.12.1"
+ namespace = "metrics-server"
+ values = {}
+ }
+}
+
+variable "anyscale_prometheus_chart" {
+ description = <<-EOT
+ (Optional) The Helm chart to install Prometheus.
+
+ ex:
+ ```
+ anyscale_prometheus_chart = {
+ enabled = true
+ name = "prometheus"
+ respository = "https://prometheus-community.github.io/helm-charts"
+ chart = "prometheus"
+ chart_version = "16.0.0"
+ namespace = "prometheus"
+ values = {
+ "some.other.config" = "value"
+ }
+ }
+ ```
+ EOT
+ type = object({
+ enabled = bool
+ name = optional(string)
+ repository = optional(string)
+ chart = optional(string)
+ chart_version = optional(string)
+ namespace = optional(string)
+ values = optional(map(string))
+ })
+ default = {
+ enabled = false
+ name = "prometheus"
+ repository = "https://prometheus-community.github.io/helm-charts"
+ chart = "prometheus"
+ chart_version = "25.26.0"
+ namespace = "prometheus"
+ values = {}
+ }
+}
diff --git a/anyscale-k8s-helm/versions.tf b/modules/anyscale-k8s-helm/versions.tf
similarity index 61%
rename from anyscale-k8s-helm/versions.tf
rename to modules/anyscale-k8s-helm/versions.tf
index 071177d..8a43595 100644
--- a/anyscale-k8s-helm/versions.tf
+++ b/modules/anyscale-k8s-helm/versions.tf
@@ -2,16 +2,6 @@ terraform {
required_version = ">= 1.0"
required_providers {
- aws = {
- source = "hashicorp/aws"
- version = "~> 5.0"
- }
-
- google = {
- source = "hashicorp/google"
- version = "~> 5.0"
- }
-
helm = {
source = "hashicorp/helm"
version = "~> 2.0"
@@ -21,6 +11,10 @@ terraform {
source = "hashicorp/kubernetes"
version = "~> 2.0"
}
+
+ time = {
+ source = "hashicorp/time"
+ version = ">= 0.12"
+ }
}
}
-
diff --git a/anyscale-k8s-helm/README.md b/modules/anyscale-k8s-namespace/README.md
similarity index 51%
rename from anyscale-k8s-helm/README.md
rename to modules/anyscale-k8s-namespace/README.md
index 887022a..dfb542a 100644
--- a/anyscale-k8s-helm/README.md
+++ b/modules/anyscale-k8s-namespace/README.md
@@ -1,9 +1,15 @@
[![Build Status][badge-build]][build-status]
[![Terraform Version][badge-terraform]](https://github.com/hashicorp/terraform/releases)
+[![OpenTofu Version][badge-opentofu]](https://github.com/opentofu/opentofu/releases)
+[![Kubernetes Provider Version][badge-tf-kubernetes]](https://github.com/terraform-providers/terraform-provider-kubernetes/releases)
[![AWS Provider Version][badge-tf-aws]](https://github.com/terraform-providers/terraform-provider-aws/releases)
+[![Google Provider Version][badge-tf-google]](https://github.com/terraform-providers/terraform-provider-google/releases)
-# anyscale-k8s-helm
-This module creates Kubernetes helm charts for Anyscale applications and workloads.
+# anyscale-k8s-namespace
+
+This module creates a Kubernetes Namespace for Anyscale.
+
+The Anyscale Namespace can also be created via the Anycsale Helm Chart.
@@ -14,4 +20,7 @@ This module creates Kubernetes helm charts for Anyscale applications and workloa
[badge-build]: https://github.com/anyscale/sa-sandbox-terraform/workflows/CI/CD%20Pipeline/badge.svg
[badge-terraform]: https://img.shields.io/badge/terraform-1.x%20-623CE4.svg?logo=terraform
[badge-tf-aws]: https://img.shields.io/badge/AWS-5.+-F8991D.svg?logo=terraform
-[build-status]: https://github.com/anyscale/sa-sandbox-terraform/actions
\ No newline at end of file
+[build-status]: https://github.com/anyscale/sa-sandbox-terraform/actions
+[badge-opentofu]: https://img.shields.io/badge/opentofu-1.x%20-623CE4.svg?logo=terraform
+[badge-tf-google]: https://img.shields.io/badge/Google-5.+-F8991D.svg?logo=terraform
+[badge-tf-kubernetes]: https://img.shields.io/badge/KUBERNETES-2.+-F8991D.svg?logo=terraform
diff --git a/modules/anyscale-k8s-namespace/main.tf b/modules/anyscale-k8s-namespace/main.tf
new file mode 100644
index 0000000..f58d6a3
--- /dev/null
+++ b/modules/anyscale-k8s-namespace/main.tf
@@ -0,0 +1,10 @@
+locals {
+ module_enabled = var.module_enabled
+}
+
+resource "kubernetes_namespace" "anyscale" {
+ count = local.module_enabled ? 1 : 0
+ metadata {
+ name = var.anyscale_kubernetes_namespace
+ }
+}
diff --git a/modules/anyscale-k8s-namespace/outputs.tf b/modules/anyscale-k8s-namespace/outputs.tf
new file mode 100644
index 0000000..6228972
--- /dev/null
+++ b/modules/anyscale-k8s-namespace/outputs.tf
@@ -0,0 +1,4 @@
+output "anyscale_kubernetes_namespace_name" {
+ description = "The name of the Kubernetes namespace."
+ value = try(kubernetes_namespace.anyscale[0].metadata[0].name, "")
+}
diff --git a/anyscale-k8s-helm/test/anyscale-aws-test/main.tf b/modules/anyscale-k8s-namespace/test/anyscale-aws-test/main.tf
similarity index 53%
rename from anyscale-k8s-helm/test/anyscale-aws-test/main.tf
rename to modules/anyscale-k8s-namespace/test/anyscale-aws-test/main.tf
index b56e2c6..1507037 100644
--- a/anyscale-k8s-helm/test/anyscale-aws-test/main.tf
+++ b/modules/anyscale-k8s-namespace/test/anyscale-aws-test/main.tf
@@ -1,5 +1,5 @@
# ---------------------------------------------------------------------------------------------------------------------
-# CREATE Anyscale K8s Helm Resources
+# CREATE Anyscale K8s ConfigMap Resources
# This template creates EKS resources for Anyscale
# Requires:
# - VPC
@@ -29,9 +29,10 @@ locals {
private_subnets = ["172.24.20.0/24", "172.24.21.0/24", "172.24.22.0/24"]
}
module "eks_vpc" {
- source = "../../../aws-anyscale-vpc"
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-vpc"
- anyscale_vpc_name = "anyscale-tftest-eks"
+ anyscale_vpc_name = "tftest-k8s-namespace"
cidr_block = "172.24.0.0/16"
public_subnets = local.public_subnets
@@ -43,11 +44,12 @@ locals {
}
module "eks_securitygroup" {
- source = "../../../aws-anyscale-securitygroups"
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-securitygroups"
vpc_id = module.eks_vpc.vpc_id
- security_group_name_prefix = "anyscale-tftest-eks-"
+ security_group_name_prefix = "tftest-k8s-namespace-"
ingress_with_self = [
{ rule = "all-all" }
@@ -55,23 +57,61 @@ module "eks_securitygroup" {
}
module "eks_iam_roles" {
- source = "../../../aws-anyscale-iam"
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-iam"
module_enabled = true
- create_anyscale_access_role = false
+ create_anyscale_access_role = true
+ anyscale_access_role_name = "tftest-k8s-namespace-controlplane-role"
create_cluster_node_instance_profile = false
create_iam_s3_policy = false
create_anyscale_eks_cluster_role = true
- anyscale_eks_cluster_role_name = "anyscale-tftest-eks-cluster-role"
+ anyscale_eks_cluster_role_name = "tftest-k8s-namespace-cluster-role"
create_anyscale_eks_node_role = true
- anyscale_eks_node_role_name = "anyscale-tftest-eks-node-role"
+ anyscale_eks_node_role_name = "tftest-k8s-namespace-node-role"
tags = local.full_tags
}
+locals {
+ coredns_config = jsonencode({
+ affinity = {
+ nodeAffinity = {
+ requiredDuringSchedulingIgnoredDuringExecution = {
+ nodeSelectorTerms = [
+ {
+ matchExpressions = [
+ {
+ key = "node-type"
+ operator = "In"
+ values = ["management"]
+ }
+ ]
+ }
+ ]
+ }
+ }
+ },
+ nodeSelector = {
+ "node-type" = "management"
+ },
+ tolerations = [
+ {
+ key = "CriticalAddonsOnly"
+ operator = "Exists"
+ },
+ {
+ effect = "NoSchedule"
+ key = "node-role.kubernetes.io/control-plane"
+ }
+ ],
+ replicaCount = 2
+ })
+}
+
module "eks_cluster" {
- source = "../../../aws-anyscale-eks-cluster"
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-eks-cluster"
module_enabled = true
@@ -79,12 +119,16 @@ module "eks_cluster" {
anyscale_subnet_count = local.anyscale_subnet_count
anyscale_security_group_id = module.eks_securitygroup.security_group_id
eks_role_arn = module.eks_iam_roles.iam_anyscale_eks_cluster_role_arn
+ anyscale_eks_name = "tftest-k8s-namespace"
tags = local.full_tags
+
+ depends_on = [module.eks_iam_roles, module.eks_vpc, module.eks_securitygroup]
}
+
# ---------------------------------------------------------------------------------------------------------------------
-# Create Helm Resources with no optional parameters
+# Create Resources with no optional parameters
# ---------------------------------------------------------------------------------------------------------------------
module "all_defaults" {
source = "../../"
@@ -92,9 +136,22 @@ module "all_defaults" {
module_enabled = true
cloud_provider = "aws"
- kubernetes_cluster_name = module.eks_cluster.eks_cluster_name
- kubernetes_endpoint_address = module.eks_cluster.eks_cluster_endpoint
- kubernetes_cluster_ca_data = module.eks_cluster.eks_cluster_ca_data
+ depends_on = [module.eks_cluster]
+
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Create Resources with as many optional parameters as possible
+# ---------------------------------------------------------------------------------------------------------------------
+module "kitchen_sink" {
+ source = "../../"
+
+ module_enabled = true
+ cloud_provider = "aws"
+
+ anyscale_kubernetes_namespace = "tftest-k8s-namespace"
+ depends_on = [module.eks_cluster]
+
}
# ---------------------------------------------------------------------------------------------------------------------
diff --git a/modules/anyscale-k8s-namespace/test/anyscale-aws-test/outputs.tf b/modules/anyscale-k8s-namespace/test/anyscale-aws-test/outputs.tf
new file mode 100644
index 0000000..6db4254
--- /dev/null
+++ b/modules/anyscale-k8s-namespace/test/anyscale-aws-test/outputs.tf
@@ -0,0 +1,23 @@
+# --------------
+# Defaults Test
+# --------------
+output "all_defaults_resources" {
+ description = "The resources of the All Defaults test"
+ value = module.all_defaults
+}
+
+# ------------------
+# Kitchen Sink Test
+# ------------------
+output "kitchen_sink_resources" {
+ description = "The resources of the Kitchen Sink test"
+ value = module.kitchen_sink
+}
+
+# -----------------
+# No resource test
+# -----------------
+output "test_no_resources" {
+ description = "The outputs of the no_resource resource - should all be empty"
+ value = module.test_no_resources
+}
diff --git a/modules/anyscale-k8s-namespace/test/anyscale-aws-test/variables.tf b/modules/anyscale-k8s-namespace/test/anyscale-aws-test/variables.tf
new file mode 100644
index 0000000..9393991
--- /dev/null
+++ b/modules/anyscale-k8s-namespace/test/anyscale-aws-test/variables.tf
@@ -0,0 +1,58 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# ENVIRONMENT VARIABLES
+# Define these secrets as environment variables
+# ---------------------------------------------------------------------------------------------------------------------
+
+# AWS_ACCESS_KEY_ID
+# AWS_SECRET_ACCESS_KEY
+
+# ---------------------------------------------------------------------------------------------------------------------
+# REQUIRED VARIABLES
+# These variables must be set when using this module.
+# ---------------------------------------------------------------------------------------------------------------------
+
+variable "aws_region" {
+ description = "The AWS region in which all resources will be created."
+ type = string
+ default = "us-east-2"
+}
+
+variable "anyscale_cloud_id" {
+ description = "(Optional) Anyscale Cloud ID. Default is `null`."
+ type = string
+ default = null
+ validation {
+ condition = (
+ var.anyscale_cloud_id == null ? true : (
+ length(var.anyscale_cloud_id) > 4 &&
+ substr(var.anyscale_cloud_id, 0, 4) == "cld_"
+ )
+ )
+ error_message = "The anyscale_cloud_id value must start with \"cld_\"."
+ }
+}
+
+# ------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These variables have defaults, but may be overridden.
+# ------------------------------------------------------------------------------
+variable "anyscale_deploy_env" {
+ description = "(Optional) Anyscale deploy environment. Used in resource names and tags."
+ type = string
+ default = "production"
+ validation {
+ condition = (
+ var.anyscale_deploy_env == "production" || var.anyscale_deploy_env == "development" || var.anyscale_deploy_env == "test"
+ )
+ error_message = "The anyscale_deploy_env only allows `production`, `test`, or `development`"
+ }
+}
+
+variable "tags" {
+ description = "(Optional) A map of tags to all resources that accept tags."
+ type = map(string)
+ default = {
+ "test" : true,
+ "environment" : "test"
+ }
+}
diff --git a/modules/anyscale-k8s-namespace/test/anyscale-aws-test/versions.tf b/modules/anyscale-k8s-namespace/test/anyscale-aws-test/versions.tf
new file mode 100644
index 0000000..c46128a
--- /dev/null
+++ b/modules/anyscale-k8s-namespace/test/anyscale-aws-test/versions.tf
@@ -0,0 +1,31 @@
+terraform {
+ required_version = ">= 1.0"
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "~> 5.0"
+ }
+
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = "~> 2.0"
+ }
+ }
+}
+
+
+
+provider "kubernetes" {
+ host = module.eks_cluster.eks_kubeconfig.endpoint
+ cluster_ca_certificate = base64decode(module.eks_cluster.eks_kubeconfig.cluster_ca_certificate)
+
+ exec {
+ api_version = "client.authentication.k8s.io/v1beta1"
+ args = ["eks", "get-token", "--cluster-name", module.eks_cluster.eks_cluster_name]
+ command = "aws"
+ }
+}
+
+provider "aws" {
+ region = var.aws_region
+}
diff --git a/modules/anyscale-k8s-namespace/variables.tf b/modules/anyscale-k8s-namespace/variables.tf
new file mode 100644
index 0000000..24b1438
--- /dev/null
+++ b/modules/anyscale-k8s-namespace/variables.tf
@@ -0,0 +1,65 @@
+# ------------------------------------------------------------------------------
+# REQUIRED PARAMETERS
+# These variables must be set when using this module.
+# ------------------------------------------------------------------------------
+variable "cloud_provider" {
+ description = <<-EOT
+ (Required) The cloud provider (aws or gcp)
+
+ ex:
+ ```
+ cloud_provider = "aws"
+ ```
+ EOT
+ type = string
+ validation {
+ condition = (
+ var.cloud_provider == "aws" || var.cloud_provider == "gcp"
+ )
+ error_message = "The cloud_provider only allows `aws` or `gcp`"
+ }
+}
+
+variable "kubernetes_cluster_name" {
+ type = string
+ description = <<-EOT
+ (Optional) The name of the Kubernetes cluster.
+
+ ex:
+ ```
+ kubernetes_cluster_name = "my-cluster"
+ ```
+ EOT
+ default = null
+}
+
+# ------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These variables have defaults, but may be overridden.
+# ------------------------------------------------------------------------------
+variable "module_enabled" {
+ description = <<-EOT
+ (Optional) Determines if this module should create resources.
+
+ If set to true, `eks_role_arn`, `anyscale_subnet_ids`, and `anyscale_security_group_id` must be provided.
+ ex:
+ ```
+ module_enabled = true
+ ```
+ EOT
+ type = bool
+ default = true
+}
+
+variable "anyscale_kubernetes_namespace" {
+ description = <<-EOT
+ (Optional) The name of the Kubernetes namespace.
+
+ ex:
+ ```
+ anyscale_kubernetes_namespace = "anyscale-k8s"
+ ```
+ EOT
+ type = string
+ default = "anyscale-k8s"
+}
diff --git a/modules/anyscale-k8s-namespace/versions.tf b/modules/anyscale-k8s-namespace/versions.tf
new file mode 100644
index 0000000..94019cd
--- /dev/null
+++ b/modules/anyscale-k8s-namespace/versions.tf
@@ -0,0 +1,10 @@
+terraform {
+ required_version = ">= 1.0"
+
+ required_providers {
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = "~> 2.0"
+ }
+ }
+}
diff --git a/modules/anyscale-k8s-persistent-volume/README.md b/modules/anyscale-k8s-persistent-volume/README.md
new file mode 100644
index 0000000..2b7207b
--- /dev/null
+++ b/modules/anyscale-k8s-persistent-volume/README.md
@@ -0,0 +1,75 @@
+[![Build Status][badge-build]][build-status]
+[![Terraform Version][badge-terraform]](https://github.com/hashicorp/terraform/releases)
+[![OpenTofu Version][badge-opentofu]](https://github.com/opentofu/opentofu/releases)
+[![Kubernetes Provider Version][badge-tf-kubernetes]](https://github.com/terraform-providers/terraform-provider-kubernetes/releases)
+[![AWS Provider Version][badge-tf-aws]](https://github.com/terraform-providers/terraform-provider-aws/releases)
+[![Google Provider Version][badge-tf-google]](https://github.com/terraform-providers/terraform-provider-google/releases)
+
+# anyscale-k8s-persistent-volume - UNUSED
+
+!!! Unused sub-module !!!
+
+This module creates the resources for a persistent volume NFS mount and persistent volume claim.
+
+
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| [terraform](#requirement\_terraform) | >= 1.0 |
+| [kubernetes](#requirement\_kubernetes) | ~> 2.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| [kubernetes](#provider\_kubernetes) | 2.32.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [kubernetes_persistent_volume.anyscale](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume) | resource |
+| [kubernetes_persistent_volume_claim.anyscale](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/persistent_volume_claim) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| [cloud\_provider](#input\_cloud\_provider) | (Required) The cloud provider (aws or gcp)
ex:cloud_provider = "aws"
| `string` | n/a | yes |
+| [anyscale\_kubernetes\_namespace](#input\_anyscale\_kubernetes\_namespace) | (Optional) The name of the Kubernetes namespace.
ex:anyscale_kubernetes_namespace = "anyscale-k8s"
| `string` | `"anyscale-k8s"` | no |
+| [aws\_efs\_file\_system\_id](#input\_aws\_efs\_file\_system\_id) | (Optional) The ID of the EFS file system.
Required if `cloud_provider` is `aws`.
ex:aws_efs_file_system_id = "fs-12345678"
| `string` | `null` | no |
+| [gcp\_filestore\_ip](#input\_gcp\_filestore\_ip) | (Optional) The Filestore IP address.
Required if `cloud_provider` is `gcp`.
ex:gcp_filestore_ip = "172.16.0.12"
| `string` | `null` | no |
+| [gcp\_filestore\_share\_name](#input\_gcp\_filestore\_share\_name) | (Optional) The Filestore share name.
Required if `cloud_provider` is `gcp`.
ex:gcp_filestore_share_name = "my-share"
| `string` | `null` | no |
+| [kubernetes\_cluster\_name](#input\_kubernetes\_cluster\_name) | (Optional) The name of the Kubernetes cluster.
ex:kubernetes_cluster_name = "my-cluster"
| `string` | `null` | no |
+| [kubernetes\_persistent\_volume\_claim\_name](#input\_kubernetes\_persistent\_volume\_claim\_name) | (Optional) The name of the Kubernetes persistent volume claim.
ex:kubernetes_persistent_volume_claim_name = "anyscale-nfs-claim"
| `string` | `"anyscale-nfs-claim"` | no |
+| [kubernetes\_persistent\_volume\_name](#input\_kubernetes\_persistent\_volume\_name) | (Optional) The name of the Kubernetes persistent volume.
ex:kubernetes_persistent_volume_name = "anyscale-nfs"
| `string` | `"anyscale-nfs"` | no |
+| [kubernetes\_persistent\_volume\_size](#input\_kubernetes\_persistent\_volume\_size) | (Optional) The size of the Kubernetes persistent volume.
When using AWS EFS, this is just a placeholder. The actual size is elastically built, making this just a placeholder
ex:kubernetes_persistent_volume_size = "20Gi"
| `string` | `"20Gi"` | no |
+| [module\_enabled](#input\_module\_enabled) | (Optional) Determines if this module should create resources.
If set to true, `eks_role_arn`, `anyscale_subnet_ids`, and `anyscale_security_group_id` must be provided.
ex:module_enabled = true
| `bool` | `false` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| [kubernetes\_persistent\_volume\_claim\_name](#output\_kubernetes\_persistent\_volume\_claim\_name) | The name of the Kubernetes persistent volume claim. |
+| [kubernetes\_persistent\_volume\_claim\_namespace](#output\_kubernetes\_persistent\_volume\_claim\_namespace) | The namespace of the Kubernetes persistent volume claim. |
+| [kubernetes\_persistent\_volume\_claim\_storageclassname](#output\_kubernetes\_persistent\_volume\_claim\_storageclassname) | The storage class name of the Kubernetes persistent volume claim. |
+| [kubernetes\_persistent\_volume\_claim\_volumename](#output\_kubernetes\_persistent\_volume\_claim\_volumename) | The volume name of the Kubernetes persistent volume claim. |
+| [kubernetes\_persistent\_volume\_name](#output\_kubernetes\_persistent\_volume\_name) | The name of the Kubernetes persistent volume. |
+
+
+
+[Terraform]: https://www.terraform.io
+[Issues]: https://github.com/anyscale/sa-sandbox-terraform/issues
+[badge-build]: https://github.com/anyscale/sa-sandbox-terraform/workflows/CI/CD%20Pipeline/badge.svg
+[badge-terraform]: https://img.shields.io/badge/terraform-1.x%20-623CE4.svg?logo=terraform
+[badge-tf-aws]: https://img.shields.io/badge/AWS-5.+-F8991D.svg?logo=terraform
+[build-status]: https://github.com/anyscale/sa-sandbox-terraform/actions
+[badge-opentofu]: https://img.shields.io/badge/opentofu-1.x%20-623CE4.svg?logo=terraform
+[badge-tf-google]: https://img.shields.io/badge/Google-5.+-F8991D.svg?logo=terraform
+[badge-tf-kubernetes]: https://img.shields.io/badge/KUBERNETES-2.+-F8991D.svg?logo=terraform
diff --git a/modules/anyscale-k8s-persistent-volume/main.tf b/modules/anyscale-k8s-persistent-volume/main.tf
new file mode 100644
index 0000000..f746764
--- /dev/null
+++ b/modules/anyscale-k8s-persistent-volume/main.tf
@@ -0,0 +1,45 @@
+locals {
+ module_enabled = var.module_enabled
+}
+
+resource "kubernetes_persistent_volume" "anyscale" {
+ count = local.module_enabled ? 1 : 0
+ metadata {
+ name = var.kubernetes_persistent_volume_name
+ }
+
+ spec {
+ capacity = {
+ storage = var.kubernetes_persistent_volume_size
+ }
+
+ access_modes = ["ReadWriteMany"]
+ persistent_volume_reclaim_policy = "Retain"
+
+ storage_class_name = var.cloud_provider == "aws" ? "efs-sc" : "filestore-sc"
+ persistent_volume_source {
+ csi {
+ driver = var.cloud_provider == "aws" ? "efs.csi.aws.com" : "filestore.csi.storage.gke.io"
+ volume_handle = var.cloud_provider == "aws" ? var.aws_efs_file_system_id : "${var.gcp_filestore_ip}/${var.gcp_filestore_share_name}"
+ }
+ }
+ }
+}
+
+resource "kubernetes_persistent_volume_claim" "anyscale" {
+ count = local.module_enabled ? 1 : 0
+ metadata {
+ name = var.kubernetes_persistent_volume_claim_name
+ namespace = var.anyscale_kubernetes_namespace
+ }
+
+ spec {
+ access_modes = ["ReadWriteMany"]
+ resources {
+ requests = {
+ storage = var.kubernetes_persistent_volume_size
+ }
+ }
+ storage_class_name = var.cloud_provider == "aws" ? "efs-sc" : "filestore-sc"
+ }
+}
diff --git a/modules/anyscale-k8s-persistent-volume/outputs.tf b/modules/anyscale-k8s-persistent-volume/outputs.tf
new file mode 100644
index 0000000..ffe73a0
--- /dev/null
+++ b/modules/anyscale-k8s-persistent-volume/outputs.tf
@@ -0,0 +1,24 @@
+output "kubernetes_persistent_volume_name" {
+ description = "The name of the Kubernetes persistent volume."
+ value = try(kubernetes_persistent_volume.anyscale[0].metadata[0].name, "")
+}
+
+output "kubernetes_persistent_volume_claim_name" {
+ description = "The name of the Kubernetes persistent volume claim."
+ value = try(kubernetes_persistent_volume_claim.anyscale[0].metadata[0].name, "")
+}
+
+output "kubernetes_persistent_volume_claim_namespace" {
+ description = "The namespace of the Kubernetes persistent volume claim."
+ value = try(kubernetes_persistent_volume_claim.anyscale[0].metadata[0].namespace, "")
+}
+
+output "kubernetes_persistent_volume_claim_volumename" {
+ description = "The volume name of the Kubernetes persistent volume claim."
+ value = try(kubernetes_persistent_volume_claim.anyscale[0].spec[0].volume_name, "")
+}
+
+output "kubernetes_persistent_volume_claim_storageclassname" {
+ description = "The storage class name of the Kubernetes persistent volume claim."
+ value = try(kubernetes_persistent_volume_claim.anyscale[0].spec[0].storage_class_name, "")
+}
diff --git a/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/main.tf b/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/main.tf
new file mode 100644
index 0000000..67d5007
--- /dev/null
+++ b/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/main.tf
@@ -0,0 +1,208 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# CREATE Anyscale K8s ConfigMap Resources
+# This template creates EKS resources for Anyscale
+# Requires:
+# - VPC
+# - Security Group
+# - IAM Roles
+# - EKS Cluster
+# ---------------------------------------------------------------------------------------------------------------------
+locals {
+ # azs = slice(data.aws_availability_zones.available.names, 0, 3)
+
+ full_tags = merge(tomap({
+ anyscale-cloud-id = var.anyscale_cloud_id,
+ anyscale-deploy-environment = var.anyscale_deploy_env
+ }),
+ var.tags
+ )
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Create resources for EKS TF Module
+# Creates a VPC
+# Creates a Security Group
+# Creates IAM Roles
+# ---------------------------------------------------------------------------------------------------------------------
+locals {
+ public_subnets = ["172.24.101.0/24", "172.24.102.0/24", "172.24.103.0/24"]
+ private_subnets = ["172.24.20.0/24", "172.24.21.0/24", "172.24.22.0/24"]
+}
+module "eks_vpc" {
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-vpc"
+
+ anyscale_vpc_name = "tftest-k8s-persistentvol"
+ cidr_block = "172.24.0.0/16"
+
+ public_subnets = local.public_subnets
+ private_subnets = local.private_subnets
+}
+locals {
+ # Because subnet ID may not be known at plan time, we cannot use it as a key
+ anyscale_subnet_count = length(local.private_subnets)
+}
+
+module "eks_securitygroup" {
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-securitygroups"
+
+ vpc_id = module.eks_vpc.vpc_id
+
+ security_group_name_prefix = "tftest-k8s-persistentvol-"
+
+ ingress_with_self = [
+ { rule = "all-all" }
+ ]
+}
+
+module "eks_iam_roles" {
+ #checkov:skip=CKV_TF_1: Test code should use the latest version of the module
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-iam"
+
+ module_enabled = true
+ create_anyscale_access_role = true
+ anyscale_access_role_name = "tftest-k8s-persistentvol-controlplane-role"
+ create_cluster_node_instance_profile = false
+ create_iam_s3_policy = false
+
+ create_anyscale_eks_cluster_role = true
+ anyscale_eks_cluster_role_name = "tftest-k8s-persistentvol-cluster-role"
+ create_anyscale_eks_node_role = true
+ anyscale_eks_node_role_name = "tftest-k8s-persistentvol-node-role"
+
+ anyscale_eks_cluster_oidc_arn = module.eks_cluster.eks_cluster_oidc_provider_arn
+ anyscale_eks_cluster_oidc_url = module.eks_cluster.eks_cluster_oidc_provider_url
+
+ create_eks_efs_csi_driver_role = true
+ eks_efs_csi_role_name = "anyscale-eks-public-efs-csi-role"
+ efs_file_system_arn = module.anyscale_efs.efs_arn
+
+ tags = local.full_tags
+}
+
+module "anyscale_efs" {
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-efs"
+
+ module_enabled = true
+
+ anyscale_efs_name = "anyscale-eks-public-efs"
+ mount_targets_subnet_count = local.anyscale_subnet_count
+ mount_targets_subnets = module.eks_vpc.private_subnet_ids
+ associated_security_group_ids = [module.eks_securitygroup.security_group_id]
+
+ tags = local.full_tags
+}
+
+locals {
+ coredns_config = jsonencode({
+ affinity = {
+ nodeAffinity = {
+ requiredDuringSchedulingIgnoredDuringExecution = {
+ nodeSelectorTerms = [
+ {
+ matchExpressions = [
+ {
+ key = "node-type"
+ operator = "In"
+ values = ["management"]
+ }
+ ]
+ }
+ ]
+ }
+ }
+ },
+ nodeSelector = {
+ "node-type" = "management"
+ },
+ tolerations = [
+ {
+ key = "CriticalAddonsOnly"
+ operator = "Exists"
+ },
+ {
+ effect = "NoSchedule"
+ key = "node-role.kubernetes.io/control-plane"
+ }
+ ],
+ replicaCount = 2
+ })
+}
+
+module "eks_cluster" {
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-eks-cluster"
+
+ module_enabled = true
+
+ anyscale_subnet_ids = module.eks_vpc.public_subnet_ids
+ anyscale_subnet_count = local.anyscale_subnet_count
+ anyscale_security_group_id = module.eks_securitygroup.security_group_id
+ eks_role_arn = module.eks_iam_roles.iam_anyscale_eks_cluster_role_arn
+ anyscale_eks_name = "tftest-k8s-persistentvol"
+
+ tags = local.full_tags
+
+ eks_addons = [
+ # Add EFS mount
+ {
+ addon_name = "aws-efs-csi-driver"
+ addon_version = "v2.0.7-eksbuild.1"
+ service_account_role_arn = module.eks_iam_roles.iam_anyscale_eks_efs_csi_driver_role_arn
+ }
+ ]
+ eks_addons_depends_on = module.anyscale_eks_nodegroups
+
+ depends_on = [module.eks_vpc, module.eks_securitygroup]
+}
+
+module "anyscale_eks_nodegroups" {
+ source = "../../../../../terraform-aws-anyscale-cloudfoundation-modules/modules/aws-anyscale-eks-nodegroups"
+
+ module_enabled = true
+
+ eks_node_role_arn = module.eks_iam_roles.iam_anyscale_eks_node_role_arn
+ eks_cluster_name = module.eks_cluster.eks_cluster_name
+ subnet_ids = module.eks_vpc.private_subnet_ids
+
+ tags = local.full_tags
+}
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Create Resources with no optional parameters
+# ---------------------------------------------------------------------------------------------------------------------
+module "all_defaults" {
+ source = "../../"
+
+ module_enabled = true
+ cloud_provider = "aws"
+
+
+ depends_on = [module.eks_cluster]
+
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Create Resources with as many optional parameters as possible
+# ---------------------------------------------------------------------------------------------------------------------
+# module "kitchen_sink" {
+# source = "../../"
+
+# module_enabled = true
+# cloud_provider = "aws"
+
+# anyscale_kubernetes_namespace = "tftest-k8s-persistentvol"
+# depends_on = [module.eks_cluster]
+
+# }
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Do not create any resources
+# ---------------------------------------------------------------------------------------------------------------------
+module "test_no_resources" {
+ source = "../.."
+
+ module_enabled = false
+ cloud_provider = "aws"
+}
diff --git a/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/outputs.tf b/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/outputs.tf
new file mode 100644
index 0000000..9b9ba14
--- /dev/null
+++ b/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/outputs.tf
@@ -0,0 +1,23 @@
+# --------------
+# Defaults Test
+# --------------
+output "all_defaults_resources" {
+ description = "The resources of the All Defaults test"
+ value = module.all_defaults
+}
+
+# ------------------
+# Kitchen Sink Test
+# ------------------
+# output "kitchen_sink_resources" {
+# description = "The resources of the Kitchen Sink test"
+# value = module.kitchen_sink
+# }
+
+# -----------------
+# No resource test
+# -----------------
+output "test_no_resources" {
+ description = "The outputs of the no_resource resource - should all be empty"
+ value = module.test_no_resources
+}
diff --git a/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/variables.tf b/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/variables.tf
new file mode 100644
index 0000000..9393991
--- /dev/null
+++ b/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/variables.tf
@@ -0,0 +1,58 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# ENVIRONMENT VARIABLES
+# Define these secrets as environment variables
+# ---------------------------------------------------------------------------------------------------------------------
+
+# AWS_ACCESS_KEY_ID
+# AWS_SECRET_ACCESS_KEY
+
+# ---------------------------------------------------------------------------------------------------------------------
+# REQUIRED VARIABLES
+# These variables must be set when using this module.
+# ---------------------------------------------------------------------------------------------------------------------
+
+variable "aws_region" {
+ description = "The AWS region in which all resources will be created."
+ type = string
+ default = "us-east-2"
+}
+
+variable "anyscale_cloud_id" {
+ description = "(Optional) Anyscale Cloud ID. Default is `null`."
+ type = string
+ default = null
+ validation {
+ condition = (
+ var.anyscale_cloud_id == null ? true : (
+ length(var.anyscale_cloud_id) > 4 &&
+ substr(var.anyscale_cloud_id, 0, 4) == "cld_"
+ )
+ )
+ error_message = "The anyscale_cloud_id value must start with \"cld_\"."
+ }
+}
+
+# ------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These variables have defaults, but may be overridden.
+# ------------------------------------------------------------------------------
+variable "anyscale_deploy_env" {
+ description = "(Optional) Anyscale deploy environment. Used in resource names and tags."
+ type = string
+ default = "production"
+ validation {
+ condition = (
+ var.anyscale_deploy_env == "production" || var.anyscale_deploy_env == "development" || var.anyscale_deploy_env == "test"
+ )
+ error_message = "The anyscale_deploy_env only allows `production`, `test`, or `development`"
+ }
+}
+
+variable "tags" {
+ description = "(Optional) A map of tags to all resources that accept tags."
+ type = map(string)
+ default = {
+ "test" : true,
+ "environment" : "test"
+ }
+}
diff --git a/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/versions.tf b/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/versions.tf
new file mode 100644
index 0000000..c46128a
--- /dev/null
+++ b/modules/anyscale-k8s-persistent-volume/test/anyscale-aws-test/versions.tf
@@ -0,0 +1,31 @@
+terraform {
+ required_version = ">= 1.0"
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "~> 5.0"
+ }
+
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = "~> 2.0"
+ }
+ }
+}
+
+
+
+provider "kubernetes" {
+ host = module.eks_cluster.eks_kubeconfig.endpoint
+ cluster_ca_certificate = base64decode(module.eks_cluster.eks_kubeconfig.cluster_ca_certificate)
+
+ exec {
+ api_version = "client.authentication.k8s.io/v1beta1"
+ args = ["eks", "get-token", "--cluster-name", module.eks_cluster.eks_cluster_name]
+ command = "aws"
+ }
+}
+
+provider "aws" {
+ region = var.aws_region
+}
diff --git a/modules/anyscale-k8s-persistent-volume/variables.tf b/modules/anyscale-k8s-persistent-volume/variables.tf
new file mode 100644
index 0000000..7644b3f
--- /dev/null
+++ b/modules/anyscale-k8s-persistent-volume/variables.tf
@@ -0,0 +1,151 @@
+# ------------------------------------------------------------------------------
+# REQUIRED PARAMETERS
+# These variables must be set when using this module.
+# ------------------------------------------------------------------------------
+variable "cloud_provider" {
+ description = <<-EOT
+ (Required) The cloud provider (aws or gcp)
+
+ ex:
+ ```
+ cloud_provider = "aws"
+ ```
+ EOT
+ type = string
+ validation {
+ condition = (
+ var.cloud_provider == "aws" || var.cloud_provider == "gcp"
+ )
+ error_message = "The cloud_provider only allows `aws` or `gcp`"
+ }
+}
+
+variable "kubernetes_cluster_name" {
+ type = string
+ description = <<-EOT
+ (Optional) The name of the Kubernetes cluster.
+
+ ex:
+ ```
+ kubernetes_cluster_name = "my-cluster"
+ ```
+ EOT
+ default = null
+}
+
+# ------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These variables have defaults, but may be overridden.
+# ------------------------------------------------------------------------------
+variable "module_enabled" {
+ description = <<-EOT
+ (Optional) Determines if this module should create resources.
+
+ If set to true, `eks_role_arn`, `anyscale_subnet_ids`, and `anyscale_security_group_id` must be provided.
+ ex:
+ ```
+ module_enabled = true
+ ```
+ EOT
+ type = bool
+ default = false
+}
+
+variable "kubernetes_persistent_volume_name" {
+ description = <<-EOT
+ (Optional) The name of the Kubernetes persistent volume.
+
+ ex:
+ ```
+ kubernetes_persistent_volume_name = "anyscale-nfs"
+ ```
+ EOT
+ type = string
+ default = "anyscale-nfs"
+}
+
+variable "kubernetes_persistent_volume_size" {
+ description = <<-EOT
+ (Optional) The size of the Kubernetes persistent volume.
+
+ When using AWS EFS, this is just a placeholder. The actual size is elastically built, making this just a placeholder
+
+ ex:
+ ```
+ kubernetes_persistent_volume_size = "20Gi"
+ ```
+ EOT
+ type = string
+ default = "20Gi"
+}
+
+variable "kubernetes_persistent_volume_claim_name" {
+ description = <<-EOT
+ (Optional) The name of the Kubernetes persistent volume claim.
+
+ ex:
+ ```
+ kubernetes_persistent_volume_claim_name = "anyscale-nfs-claim"
+ ```
+ EOT
+ type = string
+ default = "anyscale-nfs-claim"
+}
+
+variable "anyscale_kubernetes_namespace" {
+ description = <<-EOT
+ (Optional) The name of the Kubernetes namespace.
+
+ ex:
+ ```
+ anyscale_kubernetes_namespace = "anyscale-k8s"
+ ```
+ EOT
+ type = string
+ default = "anyscale-k8s"
+}
+
+variable "aws_efs_file_system_id" {
+ description = <<-EOT
+ (Optional) The ID of the EFS file system.
+
+ Required if `cloud_provider` is `aws`.
+
+ ex:
+ ```
+ aws_efs_file_system_id = "fs-12345678"
+ ```
+ EOT
+ type = string
+ default = null
+}
+
+variable "gcp_filestore_ip" {
+ description = <<-EOT
+ (Optional) The Filestore IP address.
+
+ Required if `cloud_provider` is `gcp`.
+
+ ex:
+ ```
+ gcp_filestore_ip = "172.16.0.12"
+ ```
+ EOT
+ type = string
+ default = null
+}
+
+variable "gcp_filestore_share_name" {
+ description = <<-EOT
+ (Optional) The Filestore share name.
+
+ Required if `cloud_provider` is `gcp`.
+
+ ex:
+ ```
+ gcp_filestore_share_name = "my-share"
+ ```
+ EOT
+ type = string
+ default = null
+}
diff --git a/modules/anyscale-k8s-persistent-volume/versions.tf b/modules/anyscale-k8s-persistent-volume/versions.tf
new file mode 100644
index 0000000..94019cd
--- /dev/null
+++ b/modules/anyscale-k8s-persistent-volume/versions.tf
@@ -0,0 +1,10 @@
+terraform {
+ required_version = ">= 1.0"
+
+ required_providers {
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = "~> 2.0"
+ }
+ }
+}
diff --git a/outputs.tf b/outputs.tf
new file mode 100644
index 0000000..e69de29
diff --git a/variables.tf b/variables.tf
new file mode 100644
index 0000000..d703e72
--- /dev/null
+++ b/variables.tf
@@ -0,0 +1,71 @@
+# ------------------------------------------------------------------------------
+# REQUIRED PARAMETERS
+# These variables must be set when using this module.
+# ------------------------------------------------------------------------------
+variable "cloud_provider" {
+ description = <<-EOT
+ (Required) The cloud provider (aws or gcp)
+
+ ex:
+ ```
+ cloud_provider = "aws"
+ ```
+ EOT
+ type = string
+ validation {
+ condition = (
+ var.cloud_provider == "aws" || var.cloud_provider == "gcp"
+ )
+ error_message = "The cloud_provider only allows `aws` or `gcp`"
+ }
+}
+
+variable "kubernetes_cluster_name" {
+ type = string
+ description = <<-EOT
+ (Optional) The name of the Kubernetes cluster.
+
+ ex:
+ ```
+ kubernetes_cluster_name = "my-cluster"
+ ```
+ EOT
+ default = null
+}
+
+# ------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These variables have defaults, but may be overridden.
+# ------------------------------------------------------------------------------
+
+# ------------------
+# AWS Related
+# ------------------
+variable "aws_dataplane_role_arn" {
+ description = <<-EOT
+ (Optional) The ARN of the AWS IAM role that will be used by the EKS cluster to access AWS services.
+
+ Required if `cloud_provider` is set to `aws`.
+
+ ex:
+ ```
+ aws_dataplane_role_arn = "arn:aws:iam::123456789012:role/my-eks-dataplane-role"
+ ```
+ EOT
+ type = string
+ default = null
+}
+variable "aws_controlplane_role_arn" {
+ description = <<-EOT
+ (Optional) The ARN of the AWS IAM role that will be used by the EKS cluster to access AWS services.
+
+ Required if `cloud_provider` is set to `aws`.
+
+ ex:
+ ```
+ aws_controlplane_role_arn = "arn:aws:iam::123456789012:role/my-eks-controlplane-role"
+ ```
+ EOT
+ type = string
+ default = null
+}
diff --git a/versions.tf b/versions.tf
new file mode 100644
index 0000000..8a43595
--- /dev/null
+++ b/versions.tf
@@ -0,0 +1,20 @@
+terraform {
+ required_version = ">= 1.0"
+
+ required_providers {
+ helm = {
+ source = "hashicorp/helm"
+ version = "~> 2.0"
+ }
+
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = "~> 2.0"
+ }
+
+ time = {
+ source = "hashicorp/time"
+ version = ">= 0.12"
+ }
+ }
+}