Terraform test: Add AWS persistent

ci/nightly/pipeline.template.yml

@@ -1136,6 +1136,7 @@ steps:
         - ./ci/plugins/scratch-aws-access: ~
         - ./ci/plugins/mzcompose:
             composition: terraform
+            run: aws-temporary
       branches: "main v*.*"
 
   - group: "Output consistency"

ci/qa-canary/pipeline.template.yml

@@ -54,3 +54,16 @@ steps:
             - --benchmarking-env
             - --scenario=StagingBench
             - --sqlite-store
+
+  - id: terraform-aws-persistent-test
+    label: "Terraform AWS Persistent Test"
+    timeout_in_minutes: 1440 # 24 hours
+    concurrency: 1
+    concurrency_group: 'terraform-aws-persistent-test'
+    plugins:
+      - ./ci/plugins/mzcompose:
+          composition: terraform
+          run: aws-persistent-test
+          args: ["--runtime=82800"] # 23 hours
+    agents:
+      queue: linux-aarch64-small

ci/test/lint-main/checks/check-mzcompose-files.sh

@@ -45,6 +45,7 @@ check_default_workflow_references_others() {
         -not -wholename "./test/canary-environment/mzcompose.py" `# Only run manually` \
         -not -wholename "./test/ssh-connection/mzcompose.py" `# Handled differently` \
         -not -wholename "./test/scalability/mzcompose.py" `# Other workflows are for manual usage` \
+        -not -wholename "./test/terraform/mzcompose.py" `# Other workflows are for manual usage` \
     )
 
     for file in "${MZCOMPOSE_TEST_FILES[@]}"; do

test/terraform/aws-persistent/main.tf

@@ -0,0 +1,102 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+provider "aws" {
+  region = "us-east-1"
+}
+
+module "materialize_infrastructure" {
+  source = "git::https://github.com/MaterializeInc/terraform-aws-materialize.git?ref=v0.1.3"
+
+  # Basic settings
+  environment  = "dev"
+  vpc_name     = "aws-persistent-vpc"
+  cluster_name = "aws-persistent-cluster"
+  mz_iam_service_account_name = "aws-persistent-user"
+  mz_iam_role_name = "aws-persistent-s3-role"
+
+  # VPC Configuration
+  vpc_cidr             = "10.0.0.0/16"
+  availability_zones   = ["us-east-1a", "us-east-1b"]
+  private_subnet_cidrs = ["10.0.1.0/24", "10.0.2.0/24"]
+  public_subnet_cidrs  = ["10.0.101.0/24", "10.0.102.0/24"]
+  single_nat_gateway   = true
+
+  # EKS Configuration
+  cluster_version           = "1.31"
+  node_group_instance_types = ["c7a.2xlarge"]
+  node_group_desired_size   = 2
+  node_group_min_size       = 1
+  node_group_max_size       = 3
+  node_group_capacity_type  = "ON_DEMAND"
+
+  # Storage Configuration
+  bucket_name              = "aws-persistent-storage-${random_id.suffix.hex}"
+  enable_bucket_versioning = true
+  enable_bucket_encryption = true
+  bucket_force_destroy     = true
+
+  # Database Configuration
+  database_password    = "zdUXjK4dRBBqBiTMK9gbkL9zPMYMSTsj"
+  db_identifier        = "aws-persistent-metadata-db"
+  postgres_version     = "15"
+  db_instance_class    = "db.t3.micro"
+  db_allocated_storage = 20
+  database_name        = "materialize"
+  database_username    = "materialize"
+  db_multi_az          = false
+
+  # Basic monitoring
+  enable_monitoring      = true
+  metrics_retention_days = 30
+
+  # Tags
+  tags = {
+    Environment = "dev"
+    Project     = "aws-persistent"
+    Terraform   = "true"
+  }
+}
+
+# Generate random suffix for unique S3 bucket name
+resource "random_id" "suffix" {
+  byte_length = 4
+}
+
+# outputs.tf
+output "eks_cluster_endpoint" {
+  description = "EKS cluster endpoint"
+  value       = module.materialize_infrastructure.eks_cluster_endpoint
+}
+
+output "database_endpoint" {
+  description = "RDS instance endpoint"
+  value       = module.materialize_infrastructure.database_endpoint
+}
+
+output "s3_bucket_name" {
+  description = "Name of the S3 bucket"
+  value       = module.materialize_infrastructure.s3_bucket_name
+}
+
+output "materialize_s3_role_arn" {
+  description = "The ARN of the IAM role for Materialize"
+  value       = module.materialize_infrastructure.materialize_s3_role_arn
+}
+
+output "metadata_backend_url" {
+  description = "PostgreSQL connection URL in the format required by Materialize"
+  value       = module.materialize_infrastructure.metadata_backend_url
+  sensitive   = true
+}
+
+output "persist_backend_url" {
+  description = "S3 connection URL in the format required by Materialize using IRSA"
+  value       = module.materialize_infrastructure.persist_backend_url
+}

test/terraform/aws-temporary/simple.tf

@@ -0,0 +1,14 @@
+terraform {
+  required_version = ">= 1.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.75.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = ">= 3.0"
+    }
+  }
+}