AWS Glue needs to be able to write to temp space on s3 bucket, remove…

… variables and other glue configuration that makes the networking config unnecessarily specific.
controlshift · Jul 14, 2020 · 42e1f3a · 42e1f3a
1 parent c3ef98b
commit 42e1f3a
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 24 deletions.
diff --git a/glue_job.tf b/glue_job.tf
@@ -78,7 +78,7 @@ resource "aws_iam_role_policy" "controlshift_data_export_bucket_access" {
 resource "aws_iam_role_policy" "controlshift_glue_scripts_bucket_access" {
   name = "AllowsAccessToGlueScriptsBucket"
   role = aws_iam_role.glue_service_role.id
-  policy = data.aws_iam_policy_document.controlshift_data_export_bucket.json
+  policy = data.aws_iam_policy_document.controlshift_glue_scripts_bucket.json
 }
 
 data "aws_iam_policy_document" "controlshift_data_export_bucket" {
@@ -101,18 +101,17 @@ data "aws_iam_policy_document" "controlshift_glue_scripts_bucket" {
     actions = [
       "s3:GetObject",
       "s3:GetBucketLocation",
-      "s3:ListBucket"
+      "s3:ListBucket",
+      "s3:PutObject",
+      "s3:DeleteObject"
     ]
     resources = [
-      "arn:aws:s3:::${var.glue_scripts_bucket_name}/*"
+      "arn:aws:s3:::${var.glue_scripts_bucket_name}/*",
+      "arn:aws:s3:::${var.glue_scripts_bucket_name}"
     ]
   }
 }
 
-data "aws_subnet" "redshift_subnet" {
-  id = var.redshift_subnet_id
-}
-
 resource "aws_glue_connection" "redshift_connection" {
   name = "controlshift_${var.controlshift_environment}_data_sync"
 
@@ -123,11 +122,9 @@ resource "aws_glue_connection" "redshift_connection" {
     JDBC_ENFORCE_SSL    = false
   }
 
-  physical_connection_requirements {
-    availability_zone      = data.aws_subnet.redshift_subnet.availability_zone
-    security_group_id_list = [ var.redshift_security_group_id ]
-    subnet_id              = data.aws_subnet.redshift_subnet.id
-  }
+//  physical_connection_requirements {
+//    security_group_id_list = [ var.redshift_security_group_id ]
+//  }
 }
 
 resource "aws_glue_job" "signatures_full" {

diff --git a/loader.tf b/loader.tf
@@ -1,6 +1,6 @@
 resource "aws_lambda_function" "loader" {
   s3_bucket = local.lambda_buckets[var.aws_region]
-  s3_key = "LambdaRedshiftLoader/AWSLambdaRedshiftLoader-2.7.4.zip"
+  s3_key = "LambdaRedshiftLoader/AWSLambdaRedshiftLoader-2.7.7.zip"
   function_name = "controlshift-redshift-loader"
   role          = aws_iam_role.loader_lambda_role.arn
   handler       = "index.handler"

diff --git a/variables.tf b/variables.tf
@@ -26,17 +26,6 @@ variable "redshift_schema" {
   default = "public"
   description = "The Redshift schema to load tables into"
 }
-
-variable "redshift_security_group_id" {
-  type = string
-  description = "The security group assigned to the Redshift cluster that will be used for connecting by Glue. For requirements on this Security Group see https://docs.aws.amazon.com/glue/latest/dg/setup-vpc-for-glue-access.html"
-}
-
-variable "redshift_subnet_id" {
-  type = string
-  description = "The ID of one of Redshift's cluster subnet group that Glue will use to connect"
-}
-
 variable "manifest_bucket_name" {
   type        = string
   description = "Your S3 bucket name to store manifests of ingests processed in"