From b65797fd258c23c0dc3a8579ae27b9cecd31adb2 Mon Sep 17 00:00:00 2001 From: jbeemster Date: Mon, 5 Feb 2024 17:15:55 +0100 Subject: [PATCH] Update AWS quickstart to SLULA (closes #91) --- terraform/aws/iglu_server/default/README.md | 3 ++- terraform/aws/iglu_server/default/main.tf | 4 ++- .../aws/iglu_server/default/terraform.tfvars | 3 +++ .../aws/iglu_server/default/variables.tf | 11 ++++++++ terraform/aws/iglu_server/secure/README.md | 3 ++- terraform/aws/iglu_server/secure/main.tf | 4 ++- .../aws/iglu_server/secure/terraform.tfvars | 3 +++ terraform/aws/iglu_server/secure/variables.tf | 11 ++++++++ terraform/aws/pipeline/default/README.md | 27 ++++++++++--------- terraform/aws/pipeline/default/main.tf | 8 ++++-- .../aws/pipeline/default/target_amazon_s3.tf | 12 ++++++--- .../aws/pipeline/default/target_databricks.tf | 8 ++++-- .../aws/pipeline/default/target_postgres.tf | 8 ++++-- .../aws/pipeline/default/target_redshift.tf | 8 ++++-- .../aws/pipeline/default/target_snowflake.tf | 8 ++++-- .../aws/pipeline/default/terraform.tfvars | 3 +++ terraform/aws/pipeline/default/variables.tf | 11 ++++++++ terraform/aws/pipeline/secure/README.md | 27 ++++++++++--------- terraform/aws/pipeline/secure/main.tf | 8 ++++-- .../aws/pipeline/secure/target_amazon_s3.tf | 12 ++++++--- .../aws/pipeline/secure/target_databricks.tf | 8 ++++-- .../aws/pipeline/secure/target_postgres.tf | 8 ++++-- .../aws/pipeline/secure/target_redshift.tf | 8 ++++-- .../aws/pipeline/secure/target_snowflake.tf | 8 ++++-- .../aws/pipeline/secure/terraform.tfvars | 3 +++ terraform/aws/pipeline/secure/variables.tf | 11 ++++++++ 26 files changed, 172 insertions(+), 56 deletions(-) diff --git a/terraform/aws/iglu_server/default/README.md b/terraform/aws/iglu_server/default/README.md index bc1c144..e87a797 100644 --- a/terraform/aws/iglu_server/default/README.md +++ b/terraform/aws/iglu_server/default/README.md @@ -18,7 +18,7 @@ |------|--------|---------| | [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/alb/aws | 0.2.0 | | [iglu\_rds](#module\_iglu\_rds) | snowplow-devops/rds/aws | 0.4.0 | -| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ec2/aws | 0.4.4 | +| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ec2/aws | 0.5.0 | ## Resources @@ -39,6 +39,7 @@ | [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | n/a | yes | | [ssh\_public\_key](#input\_ssh\_public\_key) | The SSH public key to use for the deployment | `string` | n/a | yes | | [vpc\_id](#input\_vpc\_id) | The VPC to deploy the components within | `string` | n/a | yes | +| [accept\_limited\_use\_license](#input\_accept\_limited\_use\_license) | Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/) | `bool` | `false` | no | | [cloudwatch\_logs\_enabled](#input\_cloudwatch\_logs\_enabled) | Whether application logs should be reported to CloudWatch | `bool` | `true` | no | | [cloudwatch\_logs\_retention\_days](#input\_cloudwatch\_logs\_retention\_days) | The length of time in days to retain logs for | `number` | `7` | no | | [iam\_permissions\_boundary](#input\_iam\_permissions\_boundary) | The permissions boundary ARN to set on IAM roles created | `string` | `""` | no | diff --git a/terraform/aws/iglu_server/default/main.tf b/terraform/aws/iglu_server/default/main.tf index 2e1c3e3..e1da115 100644 --- a/terraform/aws/iglu_server/default/main.tf +++ b/terraform/aws/iglu_server/default/main.tf @@ -36,7 +36,9 @@ module "iglu_lb" { module "iglu_server" { source = "snowplow-devops/iglu-server-ec2/aws" - version = "0.4.4" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-iglu-server" vpc_id = var.vpc_id diff --git a/terraform/aws/iglu_server/default/terraform.tfvars b/terraform/aws/iglu_server/default/terraform.tfvars index 1ced542..36dc856 100644 --- a/terraform/aws/iglu_server/default/terraform.tfvars +++ b/terraform/aws/iglu_server/default/terraform.tfvars @@ -1,3 +1,6 @@ +# Please accept the terms of the Snowplow Limited Use License Agreement to proceed. (https://docs.snowplow.io/limited-use-license-1.0/) +accept_limited_use_license = false + # Will be prefixed to all resource names # Use this to easily identify the resources created and provide entropy for subsequent environments prefix = "sp" diff --git a/terraform/aws/iglu_server/default/variables.tf b/terraform/aws/iglu_server/default/variables.tf index 81db3d5..7c1fd81 100644 --- a/terraform/aws/iglu_server/default/variables.tf +++ b/terraform/aws/iglu_server/default/variables.tf @@ -1,3 +1,14 @@ +variable "accept_limited_use_license" { + description = "Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/)" + type = bool + default = false + + validation { + condition = var.accept_limited_use_license + error_message = "Please accept the terms of the Snowplow Limited Use License Agreement to proceed." + } +} + variable "prefix" { description = "Will be prefixed to all resource names. Use to easily identify the resources created" type = string diff --git a/terraform/aws/iglu_server/secure/README.md b/terraform/aws/iglu_server/secure/README.md index 10cf2e0..c8bef9a 100644 --- a/terraform/aws/iglu_server/secure/README.md +++ b/terraform/aws/iglu_server/secure/README.md @@ -18,7 +18,7 @@ |------|--------|---------| | [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/alb/aws | 0.2.0 | | [iglu\_rds](#module\_iglu\_rds) | snowplow-devops/rds/aws | 0.4.0 | -| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ec2/aws | 0.4.4 | +| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ec2/aws | 0.5.0 | ## Resources @@ -40,6 +40,7 @@ | [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | n/a | yes | | [ssh\_public\_key](#input\_ssh\_public\_key) | The SSH public key to use for the deployment | `string` | n/a | yes | | [vpc\_id](#input\_vpc\_id) | The VPC to deploy the components within | `string` | n/a | yes | +| [accept\_limited\_use\_license](#input\_accept\_limited\_use\_license) | Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/) | `bool` | `false` | no | | [cloudwatch\_logs\_enabled](#input\_cloudwatch\_logs\_enabled) | Whether application logs should be reported to CloudWatch | `bool` | `true` | no | | [cloudwatch\_logs\_retention\_days](#input\_cloudwatch\_logs\_retention\_days) | The length of time in days to retain logs for | `number` | `7` | no | | [iam\_permissions\_boundary](#input\_iam\_permissions\_boundary) | The permissions boundary ARN to set on IAM roles created | `string` | `""` | no | diff --git a/terraform/aws/iglu_server/secure/main.tf b/terraform/aws/iglu_server/secure/main.tf index fa2637f..1d8e0eb 100644 --- a/terraform/aws/iglu_server/secure/main.tf +++ b/terraform/aws/iglu_server/secure/main.tf @@ -36,7 +36,9 @@ module "iglu_lb" { module "iglu_server" { source = "snowplow-devops/iglu-server-ec2/aws" - version = "0.4.4" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-iglu-server" vpc_id = var.vpc_id diff --git a/terraform/aws/iglu_server/secure/terraform.tfvars b/terraform/aws/iglu_server/secure/terraform.tfvars index 79cb932..b0273e2 100644 --- a/terraform/aws/iglu_server/secure/terraform.tfvars +++ b/terraform/aws/iglu_server/secure/terraform.tfvars @@ -1,3 +1,6 @@ +# Please accept the terms of the Snowplow Limited Use License Agreement to proceed. (https://docs.snowplow.io/limited-use-license-1.0/) +accept_limited_use_license = false + # Will be prefixed to all resource names # Use this to easily identify the resources created and provide entropy for subsequent environments prefix = "sp" diff --git a/terraform/aws/iglu_server/secure/variables.tf b/terraform/aws/iglu_server/secure/variables.tf index a1c619d..fa39b76 100644 --- a/terraform/aws/iglu_server/secure/variables.tf +++ b/terraform/aws/iglu_server/secure/variables.tf @@ -1,3 +1,14 @@ +variable "accept_limited_use_license" { + description = "Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/)" + type = bool + default = false + + validation { + condition = var.accept_limited_use_license + error_message = "Please accept the terms of the Snowplow Limited Use License Agreement to proceed." + } +} + variable "prefix" { description = "Will be prefixed to all resource names. Use to easily identify the resources created" type = string diff --git a/terraform/aws/pipeline/default/README.md b/terraform/aws/pipeline/default/README.md index 1504696..694f17e 100644 --- a/terraform/aws/pipeline/default/README.md +++ b/terraform/aws/pipeline/default/README.md @@ -18,24 +18,24 @@ |------|--------|---------| | [bad\_1\_stream](#module\_bad\_1\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | | [bad\_2\_stream](#module\_bad\_2\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [collector\_kinesis](#module\_collector\_kinesis) | snowplow-devops/collector-kinesis-ec2/aws | 0.8.1 | +| [collector\_kinesis](#module\_collector\_kinesis) | snowplow-devops/collector-kinesis-ec2/aws | 0.9.0 | | [collector\_lb](#module\_collector\_lb) | snowplow-devops/alb/aws | 0.2.0 | -| [db\_loader](#module\_db\_loader) | snowplow-devops/databricks-loader-ec2/aws | 0.1.3 | -| [db\_transformer\_wrp](#module\_db\_transformer\_wrp) | snowplow-devops/transformer-kinesis-ec2/aws | 0.3.9 | -| [enrich\_kinesis](#module\_enrich\_kinesis) | snowplow-devops/enrich-kinesis-ec2/aws | 0.5.3 | +| [db\_loader](#module\_db\_loader) | snowplow-devops/databricks-loader-ec2/aws | 0.2.0 | +| [db\_transformer\_wrp](#module\_db\_transformer\_wrp) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | +| [enrich\_kinesis](#module\_enrich\_kinesis) | snowplow-devops/enrich-kinesis-ec2/aws | 0.6.0 | | [enriched\_stream](#module\_enriched\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.4.3 | -| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.4.3 | +| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.0 | +| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.0 | | [postgres\_loader\_rds](#module\_postgres\_loader\_rds) | snowplow-devops/rds/aws | 0.4.0 | | [raw\_stream](#module\_raw\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [rs\_loader](#module\_rs\_loader) | snowplow-devops/redshift-loader-ec2/aws | 0.1.4 | -| [rs\_transformer\_stsv](#module\_rs\_transformer\_stsv) | snowplow-devops/transformer-kinesis-ec2/aws | 0.3.9 | -| [s3\_loader\_bad](#module\_s3\_loader\_bad) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.4.3 | -| [s3\_loader\_enriched](#module\_s3\_loader\_enriched) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.4.3 | -| [s3\_loader\_raw](#module\_s3\_loader\_raw) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.4.3 | +| [rs\_loader](#module\_rs\_loader) | snowplow-devops/redshift-loader-ec2/aws | 0.2.0 | +| [rs\_transformer\_stsv](#module\_rs\_transformer\_stsv) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | +| [s3\_loader\_bad](#module\_s3\_loader\_bad) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | +| [s3\_loader\_enriched](#module\_s3\_loader\_enriched) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | +| [s3\_loader\_raw](#module\_s3\_loader\_raw) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | | [s3\_pipeline\_bucket](#module\_s3\_pipeline\_bucket) | snowplow-devops/s3-bucket/aws | 0.2.0 | -| [sf\_loader](#module\_sf\_loader) | snowplow-devops/snowflake-loader-ec2/aws | 0.2.4 | -| [sf\_transformer\_wrj](#module\_sf\_transformer\_wrj) | snowplow-devops/transformer-kinesis-ec2/aws | 0.3.9 | +| [sf\_loader](#module\_sf\_loader) | snowplow-devops/snowflake-loader-ec2/aws | 0.3.0 | +| [sf\_transformer\_wrj](#module\_sf\_transformer\_wrj) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | ## Resources @@ -59,6 +59,7 @@ | [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | n/a | yes | | [ssh\_public\_key](#input\_ssh\_public\_key) | The SSH public key to use for the deployment | `string` | n/a | yes | | [vpc\_id](#input\_vpc\_id) | The VPC to deploy the components within | `string` | n/a | yes | +| [accept\_limited\_use\_license](#input\_accept\_limited\_use\_license) | Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/) | `bool` | `false` | no | | [cloudwatch\_logs\_enabled](#input\_cloudwatch\_logs\_enabled) | Whether application logs should be reported to CloudWatch | `bool` | `true` | no | | [cloudwatch\_logs\_retention\_days](#input\_cloudwatch\_logs\_retention\_days) | The length of time in days to retain logs for | `number` | `7` | no | | [databricks\_auth\_token](#input\_databricks\_auth\_token) | Databricks deltalake auth token | `string` | `""` | no | diff --git a/terraform/aws/pipeline/default/main.tf b/terraform/aws/pipeline/default/main.tf index 8ed4472..02348fc 100644 --- a/terraform/aws/pipeline/default/main.tf +++ b/terraform/aws/pipeline/default/main.tf @@ -86,7 +86,9 @@ module "collector_lb" { module "collector_kinesis" { source = "snowplow-devops/collector-kinesis-ec2/aws" - version = "0.8.1" + version = "0.9.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-collector-server" vpc_id = var.vpc_id @@ -114,7 +116,9 @@ module "collector_kinesis" { # 3. Deploy Enrichment module "enrich_kinesis" { source = "snowplow-devops/enrich-kinesis-ec2/aws" - version = "0.5.3" + version = "0.6.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-enrich-server" vpc_id = var.vpc_id diff --git a/terraform/aws/pipeline/default/target_amazon_s3.tf b/terraform/aws/pipeline/default/target_amazon_s3.tf index e413730..e6dae32 100644 --- a/terraform/aws/pipeline/default/target_amazon_s3.tf +++ b/terraform/aws/pipeline/default/target_amazon_s3.tf @@ -1,6 +1,8 @@ module "s3_loader_raw" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.4.3" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.s3_raw_enabled ? 1 : 0 @@ -30,7 +32,9 @@ module "s3_loader_raw" { module "s3_loader_bad" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.4.3" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.s3_bad_enabled ? 1 : 0 @@ -63,7 +67,9 @@ module "s3_loader_bad" { module "s3_loader_enriched" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.4.3" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.s3_enriched_enabled ? 1 : 0 diff --git a/terraform/aws/pipeline/default/target_databricks.tf b/terraform/aws/pipeline/default/target_databricks.tf index 7264e68..a92b8c8 100644 --- a/terraform/aws/pipeline/default/target_databricks.tf +++ b/terraform/aws/pipeline/default/target_databricks.tf @@ -9,7 +9,9 @@ resource "aws_sqs_queue" "db_message_queue" { module "db_transformer_wrp" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.3.9" + version = "0.4.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.databricks_enabled ? 1 : 0 @@ -46,7 +48,9 @@ module "db_transformer_wrp" { module "db_loader" { source = "snowplow-devops/databricks-loader-ec2/aws" - version = "0.1.3" + version = "0.2.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.databricks_enabled ? 1 : 0 diff --git a/terraform/aws/pipeline/default/target_postgres.tf b/terraform/aws/pipeline/default/target_postgres.tf index c8d62cb..8f57f3c 100644 --- a/terraform/aws/pipeline/default/target_postgres.tf +++ b/terraform/aws/pipeline/default/target_postgres.tf @@ -19,7 +19,9 @@ module "postgres_loader_rds" { module "postgres_loader_enriched" { source = "snowplow-devops/postgres-loader-kinesis-ec2/aws" - version = "0.4.3" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.postgres_db_enabled ? 1 : 0 @@ -59,7 +61,9 @@ module "postgres_loader_enriched" { module "postgres_loader_bad" { source = "snowplow-devops/postgres-loader-kinesis-ec2/aws" - version = "0.4.3" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.postgres_db_enabled ? 1 : 0 diff --git a/terraform/aws/pipeline/default/target_redshift.tf b/terraform/aws/pipeline/default/target_redshift.tf index 53d9790..f9944bc 100644 --- a/terraform/aws/pipeline/default/target_redshift.tf +++ b/terraform/aws/pipeline/default/target_redshift.tf @@ -9,7 +9,9 @@ resource "aws_sqs_queue" "rs_message_queue" { module "rs_transformer_stsv" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.3.9" + version = "0.4.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.redshift_enabled ? 1 : 0 @@ -46,7 +48,9 @@ module "rs_transformer_stsv" { module "rs_loader" { source = "snowplow-devops/redshift-loader-ec2/aws" - version = "0.1.4" + version = "0.2.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.redshift_enabled ? 1 : 0 diff --git a/terraform/aws/pipeline/default/target_snowflake.tf b/terraform/aws/pipeline/default/target_snowflake.tf index 2340294..e7acce4 100644 --- a/terraform/aws/pipeline/default/target_snowflake.tf +++ b/terraform/aws/pipeline/default/target_snowflake.tf @@ -9,7 +9,9 @@ resource "aws_sqs_queue" "sf_message_queue" { module "sf_transformer_wrj" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.3.9" + version = "0.4.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.snowflake_enabled ? 1 : 0 @@ -46,7 +48,9 @@ module "sf_transformer_wrj" { module "sf_loader" { source = "snowplow-devops/snowflake-loader-ec2/aws" - version = "0.2.4" + version = "0.3.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.snowflake_enabled ? 1 : 0 diff --git a/terraform/aws/pipeline/default/terraform.tfvars b/terraform/aws/pipeline/default/terraform.tfvars index 2fc22e4..1c48703 100644 --- a/terraform/aws/pipeline/default/terraform.tfvars +++ b/terraform/aws/pipeline/default/terraform.tfvars @@ -1,3 +1,6 @@ +# Please accept the terms of the Snowplow Limited Use License Agreement to proceed. (https://docs.snowplow.io/limited-use-license-1.0/) +accept_limited_use_license = false + # Will be prefixed to all resource names # Use this to easily identify the resources created and provide entropy for subsequent environments prefix = "sp" diff --git a/terraform/aws/pipeline/default/variables.tf b/terraform/aws/pipeline/default/variables.tf index 22b8402..397caab 100644 --- a/terraform/aws/pipeline/default/variables.tf +++ b/terraform/aws/pipeline/default/variables.tf @@ -1,3 +1,14 @@ +variable "accept_limited_use_license" { + description = "Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/)" + type = bool + default = false + + validation { + condition = var.accept_limited_use_license + error_message = "Please accept the terms of the Snowplow Limited Use License Agreement to proceed." + } +} + variable "prefix" { description = "Will be prefixed to all resource names. Use to easily identify the resources created" type = string diff --git a/terraform/aws/pipeline/secure/README.md b/terraform/aws/pipeline/secure/README.md index 9bf954b..5773ec3 100644 --- a/terraform/aws/pipeline/secure/README.md +++ b/terraform/aws/pipeline/secure/README.md @@ -18,24 +18,24 @@ |------|--------|---------| | [bad\_1\_stream](#module\_bad\_1\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | | [bad\_2\_stream](#module\_bad\_2\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [collector\_kinesis](#module\_collector\_kinesis) | snowplow-devops/collector-kinesis-ec2/aws | 0.8.1 | +| [collector\_kinesis](#module\_collector\_kinesis) | snowplow-devops/collector-kinesis-ec2/aws | 0.9.0 | | [collector\_lb](#module\_collector\_lb) | snowplow-devops/alb/aws | 0.2.0 | -| [db\_loader](#module\_db\_loader) | snowplow-devops/databricks-loader-ec2/aws | 0.1.3 | -| [db\_transformer\_wrp](#module\_db\_transformer\_wrp) | snowplow-devops/transformer-kinesis-ec2/aws | 0.3.9 | -| [enrich\_kinesis](#module\_enrich\_kinesis) | snowplow-devops/enrich-kinesis-ec2/aws | 0.5.3 | +| [db\_loader](#module\_db\_loader) | snowplow-devops/databricks-loader-ec2/aws | 0.2.0 | +| [db\_transformer\_wrp](#module\_db\_transformer\_wrp) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | +| [enrich\_kinesis](#module\_enrich\_kinesis) | snowplow-devops/enrich-kinesis-ec2/aws | 0.6.0 | | [enriched\_stream](#module\_enriched\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.4.3 | -| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.4.3 | +| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.0 | +| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-kinesis-ec2/aws | 0.5.0 | | [postgres\_loader\_rds](#module\_postgres\_loader\_rds) | snowplow-devops/rds/aws | 0.4.0 | | [raw\_stream](#module\_raw\_stream) | snowplow-devops/kinesis-stream/aws | 0.3.0 | -| [rs\_loader](#module\_rs\_loader) | snowplow-devops/redshift-loader-ec2/aws | 0.1.4 | -| [rs\_transformer\_stsv](#module\_rs\_transformer\_stsv) | snowplow-devops/transformer-kinesis-ec2/aws | 0.3.9 | -| [s3\_loader\_bad](#module\_s3\_loader\_bad) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.4.3 | -| [s3\_loader\_enriched](#module\_s3\_loader\_enriched) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.4.3 | -| [s3\_loader\_raw](#module\_s3\_loader\_raw) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.4.3 | +| [rs\_loader](#module\_rs\_loader) | snowplow-devops/redshift-loader-ec2/aws | 0.2.0 | +| [rs\_transformer\_stsv](#module\_rs\_transformer\_stsv) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | +| [s3\_loader\_bad](#module\_s3\_loader\_bad) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | +| [s3\_loader\_enriched](#module\_s3\_loader\_enriched) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | +| [s3\_loader\_raw](#module\_s3\_loader\_raw) | snowplow-devops/s3-loader-kinesis-ec2/aws | 0.5.0 | | [s3\_pipeline\_bucket](#module\_s3\_pipeline\_bucket) | snowplow-devops/s3-bucket/aws | 0.2.0 | -| [sf\_loader](#module\_sf\_loader) | snowplow-devops/snowflake-loader-ec2/aws | 0.2.4 | -| [sf\_transformer\_wrj](#module\_sf\_transformer\_wrj) | snowplow-devops/transformer-kinesis-ec2/aws | 0.3.9 | +| [sf\_loader](#module\_sf\_loader) | snowplow-devops/snowflake-loader-ec2/aws | 0.3.0 | +| [sf\_transformer\_wrj](#module\_sf\_transformer\_wrj) | snowplow-devops/transformer-kinesis-ec2/aws | 0.4.0 | ## Resources @@ -60,6 +60,7 @@ | [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | n/a | yes | | [ssh\_public\_key](#input\_ssh\_public\_key) | The SSH public key to use for the deployment | `string` | n/a | yes | | [vpc\_id](#input\_vpc\_id) | The VPC to deploy the components within | `string` | n/a | yes | +| [accept\_limited\_use\_license](#input\_accept\_limited\_use\_license) | Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/) | `bool` | `false` | no | | [cloudwatch\_logs\_enabled](#input\_cloudwatch\_logs\_enabled) | Whether application logs should be reported to CloudWatch | `bool` | `true` | no | | [cloudwatch\_logs\_retention\_days](#input\_cloudwatch\_logs\_retention\_days) | The length of time in days to retain logs for | `number` | `7` | no | | [databricks\_auth\_token](#input\_databricks\_auth\_token) | Databricks deltalake auth token | `string` | `""` | no | diff --git a/terraform/aws/pipeline/secure/main.tf b/terraform/aws/pipeline/secure/main.tf index 9e3683d..f361e14 100644 --- a/terraform/aws/pipeline/secure/main.tf +++ b/terraform/aws/pipeline/secure/main.tf @@ -86,7 +86,9 @@ module "collector_lb" { module "collector_kinesis" { source = "snowplow-devops/collector-kinesis-ec2/aws" - version = "0.8.1" + version = "0.9.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-collector-server" vpc_id = var.vpc_id @@ -116,7 +118,9 @@ module "collector_kinesis" { # 3. Deploy Enrichment module "enrich_kinesis" { source = "snowplow-devops/enrich-kinesis-ec2/aws" - version = "0.5.3" + version = "0.6.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-enrich-server" vpc_id = var.vpc_id diff --git a/terraform/aws/pipeline/secure/target_amazon_s3.tf b/terraform/aws/pipeline/secure/target_amazon_s3.tf index fc49e34..89254e4 100644 --- a/terraform/aws/pipeline/secure/target_amazon_s3.tf +++ b/terraform/aws/pipeline/secure/target_amazon_s3.tf @@ -1,6 +1,8 @@ module "s3_loader_raw" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.4.3" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.s3_raw_enabled ? 1 : 0 @@ -32,7 +34,9 @@ module "s3_loader_raw" { module "s3_loader_bad" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.4.3" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.s3_bad_enabled ? 1 : 0 @@ -67,7 +71,9 @@ module "s3_loader_bad" { module "s3_loader_enriched" { source = "snowplow-devops/s3-loader-kinesis-ec2/aws" - version = "0.4.3" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.s3_enriched_enabled ? 1 : 0 diff --git a/terraform/aws/pipeline/secure/target_databricks.tf b/terraform/aws/pipeline/secure/target_databricks.tf index d5ce29d..33304c6 100644 --- a/terraform/aws/pipeline/secure/target_databricks.tf +++ b/terraform/aws/pipeline/secure/target_databricks.tf @@ -9,7 +9,9 @@ resource "aws_sqs_queue" "db_message_queue" { module "db_transformer_wrp" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.3.9" + version = "0.4.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.databricks_enabled ? 1 : 0 @@ -48,7 +50,9 @@ module "db_transformer_wrp" { module "db_loader" { source = "snowplow-devops/databricks-loader-ec2/aws" - version = "0.1.3" + version = "0.2.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.databricks_enabled ? 1 : 0 diff --git a/terraform/aws/pipeline/secure/target_postgres.tf b/terraform/aws/pipeline/secure/target_postgres.tf index 722db8d..2c18e62 100644 --- a/terraform/aws/pipeline/secure/target_postgres.tf +++ b/terraform/aws/pipeline/secure/target_postgres.tf @@ -19,7 +19,9 @@ module "postgres_loader_rds" { module "postgres_loader_enriched" { source = "snowplow-devops/postgres-loader-kinesis-ec2/aws" - version = "0.4.3" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.postgres_db_enabled ? 1 : 0 @@ -61,7 +63,9 @@ module "postgres_loader_enriched" { module "postgres_loader_bad" { source = "snowplow-devops/postgres-loader-kinesis-ec2/aws" - version = "0.4.3" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.postgres_db_enabled ? 1 : 0 diff --git a/terraform/aws/pipeline/secure/target_redshift.tf b/terraform/aws/pipeline/secure/target_redshift.tf index 2b7ecb5..a4e8244 100644 --- a/terraform/aws/pipeline/secure/target_redshift.tf +++ b/terraform/aws/pipeline/secure/target_redshift.tf @@ -9,7 +9,9 @@ resource "aws_sqs_queue" "rs_message_queue" { module "rs_transformer_stsv" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.3.9" + version = "0.4.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.redshift_enabled ? 1 : 0 @@ -48,7 +50,9 @@ module "rs_transformer_stsv" { module "rs_loader" { source = "snowplow-devops/redshift-loader-ec2/aws" - version = "0.1.4" + version = "0.2.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.redshift_enabled ? 1 : 0 diff --git a/terraform/aws/pipeline/secure/target_snowflake.tf b/terraform/aws/pipeline/secure/target_snowflake.tf index 30c4d41..b7aa2fa 100644 --- a/terraform/aws/pipeline/secure/target_snowflake.tf +++ b/terraform/aws/pipeline/secure/target_snowflake.tf @@ -9,7 +9,9 @@ resource "aws_sqs_queue" "sf_message_queue" { module "sf_transformer_wrj" { source = "snowplow-devops/transformer-kinesis-ec2/aws" - version = "0.3.9" + version = "0.4.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.snowflake_enabled ? 1 : 0 @@ -48,7 +50,9 @@ module "sf_transformer_wrj" { module "sf_loader" { source = "snowplow-devops/snowflake-loader-ec2/aws" - version = "0.2.4" + version = "0.3.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.snowflake_enabled ? 1 : 0 diff --git a/terraform/aws/pipeline/secure/terraform.tfvars b/terraform/aws/pipeline/secure/terraform.tfvars index 88f1278..96e1a31 100644 --- a/terraform/aws/pipeline/secure/terraform.tfvars +++ b/terraform/aws/pipeline/secure/terraform.tfvars @@ -1,3 +1,6 @@ +# Please accept the terms of the Snowplow Limited Use License Agreement to proceed. (https://docs.snowplow.io/limited-use-license-1.0/) +accept_limited_use_license = false + # Will be prefixed to all resource names # Use this to easily identify the resources created and provide entropy for subsequent environments prefix = "sp" diff --git a/terraform/aws/pipeline/secure/variables.tf b/terraform/aws/pipeline/secure/variables.tf index 5d992ea..9387b97 100644 --- a/terraform/aws/pipeline/secure/variables.tf +++ b/terraform/aws/pipeline/secure/variables.tf @@ -1,3 +1,14 @@ +variable "accept_limited_use_license" { + description = "Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/)" + type = bool + default = false + + validation { + condition = var.accept_limited_use_license + error_message = "Please accept the terms of the Snowplow Limited Use License Agreement to proceed." + } +} + variable "prefix" { description = "Will be prefixed to all resource names. Use to easily identify the resources created" type = string