From 33801110585f8b13eead18df687b2b7121972ded Mon Sep 17 00:00:00 2001 From: Tirumerla <57160285+tirumerla@users.noreply.github.com> Date: Fri, 27 Aug 2021 09:57:55 -0700 Subject: [PATCH] Feature: Allow users to pass their own IAM roles (#44) * Feature: Allow users to pass their own IAM roles * Rename variables following best practices * update wording for variables & remove unused variable --- .github/CODEOWNERS | 4 +-- .github/auto-release.yml | 3 +- .github/mergify.yml | 7 +++++ .github/workflows/auto-format.yml | 4 ++- .github/workflows/auto-release.yml | 25 ++++++++++------ .github/workflows/validate-codeowners.yml | 2 ++ README.md | 6 ++++ docs/terraform.md | 6 ++++ main.tf | 24 +++++++-------- outputs.tf | 2 +- variables.tf | 36 +++++++++++++++++++++++ 11 files changed, 93 insertions(+), 26 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2537f2f..6f64b5a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -15,8 +15,8 @@ # Cloud Posse must review any changes to standard context definition, # but some changes can be rubber-stamped. -**/*.tf @cloudposse/engineering @cloudposse/approvers -README.yaml @cloudposse/engineering @cloudposse/approvers +**/*.tf @cloudposse/engineering @cloudposse/contributors @cloudposse/approvers +README.yaml @cloudposse/engineering @cloudposse/contributors @cloudposse/approvers README.md @cloudposse/engineering @cloudposse/contributors @cloudposse/approvers docs/*.md @cloudposse/engineering @cloudposse/contributors @cloudposse/approvers diff --git a/.github/auto-release.yml b/.github/auto-release.yml index c78a4d8..9976e10 100644 --- a/.github/auto-release.yml +++ b/.github/auto-release.yml @@ -17,6 +17,7 @@ version-resolver: - 'bugfix' - 'bug' - 'hotfix' + - 'no-release' default: 'minor' categories: @@ -46,7 +47,7 @@ template: | replacers: # Remove irrelevant information from Renovate bot -- search: '/---\s+^#.*Renovate configuration(?:.|\n)*?This PR has been generated .*/gm' +- search: '/(?<=---\s+)+^#.*(Renovate configuration|Configuration)(?:.|\n)*?This PR has been generated .*/gm' replace: '' # Remove Renovate bot banner image - search: '/\[!\[[^\]]*Renovate\][^\]]*\](\([^)]*\))?\s*\n+/gm' diff --git a/.github/mergify.yml b/.github/mergify.yml index b010656..ef15545 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -56,3 +56,10 @@ pull_request_rules: changes_requested: true approved: true message: "This Pull Request has been updated, so we're dismissing all reviews." + +- name: "close Pull Requests without files changed" + conditions: + - "#files=0" + actions: + close: + message: "This pull request has been automatically closed by Mergify because there are no longer any changes." diff --git a/.github/workflows/auto-format.yml b/.github/workflows/auto-format.yml index 990abed..375d0fd 100644 --- a/.github/workflows/auto-format.yml +++ b/.github/workflows/auto-format.yml @@ -6,7 +6,7 @@ on: jobs: auto-format: runs-on: ubuntu-latest - container: cloudposse/build-harness:slim-latest + container: cloudposse/build-harness:latest steps: # Checkout the pull request branch # "An action in a workflow run can’t trigger a new workflow run. For example, if an action pushes code using @@ -29,6 +29,8 @@ jobs: - name: Auto Format if: github.event.pull_request.state == 'open' shell: bash + env: + GITHUB_TOKEN: "${{ secrets.PUBLIC_REPO_ACCESS_TOKEN }}" run: make BUILD_HARNESS_PATH=/build-harness PACKAGES_PREFER_HOST=true -f /build-harness/templates/Makefile.build-harness pr/auto-format/host # Commit changes (if any) to the PR branch diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 3f48017..3a38fae 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -3,17 +3,24 @@ name: auto-release on: push: branches: - - master + - main + - master + - production jobs: publish: runs-on: ubuntu-latest steps: - # Drafts your next Release notes as Pull Requests are merged into "master" - - uses: release-drafter/release-drafter@v5 - with: - publish: true - prerelease: false - config-name: auto-release.yml - env: - GITHUB_TOKEN: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN }} + # Get PR from merged commit to master + - uses: actions-ecosystem/action-get-merged-pull-request@v1 + id: get-merged-pull-request + with: + github_token: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN }} + # Drafts your next Release notes as Pull Requests are merged into "main" + - uses: release-drafter/release-drafter@v5 + with: + publish: ${{ !contains(steps.get-merged-pull-request.outputs.labels, 'no-release') }} + prerelease: false + config-name: auto-release.yml + env: + GITHUB_TOKEN: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN }} diff --git a/.github/workflows/validate-codeowners.yml b/.github/workflows/validate-codeowners.yml index 386eb28..c5193b6 100644 --- a/.github/workflows/validate-codeowners.yml +++ b/.github/workflows/validate-codeowners.yml @@ -1,5 +1,7 @@ name: Validate Codeowners on: + workflow_dispatch: + pull_request: jobs: diff --git a/README.md b/README.md index 09e4984..4b21443 100644 --- a/README.md +++ b/README.md @@ -297,8 +297,13 @@ Available targets: | [delimiter](#input\_delimiter) | Delimiter to be used between ID elements.
Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. | `string` | `null` | no | | [descriptor\_formats](#input\_descriptor\_formats) | Describe additional descriptors to be output in the `descriptors` output map.
Map of maps. Keys are names of descriptors. Values are maps of the form
`{
format = string
labels = list(string)
}`
(Type is `any` so the map values can later be enhanced to provide additional options.)
`format` is a Terraform format string to be passed to the `format()` function.
`labels` is a list of labels, in order, to pass to `format()` function.
Label values will be normalized before being passed to `format()` so they will be
identical to how they appear in `id`.
Default is `{}` (`descriptors` output will be empty). | `any` | `{}` | no | | [ebs\_root\_volume\_size](#input\_ebs\_root\_volume\_size) | Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later | `number` | `10` | no | +| [ec2\_autoscaling\_role\_enabled](#input\_ec2\_autoscaling\_role\_enabled) | If set to `false`, will use `existing_ec2_autoscaling_role_arn` for an existing EC2 autoscaling IAM role that was created outside of this module | `bool` | `true` | no | +| [ec2\_role\_enabled](#input\_ec2\_role\_enabled) | If set to `false`, will use `existing_ec2_instance_profile_arn` for an existing EC2 IAM role that was created outside of this module | `bool` | `true` | no | | [enabled](#input\_enabled) | Set to false to prevent the module from creating any resources | `bool` | `null` | no | | [environment](#input\_environment) | ID element. Usually used for region e.g. 'uw2', 'us-west-2', OR role 'prod', 'staging', 'dev', 'UAT' | `string` | `null` | no | +| [existing\_ec2\_autoscaling\_role\_arn](#input\_existing\_ec2\_autoscaling\_role\_arn) | ARN of an existing EC2 autoscaling role to attach to the cluster | `string` | `""` | no | +| [existing\_ec2\_instance\_profile\_arn](#input\_existing\_ec2\_instance\_profile\_arn) | ARN of an existing EC2 instance profile | `string` | `""` | no | +| [existing\_service\_role\_arn](#input\_existing\_service\_role\_arn) | ARN of an existing EMR service role to attach to the cluster | `string` | `""` | no | | [id\_length\_limit](#input\_id\_length\_limit) | Limit `id` to this many characters (minimum 6).
Set to `0` for unlimited length.
Set to `null` for keep the existing setting, which defaults to `0`.
Does not affect `id_full`. | `number` | `null` | no | | [keep\_job\_flow\_alive\_when\_no\_steps](#input\_keep\_job\_flow\_alive\_when\_no\_steps) | Switch on/off run cluster with no steps or when all steps are complete | `bool` | `true` | no | | [kerberos\_ad\_domain\_join\_password](#input\_kerberos\_ad\_domain\_join\_password) | The Active Directory password for ad\_domain\_join\_user. Terraform cannot perform drift detection of this configuration. | `string` | `null` | no | @@ -334,6 +339,7 @@ Available targets: | [scale\_down\_behavior](#input\_scale\_down\_behavior) | The way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an instance group is resized | `string` | `null` | no | | [security\_configuration](#input\_security\_configuration) | The security configuration name to attach to the EMR cluster. Only valid for EMR clusters with `release_label` 4.8.0 or greater. See https://www.terraform.io/docs/providers/aws/r/emr_security_configuration.html for more info | `string` | `null` | no | | [service\_access\_security\_group](#input\_service\_access\_security\_group) | The name of the existing additional security group that will be used for EMR core & task nodes. If empty, a new security group will be created | `string` | `""` | no | +| [service\_role\_enabled](#input\_service\_role\_enabled) | If set to `false`, will use `existing_service_role_arn` for an existing IAM role that was created outside of this module | `bool` | `true` | no | | [slave\_allowed\_cidr\_blocks](#input\_slave\_allowed\_cidr\_blocks) | List of CIDR blocks to be allowed to access the slave instances | `list(string)` | `[]` | no | | [slave\_allowed\_security\_groups](#input\_slave\_allowed\_security\_groups) | List of security groups to be allowed to connect to the slave instances | `list(string)` | `[]` | no | | [stage](#input\_stage) | ID element. Usually used to indicate role, e.g. 'prod', 'staging', 'source', 'build', 'test', 'deploy', 'release' | `string` | `null` | no | diff --git a/docs/terraform.md b/docs/terraform.md index 78dc477..4c0fc33 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -91,8 +91,13 @@ | [delimiter](#input\_delimiter) | Delimiter to be used between ID elements.
Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. | `string` | `null` | no | | [descriptor\_formats](#input\_descriptor\_formats) | Describe additional descriptors to be output in the `descriptors` output map.
Map of maps. Keys are names of descriptors. Values are maps of the form
`{
format = string
labels = list(string)
}`
(Type is `any` so the map values can later be enhanced to provide additional options.)
`format` is a Terraform format string to be passed to the `format()` function.
`labels` is a list of labels, in order, to pass to `format()` function.
Label values will be normalized before being passed to `format()` so they will be
identical to how they appear in `id`.
Default is `{}` (`descriptors` output will be empty). | `any` | `{}` | no | | [ebs\_root\_volume\_size](#input\_ebs\_root\_volume\_size) | Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later | `number` | `10` | no | +| [ec2\_autoscaling\_role\_enabled](#input\_ec2\_autoscaling\_role\_enabled) | If set to `false`, will use `existing_ec2_autoscaling_role_arn` for an existing EC2 autoscaling IAM role that was created outside of this module | `bool` | `true` | no | +| [ec2\_role\_enabled](#input\_ec2\_role\_enabled) | If set to `false`, will use `existing_ec2_instance_profile_arn` for an existing EC2 IAM role that was created outside of this module | `bool` | `true` | no | | [enabled](#input\_enabled) | Set to false to prevent the module from creating any resources | `bool` | `null` | no | | [environment](#input\_environment) | ID element. Usually used for region e.g. 'uw2', 'us-west-2', OR role 'prod', 'staging', 'dev', 'UAT' | `string` | `null` | no | +| [existing\_ec2\_autoscaling\_role\_arn](#input\_existing\_ec2\_autoscaling\_role\_arn) | ARN of an existing EC2 autoscaling role to attach to the cluster | `string` | `""` | no | +| [existing\_ec2\_instance\_profile\_arn](#input\_existing\_ec2\_instance\_profile\_arn) | ARN of an existing EC2 instance profile | `string` | `""` | no | +| [existing\_service\_role\_arn](#input\_existing\_service\_role\_arn) | ARN of an existing EMR service role to attach to the cluster | `string` | `""` | no | | [id\_length\_limit](#input\_id\_length\_limit) | Limit `id` to this many characters (minimum 6).
Set to `0` for unlimited length.
Set to `null` for keep the existing setting, which defaults to `0`.
Does not affect `id_full`. | `number` | `null` | no | | [keep\_job\_flow\_alive\_when\_no\_steps](#input\_keep\_job\_flow\_alive\_when\_no\_steps) | Switch on/off run cluster with no steps or when all steps are complete | `bool` | `true` | no | | [kerberos\_ad\_domain\_join\_password](#input\_kerberos\_ad\_domain\_join\_password) | The Active Directory password for ad\_domain\_join\_user. Terraform cannot perform drift detection of this configuration. | `string` | `null` | no | @@ -128,6 +133,7 @@ | [scale\_down\_behavior](#input\_scale\_down\_behavior) | The way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an instance group is resized | `string` | `null` | no | | [security\_configuration](#input\_security\_configuration) | The security configuration name to attach to the EMR cluster. Only valid for EMR clusters with `release_label` 4.8.0 or greater. See https://www.terraform.io/docs/providers/aws/r/emr_security_configuration.html for more info | `string` | `null` | no | | [service\_access\_security\_group](#input\_service\_access\_security\_group) | The name of the existing additional security group that will be used for EMR core & task nodes. If empty, a new security group will be created | `string` | `""` | no | +| [service\_role\_enabled](#input\_service\_role\_enabled) | If set to `false`, will use `existing_service_role_arn` for an existing IAM role that was created outside of this module | `bool` | `true` | no | | [slave\_allowed\_cidr\_blocks](#input\_slave\_allowed\_cidr\_blocks) | List of CIDR blocks to be allowed to access the slave instances | `list(string)` | `[]` | no | | [slave\_allowed\_security\_groups](#input\_slave\_allowed\_security\_groups) | List of security groups to be allowed to connect to the slave instances | `list(string)` | `[]` | no | | [stage](#input\_stage) | ID element. Usually used to indicate role, e.g. 'prod', 'staging', 'source', 'build', 'test', 'deploy', 'release' | `string` | `null` | no | diff --git a/main.tf b/main.tf index 9c98dbe..49762ea 100644 --- a/main.tf +++ b/main.tf @@ -264,7 +264,7 @@ This role is required for all clusters. https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-iam-roles.html */ data "aws_iam_policy_document" "assume_role_emr" { - count = module.this.enabled ? 1 : 0 + count = module.this.enabled && var.service_role_enabled ? 1 : 0 statement { effect = "Allow" @@ -279,7 +279,7 @@ data "aws_iam_policy_document" "assume_role_emr" { } resource "aws_iam_role" "emr" { - count = module.this.enabled ? 1 : 0 + count = module.this.enabled && var.service_role_enabled ? 1 : 0 name = module.label_emr.id assume_role_policy = join("", data.aws_iam_policy_document.assume_role_emr.*.json) @@ -288,7 +288,7 @@ resource "aws_iam_role" "emr" { # https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-iam-roles.html resource "aws_iam_role_policy_attachment" "emr" { - count = module.this.enabled ? 1 : 0 + count = module.this.enabled && var.service_role_enabled ? 1 : 0 role = join("", aws_iam_role.emr.*.name) policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole" } @@ -301,7 +301,7 @@ This role is required for all clusters. https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-iam-roles.html */ data "aws_iam_policy_document" "assume_role_ec2" { - count = module.this.enabled ? 1 : 0 + count = module.this.enabled && var.ec2_role_enabled ? 1 : 0 statement { effect = "Allow" @@ -316,7 +316,7 @@ data "aws_iam_policy_document" "assume_role_ec2" { } resource "aws_iam_role" "ec2" { - count = module.this.enabled ? 1 : 0 + count = module.this.enabled && var.ec2_role_enabled ? 1 : 0 name = module.label_ec2.id assume_role_policy = join("", data.aws_iam_policy_document.assume_role_ec2.*.json) @@ -325,13 +325,13 @@ resource "aws_iam_role" "ec2" { # https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-iam-roles.html resource "aws_iam_role_policy_attachment" "ec2" { - count = module.this.enabled ? 1 : 0 + count = module.this.enabled && var.ec2_role_enabled ? 1 : 0 role = join("", aws_iam_role.ec2.*.name) policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role" } resource "aws_iam_instance_profile" "ec2" { - count = module.this.enabled ? 1 : 0 + count = module.this.enabled && var.ec2_role_enabled ? 1 : 0 name = join("", aws_iam_role.ec2.*.name) role = join("", aws_iam_role.ec2.*.name) } @@ -342,7 +342,7 @@ This role is required for all clusters. https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-iam-roles.html */ resource "aws_iam_role" "ec2_autoscaling" { - count = module.this.enabled ? 1 : 0 + count = module.this.enabled && var.ec2_autoscaling_role_enabled ? 1 : 0 name = module.label_ec2_autoscaling.id assume_role_policy = join("", data.aws_iam_policy_document.assume_role_emr.*.json) @@ -351,7 +351,7 @@ resource "aws_iam_role" "ec2_autoscaling" { # https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-iam-roles.html resource "aws_iam_role_policy_attachment" "ec2_autoscaling" { - count = module.this.enabled ? 1 : 0 + count = module.this.enabled && var.ec2_autoscaling_role_enabled ? 1 : 0 role = join("", aws_iam_role.ec2_autoscaling.*.name) policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforAutoScalingRole" } @@ -392,7 +392,7 @@ resource "aws_emr_cluster" "default" { emr_managed_master_security_group = var.use_existing_managed_master_security_group == false ? join("", aws_security_group.managed_master.*.id) : var.managed_master_security_group emr_managed_slave_security_group = var.use_existing_managed_slave_security_group == false ? join("", aws_security_group.managed_slave.*.id) : var.managed_slave_security_group service_access_security_group = var.use_existing_service_access_security_group == false && var.subnet_type == "private" ? join("", aws_security_group.managed_service_access.*.id) : var.service_access_security_group - instance_profile = join("", aws_iam_instance_profile.ec2.*.arn) + instance_profile = var.ec2_role_enabled ? join("", aws_iam_instance_profile.ec2.*.arn) : var.existing_ec2_instance_profile_arn additional_master_security_groups = var.use_existing_additional_master_security_group == false ? join("", aws_security_group.master.*.id) : var.additional_master_security_group additional_slave_security_groups = var.use_existing_additional_slave_security_group == false ? join("", aws_security_group.slave.*.id) : var.additional_slave_security_group } @@ -479,8 +479,8 @@ resource "aws_emr_cluster" "default" { log_uri = var.log_uri - service_role = join("", aws_iam_role.emr.*.arn) - autoscaling_role = join("", aws_iam_role.ec2_autoscaling.*.arn) + service_role = var.service_role_enabled ? join("", aws_iam_role.emr.*.arn) : var.existing_service_role_arn + autoscaling_role = var.ec2_autoscaling_role_enabled ? join("", aws_iam_role.ec2_autoscaling.*.arn) : var.existing_ec2_autoscaling_role_arn tags = module.this.tags diff --git a/outputs.tf b/outputs.tf index 45aec35..585fae7 100644 --- a/outputs.tf +++ b/outputs.tf @@ -29,6 +29,6 @@ output "master_host" { } output "ec2_role" { - value = join("", aws_iam_role.ec2.*.name) + value = var.ec2_role_enabled ? join("", aws_iam_role.ec2.*.name) : null description = "Role name of EMR EC2 instances so users can attach more policies" } diff --git a/variables.tf b/variables.tf index f9e4f41..fa56d3b 100644 --- a/variables.tf +++ b/variables.tf @@ -130,6 +130,42 @@ variable "custom_ami_id" { default = null } +variable "ec2_role_enabled" { + type = bool + description = "If set to `false`, will use `existing_ec2_instance_profile_arn` for an existing EC2 IAM role that was created outside of this module" + default = true +} + +variable "ec2_autoscaling_role_enabled" { + type = bool + description = "If set to `false`, will use `existing_ec2_autoscaling_role_arn` for an existing EC2 autoscaling IAM role that was created outside of this module" + default = true +} + +variable "service_role_enabled" { + type = bool + description = "If set to `false`, will use `existing_service_role_arn` for an existing IAM role that was created outside of this module" + default = true +} + +variable "existing_ec2_instance_profile_arn" { + type = string + description = "ARN of an existing EC2 instance profile" + default = "" +} + +variable "existing_ec2_autoscaling_role_arn" { + type = string + description = "ARN of an existing EC2 autoscaling role to attach to the cluster" + default = "" +} + +variable "existing_service_role_arn" { + type = string + description = "ARN of an existing EMR service role to attach to the cluster" + default = "" +} + variable "visible_to_all_users" { type = bool description = "Whether the job flow is visible to all IAM users of the AWS account associated with the job flow"