From a4b56842fb8bd66c57b95f0e8d905adca7262f96 Mon Sep 17 00:00:00 2001 From: "Dinesh.Weerapurage" Date: Sun, 9 Jul 2017 11:18:35 +0000 Subject: [PATCH 1/9] Adding boto3_action as part of current aws pack --- actions/assume_role.py | 28 ++++++++++++++++++++++++++ actions/assume_role.yaml | 32 ++++++++++++++++++++++++++++++ actions/boto3_action.py | 41 +++++++++++++++++++++++++++++++++++++++ actions/boto3_action.yaml | 27 ++++++++++++++++++++++++++ pack.yaml | 2 +- 5 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 actions/assume_role.py create mode 100755 actions/assume_role.yaml create mode 100755 actions/boto3_action.py create mode 100755 actions/boto3_action.yaml diff --git a/actions/assume_role.py b/actions/assume_role.py new file mode 100644 index 00000000..02ecd824 --- /dev/null +++ b/actions/assume_role.py @@ -0,0 +1,28 @@ +import boto3 + +from st2actions.runners.pythonrunner import Action + + +# pylint: disable=too-few-public-methods +class Boto3AssumeRoleRunner(Action): + def run(self, role_arn, role_session_name, policy, duration, external_id, serial_number, token_code): + client = boto3.client('sts') + kwargs = {} + kwargs['RoleArn'] = role_arn + kwargs['RoleSessionName'] = role_session_name + kwargs['DurationSeconds'] = duration + if policy is not None: + kwargs['Policy'] = policy + + if external_id is not None: + kwargs['ExternalId'] = external_id + + if serial_number is not None: + kwargs['SerialNumber'] = serial_number + + if token_code is not None: + kwargs['TokenCode'] = token_code + + response = client.assume_role(**kwargs) + response['Credentials']['Expiration'] = response['Credentials']['Expiration'].isoformat() + return (True, response) diff --git a/actions/assume_role.yaml b/actions/assume_role.yaml new file mode 100755 index 00000000..7b83106d --- /dev/null +++ b/actions/assume_role.yaml @@ -0,0 +1,32 @@ +--- +name: "assume_role" +runner_type: "python-script" +description: "Assume role" +enabled: true +entry_point: "assume_role.py" +pack: "aws" +parameters: + role_arn: + type: "string" + description: "ARN of the role" + required: true + role_session_name: + type: "string" + description: "Name for the session" + default: "DefaultAssumeSession" + policy: + type: "string" + description: "Policy document" + duration: + type: integer + description: "Duration for the session" + default: 3600 + external_id: + type: "string" + description: "External Id" + serial_number: + type: "string" + description: "Serial number of the MFA" + token_code: + type: "string" + description: "Token code from the MFA" diff --git a/actions/boto3_action.py b/actions/boto3_action.py new file mode 100755 index 00000000..2c56d90d --- /dev/null +++ b/actions/boto3_action.py @@ -0,0 +1,41 @@ +from datetime import date, datetime +import json + +import boto3 + +from st2actions.runners.pythonrunner import Action + + +# pylint: disable=too-few-public-methods +def json_serial(obj): + if isinstance(obj, (datetime, date)): + serial = obj.isoformat() + return serial + raise TypeError("Type %s not serializable" % type(obj)) + + +# pylint: disable=too-few-public-methods +class Boto3ActionRunner(Action): + def run(self, service, region, action_name, credentials, params): + client = None + response = None + + if credentials is not None: + session = boto3.Session( + aws_access_key_id=credentials['Credentials']['AccessKeyId'], + aws_secret_access_key=credentials['Credentials']['SecretAccessKey'], + aws_session_token=credentials['Credentials']['SessionToken']) + client = session.client(service, region_name=region) + else: + client = boto3.client(service, region_name=region) + + if client is None: + return (False, 'boto3 client creation failed') + + if params is not None: + response = getattr(client, action_name)(**params) + else: + response = getattr(client, action_name)() + + response = json.loads(json.dumps(response, default=json_serial)) + return (True, response) diff --git a/actions/boto3_action.yaml b/actions/boto3_action.yaml new file mode 100755 index 00000000..7d0b4bc2 --- /dev/null +++ b/actions/boto3_action.yaml @@ -0,0 +1,27 @@ +--- +name: "boto3_action" +runner_type: "python-script" +description: "Run any boto3 action" +enabled: true +entry_point: "boto3_action.py" +pack: "aws" +parameters: + service: + type: "string" + description: "Name of the service to create client" + required: true + region: + type: "string" + description: "Region where action is performed" + required: true + action_name: + type: "string" + description: "Name of the action to run" + required: true + credentials: + type: "object" + description: "Response from assume role" + params: + type: object + description: "Parameters for the action" + diff --git a/pack.yaml b/pack.yaml index 283a3cca..327c8f7a 100755 --- a/pack.yaml +++ b/pack.yaml @@ -19,6 +19,6 @@ keywords: - SQS - lambda -version : 0.11.0 +version : 0.12.0 author : StackStorm, Inc. email : info@stackstorm.com From 5fca0d0b18d2f35caeee366bfdea501ccee389f1 Mon Sep 17 00:00:00 2001 From: Dinesh Weerapurage Date: Sat, 15 Jul 2017 15:58:48 +0000 Subject: [PATCH 2/9] Adding boto3action to aws pack --- actions/assume_role.yaml | 2 +- actions/{boto3_action.py => boto3action.py} | 0 actions/{boto3_action.yaml => boto3action.yaml} | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename actions/{boto3_action.py => boto3action.py} (100%) rename actions/{boto3_action.yaml => boto3action.yaml} (91%) diff --git a/actions/assume_role.yaml b/actions/assume_role.yaml index 7b83106d..12574c28 100755 --- a/actions/assume_role.yaml +++ b/actions/assume_role.yaml @@ -1,7 +1,7 @@ --- name: "assume_role" runner_type: "python-script" -description: "Assume role" +description: "Assume a role to use with boto3action" enabled: true entry_point: "assume_role.py" pack: "aws" diff --git a/actions/boto3_action.py b/actions/boto3action.py similarity index 100% rename from actions/boto3_action.py rename to actions/boto3action.py diff --git a/actions/boto3_action.yaml b/actions/boto3action.yaml similarity index 91% rename from actions/boto3_action.yaml rename to actions/boto3action.yaml index 7d0b4bc2..71932f3c 100755 --- a/actions/boto3_action.yaml +++ b/actions/boto3action.yaml @@ -1,9 +1,9 @@ --- -name: "boto3_action" +name: "boto3action" runner_type: "python-script" description: "Run any boto3 action" enabled: true -entry_point: "boto3_action.py" +entry_point: "boto3action.py" pack: "aws" parameters: service: From 9bea2b370882b4b757962da31685f4ffba66c444 Mon Sep 17 00:00:00 2001 From: Dinesh Weerapurage Date: Sat, 15 Jul 2017 10:03:57 -0600 Subject: [PATCH 3/9] Using datetime serialization in assume_role for boto3action --- actions/assume_role.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/actions/assume_role.py b/actions/assume_role.py index 02ecd824..b692b7d9 100644 --- a/actions/assume_role.py +++ b/actions/assume_role.py @@ -1,11 +1,24 @@ +from datetime import date, datetime +import json + import boto3 from st2actions.runners.pythonrunner import Action +# pylint: disable=too-few-public-methods +def json_serial(obj): + if isinstance(obj, (datetime, date)): + serial = obj.isoformat() + return serial + raise TypeError("Type %s not serializable" % type(obj)) + + # pylint: disable=too-few-public-methods class Boto3AssumeRoleRunner(Action): - def run(self, role_arn, role_session_name, policy, duration, external_id, serial_number, token_code): + def run( + self, role_arn, role_session_name, policy, + duration, external_id, serial_number, token_code): client = boto3.client('sts') kwargs = {} kwargs['RoleArn'] = role_arn @@ -13,7 +26,7 @@ def run(self, role_arn, role_session_name, policy, duration, external_id, serial kwargs['DurationSeconds'] = duration if policy is not None: kwargs['Policy'] = policy - + if external_id is not None: kwargs['ExternalId'] = external_id @@ -23,6 +36,6 @@ def run(self, role_arn, role_session_name, policy, duration, external_id, serial if token_code is not None: kwargs['TokenCode'] = token_code - response = client.assume_role(**kwargs) - response['Credentials']['Expiration'] = response['Credentials']['Expiration'].isoformat() + response = client.assume_role(**kwargs) + response = json.loads(json.dumps(response, default=json_serial)) return (True, response) From ce5335e0cbfaec3fcbdcaaf5317746535a5718a7 Mon Sep 17 00:00:00 2001 From: Dinesh Weerapurage Date: Sat, 15 Jul 2017 16:07:11 +0000 Subject: [PATCH 4/9] Bumped version number for aws pack --- pack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pack.yaml b/pack.yaml index 327c8f7a..52831cd1 100755 --- a/pack.yaml +++ b/pack.yaml @@ -19,6 +19,6 @@ keywords: - SQS - lambda -version : 0.12.0 +version : 0.13.0 author : StackStorm, Inc. email : info@stackstorm.com From 671f5fdf268a534e0a5dd1ce00732f2f3802cf5c Mon Sep 17 00:00:00 2001 From: Dinesh Weerapurage Date: Sat, 15 Jul 2017 16:13:54 +0000 Subject: [PATCH 5/9] Reverting to correct pack version number --- pack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pack.yaml b/pack.yaml index 52831cd1..327c8f7a 100755 --- a/pack.yaml +++ b/pack.yaml @@ -19,6 +19,6 @@ keywords: - SQS - lambda -version : 0.13.0 +version : 0.12.0 author : StackStorm, Inc. email : info@stackstorm.com From 76f386ea5329c375a49ee06dcb128ec4aa26c50b Mon Sep 17 00:00:00 2001 From: Dinesh Weerapurage Date: Sat, 15 Jul 2017 10:34:11 -0600 Subject: [PATCH 6/9] Removed lib from .gitignore as it ignores action/libs directory, moved json_serial into lib --- .gitignore | 1 - actions/assume_role.py | 10 +--------- actions/boto3action.py | 11 +---------- actions/lib/util.py | 9 +++++++++ 4 files changed, 11 insertions(+), 20 deletions(-) create mode 100644 actions/lib/util.py diff --git a/.gitignore b/.gitignore index 72364f99..a76c9ccb 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ dist/ downloads/ eggs/ .eggs/ -lib/ lib64/ parts/ sdist/ diff --git a/actions/assume_role.py b/actions/assume_role.py index b692b7d9..4720d6f6 100644 --- a/actions/assume_role.py +++ b/actions/assume_role.py @@ -1,17 +1,9 @@ -from datetime import date, datetime import json - import boto3 from st2actions.runners.pythonrunner import Action - -# pylint: disable=too-few-public-methods -def json_serial(obj): - if isinstance(obj, (datetime, date)): - serial = obj.isoformat() - return serial - raise TypeError("Type %s not serializable" % type(obj)) +from lib.util import json_serial # pylint: disable=too-few-public-methods diff --git a/actions/boto3action.py b/actions/boto3action.py index 2c56d90d..38f4dc2c 100755 --- a/actions/boto3action.py +++ b/actions/boto3action.py @@ -1,17 +1,8 @@ -from datetime import date, datetime import json - import boto3 from st2actions.runners.pythonrunner import Action - - -# pylint: disable=too-few-public-methods -def json_serial(obj): - if isinstance(obj, (datetime, date)): - serial = obj.isoformat() - return serial - raise TypeError("Type %s not serializable" % type(obj)) +from lib.util import json_serial # pylint: disable=too-few-public-methods diff --git a/actions/lib/util.py b/actions/lib/util.py new file mode 100644 index 00000000..cdcecc9a --- /dev/null +++ b/actions/lib/util.py @@ -0,0 +1,9 @@ +from datetime import date, datetime + + +# pylint: disable=too-few-public-methods +def json_serial(obj): + if isinstance(obj, (datetime, date)): + serial = obj.isoformat() + return serial + raise TypeError("Type %s not serializable" % type(obj)) From e8f30a64311644707f2be4548b82ae4291f011f3 Mon Sep 17 00:00:00 2001 From: Dinesh Weerapurage Date: Sat, 22 Jul 2017 06:54:58 -0600 Subject: [PATCH 7/9] * boto3action.md: Add documentation on boto3action with examples. --- boto3action.md | 283 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 boto3action.md diff --git a/boto3action.md b/boto3action.md new file mode 100644 index 00000000..663dd275 --- /dev/null +++ b/boto3action.md @@ -0,0 +1,283 @@ +# aws.boto3action + +1. [Introduction](#introduction) +1. [Boto3 documentation](#boto3-documentation) +2. [Getting started](#getting-started) +3. [Create VPC workflow](#create-vpc-workflow) +4. [Create VPC workflow with assume_role](#create-vpc-workflow-with-assume_role) + +## Introduction +`aws.boto3action` runs boto3 actions in stackstorm dynamically. It has following features. + +- Uses boto3 configurations. Find more information on boto3 configuration in boto3 documentation. http://boto3.readthedocs.io/en/latest/guide/quickstart.html#configuration +- Ablity to run cross region actions +- Ablility to run cross account actions. + +## Boto3 documentation + +Boto3 contains detailed documentation and examples on each service. Follow link to find out about available services http://boto3.readthedocs.io/en/latest/reference/services/index.html + +## Getting started + +The simplest way to configure and test boto3 is to use awscli. + +``` +pip install awscli +aws configure +aws ec2 describe-vpcs --region "us-west-1" +``` + +Then go ahead and install aws pack. `aws.boto3action` is ready to use, without additional configurations. + +``` +st2 pack install aws +st2 run aws.boto3action service="ec2" action_name="describe_vpcs" region="us-west-1" +``` + +In addition, let’s assume these is a boto3 profile name `production`. Use `production` profile as follows. Boto3 documentation has more information on profiles. http://boto3.readthedocs.io/en/latest/guide/configuration.html#shared-credentials-file + +``` +st2 run aws.boto3action service="ec2" action_name="describe_vpcs" region="us-west-1" env="AWS_PROFILE=production" +``` + +## Create VPC Workflow + +action/create_vpc.yaml + +```yaml +name: "create_vpc" +runner_type: "mistral-v2" +description: "Create VPC with boto3action" +enabled: true +entry_point: "workflows/create_vpc.yaml" +parameters: + cidr_block: + type: "string" + description: "VPC CIDR block" + required: true + region: + type: "string" + description: "Region to create VPC" + required: true + subnet_cidr_block: + type: "string" + description: "Subnet CIDR block" + required: true + availability_zone: + type: "string" + description: "Availability zone to create subnet" + required: true + +``` + +action/workflows/create_vpc.yaml + + +```yaml +--- +version: '2.0' +aws.create_vpc: + type: direct + description: "Create VPC with boto3action" + input: + - cidr_block + - region + - subnet_cidr_block + - availability_zone + tasks: + create_vpc: + action: aws.boto3action + input: + service: ec2 + action_name: create_vpc + region: <% $.region %> + params: <% dict(CidrBlock => $.cidr_block, InstanceTenancy => "default") %> + publish: + vpc_id: <% task(create_vpc).result.result.Vpc.VpcId %> + on-success: + - create_subnet + - create_igw + + create_subnet: + action: aws.boto3action + input: + service: ec2 + action_name: create_subnet + region: <% $.region %> + params: <% dict(AvailabilityZone => $.availability_zone, CidrBlock => $.subnet_cidr_block, VpcId => $.vpc_id) %> + publish: + subnet_id: <% task(create_subnet).result.result.Subnet.SubnetId %> + on-success: + - create_route_table + + create_igw: + action: aws.boto3action + input: + service: ec2 + action_name: create_internet_gateway + region: <% $.region %> + publish: + igw_id: <% task(create_igw).result.result.InternetGateway.InternetGatewayId %> + on-success: + - attach_igw + + attach_igw: + action: aws.boto3action + input: + service: ec2 + action_name: attach_internet_gateway + region: <% $.region %> + params: <% dict(VpcId => $.vpc_id, InternetGatewayId => $.igw_id) %> + on-success: + - create_route_igw + + create_route_table: + action: aws.boto3action + input: + service: ec2 + action_name: create_route_table + region: <% $.region %> + params: <% dict(VpcId => $.vpc_id) %> + publish: + route_table_id: <% task(create_route_table).result.result.RouteTable.RouteTableId %> + on-success: + - attach_route_tables + + attach_route_tables: + action: aws.boto3action + input: + service: ec2 + action_name: associate_route_table + region: <% $.region %> + params: <% dict(SubnetId => $.subnet_id, RouteTableId => $.route_table_id) %> + on-success: + - create_route_igw + + create_route_igw: + join: 2 + action: aws.boto3action + input: + service: ec2 + action_name: create_route + region: <% $.region %> + params: <% dict(RouteTableId => $.route_table_id, GatewayId => $.igw_id, DestinationCidrBlock => '0.0.0.0/0') %> +``` + +Use this workflow as follows, + +``` +st2 run aws.create_vpc cidr_block="172.18.0.0/16" region="us-west-2" availability_zone="us-west-2b" subnet_cidr_block="172.18.0.0/24" +``` + +# Create VPC workflow with assume_role + + Let’s assume we have two aws accounts. First aws account, 123456, is already configured to use boto3. Second aws account, 456789, has a `IAM` role `st2_role`. We can assume this role, then use `create_vpc` workflow to create vpc in aws account 456789. + +action/workflows/create_vpc.yaml + +```yaml +--- +version: '2.0' +aws.create_vpc: + type: direct + description: "Create VPC with boto3action" + input: + - cidr_block + - region + - subnet_cidr_block + - availability_zone + tasks: + assume_role: + action: aws.assume_role + input: + role_arn: “arn:aws:iam:456789:role/st2_role” + publish: + credentials: <% task(assume_role).result.result %> + on-success: + - create_vpc + + create_vpc: + action: aws.boto3action + input: + service: ec2 + action_name: create_vpc + region: <% $.region %> + params: <% dict(CidrBlock => $.cidr_block, InstanceTenancy => "default") %> + credentials: <% $.credentials %> + publish: + vpc_id: <% task(create_vpc).result.result.Vpc.VpcId %> + on-success: + - create_subnet + - create_igw + + create_subnet: + action: aws.boto3action + input: + service: ec2 + action_name: create_subnet + region: <% $.region %> + params: <% dict(AvailabilityZone => $.availability_zone, CidrBlock => $.subnet_cidr_block, VpcId => $.vpc_id) %> + credentials: <% $.credentials %> + publish: + subnet_id: <% task(create_subnet).result.result.Subnet.SubnetId %> + on-success: + - create_route_table + + create_igw: + action: aws.boto3action + input: + service: ec2 + action_name: create_internet_gateway + region: <% $.region %> + credentials: <% $.credentials %> + publish: + igw_id: <% task(create_igw).result.result.InternetGateway.InternetGatewayId %> + on-success: + - attach_igw + + attach_igw: + action: aws.boto3action + input: + service: ec2 + action_name: attach_internet_gateway + region: <% $.region %> + params: <% dict(VpcId => $.vpc_id, InternetGatewayId => $.igw_id) %> + credentials: <% $.credentials %> + on-success: + - create_route_igw + + create_route_table: + action: aws.boto3action + input: + service: ec2 + action_name: create_route_table + region: <% $.region %> + params: <% dict(VpcId => $.vpc_id) %> + credentials: <% $.credentials %> + publish: + route_table_id: <% task(create_route_table).result.result.RouteTable.RouteTableId %> + on-success: + - attach_route_tables + + attach_route_tables: + action: aws.boto3action + input: + service: ec2 + action_name: associate_route_table + region: <% $.region %> + params: <% dict(SubnetId => $.subnet_id, RouteTableId => $.route_table_id) %> + credentials: <% $.credentials %> + on-success: + - create_route_igw + + create_route_igw: + join: 2 + action: aws.boto3action + input: + service: ec2 + action_name: create_route + region: <% $.region %> + params: <% dict(RouteTableId => $.route_table_id, GatewayId => $.igw_id, DestinationCidrBlock => '0.0.0.0/0') %> + credentials: <% $.credentials %> +``` + From 002bd02b7a4503eb577fd904187b5acbb03dfb87 Mon Sep 17 00:00:00 2001 From: Dinesh Weerapurage Date: Sat, 22 Jul 2017 07:11:26 -0600 Subject: [PATCH 8/9] Added boto3action link to README. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index f752dc2f..426500c3 100755 --- a/README.md +++ b/README.md @@ -288,3 +288,7 @@ This trigger is emitted when a single message is received from a queue. } ``` + +## Boto3Action + +`aws.boto3action` added as an option to use boto3 actions dynamically. More on [boto3action](boto3action.md). From 9a3567d0f5022bb769fc1752717fa8e4f7b1bf62 Mon Sep 17 00:00:00 2001 From: Dinesh Weerapurage Date: Tue, 1 Aug 2017 06:51:12 -0600 Subject: [PATCH 9/9] Fixing typos --- boto3action.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boto3action.md b/boto3action.md index 663dd275..470b05c9 100644 --- a/boto3action.md +++ b/boto3action.md @@ -10,8 +10,8 @@ `aws.boto3action` runs boto3 actions in stackstorm dynamically. It has following features. - Uses boto3 configurations. Find more information on boto3 configuration in boto3 documentation. http://boto3.readthedocs.io/en/latest/guide/quickstart.html#configuration -- Ablity to run cross region actions -- Ablility to run cross account actions. +- Ability to run cross region actions +- Ability to run cross account actions. ## Boto3 documentation