From 844f5b613c66b6add926b0abc85faba17dea5bf7 Mon Sep 17 00:00:00 2001 From: Elliott Spira Date: Fri, 18 Sep 2015 15:36:48 +1000 Subject: [PATCH 1/7] adding local version for development --- deploy_lambda_code.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy_lambda_code.sh b/deploy_lambda_code.sh index 37fb026..a1a91b6 100644 --- a/deploy_lambda_code.sh +++ b/deploy_lambda_code.sh @@ -8,4 +8,5 @@ mv lib/autotag.zip . zip -g autotag.zip -r node_modules/ echo --- echo 2. Uploading code via AWS CLI -aws lambda --region ap-northeast-1 update-function-code --function-name arn:aws:lambda:[AWS_REGION]:[AWS_ACCOUNT_ID]:function:gs_autotag --zip-file fileb:///[PATH_TO_AUTOTAG_DIR]/autotag.zip +aws lambda --region ap-northeast-1 update-function-code --function-name arn:aws:lambda:ap-northeast-1:002790823159:function:gs_autotag --zip-file fileb:///home/e/dev/gs/autotag/autotag.zip +# aws s3 cp autotag.zip s3://gs-lambda-functions/autotag.zip From ed57bad53b05bd5f574a7468687b3fb235d0a947 Mon Sep 17 00:00:00 2001 From: Elliott Spira Date: Fri, 18 Sep 2015 15:36:48 +1000 Subject: [PATCH 2/7] adding local version for development --- deploy_lambda_code.sh | 3 ++- sample_data.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy_lambda_code.sh b/deploy_lambda_code.sh index 37fb026..a1a91b6 100644 --- a/deploy_lambda_code.sh +++ b/deploy_lambda_code.sh @@ -8,4 +8,5 @@ mv lib/autotag.zip . zip -g autotag.zip -r node_modules/ echo --- echo 2. Uploading code via AWS CLI -aws lambda --region ap-northeast-1 update-function-code --function-name arn:aws:lambda:[AWS_REGION]:[AWS_ACCOUNT_ID]:function:gs_autotag --zip-file fileb:///[PATH_TO_AUTOTAG_DIR]/autotag.zip +aws lambda --region ap-northeast-1 update-function-code --function-name arn:aws:lambda:ap-northeast-1:002790823159:function:gs_autotag --zip-file fileb:///home/e/dev/gs/autotag/autotag.zip +# aws s3 cp autotag.zip s3://gs-lambda-functions/autotag.zip diff --git a/sample_data.js b/sample_data.js index f76b5fd..5ea1d4e 100644 --- a/sample_data.js +++ b/sample_data.js @@ -27,7 +27,7 @@ module.exports = { arn: 'arn:aws:s3:::autotag-cloudtrail-logs' }, object: { - key: 'AWSLogs/[YOUR_ACCOUNT_ID]/CloudTrail/ap-northeast-1/2015/09/18/[FILE_NAME].json.gz', + key: 'AWSLogs/002790823159/CloudTrail/ap-northeast-1/2015/09/18/002790823159_CloudTrail_ap-northeast-1_20150918T0110Z_k16qO2Q5mpKg8RYj.json.gz', size: 2858, eTag: '1dca20ce134f8165f2ec7508cda93b39', sequencer: '0055F2489B160B9FA4' From 1eb2a4e3f760b3f89143317e0b1414edc482f37a Mon Sep 17 00:00:00 2001 From: Elliott Spira Date: Mon, 28 Sep 2015 08:47:18 +1000 Subject: [PATCH 3/7] cleaning up file organisation --- README.md | 37 +++++++++++++++++-- src/autotag.js | 12 +++++- src/{ => workers}/autotag_autoscale_worker.js | 0 .../autotag_data_pipeline_worker.js | 0 src/{ => workers}/autotag_default_worker.js | 0 src/{ => workers}/autotag_ebs_worker.js | 0 src/{ => workers}/autotag_ec2_worker.js | 0 src/{ => workers}/autotag_elb_worker.js | 0 src/{ => workers}/autotag_emr_worker.js | 0 .../autotag_internet_gateway_worker.js | 0 src/{ => workers}/autotag_rds_worker.js | 0 src/{ => workers}/autotag_s3_worker.js | 0 src/{ => workers}/autotag_subnet_worker.js | 0 src/{ => workers}/autotag_vpc_worker.js | 0 14 files changed, 44 insertions(+), 5 deletions(-) rename src/{ => workers}/autotag_autoscale_worker.js (100%) rename src/{ => workers}/autotag_data_pipeline_worker.js (100%) rename src/{ => workers}/autotag_default_worker.js (100%) rename src/{ => workers}/autotag_ebs_worker.js (100%) rename src/{ => workers}/autotag_ec2_worker.js (100%) rename src/{ => workers}/autotag_elb_worker.js (100%) rename src/{ => workers}/autotag_emr_worker.js (100%) rename src/{ => workers}/autotag_internet_gateway_worker.js (100%) rename src/{ => workers}/autotag_rds_worker.js (100%) rename src/{ => workers}/autotag_s3_worker.js (100%) rename src/{ => workers}/autotag_subnet_worker.js (100%) rename src/{ => workers}/autotag_vpc_worker.js (100%) diff --git a/README.md b/README.md index 21234b1..a50fc6a 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,34 @@ -# Autotag - -This is an open-source tagging solution for AWS. Deploy autotag to lambda and set up CloudTrail and have each of your resources tagged with the resource who created it. It was written by GorillaStack. +# Auto Tag +This is an open-source tagging solution for AWS. Deploy auto tag to lambda and set up CloudTrail and have each of your resources tagged with the resource who created it. It was written by [GorillaStack](http://www.gorillastack.com/). +[Read a blog post about the project](http://blog.gorillastack.com/gorillastack-presents-auto-tag). ## Setup +### 1. Turn on CloudTrail for your region + +1. Turn on CloudTrail. +2. Create a new Amazon S3 bucket for storing your log files, or specify an existing bucket where you want the log files delivered. +3. (Optional and NOT REQUIRED for auto tag) Create a new Amazon SNS topic in order to receive notifications when new log files are delivered. + +More [documentation on creating a Trail](https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-create-and-update-a-trail.html) + +### 2. Create a lambda function + +1. Within lambda, press the 'Create a Lambda Function' button +2. Press the 'Skip' button to bypass the suggested blueprints +3. Enter the lambda function name (e.g. 'autotag') +4. Select 'Node.js' as the Runtime +5. Upload the [latest release's zip file](https://github.com/GorillaStack/auto-tag/releases) +6. Under 'Handler' add 'autotag.handler' + +More [documentation on Lambda](https://docs.aws.amazon.com/lambda/latest/dg/getting-started.html) + +### 3. Configure the access policy for your lambda role + +For the complete role's policy, scroll down for the master policy. Read on for finer details on the access permissions required below. + #### Baseline policies for your lambda IAM role If you install your lambda function and don't plan on tagging resources, at very least you will need these permissions: @@ -110,7 +133,7 @@ If you are interested in contributing, please get started by forking our github ### Development guide -Autotag is implemented in Javascript (ECMAScript 2015 - a.k.a. es6). To make this compatible with lambda and other es5 environments, we use [babel](https://babeljs.io/) to transpile the es6 code to es5. For this reason, you will need to install babel globally to get started: +Auto tag is implemented in Javascript (ECMAScript 2015 - a.k.a. es6). To make this compatible with lambda and other es5 environments, we use [babel](https://babeljs.io/) to transpile the es6 code to es5. For this reason, you will need to install babel globally to get started: ```bash $ npm install -g babel @@ -127,3 +150,9 @@ To assist you in packaging and deploying your code to your lambda function, I ha ```bash $ bash deploy_lambda_code.sh ``` + +To assist with running during development without having to deploy to lambda, use the `main.js` and `sample_data.js` files we have provided. + +```bash +$ node main.js +``` diff --git a/src/autotag.js b/src/autotag.js index 7e228e8..22b267f 100644 --- a/src/autotag.js +++ b/src/autotag.js @@ -1,7 +1,8 @@ const AwsCloudTrailListener = require('./aws_cloud_trail_listener'); const exports = {}; + exports.handler = function(cloudtrailEvent, context) { - let enabledListeners = [ + const enabledListeners = [ AwsCloudTrailListener.EC2.name, AwsCloudTrailListener.S3.name, AwsCloudTrailListener.AUTOSCALE_GROUPS.name, @@ -15,6 +16,15 @@ exports.handler = function(cloudtrailEvent, context) { AwsCloudTrailListener.DATA_PIPELINE.name ]; + /* + ** An object where: + ** Keys: linked account ids + ** Values: the arn of the role configured for cross account access on that + ** account + */ + const rolesForCrossAccountAccess = { + '002': 'arn' + }; let listener = new AwsCloudTrailListener(cloudtrailEvent, context, enabledListeners); return listener.execute(); diff --git a/src/autotag_autoscale_worker.js b/src/workers/autotag_autoscale_worker.js similarity index 100% rename from src/autotag_autoscale_worker.js rename to src/workers/autotag_autoscale_worker.js diff --git a/src/autotag_data_pipeline_worker.js b/src/workers/autotag_data_pipeline_worker.js similarity index 100% rename from src/autotag_data_pipeline_worker.js rename to src/workers/autotag_data_pipeline_worker.js diff --git a/src/autotag_default_worker.js b/src/workers/autotag_default_worker.js similarity index 100% rename from src/autotag_default_worker.js rename to src/workers/autotag_default_worker.js diff --git a/src/autotag_ebs_worker.js b/src/workers/autotag_ebs_worker.js similarity index 100% rename from src/autotag_ebs_worker.js rename to src/workers/autotag_ebs_worker.js diff --git a/src/autotag_ec2_worker.js b/src/workers/autotag_ec2_worker.js similarity index 100% rename from src/autotag_ec2_worker.js rename to src/workers/autotag_ec2_worker.js diff --git a/src/autotag_elb_worker.js b/src/workers/autotag_elb_worker.js similarity index 100% rename from src/autotag_elb_worker.js rename to src/workers/autotag_elb_worker.js diff --git a/src/autotag_emr_worker.js b/src/workers/autotag_emr_worker.js similarity index 100% rename from src/autotag_emr_worker.js rename to src/workers/autotag_emr_worker.js diff --git a/src/autotag_internet_gateway_worker.js b/src/workers/autotag_internet_gateway_worker.js similarity index 100% rename from src/autotag_internet_gateway_worker.js rename to src/workers/autotag_internet_gateway_worker.js diff --git a/src/autotag_rds_worker.js b/src/workers/autotag_rds_worker.js similarity index 100% rename from src/autotag_rds_worker.js rename to src/workers/autotag_rds_worker.js diff --git a/src/autotag_s3_worker.js b/src/workers/autotag_s3_worker.js similarity index 100% rename from src/autotag_s3_worker.js rename to src/workers/autotag_s3_worker.js diff --git a/src/autotag_subnet_worker.js b/src/workers/autotag_subnet_worker.js similarity index 100% rename from src/autotag_subnet_worker.js rename to src/workers/autotag_subnet_worker.js diff --git a/src/autotag_vpc_worker.js b/src/workers/autotag_vpc_worker.js similarity index 100% rename from src/autotag_vpc_worker.js rename to src/workers/autotag_vpc_worker.js From da996268485014b19ce1bfc8d3366023c121cc8a Mon Sep 17 00:00:00 2001 From: Elliott Spira Date: Mon, 5 Oct 2015 17:03:04 +1100 Subject: [PATCH 4/7] updates to support cross account tagging --- README.md | 91 ++++++++----------- src/autotag.js | 9 -- src/autotag_factory.js | 24 ++--- src/workers/autotag_autoscale_worker.js | 13 ++- src/workers/autotag_data_pipeline_worker.js | 13 ++- src/workers/autotag_default_worker.js | 30 ++++++ src/workers/autotag_ebs_worker.js | 7 +- src/workers/autotag_ec2_worker.js | 21 +++-- src/workers/autotag_elb_worker.js | 14 ++- src/workers/autotag_emr_worker.js | 13 ++- .../autotag_internet_gateway_worker.js | 7 +- src/workers/autotag_rds_worker.js | 13 ++- src/workers/autotag_s3_worker.js | 4 +- src/workers/autotag_subnet_worker.js | 7 +- src/workers/autotag_vpc_worker.js | 7 +- 15 files changed, 176 insertions(+), 97 deletions(-) diff --git a/README.md b/README.md index a50fc6a..bbe7842 100644 --- a/README.md +++ b/README.md @@ -27,41 +27,29 @@ More [documentation on Lambda](https://docs.aws.amazon.com/lambda/latest/dg/gett ### 3. Configure the access policy for your lambda role -For the complete role's policy, scroll down for the master policy. Read on for finer details on the access permissions required below. +Your lambda function will run as an IAM role. This is where we configure the permissions required. -#### Baseline policies for your lambda IAM role - -If you install your lambda function and don't plan on tagging resources, at very least you will need these permissions: - -1. Permissions to save logs for your lambda execution. -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Resource": "arn:aws:logs:*:*:*" - } - ] -} -``` - -2. Permissions to retrieve zipped CloudTrail log items from S3. +#### Lambda function master policy ```json { "Version": "2012-10-17", "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "arn:aws:logs:*:*:*" + }, { "Sid": "Stmt1442379848000", "Effect": "Allow", "Action": [ "s3:GetObject", - "s3:ListBucket" + "s3:ListBucket", + "sts:*" ], "Resource": [ "*" @@ -71,43 +59,28 @@ If you install your lambda function and don't plan on tagging resources, at very } ``` +This contains permissions for: -#### Necessary policies for your lambda's IAM role -Actions to allow for all resources: +1. Saving logs for your lambda execution. +2. Retrieving zipped CloudTrail log items from S3. +3. Assuming the auto-tag role on targeted accounts. -* S3: `s3:GetBucketTagging` - `s3:PutBucketTagging` -* EC2: `ec2:CreateTags` -* ELB: `elasticloadbalancing:AddTags` -* AutoScaling: `autoscaling:CreateOrUpdateTags` -* EBS: `ec2:CreateTags` (Same as EC2) -* VPC: `ec2:CreateTags` (Same as EC2) -* Subnet: `ec2:CreateTags` (Same as EC2) -* InternetGateway: `ec2:CreateTags` (Same as EC2) -* RDS: `rds:AddTagsToResource` -* EMR: `elasticmapreduce:AddTags` -* DataPipeline: `datapipeline:AddTags` +### 4. Configure the access policy for your Auto-Tag role -## Whole master policy +When auto-tag finds an event that indicated the creation of a resource, it needs permissions to tag that resource. On each account, a role for cross account access will need to be created, with the following permissions. + +*NOTE;* The role must be named 'AutoTagRole'. +(This is a temporary hard requirement, given that after uploading the code via zip file, AWS doesn't allow the user to edit the main file inline. This introduces complexity in user defined configuration for the lambda function.) + +#### Auto-Tag role master policy ```json { "Version": "2012-10-17", "Statement": [ - { - "Effect": "Allow", - "Action": [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Resource": "arn:aws:logs:*:*:*" - }, { "Sid": "Stmt1442379848000", "Effect": "Allow", "Action": [ - "s3:GetObject", - "s3:ListBucket", "s3:GetBucketTagging", "s3:PutBucketTagging", "ec2:CreateTags", @@ -125,6 +98,22 @@ Actions to allow for all resources: } ``` +Details on specific requirements for each service, in case you want a subset of this master policy: + +* S3: `s3:GetBucketTagging` + `s3:PutBucketTagging` +* EC2: `ec2:CreateTags` +* ELB: `elasticloadbalancing:AddTags` +* AutoScaling: `autoscaling:CreateOrUpdateTags` +* EBS: `ec2:CreateTags` (Same as EC2) +* VPC: `ec2:CreateTags` (Same as EC2) +* Subnet: `ec2:CreateTags` (Same as EC2) +* InternetGateway: `ec2:CreateTags` (Same as EC2) +* RDS: `rds:AddTagsToResource` +* EMR: `elasticmapreduce:AddTags` +* DataPipeline: `datapipeline:AddTags` + + ## Contributing If you have questions, feature requests or bugs to report, please do so on the github repository. diff --git a/src/autotag.js b/src/autotag.js index 22b267f..e78cd32 100644 --- a/src/autotag.js +++ b/src/autotag.js @@ -16,15 +16,6 @@ exports.handler = function(cloudtrailEvent, context) { AwsCloudTrailListener.DATA_PIPELINE.name ]; - /* - ** An object where: - ** Keys: linked account ids - ** Values: the arn of the role configured for cross account access on that - ** account - */ - const rolesForCrossAccountAccess = { - '002': 'arn' - }; let listener = new AwsCloudTrailListener(cloudtrailEvent, context, enabledListeners); return listener.execute(); diff --git a/src/autotag_factory.js b/src/autotag_factory.js index c1f103a..d933e19 100644 --- a/src/autotag_factory.js +++ b/src/autotag_factory.js @@ -1,16 +1,16 @@ const _ = require('underscore'); -const AutotagDefaultWorker = require('./autotag_default_worker.js'); -const AutotagEC2Worker = require('./autotag_ec2_worker.js'); -const AutotagS3Worker = require('./autotag_s3_worker.js'); -const AutotagELBWorker = require('./autotag_elb_worker.js'); -const AutotagEBSWorker = require('./autotag_ebs_worker.js'); -const AutotagAutoscaleWorker = require('./autotag_autoscale_worker.js'); -const AutotagVPCWorker = require('./autotag_vpc_worker.js'); -const AutotagSubnetWorker = require('./autotag_subnet_worker.js'); -const AutotagInternetGatewayWorker = require('./autotag_internet_gateway_worker.js'); -const AutotagRDSWorker = require('./autotag_rds_worker.js'); -const AutotagEMRWorker = require('./autotag_emr_worker.js'); -const AutotagDataPipelineWorker = require('./autotag_data_pipeline_worker.js'); +const AutotagDefaultWorker = require('./workers/autotag_default_worker.js'); +const AutotagEC2Worker = require('./workers/autotag_ec2_worker.js'); +const AutotagS3Worker = require('./workers/autotag_s3_worker.js'); +const AutotagELBWorker = require('./workers/autotag_elb_worker.js'); +const AutotagEBSWorker = require('./workers/autotag_ebs_worker.js'); +const AutotagAutoscaleWorker = require('./workers/autotag_autoscale_worker.js'); +const AutotagVPCWorker = require('./workers/autotag_vpc_worker.js'); +const AutotagSubnetWorker = require('./workers/autotag_subnet_worker.js'); +const AutotagInternetGatewayWorker = require('./workers/autotag_internet_gateway_worker.js'); +const AutotagRDSWorker = require('./workers/autotag_rds_worker.js'); +const AutotagEMRWorker = require('./workers/autotag_emr_worker.js'); +const AutotagDataPipelineWorker = require('./workers/autotag_data_pipeline_worker.js'); const CONFIG = require('./cloud_trail_event_config'); let AutotagFactory = { diff --git a/src/workers/autotag_autoscale_worker.js b/src/workers/autotag_autoscale_worker.js index f4c9008..8c7b595 100644 --- a/src/workers/autotag_autoscale_worker.js +++ b/src/workers/autotag_autoscale_worker.js @@ -1,10 +1,10 @@ const AutotagDefaultWorker = require('./autotag_default_worker'); const AWS = require('aws-sdk'); +const co = require('co'); class AutotagAutoscaleWorker extends AutotagDefaultWorker { constructor(event) { super(event); - this.autoscaling = new AWS.AutoScaling({region: event.awsRegion}); } /* tagResource @@ -14,6 +14,14 @@ class AutotagAutoscaleWorker extends AutotagDefaultWorker { */ tagResource() { + let _this = this; + return co(function* () { + yield _this.assumeRole(AWS); + yield _this.tagAutoscalingGroup(); + }); + } + + tagAutoscalingGroup() { let _this = this; return new Promise(function(resolve, reject) { try { @@ -21,7 +29,8 @@ class AutotagAutoscaleWorker extends AutotagDefaultWorker { tagConfig.ResourceId = _this.getAutoscalingGroupName(); tagConfig.ResourceType = 'auto-scaling-group'; tagConfig.PropagateAtLaunch = true; - _this.autoscaling.createOrUpdateTags({ + let autoscaling = new AWS.AutoScaling({region: _this.event.awsRegion}); + autoscaling.createOrUpdateTags({ Tags: [ tagConfig ] diff --git a/src/workers/autotag_data_pipeline_worker.js b/src/workers/autotag_data_pipeline_worker.js index bb2531c..3e4c158 100644 --- a/src/workers/autotag_data_pipeline_worker.js +++ b/src/workers/autotag_data_pipeline_worker.js @@ -1,11 +1,11 @@ const AutotagDefaultWorker = require('./autotag_default_worker'); const AWS = require('aws-sdk'); +const co = require('co'); const _ = require('underscore'); class AutotagDataPipelineWorker extends AutotagDefaultWorker { constructor(event) { super(event); - this.dataPipeline = new AWS.DataPipeline({region: event.awsRegion}); } /* tagResource @@ -15,10 +15,19 @@ class AutotagDataPipelineWorker extends AutotagDefaultWorker { */ tagResource() { + let _this = this; + return co(function* () { + yield _this.assumeRole(AWS); + yield _this.tagDataPipelineResource(); + }); + } + + tagDataPipelineResource() { let _this = this; return new Promise(function(resolve, reject) { try { - _this.dataPipeline.addTags({ + let dataPipeline = new AWS.DataPipeline({region: _this.event.awsRegion}); + dataPipeline.addTags({ pipelineId: _this.getDataPipelineId(), tags: [ _this.getAutotagPair() diff --git a/src/workers/autotag_default_worker.js b/src/workers/autotag_default_worker.js index 5ae7da2..c3342c3 100644 --- a/src/workers/autotag_default_worker.js +++ b/src/workers/autotag_default_worker.js @@ -1,4 +1,6 @@ const AUTOTAG_TAG_NAME = 'AutoTag_Creator'; +const ROLE_PREFIX = 'arn:aws:iam::'; +const ROLE_SUFFIX = ':role/AutoTagRole'; class AutotagDefaultWorker { constructor(event) { @@ -22,6 +24,34 @@ class AutotagDefaultWorker { }); } + assumeRole(awsSdk) { + let _this = this; + return new Promise(function(resolve, reject) { + try { + awsSdk.config.region = 'us-east-1'; + let sts = new awsSdk.STS(); + sts.assumeRole({ + RoleArn: ROLE_PREFIX + _this.event.recipientAccountId + ROLE_SUFFIX, + RoleSessionName: 'AutoTag-' + (new Date()).getTime(), + DurationSeconds: 900 + }, function(err, data) { + if (err) { + reject(err); + } else { + awsSdk.config.credentials = { + accessKeyId: data.Credentials.AccessKeyId, + secretAccessKey: data.Credentials.SecretAccessKey, + sessionToken: data.Credentials.SessionToken + }; + resolve(data); + } + }); + } catch (err) { + reject(err); + } + }); + } + dumpEventInfo() { console.log('Event Name: ' + this.event.eventName); console.log('Event Type: ' + this.event.eventType); diff --git a/src/workers/autotag_ebs_worker.js b/src/workers/autotag_ebs_worker.js index 9655c48..3cafb06 100644 --- a/src/workers/autotag_ebs_worker.js +++ b/src/workers/autotag_ebs_worker.js @@ -1,5 +1,6 @@ const AutotagEC2Worker = require('./autotag_ec2_worker'); const AWS = require('aws-sdk'); +const co = require('co'); class AutotagEBSWorker extends AutotagEC2Worker { /* tagResource @@ -9,7 +10,11 @@ class AutotagEBSWorker extends AutotagEC2Worker { */ tagResource() { - return this.tagEC2Resources([this.getVolumeId()]); + let _this = this; + return co(function* () { + yield _this.assumeRole(AWS); + yield _this.tagEC2Resources([_this.getVolumeId()]); + }); } getVolumeId() { diff --git a/src/workers/autotag_ec2_worker.js b/src/workers/autotag_ec2_worker.js index 32d057e..ae45ba5 100644 --- a/src/workers/autotag_ec2_worker.js +++ b/src/workers/autotag_ec2_worker.js @@ -1,36 +1,41 @@ const AutotagDefaultWorker = require('./autotag_default_worker'); const AWS = require('aws-sdk'); +const co = require('co'); class AutotagEC2Worker extends AutotagDefaultWorker { constructor(event) { super(event); - this.ec2 = new AWS.EC2({region: event.awsRegion}); - } + /* tagResource ** method: tagResource ** - ** Do nothing + ** Tag the ec2 instance */ - tagResource() { - return this.tagEC2Resources([this.getInstanceId()]); + let _this = this; + return co(function* () { + yield _this.assumeRole(AWS); + yield _this.tagEC2Resources([_this.getInstanceId()]); + }); } tagEC2Resources(resources) { let _this = this; return new Promise(function(resolve, reject) { try { - _this.ec2.createTags({ + let ec2 = new AWS.EC2({region: _this.event.awsRegion}); + ec2.createTags({ Resources: resources, Tags: [ _this.getAutotagPair() ] }, function(err, res) { - if (err) + if (err) { reject(err); - else + } else { resolve(true); + } }); } catch(e) { reject(e); diff --git a/src/workers/autotag_elb_worker.js b/src/workers/autotag_elb_worker.js index d2f0ec3..ad86375 100644 --- a/src/workers/autotag_elb_worker.js +++ b/src/workers/autotag_elb_worker.js @@ -1,10 +1,10 @@ const AutotagDefaultWorker = require('./autotag_default_worker'); const AWS = require('aws-sdk'); +const co = require('co'); class AutotagELBWorker extends AutotagDefaultWorker { constructor(event) { super(event); - this.elb = new AWS.ELB({region: event.awsRegion}); } /* tagResource @@ -12,12 +12,20 @@ class AutotagELBWorker extends AutotagDefaultWorker { ** ** Add tag to elastic load balancer */ - tagResource() { + let _this = this; + return co(function* () { + yield _this.assumeRole(AWS); + yield _this.tagELBResource(); + }); + } + + tagELBResource() { let _this = this; return new Promise(function(resolve, reject) { try { - _this.elb.addTags({ + let elb = new AWS.ELB({region: _this.event.awsRegion}); + elb.addTags({ LoadBalancerNames: [ _this.getLoadBalancerName() ], diff --git a/src/workers/autotag_emr_worker.js b/src/workers/autotag_emr_worker.js index fe59159..f251f97 100644 --- a/src/workers/autotag_emr_worker.js +++ b/src/workers/autotag_emr_worker.js @@ -1,10 +1,10 @@ const AutotagDefaultWorker = require('./autotag_default_worker'); const AWS = require('aws-sdk'); +const co = require('co'); class AutotagEMRWorker extends AutotagDefaultWorker { constructor(event) { super(event); - this.emr = new AWS.EMR({region: event.awsRegion}); } /* tagResource @@ -14,10 +14,19 @@ class AutotagEMRWorker extends AutotagDefaultWorker { */ tagResource() { + let _this = this; + return co(function* () { + yield _this.assumeRole(AWS); + yield _this.tagEMRResource(); + }); + } + + tagEMRResource() { let _this = this; return new Promise(function(resolve, reject) { try { - _this.emr.addTags({ + let emr = new AWS.EMR({region: _this.event.awsRegion}); + emr.addTags({ ResourceId: _this.getEMRClusterId(), Tags: [ _this.getAutotagPair() diff --git a/src/workers/autotag_internet_gateway_worker.js b/src/workers/autotag_internet_gateway_worker.js index 66dd9a0..f7eadbb 100644 --- a/src/workers/autotag_internet_gateway_worker.js +++ b/src/workers/autotag_internet_gateway_worker.js @@ -1,5 +1,6 @@ const AutotagEC2Worker = require('./autotag_ec2_worker'); const AWS = require('aws-sdk'); +const co = require('co'); class AutotagInternetGatewayWorker extends AutotagEC2Worker { /* tagResource @@ -9,7 +10,11 @@ class AutotagInternetGatewayWorker extends AutotagEC2Worker { */ tagResource() { - return this.tagEC2Resources([this.getInternetGatewayId()]); + let _this = this; + return co(function* () { + yield _this.assumeRole(AWS); + yield _this.tagEC2Resources([_this.getInternetGatewayId()]); + }); } getInternetGatewayId() { diff --git a/src/workers/autotag_rds_worker.js b/src/workers/autotag_rds_worker.js index 0ec3c02..73cf865 100644 --- a/src/workers/autotag_rds_worker.js +++ b/src/workers/autotag_rds_worker.js @@ -1,10 +1,10 @@ const AutotagDefaultWorker = require('./autotag_default_worker'); const AWS = require('aws-sdk'); +const co = require('co'); class AutotagRDSWorker extends AutotagDefaultWorker { constructor(event) { super(event); - this.rds = new AWS.RDS({region: event.awsRegion}); } /* tagResource @@ -14,10 +14,19 @@ class AutotagRDSWorker extends AutotagDefaultWorker { */ tagResource() { + let _this = this; + return co(function* () { + yield _this.assumeRole(AWS); + yield _this.tagRDSResource(); + }); + } + + tagRDSResource() { let _this = this; return new Promise(function(resolve, reject) { try { - _this.rds.addTagsToResource({ + let rds = new AWS.RDS({region: _this.event.awsRegion}); + rds.addTagsToResource({ ResourceName: _this.getDbARN(), Tags: [ _this.getAutotagPair() diff --git a/src/workers/autotag_s3_worker.js b/src/workers/autotag_s3_worker.js index 985683b..bb29471 100644 --- a/src/workers/autotag_s3_worker.js +++ b/src/workers/autotag_s3_worker.js @@ -3,10 +3,8 @@ const AWS = require('aws-sdk'); const co = require('co'); class AutotagS3Worker extends AutotagDefaultWorker { - constructor(event) { super(event); - this.s3 = new AWS.S3({region: event.awsRegion}); } /* tagResource ** method: tagResource @@ -17,6 +15,8 @@ class AutotagS3Worker extends AutotagDefaultWorker { tagResource() { let _this = this; return co(function* () { + yield _this.assumeRole(AWS); + _this.s3 = new AWS.S3({region: _this.event.awsRegion}); let tags = yield _this.getExistingTags(); tags.push(_this.getAutotagPair()); yield _this.setTags(tags); diff --git a/src/workers/autotag_subnet_worker.js b/src/workers/autotag_subnet_worker.js index 6a01375..d076d9a 100644 --- a/src/workers/autotag_subnet_worker.js +++ b/src/workers/autotag_subnet_worker.js @@ -1,5 +1,6 @@ const AutotagEC2Worker = require('./autotag_ec2_worker'); const AWS = require('aws-sdk'); +const co = require('co'); class AutotagSubnetWorker extends AutotagEC2Worker { /* tagResource @@ -9,7 +10,11 @@ class AutotagSubnetWorker extends AutotagEC2Worker { */ tagResource() { - return this.tagEC2Resources([this.getSubnetId()]); + let _this = this; + return co(function* () { + yield _this.assumeRole(AWS); + yield _this.tagEC2Resources([_this.getSubnetId()]); + }); } getSubnetId() { diff --git a/src/workers/autotag_vpc_worker.js b/src/workers/autotag_vpc_worker.js index a5f77a6..a98407c 100644 --- a/src/workers/autotag_vpc_worker.js +++ b/src/workers/autotag_vpc_worker.js @@ -1,5 +1,6 @@ const AutotagEC2Worker = require('./autotag_ec2_worker'); const AWS = require('aws-sdk'); +const co = require('co'); class AutotagVPCWorker extends AutotagEC2Worker { /* tagResource @@ -9,7 +10,11 @@ class AutotagVPCWorker extends AutotagEC2Worker { */ tagResource() { - return this.tagEC2Resources([this.getVPCId()]); + let _this = this; + return co(function* () { + yield _this.assumeRole(AWS); + yield _this.tagEC2Resources([_this.getVPCId()]); + }); } getVPCId() { From 0bce3c260d019d0d8f77d30673a5cd93a0ef4601 Mon Sep 17 00:00:00 2001 From: Elliott Spira Date: Tue, 6 Oct 2015 10:06:33 +1100 Subject: [PATCH 5/7] updating auto-tag to support cross-account tagging --- src/workers/autotag_autoscale_worker.js | 9 ++++++--- src/workers/autotag_data_pipeline_worker.js | 9 ++++++--- src/workers/autotag_default_worker.js | 11 ++++++----- src/workers/autotag_ebs_worker.js | 6 +++++- src/workers/autotag_ec2_worker.js | 9 ++++++--- src/workers/autotag_elb_worker.js | 9 ++++++--- src/workers/autotag_emr_worker.js | 9 ++++++--- src/workers/autotag_internet_gateway_worker.js | 6 +++++- src/workers/autotag_rds_worker.js | 9 ++++++--- src/workers/autotag_s3_worker.js | 7 +++++-- src/workers/autotag_subnet_worker.js | 6 +++++- src/workers/autotag_vpc_worker.js | 6 +++++- 12 files changed, 67 insertions(+), 29 deletions(-) diff --git a/src/workers/autotag_autoscale_worker.js b/src/workers/autotag_autoscale_worker.js index 8c7b595..95e38ae 100644 --- a/src/workers/autotag_autoscale_worker.js +++ b/src/workers/autotag_autoscale_worker.js @@ -16,7 +16,11 @@ class AutotagAutoscaleWorker extends AutotagDefaultWorker { tagResource() { let _this = this; return co(function* () { - yield _this.assumeRole(AWS); + let credentials = yield _this.assumeRole(); + _this.autoscaling = new AWS.AutoScaling({ + region: _this.event.awsRegion, + credentials: credentials + }); yield _this.tagAutoscalingGroup(); }); } @@ -29,8 +33,7 @@ class AutotagAutoscaleWorker extends AutotagDefaultWorker { tagConfig.ResourceId = _this.getAutoscalingGroupName(); tagConfig.ResourceType = 'auto-scaling-group'; tagConfig.PropagateAtLaunch = true; - let autoscaling = new AWS.AutoScaling({region: _this.event.awsRegion}); - autoscaling.createOrUpdateTags({ + _this.autoscaling.createOrUpdateTags({ Tags: [ tagConfig ] diff --git a/src/workers/autotag_data_pipeline_worker.js b/src/workers/autotag_data_pipeline_worker.js index 3e4c158..4012f83 100644 --- a/src/workers/autotag_data_pipeline_worker.js +++ b/src/workers/autotag_data_pipeline_worker.js @@ -17,7 +17,11 @@ class AutotagDataPipelineWorker extends AutotagDefaultWorker { tagResource() { let _this = this; return co(function* () { - yield _this.assumeRole(AWS); + let credentials = yield _this.assumeRole(); + _this.dataPipeline = new AWS.DataPipeline({ + region: _this.event.awsRegion, + credentials: credentials + }); yield _this.tagDataPipelineResource(); }); } @@ -26,8 +30,7 @@ class AutotagDataPipelineWorker extends AutotagDefaultWorker { let _this = this; return new Promise(function(resolve, reject) { try { - let dataPipeline = new AWS.DataPipeline({region: _this.event.awsRegion}); - dataPipeline.addTags({ + _this.dataPipeline.addTags({ pipelineId: _this.getDataPipelineId(), tags: [ _this.getAutotagPair() diff --git a/src/workers/autotag_default_worker.js b/src/workers/autotag_default_worker.js index c3342c3..9799f7c 100644 --- a/src/workers/autotag_default_worker.js +++ b/src/workers/autotag_default_worker.js @@ -1,3 +1,4 @@ +const AWS = require('aws-sdk'); const AUTOTAG_TAG_NAME = 'AutoTag_Creator'; const ROLE_PREFIX = 'arn:aws:iam::'; const ROLE_SUFFIX = ':role/AutoTagRole'; @@ -24,12 +25,12 @@ class AutotagDefaultWorker { }); } - assumeRole(awsSdk) { + assumeRole() { let _this = this; return new Promise(function(resolve, reject) { try { - awsSdk.config.region = 'us-east-1'; - let sts = new awsSdk.STS(); + AWS.config.region = 'us-east-1'; + let sts = new AWS.STS(); sts.assumeRole({ RoleArn: ROLE_PREFIX + _this.event.recipientAccountId + ROLE_SUFFIX, RoleSessionName: 'AutoTag-' + (new Date()).getTime(), @@ -38,12 +39,12 @@ class AutotagDefaultWorker { if (err) { reject(err); } else { - awsSdk.config.credentials = { + let credentials = { accessKeyId: data.Credentials.AccessKeyId, secretAccessKey: data.Credentials.SecretAccessKey, sessionToken: data.Credentials.SessionToken }; - resolve(data); + resolve(credentials); } }); } catch (err) { diff --git a/src/workers/autotag_ebs_worker.js b/src/workers/autotag_ebs_worker.js index 3cafb06..abc0e6f 100644 --- a/src/workers/autotag_ebs_worker.js +++ b/src/workers/autotag_ebs_worker.js @@ -12,7 +12,11 @@ class AutotagEBSWorker extends AutotagEC2Worker { tagResource() { let _this = this; return co(function* () { - yield _this.assumeRole(AWS); + let credentials = yield _this.assumeRole(); + _this.ec2 = new AWS.EC2({ + region: _this.event.awsRegion, + credentials: credentials + }); yield _this.tagEC2Resources([_this.getVolumeId()]); }); } diff --git a/src/workers/autotag_ec2_worker.js b/src/workers/autotag_ec2_worker.js index ae45ba5..633df33 100644 --- a/src/workers/autotag_ec2_worker.js +++ b/src/workers/autotag_ec2_worker.js @@ -15,7 +15,11 @@ class AutotagEC2Worker extends AutotagDefaultWorker { tagResource() { let _this = this; return co(function* () { - yield _this.assumeRole(AWS); + let credentials = yield _this.assumeRole(); + _this.ec2 = new AWS.EC2({ + region: _this.event.awsRegion, + credentials: credentials + }); yield _this.tagEC2Resources([_this.getInstanceId()]); }); } @@ -24,8 +28,7 @@ class AutotagEC2Worker extends AutotagDefaultWorker { let _this = this; return new Promise(function(resolve, reject) { try { - let ec2 = new AWS.EC2({region: _this.event.awsRegion}); - ec2.createTags({ + _this.ec2.createTags({ Resources: resources, Tags: [ _this.getAutotagPair() diff --git a/src/workers/autotag_elb_worker.js b/src/workers/autotag_elb_worker.js index ad86375..10c79e6 100644 --- a/src/workers/autotag_elb_worker.js +++ b/src/workers/autotag_elb_worker.js @@ -15,7 +15,11 @@ class AutotagELBWorker extends AutotagDefaultWorker { tagResource() { let _this = this; return co(function* () { - yield _this.assumeRole(AWS); + let credentials = yield _this.assumeRole(); + _this.elb = new AWS.ELB({ + region: _this.event.awsRegion, + credentials: credentials + }); yield _this.tagELBResource(); }); } @@ -24,8 +28,7 @@ class AutotagELBWorker extends AutotagDefaultWorker { let _this = this; return new Promise(function(resolve, reject) { try { - let elb = new AWS.ELB({region: _this.event.awsRegion}); - elb.addTags({ + _this.elb.addTags({ LoadBalancerNames: [ _this.getLoadBalancerName() ], diff --git a/src/workers/autotag_emr_worker.js b/src/workers/autotag_emr_worker.js index f251f97..1667ff6 100644 --- a/src/workers/autotag_emr_worker.js +++ b/src/workers/autotag_emr_worker.js @@ -16,7 +16,11 @@ class AutotagEMRWorker extends AutotagDefaultWorker { tagResource() { let _this = this; return co(function* () { - yield _this.assumeRole(AWS); + let credentials = yield _this.assumeRole(); + _this.emr = new AWS.EMR({ + region: _this.event.awsRegion, + credentials: credentials + }); yield _this.tagEMRResource(); }); } @@ -25,8 +29,7 @@ class AutotagEMRWorker extends AutotagDefaultWorker { let _this = this; return new Promise(function(resolve, reject) { try { - let emr = new AWS.EMR({region: _this.event.awsRegion}); - emr.addTags({ + _this.emr.addTags({ ResourceId: _this.getEMRClusterId(), Tags: [ _this.getAutotagPair() diff --git a/src/workers/autotag_internet_gateway_worker.js b/src/workers/autotag_internet_gateway_worker.js index f7eadbb..1aac071 100644 --- a/src/workers/autotag_internet_gateway_worker.js +++ b/src/workers/autotag_internet_gateway_worker.js @@ -12,7 +12,11 @@ class AutotagInternetGatewayWorker extends AutotagEC2Worker { tagResource() { let _this = this; return co(function* () { - yield _this.assumeRole(AWS); + let credentials = yield _this.assumeRole(); + _this.ec2 = new AWS.EC2({ + region: _this.event.awsRegion, + credentials: credentials + }); yield _this.tagEC2Resources([_this.getInternetGatewayId()]); }); } diff --git a/src/workers/autotag_rds_worker.js b/src/workers/autotag_rds_worker.js index 73cf865..c181c50 100644 --- a/src/workers/autotag_rds_worker.js +++ b/src/workers/autotag_rds_worker.js @@ -16,7 +16,11 @@ class AutotagRDSWorker extends AutotagDefaultWorker { tagResource() { let _this = this; return co(function* () { - yield _this.assumeRole(AWS); + let credentials = yield _this.assumeRole(); + _this.rds = new AWS.RDS({ + region: _this.event.awsRegion, + credentials: credentials + }); yield _this.tagRDSResource(); }); } @@ -25,8 +29,7 @@ class AutotagRDSWorker extends AutotagDefaultWorker { let _this = this; return new Promise(function(resolve, reject) { try { - let rds = new AWS.RDS({region: _this.event.awsRegion}); - rds.addTagsToResource({ + _this.rds.addTagsToResource({ ResourceName: _this.getDbARN(), Tags: [ _this.getAutotagPair() diff --git a/src/workers/autotag_s3_worker.js b/src/workers/autotag_s3_worker.js index bb29471..16f9027 100644 --- a/src/workers/autotag_s3_worker.js +++ b/src/workers/autotag_s3_worker.js @@ -15,8 +15,11 @@ class AutotagS3Worker extends AutotagDefaultWorker { tagResource() { let _this = this; return co(function* () { - yield _this.assumeRole(AWS); - _this.s3 = new AWS.S3({region: _this.event.awsRegion}); + let credentials = yield _this.assumeRole(); + _this.s3 = new AWS.S3({ + region: _this.event.awsRegion, + credentials: credentials + }); let tags = yield _this.getExistingTags(); tags.push(_this.getAutotagPair()); yield _this.setTags(tags); diff --git a/src/workers/autotag_subnet_worker.js b/src/workers/autotag_subnet_worker.js index d076d9a..00c7b10 100644 --- a/src/workers/autotag_subnet_worker.js +++ b/src/workers/autotag_subnet_worker.js @@ -12,7 +12,11 @@ class AutotagSubnetWorker extends AutotagEC2Worker { tagResource() { let _this = this; return co(function* () { - yield _this.assumeRole(AWS); + let credentials = yield _this.assumeRole(); + _this.ec2 = new AWS.EC2({ + region: _this.event.awsRegion, + credentials: credentials + }); yield _this.tagEC2Resources([_this.getSubnetId()]); }); } diff --git a/src/workers/autotag_vpc_worker.js b/src/workers/autotag_vpc_worker.js index a98407c..cbd43e9 100644 --- a/src/workers/autotag_vpc_worker.js +++ b/src/workers/autotag_vpc_worker.js @@ -12,7 +12,11 @@ class AutotagVPCWorker extends AutotagEC2Worker { tagResource() { let _this = this; return co(function* () { - yield _this.assumeRole(AWS); + let credentials = yield _this.assumeRole(); + _this.ec2 = new AWS.EC2({ + region: _this.event.awsRegion, + credentials: credentials + }); yield _this.tagEC2Resources([_this.getVPCId()]); }); } From 1077239af5cfb0516763ec63ef1c50bd054ba1df Mon Sep 17 00:00:00 2001 From: Elliott Spira Date: Tue, 6 Oct 2015 10:07:17 +1100 Subject: [PATCH 6/7] adding instruction to increase the timeout on the lambda function --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bbe7842..f0979a5 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ More [documentation on creating a Trail](https://docs.aws.amazon.com/awscloudtra 4. Select 'Node.js' as the Runtime 5. Upload the [latest release's zip file](https://github.com/GorillaStack/auto-tag/releases) 6. Under 'Handler' add 'autotag.handler' +7. Under 'Advanced settings' set the Timeout to 30s More [documentation on Lambda](https://docs.aws.amazon.com/lambda/latest/dg/getting-started.html) From da0326eaf9f1b3f43f7939c610052e5b93467971 Mon Sep 17 00:00:00 2001 From: Elliott Spira Date: Tue, 6 Oct 2015 10:58:27 +1100 Subject: [PATCH 7/7] adding script for testing creation of resources --- .gitignore | 3 +++ test_create_instances.sh | 58 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 test_create_instances.sh diff --git a/.gitignore b/.gitignore index 6841c51..b4bfe08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ lib/ *.bak +*.sh +sample*.js +!test_create_instances.sh diff --git a/test_create_instances.sh b/test_create_instances.sh new file mode 100644 index 0000000..9a5f33c --- /dev/null +++ b/test_create_instances.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +##### NOTE +### Requirements: +### - AWS CLI +### - jsawk +### - key-pair 'test' + +# If you need, set your default aws profile to create the resources on the right account +##export AWS_DEFAULT_PROFILE= + + +## Create EC2 instances +echo Creating EC2 instance +INSTANCE_ID=`aws --region ap-northeast-1 ec2 run-instances --image-id ami-936d9d93 --instance-type t2.micro --key-name test | jsawk "return this.Instances[0].InstanceId;"` +echo INSTANCE_ID=$INSTANCE_ID + +## Create S3 bucket +echo Creating S3 bucket +aws --region ap-northeast-1 s3 mb s3://test-bucket-gs-autotag-1 + +# ## Create ELB +echo Creating ELB +aws --region ap-northeast-1 elb create-load-balancer \ + --load-balancer-name testLoadBalancer \ + --availability-zones ap-northeast-1a ap-northeast-1c \ + --listeners Protocol=HTTP,LoadBalancerPort=80,InstanceProtocol=HTTP,InstancePort=80 + + +## Create RDS bucket +echo Creating RDS instance +aws --region ap-northeast-1 rds create-db-instance \ + --db-instance-identifier mytestrdsinstance \ + --db-instance-class db.t1.micro \ + --allocated-storage 5 \ + --engine MySQL \ + --master-username testtest \ + --master-user-password testtest + +## Create VPC +echo Creating VPC +VPC_ID=`aws --region ap-northeast-1 ec2 create-vpc --cidr-block 10.0.0.0/16 | jsawk "return this.Vpc.VpcId;"` +echo VPC_ID=$VPC_ID + +## Create VPC Subnet +echo Creating VPC Subnet +SUBNET_ID=`aws --region ap-northeast-1 ec2 create-subnet --vpc-id $VPC_ID --cidr-block 10.0.12.0/24 | jsawk "return this.Subnet.SubnetId;"` +echo SUBNET_ID=$SUBNET_ID + +## Create Internet Gateway for VPC +echo Creating Internet Gateway +INTERNET_GATEWAY_ID=`aws --region ap-northeast-1 ec2 create-internet-gateway | jsawk "return this.InternetGateway.InternetGatewayId;"` +echo INTERNET_GATEWAY_ID=$INTERNET_GATEWAY_ID + +## Create Elastic Map Reduce cluster +echo Creating EMR cluster +CLUSTER_ID`aws --region ap-northeast-1 emr create-cluster --release-label emr-4.0.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m1.medium InstanceGroupType=CORE,InstanceCount=2,InstanceType=m1.medium --use-default-roles | jsawk "return this.ClusterId;"` +echo CLUSTER_ID=$CLUSTER_ID