From 92db549586ba16973a930877a0bb9f0514b1ab9f Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Fri, 12 Jun 2020 13:37:52 -0700 Subject: [PATCH] add option to re-encrypt snapshot also fix new snapshot mode also fix empty db name handling --- README.md | 7 ++++++- architecture.svg | 2 +- architecture.xml | 2 +- cfm.py | 31 +++++++++++++++++++++++++++++-- gen-cfm.py | 15 ++++++++++----- iam.py | 17 ++++++++++++++++- lambda/rds.py | 24 ++++++++++++++++++++++-- 7 files changed, 85 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 528115d..7922849 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ The step function does the following to create the snapshot: 1. Get a snapshot of the given database by either: * Finding the latest snapshot for the given database * Creating and waiting for a new fresh snapshot + 1. Re-encrypt snapshot if KMS key is supplied 1. Create a temporary database from the snapshot 1. Wait for the database to be ready 1. Reset the master password on the temporary database to a random password @@ -50,12 +51,16 @@ and deploy it as you normally would from the terminal or in the AWS CloudFormati | Sanitization SQL statements | SQL statement used to sanitize the temporary database. Use this to remove any data you don't want in the final snapshot, or the trim the data for size. You can separate multiple statements with a semicolon. | | List of AWS accounts to share snapshot with | A comma-separated list of AWS accounts to share the final snapshot with. These accounts will see the snapshot under the "Shared with me" tab in the RDS console. | | Snapshot name format | Final snapshot name format. A new snapshot will be created periodically, so this should contain the date to provide uniqueness. Make sure it follows the [naming rules of AWS](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html). | +| KMS key id | Re-encrypt the snapshot with a different key. If left empty, it will be encrypted with the same key used for the original database. | | Network | Network parameters are required to create the temporary database. Make sure to select at least two subnets that are associated with the selected VPC | ### Encryption The new snapshot will be encrypted with the same key used by the original database. If the original database wasn't -encrypted, the snapshot won't be encrypted either. +encrypted, the snapshot won't be encrypted either. To add another step that changes the key, use the KMS key parameter. + +See [AWS documentation](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ShareSnapshot.html) for instructions +on giving other accounts access to the key. ### Known Limitations diff --git a/architecture.svg b/architecture.svg index 9d21691..2747724 100644 --- a/architecture.svg +++ b/architecture.svg @@ -1,3 +1,3 @@ -
Step Function
Step Function
CloudWatch Timer
CloudWatch Ti...
Find/take snapshot
Find/take sna...
Create temp database
Create temp d...
Run SQL to sanitize
Run SQL to sa...
Take snapshot
Take snapshot
Share snapshot
Share snapshotViewer does not support full SVG 1.1 \ No newline at end of file +
Step Function
Step Function
CloudWatch Timer
CloudWatch Ti...
Find/take snapshot
Find/take sna...
Create temp database
Create temp d...
Run SQL to sanitize
Run SQL to sa...
Take snapshot
Take snapshot
Share snapshot
Share snapshot
Re-encrypt
w/ shared key
Re-encrypt...Viewer does not support full SVG 1.1 \ No newline at end of file diff --git a/architecture.xml b/architecture.xml index 98e4255..aacdd6d 100644 --- a/architecture.xml +++ b/architecture.xml @@ -1 +1 @@ -7Vnbbts4EP0aPzbQXeljLcXdAimwrQu0fTJoiZa4oUgtNYrtfn2HEmVbkpO4gDdOsHEChHM4ImfIOYdUPHGjYvNRkTL/LFPKJ46VbiZuPHEcxwoc/KORbYvYtu+3SKZYarA9MGe/qAEtg9YspVXPEaTkwMo+mEghaAI9jCgl1323leT9WUuS0REwTwgfo99ZCnmLXvvWHv+LsizvZrYt01OQztkAVU5SuT6A3JuJGykpoW0Vm4hyvXrdurTPzR7o3QWmqIBTHijy6N91BTLPo/wDy5xPQfjxndmee8Jrk/AcaInIrBYJMClM7LDtFqSUTECzqP4Uf3HOyJr42BNp68rxB8DQDvuAPbb0GH1gaId9wB4Obw/mt4cBHgAjqze8NZjfOggQf92prIEzQaNd+VkIZoqkDLclklwqxIQUuHrTHAqOlo3Ndc6AzkuS6FVdI3cQW0kBhgC209lm4fWoWEClbhebTJPtiqwr7ypTsi6bKT8hBY72LrC5qHBbFyuzqdViLdXdiutynFag5B3tQp04bhQ7TuDpABjngxTuqQKG7PjAWaZnA6knJ8bidAV6RMyLiey2sWLXMrkcmyIlVU5Tk6CpRZyCbh4scntHHRQdKgsKaosu3QOe1z7S6Y0h33rP3cALWiw/5K1nHInRi2w39J5S2DCs+gOG2faIQjRFiTGmVJDLTArCb/boFLdNpLt12fvcSr3eTQX9QwG2plxIDbJfX7hcavvDPN8YP7WBlWzMeHPYGW97W6ADfHwDMB9Zq4Q+krhr1JqojMJTEjTeUEU5AXbfj+Psu+OO9G/iBFwX8RIbmW5EXNbpdwJJjo7fWEFV54Iz7rxGm3xUGAY0wJ+ZDvYhwTgkILpPI9v1gxFljXOPSh1Pb8mS8r9lxRotd+OlBJDFk0ROMBbMs1dST8kTqco20RXb6DiO6xW9x6EXgMu4WJKq8TsD7bsz1rDes0esD6/HpO+ws1eV41yC82fkru2cSF47vCR7uzAfo++MiRQHB3JHdeaClFWOV67/nMLYNwuvbyzvoC9miiaGiULv8JjjseVHuKhHjuVV83k1RFe0rbL2WjJF89gFhZNimZIznf1hXwXc4MIq4L12FQhPPcLdi6pAeMIhrigBLQBAC/16kxIg+vx5Fh3w4hA7/0wH8ONN3/9vdECl1XlEwLVemAgEr1wEnJPv8f4lRcA54Sb/tRboMf9y2/wTSSdPBPLk1/OowNtt4EkVWGGNaZk+ixJ47gtTguvXrgT+qUpw0ZeCLszHlODbBd4G3m4Bz3cL8IMXxv0TrqjzHNN7e0V9cUV5zlfU4P1z1SWa+6+Wmr6Db+jcm98= \ No newline at end of file +7Vldc+I2FP01PG7G3yaPix3SzKTbdtmZtE+MsIWtRpZcWQTYX98rWwYbO0C2LJBpCDPRPbrWx9U9R5IZ2EG2uhcoT3/lMaYDy4hXAzscWJZleBb8U8i6QkzTdSskESTW2BaYkO9Yg4ZGFyTGRctRck4lydtgxBnDkWxhSAi+bLvNOW33mqMEd4BJhGgXfSKxTCt06Bpb/BdMkrTu2TR0TYZqZw0UKYr5sgHZdwM7EJzLqpStAkxV9Oq4VM+NX6ndDExgJo95IEuDf5aF5GkapJ9JYj14/v0nvTwviC70hCcS54CMFyyShDM9drmuA5JzwmQZVHcEX+gzMAYu1ATKurHcHWDX9tuA2bVUG21g1/bbgLnbvLnTv7k7wAbQsVrNGzv9G40Bwtce8YWkhOFgk34GgIlAMYFlCTjlAjDGGURvlMqMgmVCcZkSiSc5ilRUl8AdwOacSU0A06ptHXjVKiRQrsrZKlFku0HLwrlJBF/kZZcPQIHe2ikUpwUs63SuF7WYLrl4nlOVjqNCCv6M66EOLDsILctz1AAIpTtTeMFCEmDHZ0oS1ZvkqnOkLYrnUrUI8yIseSyt0Db0XPq6iFGR4lhPsJvROslVr3jVgHSG32OeYSnW4FLXOk71SK03mnzLLXd9x6uwtMlbRzsirRfJpuktpaCgWfUGhplmh0I4BonRJhcy5QlniN5t0REsG4s3cdn6PHIV7zKD/sZSrnW6oIXk7fyCCIr1n/r50vhLGZDJ2gxXzcpwfWgJCr4QEd4zT1uLMxIJlocVR8Vg74IKTJEkL20ZPvnq2B39G1geVUk8g0KiCgHli/gJySgFx28kw6J2gR43Xp1F7hWGHRrA31gN9jXBaBIQ3EeBabteh7LauUWlmqePaIbp77wgpZbb4YxLybODRI5gLDDPVkodkidU5NVE52SlxtGvV/gFmp5KCON0horS7wS0r/dYzXrH7LJ+2CV9jf2XrHLzL4mxevqyttNoyB/Ib5Pwa51V56V8L69+jMum1SVz70Qvyt16lPvIOyYshsYlesZq4gzlRQoHrp9OYKgb+8M7w2nUhUTgSPOQqQXuMjw03MD0+zblefl5NzQXuEqy6lAyArPveEJRNovRiXZ+v60Btnc2Dejf55zLicAPcd4/dgO3r2oHr8e9dwsXGEklABJn6nITI4nU7nMWHXBCHyrfpgPwcUa3/xsdEHFxGhFwzOF1iYD3vkTAOvoU716VCFhHnOO/Lhh4TP54LF8hqWggBjz5fh4V+DgNHFSBOSSdkumTKIHrWNelBMN3pgTusUrgX5cSuIeV4NsFbgMfp4DznQI8z7su7h9xRJ2kMOOPK+rVJeUpr6j+7cWuqL1vby54Q33ra6rX3z4dsUVVV8Qz7Ej7Rrn3aIo/YRaJdb6h/UzUdUvVe/nroYCwW8YzXnedPraw61CL5hbW0YWeJH/9hfZt+3esnycVYG5/gy7rGj/l23f/Ag== \ No newline at end of file diff --git a/cfm.py b/cfm.py index 7000fca..1da261e 100644 --- a/cfm.py +++ b/cfm.py @@ -254,6 +254,7 @@ def add_task_state(state_name: str, task_arn: str, next_state_name: str, catch_s "new_snapshot": "${NewSnapshot}", "shared_accounts": ["${ShareAccountsJoined}"], "snapshot_format": "${SnapshotFormat}", + "kms": "${KMS}", } # TODO create add_choice() @@ -275,8 +276,24 @@ def add_task_state(state_name: str, task_arn: str, next_state_name: str, catch_s } add_state("TakeSnapshot", "WaitForSnapshot", catch_state="ErrorCleanup") - add_waiting_state("WaitForSnapshot", "CreateTempDatabase", catch_state="ErrorCleanup") - add_state("FindLatestSnapshot", "CreateTempDatabase", catch_state="ErrorCleanup") + add_waiting_state("WaitForSnapshot", "ShouldEncrypt", catch_state="ErrorCleanup") + add_state("FindLatestSnapshot", "ShouldEncrypt", catch_state="ErrorCleanup") + + states["ShouldEncrypt"] = { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.kms", + "StringEquals": "", + "Next": "CreateTempDatabase", + }, + ], + "Default": "Encrypt", + } + + add_state("Encrypt", "WaitForEncrypt", catch_state="ErrorCleanup") + add_waiting_state("WaitForEncrypt", "CreateTempDatabase", catch_state="ErrorCleanup") + add_state("CreateTempDatabase", "WaitForTempDatabase", catch_state="ErrorCleanup") add_waiting_state("WaitForTempDatabase", "SetTempPassword", catch_state="ErrorCleanup") add_state("SetTempPassword", "WaitForPassword", catch_state="ErrorCleanup") @@ -375,10 +392,20 @@ def generate_main_template(): "Options", Type="List", Default="") add_parameter(template, "SnapshotFormat", "Snapshot name format using Python .format() function", "Options", Type="String", Default="{database_identifier:.42}-sanitized-{date:%Y-%m-%d}") + add_parameter(template, "KMS", "KMS key id to re-encrypt snapshots (leave empty to not encrypt)", + "Options", Type="String", Default="") add_parameter(template, "VpcId", "VPC for temporary database", "Network", Type="AWS::EC2::VPC::Id") add_parameter(template, "SubnetIds", "Subnets for temporary database (at least two)", "Network", Type="List") + template.add_condition( + "KmsEmpty", + troposphere.Equals( + troposphere.Ref("KMS"), + "" + ) + ) + troposphere.rds.DBSubnetGroup( "SubnetGroup", template, DBSubnetGroupDescription="Temporary database used for RDS-sanitize-snapshots", diff --git a/gen-cfm.py b/gen-cfm.py index ac0bde6..77e3cfc 100644 --- a/gen-cfm.py +++ b/gen-cfm.py @@ -45,7 +45,8 @@ def validate_subnets(ctx, param, value): @click.option("--share-account", help="AWS account identifiers to share snapshots with", multiple=True) @click.option("--new-snapshot", help="Take a new snapshot instead of using the latest available", is_flag=True) @click.option("--snapshot_format", help="Snapshot name snapshot_format") -def deploy(profile, stack_name, database, vpc, subnet, sql, share_account, new_snapshot, snapshot_format): +@click.option("--kms", help="KMS ARN to encrypt snapshots") +def deploy(profile, stack_name, database, vpc, subnet, sql, share_account, new_snapshot, snapshot_format, kms): # this is more for testing and not really for user consumption... stack_template = generate_main_template() @@ -85,10 +86,14 @@ def deploy(profile, stack_name, database, vpc, subnet, sql, share_account, new_s if snapshot_format: parameters.append({ - { - "ParameterKey": "SnapshotFormat", - "ParameterValue": snapshot_format, - } + "ParameterKey": "SnapshotFormat", + "ParameterValue": snapshot_format, + }) + + if kms: + parameters.append({ + "ParameterKey": "KMS", + "ParameterValue": kms, }) if _stack_exists(cf, stack_name): diff --git a/iam.py b/iam.py index ac78daf..2644a38 100644 --- a/iam.py +++ b/iam.py @@ -15,6 +15,7 @@ "Sid": "OriginalDB", "Effect": "Allow", "Action": [ + troposphere.If("KmsEmpty", troposphere.NoValue, "rds:CopyDBSnapshot"), "rds:DescribeDBInstances", "rds:DescribeDBSnapshots", "rds:CreateDBSnapshot", @@ -22,6 +23,7 @@ "Resource": [ troposphere.Sub( "arn:${AWS::Partition}:rds:${AWS::Region}:${AWS::AccountId}:db:${Db}"), + # TODO only allow access to all snapshots when using latest snapshot option troposphere.Sub( "arn:${AWS::Partition}:rds:${AWS::Region}:${AWS::AccountId}:snapshot:*"), ] @@ -70,7 +72,20 @@ ] } } - } + }, + troposphere.If( + "KmsEmpty", + troposphere.NoValue, + { + "Sid": "Copy", + "Effect": "Allow", + "Action": [ + "kms:CreateGrant", + "kms:DescribeKey", + ], + "Resource": troposphere.Ref("KMS"), + } + ), ] } ) diff --git a/lambda/rds.py b/lambda/rds.py index 6416cfb..af05704 100644 --- a/lambda/rds.py +++ b/lambda/rds.py @@ -49,7 +49,8 @@ def decorator(f): def initialize(state, uid): orig_db = rds_client.describe_db_instances(DBInstanceIdentifier=state["db_identifier"])["DBInstances"][0] state["engine"] = orig_db["Engine"] - state["temporary_snapshot_id"] = state["db_identifier"][:55] + "-" + secrets.token_hex(5) + state["temp_snapshot_id"] = state["db_identifier"][:55] + "-" + secrets.token_hex(5) + state["temp_snapshot_id2"] = state["db_identifier"][:55] + "-" + secrets.token_hex(5) state["temp_db_id"] = state["db_identifier"][:55] + "-" + secrets.token_hex(5) state["target_snapshot_id"] = state["db_identifier"][:55] + "-" + secrets.token_hex(5) tsid = state["target_snapshot_id"] = state["snapshot_format"].format( @@ -61,6 +62,11 @@ def initialize(state, uid): # https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html raise ValueError(f"Invalid snapshot id generated from format - {tsid}") + if state["kms"] and orig_db["DBInstanceClass"] in ["db.m1.small", "db.m1.medium", "db.m1.large", "db.m1.xlarge", + "db.m2.xlarge", "db.m2.2xlarge", "db.m2.4xlarge", "db.t2.micro"]: + # https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Overview.Encryption.html + raise ValueError("Instance type doesn't support encryption.") + @state_function("FindLatestSnapshot") def find_latest_snapshot(state, uid): @@ -98,6 +104,7 @@ def take_final_snapshot(state, uid): @state_function("WaitForSnapshot") @state_function("WaitForFinalSnapshot") +@state_function("WaitForEncrypt") def wait_for_snapshot(state, uid): snapshot = rds_client.describe_db_snapshots(DBSnapshotIdentifier=state["snapshot_id"])["DBSnapshots"][0] status = snapshot["Status"] @@ -107,6 +114,19 @@ def wait_for_snapshot(state, uid): raise NotReady() +@state_function("Encrypt") +def encrypt(state, uid): + old_snapshot_id = state["snapshot_id"] + state["snapshot_id"] = state["temp_snapshot_id2"] + + rds_client.copy_db_snapshot( + SourceDBSnapshotIdentifier=old_snapshot_id, + TargetDBSnapshotIdentifier=state["snapshot_id"], + KmsKeyId=state["kms"], + Tags=_tags(uid), + ) + + @state_function("CreateTempDatabase") def create_temp_database(state, uid): rds_client.restore_db_instance_from_db_snapshot( @@ -141,7 +161,7 @@ def set_temp_password(state, uid): "port": str(db["Endpoint"]["Port"]), "user": db["MasterUsername"], "password": secrets.token_hex(32), - "database": db["DBName"], + "database": db.get("DBName", ""), } rds_client.modify_db_instance(