Added support for expiring backups of deleted resources.

[Issue(s) #7]

botocross/ec2/__init__.py

@@ -57,15 +57,15 @@ def create_snapshots(ec2, volumes, backup_set, description):
         tags = {TAG_NAME: name, TAG_BACKUP_POLICY: backup_set}
         ec2.create_tags([response.id], tags)
 
-def expire_snapshots(ec2, volumes, backup_set, backup_retention, no_origin_safeguard=False):
+def expire_snapshots(ec2, volume_ids, backup_set, backup_retention, no_origin_safeguard=False):
     log = ec2_log
-    for volume in volumes:
-        snapshot_filters = {"volume-id": volume.id, "tag:" + TAG_BACKUP_POLICY: backup_set}
+    for volume_id in volume_ids:
+        snapshot_filters = {"volume-id": volume_id, "tag:" + TAG_BACKUP_POLICY: backup_set}
         if not no_origin_safeguard:
-            snapshot_filters['description'] = CREATED_BY_BOTO_EBS_SNAPSHOT_SCRIPT_SIGNATURE + volume.id
+            snapshot_filters['description'] = CREATED_BY_BOTO_EBS_SNAPSHOT_SCRIPT_SIGNATURE + volume_id
         log.debug(snapshot_filters)
         snapshots = ec2.get_all_snapshots(owner='self', filters=snapshot_filters)
-        log.info("Deleting snapshots of " + volume.id + " (set '" + backup_set + "', " + str(len(snapshots)) + " available, retaining " + str(backup_retention) + "):")
+        log.info("Deleting snapshots of " + volume_id + " (set '" + backup_set + "', " + str(len(snapshots)) + " available, retaining " + str(backup_retention) + "):")
         # While snapshots are apparently returned in oldest to youngest order, this isn't documented;
         # therefore an explicit sort is performed to ensure this regardless.
         num_snapshots = len(snapshots);
@@ -94,24 +94,23 @@ def create_images(ec2, reservations, backup_set, description, no_reboot=False):
             tags = {TAG_NAME: name, TAG_BACKUP_POLICY: backup_set}
             ec2.create_tags([image], tags)
 
-def expire_images(ec2, reservations, backup_set, backup_retention, no_origin_safeguard=False):
+def expire_images(ec2, instance_ids, backup_set, backup_retention, no_origin_safeguard=False):
     log = ec2_log
-    for reservation in reservations:
-        for instance in reservation.instances:
-            image_filters = {"name": instance.id + "*", "tag:" + TAG_BACKUP_POLICY: backup_set}
-            if not no_origin_safeguard:
-                image_filters['description'] = CREATED_BY_BOTO_EC2_IMAGE_SCRIPT_SIGNATURE + instance.id
-            log.debug(image_filters)
-            images = ec2.get_all_images(owners=['self'], filters=image_filters)
-            log.info("Deregistering images of " + instance.id + " (set '" + backup_set + "', " + str(len(images)) + " available, retaining " + str(backup_retention) + "):")
-            # While images are apparently returned in oldest to youngest order, this isn't documented;
-            # therefore an explicit sort is performed to ensure this regardless.
-            num_images = len(images)
-            for image in sorted(images, key=attrgetter('name')):
-                log.debug(image.name)
-                if num_images <= backup_retention:
-                    log.info("... retaining last " + str(backup_retention) + " images.")
-                    break
-                num_images -= 1
-                log.info("... deregistering image '" + image.id + "' ...")
-                ec2.deregister_image(image.id, delete_snapshot=True)
+    for instance_id in instance_ids:
+        image_filters = {"name": instance_id + "*", "tag:" + TAG_BACKUP_POLICY: backup_set}
+        if not no_origin_safeguard:
+            image_filters['description'] = CREATED_BY_BOTO_EC2_IMAGE_SCRIPT_SIGNATURE + instance_id
+        log.debug(image_filters)
+        images = ec2.get_all_images(owners=['self'], filters=image_filters)
+        log.info("Deregistering images of " + instance_id + " (set '" + backup_set + "', " + str(len(images)) + " available, retaining " + str(backup_retention) + "):")
+        # While images are apparently returned in oldest to youngest order, this isn't documented;
+        # therefore an explicit sort is performed to ensure this regardless.
+        num_images = len(images)
+        for image in sorted(images, key=attrgetter('name')):
+            log.debug(image.name)
+            if num_images <= backup_retention:
+                log.info("... retaining last " + str(backup_retention) + " images.")
+                break
+            num_images -= 1
+            log.info("... deregistering image '" + image.id + "' ...")
+            ec2.deregister_image(image.id, delete_snapshot=True)

scripts/describe-instances.py

@@ -50,11 +50,11 @@
         ec2 = boto.connect_ec2(region=region, **credentials)
         reservations = ec2.get_all_instances(instance_ids=args.resource_ids, filters=filters)
         print region.name + ": " + str(len(reservations)) + " instances"
-        for reservation in reservations:
-            for instance in reservation.instances:
-                if args.verbose:
-                    pprint(vars(instance))
-                else:
-                    print instance.id
+        instances = [instance for reservation in reservations for instance in reservation.instances]
+        for instance in instances:
+            if args.verbose:
+                pprint(vars(instance))
+            else:
+                print instance.id
     except boto.exception.BotoServerError, e:
         log.error(e.error_message)

scripts/expire-images.py

@@ -32,7 +32,7 @@
 parser = argparse.ArgumentParser(description='Expire images of EC2 instances in all/some available EC2 regions',
                                  parents=[bc.build_region_parser(), bc.build_common_parser()])
 parser.add_argument("-f", "--filter", action="append", help="An EC2 instance filter. [can be used multiple times]")
-parser.add_argument("-i", "--id", dest="resource_ids", action="append", help="An EC2 instance id. [can be used multiple times]")
+parser.add_argument("-i", "--id", dest="resource_ids", action="append", help="An EC2 instance id (existing or terminated). [can be used multiple times]")
 parser.add_argument("-br", "--backup_retention", type=int, default=1, help="The number of backups to retain (correlated via backup_set). [default: 1]")
 parser.add_argument("-bs", "--backup_set", default=DEFAULT_BACKUP_SET, help="A backup set name (determines retention correlation). [default: 'default]'")
 parser.add_argument("-ns", "--no_origin_safeguard", action="store_true", help="Allow deletion of images originating from other tools. [default: False]")
@@ -55,8 +55,13 @@
 for region in regions:
     try:
         ec2 = boto.connect_ec2(region=region, **credentials)
-        reservations = ec2.get_all_instances(instance_ids=args.resource_ids, filters=filters)
-        print region.name + ": " + str(len(reservations)) + " instances"
-        expire_images(ec2, reservations, backup_set, args.backup_retention, args.no_origin_safeguard)
+        # NOTE: Not filtering by id allows expiring images of terminated instances as well.
+        reservations = ec2.get_all_instances(filters=filters)
+        instance_ids = [instance.id for reservation in reservations for instance in reservation.instances]
+        num_instances = len(instance_ids)
+        if args.resource_ids:
+            instance_ids.extend(args.resource_ids)
+        print region.name + ": " + str(num_instances) + " instances / " + str(len(instance_ids)) + " instance IDs"
+        expire_images(ec2, instance_ids, backup_set, args.backup_retention, args.no_origin_safeguard)
     except boto.exception.BotoServerError, e:
         log.error(e.error_message)

scripts/expire-snapshots.py

@@ -32,7 +32,7 @@
 parser = argparse.ArgumentParser(description='Expire snapshots of EBS volumes in all/some available EC2 regions',
                                  parents=[bc.build_region_parser(), bc.build_common_parser()])
 parser.add_argument("-f", "--filter", action="append", help="An EBS volume filter. [can be used multiple times]")
-parser.add_argument("-i", "--id", dest="resource_ids", action="append", help="An EBS volume id. [can be used multiple times]")
+parser.add_argument("-i", "--id", dest="resource_ids", action="append", help="An EBS volume id (existing or deregistered). [can be used multiple times]")
 parser.add_argument("-br", "--backup_retention", type=int, default=1, help="The number of backups to retain (correlated via backup_set). [default: 1]")
 parser.add_argument("-bs", "--backup_set", default=DEFAULT_BACKUP_SET, help="A backup set name (determines retention correlation). [default: 'default']")
 parser.add_argument("-ns", "--no_origin_safeguard", action="store_true", help="Allow deletion of snapshots originating from other tools. [default: False]")
@@ -55,8 +55,12 @@
 for region in regions:
     try:
         ec2 = boto.connect_ec2(region=region, **credentials)
-        volumes = ec2.get_all_volumes(volume_ids=args.resource_ids, filters=filters)
-        print region.name + ": " + str(len(volumes)) + " volumes"
-        expire_snapshots(ec2, volumes, backup_set, args.backup_retention, args.no_origin_safeguard)
+        # NOTE: Not filtering by id allows expiring snapshots of deregistered volumes as well.
+        volumes = ec2.get_all_volumes(filters=filters)
+        volume_ids = [volume.id for volume in volumes]
+        if args.resource_ids:
+            volume_ids.extend(args.resource_ids)
+        print region.name + ": " + str(len(volumes)) + " volumes / " + str(len(volume_ids)) + " volume IDs"
+        expire_snapshots(ec2, volume_ids, backup_set, args.backup_retention, args.no_origin_safeguard)
     except boto.exception.BotoServerError, e:
         log.error(e.error_message)