From cc59fba31fc9da98c6bd1dc7783d70c948161551 Mon Sep 17 00:00:00 2001 From: Alex Lubneuski Date: Mon, 25 Sep 2023 16:12:01 -0400 Subject: [PATCH] :tada: Added cleanup ECR operation --- d3b_cli_igor/app_ops/ecr_actions.py | 81 +++++++++++++++++++++++++---- igor | 10 +++- 2 files changed, 79 insertions(+), 12 deletions(-) diff --git a/d3b_cli_igor/app_ops/ecr_actions.py b/d3b_cli_igor/app_ops/ecr_actions.py index 8af3547..16a594e 100644 --- a/d3b_cli_igor/app_ops/ecr_actions.py +++ b/d3b_cli_igor/app_ops/ecr_actions.py @@ -5,22 +5,69 @@ __name__, testing_mode=False, log_format="detailed" ) +#TODO: +# Figure out how to find the latest image +# Currently it will delete all images that are not used in task / task definition . Which is the problem for ETL etc. + #Set app cluster name ecs_client = d3b_cli_igor.common.setup("ecs") ecr_client = d3b_cli_igor.common.setup("ecr") -def list_ecr_images(): +def query_yes_no(question, default="yes"): + """Ask a yes/no question via raw_input() and return their answer. + + "question" is a string that is presented to the user. + "default" is the presumed answer if the user just hits . + It must be "yes" (the default), "no" or None (meaning + an answer is required of the user). + + The "answer" return value is True for "yes" or False for "no". + """ + valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} + if default is None: + prompt = " [y/n] " + elif default == "yes": + prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " + else: + raise ValueError("invalid default answer: '%s'" % default) + + while True: + sys.stdout.write(question + prompt) + choice = input().lower() + if default is not None and choice == "": + return valid[default] + elif choice in valid: + return valid[choice] + else: + sys.stdout.write("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") + +def get_repo_objects(repos): + repositories = [] + for r in ecr_client.describe_repositories()['repositories']: + if r['repositoryName'] in repos: + repositories.append(r) + return repositories + +def list_ecr_images(repos): logger.info("Getting images from ECR repos") - repos = ecr_client.describe_repositories() + print(repos) + if (len(repos) > 0): + #Replace repo names with objects + repos = get_repo_objects(repos) + else: + repos = ecr_client.describe_repositories()['respositories'] container_images = [] - for r in repos['repositories']: + for r in repos: images = ecr_client.list_images( repositoryName = r['repositoryName'] ) + logger.info("Repo: " + r['repositoryName']) for i in images['imageIds']: if 'imageTag' in i: image_name = r['repositoryUri'] + ":" + i['imageTag'] - container_images.append({"image": image_name, "digest": i["imageDigest"]}) + container_images.append({"image": image_name, "digest": i["imageDigest"], "repoName": r['repositoryName'], "tag": i["imageTag"]}) return container_images def list_running_task_images(app_cluster): @@ -49,7 +96,7 @@ def list_running_task_images(app_cluster): images.append({"image": c['image'], "digest": c['imageDigest']}) return images -def list_active_task_definitions(): +def list_active_task_definitions(repos): logger.info("Getting images from Task Definitions") ecs_tasks = [] images = [] @@ -61,10 +108,11 @@ def list_active_task_definitions(): for c in td['taskDefinition']['containerDefinitions']: images.append({"image": c['image'], "digest": ""}) #Since task definiteions do not have image digest, we need to look it up - for i in list_ecr_images(): + for i in list_ecr_images(repos): for idx,ti in enumerate(images): if ( ti['image'] in i['image']): images[idx]['digest'] = i['digest'] + return images def list_clusters(): @@ -99,15 +147,28 @@ def get_list_of_old_images(active_images, ecr_images): return items_to_remove -def ecr_cleanup(): +def remove_images(imageIds): + if (query_yes_no("Would you like to remove the following images? \n " + str(imageIds), default="no")): + for i in imageIds: + ecr_client.batch_delete_image( + repositoryName = i['repoName'], + imageIds=[ + { + 'imageDigest': i['digest'], + 'imageTag': i['tag'] + } + ] + ) + +def ecr_cleanup(repos): ecs_images = [] - ecr_images = list_ecr_images() + ecr_images = list_ecr_images(repos) task_def_images = [] for c in list_clusters(): ecs_images = ecs_images + list_running_task_images(c) - task_def_images = list_active_task_definitions() + task_def_images = list_active_task_definitions(repos) print(len(task_def_images)) logger.info("Number of ECR images in all ECR repos: " + str(len(ecr_images))) @@ -117,4 +178,4 @@ def ecr_cleanup(): #Find which images to remove active_images = (task_def_images + ecs_images) logger.info("Number of total active images: " + str(len(active_images))) - get_list_of_old_images(active_images,ecr_images) + remove_images(get_list_of_old_images(active_images,ecr_images)) diff --git a/igor b/igor index 837d04b..483b044 100755 --- a/igor +++ b/igor @@ -156,10 +156,16 @@ def restart(app, environment, account): check_creds() d3b_cli_igor.app_ops.ecs_deployment.restart(app, environment, account) +@click.option( + "--repos", + nargs=1, + required=False, + help="Enter Repository Name to Cleanup(comma separated)", +) @click.command(name="ecr_cleanup") -def ecr_cleanup(): +def ecr_cleanup(repos): check_creds() - d3b_cli_igor.app_ops.ecr_actions.ecr_cleanup() + d3b_cli_igor.app_ops.ecr_actions.ecr_cleanup(repos.split(',')) @click.command(name="get-info") @click.option(