From 82c0129a6851eec1e20195325979df41e40f044b Mon Sep 17 00:00:00 2001 From: awsandy Date: Sat, 5 Oct 2024 16:59:49 +0100 Subject: [PATCH] bug fixes --- .python/common.py | 4 +-- .../aws_not_implemented.py | 1 + .python/fixtf_aws_resources/fixtf_glue.py | 3 +- .python/get_aws_resources/aws_glue.py | 31 +++++++++++++++---- .python/globals.py | 1 + aws2tf.py | 2 +- 6 files changed, 31 insertions(+), 11 deletions(-) diff --git a/.python/common.py b/.python/common.py index 1f29f02..d5f252c 100755 --- a/.python/common.py +++ b/.python/common.py @@ -226,7 +226,7 @@ def call_resource(type, id): def tfplan1(): - print("Terraform Plan - Dependancies Detection Loop ...") + rf = "resources.out" # com="terraform plan -generate-config-out="+ rf + " -out tfplan -json > plan2.json" @@ -240,8 +240,6 @@ def tfplan1(): com = "mv aws_*.tf imported" rout = rc(com) - - com = "terraform plan -generate-config-out=" + \ rf + " -out tfplan -json > plan1.json" print(com) diff --git a/.python/fixtf_aws_resources/aws_not_implemented.py b/.python/fixtf_aws_resources/aws_not_implemented.py index 38cdf06..0713465 100644 --- a/.python/fixtf_aws_resources/aws_not_implemented.py +++ b/.python/fixtf_aws_resources/aws_not_implemented.py @@ -33,6 +33,7 @@ # "aws_appstream_directory_config": True, "aws_appstream_fleet_stack_association": True, + "aws_appstream_stack": True, "aws_appstream_user_stack_association": True, "aws_appsync_api_cache": True, "aws_appsync_domain_name_api_association": True, diff --git a/.python/fixtf_aws_resources/fixtf_glue.py b/.python/fixtf_aws_resources/fixtf_glue.py index 6d6c483..08ebb68 100644 --- a/.python/fixtf_aws_resources/fixtf_glue.py +++ b/.python/fixtf_aws_resources/fixtf_glue.py @@ -4,7 +4,8 @@ def aws_glue_crawler(t1,tt1,tt2,flag1,flag2): skip=0 if tt1 == "database_name" and tt2 != "null": - t1 = tt1 + " = aws_glue_catalog_database.d-"+globals.acc+"__"+tt2+".name\n" + if globals.gluedbs[tt2]: + t1 = tt1 + " = aws_glue_catalog_database.d-"+globals.acc+"__"+tt2+".name\n" #common.add_dependancy("aws_glue_catalog_database",tt2) elif tt1 == "sample_size": diff --git a/.python/get_aws_resources/aws_glue.py b/.python/get_aws_resources/aws_glue.py index 51037c3..2d877a2 100644 --- a/.python/get_aws_resources/aws_glue.py +++ b/.python/get_aws_resources/aws_glue.py @@ -2,11 +2,12 @@ import boto3 import globals import inspect +import sys def get_aws_glue_catalog_database(type, id, clfn, descfn, topkey, key, filterid): if globals.debug: - print("--> In get_aws_glue_catalog_database doing " + type + ' with id ' + str(id) + + print("--> Inn get_aws_glue_catalog_database doing " + type + ' with id ' + str(id) + " clfn="+clfn+" descfn="+descfn+" topkey="+topkey+" key="+key+" filterid="+filterid) try: @@ -22,11 +23,13 @@ def get_aws_glue_catalog_database(type, id, clfn, descfn, topkey, key, filterid) tfid="d-"+pkey.replace(":","__") common.write_import(type,pkey,tfid) common.add_dependancy("aws_glue_catalog_table",pkey) + globals.gluedbs[j[key]]=True + #pkey2="aws_glue_catalog_table."+pkey #globals.rproc[pkey2]=True else: - if ":" in id: id =id.split(":")[1] + if ":" in id: id =id.split(":")[1] response = client.get_database(Name=id) if response == []: print("Empty response for "+type+ " id="+str(id)+" returning") @@ -37,6 +40,7 @@ def get_aws_glue_catalog_database(type, id, clfn, descfn, topkey, key, filterid) pkey=globals.acc+":"+j[key] tfid="d-"+pkey.replace(":","__") common.write_import(type,pkey,tfid) + globals.gluedbs[j[key]]=True #print("KD add aws_glue_catalog_table "+pkey) common.add_dependancy("aws_glue_catalog_table",pkey) gkey="aws_glue_catalog_table."+pkey @@ -44,7 +48,18 @@ def get_aws_glue_catalog_database(type, id, clfn, descfn, topkey, key, filterid) except Exception as e: - common.handle_error(e,str(inspect.currentframe().f_code.co_name),clfn,descfn,topkey,id) + exc_type, exc_obj, exc_tb = sys.exc_info() + exn=str(exc_type.__name__) + if exn=="AccessDeniedException" and "Insufficient Lake Formation permission" in str(e): + print("AccessDeniedException - Insufficient Lake Formation permission for",type,id) + if id is not None: + if ":" in id: id =id.split(":")[1] + pkey=globals.acc+":"+id + gkey="aws_glue_catalog_database."+pkey + + globals.rproc[gkey]=True + else: + common.handle_error(e,str(inspect.currentframe().f_code.co_name),clfn,descfn,topkey,id) return True @@ -75,7 +90,7 @@ def get_aws_glue_catalog_table(type, id, clfn, descfn, topkey, key, filterid): tkey="aws_glue_catalog_table"+"."+catalogn+":"+databasen - + #print(catalogn, databasen, tabnam,cc) response = [] client = boto3.client(clfn) @@ -83,8 +98,12 @@ def get_aws_glue_catalog_table(type, id, clfn, descfn, topkey, key, filterid): response = client.get_tables(CatalogId=catalogn,DatabaseName=databasen) if cc == 2: response = client.get_tables(CatalogId=catalogn,DatabaseName=databasen,Expression=tabnam) - - if response == []: print("Empty response for "+type+ " id="+str(id)+" returning"); return True + + if response[topkey] == []: + print("Empty response for "+type+ " id="+str(id)+" returning"); + tkey="aws_glue_catalog_table"+"."+catalogn+":"+databasen + globals.rproc[tkey]=True + return True for j in response[topkey]: #Terraform import id = "123456789012:MyDatabase:MyTable" pkey=catalogn+":"+databasen+":"+j[key] diff --git a/.python/globals.py b/.python/globals.py index fddc195..44db543 100644 --- a/.python/globals.py +++ b/.python/globals.py @@ -89,3 +89,4 @@ rolelist={} bucketlist={} tgwlist={} +gluedbs={} diff --git a/aws2tf.py b/aws2tf.py index 2372051..0aa3a47 100755 --- a/aws2tf.py +++ b/aws2tf.py @@ -403,7 +403,7 @@ def main(): lc = lc + 1 # go again plan and split / fix - + print("Terraform Plan - Dependancies Detection Loop "+str(lc)+".....") x=glob.glob("import__aws_*.tf") #print(str(x))