bug fixes

.python/common.py

@@ -226,7 +226,7 @@ def call_resource(type, id):
 
 
 def tfplan1():
-   print("Terraform Plan - Dependancies Detection Loop ...")
+
    rf = "resources.out"
    # com="terraform plan -generate-config-out="+ rf + " -out tfplan -json > plan2.json"
 
@@ -240,8 +240,6 @@ def tfplan1():
    com = "mv aws_*.tf imported"
    rout = rc(com)
 
-
-
    com = "terraform plan -generate-config-out=" + \
        rf + " -out tfplan -json > plan1.json"
    print(com)

.python/fixtf_aws_resources/aws_not_implemented.py

@@ -33,6 +33,7 @@
     #
     "aws_appstream_directory_config": True,
     "aws_appstream_fleet_stack_association": True,
+    "aws_appstream_stack": True,
     "aws_appstream_user_stack_association": True,
     "aws_appsync_api_cache": True,
     "aws_appsync_domain_name_api_association": True,

.python/fixtf_aws_resources/fixtf_glue.py

@@ -4,7 +4,8 @@
 def aws_glue_crawler(t1,tt1,tt2,flag1,flag2):
 	skip=0
 	if tt1 == "database_name" and tt2 != "null":
-		t1 = tt1 + " = aws_glue_catalog_database.d-"+globals.acc+"__"+tt2+".name\n"
+		if globals.gluedbs[tt2]:
+			t1 = tt1 + " = aws_glue_catalog_database.d-"+globals.acc+"__"+tt2+".name\n"
 		#common.add_dependancy("aws_glue_catalog_database",tt2)
 
 	elif tt1 == "sample_size":

.python/get_aws_resources/aws_glue.py

@@ -2,11 +2,12 @@
 import boto3
 import globals
 import inspect
+import sys
 
 def get_aws_glue_catalog_database(type, id, clfn, descfn, topkey, key, filterid):
 
     if globals.debug:
-        print("--> In get_aws_glue_catalog_database  doing " + type + ' with id ' + str(id) +
+        print("--> Inn get_aws_glue_catalog_database  doing " + type + ' with id ' + str(id) +
               " clfn="+clfn+" descfn="+descfn+" topkey="+topkey+" key="+key+" filterid="+filterid)
 
     try:
@@ -22,11 +23,13 @@ def get_aws_glue_catalog_database(type, id, clfn, descfn, topkey, key, filterid)
                 tfid="d-"+pkey.replace(":","__")
                 common.write_import(type,pkey,tfid) 
                 common.add_dependancy("aws_glue_catalog_table",pkey)
+                globals.gluedbs[j[key]]=True
+
                 #pkey2="aws_glue_catalog_table."+pkey
                 #globals.rproc[pkey2]=True
 
         else: 
-            if ":" in id:   id =id.split(":")[1]    
+            if ":" in id:   id =id.split(":")[1]   
             response = client.get_database(Name=id)
             if response == []: 
                 print("Empty response for "+type+ " id="+str(id)+" returning")
@@ -37,14 +40,26 @@ def get_aws_glue_catalog_database(type, id, clfn, descfn, topkey, key, filterid)
             pkey=globals.acc+":"+j[key]
             tfid="d-"+pkey.replace(":","__")
             common.write_import(type,pkey,tfid)
+            globals.gluedbs[j[key]]=True
             #print("KD add aws_glue_catalog_table "+pkey)
             common.add_dependancy("aws_glue_catalog_table",pkey)
             gkey="aws_glue_catalog_table."+pkey
             globals.rproc[gkey]=True
 
 
     except Exception as e:
-        common.handle_error(e,str(inspect.currentframe().f_code.co_name),clfn,descfn,topkey,id)
+        exc_type, exc_obj, exc_tb = sys.exc_info()
+        exn=str(exc_type.__name__)
+        if exn=="AccessDeniedException" and "Insufficient Lake Formation permission" in str(e):
+            print("AccessDeniedException - Insufficient Lake Formation permission for",type,id)
+            if id is not None:
+                if ":" in id:   id =id.split(":")[1]
+                pkey=globals.acc+":"+id
+                gkey="aws_glue_catalog_database."+pkey
+
+                globals.rproc[gkey]=True
+        else:
+            common.handle_error(e,str(inspect.currentframe().f_code.co_name),clfn,descfn,topkey,id)
 
     return True
 
@@ -75,16 +90,20 @@ def get_aws_glue_catalog_table(type, id, clfn, descfn, topkey, key, filterid):
 
 
         tkey="aws_glue_catalog_table"+"."+catalogn+":"+databasen
-
+        #print(catalogn, databasen, tabnam,cc)
         response = []
         client = boto3.client(clfn)
 
         if cc == 1:
                 response = client.get_tables(CatalogId=catalogn,DatabaseName=databasen)
         if cc == 2:
                 response = client.get_tables(CatalogId=catalogn,DatabaseName=databasen,Expression=tabnam)
-
-        if response == []: print("Empty response for "+type+ " id="+str(id)+" returning"); return True
+
+        if response[topkey] == []: 
+            print("Empty response for "+type+ " id="+str(id)+" returning"); 
+            tkey="aws_glue_catalog_table"+"."+catalogn+":"+databasen
+            globals.rproc[tkey]=True
+            return True
         for j in response[topkey]:
             #Terraform import id = "123456789012:MyDatabase:MyTable"
                 pkey=catalogn+":"+databasen+":"+j[key]

.python/globals.py

@@ -89,3 +89,4 @@
 rolelist={}
 bucketlist={}
 tgwlist={}
+gluedbs={}

aws2tf.py

@@ -403,7 +403,7 @@ def main():
         lc  = lc + 1
 
 # go again plan and split / fix
-
+        print("Terraform Plan - Dependancies Detection Loop "+str(lc)+".....")
 
         x=glob.glob("import__aws_*.tf")
         #print(str(x))