Merge pull request #180 from juanjux/fix_177

Correctly handle python2 kwargs
bblfsh · Feb 14, 2019 · cc19741 · cc19741
2 parents fe965ac + d395809
commit cc19741
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 10 deletions.
diff --git a/driver/normalizer/normalizer.go b/driver/normalizer/normalizer.go
@@ -88,8 +88,8 @@ func mapStr(nativeType string) Mapping {
 			{Name: uast.KeyType, Op: String("Boxed" + nativeType)},
 			{Name: "boxed_value", Op: UASTType(uast.String{}, Obj{
 				uast.KeyPos: Var("pos_"),
-				"Value": Var("s"),
-				"Format": String(""),
+				"Value":     Var("s"),
+				//"Format":    String(""),
 			})},
 			{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
 			{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
@@ -114,7 +114,7 @@ var Normalizers = []Mapping{
 			{Name: uast.KeyType, Op: String("BoxedName")},
 			{Name: "boxed_value", Op: UASTType(uast.Identifier{}, Obj{
 				uast.KeyPos: Var("pos_"),
-				"Name": Var("id"),
+				"Name":      Var("id"),
 			})},
 			{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
 			{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
@@ -136,7 +136,7 @@ var Normalizers = []Mapping{
 			{Name: uast.KeyType, Op: String("BoxedBoolLiteral")},
 			{Name: "boxed_value", Op: UASTType(uast.Bool{}, Obj{
 				uast.KeyPos: Var("pos_"),
-				"Value": Var("lv"),
+				"Value":     Var("lv"),
 			})},
 			{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
 			{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
@@ -225,8 +225,8 @@ var Normalizers = []Mapping{
 			uast.KeyToken: Var("name"),
 		},
 		Obj{
-			"Name":        identifierWithPos("name"),
-			"MapVariadic": Bool(true),
+			"Name": identifierWithPos("name"),
+		    "MapVariadic": Bool(true),
 		},
 	)),
 

diff --git a/native/python_package/python_driver/astimprove.py b/native/python_package/python_driver/astimprove.py
@@ -218,6 +218,18 @@ def name2arg(node: Node):
 
         kwarg = deepcopy(node.get("kwarg"))
         if kwarg:
+            if isinstance(kwarg, str):
+                # Python2 kwargs are just strings; convert to same format
+                # as Python3
+                kwarg = {
+                    "arg": kwarg,
+                    "annotation": None,
+                    # the tokenizer will fix the positions later
+                    "lineno": 1,
+                    "end_lineno": 1,
+                    "col_offset": 0,
+                    "end_col_offset": 0
+                    }
             kwarg["ast_type"] = "kwarg"
             norm_args.append(self.visit(kwarg))
 
@@ -264,13 +276,40 @@ def visit_other_field(self, node: Node) -> VisitResult:
     from pprint import pprint
 
     if len(sys.argv) > 1:
-        from pydetector.ast2dict import ast2dict
+        from pydetector import detector
         codestr = open(sys.argv[1]).read()
-        testdict = ast2dict(codestr)
+        resdict = detector.detect(codestr=codestr, stop_on_ok_ast=True)
+        codeinfo = resdict['<code_string>']
+        version = codeinfo['version']
+
+        failed = False
+        testdict = None
+
+        if version in (3, 6) and codeinfo['py3ast']:
+            testdict = codeinfo['py3ast']["PY3AST"]
+            print("Using Python3")
+        elif version in (1, 2) and codeinfo['py2ast']:
+            testdict = codeinfo['py2ast']["PY2AST"]
+            print("Using Python2")
+        else:
+            failed = True
+            errors = [
+              'Errors produced trying to get an AST for both Python versions' +
+              '\n------ Python2 errors:\n%s' % codeinfo['py2_ast_errors'] +
+              '\n------ Python3 errors:\n%s' % codeinfo['py3_ast_errors']
+            ]
+
+        if not failed and not testdict:
+            raise Exception('Empty AST generated from non empty code')
+        ast = AstImprover(codestr, testdict).parse()
+        if not ast:
+            raise Exception('Empty AST generated from non empty code')
     else:
         codestr = open("../test/fixtures/detector.py").read()
-        spec = importlib.util.spec_from_file_location("module.testmod",
-                                                      "../test/fixtures/exported_dict.py")
+        spec = importlib.util.spec_from_file_location(
+                "module.testmod",
+                "../test/fixtures/exported_dict.py"
+        )
         testmod = importlib.util.module_from_spec(spec)
 
         if spec.loader: