From 23777453e0d37c4a6f97f89aaffdc68cc35e0e97 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Fri, 11 Oct 2019 15:25:17 +0300 Subject: [PATCH] fix varargs in python 2 files; fixes #199 Signed-off-by: Denys Smirnov --- fixtures/vararg_python2.py | 5 + fixtures/vararg_python2.py.native | 143 ++++++++ fixtures/vararg_python2.py.sem.uast | 326 ++++++++++++++++++ fixtures/vararg_python2.py.uast | 267 ++++++++++++++ .../python_driver/astimprove.py | 27 +- 5 files changed, 759 insertions(+), 9 deletions(-) create mode 100644 fixtures/vararg_python2.py create mode 100644 fixtures/vararg_python2.py.native create mode 100644 fixtures/vararg_python2.py.sem.uast create mode 100644 fixtures/vararg_python2.py.uast diff --git a/fixtures/vararg_python2.py b/fixtures/vararg_python2.py new file mode 100644 index 00000000..d8eec255 --- /dev/null +++ b/fixtures/vararg_python2.py @@ -0,0 +1,5 @@ +def function6(a, *param): + print "variadicArgs" + +def function7(a, *args, **kwargs): + print "variadicAndKeywordArgs" \ No newline at end of file diff --git a/fixtures/vararg_python2.py.native b/fixtures/vararg_python2.py.native new file mode 100644 index 00000000..54c424b9 --- /dev/null +++ b/fixtures/vararg_python2.py.native @@ -0,0 +1,143 @@ +{ + 'PY2AST': { + 'ast_type': "Module", + body: [ + { + args: { + args: [ + { + '@token': "a", + 'ast_type': "arg", + 'col_offset': 15, + ctx: "Param", + 'end_col_offset': 16, + 'end_lineno': 1, + lineno: 1, + }, + { + '@token': "param", + annotation: ~, + 'ast_type': "vararg", + 'col_offset': 19, + 'end_col_offset': 24, + 'end_lineno': 1, + lineno: 1, + }, + ], + 'ast_type': "arguments", + }, + 'ast_type': "FunctionDef", + body: [ + { + 'ast_type': "Print", + 'col_offset': 5, + dest: ~, + 'end_col_offset': 10, + 'end_lineno': 2, + lineno: 2, + nl: true, + values: [ + { + 'ast_type': "Str", + 'col_offset': 11, + 'end_col_offset': 25, + 'end_lineno': 2, + lineno: 2, + s: "variadicArgs", + }, + ], + }, + ], + 'col_offset': 5, + 'decorator_list': [], + 'end_col_offset': 14, + 'end_lineno': 1, + lineno: 1, + name: "function6", + }, + { + args: { + args: [ + { + '@token': "a", + 'ast_type': "arg", + 'col_offset': 15, + ctx: "Param", + 'end_col_offset': 16, + 'end_lineno': 4, + lineno: 4, + 'noops_previous': { + 'ast_type': "PreviousNoops", + 'col_offset': 1, + 'end_col_offset': 1, + 'end_lineno': 3, + lineno: 3, + lines: [], + }, + }, + { + '@token': "kwargs", + annotation: ~, + 'ast_type': "kwarg", + 'col_offset': 1, + 'end_col_offset': 1, + 'end_lineno': 1, + lineno: 1, + }, + { + '@token': "args", + annotation: ~, + 'ast_type': "vararg", + 'col_offset': 1, + 'end_col_offset': 1, + 'end_lineno': 1, + lineno: 1, + }, + ], + 'ast_type': "arguments", + }, + 'ast_type': "FunctionDef", + body: [ + { + 'ast_type': "Print", + 'col_offset': 5, + dest: ~, + 'end_col_offset': 10, + 'end_lineno': 5, + lineno: 5, + nl: true, + values: [ + { + 'ast_type': "Str", + 'col_offset': 11, + 'end_col_offset': 35, + 'end_lineno': 5, + lineno: 5, + 'noops_sameline': { + 'ast_type': "SameLineNoops", + 'col_offset': 34, + 'end_col_offset': 35, + 'end_lineno': 5, + lineno: 5, + 'noop_lines': [ + { + 'ast_type': "NoopSameLine", + s: "", + }, + ], + }, + s: "variadicAndKeywordArgs", + }, + ], + }, + ], + 'col_offset': 5, + 'decorator_list': [], + 'end_col_offset': 14, + 'end_lineno': 4, + lineno: 4, + name: "function7", + }, + ], + }, +} \ No newline at end of file diff --git a/fixtures/vararg_python2.py.sem.uast b/fixtures/vararg_python2.py.sem.uast new file mode 100644 index 00000000..7cf77c38 --- /dev/null +++ b/fixtures/vararg_python2.py.sem.uast @@ -0,0 +1,326 @@ +{ '@type': "python:Module", + '@role': [File, Module], + '@pos': { '@type': "uast:Positions", + }, + body: [ + { '@type': "uast:FunctionGroup", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 4, + line: 1, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 13, + line: 1, + col: 14, + }, + }, + Nodes: [ + { + async: false, + comments: {}, + decorators: [], + }, + { '@type': "uast:Alias", + Name: { '@type': "uast:Identifier", + Name: "function6", + }, + Node: { '@type': "uast:Function", + Body: { '@type': "uast:Block", + Statements: [ + { '@type': "python:Print", + '@token': "print", + '@role': [Call, Callee, Expression, Function, Identifier], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 30, + line: 2, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 35, + line: 2, + col: 10, + }, + }, + dest: ~, + nl: true, + values: [ + { '@type': "python:BoxedStr", + '@role': [Argument, Call, Positional], + 'boxed_value': { '@type': "uast:String", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 36, + line: 2, + col: 11, + }, + end: { '@type': "uast:Position", + offset: 50, + line: 2, + col: 25, + }, + }, + Format: "", + Value: "variadicArgs", + }, + }, + ], + }, + ], + }, + Type: { '@type': "uast:FunctionType", + Arguments: [ + { '@type': "uast:Argument", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 14, + line: 1, + col: 15, + }, + end: { '@type': "uast:Position", + offset: 15, + line: 1, + col: 16, + }, + }, + MapVariadic: false, + Name: { '@type': "uast:Identifier", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 14, + line: 1, + col: 15, + }, + end: { '@type': "uast:Position", + offset: 15, + line: 1, + col: 16, + }, + }, + Name: "a", + }, + Receiver: false, + Type: ~, + Variadic: false, + }, + { '@type': "uast:Argument", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 18, + line: 1, + col: 19, + }, + end: { '@type': "uast:Position", + offset: 23, + line: 1, + col: 24, + }, + }, + Init: ~, + MapVariadic: false, + Name: { '@type': "uast:Identifier", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 18, + line: 1, + col: 19, + }, + end: { '@type': "uast:Position", + offset: 23, + line: 1, + col: 24, + }, + }, + Name: "param", + }, + Receiver: false, + Type: ~, + Variadic: true, + }, + ], + }, + }, + }, + ], + }, + { '@type': "uast:FunctionGroup", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 56, + line: 4, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 65, + line: 4, + col: 14, + }, + }, + Nodes: [ + { + async: false, + comments: {}, + decorators: [], + }, + { '@type': "uast:Alias", + Name: { '@type': "uast:Identifier", + Name: "function7", + }, + Node: { '@type': "uast:Function", + Body: { '@type': "uast:Block", + Statements: [ + { '@type': "python:Print", + '@token': "print", + '@role': [Call, Callee, Expression, Function, Identifier], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 91, + line: 5, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 96, + line: 5, + col: 10, + }, + }, + dest: ~, + nl: true, + values: [ + { '@type': "python:BoxedStr", + '@role': [Argument, Call, Positional], + 'boxed_value': { '@type': "uast:String", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 97, + line: 5, + col: 11, + }, + end: { '@type': "uast:Position", + offset: 121, + line: 5, + col: 35, + }, + }, + Format: "", + Value: "variadicAndKeywordArgs", + }, + 'noops_sameline': ~, + }, + ], + }, + ], + }, + Type: { '@type': "uast:FunctionType", + Arguments: [ + { '@type': "uast:Argument", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 66, + line: 4, + col: 15, + }, + end: { '@type': "uast:Position", + offset: 67, + line: 4, + col: 16, + }, + }, + MapVariadic: false, + Name: { '@type': "uast:Identifier", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 66, + line: 4, + col: 15, + }, + end: { '@type': "uast:Position", + offset: 67, + line: 4, + col: 16, + }, + }, + Name: "a", + }, + Receiver: false, + Type: ~, + Variadic: false, + }, + { '@type': "uast:Argument", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + }, + Init: ~, + MapVariadic: true, + Name: { '@type': "uast:Identifier", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + }, + Name: "kwargs", + }, + Receiver: false, + Type: ~, + Variadic: false, + }, + { '@type': "uast:Argument", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + }, + Init: ~, + MapVariadic: false, + Name: { '@type': "uast:Identifier", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + }, + Name: "args", + }, + Receiver: false, + Type: ~, + Variadic: true, + }, + ], + }, + }, + }, + ], + }, + ], +} \ No newline at end of file diff --git a/fixtures/vararg_python2.py.uast b/fixtures/vararg_python2.py.uast new file mode 100644 index 00000000..112523fd --- /dev/null +++ b/fixtures/vararg_python2.py.uast @@ -0,0 +1,267 @@ +{ '@type': "Module", + '@role': [File, Module], + '@pos': { '@type': "uast:Positions", + }, + body: [ + { '@type': "FunctionDef", + '@token': "function6", + '@role': [Declaration, Function, Identifier, Name], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 4, + line: 1, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 13, + line: 1, + col: 14, + }, + }, + args: { '@type': "arguments", + '@role': [Argument, Declaration, Function, Incomplete], + '@pos': { '@type': "uast:Positions", + }, + args: [ + { '@type': "arg", + '@token': "a", + '@role': [Argument, Declaration, Function, Name], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 14, + line: 1, + col: 15, + }, + end: { '@type': "uast:Position", + offset: 15, + line: 1, + col: 16, + }, + }, + ctx: "Param", + }, + { '@type': "vararg", + '@token': "param", + '@role': [ArgsList, Declaration, Function, List, Name], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 18, + line: 1, + col: 19, + }, + end: { '@type': "uast:Position", + offset: 23, + line: 1, + col: 24, + }, + }, + annotation: ~, + }, + ], + }, + body: { '@type': "FunctionDef.body", + '@role': [Body, Declaration, Function], + 'body_stmts': [ + { '@type': "Print", + '@token': "print", + '@role': [Call, Callee, Expression, Function, Identifier], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 30, + line: 2, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 35, + line: 2, + col: 10, + }, + }, + dest: ~, + nl: true, + values: [ + { '@type': "Str", + '@token': "variadicArgs", + '@role': [Argument, Call, Expression, Literal, Positional, Primitive, String], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 36, + line: 2, + col: 11, + }, + end: { '@type': "uast:Position", + offset: 50, + line: 2, + col: 25, + }, + }, + }, + ], + }, + ], + }, + 'decorator_list': { '@type': "FunctionDef.decorators", + '@role': [Declaration, Function, Incomplete], + decorators: [], + }, + }, + { '@type': "FunctionDef", + '@token': "function7", + '@role': [Declaration, Function, Identifier, Name], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 56, + line: 4, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 65, + line: 4, + col: 14, + }, + }, + args: { '@type': "arguments", + '@role': [Argument, Declaration, Function, Incomplete], + '@pos': { '@type': "uast:Positions", + }, + args: [ + { '@type': "arg", + '@token': "a", + '@role': [Argument, Declaration, Function, Name], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 66, + line: 4, + col: 15, + }, + end: { '@type': "uast:Position", + offset: 67, + line: 4, + col: 16, + }, + }, + ctx: "Param", + 'noops_previous': { '@type': "PreviousNoops", + '@role': [Noop], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 51, + line: 3, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 51, + line: 3, + col: 1, + }, + }, + lines: [], + }, + }, + { '@type': "kwarg", + '@token': "kwargs", + '@role': [ArgsList, Declaration, Function, Map, Name], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + }, + annotation: ~, + }, + { '@type': "vararg", + '@token': "args", + '@role': [ArgsList, Declaration, Function, List, Name], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + }, + annotation: ~, + }, + ], + }, + body: { '@type': "FunctionDef.body", + '@role': [Body, Declaration, Function], + 'body_stmts': [ + { '@type': "Print", + '@token': "print", + '@role': [Call, Callee, Expression, Function, Identifier], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 91, + line: 5, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 96, + line: 5, + col: 10, + }, + }, + dest: ~, + nl: true, + values: [ + { '@type': "Str", + '@token': "variadicAndKeywordArgs", + '@role': [Argument, Call, Expression, Literal, Positional, Primitive, String], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 97, + line: 5, + col: 11, + }, + end: { '@type': "uast:Position", + offset: 121, + line: 5, + col: 35, + }, + }, + 'noops_sameline': { '@type': "SameLineNoops", + '@role': [Comment], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 120, + line: 5, + col: 34, + }, + end: { '@type': "uast:Position", + offset: 121, + line: 5, + col: 35, + }, + }, + 'noop_lines': [ + { '@type': "NoopSameLine", + '@token': "", + '@role': [Comment, Noop], + '@pos': { '@type': "uast:Positions", + }, + }, + ], + }, + }, + ], + }, + ], + }, + 'decorator_list': { '@type': "FunctionDef.decorators", + '@role': [Declaration, Function, Incomplete], + decorators: [], + }, + }, + ], +} \ No newline at end of file diff --git a/native/python_package/python_driver/astimprove.py b/native/python_package/python_driver/astimprove.py index ecaa2f15..15424a84 100644 --- a/native/python_package/python_driver/astimprove.py +++ b/native/python_package/python_driver/astimprove.py @@ -190,6 +190,19 @@ def match_default_args(args: List[Node], defaults: List[Node]) -> List[Node]: return args + def _str2node(s: str) -> dict: + # some nodes in Python2 AST are strings instead of objects + # convert to same format + return { + "arg": s, + "annotation": None, + # the tokenizer will fix the positions later + "lineno": 1, + "end_lineno": 1, + "col_offset": 0, + "end_col_offset": 0 + } + def name2arg(node: Node): # Normalize Python2 and 3 argument types if node["ast_type"] == "Name": @@ -225,20 +238,16 @@ def name2arg(node: Node): if isinstance(kwarg, str): # Python2 kwargs are just strings; convert to same format # as Python3 - kwarg = { - "arg": kwarg, - "annotation": None, - # the tokenizer will fix the positions later - "lineno": 1, - "end_lineno": 1, - "col_offset": 0, - "end_col_offset": 0 - } + kwarg = _str2node(kwarg) kwarg["ast_type"] = "kwarg" norm_args.append(self.visit(kwarg)) vararg = deepcopy(node.get("vararg")) if vararg: + if isinstance(vararg, str): + # Python2 varargs are just strings; convert to same format + # as Python3 + vararg = _str2node(vararg) vararg["ast_type"] = "vararg" norm_args.append(self.visit(vararg))