From a2c2f75d2084895414d264a62694273af846362f Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Wed, 20 Mar 2019 14:54:30 +0100 Subject: [PATCH 1/2] sem: refactor all comment handling to 3 rules Each drops off comments from the nodes that are already mapped to smeantic UAST and thus do not expect to have such properties Signed-off-by: Alexander Bezzubov --- driver/normalizer/normalizer.go | 70 +++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/driver/normalizer/normalizer.go b/driver/normalizer/normalizer.go index f4e891a..f23249a 100644 --- a/driver/normalizer/normalizer.go +++ b/driver/normalizer/normalizer.go @@ -29,7 +29,7 @@ var Preprocessors = []Mapping{ }.Mapping(), Map( - Part("_", Obj{"loc": AnyNode(nil)}), + Part("_", Obj{"loc": Any()}), Part("_", Obj{}), ), // preserve raw string and regexp literals @@ -72,8 +72,48 @@ var Preprocessors = []Mapping{ ), } +// allNormalizedTypes are all uast.KeyType that are processed by current normalizer +// where we currently drop the coments node off. +var allNormalizedTypes = []nodes.Value{ + nodes.String("Identifier"), + nodes.String("JSXIdentifier"), + nodes.String("StringLiteral"), + nodes.String("StringLiteral"), + nodes.String("CommentLine"), + nodes.String("CommentBlock"), + nodes.String("BlockStatement"), + nodes.String("ImportDeclaration"), + nodes.String("ImportDeclaration"), + nodes.String("ImportSpecifier"), + nodes.String("ImportDefaultSpecifier"), + nodes.String("ImportNamespaceSpecifier"), + nodes.String("FunctionDeclaration"), +} + // Normalizers is the main block of normalization rules to convert native AST to semantic UAST. var Normalizers = []Mapping{ + //FIXME(bzz): save all 3 *Commnets in a new parent Group node \w order + Map( // this is not reversible + Part("_", Fields{ + {Name: uast.KeyType, Op: Check(In(allNormalizedTypes...), Var("t"))}, + {Name: "leadingComments", Drop: true, Op: Any()}, + }), + Part("_", Obj{uast.KeyType: Var("t")}), + ), + Map( // this is not reversible + Part("_", Fields{ + {Name: uast.KeyType, Op: Check(In(allNormalizedTypes...), Var("t"))}, + {Name: "innerComments", Drop: true, Op: Any()}, + }), + Part("_", Obj{uast.KeyType: Var("t")}), + ), + Map( // this is not reversible + Part("_", Fields{ + {Name: uast.KeyType, Op: Check(In(allNormalizedTypes...), Var("t"))}, + {Name: "trailingComments", Drop: true, Op: Any()}, + }), + Part("_", Obj{uast.KeyType: Var("t")}), + ), MapSemantic("Identifier", uast.Identifier{}, MapObj( Fields{ {Name: "name", Op: Var("name")}, @@ -81,9 +121,6 @@ var Normalizers = []Mapping{ {Name: "typeAnnotation", Drop: true, Op: Any()}, //FIXME(bzz): map Flow "Optional Prameter" properly {Name: "optional", Drop: true, Op: Any()}, - //FIXME(bzz): map both once we agree how - {Name: "leadingComments", Drop: true, Op: Any()}, - {Name: "trailingComments", Drop: true, Op: Any()}, }, Obj{ "Name": Var("name"), @@ -100,9 +137,6 @@ var Normalizers = []Mapping{ MapSemantic("StringLiteral", uast.String{}, MapObj( Fields{ {Name: "value", Op: singleQuote{Var("val")}}, - //FIXME(bzz): save both once we agree how - {Name: "leadingComments", Drop: true, Op: Any()}, - {Name: "trailingComments", Drop: true, Op: Any()}, }, Obj{ "Value": Var("val"), @@ -112,9 +146,6 @@ var Normalizers = []Mapping{ MapSemantic("StringLiteral", uast.String{}, MapObj( Fields{ {Name: "value", Op: Quote(Var("val"))}, - //FIXME(bzz): save both once we agree how - {Name: "leadingComments", Drop: true, Op: Any()}, - {Name: "trailingComments", Drop: true, Op: Any()}, }, Obj{ "Value": Var("val"), @@ -136,10 +167,6 @@ var Normalizers = []Mapping{ Fields{ {Name: "body", Op: Var("stmts")}, {Name: "directives", Op: Arr()}, // TODO: find an example - // FIXME(bzz): make sure such comments are linked properly - {Name: "innerComments", Drop: true, Op: Any()}, - {Name: "leadingComments", Drop: true, Op: Any()}, - {Name: "trailingComments", Drop: true, Op: Any()}, }, Obj{ "Statements": Var("stmts"), @@ -150,9 +177,6 @@ var Normalizers = []Mapping{ {Name: "source", Op: Var("path")}, // empty un-used array {Name: "specifiers", Drop: true, Op: Arr()}, - // FIXME(bzz): make sure such comments are linked properly - {Name: "leadingComments", Drop: true, Op: Any()}, - {Name: "trailingComments", Drop: true, Op: Any()}, }, Obj{ "Path": Var("path"), @@ -166,9 +190,6 @@ var Normalizers = []Mapping{ // common Fields{ {Name: "source", Op: Var("path")}, - // FIXME(bzz): make sure such comments are linked properly - {Name: "leadingComments", Drop: true, Op: Any()}, - {Name: "trailingComments", Drop: true, Op: Any()}, }, Objs{ // namespace @@ -179,9 +200,6 @@ var Normalizers = []Mapping{ {Name: uast.KeyType, Op: String("ImportNamespaceSpecifier")}, {Name: uast.KeyPos, Op: Var("local_pos")}, {Name: "local", Op: Var("local")}, - // FIXME(bzz): make sure such comments are linked properly - {Name: "leadingComments", Drop: true, Op: Any()}, - {Name: "trailingComments", Drop: true, Op: Any()}, }), }, // specific type @@ -249,9 +267,6 @@ var Normalizers = []Mapping{ MapSemantic("ImportDefaultSpecifier", uast.Alias{}, MapObj( Fields{ {Name: "local", Op: Var("local")}, - //FIXME(bzz): save this once we agree how - {Name: "leadingComments", Drop: true, Op: Any()}, - {Name: "trailingComments", Drop: true, Op: Any()}, }, Obj{ "Name": Var("local"), @@ -277,9 +292,6 @@ var Normalizers = []Mapping{ // https://flow.org/en/docs/types/generics/ // see fixtures/ext_typedecl.js#34 func makeWeakCache {Name: "typeParameters", Drop: true, Op: Any()}, - // FIXME(bzz): make sure such comments are linked properly - {Name: "leadingComments", Drop: true, Op: Any()}, - {Name: "trailingComments", Drop: true, Op: Any()}, {Name: "params", Op: Each("params", Cases("param_case", // Identifier Check( From 07e7b516ed60003754c7dfb49be18dab8429ef24 Mon Sep 17 00:00:00 2001 From: Alexander Bezzubov Date: Wed, 20 Mar 2019 22:28:19 +0100 Subject: [PATCH 2/2] sem: add new fixture for #69 Signed-off-by: Alexander Bezzubov --- fixtures/issue65-op.js | 1 + fixtures/issue65-op.js.native | 141 ++++++++++++++++++++++++++++++++ fixtures/issue65-op.js.sem.uast | 134 ++++++++++++++++++++++++++++++ fixtures/issue65-op.js.uast | 135 ++++++++++++++++++++++++++++++ 4 files changed, 411 insertions(+) create mode 100644 fixtures/issue65-op.js create mode 100644 fixtures/issue65-op.js.native create mode 100644 fixtures/issue65-op.js.sem.uast create mode 100644 fixtures/issue65-op.js.uast diff --git a/fixtures/issue65-op.js b/fixtures/issue65-op.js new file mode 100644 index 0000000..a6ddd61 --- /dev/null +++ b/fixtures/issue65-op.js @@ -0,0 +1 @@ +var a = 0 + 1; diff --git a/fixtures/issue65-op.js.native b/fixtures/issue65-op.js.native new file mode 100644 index 0000000..38263fe --- /dev/null +++ b/fixtures/issue65-op.js.native @@ -0,0 +1,141 @@ +{ + comments: [], + end: 15, + loc: { + end: { + column: 0, + line: 2, + }, + start: { + column: 0, + line: 1, + }, + }, + program: { + body: [ + { + declarations: [ + { + end: 13, + id: { + end: 5, + loc: { + end: { + column: 5, + line: 1, + }, + identifierName: "a", + start: { + column: 4, + line: 1, + }, + }, + name: "a", + start: 4, + type: "Identifier", + }, + init: { + end: 13, + left: { + end: 9, + extra: { + raw: "0", + rawValue: 0, + }, + loc: { + end: { + column: 9, + line: 1, + }, + start: { + column: 8, + line: 1, + }, + }, + start: 8, + type: "NumericLiteral", + value: 0, + }, + loc: { + end: { + column: 13, + line: 1, + }, + start: { + column: 8, + line: 1, + }, + }, + operator: "+", + right: { + end: 13, + extra: { + raw: "1", + rawValue: 1, + }, + loc: { + end: { + column: 13, + line: 1, + }, + start: { + column: 12, + line: 1, + }, + }, + start: 12, + type: "NumericLiteral", + value: 1, + }, + start: 8, + type: "BinaryExpression", + }, + loc: { + end: { + column: 13, + line: 1, + }, + start: { + column: 4, + line: 1, + }, + }, + start: 4, + type: "VariableDeclarator", + }, + ], + end: 14, + kind: "var", + loc: { + end: { + column: 14, + line: 1, + }, + start: { + column: 0, + line: 1, + }, + }, + start: 0, + type: "VariableDeclaration", + }, + ], + directives: [], + end: 15, + loc: { + end: { + column: 0, + line: 2, + }, + start: { + column: 0, + line: 1, + }, + }, + sourceType: "module", + start: 0, + type: "Program", + }, + start: 0, + type: "File", +} \ No newline at end of file diff --git a/fixtures/issue65-op.js.sem.uast b/fixtures/issue65-op.js.sem.uast new file mode 100644 index 0000000..0155ba8 --- /dev/null +++ b/fixtures/issue65-op.js.sem.uast @@ -0,0 +1,134 @@ +{ '@type': "javascript:File", + '@role': [File], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 15, + line: 2, + col: 1, + }, + }, + comments: [], + program: { '@type': "javascript:Program", + '@role': [Module], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 15, + line: 2, + col: 1, + }, + }, + body: [ + { '@type': "javascript:VariableDeclaration", + '@role': [Declaration, Statement, Variable], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 14, + line: 1, + col: 15, + }, + }, + declarations: [ + { '@type': "javascript:VariableDeclarator", + '@role': [Declaration, Variable], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 4, + line: 1, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 13, + line: 1, + col: 14, + }, + }, + id: { '@type': "uast:Identifier", + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 4, + line: 1, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 5, + line: 1, + col: 6, + }, + }, + Name: "a", + }, + init: { '@type': "javascript:BinaryExpression", + '@role': [Add, Arithmetic, Binary, Expression, Initialization, Operator], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 8, + line: 1, + col: 9, + }, + end: { '@type': "uast:Position", + offset: 13, + line: 1, + col: 14, + }, + }, + left: { '@type': "javascript:NumericLiteral", + '@token': 0, + '@role': [Binary, Expression, Left, Literal, Number], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 8, + line: 1, + col: 9, + }, + end: { '@type': "uast:Position", + offset: 9, + line: 1, + col: 10, + }, + }, + }, + operator: { '@type': "uast:Operator", + '@token': "+", + '@role': [Add, Arithmetic, Binary, Expression, Operator], + }, + right: { '@type': "javascript:NumericLiteral", + '@token': 1, + '@role': [Binary, Expression, Literal, Number, Right], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 12, + line: 1, + col: 13, + }, + end: { '@type': "uast:Position", + offset: 13, + line: 1, + col: 14, + }, + }, + }, + }, + }, + ], + kind: "var", + }, + ], + directives: [], + sourceType: "module", + }, +} \ No newline at end of file diff --git a/fixtures/issue65-op.js.uast b/fixtures/issue65-op.js.uast new file mode 100644 index 0000000..7761f13 --- /dev/null +++ b/fixtures/issue65-op.js.uast @@ -0,0 +1,135 @@ +{ '@type': "File", + '@role': [File], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 15, + line: 2, + col: 1, + }, + }, + comments: [], + program: { '@type': "Program", + '@role': [Module], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 15, + line: 2, + col: 1, + }, + }, + body: [ + { '@type': "VariableDeclaration", + '@role': [Declaration, Statement, Variable], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 0, + line: 1, + col: 1, + }, + end: { '@type': "uast:Position", + offset: 14, + line: 1, + col: 15, + }, + }, + declarations: [ + { '@type': "VariableDeclarator", + '@role': [Declaration, Variable], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 4, + line: 1, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 13, + line: 1, + col: 14, + }, + }, + id: { '@type': "Identifier", + '@token': "a", + '@role': [Expression, Identifier], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 4, + line: 1, + col: 5, + }, + end: { '@type': "uast:Position", + offset: 5, + line: 1, + col: 6, + }, + }, + }, + init: { '@type': "BinaryExpression", + '@role': [Add, Arithmetic, Binary, Expression, Initialization, Operator], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 8, + line: 1, + col: 9, + }, + end: { '@type': "uast:Position", + offset: 13, + line: 1, + col: 14, + }, + }, + left: { '@type': "NumericLiteral", + '@token': 0, + '@role': [Binary, Expression, Left, Literal, Number], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 8, + line: 1, + col: 9, + }, + end: { '@type': "uast:Position", + offset: 9, + line: 1, + col: 10, + }, + }, + }, + operator: { '@type': "uast:Operator", + '@token': "+", + '@role': [Add, Arithmetic, Binary, Expression, Operator], + }, + right: { '@type': "NumericLiteral", + '@token': 1, + '@role': [Binary, Expression, Literal, Number, Right], + '@pos': { '@type': "uast:Positions", + start: { '@type': "uast:Position", + offset: 12, + line: 1, + col: 13, + }, + end: { '@type': "uast:Position", + offset: 13, + line: 1, + col: 14, + }, + }, + }, + }, + }, + ], + kind: "var", + }, + ], + directives: [], + sourceType: "module", + }, +} \ No newline at end of file