From 62a6b0da3aee8036e7328469951144a81f8b82d5 Mon Sep 17 00:00:00 2001 From: Denys Smirnov Date: Wed, 10 Apr 2019 18:11:37 +0300 Subject: [PATCH] update sdk version Signed-off-by: Denys Smirnov --- Gopkg.lock | 6 +- Gopkg.toml | 2 +- .../bblfsh/sdk.v2/driver/transforms.go | 145 +++-- .../gopkg.in/bblfsh/sdk.v2/protocol/driver.go | 29 +- .../bblfsh/sdk.v2/protocol/driver.pb.go | 24 +- vendor/gopkg.in/bblfsh/sdk.v2/uast/iter.go | 2 + .../gopkg.in/bblfsh/sdk.v2/uast/nodes/node.go | 168 +++++- .../gopkg.in/bblfsh/sdk.v2/uast/role/role.go | 57 +- .../bblfsh/sdk.v2/uast/transformer/arrays.go | 60 +- .../bblfsh/sdk.v2/uast/transformer/ast.go | 14 +- .../bblfsh/sdk.v2/uast/transformer/errors.go | 30 +- .../bblfsh/sdk.v2/uast/transformer/ops.go | 522 +++++++++++++----- .../sdk.v2/uast/transformer/semantic.go | 240 ++++++-- .../sdk.v2/uast/transformer/transformer.go | 33 +- vendor/gopkg.in/bblfsh/sdk.v2/uast/types.go | 160 ++++-- vendor/gopkg.in/bblfsh/sdk.v2/uast/uast.go | 362 ++++++++++-- 16 files changed, 1493 insertions(+), 361 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a75f78c..978b72c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -636,8 +636,8 @@ "uast/role", "uast/transformer" ] - revision = "910019c320386fa99a2e283d80c5c1769717d545" - version = "v2.9.0" + revision = "a7cb6c25d2968d3f254149b7b878c4f5270dd1b6" + version = "v2.16.4" [[projects]] name = "gopkg.in/src-d/enry.v1" @@ -669,6 +669,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "6012e33cb2d44742fbf9227646712a87cdc76c32d5facc7961e7bbc456dcc9a7" + inputs-digest = "9993b8dc4aa7089182f23ee9d4b968f8b24b8f2fef8a17b397d42fa8c0115896" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 141e69c..1917941 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -47,7 +47,7 @@ [[constraint]] name = "gopkg.in/bblfsh/sdk.v2" - version = "2.9.x" + version = "2.16.x" [[constraint]] name = "gopkg.in/src-d/enry.v1" diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/driver/transforms.go b/vendor/gopkg.in/bblfsh/sdk.v2/driver/transforms.go index a36d29a..cd4b678 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/driver/transforms.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/driver/transforms.go @@ -9,26 +9,69 @@ import ( "gopkg.in/bblfsh/sdk.v2/uast/transformer" ) -// Transforms describes a set of AST transformations this driver requires. +// Transforms describes a set of AST transformations the driver requires. +// +// The pipeline can be illustrated as: +// ( AST )--------------> ( ModeNative ) +// V +// [ Preprocess ] +// [ PreprocessCode ]--------> ( ModePreprocessed ) +// | +// |----------------\ +// | [ Normalize ] +// [ Annotations ] <-------/ +// | +// V +// [ Code ]-------------\ +// | [ Namespace ] +// | | +// V V +// ( ModeAnnotated ) ( ModeSemantic ) type Transforms struct { - // Namespace for this language driver + // Namespace for native AST nodes of this language. Only enabled in Semantic mode. + // + // Namespace will be set at the end of the pipeline, thus all transforms can + // use type names without the driver namespace. Namespace string - // Preprocess transforms normalizes native AST. - // It usually includes: - // * changing type key to uast.KeyType - // * changing token key to uast.KeyToken - // * restructure positional information + + // Preprocess stage normalizes native AST for both Annotated and Semantic stages. + // + // It usually: + // * changes type key to uast.KeyType + // * restructures positional information + // * fixes any issues with native AST structure Preprocess []transformer.Transformer - // Normalize converts known AST structures to high-level AST representation (UAST). + + // PreprocessCode stage runs code-assisted transformations after the Preprocess stage. + // It can be used to fix node tokens or positional information based on the source. + PreprocessCode []transformer.CodeTransformer + + // Normalize stage converts a known native AST structures to a canonicalized + // high-level AST representation (UAST). It is executed after PreprocessCode + // and before the Annotations stage. Normalize []transformer.Transformer - // Annotations transforms annotates the tree with roles. + + // Annotations stage applies UAST role annotations and is executed after + // Semantic stage, or after PreprocessCode if Semantic is disabled. + // + // It also changes token key to uast.KeyToken. It should not be done in the + // Preprocess stage, because Semantic annotations are easier on clean native AST. Annotations []transformer.Transformer - // Code transforms are applied directly after Native and provide a way + + // Code stage is applied directly after annotations and provides a way // to extract more information from source files, fix positional info, etc. + // + // TODO(dennwc): deprecate and examine drivers; documentation was incorrect + // it was described that the stage runs after Native, but it + // was always running after all annotation stages + // + // Deprecated: see PreprocessCode Code []transformer.CodeTransformer } -// Do applies AST transformation pipeline for specified nodes. +// Do applies AST transformation pipeline for specified AST subtree. +// +// Mode can be specified to stop the pipeline at a specific abstraction level. func (t Transforms) Do(rctx context.Context, mode Mode, code string, nd nodes.Node) (nodes.Node, error) { sp, ctx := opentracing.StartSpanFromContext(rctx, "uast.Transform") defer sp.Finish() @@ -43,27 +86,10 @@ func (t Transforms) Do(rctx context.Context, mode Mode, code string, nd nodes.No return nd, nil } - var err error - - nd, err = t.do(ctx, mode, nd) - if err != nil { - return nd, err - } - - nd, err = t.doCode(ctx, code, nd) - if err != nil { - return nd, err - } - - nd, err = t.doPost(ctx, mode, nd) - if err != nil { - return nd, err - } - - return nd, nil + return t.do(ctx, mode, code, nd) } -func (t Transforms) do(ctx context.Context, mode Mode, nd nodes.Node) (nodes.Node, error) { +func (t Transforms) do(ctx context.Context, mode Mode, code string, nd nodes.Node) (nodes.Node, error) { var err error runAll := func(name string, list []transformer.Transformer) error { sp, _ := opentracing.StartSpanFromContext(ctx, "uast.Transform."+name) @@ -77,47 +103,62 @@ func (t Transforms) do(ctx context.Context, mode Mode, nd nodes.Node) (nodes.Nod } return nil } + runAllCode := func(name string, list []transformer.CodeTransformer) error { + sp, _ := opentracing.StartSpanFromContext(ctx, "uast.Transform."+name) + defer sp.Finish() + + for _, ct := range list { + t := ct.OnCode(code) + nd, err = t.Do(nd) + if err != nil { + return err + } + } + return nil + } + + // Preprocess AST and optionally use the second pre-processing stage + // that can access the source code (to fix tokens, for example). if err := runAll("preprocess", t.Preprocess); err != nil { return nd, err } + if err := runAllCode("preprocess-code", t.PreprocessCode); err != nil { + return nd, err + } + + // First run Semantic mode (UAST canonicalization). + // It's considered a more high-level representation, but it needs + // a clean AST to run, so we execute it before Annotated mode. if mode >= ModeSemantic { if err := runAll("semantic", t.Normalize); err != nil { return nd, err } } + + // Next run Annotated mode. It won't see nodes converted by Semantic nodes, + // because it expects a clean native AST. + // This is intentional — Semantic nodes are already defined with specific + // roles in mind, thus they shouldn't be annotated further on this stage. if mode >= ModeAnnotated { if err := runAll("annotated", t.Annotations); err != nil { return nd, err } } - return nd, nil -} -func (t Transforms) doCode(ctx context.Context, code string, nd nodes.Node) (nodes.Node, error) { - sp, _ := opentracing.StartSpanFromContext(ctx, "uast.Transform.onCode") - defer sp.Finish() - - var err error - for _, ct := range t.Code { - t := ct.OnCode(code) - nd, err = t.Do(nd) - if err != nil { - return nd, err - } + // Run a code-assisted post-processing. Deprecated. + // There is no real reason to run it after all other stages except Preprocess. + if err := runAllCode("on-code", t.Code); err != nil { + return nd, err } - return nd, nil -} - -func (t Transforms) doPost(ctx context.Context, mode Mode, nd nodes.Node) (nodes.Node, error) { - sp, _ := opentracing.StartSpanFromContext(ctx, "uast.Transform.namespace") - defer sp.Finish() - var err error + // All native nodes should have a namespace in Semantic mode. + // Set if it was specified in the transform configuration. if mode >= ModeSemantic && t.Namespace != "" { - nd, err = transformer.DefaultNamespace(t.Namespace).Do(nd) - if err != nil { + tr := transformer.DefaultNamespace(t.Namespace) + if err := runAll("namespace", []transformer.Transformer{tr}); err != nil { return nd, err } } + return nd, nil } diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/protocol/driver.go b/vendor/gopkg.in/bblfsh/sdk.v2/protocol/driver.go index ee742e6..fdc73ac 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/protocol/driver.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/protocol/driver.go @@ -20,32 +20,49 @@ import ( //go:generate protoc --proto_path=$GOPATH/src:. --gogo_out=plugins=grpc:. ./driver.proto +const ( + mb = 1 << 20 + + // DefaultGRPCMaxMessageBytes is maximum msg size for gRPC. + DefaultGRPCMaxMessageBytes = 100 * mb +) + // ServerOptions returns a set of common options that should be used in bblfsh server. // // It automatically enables OpenTrace if a global tracer is set. func ServerOptions() []grpc.ServerOption { + opts := []grpc.ServerOption{ + grpc.MaxSendMsgSize(DefaultGRPCMaxMessageBytes), + grpc.MaxRecvMsgSize(DefaultGRPCMaxMessageBytes), + } tracer := opentracing.GlobalTracer() if _, ok := tracer.(opentracing.NoopTracer); ok { - return nil + return opts } - return []grpc.ServerOption{ + opts = append(opts, grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(tracer)), grpc.StreamInterceptor(otgrpc.OpenTracingStreamServerInterceptor(tracer)), - } + ) + return opts } // DialOptions returns a set of common options that should be used when dialing bblfsh server. // // It automatically enables OpenTrace if a global tracer is set. func DialOptions() []grpc.DialOption { + opts := []grpc.DialOption{grpc.WithDefaultCallOptions( + grpc.MaxCallSendMsgSize(DefaultGRPCMaxMessageBytes), + grpc.MaxCallRecvMsgSize(DefaultGRPCMaxMessageBytes), + )} tracer := opentracing.GlobalTracer() if _, ok := tracer.(opentracing.NoopTracer); ok { - return nil + return opts } - return []grpc.DialOption{ + opts = append(opts, grpc.WithUnaryInterceptor(otgrpc.OpenTracingClientInterceptor(tracer)), grpc.WithStreamInterceptor(otgrpc.OpenTracingStreamClientInterceptor(tracer)), - } + ) + return opts } func RegisterDriver(srv *grpc.Server, d driver.Driver) { diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/protocol/driver.pb.go b/vendor/gopkg.in/bblfsh/sdk.v2/protocol/driver.pb.go index f80f58b..0f1cd29 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/protocol/driver.pb.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/protocol/driver.pb.go @@ -19,8 +19,10 @@ import fmt "fmt" import math "math" import _ "github.com/gogo/protobuf/gogoproto" -import context "golang.org/x/net/context" -import grpc "google.golang.org/grpc" +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) import io "io" @@ -301,6 +303,24 @@ func (m *ParseError) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Driver(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Driver(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintDriver(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/uast/iter.go b/vendor/gopkg.in/bblfsh/sdk.v2/uast/iter.go index 9ed2960..3b8dffc 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/uast/iter.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/uast/iter.go @@ -77,6 +77,7 @@ func (it *positionIter) sort() { plist = nil } +// Next implements nodes.Iterator. func (it *positionIter) Next() bool { if it.nodes == nil { it.sort() @@ -89,6 +90,7 @@ func (it *positionIter) Next() bool { return len(it.nodes) != 0 } +// Node implements nodes.Iterator. func (it *positionIter) Node() nodes.External { if len(it.nodes) == 0 { return nil diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/uast/nodes/node.go b/vendor/gopkg.in/bblfsh/sdk.v2/uast/nodes/node.go index 8fd3ef6..2146fa6 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/uast/nodes/node.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/uast/nodes/node.go @@ -18,8 +18,61 @@ func Equal(n1, n2 External) bool { } else if n1 == nil || n2 == nil { return false } - if Same(n1, n2) { - return true + switch n1 := n1.(type) { + case String: + n2, ok := n2.(String) + if ok { + return n1 == n2 + } + case Int: + n2, ok := n2.(Int) + if ok { + return n1 == n2 + } + case Uint: + n2, ok := n2.(Uint) + if ok { + return n1 == n2 + } + case Bool: + n2, ok := n2.(Bool) + if ok { + return n1 == n2 + } + case Float: + n2, ok := n2.(Float) + if ok { + return n1 == n2 + } + case Object: + if n2, ok := n2.(Object); ok { + if len(n1) != len(n2) { + return false + } + if pointerOf(n1) == pointerOf(n2) { + return true + } + return n1.EqualObject(n2) + } + if _, ok := n2.(Node); ok { + return false + } + return n1.Equal(n2) + case Array: + if n2, ok := n2.(Array); ok { + if len(n1) != len(n2) { + return false + } + return len(n1) == 0 || &n1[0] == &n2[0] || n1.EqualArray(n2) + } + if _, ok := n2.(Node); ok { + return false + } + return n1.Equal(n2) + default: + if Same(n1, n2) { + return true + } } if n, ok := n1.(Node); ok { return n.Equal(n2) @@ -29,6 +82,61 @@ func Equal(n1, n2 External) bool { return equalExt(n1, n2) } +// NodeEqual compares two subtrees. +// Equality is checked by value (deep), not by reference. +func NodeEqual(n1, n2 Node) bool { + if n1 == nil && n2 == nil { + return true + } else if n1 == nil || n2 == nil { + return false + } + switch n1 := n1.(type) { + case String: + n2, ok := n2.(String) + return ok && n1 == n2 + case Int: + n2, ok := n2.(Int) + if ok { + return n1 == n2 + } + case Uint: + n2, ok := n2.(Uint) + if ok { + return n1 == n2 + } + case Bool: + n2, ok := n2.(Bool) + return ok && n1 == n2 + case Float: + n2, ok := n2.(Float) + if ok { + return n1 == n2 + } + case Object: + n2, ok := n2.(Object) + if !ok { + return false + } + if len(n1) != len(n2) { + return false + } + if pointerOf(n1) == pointerOf(n2) { + return true + } + return n1.EqualObject(n2) + case Array: + n2, ok := n2.(Array) + if !ok { + return false + } + if len(n1) != len(n2) { + return false + } + return len(n1) == 0 || &n1[0] == &n2[0] || n1.EqualArray(n2) + } + return n1.Equal(n2) +} + // Node is a generic interface for a tree structure. // // Can be one of: @@ -896,12 +1004,40 @@ func Same(n1, n2 External) bool { // second node is external, need to call SameAs on it return n2.SameAs(n1) } + // fast path + switch i1 := i1.(type) { + case Object: + i2, ok := i2.(Object) + if !ok || len(i1) != len(i2) { + return false + } + return pointerOf(i1) == pointerOf(i2) + case Array: + i2, ok := i2.(Array) + if !ok || len(i1) != len(i2) { + return false + } + if i1 == nil && i2 == nil { + return true + } else if i1 == nil || i2 == nil { + return false + } else if len(i1) == 0 { + return true + } + return &i1[0] == &i2[0] + case Value: + i2, ok := i2.(Value) + if !ok { + return false + } + return i1 == i2 + } // both nodes are internal - compare unique key return UniqueKey(i1) == UniqueKey(i2) } // pointerOf returns a Go pointer for Node that is a reference type (Arrays and Objects). -func pointerOf(n Node) uintptr { +func pointerOf(n interface{}) uintptr { if n == nil { return 0 } @@ -948,3 +1084,29 @@ func UniqueKey(n Node) Comparable { return unkPtr(ptr) } } + +// ChildrenCount reports the number of immediate children of n. If n is an Array this is +// the length of the array. If n is an Object, each object in a field of n counts as +// one child and each array is counted as its length. +func ChildrenCount(n Node) int { + switch n := n.(type) { + case nil: + return 0 + case Value: + return 0 + case Array: + return len(n) + case Object: + c := 0 + for _, v := range n { + switch v := v.(type) { + case Object: + c++ + case Array: + c += len(v) + } + } + return c + } + return 0 +} diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/uast/role/role.go b/vendor/gopkg.in/bblfsh/sdk.v2/uast/role/role.go index 5c30a37..4b41fd7 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/uast/role/role.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/uast/role/role.go @@ -49,10 +49,11 @@ const ( // Operator is any form of operator. Operator - // Binary is any form of binary operator, in contrast with unary operators. + // Binary is any form of binary operator or expression in contrast with unary operators / + // expressions. Binary - // Unary is any form of unary operator, in contrast with binary operators. + // Unary is any form of unary operator or expression, in contrast with binary. Unary // Left is a left hand side in a binary expression. @@ -69,13 +70,13 @@ const ( // Nodes without Infix or Postfix mark are considered in prefix order by default. Postfix - // Bitwise is any form of bitwise operation. + // Bitwise is any form of bitwise operation, expression or declaration. Bitwise - // Boolean is any form of boolean operation. + // Boolean is any form of boolean operation, expression or declaration. Boolean - // Unsigned is an form of unsigned operation. + // Unsigned is an form of unsigned operation, expression or declaration. Unsigned // LeftShift is a left shift operation (i.e. `<<`, `rol`, etc.) @@ -96,7 +97,7 @@ const ( // Expression is a construct computed to produce some value. Expression - // Statement is some action to be carried out. + // Statement is some action to be carried out usually without producing a value. Statement // Equal is an eaquality predicate (i.e. `=`, `==`, etc.) @@ -111,7 +112,7 @@ const ( // LessThanOrEqual is a comparison predicate that checks if the lhs value is smaller or equal to the rhs value (i.e. `<=`.) LessThanOrEqual - // GreaterThan is a comparison predicate that checks if the lhs value is greather than the rhs value (i. e. `>`.) + // GreaterThan is a comparison predicate that checks if the lhs value is greater than the rhs value (i. e. `>`.) GreaterThan // GreaterThanOrEqual is a comparison predicate that checks if the lhs value is greather than or equal to the rhs value (i.e. 1>=`.) @@ -123,10 +124,10 @@ const ( // Contains is a membership predicate that checks if the lhs value is a member of the rhs container (i.e. `in` in Python.) Contains - // Increment is an arithmetic operator that increments a value (i. e. `++i`.) + // Increment is an arithmetic operator that increments a value (i.e. `++i`.) Increment - // Decrement is an arithmetic operator that decrements a value (i. e. `--i`.) + // Decrement is an arithmetic operator that decrements a value (i.e. `--i`.) Decrement // Negative is an arithmetic operator that negates a value (i.e. `-x`.) @@ -213,10 +214,10 @@ const ( // Module is a set of funcitonality grouped. Module - // Friend is an access granter for some private resources. + // Friend is an access granter for some private resources to "friend" types. Friend - // World is a set of every component. + // World is a public access granter. World // If is used for if-then[-else] statements or expressions. @@ -253,7 +254,7 @@ const ( // Else is the clause executed when the Condition is false. Else - // Switch is used to represent a broad of switch flavors. An expression + // Switch is used to represent many switch flavors. An expression // is evaluated and then compared to the values returned by different // case expressions, executing a body associated to the first case that // matches. Similar constructions that go beyond expression comparison @@ -293,11 +294,11 @@ const ( // Continue is a construct for continuation with the next iteration of a loop. Continue - // Goto is an unconditional transfer of control statement. + // Goto is an unconditional transfer of the program flow to another place. Goto // Block is a group of statements. If the source language has block scope, - // it should be annotated both with Block and BlockScope. + // it should be annotated both with Block and Scope. Block // Scope is a range in which a variable can be referred. @@ -313,14 +314,15 @@ const ( // Catch is a clause to capture exceptions. Catch - // Finally is a clause for a block executed after a block with exception handling. + // Finally defines a block or expression to be run inconditionally after an + // expression handling block. Finally - // Throw is a statement that creates an exception. + // Throw is a statement that creates and throw an exception. Throw - // Assert checks if an expression is true and if it is not, it signals - // an error/exception, possibly stopping the execution. + // Assert checks if an expression is true and, if it is not, signals + // an error/exception, usually stopping the program's execution. Assert // Call is any call, whether it is a function, procedure, method or macro. @@ -337,7 +339,8 @@ const ( // Positional is an element which position has meaning (i.e. a positional argument in a call). Positional - // Noop is a construct that does nothing. + // Noop is a construct that does nothing like comments, documentation or, in some + // languages like Python, annotations. Noop // Literal is a literal value. @@ -417,27 +420,29 @@ const ( // Unannotated will be automatically added by the SDK for nodes that did not receive // any annotations with the current version of the driver's `annotations.go` file. - // Added in BIP-002. + // Added in BIP-002. Having a driver produce Unannotated nodes should be considered + // a bug. Unannotated // Visibility is an access granter role, usually together with an specifier role Visibility - // Annotation is syntactic metadata + // Annotation is syntactic metadata with or without semantic meaning. Annotation - // Anonymous is an unbound construct + // Anonymous is an construct not bound to name i.e. lambda functions. Anonymous - // Enumeration is a distinct type that represents a set of named constants + // Enumeration is a distinct type that represents a set of named constants. Enumeration - // Arithmetic is a type of operation + // Arithmetic is a type of arithmetic operation. Arithmetic - // Relational is a type of operation + // Relational is a type of relational operation i.e. "equal", "not equal", + // "less than"... Relational - // Variable is a symbolic name associatend with a value + // Variable is a symbolic name associatend with a value. Variable ) diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/arrays.go b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/arrays.go index 82d5f0d..1c5da50 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/arrays.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/arrays.go @@ -67,6 +67,63 @@ func One(op Op) ArrayOp { return Arr(op) } +// AnyElem check matches if any of list elements matches sub-check. +func AnyElem(s Sel) Sel { + if s == nil { + s = Is(nil) + } + return &opAnyElem{sel: s} +} + +type opAnyElem struct { + sel Sel +} + +func (*opAnyElem) Kinds() nodes.Kind { + return nodes.KindArray +} + +func (op *opAnyElem) Check(st *State, n nodes.Node) (bool, error) { + l, ok := n.(nodes.Array) + if !ok { + return false, nil + } + for _, o := range l { + if ok, err := op.sel.Check(st.Clone(), o); err != nil { + return false, err + } else if ok { + return true, nil + } + } + return false, nil +} + +// All check matches if all list elements matches sub-check. +func All(s Sel) Sel { + return &opAll{sel: s} +} + +type opAll struct { + sel Sel +} + +func (*opAll) Kinds() nodes.Kind { + return nodes.KindArray +} + +func (op *opAll) Check(st *State, n nodes.Node) (bool, error) { + l, ok := n.(nodes.Array) + if !ok { + return false, nil + } + for _, o := range l { + if ok, err := op.sel.Check(st.Clone(), o); err != nil || !ok { + return false, err + } + } + return true, nil +} + // LookupArrOpVar is like LookupOpVar but returns an array operation. // Default value can be specified by setting the nil key. func LookupArrOpVar(vr string, cases map[nodes.Value]ArrayOp) ArrayOp { @@ -203,9 +260,6 @@ func (op opAppend) Check(st *State, n nodes.Node) (bool, error) { // and the part we will use for sub-array checks tail := len(arr) - len(sarr) sub, arrs := arr[:tail], arr[tail:] - if len(sub) == 0 { - sub = nil - } if ok, err := op.op.Check(st, sub); err != nil { return false, errAppend.Wrap(err) } else if !ok { diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/ast.go b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/ast.go index f492d94..1c85df9 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/ast.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/ast.go @@ -72,7 +72,7 @@ func RolesFieldOp(vr string, op ArrayOp, roles ...role.Role) Field { func ASTObjectLeft(typ string, ast ObjectOp) ObjectOp { if fields, ok := ast.Fields(); !ok { panic("unexpected partial transform") - } else if _, ok := fields[uast.KeyRoles]; ok { + } else if fields.Has(uast.KeyRoles) { panic("unexpected roles filed") } var obj Fields @@ -95,7 +95,7 @@ type RolesByType func(typ string) role.Roles func ASTObjectRightCustom(typ string, norm ObjectOp, fnc RolesByType, rop ArrayOp, roles ...role.Role) ObjectOp { if fields, ok := norm.Fields(); !ok { panic("unexpected partial transform") - } else if _, ok := fields[uast.KeyRoles]; ok { + } else if fields.Has(uast.KeyRoles) { panic("unexpected roles field") } var obj Fields @@ -412,11 +412,11 @@ func StringToRolesMap(m map[string][]role.Role) map[nodes.Value]ArrayOp { // Since rules are applied depth-first, this operation will work properly only in a separate mapping step. // In other cases it will apply itself before parent node appends field roles. func AnnotateIfNoRoles(typ string, roles ...role.Role) Mapping { - return Map( - Check( // TODO: CheckObj - Not(Has{ - uast.KeyRoles: AnyNode(nil), - }), + return MapObj( + CheckObj( + HasFields{ + uast.KeyRoles: false, + }, Part("_", Obj{ uast.KeyType: String(typ), }), diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/errors.go b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/errors.go index 89535dd..8137c98 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/errors.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/errors.go @@ -5,6 +5,9 @@ import ( "fmt" "gopkg.in/src-d/go-errors.v1" + + "gopkg.in/bblfsh/sdk.v2/uast" + "gopkg.in/bblfsh/sdk.v2/uast/nodes" ) var ( @@ -17,6 +20,12 @@ var ( // in Lookup or If, for example, before another transformation step that sets this variable is executed. // If this is the case, see Fields vs Obj comparison in regards to execution order. ErrVariableNotDefined = errors.NewKind("variable %q is not defined") + // ErrVariableUnused is returned when a first half of transformation defines a variable and the second part + // never uses it in any operations. + // + // If you receive this error and still think that every variable is used - double check conditional branches + // like Opt, If, Case, etc. + ErrVariableUnused = errors.NewKind("variables %q unused in the second part of the transform") // ErrExpectedObject is returned when transformation expected an object in the tree or variable, but got other type. ErrExpectedObject = errors.NewKind("expected object, got %T") // ErrExpectedList is returned when transformation expected an array in the tree or variable, but got other type. @@ -38,7 +47,9 @@ var ( // ErrUnusedField is returned when a transformation is not defined as partial, but does not process a specific key // found in object. This usually means that an AST has a field that is not covered by transformation code and it // should be added to the mapping. - ErrUnusedField = errors.NewKind("field was not used: %v") + // + // Use NewErrUnusedField for constructing this error. + ErrUnusedField = errors.NewKind("unused field(s) on node %v: %v") // ErrDuplicateField is returned when trying to create a Fields definition with two items with the same name. ErrDuplicateField = errors.NewKind("duplicate field: %v") // ErrUndefinedField is returned when trying to create an object with a field that is not defined in the type spec. @@ -83,3 +94,20 @@ func (e *MultiError) Error() string { } return buf.String() } + +// NewErrUnusedField is a helper for creating ErrUnusedField. +// +// It will include a short type information for the node to simplify debugging. +func NewErrUnusedField(n nodes.Object, fields []string) error { + var t interface{} + if typ, _ := n[uast.KeyType].(nodes.String); typ != "" { + t = typ + } else { + t = n.Keys() + } + var f interface{} = fields + if len(fields) == 1 { + f = fields[0] + } + return ErrUnusedField.New(t, f) +} diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/ops.go b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/ops.go index 08e739a..d8e0fe4 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/ops.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/ops.go @@ -69,7 +69,7 @@ func (op opIs) Kinds() nodes.Kind { } func (op opIs) Check(st *State, n nodes.Node) (bool, error) { - return nodes.Equal(op.n, n), nil + return nodes.NodeEqual(op.n, n), nil } func (op opIs) Construct(st *State, n nodes.Node) (nodes.Node, error) { @@ -122,6 +122,13 @@ func (op opVar) Construct(st *State, n nodes.Node) (nodes.Node, error) { } // AnyNode matches any node and throws it away. Reversal will create a node with create op. +// +// This operation should not be used thoughtlessly. Each field that is dropped this way +// is an information loss and may become a source of bugs. +// +// The preferred way is to assert an exact value with Is or similar operator. If possible, +// assert the type of expected node or any other field that indicates that this node +// is useless. func AnyNode(create Mod) Op { if create == nil { create = Is(nil) @@ -145,7 +152,14 @@ func (op opAnyNode) Construct(st *State, n nodes.Node) (nodes.Node, error) { return op.create.Construct(st, n) } -// AnyVal accept any value and aways creates a node with a provided one. +// Any matches any node and throws it away. It creates a nil node on reversal. +// See AnyNode for details. +func Any() Op { + return AnyNode(nil) +} + +// AnyVal accept any value and creates a provided value on reversal. +// See AnyNode for details. func AnyVal(val nodes.Value) Op { return AnyNode(Is(val)) } @@ -207,6 +221,7 @@ func (o Objs) ObjectOps() []ObjectOp { return l } +// EmptyObj checks that a node is an empty object. func EmptyObj() Op { return Is(nodes.Object{}) } @@ -220,19 +235,16 @@ func (Obj) Kinds() nodes.Kind { } func (o Obj) Fields() (FieldDescs, bool) { - fields := make(FieldDescs, len(o)) + d := NewFieldDescs(len(o)) for k, v := range o { f := FieldDesc{Optional: false} - if is, ok := v.(opIs); ok { - n := is.n - f.Fixed = &n - } - fields[k] = f + f.SetValue(v) + d.Set(k, f) } - return fields, true + return d, true } -// Object converts this helper to a full Object description. +// fields converts this helper to a full Fields description. func (o Obj) fields() Fields { fields := make(Fields, 0, len(o)) for k, op := range o { @@ -242,74 +254,175 @@ func (o Obj) fields() Fields { return fields } -// Check will make an Object operation from this helper and call Check on it. +// Check will convert the operation to Fields and will call Check on it. func (o Obj) Check(st *State, n nodes.Node) (bool, error) { return o.fields().Check(st, n) } -// Construct will make an Object operation from this helper and call Construct on it. +// Construct will convert the operation to Fields and will call Construct on it. func (o Obj) Construct(st *State, n nodes.Node) (nodes.Node, error) { return o.fields().Construct(st, n) } -// CheckObj will make an Object operation from this helper and call Check on it. +// CheckObj will convert the operation to Fields and will call CheckObj on it. func (o Obj) CheckObj(st *State, n nodes.Object) (bool, error) { return o.fields().CheckObj(st, n) } -// ConstructObj will make an Object operation from this helper and call Construct on it. +// ConstructObj will convert the operation to Fields and will call ConstructObj on it. func (o Obj) ConstructObj(st *State, n nodes.Object) (nodes.Object, error) { return o.fields().ConstructObj(st, n) } +// FieldDesc is a field descriptor for operations that act on objects. +// +// It is used for transformation optimizer to filter candidate nodes upfront +// without running the full transformation tree. type FieldDesc struct { - Optional bool // field might not exists in the object - Fixed *nodes.Node // field is required to have a fixed value; the value may be nil + // Optional indicates that field might not exists in the object. + Optional bool + // Fixed is set if a field is required to have a specific value. The value may be nil. + Fixed *nodes.Node } -// FieldDescs is a descriptions of static fields of an object. -// Transformation operation may return this struct to indicate what fields they will require. -type FieldDescs map[string]FieldDesc +// SetValue checks the selector for a fixed value and sets it for the field descriptor. +func (f *FieldDesc) SetValue(sel Sel) { + if is, ok := sel.(opIs); ok { + n := is.n + f.Fixed = &n + } +} -// Clone makes a copy of field description, without cloning each field values. -func (f FieldDescs) Clone() FieldDescs { +func NewFieldDescs(n int) FieldDescs { + if n == 0 { + return FieldDescs{} + } + return FieldDescs{ + fields: make([]fieldDesc, 0, n), + m: make(map[string]int, n), + } +} + +type fieldDesc struct { + name string + FieldDesc +} + +// FieldDescs contains descriptions of static fields of an object. +// +// Transformations may return this type to indicate what fields they will require. +// +// See FieldDesc for details. +type FieldDescs struct { + fields []fieldDesc + m map[string]int +} + +// Len returns a number of fields. +func (f *FieldDescs) Len() int { if f == nil { - return nil + return 0 } - fields := make(FieldDescs, len(f)) - for k, v := range f { - fields[k] = v + return len(f.fields) +} + +// Index return the field descriptor and its name, given an index. +func (f *FieldDescs) Index(i int) (FieldDesc, string) { + if f == nil || i < 0 || i >= len(f.fields) { + return FieldDesc{}, "" } - return fields + d := f.fields[i] + return d.FieldDesc, d.name +} + +// Has checks if a field with a given name exists. +func (f *FieldDescs) Has(k string) bool { + if f == nil { + return false + } + _, ok := f.m[k] + return ok +} + +// Get the field descriptor by its name. +func (f *FieldDescs) Get(k string) (FieldDesc, bool) { + if f == nil { + return FieldDesc{}, false + } + i, ok := f.m[k] + if !ok { + return FieldDesc{}, false + } + return f.fields[i].FieldDesc, true +} + +// Set a field descriptor by name. +func (f *FieldDescs) Set(k string, d FieldDesc) { + if i, ok := f.m[k]; ok { + f.fields[i].FieldDesc = d + return + } + i := len(f.fields) + f.fields = append(f.fields, fieldDesc{name: k, FieldDesc: d}) + if f.m == nil { + f.m = make(map[string]int) + } + f.m[k] = i +} + +// Clone makes a copy of field description, without cloning each field values. +func (f *FieldDescs) Clone() FieldDescs { + if f == nil || len(f.fields) == 0 { + return FieldDescs{} + } + f2 := NewFieldDescs(len(f.fields)) + f2.fields = f2.fields[:len(f.fields)] + copy(f2.fields, f.fields) + for k, v := range f.m { + f2.m[k] = v + } + return f2 } // CheckObj verifies that an object matches field descriptions. // It ignores all fields in the object that are not described. -func (f FieldDescs) CheckObj(n nodes.Object) bool { - for k, d := range f { +func (f *FieldDescs) CheckObj(n nodes.Object) bool { + if f == nil { + return true + } + for _, d := range f.fields { if d.Optional { continue } - v, ok := n[k] + v, ok := n[d.name] if !ok { return false } - if d.Fixed != nil && !nodes.Equal(*d.Fixed, v) { + if d.Fixed != nil && !nodes.NodeEqual(*d.Fixed, v) { return false } } return true } -// ObjectOp is an operation that is executed on an object. See Object. -type ObjectOp interface { - Op +// ObjectSel is a selector that matches objects. See Object. +type ObjectSel interface { + Sel // Fields returns a map of field names that will be processed by this operation. - // The flag if the map indicates if the field is required. - // False bool value returned as a second argument indicates that implementation will process all fields. + // The flag in the map indicates if the field is required. + // + // Returning true as a second argument indicates that the operation will always + // use all fields. Returning false means that an operation is partial. Fields() (FieldDescs, bool) CheckObj(st *State, n nodes.Object) (bool, error) +} + +// ObjectOp is an operation that is executed on an object. See Object. +type ObjectOp interface { + Mod + ObjectSel + ConstructObj(st *State, n nodes.Object) (nodes.Object, error) } @@ -387,7 +500,8 @@ func (op *opPartialObj) CheckObj(st *State, n nodes.Object) (bool, error) { // TODO: consider throwing an error if a transform is defined as partial, but in fact it's not other := n.CloneObject() n = make(nodes.Object) - for k := range op.used { + for _, d := range op.used.fields { + k := d.name if _, ok := other[k]; ok { n[k] = other[k] delete(other, k) @@ -442,7 +556,7 @@ func JoinObj(ops ...ObjectOp) ObjectOp { partial ObjectOp out []processedOp ) - required := make(FieldDescs) + required := NewFieldDescs(0) for _, s := range ops { if j, ok := s.(*opObjJoin); ok { if j.partial != nil { @@ -451,14 +565,15 @@ func JoinObj(ops ...ObjectOp) ObjectOp { } partial = j.partial } - for k, req := range j.allFields { - if req2, ok := required[k]; ok { + for _, req := range j.allFields.fields { + k := req.name + if req2, ok := required.Get(k); ok { // only allow this if values are fixed and equal - if req.Fixed == nil || req2.Fixed == nil || !nodes.Equal(*req.Fixed, *req2.Fixed) { + if req.Fixed == nil || req2.Fixed == nil || !nodes.NodeEqual(*req.Fixed, *req2.Fixed) { panic(ErrDuplicateField.New(k)) } } - required[k] = req + required.Set(k, req.FieldDesc) } out = append(out, j.ops...) continue @@ -471,20 +586,21 @@ func JoinObj(ops ...ObjectOp) ObjectOp { partial = s continue } - for k, req := range fields { - if _, ok := required[k]; ok { + for _, req := range fields.fields { + k := req.name + if required.Has(k) { panic(ErrDuplicateField.New(k)) } - required[k] = req + required.Set(k, req.FieldDesc) } out = append(out, processedOp{op: s, fields: fields}) } if partial != nil { - required = nil + required = NewFieldDescs(0) } for i := 0; i < len(out); i++ { op := out[i] - if len(op.fields) != 0 { + if len(op.fields.fields) != 0 { continue } if o, ok := op.op.(Obj); ok && len(o) == 0 && len(out) > 1 { @@ -529,8 +645,9 @@ func (op *opObjJoin) CheckObj(st *State, n nodes.Object) (bool, error) { src := n n = n.CloneObject() for _, s := range op.ops { - sub := make(nodes.Object, len(s.fields)) - for k := range s.fields { + sub := make(nodes.Object, len(s.fields.fields)) + for _, d := range s.fields.fields { + k := d.name if v, ok := src[k]; ok { sub[k] = v delete(n, k) @@ -544,6 +661,8 @@ func (op *opObjJoin) CheckObj(st *State, n nodes.Object) (bool, error) { if ok, err := op.partial.CheckObj(st, n); err != nil || !ok { return false, err } + } else if len(n) != 0 { + return false, NewErrUnusedField(src, n.Keys()) } return true, nil } @@ -559,7 +678,7 @@ func (op *opObjJoin) ConstructObj(st *State, n nodes.Object) (nodes.Object, erro return nil, err } for k, v := range np { - if v2, ok := n[k]; ok && !nodes.Equal(v, v2) { + if v2, ok := n[k]; ok && !nodes.NodeEqual(v, v2) { return nil, ErrDuplicateField.New(k) } n[k] = v @@ -571,9 +690,9 @@ func (op *opObjJoin) ConstructObj(st *State, n nodes.Object) (nodes.Object, erro return nil, err } for k, v := range n2 { - if v2, ok := n[k]; ok && !nodes.Equal(v, v2) { + if v2, ok := n[k]; ok && !nodes.NodeEqual(v, v2) { return nil, ErrDuplicateField.New(k) - } else if _, ok = s.fields[k]; !ok { + } else if !s.fields.Has(k) { return nil, fmt.Errorf("undeclared field was set: %v", k) } n[k] = v @@ -706,11 +825,26 @@ func (op opScope) Construct(st *State, n nodes.Node) (nodes.Node, error) { // Field is an operation on a specific field of an object. type Field struct { Name string // name of the field - // Optional can be set to make a field optional. Provided string is used as a variable name to the state of the field. - // Note that "optional" means that the field may not exists in the object, and it does not mean that the field can be nil. + // Optional can be set to make a field optional. Provided string is used as a variable + // name to the state of the field. Note that "optional" means that the field may not + // exists in the object, and it does not mean that the field can be nil. // To handle nil fields, see Opt operation. Optional string - Op Op // operation used to check/construct the field value + // Drop the field if it exists. Optional is implied, but the variable won't be created + // in this case. Op should be set and it will be called to check the value before + // dropping it. If the check fails, the whole transform will be canceled. + // + // Please note that you should avoid dropping fields with Any unless there is no + // reasonable alternative. + Drop bool + Op Op // operation used to check/construct the field value +} + +// Desc returns a field descriptor. +func (f Field) Desc() FieldDesc { + d := FieldDesc{Optional: f.Optional != "" || f.Drop} + d.SetValue(f.Op) + return d } var _ ObjectOp = Fields{} @@ -728,14 +862,9 @@ func (Fields) Kinds() nodes.Kind { } func (o Fields) Fields() (FieldDescs, bool) { - fields := make(FieldDescs, len(o)) + fields := NewFieldDescs(len(o)) for _, f := range o { - fld := FieldDesc{Optional: f.Optional != ""} - if is, ok := f.Op.(opIs); ok { - n := is.n - fld.Fixed = &n - } - fields[f.Name] = fld + fields.Set(f.Name, f.Desc()) } return fields, true } @@ -763,11 +892,11 @@ func (o Fields) CheckObj(st *State, n nodes.Object) (bool, error) { if err := st.SetVar(f.Optional, nodes.Bool(ok)); err != nil { return false, errKey.Wrap(err, f.Name) } - if !ok { - continue - } } if !ok { + if f.Optional != "" || f.Drop { + continue + } if errorOnFilterCheck { return filtered("field %+v is missing in %+v\n%+v", f, n, o) } @@ -783,8 +912,8 @@ func (o Fields) CheckObj(st *State, n nodes.Object) (bool, error) { if !allowUnusedFields { set, _ := o.Fields() // TODO: optimize for k := range n { - if _, ok := set[k]; !ok { - return false, ErrUnusedField.New(k) + if !set.Has(k) { + return false, NewErrUnusedField(n, []string{k}) } } } @@ -1224,8 +1353,72 @@ func (op *opCheck) Construct(st *State, n nodes.Node) (nodes.Node, error) { return op.op.Construct(st, n) } +// CheckObj is similar to Check, but accepts only object operators. +func CheckObj(s ObjectSel, op ObjectOp) ObjectOp { + // merge field descriptor once, so we don't have to compute them later + + // doesn't matter if check is marked as partial or not + // we always consider it as such + checks, _ := s.Fields() + // optional selectors doesn't make sense + for _, f := range checks.fields { + if f.Optional { + panic("optional fields are not allowed in CheckObj") + } + } + + // merge maps, prefer fields from op + fields, full := op.Fields() + for _, f := range fields.fields { + checks.Set(f.name, f.FieldDesc) + } + + return &opCheckObj{sel: s, op: op, fields: checks, full: full} +} + +type opCheckObj struct { + sel ObjectSel + op ObjectOp + fields FieldDescs + full bool +} + +func (op *opCheckObj) Kinds() nodes.Kind { + return nodes.KindObject +} + +func (op *opCheckObj) Fields() (FieldDescs, bool) { + return op.fields, op.full +} + +func (op *opCheckObj) CheckObj(st *State, n nodes.Object) (bool, error) { + if ok, err := op.sel.CheckObj(st.Clone(), n); err != nil || !ok { + return ok, err + } + return op.op.CheckObj(st, n) +} + +func (op *opCheckObj) ConstructObj(st *State, n nodes.Object) (nodes.Object, error) { + return op.op.ConstructObj(st, n) +} + +func (op *opCheckObj) Check(st *State, n nodes.Node) (bool, error) { + if ok, err := op.sel.Check(st.Clone(), n); err != nil || !ok { + return ok, err + } + return op.op.Check(st, n) +} + +func (op *opCheckObj) Construct(st *State, n nodes.Node) (nodes.Node, error) { + return op.op.Construct(st, n) +} + // Not negates the check. func Not(s Sel) Sel { + if k, ok := s.(opKind); ok { + // invert the kind mask + return opKind{k: nodes.KindsAny &^ k.k} + } return &opNot{sel: s} } @@ -1245,6 +1438,40 @@ func (op *opNot) Check(st *State, n nodes.Node) (bool, error) { return !ok, nil } +// ObjNot negates all checks on an object, while still asserting this node as an object. +func ObjNot(s ObjectSel) ObjectSel { + return &opObjNot{sel: s} +} + +type opObjNot struct { + sel ObjectSel +} + +func (*opObjNot) Kinds() nodes.Kind { + return nodes.KindObject +} + +func (op *opObjNot) Fields() (FieldDescs, bool) { + // TODO(dennwc): FieldDescs should contain negative checks as well + return FieldDescs{}, false // not sure; can be anything +} + +func (op *opObjNot) CheckObj(st *State, n nodes.Object) (bool, error) { + ok, err := op.sel.CheckObj(st.Clone(), n) + if err != nil { + return false, err + } + return !ok, nil +} + +func (op *opObjNot) Check(st *State, n nodes.Node) (bool, error) { + ok, err := op.sel.Check(st.Clone(), n) + if err != nil { + return false, err + } + return !ok, nil +} + // Not nil is a condition that ensures that node is not nil. func NotNil() Sel { return Not(Is(nil)) @@ -1276,63 +1503,6 @@ func (op opAnd) Check(st *State, n nodes.Node) (bool, error) { return true, nil } -// Any check matches if any of list elements matches sub-check. -func Any(s Sel) Sel { - if s == nil { - s = Is(nil) - } - return &opAny{sel: s} -} - -type opAny struct { - sel Sel -} - -func (*opAny) Kinds() nodes.Kind { - return nodes.KindArray -} - -func (op *opAny) Check(st *State, n nodes.Node) (bool, error) { - l, ok := n.(nodes.Array) - if !ok { - return false, nil - } - for _, o := range l { - if ok, err := op.sel.Check(st.Clone(), o); err != nil { - return false, err - } else if ok { - return true, nil - } - } - return false, nil -} - -// All check matches if all list elements matches sub-check. -func All(s Sel) Sel { - return &opAll{sel: s} -} - -type opAll struct { - sel Sel -} - -func (*opAll) Kinds() nodes.Kind { - return nodes.KindArray -} - -func (op *opAll) Check(st *State, n nodes.Node) (bool, error) { - l, ok := n.(nodes.Array) - if !ok { - return false, nil - } - for _, o := range l { - if ok, err := op.sel.Check(st.Clone(), o); err != nil || !ok { - return false, err - } - } - return true, nil -} - var _ Sel = Has{} func HasType(o interface{}) Sel { @@ -1345,6 +1515,8 @@ func HasType(o interface{}) Sel { return Has{uast.KeyType: String(typ)} } +var _ ObjectSel = Has{} + // Has is a check-only operation that verifies that object has specific fields and they match given checks. type Has map[string]Sel @@ -1352,14 +1524,20 @@ func (Has) Kinds() nodes.Kind { return nodes.KindObject } -// Check verifies that specified fields exists and matches the provided sub-operations. -func (m Has) Check(st *State, n nodes.Node) (bool, error) { - o, ok := n.(nodes.Object) - if !ok { - return false, nil +func (m Has) Fields() (FieldDescs, bool) { + desc := NewFieldDescs(len(m)) + for k, sel := range m { + f := FieldDesc{Optional: false} + f.SetValue(sel) + desc.Set(k, f) } + return desc, false +} + +// CheckObj verifies that specified fields exist and matches the provided sub-operations. +func (m Has) CheckObj(st *State, n nodes.Object) (bool, error) { for k, sel := range m { - v, ok := o[k] + v, ok := n[k] if !ok { return false, nil } @@ -1370,6 +1548,54 @@ func (m Has) Check(st *State, n nodes.Node) (bool, error) { return true, nil } +// Check verifies that specified fields exist and matches the provided sub-operations. +func (m Has) Check(st *State, n nodes.Node) (bool, error) { + o, ok := n.(nodes.Object) + if !ok { + return false, nil + } + return m.CheckObj(st, o) +} + +var _ ObjectSel = HasFields{} + +// HasFields is a check-only operation that verifies existence of specific fields. +type HasFields map[string]bool + +func (HasFields) Kinds() nodes.Kind { + return nodes.KindObject +} + +func (m HasFields) Fields() (FieldDescs, bool) { + desc := NewFieldDescs(len(m)) + for k, expect := range m { + if expect { + desc.Set(k, FieldDesc{Optional: false}) + } + } + return desc, false +} + +// CheckObj verifies that specified fields exist and matches the provided sub-operations. +func (m HasFields) CheckObj(st *State, n nodes.Object) (bool, error) { + for k, expect := range m { + _, ok := n[k] + if ok != expect { + return false, nil + } + } + return true, nil +} + +// Check verifies that specified fields exist and matches the provided sub-operations. +func (m HasFields) Check(st *State, n nodes.Node) (bool, error) { + o, ok := n.(nodes.Object) + if !ok { + return false, nil + } + return m.CheckObj(st, o) +} + // In check that the node is a value from a given list. func In(vals ...nodes.Value) Sel { m := make(map[nodes.Value]struct{}, len(vals)) @@ -1400,10 +1626,32 @@ func (op *opIn) Check(st *State, n nodes.Node) (bool, error) { return ok, nil } +// Cases acts like a switch statement: it checks multiple operations, picks one that +// matches node structure and writes the number of the taken branch to the variable. +// +// Operations in branches should not be ambiguous. They should not overlap. func Cases(vr string, cases ...Op) Op { return &opCases{vr: vr, cases: cases} } +// CasesObj is similar to Cases, but only works on object nodes. It also allows to specify +// a common set of operations that will be executed for each branch. +// +// It is also required for all branches in CasesObj to have exactly the same set of keys. +// +// Example: +// CasesObj("case", +// // common +// Obj{ +// "type": String("ident"), +// }, +// Objects{ +// // case 1: A +// {"name": String("A")}, +// // case 1: B +// {"name": String("B")}, +// }, +// ) func CasesObj(vr string, common ObjectOp, cases ObjectOps) ObjectOp { list := cases.ObjectOps() if len(list) == 0 { @@ -1416,26 +1664,26 @@ func CasesObj(vr string, common ObjectOp, cases ObjectOps) ObjectOp { if !ok { panic("partial transforms are not allowed in Cases") } - for _, f := range arr { + for _, f := range arr.fields { if f.Optional { panic("optional fields are not allowed in Cases") } } if i == 0 { // use as a baseline wipe all specific constraints (might differ in cases) - fields = make(FieldDescs, len(arr)) - for k := range arr { - fields[k] = FieldDesc{Optional: false} + fields = NewFieldDescs(len(arr.fields)) + for _, f := range arr.fields { + fields.Set(f.name, FieldDesc{Optional: false}) } continue } // check that all other cases mention the case set of fields - if len(arr) != len(fields) { + if len(arr.fields) != len(fields.fields) { panic("all cases should have the same number of fields") } - for k := range arr { - if _, ok := fields[k]; !ok { - panic(fmt.Errorf("field %s does not exists in case %d", k, i)) + for _, f := range arr.fields { + if !fields.Has(f.name) { + panic(fmt.Errorf("field %s does not exists in case %d", f.name, i)) } } } diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/semantic.go b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/semantic.go index 234c552..e487510 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/semantic.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/semantic.go @@ -30,10 +30,11 @@ func uastType(uobj interface{}, op ObjectOp, part string) ObjectOp { if len(zero) == 0 { return JoinObj(obj, op) } - for k := range fields { - if k == uast.KeyType { + for _, f := range fields.fields { + if f.name == uast.KeyType { continue } + k := f.name _, ok := zero[k] _, ok2 := opt[k] if !ok && !ok2 { @@ -94,9 +95,26 @@ func MapSemanticPos(nativeType string, semType interface{}, pos map[string]strin } func CommentText(tokens [2]string, vr string) Op { - return commentUAST{ - tokens: tokens, - text: vr + "_text", pref: vr + "_pref", suff: vr + "_suff", tab: vr + "_tab", + return &commentUAST{ + startToken: tokens[0], + endToken: tokens[1], + textVar: vr + "_text", + prefVar: vr + "_pref", + suffVar: vr + "_suff", + indentVar: vr + "_tab", + doTrim: false, + } +} + +func CommentTextTrimmed(tokens [2]string, vr string) Op { + return &commentUAST{ + startToken: tokens[0], + endToken: tokens[1], + textVar: vr + "_text", + prefVar: vr + "_pref", + suffVar: vr + "_suff", + indentVar: vr + "_tab", + doTrim: true, } } @@ -114,72 +132,204 @@ func CommentNode(block bool, vr string, pos Op) ObjectOp { return UASTType(uast.Comment{}, obj) } -type commentUAST struct { - tokens [2]string - text string - pref, suff, tab string +// commentElems contains individual comment elements. +// See uast.Comment for details. +type commentElems struct { + StartToken string + EndToken string + Text string + Prefix string + Suffix string + Indent string + DoTrim bool } -func (commentUAST) Kinds() nodes.Kind { - return nodes.KindString +func (c *commentElems) isTab(r rune) bool { + if unicode.IsSpace(r) { + return true + } + for _, r2 := range c.StartToken { + if r == r2 { + return true + } + } + for _, r2 := range c.EndToken { + if r == r2 { + return true + } + } + return false } -func (op commentUAST) Check(st *State, n nodes.Node) (bool, error) { - s, ok := n.(nodes.String) - if !ok { - return false, nil +func (c *commentElems) Split(text string) bool { + if c.DoTrim { + text = strings.TrimLeftFunc(text, unicode.IsSpace) } - text := string(s) - if !strings.HasPrefix(text, op.tokens[0]) || !strings.HasSuffix(text, op.tokens[1]) { - return false, nil + + if !strings.HasPrefix(text, c.StartToken) || !strings.HasSuffix(text, c.EndToken) { + return false } - text = strings.TrimPrefix(text, op.tokens[0]) - text = strings.TrimSuffix(text, op.tokens[1]) - var ( - pref, suff, tab string - ) + text = strings.TrimPrefix(text, c.StartToken) + text = strings.TrimSuffix(text, c.EndToken) + + // find prefix (if not already trimmed) + i := strings.IndexFunc(text, func(r rune) bool { + return !c.isTab(r) + }) + + c.Prefix = "" + + if i >= 0 { + c.Prefix = text[:i] + text = text[i:] + } else { + c.Prefix = text + text = "" + } + + // find suffix + i = strings.LastIndexFunc(text, func(r rune) bool { + return !c.isTab(r) + }) + + c.Suffix = "" + if i >= 0 { + c.Suffix = text[i+1:] + text = text[:i+1] + } + c.Text = text + + sub := strings.Split(text, "\n") + if len(sub) == 1 { + // fast path, no tabs + return true + } + + // find minimal common prefix for other lines + // first line is special, it won't contain tab + for i, line := range sub[1:] { + if i == 0 { + j := strings.IndexFunc(line, func(r rune) bool { + return !c.isTab(r) + }) + + c.Indent = "" + if j >= 0 { + c.Indent = line[:j] + } else { + return true // no tabs + } + continue + } + if strings.HasPrefix(line, c.Indent) { + continue + } + j := strings.IndexFunc(line, func(r rune) bool { + return !c.isTab(r) + }) - // find prefix - i := 0 - for ; i < len(text); i++ { - if r := rune(text[i]); unicode.IsLetter(r) || unicode.IsNumber(r) { + tab := "" + if j >= 0 { + tab = line[:j] + } else { + return true // no tabs + } + + for j := 0; j < len(c.Indent) && j < len(tab); j++ { + if c.Indent[j] == tab[j] { + continue + } + if j == 0 { + return true // inconsistent, no tabs + } + tab = tab[:j] break } + c.Indent = tab } - pref = text[:i] - text = text[i:] + for i, line := range sub { + if i == 0 { + continue + } + sub[i] = strings.TrimPrefix(line, c.Indent) + } + c.Text = strings.Join(sub, "\n") + return true +} - // find suffix - i = len(text) - 1 - for ; i >= 0 && unicode.IsSpace(rune(text[i])); i-- { +func (c commentElems) Join() string { + if c.Indent != "" { + sub := strings.Split(c.Text, "\n") + for i, line := range sub { + if i == 0 { + continue + } + sub[i] = c.Indent + line + } + c.Text = strings.Join(sub, "\n") + } + return strings.Join([]string{ + c.StartToken, c.Prefix, + c.Text, + c.Suffix, c.EndToken, + }, "") +} + +type commentUAST struct { + startToken string + endToken string + textVar string + prefVar string + suffVar string + indentVar string + doTrim bool +} + +func (*commentUAST) Kinds() nodes.Kind { + return nodes.KindString +} + +func (op *commentUAST) Check(st *State, n nodes.Node) (bool, error) { + s, ok := n.(nodes.String) + if !ok { + return false, nil } - suff = text[i+1:] - text = text[:i+1] - // TODO: set tab + c := commentElems{StartToken: op.startToken, EndToken: op.endToken, DoTrim: op.doTrim} + if !c.Split(string(s)) { + return false, nil + } err := st.SetVars(Vars{ - op.text: nodes.String(text), - op.pref: nodes.String(pref), - op.suff: nodes.String(suff), - op.tab: nodes.String(tab), + op.textVar: nodes.String(c.Text), + op.prefVar: nodes.String(c.Prefix), + op.suffVar: nodes.String(c.Suffix), + op.indentVar: nodes.String(c.Indent), }) return err == nil, err } -func (op commentUAST) Construct(st *State, n nodes.Node) (nodes.Node, error) { +func (op *commentUAST) Construct(st *State, n nodes.Node) (nodes.Node, error) { var ( text, pref, suff, tab nodes.String ) err := st.MustGetVars(VarsPtrs{ - op.text: &text, - op.pref: &pref, op.suff: &suff, op.tab: &tab, + op.textVar: &text, + op.prefVar: &pref, op.suffVar: &suff, op.indentVar: &tab, }) if err != nil { return nil, err } - // FIXME: handle tab - text = pref + text + suff - return nodes.String(op.tokens[0] + string(text) + op.tokens[1]), nil + + c := commentElems{ + StartToken: op.startToken, + EndToken: op.endToken, + Text: string(text), + Prefix: string(pref), + Suffix: string(suff), + Indent: string(tab), + } + + return nodes.String(c.Join()), nil } diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/transformer.go b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/transformer.go index ee60574..c4b078d 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/transformer.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/uast/transformer/transformer.go @@ -267,7 +267,7 @@ func (m *mappings) index() { case ObjectOp: specific := false fields, _ := op.Fields() - if f, ok := fields[uast.KeyType]; ok && !f.Optional { + if f, ok := fields.Get(uast.KeyType); ok && !f.Optional { if f.Fixed != nil { typ := *f.Fixed if typ, ok := typ.(nodes.String); ok { @@ -345,6 +345,9 @@ func (m mappings) Do(root nodes.Node) (nodes.Node, error) { return n, true }) err := NewMultiError(errs...) + if err == nil { + err = st.Validate() + } if ok { return nn, err } @@ -364,25 +367,44 @@ type Vars map[string]nodes.Node // State stores all variables (placeholder values, flags and wny other state) between Check and Construct steps. type State struct { vars Vars + unused map[string]struct{} states map[string][]*State } // Reset clears the state and allows to reuse an object. func (st *State) Reset() { st.vars = nil + st.unused = nil st.states = nil } +// Validate should be called after a successful transformation to check if there are any errors related to unused state. +func (st *State) Validate() error { + if len(st.unused) == 0 { + return nil + } + names := make([]string, 0, len(st.unused)) + for name := range st.unused { + names = append(names, name) + } + sort.Strings(names) + return ErrVariableUnused.New(names) +} + // Clone will return a copy of the State. This can be used to apply Check and throw away any variables produced by it. // To merge a cloned state back use ApplyFrom on a parent state. func (st *State) Clone() *State { st2 := NewState() if len(st.vars) != 0 { st2.vars = make(Vars) + st2.unused = make(map[string]struct{}) } for k, v := range st.vars { st2.vars[k] = v } + for k := range st.unused { + st2.unused[k] = struct{}{} + } if len(st.states) != 0 { st2.states = make(map[string][]*State) } @@ -396,12 +418,16 @@ func (st *State) Clone() *State { func (st *State) ApplyFrom(st2 *State) { if len(st2.vars) != 0 && st.vars == nil { st.vars = make(Vars) + st.unused = make(map[string]struct{}) } for k, v := range st2.vars { if _, ok := st.vars[k]; !ok { st.vars[k] = v } } + for k := range st2.unused { + st.unused[k] = struct{}{} + } if len(st2.states) != 0 && st.states == nil { st.states = make(map[string][]*State) } @@ -415,6 +441,9 @@ func (st *State) ApplyFrom(st2 *State) { // GetVar looks up a named variable. func (st *State) GetVar(name string) (nodes.Node, bool) { n, ok := st.vars[name] + if ok { + delete(st.unused, name) + } return n, ok } @@ -452,8 +481,10 @@ func (st *State) SetVar(name string, val nodes.Node) error { // not declared if st.vars == nil { st.vars = make(Vars) + st.unused = make(map[string]struct{}) } st.vars[name] = val + st.unused[name] = struct{}{} return nil } if nodes.Equal(cur, val) { diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/uast/types.go b/vendor/gopkg.in/bblfsh/sdk.v2/uast/types.go index ef9e4ea..a594abe 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/uast/types.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/uast/types.go @@ -10,15 +10,20 @@ import ( ) var ( - ErrIncorrectType = errors.NewKind("incorrect object type: %q, expected: %q") + // ErrIncorrectType is returned when trying to load a generic UAST node into a Go value + // of an incorrect type. + ErrIncorrectType = errors.NewKind("incorrect object type: %q, expected: %q") + // ErrTypeNotRegistered is returned when trying to create a UAST type that was not associated + // with any Go type. See RegisterPackage. ErrTypeNotRegistered = errors.NewKind("type is not registered: %q") ) var ( - namespaces = make(map[string]string) - package2ns = make(map[string]string) - type2name = make(map[reflect.Type]nodeID) - name2type = make(map[nodeID]reflect.Type) + namespaces = make(map[string]string) // namespace to package + package2ns = make(map[string]string) // package to namespace + type2name = make(map[reflect.Type]nodeID) + name2type = make(map[nodeID]reflect.Type) + typeContentKey = make(map[string]string) // ns:type to "content" field name ) func parseNodeID(s string) nodeID { @@ -46,6 +51,18 @@ func (n nodeID) String() string { return n.NS + ":" + n.Name } +// RegisterPackage registers a new UAST namespace and associates the concrete types +// of the specified values with it. All types should be in the same Go package. +// The name of each type is derived from its reflect.Type name. +// +// Example: +// type Node struct{} +// +// func init(){ +// // will register a UAST type "my:Node" associated with +// // a Node type from this package +// RegisterPackage("my", Node{}) +// } func RegisterPackage(ns string, types ...interface{}) { if _, ok := namespaces[ns]; ok { panic("namespace already registered") @@ -60,19 +77,42 @@ func RegisterPackage(ns string, types ...interface{}) { package2ns[pkg] = ns for _, o := range types { - rt := reflect.TypeOf(o) - if rt.Kind() == reflect.Ptr { - rt = rt.Elem() + registerType(ns, o) + } +} + +func registerType(ns string, o interface{}) { + rt := reflect.TypeOf(o) + if rt.Kind() == reflect.Ptr { + rt = rt.Elem() + } + if name, ok := type2name[rt]; ok { + panic(fmt.Errorf("type %v already registered under %s name", rt, name)) + } + id := nodeID{NS: ns, Name: rt.Name()} + type2name[rt] = id + name2type[id] = rt + if rt.Kind() != reflect.Struct { + return + } + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + if f.Anonymous { + continue // do not inherit content field } - if name, ok := type2name[rt]; ok { - panic(fmt.Errorf("type %v already registered under %s name", rt, name)) + d, err := getFieldDesc(f) + if err != nil { + panic(err) + } + if d.Content { + typeContentKey[id.String()] = d.Name } - name := nodeID{NS: ns, Name: rt.Name()} - type2name[rt] = name - name2type[name] = rt } } +// LookupType finds a Go type corresponding to a specified UAST type. +// +// It only returns types registered via RegisterPackage. func LookupType(typ string) (reflect.Type, bool) { name := parseNodeID(typ) rt, ok := name2type[name] @@ -88,7 +128,7 @@ func zeroFieldsTo(obj, opt nodes.Object, rt reflect.Type) error { } continue } - name, omit, err := fieldName(f) + d, err := getFieldDesc(f) if err != nil { return err } @@ -105,10 +145,10 @@ func zeroFieldsTo(obj, opt nodes.Object, rt reflect.Type) error { case reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8: v = nodes.Uint(0) } - if omit { - opt[name] = v + if d.OmitEmpty { + opt[d.Name] = v } else { - obj[name] = v + obj[d.Name] = v } } return nil @@ -139,6 +179,9 @@ func NewObjectByTypeOpt(typ string) (obj, opt nodes.Object) { return obj, opt } +// NewValue creates a new Go value corresponding to a specified UAST type. +// +// It only creates types registered via RegisterPackage. func NewValue(typ string) (reflect.Value, error) { rt, ok := LookupType(typ) if !ok { @@ -152,6 +195,11 @@ func NewValue(typ string) (reflect.Value, error) { } } +// TypeOf returns the UAST type of a value. +// +// If the value is a generic UAST node, the function returns the value of its KeyType. +// +// If an object is registered as a UAST schema type, the function returns the associated type. func TypeOf(o interface{}) string { switch obj := o.(type) { case nil: @@ -190,34 +238,49 @@ func typeOf(tp reflect.Type) nodeID { return nodeID{NS: ns, Name: name} } -func fieldName(f reflect.StructField) (string, bool, error) { - name := strings.SplitN(f.Tag.Get("uast"), ",", 2)[0] - omitempty := false - if name == "" { +type fieldDesc struct { + Name string + OmitEmpty bool + Content bool +} + +func getFieldDesc(f reflect.StructField) (fieldDesc, error) { + uastTag := strings.Split(f.Tag.Get("uast"), ",") + desc := fieldDesc{ + Name: uastTag[0], + } + for _, s := range uastTag[1:] { + if s == "content" { + desc.Content = true + break + } + } + if desc.Name == "" { tags := strings.Split(f.Tag.Get("json"), ",") for _, s := range tags[1:] { if s == "omitempty" { - omitempty = true + desc.OmitEmpty = true break } } - name = tags[0] + desc.Name = tags[0] } - if name == "" { - return "", false, fmt.Errorf("field %s should have uast or json name", f.Name) + if desc.Name == "" { + return desc, fmt.Errorf("field %s should have uast or json name", f.Name) } - return name, omitempty, nil + return desc, nil } var ( - reflString = reflect.TypeOf("") - reflAny = reflect.TypeOf((*Any)(nil)).Elem() - reflNode = reflect.TypeOf((*nodes.Node)(nil)).Elem() - reflNodeExt = reflect.TypeOf((*nodes.External)(nil)).Elem() + reflString = reflect.TypeOf("") + reflAny = reflect.TypeOf((*Any)(nil)).Elem() + reflAnySlice = reflect.TypeOf([]Any{}) + reflNode = reflect.TypeOf((*nodes.Node)(nil)).Elem() + reflNodeExt = reflect.TypeOf((*nodes.External)(nil)).Elem() ) -// ToNode converts objects returned by schema-less encodings such as JSON to Node objects. -// It also supports types from packages registered via RegisterPackage. +// ToNode converts generic values returned by schema-less encodings such as JSON to Node objects. +// It also supports values registered via RegisterPackage. func ToNode(o interface{}) (nodes.Node, error) { return nodes.ToNode(o, toNodeFallback) } @@ -316,7 +379,7 @@ func structToNode(obj nodes.Object, rv reflect.Value, rt reflect.Type) error { } continue } - name, omit, err := fieldName(ft) + d, err := getFieldDesc(ft) if err != nil { return fmt.Errorf("type %s: %v", rt.Name(), err) } @@ -324,14 +387,17 @@ func structToNode(obj nodes.Object, rv reflect.Value, rt reflect.Type) error { if err != nil { return err } - if v == nil && omit { + if v == nil && d.OmitEmpty { continue } - obj[name] = v + obj[d.Name] = v } return nil } +// NodeAs loads a generic UAST node into provided Go value. +// +// It returns ErrIncorrectType in case of type mismatch. func NodeAs(n nodes.External, dst interface{}) error { var rv reflect.Value if v, ok := dst.(reflect.Value); ok { @@ -354,6 +420,26 @@ func setAnyOrNode(dst reflect.Value, n nodes.External) (bool, error) { } dst.Set(reflect.ValueOf(nd)) return true, nil + } else if rt == reflAnySlice { + narr, ok := n.(nodes.ExternalArray) + if !ok { + return false, nil + } + sz := narr.Size() + arr := make([]Any, 0, sz) + for i := 0; i < sz; i++ { + e := narr.ValueAt(i) + var v Any = e + if nv, err := NewValue(TypeOf(e)); err == nil { + if err = nodeAs(e, nv); err != nil { + return false, err + } + v = nv.Interface() + } + arr = append(arr, v) + } + dst.Set(reflect.ValueOf(arr)) + return true, nil } return false, nil } @@ -482,11 +568,11 @@ func nodeToStruct(rv reflect.Value, rt reflect.Type, obj nodes.ExternalObject) e } continue } - name, _, err := fieldName(ft) + d, err := getFieldDesc(ft) if err != nil { return fmt.Errorf("type %s: %v", rt.Name(), err) } - v, ok := obj.ValueAt(name) + v, ok := obj.ValueAt(d.Name) if !ok { continue } diff --git a/vendor/gopkg.in/bblfsh/sdk.v2/uast/uast.go b/vendor/gopkg.in/bblfsh/sdk.v2/uast/uast.go index e51d973..12e8a5c 100644 --- a/vendor/gopkg.in/bblfsh/sdk.v2/uast/uast.go +++ b/vendor/gopkg.in/bblfsh/sdk.v2/uast/uast.go @@ -31,6 +31,7 @@ func init() { GenNode{}, Identifier{}, String{}, + Bool{}, QualifiedIdentifier{}, Comment{}, Group{}, @@ -40,6 +41,7 @@ func init() { Import{}, RuntimeImport{}, RuntimeReImport{}, + InlineImport{}, Argument{}, FunctionType{}, Function{}, @@ -48,10 +50,10 @@ func init() { // Special field keys for nodes.Object const ( - KeyType = "@type" // InternalType - KeyToken = "@token" // Token - KeyRoles = "@role" // Roles, for representations see RoleList - KeyPos = "@pos" // All positional information is stored in this field + KeyType = "@type" // the type of UAST node (InternalType in v1) + KeyToken = "@token" // token of the UAST node (Native and Annotated nodes only) + KeyRoles = "@role" // roles of UAST node (Annotated nodes only); for representations see RoleList + KeyPos = "@pos" // positional information is stored in this field, see Positions ) const ( @@ -81,23 +83,32 @@ type Position struct { Offset uint32 `json:"offset"` // Line is the line number. It is a 1-based index. Line uint32 `json:"line"` - // Col is the column number (the byte offset of the position relative to + // Col is the column number — the byte offset of the position relative to // a line. It is a 1-based index. Col uint32 `json:"col"` } +// HasOffset checks if a position has a valid offset value. func (p Position) HasOffset() bool { return p.Offset != 0 || (p.Line == 1 && p.Col == 1) } +// HasLineCol checks if a position has a valid line-column pair. func (p Position) HasLineCol() bool { return p.Line != 0 && p.Col != 0 } +// Valid checks if position value is valid. func (p Position) Valid() bool { return p != (Position{}) } +// Less reports whether position p is strictly less than p2. +// +// If both positions have offsets, they will be used for comparison. +// Otherwise, line-column pair will be used. +// +// Invalid positions are sorted last. func (p Position) Less(p2 Position) bool { if !p.Valid() { return false @@ -116,7 +127,11 @@ func (p Position) Less(p2 Position) bool { return p.Col != 0 && p.Col < p2.Col } -// Positions is a container object that stores all positional information for a node. +// Positions is a container that stores all positional information for a UAST node. +// +// The string key is a name of a position, for example KeyStart is a start position +// of a node and KeyEnd is an end position of a node. Driver may provide additional +// positional information for other tokens that the node consists of. type Positions map[string]Position // Keys returns a sorted slice of position names. @@ -145,7 +160,7 @@ func (p Positions) End() *Position { return nil } -// ToObject converts positions to a generic object. +// ToObject converts a positions map to a generic UAST node. func (p Positions) ToObject() nodes.Object { n, err := toNodeReflect(reflect.ValueOf(p)) if err != nil { @@ -166,8 +181,14 @@ func AsPosition(m nodes.Object) *Position { return &p } -// PositionsOf returns an object with all positional information for a node. -func PositionsOf(m nodes.Object) Positions { +// PositionsOf returns a complete positions map for the given UAST node. +// The function will return nil for non-object nodes like arrays and values. To get +// positions for these nodes, PositionsOf should be called on their parent node. +func PositionsOf(n nodes.Node) Positions { + m, ok := n.(nodes.Object) + if !ok { + return nil + } o, _ := m[KeyPos].(nodes.Object) if len(o) == 0 { return nil @@ -201,7 +222,12 @@ func RoleList(roles ...role.Role) nodes.Array { } // RolesOf is a helper for getting node UAST roles (see KeyRoles). -func RolesOf(m nodes.Object) role.Roles { +// The function will returns nil roles array for non-object nodes like arrays and values. +func RolesOf(n nodes.Node) role.Roles { + m, ok := n.(nodes.Object) + if !ok { + return nil + } arr, ok := m[KeyRoles].(nodes.Array) if !ok || len(arr) == 0 { if tp := TypeOf(m); tp == "" || strings.HasPrefix(tp, NS+":") { @@ -219,20 +245,29 @@ func RolesOf(m nodes.Object) role.Roles { } // TokenOf is a helper for getting node token (see KeyToken). -func TokenOf(m nodes.Object) string { - t := m[KeyToken] - s, ok := t.(nodes.String) - if ok { - return string(s) - } - v, _ := t.(nodes.Value) - if v != nil { - return fmt.Sprint(v) +// +// The token is an exact code snippet that represents a given AST node. It only works for +// primitive nodes like identifiers and string literals, and is only available in Native +// and Annotated parsing modes. For Semantic mode, see ContentOf. +// +// It returns an empty string if the node is not an object, or there is no token. +func TokenOf(n nodes.Node) string { + switch n := n.(type) { + case nodes.String: + return string(n) + case nodes.Value: + return fmt.Sprint(n) + case nodes.Object: + t := n[KeyToken] + if t == nil { + return "" + } + return TokenOf(t) } return "" } -// Tokens collects all tokens of the tree recursively (pre-order). +// Tokens collects all tokens of the tree recursively (pre-order). See TokenOf. func Tokens(n nodes.Node) []string { var tokens []string nodes.WalkPreOrder(n, func(n nodes.Node) bool { @@ -246,6 +281,26 @@ func Tokens(n nodes.Node) []string { return tokens } +// ContentOf returns any relevant string content of a node. It returns a Name for +// Identifiers, Value for Strings, etc and uses TokenOf for non-Semantic nodes. +// +// The result may not exactly match the source file since values in Semantic nodes +// are normalized. +// +// It returns an empty string if the node has no string content. +func ContentOf(n nodes.Node) string { + if obj, ok := n.(nodes.Object); ok { + typ, _ := obj[KeyType].(nodes.String) + + if field, ok := typeContentKey[string(typ)]; ok { + // allow nested objects + return ContentOf(obj[field]) + } + } + // fallback to token + return TokenOf(n) +} + // HashNoPos hashes the node, but skips positional information. func HashNoPos(n nodes.External) nodes.Hash { h := nodes.NewHasher() @@ -261,87 +316,320 @@ type Any interface{} // Scope is a temporary definition of a scope semantic type. type Scope = Any +// GenNode is embedded into every UAST node to store positional information. type GenNode struct { Positions Positions `json:"@pos,omitempty"` } +// Identifier is a name of an entity. +// +// What is considered an Identifier: +// - variable, type, function names; +// - builtin type names; +// - package name consisting of a single name element; +// - goto labels; +// +// Not considered an Identifier: +// - qualified names (see QualifiedIdentifier); +// - path-like or url-like package names (see String); type Identifier struct { GenNode - Name string `json:"Name"` + // Name of an entity. Can be any valid UTF8 string. + Name string `json:"Name" uast:",content"` } +// Roles returns a list of UAST node roles that apply to this node. +func (Identifier) Roles() []role.Role { + return []role.Role{ + role.Identifier, + } +} + +// String is an unescaped UTF8 string literal. +// +// What is considered a String literal: +// - escaped string literals; +// - raw string literals; +// - path-like or url-like package names; +// +// Not considered a String literal: +// - identifiers (see Identifier); +// - qualified names (see QualifiedIdentifier); +// - numeric and boolean literals; +// - special regexp literals; type String struct { GenNode - Value string `json:"Value"` - Format string `json:"Format"` // TODO: make an enum later + // Value is a UTF8 string literal value. + // + // Drivers should remove any quotes and unescape the value according to the language rules. + Value string `json:"Value" uast:",content"` + + // Format is an optional language-specific string that describes the format of the literal. + // + // This field can be empty for the most common string literal type of a specific language. + // The priority is given to a one-line literal that escapes newline characters. + // + // TODO: define some well-known formats and maybe make it an enum + Format string `json:"Format"` } +// QualifiedIdentifier is a name of an entity that consists of multiple simple identifiers, +// organized in a hierarchy, similar to filesystem paths. +// +// What is considered a QualifiedIdentifier: +// - qualified names that consist of Identifier-like elements; +// +// Not considered a QualifiedIdentifier: +// - path-like or url-like package names (see String); +// - selector expressions (a->b and a.b in C++); type QualifiedIdentifier struct { GenNode + // Names is a list of simple identifiers starting from a root level of hierarchy + // and ending with leaf identifier. Names should not be empty. Names []Identifier `json:"Names"` } +// Comment is a no-op node that can span multiple lines and provides a human-readable +// description for code around it. +// +// TODO: currently some annotations are also considered a Comment; need to clarify this type Comment struct { GenNode - Text string `json:"Text"` + + // Block is set to true for block-style comments. + // + // TODO: should be a string similar to Format field in String literal; + // may have more than 2 possible values (line, block, doc?) + Block bool `json:"Block"` + + // Text is an unescaped UTF8 string with the comment text. + // + // Drivers must trim any comment-related tokens as well as whitespaces and + // stylistic characters at the beginning of ToObjecteach line. See Prefix, Suffix, Tab. + // + // Example: + // /* + // * some comment + // */ + // + // only "some comment" is considered a text + Text string `json:"Text" uast:",content"` + + // Prefix is a set of whitespaces and stylistic characters that appear before + // the first line of an actual comment text. + // + // Example: + // /* + // * some comment + // */ + // + // the "\n" after the "/*" token is considered a prefix Prefix string `json:"Prefix"` + + // Suffix is a set of whitespaces and stylistic characters that appear after + // the last line of an actual comment text. + // + // Example: + // /* + // * some comment + // */ + // + // the "\n " before the "*/" token is considered a suffix Suffix string `json:"Suffix"` - Tab string `json:"Tab"` - Block bool `json:"Block"` + + // Tab is a set of whitespace and stylistic characters that appears at the beginning + // of each comment line, except the first one, which uses Prefix. + // + // Example: + // /* + // * some comment + // */ + // + // the " *" before the comment text is considered a tab + // + // TODO(dennwc): rename to Indent? + Tab string `json:"Tab"` } +// Group is a no-op UAST node that groups multiple nodes together. +// +// Drivers may use it when for grouping statements that are represented by a single statement +// in the native AST. +// +// For example, a language may describe a way to define multiple variables in one statement. +// This statement should be split into separate UAST nodes that become a children of a single Group. +// +// Groups should never convey any semantic meaning. type Group struct { GenNode + // Nodes is a list of UAST nodes in a group. Nodes []Any `json:"Nodes"` } +// FunctionGroup is a special group node that joins multiple UAST nodes related to a function +// declaration. +// +// FunctionGroup usually contains at least an Alias node that specifies the function name and +// may contain additional nodes such as annotations and comments and docs related to it. +// +// See Function for more details about function declarations. type FunctionGroup Group +// Block is a logical code block. It groups multiple statements and enforces a sequential execution +// of these statements. +// +// When the Block should be used: +// - for function bodies; +// - when the statement defines a new scope; type Block struct { GenNode Statements []Any `json:"Statements"` + + // TODO: block is logical and should have a reference to a corresponding scope; should be used in "if"s, etc // Scope *Scope } +// Alias provides a way to assign a permanent name to an entity, or give an alternative name. +// +// Aliases are immutable and the only way to redefine it is to shadow it in the child scope. +// +// What is considered an Alias: +// - a name of a function in a function declaration; +// - a name of a constant and its value; +// - a name of a preprocessor macros and its substitution; +// - variable declaration; // TODO: should point to some Variable node +// +// Not considered an Alias: +// - value assignments to a variable, even if it defines a variable; type Alias struct { GenNode + // Name assigned to an entity. + // + // TODO: define a different node to handle QualifiedIdentifier as a name Name Identifier `json:"Name"` - Node Any `json:"Node"` + + // A UAST node to assign a name to. + Node Any `json:"Node"` + + // TODO: should include a pointer to a scope where an alias is defined // Target *Scope } +// Import is a statement that can load other modules into the program or library. +// +// This is a declarative import statement. Its position in the UAST does not affect +// the way and the time when the module is imported and the side-effects are executed +// only once a package is initialized. +// +// This describes imports in Go, Java, and C#, for example. +// +// For more specific types see RuntimeImport, RuntimeReImport, InlineImport. type Import struct { GenNode - Path Any `json:"Path"` + // Path is a path of a modules or package to load. + // + // May have a value of: + // - String (specifies relative or absolute module path); + // - QualifiedIdentifier (specifies a canonical module name); + // - Alias (contains any of the above and defines a local package name within a file/scope); + Path Any `json:"Path"` + + // All is set to true when the statement defines all exported symbols from + // a module in the local scope (usually file). All bool `json:"All"` Names []Any `json:"Names"` Target Scope `json:"Target"` } +// RuntimeImport is a type of an import statement that imports a module only when an execution +// reaches this UAST node. The import side effects are executed only once, regardless of how many +// times a statement is reached. +// +// This describes imports in PHP, Python and JS for example. +// +// For other import types, see Import. type RuntimeImport Import +// RuntimeReImport is a subset of RuntimeImport statement that will re-execute +// an import and its side-effects statement each time an execution reaches the +// statement. +// +// This describes imports in PHP and Python for example. +// +// For other import types, see Import. type RuntimeReImport RuntimeImport -//type InlineImport Import +// InlineImport is a subset of import statement that acts like a preprocessor - all statements in +// the imported module are copied into a position of the UAST node. +// +// This describes #include in C and C++. +// +// For other import types, see Import. +type InlineImport Import +// Argument is a named argument or return of a function. type Argument struct { GenNode - Name *Identifier `json:"Name"` - Type Any `json:"Type"` - Init Any `json:"Init"` - Variadic bool `json:"Variadic"` - MapVariadic bool `json:"MapVariadic"` - Receiver bool `json:"Receiver"` + // Name is an optional name of an argument. + Name *Identifier `json:"Name"` + + // Type is an optional type of an argument. + Type Any `json:"Type"` + + // Init is an optional expression used to initialize the argument + // in case no value is provided. + Init Any `json:"Init"` + + // Variadic is set for the last argument of a function with a + // variadic number of arguments. + Variadic bool `json:"Variadic"` + + // MapVariadic is set for the last argument of a function that accepts a + // map/dictionary value that is mapped to function arguments. + MapVariadic bool `json:"MapVariadic"` + + // Receiver is set to true if an argument is a receiver of a method call. + Receiver bool `json:"Receiver"` } +// FunctionType is a signature of a function. type FunctionType struct { GenNode + // Arguments is a set of arguments the function accepts. + // + // Methods defined on structures and classes must have the first argument + // that corresponds to a method's receiver ("this" in most languages). Arguments []Argument `json:"Arguments"` - Returns []Argument `json:"Returns"` + + // Returns is a set of values returned by a function. + // + // Languages with an implicit return should specify a single return with an + // unspecified type. + Returns []Argument `json:"Returns"` } +// Function is a declaration of a function with a specific signature and implementation. +// +// Name is not a part of function declaration. Use Alias as a parent node to specify +// the name of a function. +// +// What is considered a Function: +// - function declaration; +// - anonymous functions; type Function struct { GenNode + // Type is a signature of a function. Should always be set. Type FunctionType `json:"Type"` - Body *Block `json:"Body"` + + // Body is an optional implementation of a function. should point to a Block with + // a set of statements. Each code path in those statements should end with return. + // + // TODO: we don't have return statements yet + Body *Block `json:"Body"` +} + +// Bool is a boolean literal. +type Bool struct { + GenNode + Value bool `json:"Value" uast:",content"` }