Skip to content

Commit

Permalink
feat: bump tree-sitter-javascript to 0.23
Browse files Browse the repository at this point in the history
  • Loading branch information
@amaanq
amaanq committed Sep 2, 2024
1 parent 198d035 commit 71fe641
Show file tree
Hide file tree
Showing 11 changed files with 352,825 additions and 343,374 deletions.
6 changes: 5 additions & 1 deletion common/define-grammar.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,11 @@ module.exports = function defineGrammar(dialect) {
prec('call', seq(
field('function', choice($.expression, $.import)),
field('type_arguments', optional($.type_arguments)),
field('arguments', choice($.arguments, $.template_string)),
field('arguments', $.arguments),
)),
prec('template_call', seq(
field('function', choice($.primary_expression, $.new_expression)),
field('arguments', $.template_string),
)),
prec('member', seq(
field('function', $.primary_expression),
Expand Down
48 changes: 48 additions & 0 deletions common/scanner.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ enum TokenType {
LOGICAL_OR,
ESCAPE_SEQUENCE,
REGEX_PATTERN,
JSX_TEXT,
FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON,
ERROR_RECOVERY,
};
Expand Down Expand Up @@ -115,6 +116,7 @@ static bool scan_automatic_semicolon(TSLexer *lexer, const bool *valid_symbols,
}

switch (lexer->lookahead) {
case '`':
case ',':
case '.':
case ';':
Expand Down Expand Up @@ -271,13 +273,59 @@ static bool scan_closing_comment(TSLexer *lexer) {
return true;
}

static bool scan_jsx_text(TSLexer *lexer) {
// saw_text will be true if we see any non-whitespace content, or any whitespace content that is not a newline and
// does not immediately follow a newline.
bool saw_text = false;
// at_newline will be true if we are currently at a newline, or if we are at whitespace that is not a newline but
// immediately follows a newline.
bool at_newline = false;

while (lexer->lookahead != 0 && lexer->lookahead != '<' && lexer->lookahead != '>' && lexer->lookahead != '{' &&
lexer->lookahead != '}' && lexer->lookahead != '&') {
bool is_wspace = iswspace(lexer->lookahead);
if (lexer->lookahead == '\n') {
at_newline = true;
} else {
// If at_newline is already true, and we see some whitespace, then it must stay true.
// Otherwise, it should be false.
//
// See the table below to determine the logic for computing `saw_text`.
//
// |------------------------------------|
// | at_newline | is_wspace | saw_text |
// |------------|-----------|-----------|
// | false (0) | false (0) | true (1) |
// | false (0) | true (1) | true (1) |
// | true (1) | false (0) | true (1) |
// | true (1) | true (1) | false (0) |
// |------------------------------------|

at_newline &= is_wspace;
if (!at_newline) {
saw_text = true;
}
}

advance(lexer);
}

lexer->result_symbol = JSX_TEXT;
return saw_text;
}

static inline bool external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
if (valid_symbols[TEMPLATE_CHARS]) {
if (valid_symbols[AUTOMATIC_SEMICOLON]) {
return false;
}
return scan_template_chars(lexer);
}

if (valid_symbols[JSX_TEXT] && scan_jsx_text(lexer)) {
return true;
}

if (valid_symbols[AUTOMATIC_SEMICOLON] || valid_symbols[FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON]) {
bool scanned_comment = false;
bool ret = scan_automatic_semicolon(lexer, valid_symbols, &scanned_comment);
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
"eslint": ">=8.57.0",
"eslint-config-google": "^0.14.0",
"tree-sitter-cli": "^0.22.6",
"tree-sitter-javascript": "^0.21.4",
"tree-sitter-javascript": "^0.23.0",
"prebuildify": "^6.0.1"
},
"scripts": {
Expand Down
201 changes: 149 additions & 52 deletions tsx/src/grammar.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1204,29 +1204,51 @@
"value": "("
},
{
"type": "FIELD",
"name": "initializer",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "lexical_declaration"
},
{
"type": "SYMBOL",
"name": "variable_declaration"
},
{
"type": "SYMBOL",
"name": "expression_statement"
},
{
"type": "CHOICE",
"members": [
{
"type": "FIELD",
"name": "initializer",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "lexical_declaration"
},
{
"type": "SYMBOL",
"name": "variable_declaration"
}
]
}
},
{
"type": "SEQ",
"members": [
{
"type": "FIELD",
"name": "initializer",
"content": {
"type": "SYMBOL",
"name": "_expressions"
}
},
{
"type": "STRING",
"value": ";"
}
]
},
{
"type": "FIELD",
"name": "initializer",
"content": {
"type": "SYMBOL",
"name": "empty_statement"
}
]
}
}
]
},
{
"type": "FIELD",
Expand All @@ -1235,8 +1257,17 @@
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "expression_statement"
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expressions"
},
{
"type": "STRING",
"value": ";"
}
]
},
{
"type": "SYMBOL",
Expand Down Expand Up @@ -1413,6 +1444,18 @@
}
]
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_automatic_semicolon"
},
{
"type": "BLANK"
}
]
}
]
}
Expand Down Expand Up @@ -2815,25 +2858,6 @@
}
]
},
"jsx_text": {
"type": "CHOICE",
"members": [
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "PATTERN",
"value": "[^{}<>&]*[^{}<>\\s\\p{Zs}\\uFEFF\\u2028\\u2029\\u2060\\u200B&][^{}<>&]*"
}
},
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "PATTERN",
"value": "[^{}<>\\n&]+"
}
}
]
},
"html_character_reference": {
"type": "PATTERN",
"value": "&(#([xX][0-9a-fA-F]{1,6}|[0-9]{1,5})|[A-Za-z]{1,30});"
Expand Down Expand Up @@ -3850,19 +3874,44 @@
{
"type": "FIELD",
"name": "arguments",
"content": {
"type": "SYMBOL",
"name": "arguments"
}
}
]
}
},
{
"type": "PREC",
"value": "template_call",
"content": {
"type": "SEQ",
"members": [
{
"type": "FIELD",
"name": "function",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "arguments"
"name": "primary_expression"
},
{
"type": "SYMBOL",
"name": "template_string"
"name": "new_expression"
}
]
}
},
{
"type": "FIELD",
"name": "arguments",
"content": {
"type": "SYMBOL",
"name": "template_string"
}
}
]
}
Expand Down Expand Up @@ -6303,19 +6352,41 @@
}
},
"meta_property": {
"type": "SEQ",
"type": "CHOICE",
"members": [
{
"type": "STRING",
"value": "new"
},
{
"type": "STRING",
"value": "."
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "new"
},
{
"type": "STRING",
"value": "."
},
{
"type": "STRING",
"value": "target"
}
]
},
{
"type": "STRING",
"value": "target"
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "import"
},
{
"type": "STRING",
"value": "."
},
{
"type": "STRING",
"value": "meta"
}
]
}
]
},
Expand Down Expand Up @@ -11124,6 +11195,10 @@
"primary_expression",
"_for_header"
],
[
"variable_declarator",
"_for_header"
],
[
"array",
"array_pattern"
Expand Down Expand Up @@ -11310,6 +11385,10 @@
"type": "STRING",
"value": "member"
},
{
"type": "STRING",
"value": "template_call"
},
{
"type": "STRING",
"value": "call"
Expand Down Expand Up @@ -11398,6 +11477,10 @@
"type": "STRING",
"value": "member"
},
{
"type": "STRING",
"value": "template_call"
},
{
"type": "STRING",
"value": "new"
Expand Down Expand Up @@ -11435,6 +11518,16 @@
"value": "object"
}
],
[
{
"type": "SYMBOL",
"name": "meta_property"
},
{
"type": "SYMBOL",
"name": "import"
}
],
[
{
"type": "SYMBOL",
Expand Down Expand Up @@ -11913,6 +12006,10 @@
"type": "SYMBOL",
"name": "regex_pattern"
},
{
"type": "SYMBOL",
"name": "jsx_text"
},
{
"type": "SYMBOL",
"name": "_function_signature_automatic_semicolon"
Expand Down
Loading

0 comments on commit 71fe641

Please sign in to comment.