From 8a5a26c14120d5330f035ed623095fd3fe0df215 Mon Sep 17 00:00:00 2001
From: ecmel <ecmel.ercan@gmail.com>
Date: Fri, 26 Apr 2024 07:53:40 +0300
Subject: [PATCH] update language

---
 server/src/parser/language.ts | 291 ++++++++++++--------------
 server/src/parser/lexer.ts    |   6 +-
 server/src/parser/parser.ts   |   3 +-
 syntaxes/q.tmLanguage.json    | 378 ++++++++++++++++------------------
 4 files changed, 314 insertions(+), 364 deletions(-)

diff --git a/server/src/parser/language.ts b/server/src/parser/language.ts
index 8c2f83b1..5e3c7447 100644
--- a/server/src/parser/language.ts
+++ b/server/src/parser/language.ts
@@ -28,6 +28,7 @@ import { Identifier, Keyword, Reserved } from "./keywords";
 import {
   Colon,
   Command,
+  Comparator,
   DoubleColon,
   Iterator,
   LineComment,
@@ -45,10 +46,7 @@ function _(token: TokenType | RegExp) {
 
 const includes = [
   {
-    include: "#comments",
-  },
-  {
-    include: "#strings",
+    include: "#multiline",
   },
   {
     include: "#literals",
@@ -57,13 +55,7 @@ const includes = [
     include: "#keywords",
   },
   {
-    include: "#identifiers",
-  },
-  {
-    include: "#commands",
-  },
-  {
-    include: "#operators",
+    include: "#tokens",
   },
 ];
 
@@ -140,7 +132,138 @@ const quke = {
 
 const BlockComment = [/^\/\s*$/, /^\\\s*$/];
 const StringLiteral = [/"/, /\\["\\]/];
-const ControlKeyword = /[$!?#@'^]/;
+
+const repository = {
+  quke,
+  qdoc,
+  multiline: {
+    patterns: [
+      {
+        name: "comment.block.q",
+        begin: _(BlockComment[0]),
+        end: _(BlockComment[1]),
+      },
+      {
+        name: "comment.last.q",
+        begin: _(BlockComment[1]),
+      },
+      {
+        include: "#qdoc",
+      },
+      {
+        name: "comment.line.q",
+        match: _(LineComment),
+      },
+      {
+        name: "string.quoted.q",
+        begin: _(StringLiteral[0]),
+        end: _(StringLiteral[0]),
+        patterns: [
+          {
+            name: "constant.character.escape.q",
+            match: _(StringLiteral[1]),
+          },
+        ],
+      },
+    ],
+  },
+  literals: {
+    patterns: [
+      {
+        name: "support.type.symbol.q",
+        match: _(SymbolLiteral),
+      },
+      {
+        name: "constant.numeric.datetime.q",
+        match: _(DateTimeLiteral),
+      },
+      {
+        name: "constant.numeric.timestamp.q",
+        match: _(TimeStampLiteral),
+      },
+      {
+        name: "constant.numeric.date.q",
+        match: _(DateLiteral),
+      },
+      {
+        name: "constant.numeric.month.q",
+        match: _(MonthLiteral),
+      },
+      {
+        name: "constant.numeric.time.q",
+        match: _(TimeLiteral),
+      },
+      {
+        name: "constant.numeric.file.q",
+        match: _(FileLiteral),
+      },
+      {
+        name: "constant.language.infinity.q",
+        match: _(InfinityLiteral),
+      },
+      {
+        name: "constant.numeric.binary.q",
+        match: _(BinaryLiteral),
+      },
+      {
+        name: "constant.numeric.byte.q",
+        match: _(ByteLiteral),
+      },
+      {
+        name: "constant.numeric.number.q",
+        match: _(NumberLiteral),
+      },
+    ],
+  },
+  keywords: {
+    patterns: [
+      {
+        name: "keyword.other.reserved.q",
+        match: `${_(Reserved)}\\b`,
+      },
+      {
+        name: "keyword.other.q",
+        match: `\\b${_(Keyword)}\\b`,
+      },
+      {
+        name: "variable.other.q",
+        match: `${_(Identifier)}\\b`,
+      },
+    ],
+  },
+  tokens: {
+    patterns: [
+      {
+        name: "constant.character.q",
+        match: _(Command),
+      },
+      {
+        name: "keyword.other.iterator.q",
+        match: _(Iterator),
+      },
+      {
+        name: "punctuation.assignment.q",
+        match: _(DoubleColon),
+      },
+      {
+        name: "keyword.operator.arithmetic.q",
+        match: _(Operator),
+      },
+      {
+        name: "keyword.operator.arithmetic.q",
+        match: _(Comparator),
+      },
+      {
+        name: "punctuation.assignment.q",
+        match: _(Colon),
+      },
+      {
+        name: "punctuation.terminator.statement.q",
+        match: _(SemiColon),
+      },
+    ],
+  },
+};
 
 const language = {
   description: "This file is auto generated DO NOT EDIT",
@@ -152,149 +275,7 @@ const language = {
     },
     ...includes,
   ],
-  repository: {
-    comments: {
-      patterns: [
-        quke,
-        qdoc,
-        {
-          name: "comment.block.q",
-          begin: _(BlockComment[0]),
-          end: _(BlockComment[1]),
-        },
-        {
-          name: "comment.last.q",
-          begin: _(BlockComment[1]),
-        },
-        {
-          include: "#qdoc",
-        },
-        {
-          name: "comment.line.q",
-          match: _(LineComment),
-        },
-      ],
-    },
-    strings: {
-      patterns: [
-        {
-          name: "string.quoted.q",
-          begin: _(StringLiteral[0]),
-          end: _(StringLiteral[0]),
-          patterns: [
-            {
-              name: "constant.character.escape.q",
-              match: _(StringLiteral[1]),
-            },
-          ],
-        },
-      ],
-    },
-    literals: {
-      patterns: [
-        {
-          name: "support.type.symbol.q",
-          match: _(SymbolLiteral),
-        },
-        {
-          name: "constant.numeric.datetime.q",
-          match: _(DateTimeLiteral),
-        },
-        {
-          name: "constant.numeric.timestamp.q",
-          match: _(TimeStampLiteral),
-        },
-        {
-          name: "constant.numeric.date.q",
-          match: _(DateLiteral),
-        },
-        {
-          name: "constant.numeric.month.q",
-          match: _(MonthLiteral),
-        },
-        {
-          name: "constant.numeric.time.q",
-          match: _(TimeLiteral),
-        },
-        {
-          name: "constant.numeric.file.q",
-          match: _(FileLiteral),
-        },
-        {
-          name: "constant.language.infinity.q",
-          match: _(InfinityLiteral),
-        },
-        {
-          name: "constant.numeric.binary.q",
-          match: _(BinaryLiteral),
-        },
-        {
-          name: "constant.numeric.byte.q",
-          match: _(ByteLiteral),
-        },
-        {
-          name: "constant.numeric.number.q",
-          match: _(NumberLiteral),
-        },
-      ],
-    },
-    keywords: {
-      patterns: [
-        {
-          name: "keyword.other.reserved.q",
-          match: `${_(Reserved)}\\b`,
-        },
-        {
-          name: "keyword.other.q",
-          match: `\\b${_(Keyword)}\\b`,
-        },
-      ],
-    },
-    identifiers: {
-      patterns: [
-        {
-          name: "variable.other.q",
-          match: `${_(Identifier)}\\b`,
-        },
-      ],
-    },
-    commands: {
-      patterns: [
-        {
-          name: "constant.character.q",
-          match: _(Command),
-        },
-      ],
-    },
-    operators: {
-      patterns: [
-        {
-          name: "keyword.other.iterator.q",
-          match: _(Iterator),
-        },
-        {
-          name: "keyword.other.control.q",
-          match: _(ControlKeyword),
-        },
-        {
-          name: "keyword.operator.arithmetic.q",
-          match: _(Operator),
-        },
-        {
-          name: "punctuation.assignment.q",
-          match: _(Colon),
-        },
-        {
-          name: "punctuation.assignment.q",
-          match: _(DoubleColon),
-        },
-        {
-          name: "punctuation.terminator.statement.q",
-          match: _(SemiColon),
-        },
-      ],
-    },
-  },
+  repository,
 };
 
 export function generateTextMateGrammar() {
diff --git a/server/src/parser/lexer.ts b/server/src/parser/lexer.ts
index c50955e5..f8c7f823 100644
--- a/server/src/parser/lexer.ts
+++ b/server/src/parser/lexer.ts
@@ -68,7 +68,7 @@ import {
   Tolerance,
 } from "./quke";
 
-const Prelude = [BlockComment, LastComment, LineComment, CharLiteral];
+const MultiLine = [BlockComment, LastComment, LineComment, CharLiteral];
 
 const QTokens = [
   Command,
@@ -128,8 +128,8 @@ export const QLexer = new Lexer(
   {
     defaultMode: "q_mode",
     modes: {
-      q_mode: [...Prelude, Quke, ...QTokens],
-      quke_mode: [...Prelude, ...QukeTokens, ...QTokens],
+      q_mode: [...MultiLine, Quke, ...QTokens],
+      quke_mode: [...MultiLine, ...QukeTokens, ...QTokens],
     },
   },
   { safeMode: true },
diff --git a/server/src/parser/parser.ts b/server/src/parser/parser.ts
index afcf8c34..5f0b08be 100644
--- a/server/src/parser/parser.ts
+++ b/server/src/parser/parser.ts
@@ -83,7 +83,6 @@ export interface Token extends IToken {
   identifierKind?: IdentifierKind;
   scope?: Token;
   lambda?: Token;
-  description?: string;
 }
 
 export function parse(text: string): Token[] {
@@ -171,7 +170,7 @@ export function parse(text: string): Token[] {
         const [cmd, arg] = args(token.image, 2);
         switch (cmd) {
           case "\\d":
-            if (arg && arg.startsWith(".") && isIdentifier(arg)) {
+            if (arg?.startsWith(".") && isIdentifier(arg)) {
               namespace = arg === "." ? "" : arg;
             }
             break;
diff --git a/syntaxes/q.tmLanguage.json b/syntaxes/q.tmLanguage.json
index ca0fb972..7daf63fa 100644
--- a/syntaxes/q.tmLanguage.json
+++ b/syntaxes/q.tmLanguage.json
@@ -7,10 +7,7 @@
       "include": "#quke"
     },
     {
-      "include": "#comments"
-    },
-    {
-      "include": "#strings"
+      "include": "#multiline"
     },
     {
       "include": "#literals"
@@ -19,22 +16,25 @@
       "include": "#keywords"
     },
     {
-      "include": "#identifiers"
-    },
-    {
-      "include": "#commands"
-    },
-    {
-      "include": "#operators"
+      "include": "#tokens"
     }
   ],
   "repository": {
-    "comments": {
+    "quke": {
       "patterns": [
         {
+          "begin": "(?<!.)\\s*(x?feature)\\b(.*)",
+          "captures": {
+            "1": {
+              "name": "support.function.q"
+            },
+            "2": {
+              "name": "string.quoted.q"
+            }
+          },
           "patterns": [
             {
-              "begin": "(?<!.)\\s*(x?feature)\\b(.*)",
+              "match": "\\b(x?feature)\\b(.*)",
               "captures": {
                 "1": {
                   "name": "support.function.q"
@@ -42,195 +42,177 @@
                 "2": {
                   "name": "string.quoted.q"
                 }
-              },
-              "patterns": [
-                {
-                  "match": "\\b(x?feature)\\b(.*)",
-                  "captures": {
-                    "1": {
-                      "name": "support.function.q"
-                    },
-                    "2": {
-                      "name": "string.quoted.q"
-                    }
-                  }
-                },
-                {
-                  "match": "\\b(x?should)\\b(.*)",
-                  "captures": {
-                    "1": {
-                      "name": "support.function.q"
-                    },
-                    "2": {
-                      "name": "string.quoted.q"
-                    }
-                  }
-                },
-                {
-                  "match": "\\b(x?expect)\\b(.*)",
-                  "captures": {
-                    "1": {
-                      "name": "support.function.q"
-                    },
-                    "2": {
-                      "name": "string.quoted.q"
-                    }
-                  }
-                },
-                {
-                  "match": "\\b(x?to match)\\b(.*)",
-                  "captures": {
-                    "1": {
-                      "name": "support.function.q"
-                    },
-                    "2": {
-                      "name": "string.quoted.q"
-                    }
-                  }
-                },
-                {
-                  "match": "\\b(x?behaviour)\\b(.*)",
-                  "captures": {
-                    "1": {
-                      "name": "support.function.q"
-                    },
-                    "2": {
-                      "name": "string.quoted.q"
-                    }
-                  }
-                },
-                {
-                  "match": "\\b(x?baseline)\\b(.*)",
-                  "captures": {
-                    "1": {
-                      "name": "support.function.q"
-                    },
-                    "2": {
-                      "name": "string.quoted.q"
-                    }
-                  }
-                },
-                {
-                  "match": "\\b(x?bench)\\b(.*)",
-                  "captures": {
-                    "1": {
-                      "name": "support.function.q"
-                    },
-                    "2": {
-                      "name": "string.quoted.q"
-                    }
-                  }
-                },
-                {
-                  "match": "\\b(x?property)\\b(.*)",
-                  "captures": {
-                    "1": {
-                      "name": "support.function.q"
-                    },
-                    "2": {
-                      "name": "string.quoted.q"
-                    }
-                  }
-                },
-                {
-                  "name": "support.function.q",
-                  "match": "\\bx?after each\\b"
-                },
-                {
-                  "name": "support.function.q",
-                  "match": "\\bx?after\\b"
-                },
-                {
-                  "name": "support.function.q",
-                  "match": "\\bx?before each\\b"
-                },
-                {
-                  "name": "support.function.q",
-                  "match": "\\bx?before\\b"
-                },
-                {
-                  "name": "support.function.q",
-                  "match": "\\bx?setup\\b"
-                },
-                {
-                  "name": "support.function.q",
-                  "match": "\\bx?teardown\\b"
-                },
-                {
-                  "name": "support.function.q",
-                  "match": "\\bx?skip if\\b"
-                },
-                {
-                  "name": "support.function.q",
-                  "match": "\\bx?timelimit\\b"
-                },
-                {
-                  "name": "support.function.q",
-                  "match": "\\bx?tolerance\\b"
-                },
-                {
-                  "name": "support.function.q",
-                  "match": "\\bx?replicate\\b"
+              }
+            },
+            {
+              "match": "\\b(x?should)\\b(.*)",
+              "captures": {
+                "1": {
+                  "name": "support.function.q"
                 },
-                {
-                  "include": "#comments"
+                "2": {
+                  "name": "string.quoted.q"
+                }
+              }
+            },
+            {
+              "match": "\\b(x?expect)\\b(.*)",
+              "captures": {
+                "1": {
+                  "name": "support.function.q"
                 },
-                {
-                  "include": "#strings"
+                "2": {
+                  "name": "string.quoted.q"
+                }
+              }
+            },
+            {
+              "match": "\\b(x?to match)\\b(.*)",
+              "captures": {
+                "1": {
+                  "name": "support.function.q"
                 },
-                {
-                  "include": "#literals"
+                "2": {
+                  "name": "string.quoted.q"
+                }
+              }
+            },
+            {
+              "match": "\\b(x?behaviour)\\b(.*)",
+              "captures": {
+                "1": {
+                  "name": "support.function.q"
                 },
-                {
-                  "include": "#keywords"
+                "2": {
+                  "name": "string.quoted.q"
+                }
+              }
+            },
+            {
+              "match": "\\b(x?baseline)\\b(.*)",
+              "captures": {
+                "1": {
+                  "name": "support.function.q"
                 },
-                {
-                  "include": "#identifiers"
+                "2": {
+                  "name": "string.quoted.q"
+                }
+              }
+            },
+            {
+              "match": "\\b(x?bench)\\b(.*)",
+              "captures": {
+                "1": {
+                  "name": "support.function.q"
                 },
-                {
-                  "include": "#commands"
+                "2": {
+                  "name": "string.quoted.q"
+                }
+              }
+            },
+            {
+              "match": "\\b(x?property)\\b(.*)",
+              "captures": {
+                "1": {
+                  "name": "support.function.q"
                 },
-                {
-                  "include": "#operators"
+                "2": {
+                  "name": "string.quoted.q"
                 }
-              ]
+              }
+            },
+            {
+              "name": "support.function.q",
+              "match": "\\bx?after each\\b"
+            },
+            {
+              "name": "support.function.q",
+              "match": "\\bx?after\\b"
+            },
+            {
+              "name": "support.function.q",
+              "match": "\\bx?before each\\b"
+            },
+            {
+              "name": "support.function.q",
+              "match": "\\bx?before\\b"
+            },
+            {
+              "name": "support.function.q",
+              "match": "\\bx?setup\\b"
+            },
+            {
+              "name": "support.function.q",
+              "match": "\\bx?teardown\\b"
+            },
+            {
+              "name": "support.function.q",
+              "match": "\\bx?skip if\\b"
+            },
+            {
+              "name": "support.function.q",
+              "match": "\\bx?timelimit\\b"
+            },
+            {
+              "name": "support.function.q",
+              "match": "\\bx?tolerance\\b"
+            },
+            {
+              "name": "support.function.q",
+              "match": "\\bx?replicate\\b"
+            },
+            {
+              "include": "#multiline"
+            },
+            {
+              "include": "#literals"
+            },
+            {
+              "include": "#keywords"
+            },
+            {
+              "include": "#tokens"
             }
           ]
-        },
+        }
+      ]
+    },
+    "qdoc": {
+      "patterns": [
         {
+          "name": "comment.qdoc",
+          "begin": "(?:(?<=\\r?\\n|[ \\t])|(?<!.))\\/\\/",
+          "end": "\\r?\\n",
           "patterns": [
             {
-              "name": "comment.qdoc",
-              "begin": "(?:(?<=\\r?\\n|[ \\t])|(?<!.))\\/\\/",
-              "end": "\\r?\\n",
+              "name": "storage.type.qdoc",
+              "match": "@\\b(?:author|category|deprecated|doctest|end|example|fileoverview|kind|name|private|see|subcategory|throws|todo|default-category|default-subcategory|typedef|fileOverview|param|desc|return[s]?|overview)\\b"
+            },
+            {
+              "name": "keyword.control.qdoc",
+              "begin": "{",
+              "end": "}",
               "patterns": [
                 {
-                  "name": "storage.type.qdoc",
-                  "match": "@\\b(?:author|category|deprecated|doctest|end|example|fileoverview|kind|name|private|see|subcategory|throws|todo|default-category|default-subcategory|typedef|fileOverview|param|desc|return[s]?|overview)\\b"
-                },
-                {
-                  "name": "keyword.control.qdoc",
-                  "begin": "{",
-                  "end": "}",
-                  "patterns": [
-                    {
-                      "name": "entity.name.type.qdoc",
-                      "match": "\\b(type|atom|anything|dict|enum|function|hsym|option|string|table|tuple|typedef|vector|bool|boolean|byte|char|character|date|datetime|float|guid|int|integer|long|minute|month|real|second|short|string|symbol|time|timespan|timestamp)\\b"
-                    }
-                  ]
-                },
-                {
-                  "name": "variable.other.qdoc",
-                  "match": "[\\w.]+?(?=\\s*{.+})"
-                },
-                {
-                  "name": "variable.other.qdoc",
-                  "match": "\\s\\.[\\w.]+?\\s"
+                  "name": "entity.name.type.qdoc",
+                  "match": "\\b(type|atom|anything|dict|enum|function|hsym|option|string|table|tuple|typedef|vector|bool|boolean|byte|char|character|date|datetime|float|guid|int|integer|long|minute|month|real|second|short|string|symbol|time|timespan|timestamp)\\b"
                 }
               ]
+            },
+            {
+              "name": "variable.other.qdoc",
+              "match": "[\\w.]+?(?=\\s*{.+})"
+            },
+            {
+              "name": "variable.other.qdoc",
+              "match": "\\s\\.[\\w.]+?\\s"
             }
           ]
-        },
+        }
+      ]
+    },
+    "multiline": {
+      "patterns": [
         {
           "name": "comment.block.q",
           "begin": "^\\/\\s*$",
@@ -246,11 +228,7 @@
         {
           "name": "comment.line.q",
           "match": "(?:(?<=[ \\t])|(?<!.))\\/.*"
-        }
-      ]
-    },
-    "strings": {
-      "patterns": [
+        },
         {
           "name": "string.quoted.q",
           "begin": "\"",
@@ -321,46 +299,38 @@
         {
           "name": "keyword.other.q",
           "match": "\\b(?:reciprocal|distinct|ceiling|reverse|sublist|ungroup|delete|deltas|differ|enlist|except|getenv|hclose|hcount|insert|mcount|ratios|rotate|select|setenv|signum|string|system|tables|update|upsert|within|xgroup|count|cross|dsave|fills|first|fkeys|floor|group|gtime|hopen|idesc|inter|lower|ltime|ltrim|parse|peach|prior|read0|read1|reval|rload|rsave|rtrim|union|upper|value|views|where|while|xcols|xdesc|xprev|xrank|acos|ajf0|asin|asof|atan|attr|avgs|binr|cols|desc|each|eval|exec|exit|flip|from|hdel|hsym|iasc|keys|last|like|load|mavg|maxs|mdev|meta|mins|mmax|mmin|msum|next|null|over|prds|prev|rand|rank|raze|save|scan|scov|sdev|show|sqrt|sums|svar|trim|type|view|wavg|wsum|xasc|xbar|xcol|xexp|xkey|xlog|abs|aj0|ajf|all|and|any|asc|avg|bin|cor|cos|cov|csv|cut|dev|div|ema|exp|fby|get|ijf|inv|key|ljf|log|lsq|max|md5|med|min|mmu|mod|neg|not|prd|set|sin|ssr|sum|tan|til|ujf|var|wj1|aj|do|ej|if|ij|in|lj|or|pj|ss|sv|uj|vs|wj)\\b"
-        }
-      ]
-    },
-    "identifiers": {
-      "patterns": [
+        },
         {
           "name": "variable.other.q",
           "match": "\\.?[a-zA-Z][a-zA-Z0-9_]*(?:\\.[a-zA-Z0-9_]+)*\\b"
         }
       ]
     },
-    "commands": {
+    "tokens": {
       "patterns": [
         {
           "name": "constant.character.q",
           "match": "(?<!.)\\\\(?:cd|ts|[abBcCdefglopPrsStTuvwWxz12_\\\\]).*"
-        }
-      ]
-    },
-    "operators": {
-      "patterns": [
+        },
         {
           "name": "keyword.other.iterator.q",
           "match": "[\\\\'/]:"
         },
         {
-          "name": "keyword.other.control.q",
-          "match": "[$!?#@'^]"
+          "name": "punctuation.assignment.q",
+          "match": "::"
         },
         {
           "name": "keyword.operator.arithmetic.q",
           "match": "[_,.'^?!#@$&|%*+-]"
         },
         {
-          "name": "punctuation.assignment.q",
-          "match": ":"
+          "name": "keyword.operator.arithmetic.q",
+          "match": "[~<=>]"
         },
         {
           "name": "punctuation.assignment.q",
-          "match": "::"
+          "match": ":"
         },
         {
           "name": "punctuation.terminator.statement.q",