More misc fixes

CHANGELOG.md

@@ -51,7 +51,9 @@ All notable changes to the "nwscript-ee-language-server" extension will be docum
 
 ## [1.5.3]
 
-- Goto will now work for functions from `nwscript.nss` if the file is in your project.
+- Goto will now work for functions and constants from `nwscript.nss` if the file is in your project.
+- Fixed a few issues with the tokenizer.
+- Fixed a small issue with `const` expressions resolution.
 - Fixed the compilation provider not reporting warnings.
 - New compiler setting `reportWarnings`. True by default.
 - New formatter setting `verbose`. False by default.

server/src/Documents/Document.ts

@@ -39,7 +39,13 @@ export default class Document {
   }
 
   public getGlobalComplexTokensWithRef(computedChildren: string[] = []): OwnedComplexTokens[] {
-    return [{ owner: this.uri, tokens: this.complexTokens }].concat(
+    const localStandardLibDefinitions = this.collection.get("nwscript");
+    return [
+      { owner: this.uri, tokens: this.complexTokens },
+      ...(localStandardLibDefinitions
+        ? [{ owner: localStandardLibDefinitions.uri, tokens: localStandardLibDefinitions.complexTokens }]
+        : []),
+    ].concat(
       this.children.flatMap((child) => {
         // Cycling children or/and duplicates
         if (computedChildren.includes(child)) {
@@ -59,25 +65,27 @@ export default class Document {
     );
   }
 
-  public getGlobalComplexTokens(computedChildren: string[] = []): ComplexToken[] {
-    return this.complexTokens.concat(
-      this.children.flatMap((child) => {
-        // Cycling children or/and duplicates
-        if (computedChildren.includes(child)) {
-          return [];
-        } else {
-          computedChildren.push(child);
-        }
-
-        const childDocument = this.collection.get(child);
-
-        if (!childDocument) {
-          return [];
-        }
-
-        return childDocument.getGlobalComplexTokens(computedChildren);
-      }),
-    );
+  public getGlobalComplexTokens(computedChildren: string[] = [], localFunctionIdentifiers: string[] = []): ComplexToken[] {
+    return this.complexTokens
+      .filter((token) => !localFunctionIdentifiers.includes(token.identifier))
+      .concat(
+        this.children.flatMap((child) => {
+          // Cycling children or/and duplicates
+          if (computedChildren.includes(child)) {
+            return [];
+          } else {
+            computedChildren.push(child);
+          }
+
+          const childDocument = this.collection.get(child);
+
+          if (!childDocument) {
+            return [];
+          }
+
+          return childDocument.getGlobalComplexTokens(computedChildren);
+        }),
+      );
   }
 
   public getGlobalStructComplexTokensWithRef(computedChildren: string[] = []): OwnedStructComplexTokens[] {

server/src/Providers/CompletionItemsProvider.ts

@@ -61,7 +61,7 @@ export default class CompletionItemsProvider extends Provider {
           }
 
           return this.getLocalScopeCompletionItems(localScope)
-            .concat(this.getGlobalScopeCompletionItems(document))
+            .concat(this.getGlobalScopeCompletionItems(document, localScope))
             .concat(this.getStandardLibCompletionItems());
         }
       }
@@ -77,8 +77,15 @@ export default class CompletionItemsProvider extends Provider {
     return functionVariablesCompletionItems.concat(functionsCompletionItems);
   }
 
-  private getGlobalScopeCompletionItems(document: Document | undefined) {
-    return document?.getGlobalComplexTokens().map((token) => CompletionItemBuilder.buildItem(token)) || [];
+  private getGlobalScopeCompletionItems(document: Document | undefined, localScope: LocalScopeTokenizationResult) {
+    return (
+      document
+        ?.getGlobalComplexTokens(
+          [],
+          localScope.functionsComplexTokens.map((token) => token.identifier),
+        )
+        .map((token) => CompletionItemBuilder.buildItem(token)) || []
+    );
   }
 
   private getStandardLibCompletionItems() {

server/src/Providers/DiagnosticsProvider.ts

@@ -42,7 +42,7 @@ export default class DiagnoticsProvider extends Provider {
             start: { line: linePosition, character: 0 },
             end: { line: linePosition, character: Number.MAX_VALUE },
           },
-          message: lineMessage.exec(line)![1].trim(),
+          message: lineMessage.exec(line)![2].trim(),
         };
 
         files[uri].push(diagnostic);
@@ -71,7 +71,7 @@ export default class DiagnoticsProvider extends Provider {
     return async () => {
       return await new Promise<boolean>((resolve, reject) => {
         const { enabled, nwnHome, reportWarnings, nwnInstallation, verbose } = this.server.config.compiler;
-        if (!enabled) {
+        if (!enabled || uri.includes("nwscript.nss")) {
           return resolve(true);
         }
 

server/src/Providers/Formatters/ClangFormatter.ts

@@ -1,25 +1,28 @@
 import { spawn } from "child_process";
 import { parser, Tag } from "sax";
-import { Range } from "vscode-languageserver";
-import { TextDocument, TextEdit } from "vscode-languageserver-textdocument";
+import { TextDocument, Range, TextEdit } from "vscode-languageserver-textdocument";
 
 import Formatter from "./Formatter";
 
-type CurrentEdit = { length: number; offset: number; text: string } | null;
+type CurrentEdit = { length: number; offset: number; text: string };
+
 export default class ClangFormatter extends Formatter {
-  private xmlParseOnText(currentEdit: CurrentEdit) {
+  edits: TextEdit[] = [];
+  currentEdit: CurrentEdit | null = null;
+
+  private xmlParseOnText() {
     return (text: string) => {
-      if (!currentEdit) {
+      if (!this.currentEdit) {
         return;
       }
 
-      currentEdit.text = text;
+      this.currentEdit.text = text;
     };
   }
 
-  private xmlParserOnOpenTag(currentEdit: CurrentEdit, reject: (reason: any) => void) {
+  private xmlParserOnOpenTag(reject: (reason: any) => void) {
     return (tag: Tag) => {
-      if (currentEdit) {
+      if (this.currentEdit) {
         reject(new Error("Malformed output."));
       }
 
@@ -28,7 +31,7 @@ export default class ClangFormatter extends Formatter {
           return;
 
         case "replacement":
-          currentEdit = {
+          this.currentEdit = {
             length: parseInt(tag.attributes.length.toString()),
             offset: parseInt(tag.attributes.offset.toString()),
             text: "",
@@ -41,17 +44,17 @@ export default class ClangFormatter extends Formatter {
     };
   }
 
-  private xmlParserOnCloseTag(document: TextDocument, edits: TextEdit[], currentEdit: CurrentEdit) {
+  private xmlParserOnCloseTag(document: TextDocument) {
     return () => {
-      if (!currentEdit) {
+      if (!this.currentEdit) {
         return;
       }
 
-      const start = document.positionAt(currentEdit.offset);
-      const end = document.positionAt(currentEdit.offset + currentEdit.length);
+      const start = document.positionAt(this.currentEdit.offset);
+      const end = document.positionAt(this.currentEdit.offset + this.currentEdit.length);
 
-      edits.push({ range: { start, end }, newText: currentEdit.text });
-      currentEdit = null;
+      this.edits.push({ range: { start, end }, newText: this.currentEdit.text });
+      this.currentEdit = null;
     };
   }
 
@@ -81,7 +84,9 @@ export default class ClangFormatter extends Formatter {
         this.logger.info(`Resolving clang-format's executable with: ${this.executable}.`);
       }
 
-      const child = spawn(this.executable, args, { shell: true });
+      const child = spawn(this.executable, args, {
+        cwd: this.workspaceFilesSystem.getWorkspaceRootPath(),
+      });
 
       child.stdin.end(document.getText());
       child.stdout.on("data", (chunk: string) => (stdout += chunk));
@@ -98,25 +103,23 @@ export default class ClangFormatter extends Formatter {
           reject(new Error(stderr));
         }
 
-        let currentEdit: CurrentEdit = null;
-        const edits: TextEdit[] = [];
         const xmlParser = parser(true, {
           trim: false,
           normalize: false,
         });
 
         xmlParser.onerror = (err) => reject(err);
-        xmlParser.ontext = this.xmlParseOnText(currentEdit);
-        xmlParser.onopentag = this.xmlParserOnOpenTag(currentEdit, reject);
-        xmlParser.onclosetag = this.xmlParserOnCloseTag(document, edits, currentEdit);
+        xmlParser.ontext = this.xmlParseOnText();
+        xmlParser.onopentag = this.xmlParserOnOpenTag(reject);
+        xmlParser.onclosetag = this.xmlParserOnCloseTag(document);
         xmlParser.write(stdout);
         xmlParser.end();
 
         if (this.verbose) {
           this.logger.info("Done.\n");
         }
 
-        resolve(edits);
+        resolve(this.edits);
       });
     });
   }

server/src/Providers/Formatters/Formatter.ts

@@ -1,9 +1,10 @@
 import { fileURLToPath } from "url";
+import { Range, TextDocument, TextEdit } from "vscode-languageserver-textdocument";
 
 import { WorkspaceFilesSystem } from "../../WorkspaceFilesSystem";
 import { Logger } from "../../Logger";
 
-export default class Formatter {
+export default abstract class Formatter {
   constructor(
     protected readonly workspaceFilesSystem: WorkspaceFilesSystem,
     protected readonly enabled: boolean,
@@ -21,4 +22,6 @@ export default class Formatter {
       return this.workspaceFilesSystem.getGlobPaths(glob).some((path) => path === documentPath);
     });
   }
+
+  protected abstract formatDocument(document: TextDocument, range: Range | null): Promise<TextEdit[] | null>;
 }

server/src/Tokenizer/Tokenizer.ts

@@ -110,7 +110,8 @@ export default class Tokenizer {
   private getInlineFunctionParams(line: string, lineIndex: number, tokensArray: IToken[]) {
     const functionParamTokens = tokensArray.filter(
       (token) =>
-        token.scopes.includes(LanguageScopes.functionParameter) || token.scopes.includes(LanguageScopes.variableIdentifer),
+        token.scopes.includes(LanguageScopes.functionParameters) &&
+        (token.scopes.includes(LanguageScopes.functionParameter) || token.scopes.includes(LanguageScopes.variableIdentifer)),
     );
 
     return functionParamTokens.map((token) => {
@@ -167,7 +168,14 @@ export default class Tokenizer {
     while (!isLastParamsLine) {
       isLastParamsLine = Boolean(tokensArray.find((token) => token.scopes.includes(LanguageScopes.rightParametersRoundBracket)));
 
-      if (isLastParamsLine && Boolean(tokensArray.find((token) => token.scopes.includes(LanguageScopes.terminatorStatement)))) {
+      if (
+        isLastParamsLine &&
+        Boolean(
+          tokensArray.find(
+            (token) => token.scopes.includes(LanguageScopes.terminatorStatement) && !token.scopes.includes(LanguageScopes.block),
+          ),
+        )
+      ) {
         isFunctionDeclaration = true;
       }
 

server/src/Tokenizer/constants.ts

@@ -35,6 +35,7 @@ export enum LanguageScopes {
   functionCall = "meta.function-call.nss",
   functionDeclaration = "meta.function.nss",
   functionParameter = "variable.parameter.nss",
+  functionParameters = "meta.function.definition.parameters.nss",
   block = "meta.block.nss",
   blockDeclaraction = "punctuation.section.block.begin.bracket.curly.nss",
   blockTermination = "punctuation.section.block.end.bracket.curly.nss",

server/src/WorkspaceFilesSystem/WorkspaceFilesSystem.ts

@@ -18,4 +18,8 @@ export default class WorkspaceFilesSystem {
   public getGlobPaths(glob: string) {
     return new GlobSync(glob).found.map((filename) => this.normalizedAbsolutePath(filename));
   }
+
+  public getWorkspaceRootPath() {
+    return this.rootPath;
+  }
 }

syntaxes/nwscript-ee.tmLanguage

@@ -3730,7 +3730,7 @@ U[a-fA-F0-9]{,8} )</string>
         <array>
           <dict>
             <key>match</key>
-            <string>(const){0,1} *\b(action|effect|event|float|int|itemproperty|location|object|string|talent|vector|void|json|sqlquery|cassowary)\b (?(1)(\S*)|)</string>
+            <string>(const){0,1} *\b(action|effect|event|float|int|itemproperty|location|object|string|talent|vector|void|json|sqlquery|cassowary)\b (?(1)([a-zA-Z0-9_]+)|)</string>
             <key>captures</key>
             <dict>
               <key>1</key>

syntaxes/nwscript-ee.tmLanguage.json

@@ -2402,7 +2402,7 @@
     "storage_types": {
       "patterns": [
         {
-          "match": "(const){0,1} *\\b(action|effect|event|float|int|itemproperty|location|object|string|talent|vector|void|json|sqlquery|cassowary)\\b (?(1)(\\S*)|)",
+          "match": "(const){0,1} *\\b(action|effect|event|float|int|itemproperty|location|object|string|talent|vector|void|json|sqlquery|cassowary)\\b (?(1)([a-zA-Z0-9_]+)|)",
           "captures": {
             "1": {
               "name": "storage.modifier.nss"