improvement: Catch an error if thrown during tokenization

kasiaMarek · Jun 29, 2023 · 66cdf3d · 66cdf3d
1 parent f460b18
commit 66cdf3d
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 15 deletions.
diff --git a/metals-bench/src/main/scala/bench/SemanticTokensBench.scala b/metals-bench/src/main/scala/bench/SemanticTokensBench.scala
@@ -3,7 +3,6 @@ package bench
 import java.net.URI
 import java.nio.charset.StandardCharsets
 import java.util.concurrent.TimeUnit
-import java.{util => ju}
 
 import scala.meta.internal.io.FileIO
 import scala.meta.internal.jdk.CollectionConverters._
@@ -71,7 +70,7 @@ class SemanticTokensBench extends PcBenchmark {
   @Benchmark
   @BenchmarkMode(Array(Mode.SingleShotTime))
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
-  def semanticHighlight(): ju.List[Integer] = {
+  def semanticHighlight(): List[Integer] = {
     val pc = presentationCompiler()
     val text = currentHighlight
     val vFile = CompilerVirtualFileParams(

diff --git a/metals/src/main/scala/scala/meta/internal/metals/Compilers.scala b/metals/src/main/scala/scala/meta/internal/metals/Compilers.scala
@@ -11,6 +11,7 @@ import scala.collection.concurrent.TrieMap
 import scala.concurrent.ExecutionContextExecutorService
 import scala.concurrent.Future
 import scala.util.Try
+import scala.util.control.NonFatal
 
 import scala.meta.inputs.Input
 import scala.meta.inputs.Position
@@ -474,14 +475,24 @@ class Compilers(
             .semanticTokens(vFile)
             .asScala
             .map { nodes =>
-              val plist = SemanticTokensProvider.provide(
-                nodes.asScala.toList,
-                vFile,
-                isScala3,
-              )
+              val plist =
+                try {
+                  SemanticTokensProvider.provide(
+                    nodes.asScala.toList,
+                    vFile,
+                    isScala3,
+                  )
+                } catch {
+                  case NonFatal(e) =>
+                    scribe.error(
+                      s"Failed to tokenize input for semantic tokens for $path",
+                      e,
+                    )
+                    Nil
+                }
 
               val tokens =
-                findCorrectStart(0, 0, plist.asScala.toList)
+                findCorrectStart(0, 0, plist.toList)
               if (isScala3 && path.isWorksheet) {
                 new SemanticTokens(adjustForScala3Worksheet(tokens).asJava)
               } else {

diff --git a/metals/src/main/scala/scala/meta/internal/metals/SemanticTokensProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/SemanticTokensProvider.scala
@@ -1,7 +1,5 @@
 package scala.meta.internal.metals
 
-import java.{util => ju}
-
 import scala.annotation.switch
 import scala.annotation.tailrec
 import scala.collection.mutable.ListBuffer
@@ -80,11 +78,11 @@ object SemanticTokensProvider {
       nodes: List[Node],
       params: VirtualFileParams,
       isScala3: Boolean,
-  ): ju.List[Integer] = {
+  ): List[Integer] = {
     // no semantic data was available, we can revert to default highlighting
     if (nodes.isEmpty) {
       scribe.warn("Could not find semantic tokens for: " + params.uri())
-      List.empty[Integer].asJava
+      List.empty[Integer]
     } else {
       val tokens = getTokens(isScala3, params.text())
       val buffer = ListBuffer.empty[Integer]
@@ -100,7 +98,7 @@ object SemanticTokensProvider {
         )
         delta = delta0
       }
-      buffer.toList.asJava
+      buffer.toList
     }
   }
 

diff --git a/tests/slow/src/test/scala/tests/feature/SemanticTokensScala3ExpectSuite.scala b/tests/slow/src/test/scala/tests/feature/SemanticTokensScala3ExpectSuite.scala
@@ -70,7 +70,7 @@ class SemanticTokensScala3ExpectSuite(
 
           TestSemanticTokens.semanticString(
             file.code,
-            tokens.asScala.toList.map(_.toInt),
+            tokens.toList.map(_.toInt),
           )
         },
       )

diff --git a/tests/unit/src/test/scala/tests/SemanticTokensExpectSuite.scala b/tests/unit/src/test/scala/tests/SemanticTokensExpectSuite.scala
@@ -32,7 +32,7 @@ class SemanticTokensExpectSuite extends DirectoryExpectSuite("semanticTokens") {
 
           TestSemanticTokens.semanticString(
             file.code,
-            tokens.asScala.toList.map(_.toInt),
+            tokens.toList.map(_.toInt),
           )
         },
       )