From 612fadeedadfd09bf3f8f7626596d78a899813f8 Mon Sep 17 00:00:00 2001
From: Tomasz Godzik <tgodzik@virtuslab.com>
Date: Tue, 10 Sep 2024 12:12:54 +0200
Subject: [PATCH] improvement: Add report when tokenization fails

---
 .../src/main/scala/bench/SemanticTokensBench.scala  |  3 ++-
 .../internal/metals/SemanticTokensProvider.scala    | 13 +++++++++++--
 .../feature/SemanticTokensScala3ExpectSuite.scala   |  3 ++-
 .../scala/tests/SemanticTokensExpectSuite.scala     |  3 ++-
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/metals-bench/src/main/scala/bench/SemanticTokensBench.scala b/metals-bench/src/main/scala/bench/SemanticTokensBench.scala
index cff19757385..49f4f75b4fa 100644
--- a/metals-bench/src/main/scala/bench/SemanticTokensBench.scala
+++ b/metals-bench/src/main/scala/bench/SemanticTokensBench.scala
@@ -9,6 +9,7 @@ import scala.meta.internal.io.FileIO
 import scala.meta.internal.jdk.CollectionConverters._
 import scala.meta.internal.metals.CompilerVirtualFileParams
 import scala.meta.internal.metals.EmptyCancelToken
+import scala.meta.internal.metals.EmptyReportContext
 import scala.meta.internal.metals.ScalaVersions
 import scala.meta.internal.metals.SemanticTokensProvider
 import scala.meta.io.AbsolutePath
@@ -92,7 +93,7 @@ class SemanticTokensBench extends PcBenchmark {
       AbsolutePath(Paths.get(vFile.uri)),
       isScala3,
       MetalsTestEnrichments.emptyTrees,
-    )
+    )(EmptyReportContext)
   }
 
   def currentHighlight: String = highlightRequests(currentHighlightRequest)
diff --git a/metals/src/main/scala/scala/meta/internal/metals/SemanticTokensProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/SemanticTokensProvider.scala
index 67b41b9572b..dec331e2a77 100644
--- a/metals/src/main/scala/scala/meta/internal/metals/SemanticTokensProvider.scala
+++ b/metals/src/main/scala/scala/meta/internal/metals/SemanticTokensProvider.scala
@@ -86,7 +86,7 @@ object SemanticTokensProvider {
       path: AbsolutePath,
       isScala3: Boolean,
       trees: Trees,
-  ): List[Integer] = {
+  )(implicit rc: ReportContext): List[Integer] = {
     // no semantic data was available, we can revert to default highlighting
     if (nodes.isEmpty) {
       if (params.uri().toString().isScalaFilename)
@@ -94,7 +94,16 @@ object SemanticTokensProvider {
       List.empty[Integer]
     } else {
       val tokens = Try(getTokens(isScala3, params.text())) match {
-        case Failure(_) =>
+        case Failure(t) =>
+          rc.unsanitized.create(
+            Report(
+              "semantic-tokens-provider",
+              params.text(),
+              s"Could not find semantic tokens for: ${params.uri()}",
+              Some(params.uri().toString()),
+              error = Some(t),
+            )
+          )
           /* Try to get tokens from trees as a fallback to avoid
            * things being unhighlighted too often.
            */
diff --git a/tests/slow/src/test/scala/tests/feature/SemanticTokensScala3ExpectSuite.scala b/tests/slow/src/test/scala/tests/feature/SemanticTokensScala3ExpectSuite.scala
index 9300248ac84..c8336d7233c 100644
--- a/tests/slow/src/test/scala/tests/feature/SemanticTokensScala3ExpectSuite.scala
+++ b/tests/slow/src/test/scala/tests/feature/SemanticTokensScala3ExpectSuite.scala
@@ -2,6 +2,7 @@ package tests.feature
 
 import scala.meta.internal.metals.CompilerVirtualFileParams
 import scala.meta.internal.metals.EmptyCancelToken
+import scala.meta.internal.metals.EmptyReportContext
 import scala.meta.internal.metals.MetalsEnrichments._
 import scala.meta.internal.metals.SemanticTokensProvider
 import scala.meta.internal.metals.{BuildInfo => V}
@@ -40,7 +41,7 @@ class SemanticTokensScala3ExpectSuite(
             file.file,
             isScala3 = true,
             MetalsTestEnrichments.emptyTrees,
-          )
+          )(EmptyReportContext)
 
           TestSemanticTokens.semanticString(
             file.code,
diff --git a/tests/unit/src/test/scala/tests/SemanticTokensExpectSuite.scala b/tests/unit/src/test/scala/tests/SemanticTokensExpectSuite.scala
index 8c9b5dd3f6e..7c79f2180ed 100644
--- a/tests/unit/src/test/scala/tests/SemanticTokensExpectSuite.scala
+++ b/tests/unit/src/test/scala/tests/SemanticTokensExpectSuite.scala
@@ -2,6 +2,7 @@ package tests
 
 import scala.meta.internal.metals.CompilerVirtualFileParams
 import scala.meta.internal.metals.EmptyCancelToken
+import scala.meta.internal.metals.EmptyReportContext
 import scala.meta.internal.metals.MetalsEnrichments._
 import scala.meta.internal.metals.SemanticTokensProvider
 import scala.meta.internal.pc.ScalaPresentationCompiler
@@ -30,7 +31,7 @@ class SemanticTokensExpectSuite extends DirectoryExpectSuite("semanticTokens") {
             file.file,
             isScala3 = false,
             MetalsTestEnrichments.emptyTrees,
-          )
+          )(EmptyReportContext)
 
           TestSemanticTokens.semanticString(
             file.code,