TH_Tokenized: fix getPosInfoAtOffset for 2-byte tokens.

adriweb · May 26, 2024 · 50eeb42 · 50eeb42
1 parent 94362b7
commit 50eeb42
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 2 deletions.
diff --git a/TIVarsLib.wasm b/TIVarsLib.wasm
diff --git a/src/TypeHandlers/TH_Tokenized.cpp b/src/TypeHandlers/TH_Tokenized.cpp
@@ -407,7 +407,7 @@ namespace tivars
         }
 
         // Find column number and token length if byteOffset is reached
-        for (uint16_t i = lastNewLineOffset+1; i <= byteOffset; i++)
+        for (uint16_t i = std::max(2, lastNewLineOffset+1); i <= byteOffset; i++)
         {
             const uint8_t currentToken = data[i];
             uint8_t nextToken = (i < dataSize-1) ? data[i+1] : (uint8_t)-1;
@@ -440,7 +440,7 @@ namespace tivars
 
             posinfo.column += (uint16_t)tokStr.size();
 
-            if (posinfo.len == 0 && ((currIdx == byteOffset && !is2ByteTok) || (currIdx == byteOffset-1 && is2ByteTok)))
+            if (posinfo.len == 0 && ((currIdx == byteOffset && !is2ByteTok) || (currIdx >= byteOffset-1 && is2ByteTok)))
             {
                 posinfo.len = (uint8_t)tokStr.size();
                 posinfo.column -= posinfo.len; // column will be the beginning of the token

diff --git a/tests.cpp b/tests.cpp
@@ -100,6 +100,17 @@ int main(int argc, char** argv)
         assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
     }
 
+    {
+        TH_Tokenized::token_posinfo actual{}, expected{};
+        const std::string hexStr = "0700DEEF983170323F";
+        actual = TH_Tokenized::getPosInfoAtOffsetFromHexStr(hexStr, 2);
+        expected = { 0, 0, 5 };
+        assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
+        actual = TH_Tokenized::getPosInfoAtOffsetFromHexStr(hexStr, 3);
+        expected = { 0, 5, 5 };
+        assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
+    }
+
     {
         // Test string interpolation behaviour
         TIVarFile testPrgm = TIVarFile::createNew("Program", "INTERP");