TH_Tokenized: use true lower-alpha tokens in strings.

adriweb · May 31, 2024 · 7be5c1b · 7be5c1b
1 parent c125d9d
commit 7be5c1b
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 0 deletions.
diff --git a/TIVarsLib.wasm b/TIVarsLib.wasm
diff --git a/src/TypeHandlers/TH_Tokenized.cpp b/src/TypeHandlers/TH_Tokenized.cpp
@@ -96,6 +96,19 @@ namespace tivars
                  currentLength += (needMinMunch ? 1 : -1))
             {
                 std::string currentSubString = str_new.substr(strCursorPos, currentLength);
+
+                // We want to use true-lowercase alpha tokens in this case.
+                if ((isWithinString && !inEvaluatedString) && currentLength == 1 && std::islower(currentSubString[0]))
+                {
+                    // 0xBBB0 is 'a', etc. But we skip what would be 'l' at 0xBBBB which doesn't exist (prefix conflict)
+                    const char letter = currentSubString[0];
+                    uint16_t tokenValue = 0xBBB0 + (letter - 'a') + (letter >= 'l' ? 1 : 0);
+                    data.push_back(tokenValue >> 8);
+                    data.push_back(tokenValue & 0xFF);
+                    lastTokenBytes = tokenValue;
+                    break;
+                }
+
                 if (tokens_NameToBytes.count(currentSubString))
                 {
                     uint16_t tokenValue = tokens_NameToBytes[currentSubString];

diff --git a/tests.cpp b/tests.cpp
@@ -119,6 +119,18 @@ int main(int argc, char** argv)
         assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
     }
 
+    {
+        // Test lower alpha being the expected lowercase tokens, not special-meaning ones
+        TIVarFile testPrgm = TIVarFile::createNew("Program", "INTERP");
+        testPrgm.setContentFromString("Disp \"abcdefghijklmnopqrstuvwxyz\"");
+        string detok_fr = testPrgm.getReadableContent({{"lang", LANG_FR}});
+        string detok_en = testPrgm.getReadableContent({{"lang", LANG_EN}});
+        string hex = testPrgm.getRawContentHexStr();
+        assert(detok_fr == "Disp \"abcdefghijklmnopqrstuvwxyz\"");
+        assert(detok_en == "Disp \"abcdefghijklmnopqrstuvwxyz\"");
+        assert(hex == "3700de2abbb0bbb1bbb2bbb3bbb4bbb5bbb6bbb7bbb8bbb9bbbabbbcbbbdbbbebbbfbbc0bbc1bbc2bbc3bbc4bbc5bbc6bbc7bbc8bbc9bbca2a");
+    }
+
     {
         // Test string interpolation behaviour
         TIVarFile testPrgm = TIVarFile::createNew("Program", "INTERP");