From 50eeb42026d61d154463482093f89b52644492fe Mon Sep 17 00:00:00 2001
From: Adrien Bertrand <bertrand.adrien@gmail.com>
Date: Mon, 27 May 2024 00:50:45 +0200
Subject: [PATCH] TH_Tokenized: fix getPosInfoAtOffset for 2-byte tokens.

---
 TIVarsLib.wasm                    | Bin 526372 -> 526389 bytes
 src/TypeHandlers/TH_Tokenized.cpp |   4 ++--
 tests.cpp                         |  11 +++++++++++
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/TIVarsLib.wasm b/TIVarsLib.wasm
index 351dc3215cdc66dc8d11712ee702cadc4c991264..51e300edaef2c27014b6df56d8e59e71ca849409 100755
GIT binary patch
delta 1160
zcmZuwziSjh6rS0c+1b6BP1g7eJcKucArTCU3PK`yBPJJ#lGfJJJ&Rnhv)IDI!p1__
zCP)Mk6-~5|Sc?7yqMe}r1y<Ic-`sj=;lbR_yr19u-g}Sl)0_MBs%QE8JHg%!mg6Ki
zVlP;^wZ-2zU!Th32a51^q}K{YMc~8uSuRBLTBQf;yz3=pmy3k>e8QY25fXxA0I>|o
zI@F8IkRZ9~(xfK2EPIp&w#XO>%cDjUr@Tv9MY5dpdVv`yCng#}VIbcHw%Rp;g|fUS
z-Bi*eMV&R-tEod%F;*w=%5`GE4w->Z@$@;IiWq8ypN0n%I`CV7mIGtH)+IgG3Bb%{
zO?f%10yAD#UbkT-8A^c))|FE*k6Fl|22&n0B!x4kxjqv6vn(W)&!}jwJeRkN7Q|)Q
z1JZQw^K*HE*ZT}5s6H^qZHhan1Yh{9jD^VTI8Mi^sQfh&Ll<S)L@s)ik*O9K3m5u8
z%N4bC`_2+)lz}#6Gn56s>G{O2M-utK%fWw)4ajH#HxFLOi?v=xQ^C(4Cp=o?7*{zs
z10`{AC5Ap;_*L{ZQrybc!3Qh`pa#SbTtRuTH{BGF=Rj>HuG8QZ@SP1K4mJy#Mhi(V
z)C|u!3U-N~PkLedkwV{n79O`3y;87PJ$%Wo6g-v~$=3Y>v!z{zKgw2GP6R20AL2j|
zzC0A+6Dyy9Sw+f34cU<0+VnA39MQ_S=EF;Q^h^p8!q4rP8iN`OAv>h>M&N5+L3e~K
zrDU-N<J*SuGtIYi$>CbHGmXMi5gTsfE=aHXC{9s2q#0ng5;UCTSu7x4V}Ts-(VG7`
zt)m$DZbZ<nzxeAPuM>Ym&9faj$8Ga*M;_8sRe9RbzUIe{oSng*t$NjsEh90XoTapx
z-IeF-vVDmKwiI>mJmzc_W|c3Oc!CP@$P=ajtoE@A2~GDm$3vQ9y{4~_9xfM{3Pya9
rNqoxKVLlv;{fWCA1BQ^OA8AjGH|6iN(B1F55=Y^rd*65UrE9+dJb%Rz

delta 1084
zcmYjQziSjh6rS0g*<Z7pY$8GQK;8^WA{Z1E14+OeA$JK8Y;7Yp*|YFMI*Sw*DQqGL
z+XPJ@hy+bcArWj++ZroD{|Rf)?`=5oU~XpK`{upxeedn3E!x|nkL_F@J@@&h&m~rX
z6(0LsZEwq?`sX`!btCbg2k~+tDDkq-+d>hBZIV-gA~`LLQ>0G3SaeYf$9Sm-*e7g8
zp$`a_9wU740Pa~~C?vIJb+AEdLA6PHF)IXxxnxg9lA;#H6{&L4Wr%K9)ew^*DiBSw
zmhr-u<qKugpiMay%ZOD_%CQPBD_d~Df=th8TwMf$1xFg-7vljOnV0LJ7J*~5+@h==
zd{-0Gs&d&`97TpSXuw_BoB&5yGDu{~%Z8LSrd}C@=aSe5m5V8<@9(Q;MUymHQf&}R
zj!*a1HMvn53Q(;VR^<`SP|`-cbs8f?unJ4&nw@_JVc)VX9@?RIOs1L^!r$#GNT*4V
z<ig~f6cihqA)BGt%QdHlxHe_T2OTE<1EVIum~j2-y}Gv%Eug3RUvON`z#Law1wLpV
zCxUcg{{gB8?VcUuap>7`N`3ys79g5|qyZ_0xe-VXwy0ttq1_0LLlA-qH*U0^lk-8_
z@9aL-=&FnT-fnY<I)&h3@s@O_kX#XK8+8kePGn>JQQUz9QX@=`@O>;8{Eye6|0c?}
z(hE(wVGsJ9tgks;fBm2)Z((qS|GmR`Fnvs)bf1z{FPB{k-$lrvDOox@B~OjZ>t8d$
z`Nfn6wJA3VxvRPvewTLHOlhBTFwG&Aye<=_%||Xvf;99WKC0h*(HBM<a5x0l`v+>~
z5^C}DRrxZ&exssrDxj$VQ(cue4VCraKwTM6`5V(MC-IE{aGTQl&Y_xrG<tRmr^Oum
zPB8Fd5wOaQ&dC6DGnN8<g>YSmsiTqZS~GST`ZCs`sJ~EPNYM6Kp|Fi|6J@{GyeDb1
R&C^4LrLm&p)5Cb>;a`m+wnYE{

diff --git a/src/TypeHandlers/TH_Tokenized.cpp b/src/TypeHandlers/TH_Tokenized.cpp
index 1af3c4a..8b30050 100644
--- a/src/TypeHandlers/TH_Tokenized.cpp
+++ b/src/TypeHandlers/TH_Tokenized.cpp
@@ -407,7 +407,7 @@ namespace tivars
         }
 
         // Find column number and token length if byteOffset is reached
-        for (uint16_t i = lastNewLineOffset+1; i <= byteOffset; i++)
+        for (uint16_t i = std::max(2, lastNewLineOffset+1); i <= byteOffset; i++)
         {
             const uint8_t currentToken = data[i];
             uint8_t nextToken = (i < dataSize-1) ? data[i+1] : (uint8_t)-1;
@@ -440,7 +440,7 @@ namespace tivars
 
             posinfo.column += (uint16_t)tokStr.size();
 
-            if (posinfo.len == 0 && ((currIdx == byteOffset && !is2ByteTok) || (currIdx == byteOffset-1 && is2ByteTok)))
+            if (posinfo.len == 0 && ((currIdx == byteOffset && !is2ByteTok) || (currIdx >= byteOffset-1 && is2ByteTok)))
             {
                 posinfo.len = (uint8_t)tokStr.size();
                 posinfo.column -= posinfo.len; // column will be the beginning of the token
diff --git a/tests.cpp b/tests.cpp
index 8b7a059..d730d8e 100644
--- a/tests.cpp
+++ b/tests.cpp
@@ -100,6 +100,17 @@ int main(int argc, char** argv)
         assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
     }
 
+    {
+        TH_Tokenized::token_posinfo actual{}, expected{};
+        const std::string hexStr = "0700DEEF983170323F";
+        actual = TH_Tokenized::getPosInfoAtOffsetFromHexStr(hexStr, 2);
+        expected = { 0, 0, 5 };
+        assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
+        actual = TH_Tokenized::getPosInfoAtOffsetFromHexStr(hexStr, 3);
+        expected = { 0, 5, 5 };
+        assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
+    }
+
     {
         // Test string interpolation behaviour
         TIVarFile testPrgm = TIVarFile::createNew("Program", "INTERP");