From 7be5c1b5f5b8365303e17446e6606ff404471cd3 Mon Sep 17 00:00:00 2001 From: Adrien Bertrand Date: Fri, 31 May 2024 11:06:00 +0200 Subject: [PATCH] TH_Tokenized: use true lower-alpha tokens in strings. --- TIVarsLib.wasm | Bin 526736 -> 527129 bytes src/TypeHandlers/TH_Tokenized.cpp | 13 +++++++++++++ tests.cpp | 12 ++++++++++++ 3 files changed, 25 insertions(+) diff --git a/TIVarsLib.wasm b/TIVarsLib.wasm index 8607e2d5d01754b12db912aebf16275ef51e04e2..8ea1539a03a74e21896a012e8c0e71152cdf7716 100755 GIT binary patch delta 2627 zcmb7GO>9(E6u#%)_cNV0(|M&Wo%R(u@6ta)8yY38QlfJkTYd_%aAOP`F$SHwU?^!^ zXr}6durVY?Ux*vo7)&(LjdcNpg$ZFnBw|0Orb%_TUwnh(Xq(;&QfpoWgksvzxUD6X+L7m#qqE;Q@Y09 z><26Oyob@8*3YwXzuwZ@m6}!o=C|8(#U^{VN&_?9CE8$qogT9PksjmVW|fADCLUt# z)ZVl^^ZIFs{A_zaZKE(d)K5Dp$lmFvEotDmZZ%W<6bDXC>2UxJ#`L5mS7>-m4zlA% zy`E6khH!JUpC-ovcS^?`Fiq)F>r%*#e5T07jk&VIYTM$;0eTUe!AtL)ZZA%#9oD!t z)j$X<)NtW)QlHX;Fw#@Hr*)}vFrg-$5pSl(S+gF z^0*_LNoY@C*c;QF-Fw>%h1-;i7pKZ4B{sgW{MSd&~+P#V`O1Ud$N@6+e%w2X0EJ(a#ua= za|$snZ>4Qbuna>4A)XvSM2jg2JrBOz?Fm(7x3*>)R=YcD9vRl|F3A=!ZM?G8JC{x=U)&|xlx7OU*NU;M~;2%EP?f|2;*T$`KNIqyh!!*)- z^m7T|(NA*EW!=iPgt5_egy0b_S6p+ckp~1-P)Zm*7`Z97YNaQhZk(sJpLE)1ocA zmA_2G6_(RL*Rt1!X*vyI1Py@%krUqc7hz8)Z2`bSF6_@P4pVh^SjR%ps(yhC`0=O> z>e6;a1qBfxViqYt5zvElC?8_Vx<;tKi8O0gV_I4G59H;1WBjCD82`R~B7FlNb2zaE zu^WgSfuqgA1(#chw3ILK2;VI{A+~6t0Vu^yz(Hv75b7=doRlz!{2&TRdnvIY64Is- zcZF2?PY9sieuNQ%5l>)yjX6Llj49fFbr zAQ8|(wqmKL8Hi1A-$rk6_s}DM-QD_C%0^ls0(Au>DwSi+ERO*$1Hf4o6ON{T99;s&yQA0m|ueU!3zLO&zFCcRru%ChD=g3@2?Eh`J(C+Y$n# z#Rhl0;b_64_o)HK8kc}5pB<0vW~Wcq#iP_a>oNUGYbt2j2%q9Z&KBI`u?KyW_DZI? zqrsc!%Tx*heGJDS;C5S_N|)^>Nt>T=xHs>TawwqUBsXAaQ?_=I(nxr)0jMc?*^Wz; z9EA^Toi9X>+U+pTVr+WcXE2v<9Q@ynKf6Tf^n;DzCzQf7*jc2NFjN(fV<~ssMk6<9 zSK0PqT59PLhxFt1nEt08vph`a5gb4Qgt%BjM)HlvAqgSkLKH5MIB>}yK<4Y&^#dzN*;ViM^mKn! z_0?C^AKl<vZY!GdmukEO3$%xHmD2U@H-#L zIxwd@EZ5^>tPFLK%kM_PqRM|CrB`28I0yV0EluexoN`#65W_eVbW*bUljhOJ#)f}B z3NRg727>nEi4@6%A)C!Nmo0maWiPPDL1M_niskC{15_m>>YIa9_uY7n>g?sW57N08 zl@TX|-9%FYGre&$GSM^_Nw_-GNka`Jd9Q;fj?AB89?6Z&-(>$`rKLT~n+{BA-f01J5F;)_ z_<_-Oh0-A&G_;}zaqKUe@1`+$S-WYiTGu#}?IazrR3qC?*$rWcj9TKb4SE$s`txmR zR6Ge?YwAX{eH@grTC}H$2fOJY{$k(*qYno+{$r~!Rk)%aZ@zhn~fJ@OqQh3)xLl2M*;cNo97QkJpZzAB`7I^PnzHwY0*duM_q92+7Y^G>o z`R%9WXol`5O?-(JEJ6fR0vk4;zzWp6=^NgguWq813GE{aB$&!hXuc?xWCoU^%YfPR zi_DM>45avmmb&b}Zm4Z5Q`8x2TyEQ0hUvvn?>F*N$0EyZ9csLUvxIK za0l5aFBCfQyS#rsqQbKP!8Fje5y7xVBVE^fY`#>&-U2dGuz^0475M=RNLiLmU2?9D}ZVJ{GXBpk38PhzPsiYTzF@WL&0ZavxQ=6PH{ zgt(D}EZlb4e=?VMc&wh(V{{65jO_3B_^90NvV(;=q*jLUmxM3`LPtDBIFx_(6s@EI zn2d0N8C(DxzU^JOdkkIDK8V0*=jb`rG{B9(UayJc;= 'l' ? 1 : 0); + data.push_back(tokenValue >> 8); + data.push_back(tokenValue & 0xFF); + lastTokenBytes = tokenValue; + break; + } + if (tokens_NameToBytes.count(currentSubString)) { uint16_t tokenValue = tokens_NameToBytes[currentSubString]; diff --git a/tests.cpp b/tests.cpp index faf5bd0..56f8275 100644 --- a/tests.cpp +++ b/tests.cpp @@ -119,6 +119,18 @@ int main(int argc, char** argv) assert(memcmp(&actual, &expected, sizeof(actual)) == 0); } + { + // Test lower alpha being the expected lowercase tokens, not special-meaning ones + TIVarFile testPrgm = TIVarFile::createNew("Program", "INTERP"); + testPrgm.setContentFromString("Disp \"abcdefghijklmnopqrstuvwxyz\""); + string detok_fr = testPrgm.getReadableContent({{"lang", LANG_FR}}); + string detok_en = testPrgm.getReadableContent({{"lang", LANG_EN}}); + string hex = testPrgm.getRawContentHexStr(); + assert(detok_fr == "Disp \"abcdefghijklmnopqrstuvwxyz\""); + assert(detok_en == "Disp \"abcdefghijklmnopqrstuvwxyz\""); + assert(hex == "3700de2abbb0bbb1bbb2bbb3bbb4bbb5bbb6bbb7bbb8bbb9bbbabbbcbbbdbbbebbbfbbc0bbc1bbc2bbc3bbc4bbc5bbc6bbc7bbc8bbc9bbca2a"); + } + { // Test string interpolation behaviour TIVarFile testPrgm = TIVarFile::createNew("Program", "INTERP");