Fix incorrect encoding literals

C++: add a method to translator to get the raw string literal instead of std::string object Fixes the following KST tests: - str_encodings_escaping_enc - str_encodings_escaping_to_s
kaitai-io · Jul 9, 2024 · 7eb8775 · 7eb8775
1 parent 27d5020
commit 7eb8775
Show file tree

Hide file tree

Showing 12 changed files with 28 additions and 11 deletions.
diff --git a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
@@ -404,6 +404,21 @@ class ExpressionsSpec extends AnyFunSpec {
       Expressions.parse("foo.bar") should be (Attribute(Name(identifier("foo")),identifier("bar")))
     }
 
+    describe("strings") {
+      it("single-quoted") {
+        // \" -> \"
+        // \\ -> \\
+        Expressions.parse(""" ' \" \\ ' """) should be(Str(" \\\" \\\\ "))
+        Expressions.parse(""" 'ASCII\\x' """) should be(Str("ASCII\\\\x"))
+      }
+      it("double-quoted") {
+        // \" -> "
+        // \\ -> \
+        Expressions.parse(""" " \" \\ " """) should be(Str(" \" \\ "))
+        Expressions.parse(""" "ASCII\\'x" """) should be(Str("ASCII\\'x"))
+      }
+    }
+
     describe("f-strings") {
       it("parses f-string with just a string") {
         Expressions.parse("f\"abc\"") should be(InterpolatedStr(Seq(

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
@@ -103,7 +103,7 @@ class CSharpTranslator(provider: TypeProvider, importList: ImportList) extends B
   override def intToStr(i: expr): String =
     s"${translate(i, METHOD_PRECEDENCE)}.ToString()"
   override def bytesToStr(bytesExpr: String, encoding: String): String =
-    s"""System.Text.Encoding.GetEncoding("$encoding").GetString($bytesExpr)"""
+    s"""System.Text.Encoding.GetEncoding(${doStringLiteral(encoding)}).GetString($bytesExpr)"""
   override def strLength(s: expr): String =
     s"${translate(s, METHOD_PRECEDENCE)}.Length"
 

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -60,6 +60,8 @@ class CppTranslator(provider: TypeProvider, importListSrc: CppImportList, import
     }
   }
 
+  def doRawStringLiteral(s: String): String = super.doStringLiteral(s)
+
   /**
     * Handles string literal for C++ by wrapping a C `const char*`-style string
     * into a std::string constructor. Note that normally std::string
@@ -188,7 +190,7 @@ class CppTranslator(provider: TypeProvider, importListSrc: CppImportList, import
     //s"std::to_string(${translate(i)})"
     s"${CppCompiler.kstreamName}::to_string(${translate(i)})"
   override def bytesToStr(bytesExpr: String, encoding: String): String =
-    s"""${CppCompiler.kstreamName}::bytes_to_str($bytesExpr, "$encoding")"""
+    s"""${CppCompiler.kstreamName}::bytes_to_str($bytesExpr, ${doRawStringLiteral(encoding)})"""
   override def bytesLength(b: Ast.expr): String =
     s"${translate(b, METHOD_PRECEDENCE)}.length()"
 

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
@@ -114,7 +114,7 @@ class JavaScriptTranslator(provider: TypeProvider, importList: ImportList) exten
     s"(${translate(i)}).toString()"
 
   override def bytesToStr(bytesExpr: String, encoding: String): String =
-    s"""${JavaScriptCompiler.kstreamName}.bytesToStr($bytesExpr, "$encoding")"""
+    s"""${JavaScriptCompiler.kstreamName}.bytesToStr($bytesExpr, ${doStringLiteral(encoding)})"""
 
   override def strLength(s: expr): String =
     s"${translate(s, METHOD_PRECEDENCE)}.length"

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -132,7 +132,7 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas
         s"StandardCharsets.${charsetConst}"
       case None =>
         importList.add("java.nio.charset.Charset")
-        s"""Charset.forName("$encoding")"""
+        s"""Charset.forName(${doStringLiteral(encoding)})"""
     }
     s"new String($bytesExpr, $charsetExpr)"
   }

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala
@@ -108,7 +108,7 @@ class LuaTranslator(provider: TypeProvider, importList: ImportList) extends Base
   override def bytesToStr(bytesExpr: String, encoding: String): String = {
     importList.add("local str_decode = require(\"string_decode\")")
 
-    s"""str_decode.decode($bytesExpr, "$encoding")"""
+    s"""str_decode.decode($bytesExpr, ${doStringLiteral(encoding)})"""
   }
   override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = {
     s"string.byte(${translate(container)}, ${translate(idx)} + 1)"

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala
@@ -12,7 +12,7 @@ import io.kaitai.struct.languages.NimCompiler.{ksToNim, namespaced, camelCase}
 class NimTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) {
   // Members declared in io.kaitai.struct.translators.BaseTranslator
   override def bytesToStr(bytesExpr: String, encoding: String): String = {
-    s"""encode($bytesExpr, "$encoding")"""
+    s"""encode($bytesExpr, ${doStringLiteral(encoding)})"""
   }
   override def doEnumById(enumSpec: EnumSpec, id: String): String = s"${namespaced(enumSpec.name)}($id)"
 //  override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = s"${namespaced(enumSpec.name)}($label)"

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
@@ -103,7 +103,7 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT
     s"strval(${translate(i)})"
 
   override def bytesToStr(bytesExpr: String, encoding: String): String =
-    s"""${PHPCompiler.kstreamName}::bytesToStr($bytesExpr, "$encoding")"""
+    s"""${PHPCompiler.kstreamName}::bytesToStr($bytesExpr, ${doStringLiteral(encoding)})"""
 
   override def bytesLength(b: Ast.expr): String =
     s"strlen(${translate(b)})"

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
@@ -142,7 +142,7 @@ class PerlTranslator(provider: TypeProvider, importList: ImportList) extends Bas
     s"sprintf('%d', ${translate(i)})"
   override def bytesToStr(bytesExpr: String, encoding: String): String = {
     importList.add("Encode")
-    s"""Encode::decode("$encoding", $bytesExpr)"""
+    s"""Encode::decode(${doStringLiteral(encoding)}, $bytesExpr)"""
   }
   override def bytesLength(b: Ast.expr): String =
     strLength(b)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
@@ -99,7 +99,7 @@ class PythonTranslator(provider: TypeProvider, importList: ImportList, config: R
   override def intToStr(i: Ast.expr): String =
     s"str(${translate(i)})"
   override def bytesToStr(bytesExpr: String, encoding: String): String =
-    s"""($bytesExpr).decode("$encoding")"""
+    s"""($bytesExpr).decode(${doStringLiteral(encoding)})"""
 
   override def bytesLength(value: Ast.expr): String =
     s"len(${translate(value)})"

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
@@ -90,7 +90,7 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider)
   override def bytesToStr(bytesExpr: String, encoding: String): String = {
     // We can skip "encode to UTF8" if we're 100% sure that the string we're handling is already
     // in UTF8.
-    s"""($bytesExpr).force_encoding("$encoding")""" + (if (encoding != "UTF-8") {
+    s"""($bytesExpr).force_encoding(${doStringLiteral(encoding)})""" + (if (encoding != "UTF-8") {
       ".encode('UTF-8')"
     } else {
       ""

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -81,7 +81,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
       case "ASCII" =>
         s"String::from_utf8_lossy($bytesExpr)"
       case _ =>
-        "panic!(\"Unimplemented encoding for bytesToStr: {}\", \"" + encoding + "\")"
+        s"panic!(\"Unimplemented encoding for bytesToStr: {}\", \"${doStringLiteral(encoding)}\")"
     }
   override def bytesLength(b: Ast.expr): String =
     s"${translate(b, METHOD_PRECEDENCE)}.len()"