From 7eb877505363d924b72f0b864ba1d0a7ff869ad3 Mon Sep 17 00:00:00 2001 From: Mingun Date: Mon, 8 Apr 2024 00:18:27 +0500 Subject: [PATCH] Fix incorrect encoding literals C++: add a method to translator to get the raw string literal instead of std::string object Fixes the following KST tests: - str_encodings_escaping_enc - str_encodings_escaping_to_s --- .../kaitai/struct/exprlang/ExpressionsSpec.scala | 15 +++++++++++++++ .../struct/translators/CSharpTranslator.scala | 2 +- .../kaitai/struct/translators/CppTranslator.scala | 4 +++- .../struct/translators/JavaScriptTranslator.scala | 2 +- .../struct/translators/JavaTranslator.scala | 2 +- .../kaitai/struct/translators/LuaTranslator.scala | 2 +- .../kaitai/struct/translators/NimTranslator.scala | 2 +- .../kaitai/struct/translators/PHPTranslator.scala | 2 +- .../struct/translators/PerlTranslator.scala | 2 +- .../struct/translators/PythonTranslator.scala | 2 +- .../struct/translators/RubyTranslator.scala | 2 +- .../struct/translators/RustTranslator.scala | 2 +- 12 files changed, 28 insertions(+), 11 deletions(-) diff --git a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala index 488c81861..309630fac 100644 --- a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala +++ b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala @@ -404,6 +404,21 @@ class ExpressionsSpec extends AnyFunSpec { Expressions.parse("foo.bar") should be (Attribute(Name(identifier("foo")),identifier("bar"))) } + describe("strings") { + it("single-quoted") { + // \" -> \" + // \\ -> \\ + Expressions.parse(""" ' \" \\ ' """) should be(Str(" \\\" \\\\ ")) + Expressions.parse(""" 'ASCII\\x' """) should be(Str("ASCII\\\\x")) + } + it("double-quoted") { + // \" -> " + // \\ -> \ + Expressions.parse(""" " \" \\ " """) should be(Str(" \" \\ ")) + Expressions.parse(""" "ASCII\\'x" """) should be(Str("ASCII\\'x")) + } + } + describe("f-strings") { it("parses f-string with just a string") { Expressions.parse("f\"abc\"") should be(InterpolatedStr(Seq( diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala index a105a1b00..5eda64bdc 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala @@ -103,7 +103,7 @@ class CSharpTranslator(provider: TypeProvider, importList: ImportList) extends B override def intToStr(i: expr): String = s"${translate(i, METHOD_PRECEDENCE)}.ToString()" override def bytesToStr(bytesExpr: String, encoding: String): String = - s"""System.Text.Encoding.GetEncoding("$encoding").GetString($bytesExpr)""" + s"""System.Text.Encoding.GetEncoding(${doStringLiteral(encoding)}).GetString($bytesExpr)""" override def strLength(s: expr): String = s"${translate(s, METHOD_PRECEDENCE)}.Length" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala index bedc1f052..197192cf8 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala @@ -60,6 +60,8 @@ class CppTranslator(provider: TypeProvider, importListSrc: CppImportList, import } } + def doRawStringLiteral(s: String): String = super.doStringLiteral(s) + /** * Handles string literal for C++ by wrapping a C `const char*`-style string * into a std::string constructor. Note that normally std::string @@ -188,7 +190,7 @@ class CppTranslator(provider: TypeProvider, importListSrc: CppImportList, import //s"std::to_string(${translate(i)})" s"${CppCompiler.kstreamName}::to_string(${translate(i)})" override def bytesToStr(bytesExpr: String, encoding: String): String = - s"""${CppCompiler.kstreamName}::bytes_to_str($bytesExpr, "$encoding")""" + s"""${CppCompiler.kstreamName}::bytes_to_str($bytesExpr, ${doRawStringLiteral(encoding)})""" override def bytesLength(b: Ast.expr): String = s"${translate(b, METHOD_PRECEDENCE)}.length()" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala index b26f47bab..f32c68cd6 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala @@ -114,7 +114,7 @@ class JavaScriptTranslator(provider: TypeProvider, importList: ImportList) exten s"(${translate(i)}).toString()" override def bytesToStr(bytesExpr: String, encoding: String): String = - s"""${JavaScriptCompiler.kstreamName}.bytesToStr($bytesExpr, "$encoding")""" + s"""${JavaScriptCompiler.kstreamName}.bytesToStr($bytesExpr, ${doStringLiteral(encoding)})""" override def strLength(s: expr): String = s"${translate(s, METHOD_PRECEDENCE)}.length" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index d303d3b9b..3eec364db 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -132,7 +132,7 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas s"StandardCharsets.${charsetConst}" case None => importList.add("java.nio.charset.Charset") - s"""Charset.forName("$encoding")""" + s"""Charset.forName(${doStringLiteral(encoding)})""" } s"new String($bytesExpr, $charsetExpr)" } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala index 764bb1dae..bf07bac90 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala @@ -108,7 +108,7 @@ class LuaTranslator(provider: TypeProvider, importList: ImportList) extends Base override def bytesToStr(bytesExpr: String, encoding: String): String = { importList.add("local str_decode = require(\"string_decode\")") - s"""str_decode.decode($bytesExpr, "$encoding")""" + s"""str_decode.decode($bytesExpr, ${doStringLiteral(encoding)})""" } override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = { s"string.byte(${translate(container)}, ${translate(idx)} + 1)" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala index 7feaee04c..f82e00eb7 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala @@ -12,7 +12,7 @@ import io.kaitai.struct.languages.NimCompiler.{ksToNim, namespaced, camelCase} class NimTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { // Members declared in io.kaitai.struct.translators.BaseTranslator override def bytesToStr(bytesExpr: String, encoding: String): String = { - s"""encode($bytesExpr, "$encoding")""" + s"""encode($bytesExpr, ${doStringLiteral(encoding)})""" } override def doEnumById(enumSpec: EnumSpec, id: String): String = s"${namespaced(enumSpec.name)}($id)" // override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = s"${namespaced(enumSpec.name)}($label)" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala index e3b33e0bd..43e58a833 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala @@ -103,7 +103,7 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT s"strval(${translate(i)})" override def bytesToStr(bytesExpr: String, encoding: String): String = - s"""${PHPCompiler.kstreamName}::bytesToStr($bytesExpr, "$encoding")""" + s"""${PHPCompiler.kstreamName}::bytesToStr($bytesExpr, ${doStringLiteral(encoding)})""" override def bytesLength(b: Ast.expr): String = s"strlen(${translate(b)})" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala index c0ef2c1a1..1b7013c61 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala @@ -142,7 +142,7 @@ class PerlTranslator(provider: TypeProvider, importList: ImportList) extends Bas s"sprintf('%d', ${translate(i)})" override def bytesToStr(bytesExpr: String, encoding: String): String = { importList.add("Encode") - s"""Encode::decode("$encoding", $bytesExpr)""" + s"""Encode::decode(${doStringLiteral(encoding)}, $bytesExpr)""" } override def bytesLength(b: Ast.expr): String = strLength(b) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala index 8bdd599c7..d1e72ab1b 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala @@ -99,7 +99,7 @@ class PythonTranslator(provider: TypeProvider, importList: ImportList, config: R override def intToStr(i: Ast.expr): String = s"str(${translate(i)})" override def bytesToStr(bytesExpr: String, encoding: String): String = - s"""($bytesExpr).decode("$encoding")""" + s"""($bytesExpr).decode(${doStringLiteral(encoding)})""" override def bytesLength(value: Ast.expr): String = s"len(${translate(value)})" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala index 40b012c96..5e1cfde4e 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala @@ -90,7 +90,7 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) override def bytesToStr(bytesExpr: String, encoding: String): String = { // We can skip "encode to UTF8" if we're 100% sure that the string we're handling is already // in UTF8. - s"""($bytesExpr).force_encoding("$encoding")""" + (if (encoding != "UTF-8") { + s"""($bytesExpr).force_encoding(${doStringLiteral(encoding)})""" + (if (encoding != "UTF-8") { ".encode('UTF-8')" } else { "" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index af1c5efd6..6f77cc4ce 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -81,7 +81,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base case "ASCII" => s"String::from_utf8_lossy($bytesExpr)" case _ => - "panic!(\"Unimplemented encoding for bytesToStr: {}\", \"" + encoding + "\")" + s"panic!(\"Unimplemented encoding for bytesToStr: {}\", \"${doStringLiteral(encoding)}\")" } override def bytesLength(b: Ast.expr): String = s"${translate(b, METHOD_PRECEDENCE)}.len()"