Added basic interpolated string (f-string) translation into target la…

…nguages, mostly working off concatenation and existing other-types-to-string conversions. Supports only integers and strings now. Added basic unit tests. * GoTranslator: given non-string nature, added custom implementation using `fmt.Sprintf` * CommonLiterals: split generation of string "body" (without quotes) and adding quotes to it * CommonMethods: param name cleanup
kaitai-io · Oct 14, 2023 · 97ffceb · 97ffceb
1 parent 1c7c759
commit 97ffceb
Show file tree

Hide file tree

Showing 8 changed files with 97 additions and 13 deletions.
diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -3,14 +3,13 @@ package io.kaitai.struct.translators
 import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.{Ast, Expressions}
-import io.kaitai.struct.format.{ClassSpec, FixedSized}
+import io.kaitai.struct.format.{ClassSpec, FixedSized, Identifier}
 import io.kaitai.struct.languages._
 import io.kaitai.struct.languages.components.{CppImportList, LanguageCompilerStatic}
 import io.kaitai.struct.{ImportList, RuntimeConfig, StringLanguageOutputWriter}
 import org.scalatest.Tag
 import org.scalatest.funsuite.AnyFunSuite
 import org.scalatest.matchers.should.Matchers._
-import io.kaitai.struct.format.Identifier
 
 class TranslatorSpec extends AnyFunSuite {
 
@@ -673,6 +672,25 @@ class TranslatorSpec extends AnyFunSuite {
   // sizeof of fixed user type
   everybody("bitsizeof<block>", "56", CalcIntType)
 
+  // f-strings
+  everybodyExcept("f\"abc\"", "\"abc\"", Map(
+    CppCompiler -> "std::string(\"abc\")",
+    PythonCompiler -> "u\"abc\""
+  ), CalcStrType)
+
+  full("f\"abc{1}def\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
+    CppCompiler -> "std::string(\"abc\") + kaitai::kstream::to_string(1) + std::string(\"def\")",
+    CSharpCompiler -> "\"abc\" + Convert.ToString((long) (1), 10) + \"def\"",
+    GoCompiler -> "fmt.Sprintf(\"abc%ddef\", 1)",
+    JavaCompiler -> "\"abc\" + Long.toString(1, 10) + \"def\"",
+    JavaScriptCompiler -> "\"abc\" + (1).toString(10) + \"def\"",
+    LuaCompiler -> "\"abc\" + tostring(1) + \"def\"",
+    PerlCompiler -> "\"abc\" . sprintf('%d', 1) . \"def\"",
+    PHPCompiler -> "\"abc\" . strval(1) . \"def\"",
+    PythonCompiler -> "u\"abc\" + str(1) + u\"def\"",
+    RubyCompiler -> "\"abc\" + 1.to_s(10) + \"def\"",
+  ))
+
   /**
     * Checks translation of expression `src` into target languages
     *

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -54,6 +54,8 @@ abstract class BaseTranslator(val provider: TypeProvider)
         doFloatLiteral(n)
       case Ast.expr.Str(s) =>
         doStringLiteral(s)
+      case Ast.expr.InterpolatedStr(s) =>
+        doInterpolatedStringLiteral(s)
       case Ast.expr.Bool(n) =>
         doBoolLiteral(n)
       case Ast.expr.EnumById(enumType, id, inType) =>
@@ -204,4 +206,19 @@ abstract class BaseTranslator(val provider: TypeProvider)
   // for the language
   def anyField(value: Ast.expr, attrName: String): String =
     s"${translate(value)}.${doName(attrName)}"
+
+  // f-strings
+  def doInterpolatedStringLiteral(exprs: Seq[Ast.expr]): String =
+    exprs.map(anyToStr).mkString(" + ")
+
+  def anyToStr(value: Ast.expr): String = {
+    detectType(value) match {
+      case _: IntType =>
+        intToStr(value, Ast.expr.IntNum(10))
+      case _: StrType =>
+        translate(value)
+      case otherType =>
+        throw new UnsupportedOperationException(s"unable to convert $otherType to string in format string")
+    }
+  }
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonLiterals.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonLiterals.scala
@@ -8,16 +8,27 @@ trait CommonLiterals {
   def doIntLiteral(n: BigInt): String = n.toString
   def doFloatLiteral(n: Any): String = n.toString
 
-  def doStringLiteral(s: String): String = {
-    val encoded = s.toCharArray.map((code) =>
-      if (code <= 0xff) {
-        strLiteralAsciiChar(code)
-      } else {
-        strLiteralUnicode(code)
-      }
-    ).mkString
-    "\"" + encoded + "\""
-  }
+  /**
+   * Generates string literal with double quotes framing it.
+   * @param s string to put in as literal
+   * @return string literal
+   */
+  def doStringLiteral(s: String): String =
+    "\"" + doStringLiteralBody(s) + "\""
+
+  /**
+   * Generates body of string literal for a given string, without framing quotes.
+   * @param s string to put in as literal
+   * @return body of a string literal
+   */
+  def doStringLiteralBody(s: String): String = s.toCharArray.map((code) =>
+    if (code <= 0xff) {
+      strLiteralAsciiChar(code)
+    } else {
+      strLiteralUnicode(code)
+    }
+  ).mkString
+
   def doBoolLiteral(n: Boolean): String = n.toString
 
   /**

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala
@@ -128,7 +128,7 @@ abstract trait CommonMethods[T] extends TypeDetector {
 
   def bytesToStr(value: Ast.expr, encoding: String): T
 
-  def intToStr(value: Ast.expr, num: Ast.expr): T
+  def intToStr(value: Ast.expr, base: Ast.expr): T
 
   def floatToInt(value: Ast.expr): T
 

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -43,6 +43,8 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
         trFloatLiteral(n)
       case Ast.expr.Str(s) =>
         trStringLiteral(s)
+      case Ast.expr.InterpolatedStr(s) =>
+        trInterpolatedStringLiteral(s)
       case Ast.expr.Bool(n) =>
         trBoolLiteral(n)
       case Ast.expr.EnumById(enumType, id, inType) =>
@@ -469,6 +471,35 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     ResultLocalVar(v)
   }
 
+  def trInterpolatedStringLiteral(exprs: Seq[Ast.expr]): TranslatorResult = {
+
+    exprs match {
+      case Seq(Ast.expr.Str(s)) =>
+        // exactly one string literal, no need for printf at all
+        trStringLiteral(s)
+
+      case _ =>
+        importList.add("fmt")
+
+        val piecesAndArgs: Seq[(String, Option[String])] = exprs.map {
+          case Ast.expr.Str(s) => (doStringLiteralBody(s), None)
+          case e =>
+            detectType(e) match {
+              case _: IntType => ("%d", Some(translate(e)))
+              case _: StrType => ("%s", Some(translate(e)))
+              case _: BooleanType => ("%b", Some(translate(e)))
+              case otherType =>
+                throw new UnsupportedOperationException(s"unable to convert $otherType to string in format string")
+            }
+        }
+
+        val fmtString = piecesAndArgs.map(x => x._1).mkString
+        val fmtArgs = piecesAndArgs.flatMap(x => x._2)
+
+        ResultString("fmt.Sprintf(\"" + fmtString + "\", " + fmtArgs.mkString(", ") + ")")
+    }
+  }
+
   def outVarCheckRes(expr: String): ResultLocalVar = {
     val v1 = allocateLocalVar()
     out.puts(s"${localVarName(v1)}, err := $expr")

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
@@ -146,6 +146,9 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT
   override def arrayMax(a: Ast.expr): String =
     s"max(${translate(a)})"
 
+  override def doInterpolatedStringLiteral(exprs: Seq[Ast.expr]): String =
+    exprs.map(anyToStr).mkString(" . ")
+
   val namespaceRef = if (config.phpNamespace.isEmpty) {
     ""
   } else {

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
@@ -203,4 +203,7 @@ class PerlTranslator(provider: TypeProvider, importList: ImportList) extends Bas
 
   override def kaitaiStreamSize(value: Ast.expr): String =
     s"${translate(value)}->size()"
+
+  override def doInterpolatedStringLiteral(exprs: Seq[Ast.expr]): String =
+    exprs.map(anyToStr).mkString(" . ")
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
@@ -48,6 +48,7 @@ class TypeDetector(provider: TypeProvider) {
         }
       case Ast.expr.FloatNum(_) => CalcFloatType
       case Ast.expr.Str(_) => CalcStrType
+      case Ast.expr.InterpolatedStr(_) => CalcStrType
       case Ast.expr.Bool(_) => CalcBooleanType
       case Ast.expr.EnumByLabel(enumType, _, inType) =>
         val t = EnumType(List(enumType.name), CalcIntType)