Squashed commit of the following:

commit 1dbafe3
Author: Mikhail Yakshin <[email protected]>
Date:   Thu Feb 29 15:08:24 2024 +0000

    ExpressionsSpec: f-strings: added test with newline in the middle, fixed name of test with double quote in the middle

commit 8605f3f
Author: Mikhail Yakshin <[email protected]>
Date:   Thu Feb 29 15:04:40 2024 +0000

    Lexical: moved fstringItem + fstringChar from Expressions; Expressions: fixed rep -> repX to avoid whitespace problem

commit 17f9e40
Author: Mikhail Yakshin <[email protected]>
Date:   Sat Oct 14 21:07:58 2023 +0100

    Added similar escaped quote+space test for double-quoted string

commit c0083fb
Author: Mikhail Yakshin <[email protected]>
Date:   Sat Oct 14 20:48:19 2023 +0100

    Added tests suggested in PR review: quote in f-string, regular string in f-string, f-string in f-string

commit 1b22258
Author: Mikhail Yakshin <[email protected]>
Date:   Sat Oct 14 20:38:19 2023 +0100

    Apply suggestions from code review

    Co-authored-by: Petr Pučil <[email protected]>

commit 97ffceb
Author: Mikhail Yakshin <[email protected]>
Date:   Sat Oct 14 08:50:41 2023 +0100

    Added basic interpolated string (f-string) translation into target languages, mostly working off concatenation and existing other-types-to-string conversions. Supports only integers and strings now. Added basic unit tests.

    * GoTranslator: given non-string nature, added custom implementation using `fmt.Sprintf`
    * CommonLiterals: split generation of string "body" (without quotes) and adding quotes to it
    * CommonMethods: param name cleanup

commit 1c7c759
Author: Mikhail Yakshin <[email protected]>
Date:   Sat Oct 14 08:23:12 2023 +0100

    Implemented basic f-string parsing in expression language

jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala

@@ -261,6 +261,10 @@ class ExpressionsSpec extends AnyFunSpec {
       Expressions.parse("\"abc\\u21bbdef\"") should be (Str("abc\u21bbdef"))
     }
 
+    it("parses double-quoted string with double quote") {
+      Expressions.parse("\"this \\\" is a quote\"") should be(Str("this \" is a quote"))
+    }
+
     // Casts
     it("parses 123.as<u4>") {
       Expressions.parse("123.as<u4>") should be (
@@ -388,5 +392,67 @@ class ExpressionsSpec extends AnyFunSpec {
     it("parses foo.bar") {
       Expressions.parse("foo.bar") should be (Attribute(Name(identifier("foo")),identifier("bar")))
     }
+
+    describe("f-strings") {
+      it("parses f-string with just a string") {
+        Expressions.parse("f\"abc\"") should be(InterpolatedStr(ArrayBuffer(
+          Str("abc")
+        )))
+      }
+
+      it("parses f-string with just one expression") {
+        Expressions.parse("f\"{123}\"") should be(InterpolatedStr(ArrayBuffer(
+          IntNum(123)
+        )))
+      }
+
+      it("parses f-string with string + expression") {
+        Expressions.parse("f\"foo={123}\"") should be(InterpolatedStr(ArrayBuffer(
+          Str("foo="),
+          IntNum(123)
+        )))
+      }
+
+      it("parses f-string with expression + string") {
+        Expressions.parse("f\"{123}=abc\"") should be(InterpolatedStr(ArrayBuffer(
+          IntNum(123),
+          Str("=abc")
+        )))
+      }
+
+      it("parses f-string with str + expression + str") {
+        Expressions.parse("f\"abc={123}=def\"") should be(InterpolatedStr(ArrayBuffer(
+          Str("abc="),
+          IntNum(123),
+          Str("=def")
+        )))
+      }
+
+      it("parses f-string string with newline in the middle") {
+        Expressions.parse("f\"abc\\ndef\"") should be(InterpolatedStr(ArrayBuffer(Str("abc\ndef"))))
+      }
+
+      it("parses f-string with double quote in the middle") {
+        Expressions.parse("f\"this \\\" is a quote\"") should be(InterpolatedStr(ArrayBuffer(
+          Str("this \" is a quote")
+        )))
+      }
+
+      it("parses f-string with string in it") {
+        Expressions.parse("f\"abc{\"def\"}ghi\"") should be(InterpolatedStr(ArrayBuffer(
+          Str("abc"),
+          Str("def"),
+          Str("ghi"),
+        )))
+      }
+
+      it("parses f-string with f-string in it") {
+        Expressions.parse("f\"abc{f\"def\"}ghi\"") should be(InterpolatedStr(ArrayBuffer(
+          Str("abc"),
+          InterpolatedStr(ArrayBuffer(Str("def"))),
+          Str("ghi"),
+        )))
+      }
+    }
   }
 }

jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala

@@ -3,14 +3,13 @@ package io.kaitai.struct.translators
 import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.{Ast, Expressions}
-import io.kaitai.struct.format.{ClassSpec, FixedSized}
+import io.kaitai.struct.format.{ClassSpec, FixedSized, Identifier}
 import io.kaitai.struct.languages._
 import io.kaitai.struct.languages.components.{CppImportList, LanguageCompilerStatic}
 import io.kaitai.struct.{ImportList, RuntimeConfig, StringLanguageOutputWriter}
 import org.scalatest.Tag
 import org.scalatest.funsuite.AnyFunSuite
 import org.scalatest.matchers.should.Matchers._
-import io.kaitai.struct.format.Identifier
 
 class TranslatorSpec extends AnyFunSuite {
 
@@ -677,6 +676,25 @@ class TranslatorSpec extends AnyFunSuite {
   // sizeof of fixed user type
   everybody("bitsizeof<block>", "56", CalcIntType)
 
+  // f-strings
+  everybodyExcept("f\"abc\"", "\"abc\"", Map(
+    CppCompiler -> "std::string(\"abc\")",
+    PythonCompiler -> "u\"abc\""
+  ), CalcStrType)
+
+  full("f\"abc{1}def\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
+    CppCompiler -> "std::string(\"abc\") + kaitai::kstream::to_string(1) + std::string(\"def\")",
+    CSharpCompiler -> "\"abc\" + Convert.ToString((long) (1), 10) + \"def\"",
+    GoCompiler -> "fmt.Sprintf(\"abc%ddef\", 1)",
+    JavaCompiler -> "\"abc\" + Long.toString(1, 10) + \"def\"",
+    JavaScriptCompiler -> "\"abc\" + (1).toString(10) + \"def\"",
+    LuaCompiler -> "\"abc\" + tostring(1) + \"def\"",
+    PerlCompiler -> "\"abc\" . sprintf('%d', 1) . \"def\"",
+    PHPCompiler -> "\"abc\" . strval(1) . \"def\"",
+    PythonCompiler -> "u\"abc\" + str(1) + u\"def\"",
+    RubyCompiler -> "\"abc\" + 1.to_s(10) + \"def\"",
+  ))
+
   /**
     * Checks translation of expression `src` into target languages
     *

shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala

@@ -89,6 +89,7 @@ object Ast {
     /** For internal use in the compiler. It cannot appear in an AST parsed from a user-supplied string. */
     case class InternalName(id: Identifier) extends expr
     case class List(elts: Seq[expr]) extends expr
+    case class InterpolatedStr(elts: Seq[expr]) extends expr
   }
 
   sealed trait boolop

shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala

@@ -35,6 +35,15 @@ object Expressions {
   def FLOAT_NUMBER[_: P] = Lexical.floatnumber
   def STRING[_: P]: P[String] = Lexical.stringliteral
 
+  def fstring[_: P]: P[Ast.expr.InterpolatedStr] = P("f\"" ~/ fstringElement.rep ~ "\"").map(Ast.expr.InterpolatedStr)
+  def fstringElement[_: P]: P[Ast.expr] = P(
+    formatExpr |
+      Lexical.fstringItem.repX(min = 1).
+        map(_.mkString).
+        map(Ast.expr.Str)
+  )
+  def formatExpr[_: P]: P[Ast.expr] = P("{" ~/ test ~ "}")
+
   def test[_: P]: P[Ast.expr] = P( or_test ~ ("?" ~ test ~ ":" ~ test).? ).map {
     case (x, None) => x
     case (condition, Some((ifTrue, ifFalse))) => Ast.expr.IfExp(condition, ifTrue, ifFalse)
@@ -119,6 +128,7 @@ object Expressions {
     enumByName |
     byteSizeOfType |
     bitSizeOfType |
+    fstring |
     STRING.rep(1).map(_.mkString).map(Ast.expr.Str) |
     NAME.map((x) => x.name match {
       case "true" => Ast.expr.Bool(true)

shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala

@@ -43,6 +43,10 @@ object Lexical {
   def doublestring[_: P]: P[String] = P("\"" ~/ doublestringitem.rep ~ "\"").map(_.mkString)
   def doublestringitem[_: P] = P( doublestringchar.! | escapeseq )
   def doublestringchar[_: P] = P( CharsWhile(!"\\\"".contains(_)) )
+
+  def fstringItem[_: P] = P(fstringChar.! | Lexical.escapeseq)
+  def fstringChar[_: P] = P(CharsWhile(!"{\\\"".contains(_)))
+
   def escapeseq[_: P] = P( "\\" ~/ (quotedchar | quotedoctal | quotedhex) )
 
   val QUOTED_CC = Map(

shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala

@@ -54,6 +54,8 @@ abstract class BaseTranslator(val provider: TypeProvider)
         doFloatLiteral(n)
       case Ast.expr.Str(s) =>
         doStringLiteral(s)
+      case Ast.expr.InterpolatedStr(s) =>
+        doInterpolatedStringLiteral(s)
       case Ast.expr.Bool(n) =>
         doBoolLiteral(n)
       case Ast.expr.EnumById(enumType, id, inType) =>
@@ -204,4 +206,19 @@ abstract class BaseTranslator(val provider: TypeProvider)
   // for the language
   def anyField(value: Ast.expr, attrName: String): String =
     s"${translate(value)}.${doName(attrName)}"
+
+  // f-strings
+  def doInterpolatedStringLiteral(exprs: Seq[Ast.expr]): String =
+    exprs.map(anyToStr).mkString(" + ")
+
+  def anyToStr(value: Ast.expr): String = {
+    detectType(value) match {
+      case _: IntType =>
+        intToStr(value, Ast.expr.IntNum(10))
+      case _: StrType =>
+        translate(value)
+      case otherType =>
+        throw new UnsupportedOperationException(s"unable to convert $otherType to string in format string (only integers and strings are supported)")
+    }
+  }
 }

shared/src/main/scala/io/kaitai/struct/translators/CommonLiterals.scala

@@ -8,16 +8,27 @@ trait CommonLiterals {
   def doIntLiteral(n: BigInt): String = n.toString
   def doFloatLiteral(n: Any): String = n.toString
 
-  def doStringLiteral(s: String): String = {
-    val encoded = s.toCharArray.map((code) =>
-      if (code <= 0xff) {
-        strLiteralAsciiChar(code)
-      } else {
-        strLiteralUnicode(code)
-      }
-    ).mkString
-    "\"" + encoded + "\""
-  }
+  /**
+   * Generates string literal enclosed in double quotes.
+   * @param s string to put in as literal
+   * @return string literal
+   */
+  def doStringLiteral(s: String): String =
+    "\"" + doStringLiteralBody(s) + "\""
+
+  /**
+   * Generates body of string literal for a given string, without enclosing quotes.
+   * @param s string to put in as literal
+   * @return body of a string literal
+   */
+  def doStringLiteralBody(s: String): String = s.toCharArray.map((code) =>
+    if (code <= 0xff) {
+      strLiteralAsciiChar(code)
+    } else {
+      strLiteralUnicode(code)
+    }
+  ).mkString
+
   def doBoolLiteral(n: Boolean): String = n.toString
 
   /**

shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala

@@ -128,7 +128,7 @@ abstract trait CommonMethods[T] extends TypeDetector {
 
   def bytesToStr(value: Ast.expr, encoding: String): T
 
-  def intToStr(value: Ast.expr, num: Ast.expr): T
+  def intToStr(value: Ast.expr, base: Ast.expr): T
 
   def floatToInt(value: Ast.expr): T
 

shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala

@@ -43,6 +43,8 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
         trFloatLiteral(n)
       case Ast.expr.Str(s) =>
         trStringLiteral(s)
+      case Ast.expr.InterpolatedStr(s) =>
+        trInterpolatedStringLiteral(s)
       case Ast.expr.Bool(n) =>
         trBoolLiteral(n)
       case Ast.expr.EnumById(enumType, id, inType) =>
@@ -469,6 +471,34 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     ResultLocalVar(v)
   }
 
+  def trInterpolatedStringLiteral(exprs: Seq[Ast.expr]): TranslatorResult = {
+    exprs match {
+      case Seq(Ast.expr.Str(s)) =>
+        // exactly one string literal, no need for printf at all
+        trStringLiteral(s)
+
+      case _ =>
+        importList.add("fmt")
+
+        val piecesAndArgs: Seq[(String, Option[String])] = exprs.map {
+          case Ast.expr.Str(s) => (doStringLiteralBody(s), None)
+          case e =>
+            detectType(e) match {
+              case _: IntType => ("%d", Some(translate(e)))
+              case _: StrType => ("%s", Some(translate(e)))
+              case _: BooleanType => ("%b", Some(translate(e)))
+              case otherType =>
+                throw new UnsupportedOperationException(s"unable to convert $otherType to string in format string")
+            }
+        }
+
+        val fmtString = piecesAndArgs.map(x => x._1).mkString
+        val fmtArgs = piecesAndArgs.flatMap(x => x._2)
+
+        ResultString("fmt.Sprintf(\"" + fmtString + "\", " + fmtArgs.mkString(", ") + ")")
+    }
+  }
+
   def outVarCheckRes(expr: String): ResultLocalVar = {
     val v1 = allocateLocalVar()
     out.puts(s"${localVarName(v1)}, err := $expr")

shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala

@@ -146,6 +146,9 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT
   override def arrayMax(a: Ast.expr): String =
     s"max(${translate(a)})"
 
+  override def doInterpolatedStringLiteral(exprs: Seq[Ast.expr]): String =
+    exprs.map(anyToStr).mkString(" . ")
+
   val namespaceRef = if (config.phpNamespace.isEmpty) {
     ""
   } else {

shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala

@@ -203,4 +203,7 @@ class PerlTranslator(provider: TypeProvider, importList: ImportList) extends Bas
 
   override def kaitaiStreamSize(value: Ast.expr): String =
     s"${translate(value)}->size()"
+
+  override def doInterpolatedStringLiteral(exprs: Seq[Ast.expr]): String =
+    exprs.map(anyToStr).mkString(" . ")
 }

shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala

@@ -48,6 +48,7 @@ class TypeDetector(provider: TypeProvider) {
         }
       case Ast.expr.FloatNum(_) => CalcFloatType
       case Ast.expr.Str(_) => CalcStrType
+      case Ast.expr.InterpolatedStr(_) => CalcStrType
       case Ast.expr.Bool(_) => CalcBooleanType
       case Ast.expr.EnumByLabel(enumType, _, inType) =>
         val t = EnumType(List(enumType.name), CalcIntType)