Introduce a Terminator type that holds all information about termin…

…ator sequence

jvm/src/test/scala/io/kaitai/struct/CalculateSeqSizes$Test.scala

@@ -1,7 +1,7 @@
 package io.kaitai.struct
 
 import io.kaitai.struct.datatype.{BigEndian, BigBitEndian}
-import io.kaitai.struct.datatype.DataType
+import io.kaitai.struct.datatype.{DataType, Terminator}
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.{Ast, Expressions}
 import io.kaitai.struct.format.{DynamicSized, FixedSized, MetaSpec, Sized, YamlAttrArgs}
@@ -39,12 +39,14 @@ class CalculateSeqSizes$Test extends AnyFunSpec {
       YamlAttrArgs(
         size.map(s => Ast.expr.IntNum(s)),
         false,// sizeEos
-        None, // encoding
-        terminator,
-        false,// include
-        false,// consume
-        false,// eosError
+        terminator.map(value => Terminator(
+          value,
+          false,// include
+          false,// consume
+          false,// eosError
+        )),
         None, // padRight
+        None, // encoding
         contents,
         None, // enumRef
         None, // parent
@@ -82,7 +84,7 @@ class CalculateSeqSizes$Test extends AnyFunSpec {
     CalculateSeqSizes.dataTypeBitsSize(SwitchType(
       Ast.expr.IntNum(0),
       cases.map { case (condition, typeName) =>
-        Expressions.parse(condition) -> parse(Some(typeName), None, None, None)
+        Expressions.parse(condition) -> parse(Some(typeName), None, Some(0), None)
       }
     ))
   }
@@ -164,8 +166,9 @@ class CalculateSeqSizes$Test extends AnyFunSpec {
 
       //-----------------------------------------------------------------------
 
-      sizeof("str", 0) should be (DynamicSized)
-      sizeof("strz"  ) should be (DynamicSized)
+      // We do not auto-calculate terminator in our test, so in both cases terminator is necessary
+      sizeof("str",  0) should be (DynamicSized)
+      sizeof("strz", 0) should be (DynamicSized)
 
       //TODO: Uncomment when https://github.com/kaitai-io/kaitai_struct/issues/799
       // will be implemented

shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala

@@ -127,15 +127,15 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
       s"Int${width.width * 8}${signToStr(signed)}${fixedEndianToStr(endianOpt.get)}"
     case FloatMultiType(width, endianOpt) =>
       s"Float${width.width * 8}${fixedEndianToStr(endianOpt.get)}"
-    case BytesEosType(terminator, include, padRight, process) =>
+    case BytesEosType(terminator, padRight, process) =>
       "GreedyBytes"
     case blt: BytesLimitType =>
       attrBytesLimitType(blt)
     case btt: BytesTerminatedType =>
       attrBytesTerminatedType(btt, "GreedyBytes")
     case StrFromBytesType(bytes, encoding, _) =>
       bytes match {
-        case BytesEosType(terminator, include, padRight, process) =>
+        case BytesEosType(terminator, padRight, process) =>
           s"GreedyString(encoding='$encoding')"
         case blt: BytesLimitType =>
           attrBytesLimitType(blt, s"GreedyString(encoding='$encoding')")
@@ -146,10 +146,10 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
       s"LazyBound(lambda: ${types2class(ut.classSpec.get.name)})"
     case utb: UserTypeFromBytes =>
       utb.bytes match {
-        //case BytesEosType(terminator, include, padRight, process) =>
-        case BytesLimitType(size, terminator, include, padRight, process) =>
+        //case BytesEosType(terminator, padRight, process) =>
+        case BytesLimitType(size, terminator, padRight, process) =>
           s"FixedSized(${translator.translate(size)}, LazyBound(lambda: ${types2class(utb.classSpec.get.name)}))"
-        //case BytesTerminatedType(terminator, include, consume, eosError, process) =>
+        //case BytesTerminatedType(terminator, process) =>
         case _ => "???"
       }
     case et: EnumType =>
@@ -162,9 +162,9 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
   def attrBytesLimitType(blt: BytesLimitType, subcon: String = "GreedyBytes"): String = {
     val subcon2 = blt.terminator match {
       case None => subcon
-      case Some(term) =>
+      case Some(Terminator(term, include, _, _)) =>
         val termStr = "\\x%02X".format(term & 0xff)
-        s"NullTerminated($subcon, term=b'$termStr', include=${translator.doBoolLiteral(blt.include)})"
+        s"NullTerminated($subcon, term=b'$termStr', include=${translator.doBoolLiteral(include)})"
     }
     val subcon3 = blt.padRight match {
       case None => subcon2
@@ -176,11 +176,11 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
   }
 
   def attrBytesTerminatedType(btt: BytesTerminatedType, subcon: String): String = {
-    val termStr = "\\x%02X".format(btt.terminator & 0xff)
+    val termStr = "\\x%02X".format(btt.terminator.value & 0xff)
     s"NullTerminated($subcon, " +
       s"term=b'$termStr', " +
-      s"include=${translator.doBoolLiteral(btt.include)}, " +
-      s"consume=${translator.doBoolLiteral(btt.consume)})"
+      s"include=${translator.doBoolLiteral(btt.terminator.include)}, " +
+      s"consume=${translator.doBoolLiteral(btt.terminator.consume)})"
   }
 
   def attrSwitchType(st: SwitchType): String = {

shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala

@@ -1,6 +1,6 @@
 package io.kaitai.struct
 
-import io.kaitai.struct.datatype.DataType
+import io.kaitai.struct.datatype.{DataType, Terminator}
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format._
@@ -413,7 +413,7 @@ object GraphvizClassCompiler extends LanguageCompilerStatic {
       case rt: ReadableType => rt.apiCall(None) // FIXME
       case ut: UserType => type2display(ut.name)
       //case FixedBytesType(contents, _) => contents.map(_.formatted("%02X")).mkString(" ")
-      case BytesTerminatedType(terminator, include, consume, eosError, _) =>
+      case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) =>
         val args = ListBuffer[String]()
         if (terminator != 0)
           args += s"term=$terminator"

shared/src/main/scala/io/kaitai/struct/datatype/Chunk.scala

@@ -0,0 +1,22 @@
+package io.kaitai.struct.datatype
+
+import io.kaitai.struct.exprlang.Ast
+
+/** Defines "working" sub-stream, as beginning part of full stream.
+  * All data after terminator byte is ignored and not available for parsing.
+  *
+  * @param value Byte at which stop reading stream
+  * @param include Specifies if terminator byte should be included in the final value
+  * @param consume Specify if terminator byte should be "consumed" when reading.
+  *        If `true`: the stream pointer will point to the byte after the terminator byte
+  *        If `false`: the stream pointer will point to the terminator byte itself
+  * @param mandatory If `true`, terminator must be present in the input stream, otherwise
+  *        reaching end of stream before encountering terminator also possible.
+  *        Corresponds to an `eos-error` key
+  */
+sealed case class Terminator(
+  value: Int,
+  include: Boolean,
+  consume: Boolean,
+  mandatory: Boolean,
+)

shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala

@@ -143,23 +143,18 @@ object DataType {
     override def process = None
   }
   case class BytesEosType(
-    terminator: Option[Int],
-    include: Boolean,
+    terminator: Option[Terminator],
     padRight: Option[Int],
     override val process: Option[ProcessExpr]
   ) extends BytesType
   case class BytesLimitType(
     size: Ast.expr,
-    terminator: Option[Int],
-    include: Boolean,
+    terminator: Option[Terminator],
     padRight: Option[Int],
     override val process: Option[ProcessExpr]
   ) extends BytesType
   case class BytesTerminatedType(
-    terminator: Int,
-    include: Boolean,
-    consume: Boolean,
-    eosError: Boolean,
+    terminator: Terminator,
     override val process: Option[ProcessExpr]
   ) extends BytesType
 
@@ -499,9 +494,9 @@ object DataType {
       } else {
         (arg.size, arg.sizeEos) match {
           case (Some(sizeValue), false) =>
-            Map(SwitchType.ELSE_CONST -> BytesLimitType(sizeValue, None, false, None, arg.process))
+            Map(SwitchType.ELSE_CONST -> BytesLimitType(sizeValue, None, None, arg.process))
           case (None, true) =>
-            Map(SwitchType.ELSE_CONST -> BytesEosType(None, false, None, arg.process))
+            Map(SwitchType.ELSE_CONST -> BytesEosType(None, None, arg.process))
           case (None, false) =>
             Map()
           case (Some(_), true) =>
@@ -533,7 +528,7 @@ object DataType {
     val r = dto match {
       case None => // `type:` key is missing in the KSY definition
         arg.contents match {
-          case Some(c) => BytesLimitType(Ast.expr.IntNum(c.length), None, false, None, arg.process)
+          case Some(c) => BytesLimitType(Ast.expr.IntNum(c.length), None, None, arg.process)
           case _ => arg.getByteArrayType(path)
         }
       case Some(dt) => dt match {// type: dt
@@ -576,15 +571,7 @@ object DataType {
           }
         case "str" | "strz" =>
           val enc = getEncoding(arg.encoding, metaDef, path)
-
-          // "strz" makes terminator = 0 by default
-          val arg2 = if (dt == "strz") {
-            arg.copy(terminator = arg.terminator.orElse(Some(0)))
-          } else {
-            arg
-          }
-
-          val bat = arg2.getByteArrayType(path)
+          val bat = arg.getByteArrayType(path)
           StrFromBytesType(bat, enc, arg.encoding.isEmpty)
         case _ =>
           val typeWithArgs = Expressions.parseTypeRef(dt)

shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala

@@ -2,7 +2,7 @@ package io.kaitai.struct.format
 
 import java.nio.charset.Charset
 import io.kaitai.struct.Utils
-import io.kaitai.struct.datatype.DataType
+import io.kaitai.struct.datatype.{DataType, Terminator}
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast.expr
 import io.kaitai.struct.exprlang.{Ast, Expressions}
@@ -82,32 +82,26 @@ case class AttrSpec(
 case class YamlAttrArgs(
   size: Option[Ast.expr],
   sizeEos: Boolean,
-  encoding: Option[String],
-  terminator: Option[Int],
-  include: Boolean,
-  consume: Boolean,
-  eosError: Boolean,
+  terminator: Option[Terminator],
   padRight: Option[Int],
+
+  encoding: Option[String],
   contents: Option[Array[Byte]],
   enumRef: Option[String],
   parent: Option[Ast.expr],
   process: Option[ProcessExpr]
 ) {
   def getByteArrayType(path: List[String]) = {
-    (size, sizeEos) match {
-      case (Some(bs: expr), false) =>
-        BytesLimitType(bs, terminator, include, padRight, process)
-      case (None, true) =>
-        BytesEosType(terminator, include, padRight, process)
-      case (None, false) =>
-        terminator match {
-          case Some(term) =>
-            BytesTerminatedType(term, include, consume, eosError, process)
-          case None =>
-            throw KSYParseError("'size', 'size-eos' or 'terminator' must be specified", path).toException
-        }
-      case (Some(_), true) =>
+    (size, sizeEos, terminator) match {
+      case (None,        true,   until) => BytesEosType(until, padRight, process)
+      case (None,       false, Some(t)) => BytesTerminatedType(t, process)
+      case (None,       false,    None) =>
+        throw KSYParseError("'size', 'size-eos' or 'terminator' must be specified", path).toException
+
+      case (Some(_),     true,       _) =>
         throw KSYParseError("only one of 'size' or 'size-eos' must be specified", path).toException
+      // TODO: Warning or even error, if natural type size is less that `size`
+      case (Some(size), false,   until) => BytesLimitType(size, until, padRight, process)
     }
   }
 }
@@ -180,9 +174,9 @@ object AttrSpec {
     val ifExpr = ParseUtils.getOptValueExpression(srcMap, "if", path)
     val encoding = ParseUtils.getOptValueStr(srcMap, "encoding", path)
     val terminator = ParseUtils.getOptValueInt(srcMap, "terminator", path)
-    val consume = ParseUtils.getOptValueBool(srcMap, "consume", path).getOrElse(true)
-    val include = ParseUtils.getOptValueBool(srcMap, "include", path).getOrElse(false)
-    val eosError = ParseUtils.getOptValueBool(srcMap, "eos-error", path).getOrElse(true)
+    val consume = ParseUtils.getOptValueBool(srcMap, "consume", path)
+    val include = ParseUtils.getOptValueBool(srcMap, "include", path)
+    val eosError = ParseUtils.getOptValueBool(srcMap, "eos-error", path)
     val padRight = ParseUtils.getOptValueInt(srcMap, "pad-right", path)
     val enumOpt = ParseUtils.getOptValueStr(srcMap, "enum", path)
     val parent = ParseUtils.getOptValueExpression(srcMap, "parent", path)
@@ -201,9 +195,30 @@ object AttrSpec {
 
     val typObj = srcMap.get("type")
 
+    val until = (typObj, terminator, consume, include, eosError) match {
+      // "strz" makes terminator = 0 by default
+      case (Some("strz"), None, consume, include, mandatory) => Some(Terminator(
+        0,
+        include.getOrElse(false),
+        consume.getOrElse(true),
+        mandatory.getOrElse(true),
+      ))
+      case (_, Some(value), consume, include, mandatory) => Some(Terminator(
+        value,
+        include.getOrElse(false),
+        consume.getOrElse(true),
+        mandatory.getOrElse(true),
+      ))
+      case (_, None, None, None, None) => None
+      // TODO: Emit warning instead here, but error also an option until warnings is not implemented
+      // case (_, None, _, _, _) =>
+      //   throw KSYParseError.withText("`consume`, `include` or `eos-error` has no effect without `terminator`", path)
+      case (_, None, _, _, _) => None
+    }
+
     val yamlAttrArgs = YamlAttrArgs(
-      size, sizeEos,
-      encoding, terminator, include, consume, eosError, padRight,
+      size, sizeEos, until, padRight,
+      encoding,
       contents, enumOpt, parent, process
     )
 
@@ -303,9 +318,9 @@ object AttrSpec {
     } else {
       (arg.size, arg.sizeEos) match {
         case (Some(sizeValue), false) =>
-          Map(SwitchType.ELSE_CONST -> BytesLimitType(sizeValue, None, false, None, arg.process))
+          Map(SwitchType.ELSE_CONST -> BytesLimitType(sizeValue, None, None, arg.process))
         case (None, true) =>
-          Map(SwitchType.ELSE_CONST -> BytesEosType(None, false, None, arg.process))
+          Map(SwitchType.ELSE_CONST -> BytesEosType(None, None, arg.process))
         case (None, false) =>
           Map()
         case (Some(_), true) =>

shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala

@@ -362,7 +362,7 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
         s"$io.ReadBytes(${expression(blt.size)})"
       case _: BytesEosType =>
         s"$io.ReadBytesFull()"
-      case BytesTerminatedType(terminator, include, consume, eosError, _) =>
+      case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) =>
         s"$io.ReadBytesTerm($terminator, $include, $consume, $eosError)"
       case BitsType1(bitEndian) =>
         s"$io.ReadBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1) != 0"
@@ -388,13 +388,13 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     }
   }
 
-  override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = {
+  override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]) = {
     val expr1 = padRight match {
       case Some(padByte) => s"$kstreamName.BytesStripRight($expr0, $padByte)"
       case None => expr0
     }
     val expr2 = terminator match {
-      case Some(term) => s"$kstreamName.BytesTerminate($expr1, $term, $include)"
+      case Some(Terminator(term, include, _, _)) => s"$kstreamName.BytesTerminate($expr1, $term, $include)"
       case None => expr1
     }
     expr2

shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala

@@ -673,7 +673,7 @@ class CppCompiler(
         s"$io->read_bytes(${expression(blt.size)})"
       case _: BytesEosType =>
         s"$io->read_bytes_full()"
-      case BytesTerminatedType(terminator, include, consume, eosError, _) =>
+      case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) =>
         s"$io->read_bytes_term($terminator, $include, $consume, $eosError)"
       case BitsType1(bitEndian) =>
         s"$io->read_bits_int_${bitEndian.toSuffix}(1)"
@@ -717,13 +717,13 @@ class CppCompiler(
     }
   }
 
-  override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = {
+  override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]) = {
     val expr1 = padRight match {
       case Some(padByte) => s"$kstreamName::bytes_strip_right($expr0, $padByte)"
       case None => expr0
     }
     val expr2 = terminator match {
-      case Some(term) => s"$kstreamName::bytes_terminate($expr1, $term, $include)"
+      case Some(Terminator(term, include, _, _)) => s"$kstreamName::bytes_terminate($expr1, $term, $include)"
       case None => expr1
     }
     expr2