From d972411b4843c611e9b4f06682dc86a5ea0674cb Mon Sep 17 00:00:00 2001 From: Mingun Date: Sun, 12 Dec 2021 01:51:59 +0500 Subject: [PATCH] Introduce a `Terminator` type that holds all information about terminator sequence --- .../struct/CalculateSeqSizes$Test.scala | 21 +++--- .../struct/ConstructClassCompiler.scala | 20 +++--- .../kaitai/struct/GraphvizClassCompiler.scala | 4 +- .../io/kaitai/struct/datatype/Chunk.scala | 22 ++++++ .../io/kaitai/struct/datatype/DataType.scala | 27 ++------ .../io/kaitai/struct/format/AttrSpec.scala | 67 ++++++++++++------- .../struct/languages/CSharpCompiler.scala | 6 +- .../kaitai/struct/languages/CppCompiler.scala | 6 +- .../kaitai/struct/languages/GoCompiler.scala | 6 +- .../struct/languages/JavaCompiler.scala | 6 +- .../struct/languages/JavaScriptCompiler.scala | 6 +- .../kaitai/struct/languages/LuaCompiler.scala | 8 +-- .../kaitai/struct/languages/NimCompiler.scala | 6 +- .../kaitai/struct/languages/PHPCompiler.scala | 8 +-- .../struct/languages/PerlCompiler.scala | 8 +-- .../struct/languages/PythonCompiler.scala | 6 +- .../struct/languages/RubyCompiler.scala | 6 +- .../struct/languages/RustCompiler.scala | 8 +-- .../components/EveryReadIsExpression.scala | 12 ++-- .../struct/languages/components/GoReads.scala | 14 ++-- 20 files changed, 147 insertions(+), 120 deletions(-) create mode 100644 shared/src/main/scala/io/kaitai/struct/datatype/Chunk.scala diff --git a/jvm/src/test/scala/io/kaitai/struct/CalculateSeqSizes$Test.scala b/jvm/src/test/scala/io/kaitai/struct/CalculateSeqSizes$Test.scala index 0a0e11cc1..b0a4822e2 100644 --- a/jvm/src/test/scala/io/kaitai/struct/CalculateSeqSizes$Test.scala +++ b/jvm/src/test/scala/io/kaitai/struct/CalculateSeqSizes$Test.scala @@ -1,7 +1,7 @@ package io.kaitai.struct import io.kaitai.struct.datatype.{BigEndian, BigBitEndian} -import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.{DataType, Terminator} import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.{Ast, Expressions} import io.kaitai.struct.format.{DynamicSized, FixedSized, MetaSpec, Sized, YamlAttrArgs} @@ -39,12 +39,14 @@ class CalculateSeqSizes$Test extends AnyFunSpec { YamlAttrArgs( size.map(s => Ast.expr.IntNum(s)), false,// sizeEos - None, // encoding - terminator, - false,// include - false,// consume - false,// eosError + terminator.map(value => Terminator( + value, + false,// include + false,// consume + false,// eosError + )), None, // padRight + None, // encoding contents, None, // enumRef None, // parent @@ -82,7 +84,7 @@ class CalculateSeqSizes$Test extends AnyFunSpec { CalculateSeqSizes.dataTypeBitsSize(SwitchType( Ast.expr.IntNum(0), cases.map { case (condition, typeName) => - Expressions.parse(condition) -> parse(Some(typeName), None, None, None) + Expressions.parse(condition) -> parse(Some(typeName), None, Some(0), None) } )) } @@ -164,8 +166,9 @@ class CalculateSeqSizes$Test extends AnyFunSpec { //----------------------------------------------------------------------- - sizeof("str", 0) should be (DynamicSized) - sizeof("strz" ) should be (DynamicSized) + // We do not auto-calculate terminator in our test, so in both cases terminator is necessary + sizeof("str", 0) should be (DynamicSized) + sizeof("strz", 0) should be (DynamicSized) //TODO: Uncomment when https://github.com/kaitai-io/kaitai_struct/issues/799 // will be implemented diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala index 192259fff..a91023cd8 100644 --- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala @@ -127,7 +127,7 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend s"Int${width.width * 8}${signToStr(signed)}${fixedEndianToStr(endianOpt.get)}" case FloatMultiType(width, endianOpt) => s"Float${width.width * 8}${fixedEndianToStr(endianOpt.get)}" - case BytesEosType(terminator, include, padRight, process) => + case BytesEosType(terminator, padRight, process) => "GreedyBytes" case blt: BytesLimitType => attrBytesLimitType(blt) @@ -135,7 +135,7 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend attrBytesTerminatedType(btt, "GreedyBytes") case StrFromBytesType(bytes, encoding, _) => bytes match { - case BytesEosType(terminator, include, padRight, process) => + case BytesEosType(terminator, padRight, process) => s"GreedyString(encoding='$encoding')" case blt: BytesLimitType => attrBytesLimitType(blt, s"GreedyString(encoding='$encoding')") @@ -146,10 +146,10 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend s"LazyBound(lambda: ${types2class(ut.classSpec.get.name)})" case utb: UserTypeFromBytes => utb.bytes match { - //case BytesEosType(terminator, include, padRight, process) => - case BytesLimitType(size, terminator, include, padRight, process) => + //case BytesEosType(terminator, padRight, process) => + case BytesLimitType(size, terminator, padRight, process) => s"FixedSized(${translator.translate(size)}, LazyBound(lambda: ${types2class(utb.classSpec.get.name)}))" - //case BytesTerminatedType(terminator, include, consume, eosError, process) => + //case BytesTerminatedType(terminator, process) => case _ => "???" } case et: EnumType => @@ -162,9 +162,9 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend def attrBytesLimitType(blt: BytesLimitType, subcon: String = "GreedyBytes"): String = { val subcon2 = blt.terminator match { case None => subcon - case Some(term) => + case Some(Terminator(term, include, _, _)) => val termStr = "\\x%02X".format(term & 0xff) - s"NullTerminated($subcon, term=b'$termStr', include=${translator.doBoolLiteral(blt.include)})" + s"NullTerminated($subcon, term=b'$termStr', include=${translator.doBoolLiteral(include)})" } val subcon3 = blt.padRight match { case None => subcon2 @@ -176,11 +176,11 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend } def attrBytesTerminatedType(btt: BytesTerminatedType, subcon: String): String = { - val termStr = "\\x%02X".format(btt.terminator & 0xff) + val termStr = "\\x%02X".format(btt.terminator.value & 0xff) s"NullTerminated($subcon, " + s"term=b'$termStr', " + - s"include=${translator.doBoolLiteral(btt.include)}, " + - s"consume=${translator.doBoolLiteral(btt.consume)})" + s"include=${translator.doBoolLiteral(btt.terminator.include)}, " + + s"consume=${translator.doBoolLiteral(btt.terminator.consume)})" } def attrSwitchType(st: SwitchType): String = { diff --git a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala index 0e8386cb1..a295ea958 100644 --- a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala @@ -1,6 +1,6 @@ package io.kaitai.struct -import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.{DataType, Terminator} import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ @@ -413,7 +413,7 @@ object GraphvizClassCompiler extends LanguageCompilerStatic { case rt: ReadableType => rt.apiCall(None) // FIXME case ut: UserType => type2display(ut.name) //case FixedBytesType(contents, _) => contents.map(_.formatted("%02X")).mkString(" ") - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => val args = ListBuffer[String]() if (terminator != 0) args += s"term=$terminator" diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/Chunk.scala b/shared/src/main/scala/io/kaitai/struct/datatype/Chunk.scala new file mode 100644 index 000000000..9bab46cbf --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/datatype/Chunk.scala @@ -0,0 +1,22 @@ +package io.kaitai.struct.datatype + +import io.kaitai.struct.exprlang.Ast + +/** Defines "working" sub-stream, as beginning part of full stream. + * All data after terminator byte is ignored and not available for parsing. + * + * @param value Byte at which stop reading stream + * @param include Specifies if terminator byte should be included in the final value + * @param consume Specify if terminator byte should be "consumed" when reading. + * If `true`: the stream pointer will point to the byte after the terminator byte + * If `false`: the stream pointer will point to the terminator byte itself + * @param mandatory If `true`, terminator must be present in the input stream, otherwise + * reaching end of stream before encountering terminator also possible. + * Corresponds to an `eos-error` key + */ +sealed case class Terminator( + value: Int, + include: Boolean, + consume: Boolean, + mandatory: Boolean, +) diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala index 14fe46708..b9cda51aa 100644 --- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala +++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala @@ -143,23 +143,18 @@ object DataType { override def process = None } case class BytesEosType( - terminator: Option[Int], - include: Boolean, + terminator: Option[Terminator], padRight: Option[Int], override val process: Option[ProcessExpr] ) extends BytesType case class BytesLimitType( size: Ast.expr, - terminator: Option[Int], - include: Boolean, + terminator: Option[Terminator], padRight: Option[Int], override val process: Option[ProcessExpr] ) extends BytesType case class BytesTerminatedType( - terminator: Int, - include: Boolean, - consume: Boolean, - eosError: Boolean, + terminator: Terminator, override val process: Option[ProcessExpr] ) extends BytesType @@ -499,9 +494,9 @@ object DataType { } else { (arg.size, arg.sizeEos) match { case (Some(sizeValue), false) => - Map(SwitchType.ELSE_CONST -> BytesLimitType(sizeValue, None, false, None, arg.process)) + Map(SwitchType.ELSE_CONST -> BytesLimitType(sizeValue, None, None, arg.process)) case (None, true) => - Map(SwitchType.ELSE_CONST -> BytesEosType(None, false, None, arg.process)) + Map(SwitchType.ELSE_CONST -> BytesEosType(None, None, arg.process)) case (None, false) => Map() case (Some(_), true) => @@ -533,7 +528,7 @@ object DataType { val r = dto match { case None => // `type:` key is missing in the KSY definition arg.contents match { - case Some(c) => BytesLimitType(Ast.expr.IntNum(c.length), None, false, None, arg.process) + case Some(c) => BytesLimitType(Ast.expr.IntNum(c.length), None, None, arg.process) case _ => arg.getByteArrayType(path) } case Some(dt) => dt match {// type: dt @@ -576,15 +571,7 @@ object DataType { } case "str" | "strz" => val enc = getEncoding(arg.encoding, metaDef, path) - - // "strz" makes terminator = 0 by default - val arg2 = if (dt == "strz") { - arg.copy(terminator = arg.terminator.orElse(Some(0))) - } else { - arg - } - - val bat = arg2.getByteArrayType(path) + val bat = arg.getByteArrayType(path) StrFromBytesType(bat, enc, arg.encoding.isEmpty) case _ => val typeWithArgs = Expressions.parseTypeRef(dt) diff --git a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala index 09e1b941f..5b850cd77 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala @@ -2,7 +2,7 @@ package io.kaitai.struct.format import java.nio.charset.Charset import io.kaitai.struct.Utils -import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.{DataType, Terminator} import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.exprlang.{Ast, Expressions} @@ -82,32 +82,26 @@ case class AttrSpec( case class YamlAttrArgs( size: Option[Ast.expr], sizeEos: Boolean, - encoding: Option[String], - terminator: Option[Int], - include: Boolean, - consume: Boolean, - eosError: Boolean, + terminator: Option[Terminator], padRight: Option[Int], + + encoding: Option[String], contents: Option[Array[Byte]], enumRef: Option[String], parent: Option[Ast.expr], process: Option[ProcessExpr] ) { def getByteArrayType(path: List[String]) = { - (size, sizeEos) match { - case (Some(bs: expr), false) => - BytesLimitType(bs, terminator, include, padRight, process) - case (None, true) => - BytesEosType(terminator, include, padRight, process) - case (None, false) => - terminator match { - case Some(term) => - BytesTerminatedType(term, include, consume, eosError, process) - case None => - throw KSYParseError("'size', 'size-eos' or 'terminator' must be specified", path).toException - } - case (Some(_), true) => + (size, sizeEos, terminator) match { + case (None, true, until) => BytesEosType(until, padRight, process) + case (None, false, Some(t)) => BytesTerminatedType(t, process) + case (None, false, None) => + throw KSYParseError("'size', 'size-eos' or 'terminator' must be specified", path).toException + + case (Some(_), true, _) => throw KSYParseError("only one of 'size' or 'size-eos' must be specified", path).toException + // TODO: Warning or even error, if natural type size is less that `size` + case (Some(size), false, until) => BytesLimitType(size, until, padRight, process) } } } @@ -180,9 +174,9 @@ object AttrSpec { val ifExpr = ParseUtils.getOptValueExpression(srcMap, "if", path) val encoding = ParseUtils.getOptValueStr(srcMap, "encoding", path) val terminator = ParseUtils.getOptValueInt(srcMap, "terminator", path) - val consume = ParseUtils.getOptValueBool(srcMap, "consume", path).getOrElse(true) - val include = ParseUtils.getOptValueBool(srcMap, "include", path).getOrElse(false) - val eosError = ParseUtils.getOptValueBool(srcMap, "eos-error", path).getOrElse(true) + val consume = ParseUtils.getOptValueBool(srcMap, "consume", path) + val include = ParseUtils.getOptValueBool(srcMap, "include", path) + val eosError = ParseUtils.getOptValueBool(srcMap, "eos-error", path) val padRight = ParseUtils.getOptValueInt(srcMap, "pad-right", path) val enumOpt = ParseUtils.getOptValueStr(srcMap, "enum", path) val parent = ParseUtils.getOptValueExpression(srcMap, "parent", path) @@ -201,9 +195,30 @@ object AttrSpec { val typObj = srcMap.get("type") + val until = (typObj, terminator, consume, include, eosError) match { + // "strz" makes terminator = 0 by default + case (Some("strz"), None, consume, include, mandatory) => Some(Terminator( + 0, + include.getOrElse(false), + consume.getOrElse(true), + mandatory.getOrElse(true), + )) + case (_, Some(value), consume, include, mandatory) => Some(Terminator( + value, + include.getOrElse(false), + consume.getOrElse(true), + mandatory.getOrElse(true), + )) + case (_, None, None, None, None) => None + // TODO: Emit warning instead here, but error also an option until warnings is not implemented + // case (_, None, _, _, _) => + // throw KSYParseError.withText("`consume`, `include` or `eos-error` has no effect without `terminator`", path) + case (_, None, _, _, _) => None + } + val yamlAttrArgs = YamlAttrArgs( - size, sizeEos, - encoding, terminator, include, consume, eosError, padRight, + size, sizeEos, until, padRight, + encoding, contents, enumOpt, parent, process ) @@ -303,9 +318,9 @@ object AttrSpec { } else { (arg.size, arg.sizeEos) match { case (Some(sizeValue), false) => - Map(SwitchType.ELSE_CONST -> BytesLimitType(sizeValue, None, false, None, arg.process)) + Map(SwitchType.ELSE_CONST -> BytesLimitType(sizeValue, None, None, arg.process)) case (None, true) => - Map(SwitchType.ELSE_CONST -> BytesEosType(None, false, None, arg.process)) + Map(SwitchType.ELSE_CONST -> BytesEosType(None, None, arg.process)) case (None, false) => Map() case (Some(_), true) => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala index ecc8c0795..7a68ee14a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala @@ -362,7 +362,7 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.ReadBytes(${expression(blt.size)})" case _: BytesEosType => s"$io.ReadBytesFull()" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io.ReadBytesTerm($terminator, $include, $consume, $eosError)" case BitsType1(bitEndian) => s"$io.ReadBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1) != 0" @@ -388,13 +388,13 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName.BytesStripRight($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.BytesTerminate($expr1, $term, $include)" + case Some(Terminator(term, include, _, _)) => s"$kstreamName.BytesTerminate($expr1, $term, $include)" case None => expr1 } expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala index 882c476c8..f04b0c8c6 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala @@ -673,7 +673,7 @@ class CppCompiler( s"$io->read_bytes(${expression(blt.size)})" case _: BytesEosType => s"$io->read_bytes_full()" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io->read_bytes_term($terminator, $include, $consume, $eosError)" case BitsType1(bitEndian) => s"$io->read_bits_int_${bitEndian.toSuffix}(1)" @@ -717,13 +717,13 @@ class CppCompiler( } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName::bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName::bytes_terminate($expr1, $term, $include)" + case Some(Terminator(term, include, _, _)) => s"$kstreamName::bytes_terminate($expr1, $term, $include)" case None => expr1 } expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala index 9829a0435..8a1076d7f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala @@ -404,7 +404,7 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.ReadBytes(int(${expression(blt.size)}))" case _: BytesEosType => s"$io.ReadBytesFull()" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io.ReadBytesTerm($terminator, $include, $consume, $eosError)" case BitsType1(bitEndian) => s"$io.ReadBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1)" @@ -425,13 +425,13 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } -// override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { +// override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]) = { // val expr1 = padRight match { // case Some(padByte) => s"$kstreamName.bytesStripRight($expr0, (byte) $padByte)" // case None => expr0 // } // val expr2 = terminator match { -// case Some(term) => s"$kstreamName.bytesTerminate($expr1, (byte) $term, $include)" +// case Some(Terminator(term, include, _, _)) => s"$kstreamName.bytesTerminate($expr1, (byte) $term, $include)" // case None => expr1 // } // expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 6b05e034e..60c290111 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -437,7 +437,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.readBytes(${expression(blt.size)})" case _: BytesEosType => s"$io.readBytesFull()" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io.readBytesTerm((byte) $terminator, $include, $consume, $eosError)" case BitsType1(bitEndian) => s"$io.readBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1) != 0" @@ -491,13 +491,13 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName.bytesStripRight($expr0, (byte) $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.bytesTerminate($expr1, (byte) $term, $include)" + case Some(Terminator(term, include, _, _)) => s"$kstreamName.bytesTerminate($expr1, (byte) $term, $include)" case None => expr1 } expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala index 6d397f268..6d62220e3 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala @@ -366,7 +366,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.readBytes(${expression(blt.size)})" case _: BytesEosType => s"$io.readBytesFull()" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io.readBytesTerm($terminator, $include, $consume, $eosError)" case BitsType1(bitEndian) => s"$io.readBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1) != 0" @@ -392,13 +392,13 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName.bytesStripRight($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.bytesTerminate($expr1, $term, $include)" + case Some(Terminator(term, include, _, _)) => s"$kstreamName.bytesTerminate($expr1, $term, $include)" case None => expr1 } expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala index 1093caf1b..d062811a6 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala @@ -1,7 +1,7 @@ package io.kaitai.struct.languages import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils, ExternalType} -import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError, ValidationNotEqualError} +import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError, ValidationNotEqualError, Terminator} import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ @@ -326,7 +326,7 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io:read_bytes(${expression(blt.size)})" case _: BytesEosType => s"$io:read_bytes_full()" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io:read_bytes_term($terminator, $include, $consume, $eosError)" case BitsType1(bitEndian) => s"$io:read_bits_int_${bitEndian.toSuffix}(1) ~= 0" @@ -350,13 +350,13 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } s"${types2class(t.classSpec.get.name)}($addParams$io$addArgs)" } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]): String = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName.bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.bytes_terminate($expr1, $term, $include)" + case Some(Terminator(term, include, _, _)) => s"$kstreamName.bytes_terminate($expr1, $term, $include)" case None => expr1 } expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala index b5ac4366b..a3e3391af 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala @@ -349,13 +349,13 @@ class NimCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def publicMemberName(id: Identifier): String = idToStr(id) // Members declared in io.kaitai.struct.languages.components.EveryReadIsExpression - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]): String = { val expr1 = padRight match { case Some(padByte) => s"$expr0.bytesStripRight($padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$expr1.bytesTerminate($term, $include)" + case Some(Terminator(term, include, _, _)) => s"$expr1.bytesTerminate($term, $include)" case None => expr1 } expr2 @@ -393,7 +393,7 @@ class NimCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.readBytes(int(${expression(blt.size)}))" case _: BytesEosType => s"$io.readBytesFull()" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io.readBytesTerm($terminator, $include, $consume, $eosError)" case BitsType1(bitEndian) => s"$io.readBitsInt${camelCase(bitEndian.toSuffix, true)}(1) != 0" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala index d3f11a74a..fe2481d2e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala @@ -1,7 +1,7 @@ package io.kaitai.struct.languages import io.kaitai.struct.datatype.DataType._ -import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian, KSError, UndecidedEndiannessError} +import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian, KSError, UndecidedEndiannessError, Terminator} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format.{NoRepeat, RepeatEos, RepeatExpr, RepeatSpec, _} import io.kaitai.struct.languages.components._ @@ -340,7 +340,7 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io->readBytes(${expression(blt.size)})" case _: BytesEosType => s"$io->readBytesFull()" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io->readBytesTerm($terminator, $include, $consume, $eosError)" case BitsType1(bitEndian) => s"$io->readBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1) != 0" @@ -366,13 +366,13 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName::bytesStripRight($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName::bytesTerminate($expr1, $term, $include)" + case Some(Terminator(term, include, _, _)) => s"$kstreamName::bytesTerminate($expr1, $term, $include)" case None => expr1 } expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala index 041c75cea..b91541a4a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala @@ -1,6 +1,6 @@ package io.kaitai.struct.languages -import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError} +import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError, Terminator} import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr @@ -296,7 +296,7 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io->read_bytes(${expression(blt.size)})" case _: BytesEosType => s"$io->read_bytes_full()" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io->read_bytes_term($terminator, ${boolLiteral(include)}, ${boolLiteral(consume)}, ${boolLiteral(eosError)})" case BitsType1(bitEndian) => s"$io->read_bits_int_${bitEndian.toSuffix}(1)" @@ -320,13 +320,13 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName::bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName::bytes_terminate($expr1, $term, ${boolLiteral(include)})" + case Some(Terminator(term, include, _, _)) => s"$kstreamName::bytes_terminate($expr1, $term, ${boolLiteral(include)})" case None => expr1 } expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala index 18b5d1299..4da555c4c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala @@ -356,7 +356,7 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.read_bytes(${expression(blt.size)})" case _: BytesEosType => s"$io.read_bytes_full()" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io.read_bytes_term($terminator, ${bool2Py(include)}, ${bool2Py(consume)}, ${bool2Py(eosError)})" case BitsType1(bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}(1) != 0" @@ -382,13 +382,13 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName.bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.bytes_terminate($expr1, $term, ${bool2Py(include)})" + case Some(Terminator(term, include, _, _)) => s"$kstreamName.bytes_terminate($expr1, $term, ${bool2Py(include)})" case None => expr1 } expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala index 28573e8ff..8d428bb72 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala @@ -364,7 +364,7 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.read_bytes(${expression(blt.size)})" case _: BytesEosType => s"$io.read_bytes_full" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io.read_bytes_term($terminator, $include, $consume, $eosError)" case BitsType1(bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}(1) != 0" @@ -396,13 +396,13 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ioName } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName::bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName::bytes_terminate($expr1, $term, $include)" + case Some(Terminator(term, include, _, _)) => s"$kstreamName::bytes_terminate($expr1, $term, $include)" case None => expr1 } expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 17be18991..81e354ed8 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -1,7 +1,7 @@ package io.kaitai.struct.languages import io.kaitai.struct.datatype.DataType._ -import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError} +import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError, Terminator} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format.{NoRepeat, RepeatEos, RepeatExpr, RepeatSpec, _} import io.kaitai.struct.languages.components._ @@ -311,7 +311,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.read_bytes(${expression(blt.size)})?" case _: BytesEosType => s"$io.read_bytes_full()?" - case BytesTerminatedType(terminator, include, consume, eosError, _) => + case BytesTerminatedType(Terminator(terminator, include, consume, eosError), _) => s"$io.read_bytes_term($terminator, $include, $consume, $eosError)?" case BitsType1(bitEndian) => s"$io.read_bits_int(1)? != 0" @@ -338,13 +338,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]): String = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName::bytesStripRight($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName::bytesTerminate($expr1, $term, $include)" + case Some(Terminator(term, include, _, _)) => s"$kstreamName::bytesTerminate($expr1, $term, $include)" case None => expr1 } expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala index 9db2e5c44..579c5f98a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala @@ -2,7 +2,7 @@ package io.kaitai.struct.languages.components import io.kaitai.struct.Utils import io.kaitai.struct.datatype.DataType._ -import io.kaitai.struct.datatype.{DataType, FixedEndian} +import io.kaitai.struct.datatype.{DataType, FixedEndian, Terminator} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ import io.kaitai.struct.translators.BaseTranslator @@ -91,10 +91,10 @@ trait EveryReadIsExpression // apply pad stripping and termination dataType match { - case BytesEosType(terminator, include, padRight, _) => - bytesPadTermExpr(expr, padRight, terminator, include) - case BytesLimitType(_, terminator, include, padRight, _) => - bytesPadTermExpr(expr, padRight, terminator, include) + case BytesEosType(terminator, padRight, _) => + bytesPadTermExpr(expr, padRight, terminator) + case BytesLimitType(_, terminator, padRight, _) => + bytesPadTermExpr(expr, padRight, terminator) case _ => expr } @@ -247,7 +247,7 @@ trait EveryReadIsExpression def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = ??? def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String - def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String + def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Terminator]): String def userTypeDebugRead(id: String, dataType: DataType, assignType: DataType): Unit = ??? def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala index 600d0cb28..e1008bc2c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala @@ -1,7 +1,7 @@ package io.kaitai.struct.languages.components import io.kaitai.struct.datatype.DataType._ -import io.kaitai.struct.datatype.{DataType, FixedEndian} +import io.kaitai.struct.datatype.{DataType, FixedEndian, Terminator} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ import io.kaitai.struct.translators.{GoTranslator, ResultLocalVar, ResultString, TranslatorResult} @@ -85,14 +85,14 @@ trait GoReads extends CommonReads with ObjectOrientedLanguage with GoSwitchOps { } } - def bytesPadTermExpr(id: ResultLocalVar, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { + def bytesPadTermExpr(id: ResultLocalVar, padRight: Option[Int], terminator: Option[Terminator]): String = { val expr0 = translator.resToStr(id) val expr1 = padRight match { case Some(padByte) => s"kaitai.BytesStripRight($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"kaitai.BytesTerminate($expr1, $term, $include)" + case Some(Terminator(term, include, _, _)) => s"kaitai.BytesTerminate($expr1, $term, $include)" case None => expr1 } expr2 @@ -100,10 +100,10 @@ trait GoReads extends CommonReads with ObjectOrientedLanguage with GoSwitchOps { def parseExprBytes(id: ResultLocalVar, dataType: BytesType): ResultLocalVar = { dataType match { - case BytesEosType(terminator, include, padRight, _) => - translator.outTransform(id, bytesPadTermExpr(id, padRight, terminator, include)) - case BytesLimitType(_, terminator, include, padRight, _) => - translator.outTransform(id, bytesPadTermExpr(id, padRight, terminator, include)) + case BytesEosType(terminator, padRight, _) => + translator.outTransform(id, bytesPadTermExpr(id, padRight, terminator)) + case BytesLimitType(_, terminator, padRight, _) => + translator.outTransform(id, bytesPadTermExpr(id, padRight, terminator)) case _ => id }