diff --git a/jvm/src/test/scala/io/kaitai/struct/CalculateSeqSizes$Test.scala b/jvm/src/test/scala/io/kaitai/struct/CalculateSeqSizes$Test.scala new file mode 100644 index 000000000..4459aef66 --- /dev/null +++ b/jvm/src/test/scala/io/kaitai/struct/CalculateSeqSizes$Test.scala @@ -0,0 +1,298 @@ +package io.kaitai.struct + +import io.kaitai.struct.datatype.{BigEndian, BigBitEndian} +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.{Ast, Expressions} +import io.kaitai.struct.format.{DynamicSized, FixedSized, MetaSpec, Sized, YamlAttrArgs} +import io.kaitai.struct.format.{RepeatSpec, NoRepeat, RepeatExpr, RepeatUntil, RepeatEos} +import io.kaitai.struct.precompile.CalculateSeqSizes +import io.kaitai.struct.problems.CompilationProblemException + +import scala.collection.immutable.SortedMap + +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers._ + +class CalculateSeqSizes$Test extends AnyFunSpec { + private def parse( + typeName: Option[String], + size: Option[Int], + terminator: Option[Seq[Byte]], + contents: Option[Array[Byte]], + ): DataType = { + DataType.fromYaml( + typeName, + List(), + MetaSpec( + List(), // path + false, // isOpaque + None, // id + Some(BigEndian), // endian + Some(BigBitEndian), // bitEndian + Some("utf-8"), // encoding + false, // forceDebug + None, // opaqueTypes + None, // zeroCopySubstream + List() // imports + ), + YamlAttrArgs( + size.map(s => Ast.expr.IntNum(s)), + false,// sizeEos + None, // encoding + terminator, + false,// include + false,// consume + false,// eosError + None, // padRight + contents, + None, // enumRef + None, // parent + None, // process + ) + ) + } + private def sizeof( + typeName: Option[String], + size: Option[Int], + terminator: Option[Seq[Byte]], + contents: Option[Array[Byte]], + ): Sized = { + CalculateSeqSizes.dataTypeBitsSize(parse(typeName, size, terminator, contents)) + } + /** Helper for testing built-in types */ + private def sizeof(typeName: String): Sized = { + sizeof(Some(typeName), None, None, None) + } + /** Helper for testing unsized built-in types which requires explicit size boundary. */ + private def sizeof(typeName: String, terminator: Seq[Byte]): Sized = { + sizeof(Some(typeName), None, Some(terminator), None) + } + /** Helper for testing the unnamed "bytes" built-in type defined implicitly via `size` key. */ + private def sizeof(size: Int): Sized = { + sizeof(None, Some(size), None, None) + } + /** Helper for testing the `contents` size. */ + private def sizeof(contents: Array[Byte]): Sized = { + sizeof(None, None, None, Some(contents)) + } + + /** Helper for testing the `switch-on` types. */ + private def switchOn(cases: SortedMap[String, String]): Sized = { + CalculateSeqSizes.dataTypeBitsSize(SwitchType( + Ast.expr.IntNum(0), + cases.map { case (condition, typeName) => + Expressions.parse(condition) -> parse(Some(typeName), None, None, None) + } + )) + } + + private def repeat(size: Sized, repeat: RepeatSpec): Sized = { + CalculateSeqSizes.sizeMultiply(size, repeat, List()) + } + + describe("CalculateSeqSizes") { + it("built-in types has correct size") { + sizeof("s1") should be (FixedSized( 8)) + sizeof("s2") should be (FixedSized(16)) + sizeof("s4") should be (FixedSized(32)) + sizeof("s8") should be (FixedSized(64)) + + sizeof("s2be") should be (FixedSized(16)) + sizeof("s4be") should be (FixedSized(32)) + sizeof("s8be") should be (FixedSized(64)) + + sizeof("s2le") should be (FixedSized(16)) + sizeof("s4le") should be (FixedSized(32)) + sizeof("s8le") should be (FixedSized(64)) + + //----------------------------------------------------------------------- + + sizeof("u1") should be (FixedSized( 8)) + sizeof("u2") should be (FixedSized(16)) + sizeof("u4") should be (FixedSized(32)) + sizeof("u8") should be (FixedSized(64)) + + sizeof("u2be") should be (FixedSized(16)) + sizeof("u4be") should be (FixedSized(32)) + sizeof("u8be") should be (FixedSized(64)) + + sizeof("u2le") should be (FixedSized(16)) + sizeof("u4le") should be (FixedSized(32)) + sizeof("u8le") should be (FixedSized(64)) + + //----------------------------------------------------------------------- + + sizeof("f4") should be (FixedSized(32)) + sizeof("f8") should be (FixedSized(64)) + + sizeof("f4be") should be (FixedSized(32)) + sizeof("f8be") should be (FixedSized(64)) + + sizeof("f4le") should be (FixedSized(32)) + sizeof("f8le") should be (FixedSized(64)) + + //----------------------------------------------------------------------- + + sizeof("b1") should be (FixedSized(1)) + sizeof("b2") should be (FixedSized(2)) + sizeof("b3") should be (FixedSized(3)) + sizeof("b4") should be (FixedSized(4)) + sizeof("b5") should be (FixedSized(5)) + sizeof("b6") should be (FixedSized(6)) + sizeof("b7") should be (FixedSized(7)) + sizeof("b8") should be (FixedSized(8)) + sizeof("b9") should be (FixedSized(9)) + + sizeof("b2be") should be (FixedSized(2)) + sizeof("b3be") should be (FixedSized(3)) + sizeof("b4be") should be (FixedSized(4)) + sizeof("b5be") should be (FixedSized(5)) + sizeof("b6be") should be (FixedSized(6)) + sizeof("b7be") should be (FixedSized(7)) + sizeof("b8be") should be (FixedSized(8)) + sizeof("b9be") should be (FixedSized(9)) + + sizeof("b2le") should be (FixedSized(2)) + sizeof("b3le") should be (FixedSized(3)) + sizeof("b4le") should be (FixedSized(4)) + sizeof("b5le") should be (FixedSized(5)) + sizeof("b6le") should be (FixedSized(6)) + sizeof("b7le") should be (FixedSized(7)) + sizeof("b8le") should be (FixedSized(8)) + sizeof("b9le") should be (FixedSized(9)) + + //----------------------------------------------------------------------- + + sizeof("str", Seq(0)) should be (DynamicSized) + sizeof("strz" ) should be (DynamicSized) + + //TODO: Uncomment when https://github.com/kaitai-io/kaitai_struct/issues/799 + // will be implemented + // sizeof("bytes") should be (DynamicSized) + sizeof(10) should be (FixedSized(10*8))// size: 10 + + sizeof("abcdef".getBytes()) should be (FixedSized(6*8))// content: 'abcdef' + } + + describe("switch-on") { + it("has a zero size if no cases present") { + switchOn(SortedMap()) should be (FixedSized(0)) + } + + it("has a fixed size when all cases have the same fixed size") { + switchOn(SortedMap( + "0" -> "f4be", + "1" -> "u4be", + "_" -> "s4be", + )) should be (FixedSized(4*8)) + } + + it("has a dynamic size when not all cases have the same size") { + switchOn(SortedMap( + "0" -> "f4be", + "1" -> "u4be", + "_" -> "u1", + )) should be (DynamicSized) + } + + it("has a dynamic size when contains a case with a dynamic size") { + // Fixed + Dynamic + switchOn(SortedMap( + "0" -> "f4be", + "1" -> "u4be", + "_" -> "strz", + )) should be (DynamicSized) + + // Dynamic + Fixed + switchOn(SortedMap( + "0" -> "strz", + "1" -> "u4be", + "_" -> "f4be", + )) should be (DynamicSized) + + // Dynamic + Dynamic + switchOn(SortedMap( + "1" -> "strz", + "_" -> "strz", + )) should be (DynamicSized) + } + } + + describe("repeat") { + it("no repeat does not change calculated size") { + repeat(FixedSized(42), NoRepeat) should be (FixedSized(42)) + repeat(DynamicSized, NoRepeat) should be (DynamicSized) + } + + it("`repeat-expr: ` produces compilation error") { + intercept[CompilationProblemException] { + repeat(FixedSized(0), RepeatExpr(Ast.expr.IntNum(-1))) + }.getMessage() should be ("(main): /:\n\terror: negative count of repetitions: -1\n") + + intercept[CompilationProblemException] { + repeat(FixedSized(42), RepeatExpr(Ast.expr.IntNum(-1))) + }.getMessage() should be ("(main): /:\n\terror: negative count of repetitions: -1\n") + + intercept[CompilationProblemException] { + repeat(DynamicSized, RepeatExpr(Ast.expr.IntNum(-1))) + }.getMessage() should be ("(main): /:\n\terror: negative count of repetitions: -1\n") + } + + it("`repeat-expr: ` produces FixedSized(0)") { + repeat(FixedSized(42), RepeatExpr(Ast.expr.IntNum(0))) should be (FixedSized(0)) + repeat(DynamicSized, RepeatExpr(Ast.expr.IntNum(0))) should be (FixedSized(0)) + } + + it("`repeat-expr: ` multiplies size") { + repeat(FixedSized(42), RepeatExpr(Ast.expr.IntNum(1))) should be (FixedSized(42)) + repeat(DynamicSized, RepeatExpr(Ast.expr.IntNum(1))) should be (DynamicSized) + + repeat(FixedSized(42), RepeatExpr(Ast.expr.IntNum(2))) should be (FixedSized(84)) + repeat(DynamicSized, RepeatExpr(Ast.expr.IntNum(2))) should be (DynamicSized) + } + + it("`repeat-until: ` produces compilation error") { + intercept[CompilationProblemException] { + repeat(FixedSized(0), RepeatUntil(Ast.expr.Bool(false))) + }.getMessage() should be ("(main): /:\n\terror: infinity cycle: stop condition is always `false`\n") + + intercept[CompilationProblemException] { + repeat(FixedSized(42), RepeatUntil(Ast.expr.Bool(false))) + }.getMessage() should be ("(main): /:\n\terror: infinity cycle: stop condition is always `false`\n") + + intercept[CompilationProblemException] { + repeat(DynamicSized, RepeatUntil(Ast.expr.Bool(false))) + }.getMessage() should be ("(main): /:\n\terror: infinity cycle: stop condition is always `false`\n") + } + + it("`repeat-until: ` returns the same size") { + repeat(FixedSized(42), RepeatUntil(Ast.expr.Bool(true))) should be (FixedSized(42)) + repeat(DynamicSized, RepeatUntil(Ast.expr.Bool(true))) should be (DynamicSized) + } + + it("`repeat-until: ` produces DynamicSized") { + val x = Ast.expr.Name(Ast.identifier("x")) + + repeat(FixedSized(42), RepeatUntil(x)) should be (DynamicSized) + repeat(DynamicSized, RepeatUntil(x)) should be (DynamicSized) + } + + it("FixedSize(0) always produce FixedSize(0)") { + val x = Ast.expr.Name(Ast.identifier("x")) + + repeat(FixedSized(0), NoRepeat) should be (FixedSized(0)) + + repeat(FixedSized(0), RepeatExpr(Ast.expr.IntNum(0))) should be (FixedSized(0)) + repeat(FixedSized(0), RepeatExpr(Ast.expr.IntNum(1))) should be (FixedSized(0)) + repeat(FixedSized(0), RepeatExpr(Ast.expr.IntNum(2))) should be (FixedSized(0)) + repeat(FixedSized(0), RepeatExpr(x)) should be (FixedSized(0)) + + repeat(FixedSized(0), RepeatUntil(Ast.expr.Bool(true))) should be (FixedSized(0)) + repeat(FixedSized(0), RepeatUntil(x)) should be (FixedSized(0)) + + repeat(FixedSized(0), RepeatEos) should be (FixedSized(0)) + } + } + } +} diff --git a/jvm/src/test/scala/io/kaitai/struct/exprlang/Ast$Test.scala b/jvm/src/test/scala/io/kaitai/struct/exprlang/Ast$Test.scala index bebe464e7..305feec02 100644 --- a/jvm/src/test/scala/io/kaitai/struct/exprlang/Ast$Test.scala +++ b/jvm/src/test/scala/io/kaitai/struct/exprlang/Ast$Test.scala @@ -51,4 +51,138 @@ class Ast$Test extends AnyFunSpec { Expressions.parse("(x + 1) * (x + 1) - (x * x + 2 * x + 2)").evaluateIntConst should be(None) // be(Some(0)) } } + + describe("Ast.expr.evaluateBoolConst") { + it ("considers `true` constant") { + Expressions.parse("true").evaluateBoolConst should be(Some(true)) + } + + it ("considers `false` constant") { + Expressions.parse("false").evaluateBoolConst should be(Some(false)) + } + + it ("considers `false and ?` constant") { + Expressions.parse("false and false").evaluateBoolConst should be(Some(false)) + Expressions.parse("false and true").evaluateBoolConst should be(Some(false)) + Expressions.parse("false and x").evaluateBoolConst should be(Some(false)) + Expressions.parse("false and (1==1)").evaluateBoolConst should be(Some(false)) + } + + it ("considers `? and false` constant") { + Expressions.parse("false and false").evaluateBoolConst should be(Some(false)) + Expressions.parse("true and false").evaluateBoolConst should be(Some(false)) + Expressions.parse("x and false").evaluateBoolConst should be(Some(false)) + Expressions.parse("(1==1) and false").evaluateBoolConst should be(Some(false)) + } + + it ("considers `true or ?` constant") { + Expressions.parse("true or false" ).evaluateBoolConst should be(Some(true)) + Expressions.parse("true or true" ).evaluateBoolConst should be(Some(true)) + Expressions.parse("true or x" ).evaluateBoolConst should be(Some(true)) + Expressions.parse("true or (1==1)").evaluateBoolConst should be(Some(true)) + } + + it ("considers `? or true` constant") { + Expressions.parse("false or true").evaluateBoolConst should be(Some(true)) + Expressions.parse("true or true").evaluateBoolConst should be(Some(true)) + Expressions.parse("x or true").evaluateBoolConst should be(Some(true)) + Expressions.parse("(1==1) or true").evaluateBoolConst should be(Some(true)) + } + + it ("evaluates `? == ?`") { + Expressions.parse("true == true" ).evaluateBoolConst should be(Some(true)) + Expressions.parse("false == false").evaluateBoolConst should be(Some(true)) + Expressions.parse("42 == 42" ).evaluateBoolConst should be(Some(true)) + Expressions.parse("field == field").evaluateBoolConst should be(None)//(Some(true))//TODO: symbolic calculations + + Expressions.parse("true == false").evaluateBoolConst should be(Some(false)) + Expressions.parse("false == true" ).evaluateBoolConst should be(Some(false)) + Expressions.parse("42 == 420" ).evaluateBoolConst should be(Some(false)) + Expressions.parse("field == other").evaluateBoolConst should be(None) + } + + it ("evaluates `? != ?`") { + Expressions.parse("true != true" ).evaluateBoolConst should be(Some(false)) + Expressions.parse("false != false").evaluateBoolConst should be(Some(false)) + Expressions.parse("42 != 42" ).evaluateBoolConst should be(Some(false)) + Expressions.parse("field != field").evaluateBoolConst should be(None)//(Some(false))//TODO: symbolic calculations + + Expressions.parse("true != false").evaluateBoolConst should be(Some(true)) + Expressions.parse("false != true" ).evaluateBoolConst should be(Some(true)) + Expressions.parse("42 != 420" ).evaluateBoolConst should be(Some(true)) + Expressions.parse("field != other").evaluateBoolConst should be(None) + } + + it ("evaluates `? < ?`") { + Expressions.parse("42 < 10").evaluateBoolConst should be(Some(false)) + Expressions.parse("42 < 42").evaluateBoolConst should be(Some(false)) + Expressions.parse("42 < 99").evaluateBoolConst should be(Some(true)) + Expressions.parse("42 < xx").evaluateBoolConst should be(None) + Expressions.parse("xx < xx").evaluateBoolConst should be(None)//(Some(false))//TODO: symbolic calculations + Expressions.parse("xx < yy").evaluateBoolConst should be(None) + + Expressions.parse("10 < 42").evaluateBoolConst should be(Some(true)) + Expressions.parse("42 < 42").evaluateBoolConst should be(Some(false)) + Expressions.parse("99 < 42").evaluateBoolConst should be(Some(false)) + Expressions.parse("xx < 42").evaluateBoolConst should be(None) + } + + it ("evaluates `? <= ?`") { + Expressions.parse("42 <= 10").evaluateBoolConst should be(Some(false)) + Expressions.parse("42 <= 42").evaluateBoolConst should be(Some(true)) + Expressions.parse("42 <= 99").evaluateBoolConst should be(Some(true)) + Expressions.parse("42 <= xx").evaluateBoolConst should be(None) + Expressions.parse("xx <= xx").evaluateBoolConst should be(None)//(Some(true))//TODO: symbolic calculations + Expressions.parse("xx <= yy").evaluateBoolConst should be(None) + + Expressions.parse("10 <= 42").evaluateBoolConst should be(Some(true)) + Expressions.parse("42 <= 42").evaluateBoolConst should be(Some(true)) + Expressions.parse("99 <= 42").evaluateBoolConst should be(Some(false)) + Expressions.parse("xx <= 42").evaluateBoolConst should be(None) + } + + it ("evaluates `? > ?`") { + Expressions.parse("42 > 10").evaluateBoolConst should be(Some(true)) + Expressions.parse("42 > 42").evaluateBoolConst should be(Some(false)) + Expressions.parse("42 > 99").evaluateBoolConst should be(Some(false)) + Expressions.parse("42 > xx").evaluateBoolConst should be(None) + Expressions.parse("xx > xx").evaluateBoolConst should be(None)//(Some(false))//TODO: symbolic calculations + Expressions.parse("xx > yy").evaluateBoolConst should be(None) + + Expressions.parse("10 > 42").evaluateBoolConst should be(Some(false)) + Expressions.parse("42 > 42").evaluateBoolConst should be(Some(false)) + Expressions.parse("99 > 42").evaluateBoolConst should be(Some(true)) + Expressions.parse("xx > 42").evaluateBoolConst should be(None) + } + + it ("evaluates `? >= ?`") { + Expressions.parse("42 >= 10").evaluateBoolConst should be(Some(true)) + Expressions.parse("42 >= 42").evaluateBoolConst should be(Some(true)) + Expressions.parse("42 >= 99").evaluateBoolConst should be(Some(false)) + Expressions.parse("42 >= xx").evaluateBoolConst should be(None) + Expressions.parse("xx >= xx").evaluateBoolConst should be(None)//(Some(true))//TODO: symbolic calculations + Expressions.parse("xx >= yy").evaluateBoolConst should be(None) + + Expressions.parse("10 >= 42").evaluateBoolConst should be(Some(false)) + Expressions.parse("42 >= 42").evaluateBoolConst should be(Some(true)) + Expressions.parse("99 >= 42").evaluateBoolConst should be(Some(true)) + Expressions.parse("xx >= 42").evaluateBoolConst should be(None) + } + + it ("considers `[true, false, 7==7][2]` constant") { + Expressions.parse("[true, false, 7==7][2]").evaluateBoolConst should be(Some(true)) + } + + it ("considers `4 > 2 ? true : false` constant") { + Expressions.parse("4 > 2 ? true : false").evaluateBoolConst should be(Some(true)) + } + + it ("considers `x` variable") { + Expressions.parse("x").evaluateBoolConst should be(None) + } + + it ("considers `[true, false, 7==7][x]` variable") { + Expressions.parse("[true, false, 7==7][x]").evaluateBoolConst should be(None) + } + } } diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala index 96e6f4036..b7a5254dc 100644 --- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala +++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala @@ -21,7 +21,7 @@ sealed trait DataType { * Kaitai Struct type system. */ object DataType { - abstract class IntWidth(val width: Int) + abstract sealed class IntWidth(val width: Int) case object Width1 extends IntWidth(1) case object Width2 extends IntWidth(2) case object Width4 extends IntWidth(4) @@ -35,14 +35,58 @@ object DataType { def apiCall(defEndian: Option[FixedEndian]): String } + /** A generic number type. */ abstract sealed class NumericType extends DataType + /** A generic boolean type. */ abstract sealed class BooleanType extends DataType + /** A generic integer type. */ abstract sealed class IntType extends NumericType + /** + * An integer type that occupies undecided number of bytes in the stream. + * + * If it possible to determine, the more narrow `Int1Type` will be inferred + * for an expression instead of this type. + * + * Represents a type of the following constructions: + * + * - `sizeof<>` and `bitsizeof<>` built-in operators + * - `_sizeof` special property of fields and self object + * - `size` and `pos` properties of streams + * - `to_i` result when converting other types (strings, floats, enums, and booleans) to integers + * - `length` and `size` properties of arrays + * - `length` property of strings + * - `_index` context variable + */ case object CalcIntType extends IntType + /** + * An integer type that fit into the byte and therefore can represent element + * of the byte array. + * + * Parameters have this type when it's declared with `type: u1` or `type: s1`. + * + * Represents a type of the following constructions: + * + * - `Int1Type(true)` -- a constant in the [0..127] range + * - `Int1Type(false)` -- a constant in the [128..255] range + * - element type of byte arrays + * - `first`, `last`, `min`, and `max` properties of byte arrays + * - result of the `[]` operator of byte arrays + * + * @param signed Determines if most significant bit of number contains sign + */ case class Int1Type(signed: Boolean) extends IntType with ReadableType { override def apiCall(defEndian: Option[FixedEndian]): String = if (signed) "s1" else "u1" } + /** + * An integer type that occupies some predefined size in bytes in the stream. + * + * Parameters have this type when it's declared with `type: u` or `type: s`. + * + * @param signed Determines if most significant bit of number contains sign + * @param width Size of type in bytes + * @param endian Byte order used to represent the number + */ case class IntMultiType(signed: Boolean, width: IntWidth, endian: Option[FixedEndian]) extends IntType with ReadableType { override def apiCall(defEndian: Option[FixedEndian]): String = { val ch1 = if (signed) 's' else 'u' @@ -50,11 +94,34 @@ object DataType { s"$ch1${width.width}${finalEnd.map(_.toSuffix).getOrElse("")}" } } + /** + * A boolean type that occupies one bit in the stream. + * + * Parameters have this type when it's declared with `type: b1`. + */ case class BitsType1(bitEndian: BitEndianness) extends BooleanType + /** + * An integer number type that occupies some predefined size in _bits_ in the stream. + * + * Parameters have this type when it's declared with `type: bX`. + * + * @param width Size of type in bits + * @param bitEndian Bit order inside of byte used to represent the number + */ case class BitsType(width: Int, bitEndian: BitEndianness) extends IntType - abstract class FloatType extends NumericType + /** A generic floating-point number type. */ + abstract sealed class FloatType extends NumericType + /** A floating-point number type that occupies undecided number of bytes in the stream. */ case object CalcFloatType extends FloatType + /** + * A floating-point number type that occupies some predefined size in bytes in the stream. + * + * Parameters have this type when it's declared with `type: fX`. + * + * @param width Size of type in bytes + * @param endian Byte order used to represent the number + */ case class FloatMultiType(width: IntWidth, endian: Option[FixedEndian]) extends FloatType with ReadableType { override def apiCall(defEndian: Option[FixedEndian]): String = { val finalEnd = endian.orElse(defEndian) @@ -66,7 +133,13 @@ object DataType { def process: Option[ProcessExpr] } - abstract class BytesType extends DataType with Processing + /** A generic raw bytes type. */ + abstract sealed class BytesType extends DataType with Processing + /** + * A raw bytes type that occupies undecided number of bytes in the stream. + * + * Parameters have this type when it's declared with `type: bytes`. + */ case object CalcBytesType extends BytesType { override def process = None } @@ -91,7 +164,13 @@ object DataType { override val process: Option[ProcessExpr] ) extends BytesType - abstract class StrType extends DataType + /** A generic string type. */ + abstract sealed class StrType extends DataType + /** + * A pure string type that occupies undecided number of bytes in the stream. + * + * Parameters have this type when it's declared with `type: str`. + */ case object CalcStrType extends StrType /** * A type that have the `str` and `strz` built-in Kaitai types. @@ -112,6 +191,11 @@ object DataType { isEncodingDerived: Boolean, ) extends StrType + /** + * A boolean type that occupies undecided number of bytes in the stream. + * + * Parameters have this type when it's declared with `type: bool`. + */ case object CalcBooleanType extends BooleanType /** @@ -148,7 +232,7 @@ object DataType { def isOwningInExpr: Boolean = false } - abstract class StructType extends ComplexDataType + abstract sealed class StructType extends ComplexDataType /** * Common abstract ancestor for all types which can treated as "user types". @@ -158,7 +242,7 @@ object DataType { * @param forcedParent optional parent enforcement expression * @param args parameters passed into this type as extra arguments */ - abstract class UserType( + abstract sealed class UserType( val name: List[String], val forcedParent: Option[Ast.expr], var args: Seq[Ast.expr] @@ -178,6 +262,7 @@ object DataType { cs.meta.isOpaque } } + /** User type which isn't restricted in size (i.e. without `size`, `terminator`, or `size-eos`). */ case class UserTypeInstream( _name: List[String], _forcedParent: Option[Ast.expr], @@ -190,6 +275,7 @@ object DataType { r } } + /** User type which is restricted in size either via `size`, `terminator`, or `size-eos`. */ case class UserTypeFromBytes( _name: List[String], _forcedParent: Option[Ast.expr], @@ -204,6 +290,12 @@ object DataType { r } } + /** + * Reference to the user type which isn't restricted in size (i.e. without + * `size`, `terminator`, or `size-eos`). + * + * Parameters have this type when it's declared with any non-built-in type. + */ case class CalcUserType( _name: List[String], _forcedParent: Option[Ast.expr], @@ -212,6 +304,10 @@ object DataType { ) extends UserType(_name, _forcedParent, _args) { override def isOwning = false } + /** + * Reference to the user type which restricted in size either via `size`, + * `terminator`, or `size-eos`. + */ case class CalcUserTypeFromBytes( _name: List[String], _forcedParent: Option[Ast.expr], @@ -223,12 +319,29 @@ object DataType { override def isOwning = false } + /** + * A generic collection type. + * + * @param elType Type of elements in the collection + */ abstract sealed class ArrayType(val elType: DataType) extends ComplexDataType - + /** + * An owned slice of array type. This type is used for holding data in attributes + * with `repeat` key. Number of elements in that type is unknown + * + * @param _elType Type of elements in the slice + */ case class ArrayTypeInStream(_elType: DataType) extends ArrayType(_elType) { override def isOwning: Boolean = true override def asNonOwning(isOwningInExpr: Boolean = false): CalcArrayType = CalcArrayType(elType, isOwningInExpr) } + /** + * A borrowed slice of an array. This type is used when array is passed as a + * parameter to the user type (parameter have type `bytes` or haven't any + * explicitly defined type). Number of elements in that type is unknown + * + * @param _elType Type of elements in the slice + */ case class CalcArrayType(_elType: DataType, override val isOwningInExpr: Boolean = false) extends ArrayType(_elType) { override def isOwning: Boolean = false } @@ -236,18 +349,46 @@ object DataType { /** Represents `_parent: false` expression which means that type explicitly has no parent. */ val USER_TYPE_NO_PARENT = Ast.expr.Bool(false) + /** + * A very generic type that can hold any other type. Used when type of expression + * is completely unknown to the compiler. + * + * Parameters have this type when it's declared with `type: any`. + */ case object AnyType extends DataType + + /** + * A type that can hold any Kaitai generated type. Can be used as common ancestor + * of `switch-on` types, when all alternative types is owned. + */ case object KaitaiStructType extends StructType { def isOwning = true override def asNonOwning(isOwningInExpr: Boolean = false): DataType = CalcKaitaiStructType(isOwningInExpr) } + /** + * A type that can hold any Kaitai generated type. Can be used as common ancestor + * of `switch-on` types, when at least one of the alternative types is borrowed. + * + * Parameters have this type when it's declared with `type: struct`. + */ case class CalcKaitaiStructType(override val isOwningInExpr: Boolean = false) extends StructType { def isOwning = false } + /** + * A type that hold and own Kaitai stream object. This type is used when a new IO object + * is allocated (i.e. when new sub-stream is created for types with `size`, `terminator`, + * or `size-eos: true` attributes). + */ case object OwnedKaitaiStreamType extends ComplexDataType { def isOwning = true override def asNonOwning(isOwningInExpr: Boolean = false): DataType = KaitaiStreamType } + /** + * A type that hold and borrow Kaitai stream object. This type is used + * when an IO object is passed as parameter to the user type. + * + * Parameters have this type when it's declared with `type: io`. + */ case object KaitaiStreamType extends ComplexDataType { def isOwning = false } @@ -377,6 +518,13 @@ object DataType { private val ReFloatType = """f(4|8)(le|be)?""".r private val ReBitType = """b(\d+)(le|be)?""".r + /** + * @param dto Content of the `type` key or the case variant in case of `switch-on` type + * @param path YAML path to the type definition, for use in errors + * @param metaDef Default properties of the attribute from `meta` section of type definition + * @param arg Other properties of the attribute in the KSY definition, such as `size`, + `content`, `terminator`, etc. + */ def fromYaml( dto: Option[String], path: List[String], @@ -384,12 +532,12 @@ object DataType { arg: YamlAttrArgs ): DataType = { val r = dto match { - case None => + case None => // `type:` key is missing in the KSY definition arg.contents match { case Some(c) => BytesLimitType(Ast.expr.IntNum(c.length), None, false, None, arg.process) case _ => arg.getByteArrayType(path) } - case Some(dt) => dt match { + case Some(dt) => dt match {// type: dt case "u1" => Int1Type(false) case "s1" => Int1Type(true) case ReIntType(signStr, widthStr, endianStr) => @@ -459,6 +607,8 @@ object DataType { StrFromBytesType(bat, enc, arg.encoding.isEmpty) case _ => val typeWithArgs = Expressions.parseTypeRef(dt) + // if `size`, `terminator` and `size-eos: true` isn't defined, + // user type uses parent stream, otherwise creates an own stream if (arg.size.isEmpty && !arg.sizeEos && arg.terminator.isEmpty) { if (arg.process.isDefined) throw KSYParseError(s"user type '$dt': need 'size' / 'size-eos' / 'terminator' if 'process' is used", path).toException diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala index 2acf1943b..276dc9592 100644 --- a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala +++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala @@ -61,6 +61,17 @@ object Ast { case ConstEvaluator.value.Str(x) => Some(x) case _ => None } + /** + * Evaluates the expression, if it's possible to get a static boolean + * constant as the result of evaluation (i.e. if it does not involve any + * variables or anything like that). Expect no complex logic or symbolic + * simplification of expressions here: something like "x and !x", which is + * known to be always `false`, will still report it as "None". + * + * @return boolean result of evaluation if it's constant or None, if it's + * variable + */ + def evaluateBoolConst: Option[Boolean] = ConstEvaluator.evaluateBoolConst(this) } object expr{ diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/ConstEvaluator.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/ConstEvaluator.scala index d146b3b50..636a0e0e8 100644 --- a/shared/src/main/scala/io/kaitai/struct/exprlang/ConstEvaluator.scala +++ b/shared/src/main/scala/io/kaitai/struct/exprlang/ConstEvaluator.scala @@ -29,6 +29,22 @@ object ConstEvaluator { } } + /** + * Evaluates the expression, if it's possible to get a boolean constant + * as the result of evaluation (i.e. if it does not involve any variables + * or anything like that). + * + * @param ex expression to evaluate. + * @return boolean result of evaluation if it's constant or None, if it's + * variable or potentially variable. + */ + def evaluateBoolConst(ex: Ast.expr): Option[Boolean] = { + evaluate(ex) match { + case value.Bool(x) => Some(x) + case _ => None + } + } + /** * Evaluates the expression, if it's possible to get a constant as the result * of evaluation (i.e. if it does not involve any variables or anything like @@ -72,17 +88,22 @@ object ConstEvaluator { case _ => value.NonConst } - case expr.BoolOp(op, values) => - value.Bool(values.foldLeft(true)((acc, right) => { - val rightValue = evaluate(right) match { - case value.Bool(x) => x - case _ => return value.NonConst - } - op match { - case boolop.And => acc && rightValue - case boolop.Or => acc || rightValue - } - })) + case expr.BoolOp(boolop.And, values) => values.foldLeft[value](value.Bool(true))( + (acc, right) => evaluate(right) match { + // `... && false` always produce `false`, so do early return + case value.Bool(false) => return value.Bool(false) + case value.Bool(true) => value.Bool(true) + case _ => value.NonConst + } + ) + case expr.BoolOp(boolop.Or, values) => values.foldLeft[value](value.Bool(false))( + (acc, right) => evaluate(right) match { + // `... || true` always produce `false`, so do early return + case value.Bool(true) => return value.Bool(true) + case value.Bool(false) => value.Bool(false) + case _ => value.NonConst + } + ) case expr.Compare(left, op, right) => val leftValue = evaluate(left) diff --git a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala index 6a3cb0135..fa89f509c 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala @@ -33,10 +33,48 @@ case object GenericStructClassSpec extends ClassSpecLike { override def toDataType: DataType = CalcKaitaiStructType() } -sealed trait Sized +/** + * Type that represent result of the `_sizeof` special property and `sizeof<>` + * meta-function. + */ +sealed trait Sized { + /** + * Combines two sizes into one object which covers both size requirements + * + * @param other Size to combine with + */ + final def or(other: Sized): Sized = { + (this, other) match { + case (FixedSized(l), FixedSized(r)) if l == r => FixedSized(l) + case (FixedSized(_), FixedSized(_)) => DynamicSized + + case (FixedSized(_), DynamicSized) => DynamicSized + case (DynamicSized, FixedSized(_)) => DynamicSized + + case (DynamicSized, DynamicSized) => DynamicSized + + // other combinations should produce an error + } + } +} +/** + * The size of type have no constant value. The examples is built-in unsized + * types: `str`, `strz`, and `bytes`. Those types has no natural size in contrary + * to the sized types, such as `u1` or `f4be`. + */ case object DynamicSized extends Sized +/** + * A marker object that indicates that size of the type has not been yet calculated + * and calculation should be performed when size will be requested. + */ case object NotCalculatedSized extends Sized +/** + * A marker object that indicates that calculation of the size of the type in the + * progress. If that object will be seen during calculation process it is mean that + * type is defined recursively. + */ case object StartedCalculationSized extends Sized +/** The size of type is `n` bits. */ case class FixedSized(n: Int) extends Sized case class ClassSpec( diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala index 4d28c263b..816a0131f 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala @@ -5,6 +5,7 @@ import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ +import io.kaitai.struct.problems.KSYParseError class CalculateSeqSizes(specs: ClassSpecs) { def run(): Unit = { @@ -13,24 +14,54 @@ class CalculateSeqSizes(specs: ClassSpecs) { } object CalculateSeqSizes { - def sizeMultiply(sizeElement: Sized, repeat: RepeatSpec) = { - sizeElement match { - case FixedSized(elementSize) => - repeat match { - case NoRepeat => - sizeElement - case RepeatExpr(expr) => - expr.evaluateIntConst match { - case Some(count) => FixedSized(elementSize * count.toInt) - case None => DynamicSized - } - case _: RepeatUntil | RepeatEos => - DynamicSized + def sizeMultiply(sizeElement: Sized, repeat: RepeatSpec, path: List[String]): Sized = { + repeat match { + case NoRepeat => sizeElement + + case RepeatExpr(expr) => (expr.evaluateIntConst, sizeElement) match { + case (Some(count), _) if count < 0 => + throw KSYParseError.withText(s"negative count of repetitions: ${count}", path) + + case (Some(count), _) if count == 0 => { + //TODO: add user visible warning + Log.seqSizes.warn(() => s"repetition count expression ${expr} is always `0`, no iterations will be performed") + FixedSized(0) + } + case (Some(count), FixedSized(size)) => FixedSized(size * count.toInt)//FIXME: toInt is a potential footgun + + case (_, FixedSized(0)) => FixedSized(0) + case (_, _) => DynamicSized + } + + case RepeatUntil(expr) => expr.evaluateBoolConst match { + case Some(false) => + throw KSYParseError.withText("infinity cycle: stop condition is always `false`", path) + + case Some(true) => { + //TODO: add user visible warning + Log.seqSizes.warn(() => s"expression ${expr} is always `true`, cycle will be stopped after first iteration") + sizeElement + } + + case None => sizeElement match { + case FixedSized(0) => FixedSized(0) + case _ => DynamicSized } - case _ => sizeElement + } + + case RepeatEos => sizeElement match { + case FixedSized(0) => FixedSized(0) + case _ => DynamicSized + } } } + /** + * Calculates size of the specified class instance for use in the `_sizeof` + * special property and `sizeof<>` meta-function. + * + * @param curClass + */ def getSeqSize(curClass: ClassSpec): Sized = { curClass.seqSize match { case DynamicSized | _: FixedSized => @@ -64,7 +95,23 @@ object CalculateSeqSizes { var seqPos: Option[Int] = Some(0) curClass.seq.foreach { attr => val sizeElement = dataTypeBitsSize(attr.dataType) - val sizeContainer = sizeMultiply(sizeElement, attr.cond.repeat) + val size = sizeMultiply(sizeElement, attr.cond.repeat, attr.path) + val sizeContainer = attr.cond.ifExpr match { + case Some(expr) => expr.evaluateBoolConst match { + case Some(true) => { + //TODO: add user visible warning + Log.seqSizes.warn(() => s"${attr.path}: condition is always `true`") + size + } + case Some(false) => { + //TODO: add user visible warning + Log.seqSizes.warn(() => s"${attr.path}: condition is always `false`") + FixedSized(0) + } + case None => DynamicSized + } + case None => size + } op(attr, seqPos, sizeElement, sizeContainer) @@ -80,43 +127,51 @@ object CalculateSeqSizes { * Determines how many bits occupies given data type. * * @param dataType data type to analyze - * @return number of bits or None, if it's impossible to determine a priori + * @return number of bits or [[DynamicSized]], if it's impossible to determine a priori */ def dataTypeBitsSize(dataType: DataType): Sized = { dataType match { case BitsType1(_) => FixedSized(1) - case BitsType(width, _) => FixedSized(width) + case CalcBooleanType => DynamicSized + case EnumType(_, basedOn) => dataTypeBitsSize(basedOn) + case ut: UserTypeInstream => getSeqSize(ut.classSpec.get) - case _ => - dataTypeByteSize(dataType) match { - case FixedSized(x) => FixedSized(x * 8) - case otherSize => otherSize - } - } - } + case ut: UserTypeFromBytes => dataTypeBitsSize(ut.bytes) + case ut: CalcUserTypeFromBytes => dataTypeBitsSize(ut.bytes) + case _: StructType => DynamicSized + + case BitsType(width, _) => FixedSized(width) + case Int1Type(_) => FixedSized(8) + case IntMultiType(_, width, _) => FixedSized(width.width * 8) + case CalcIntType => DynamicSized + + case FloatMultiType(width, _) => FixedSized(width.width * 8) + case CalcFloatType => DynamicSized - /** - * Determines how many bytes occupies a given data type. - * - * @param dataType data type to analyze - * @return number of bytes or None, if it's impossible to determine a priori - */ - def dataTypeByteSize(dataType: DataType): Sized = { - dataType match { - case _: Int1Type => FixedSized(1) - case IntMultiType(_, width, _) => FixedSized(width.width) - case FloatMultiType(width, _) => FixedSized(width.width) case _: BytesEosType => DynamicSized case blt: BytesLimitType => blt.size.evaluateIntConst match { - case Some(x) => FixedSized(x.toInt) + case Some(x) => FixedSized(x.toInt * 8) case None => DynamicSized } case _: BytesTerminatedType => DynamicSized - case StrFromBytesType(basedOn, _, _) => dataTypeByteSize(basedOn) - case utb: UserTypeFromBytes => dataTypeByteSize(utb.bytes) - case cutb: CalcUserTypeFromBytes => dataTypeByteSize(cutb.bytes) - case st: SwitchType => DynamicSized // FIXME: it's really possible get size if st.hasSize + case CalcBytesType => DynamicSized + + case StrFromBytesType(basedOn, _, _) => dataTypeBitsSize(basedOn) + case CalcStrType => DynamicSized + + case SwitchType(_, cases, _, _) => cases.values.foldLeft[Option[Sized]](None) { + case (acc, dataType) => Some(acc match { + case None => dataTypeBitsSize(dataType) + case Some(s) => s.or(dataTypeBitsSize(dataType)) + }) + }.getOrElse(FixedSized(0)) + + case OwnedKaitaiStreamType | KaitaiStreamType => DynamicSized + + // TODO: Add special type or attribute to ArrayType for arrays of known size + case _: ArrayType => DynamicSized + case AnyType => DynamicSized } } } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala index 49cf61ef2..a167f3f9c 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala @@ -102,20 +102,10 @@ class TypeDetector(provider: TypeProvider) { } case Ast.expr.Subscript(container: Ast.expr, idx: Ast.expr) => detectType(container) match { - case ArrayTypeInStream(elType: DataType) => + case arr: ArrayType => detectType(idx) match { - case _: IntType => elType.asNonOwning( - elType match { - case ct: ComplexDataType => ct.isOwning - case _ => false - } - ) - case idxType => throw new TypeMismatchError(s"unable to index an array using $idxType") - } - case CalcArrayType(elType: DataType, _) => - detectType(idx) match { - case _: IntType => elType.asNonOwning( - elType match { + case _: IntType => arr.elType.asNonOwning( + arr.elType match { case ct: ComplexDataType => ct.isOwning case _ => false }