From 662ddf9880af0b51f5f220ecca5f2fed86f6bf4b Mon Sep 17 00:00:00 2001 From: Mingun Date: Thu, 26 Nov 2020 00:58:39 +0500 Subject: [PATCH 1/4] Seal all abstract classes in DataType.scala Part of them already was sealed, so there is no reasons to not seal other --- .../scala/io/kaitai/struct/datatype/DataType.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala index 96e6f4036..81bd6de4a 100644 --- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala +++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala @@ -21,7 +21,7 @@ sealed trait DataType { * Kaitai Struct type system. */ object DataType { - abstract class IntWidth(val width: Int) + abstract sealed class IntWidth(val width: Int) case object Width1 extends IntWidth(1) case object Width2 extends IntWidth(2) case object Width4 extends IntWidth(4) @@ -53,7 +53,7 @@ object DataType { case class BitsType1(bitEndian: BitEndianness) extends BooleanType case class BitsType(width: Int, bitEndian: BitEndianness) extends IntType - abstract class FloatType extends NumericType + abstract sealed class FloatType extends NumericType case object CalcFloatType extends FloatType case class FloatMultiType(width: IntWidth, endian: Option[FixedEndian]) extends FloatType with ReadableType { override def apiCall(defEndian: Option[FixedEndian]): String = { @@ -66,7 +66,7 @@ object DataType { def process: Option[ProcessExpr] } - abstract class BytesType extends DataType with Processing + abstract sealed class BytesType extends DataType with Processing case object CalcBytesType extends BytesType { override def process = None } @@ -91,7 +91,7 @@ object DataType { override val process: Option[ProcessExpr] ) extends BytesType - abstract class StrType extends DataType + abstract sealed class StrType extends DataType case object CalcStrType extends StrType /** * A type that have the `str` and `strz` built-in Kaitai types. @@ -148,7 +148,7 @@ object DataType { def isOwningInExpr: Boolean = false } - abstract class StructType extends ComplexDataType + abstract sealed class StructType extends ComplexDataType /** * Common abstract ancestor for all types which can treated as "user types". @@ -158,7 +158,7 @@ object DataType { * @param forcedParent optional parent enforcement expression * @param args parameters passed into this type as extra arguments */ - abstract class UserType( + abstract sealed class UserType( val name: List[String], val forcedParent: Option[Ast.expr], var args: Seq[Ast.expr] From 519f06570b52a7243435421a6036abce82b7fd35 Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 27 Nov 2020 23:13:31 +0500 Subject: [PATCH 2/4] Unify processing of array types in TypeDetector Both ArrayTypeInStream and CalcArrayType the only existing successors of the sealed class TypeDetector so we can do that simplification --- .../kaitai/struct/translators/TypeDetector.scala | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala index 49cf61ef2..a167f3f9c 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala @@ -102,20 +102,10 @@ class TypeDetector(provider: TypeProvider) { } case Ast.expr.Subscript(container: Ast.expr, idx: Ast.expr) => detectType(container) match { - case ArrayTypeInStream(elType: DataType) => + case arr: ArrayType => detectType(idx) match { - case _: IntType => elType.asNonOwning( - elType match { - case ct: ComplexDataType => ct.isOwning - case _ => false - } - ) - case idxType => throw new TypeMismatchError(s"unable to index an array using $idxType") - } - case CalcArrayType(elType: DataType, _) => - detectType(idx) match { - case _: IntType => elType.asNonOwning( - elType match { + case _: IntType => arr.elType.asNonOwning( + arr.elType match { case ct: ComplexDataType => ct.isOwning case _ => false } From bd7e979f35bdded0237339dfcb9a503ba5ba68dc Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 27 Nov 2020 23:11:01 +0500 Subject: [PATCH 3/4] Document DataTypes --- .../io/kaitai/struct/datatype/DataType.scala | 145 +++++++++++++++++- 1 file changed, 144 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala index 81bd6de4a..b8b07b035 100644 --- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala +++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala @@ -35,14 +35,58 @@ object DataType { def apiCall(defEndian: Option[FixedEndian]): String } + /** A generic number type. */ abstract sealed class NumericType extends DataType + /** A generic boolean type. */ abstract sealed class BooleanType extends DataType + /** A generic integer type. */ abstract sealed class IntType extends NumericType + /** + * An integer type that occupies undecided number of bytes in the stream. + * + * If it possible to determine, the more narrow `Int1Type` will be inferred + * for an expression instead of this type. + * + * Represents a type of the following constructions: + * + * - `sizeof<>` and `bitsizeof<>` built-in operators + * - `_sizeof` special property of fields and self object + * - `size` and `pos` properties of streams + * - `to_i` result when converting other types (strings, floats, enums, and booleans) to integers + * - `length` and `size` properties of arrays + * - `length` property of strings + * - `_index` context variable + */ case object CalcIntType extends IntType + /** + * An integer type that fit into the byte and therefore can represent element + * of the byte array. + * + * Parameters have this type when it's declared with `type: u1` or `type: s1`. + * + * Represents a type of the following constructions: + * + * - `Int1Type(true)` -- a constant in the [0..127] range + * - `Int1Type(false)` -- a constant in the [128..255] range + * - element type of byte arrays + * - `first`, `last`, `min`, and `max` properties of byte arrays + * - result of the `[]` operator of byte arrays + * + * @param signed Determines if most significant bit of number contains sign + */ case class Int1Type(signed: Boolean) extends IntType with ReadableType { override def apiCall(defEndian: Option[FixedEndian]): String = if (signed) "s1" else "u1" } + /** + * An integer type that occupies some predefined size in bytes in the stream. + * + * Parameters have this type when it's declared with `type: u` or `type: s`. + * + * @param signed Determines if most significant bit of number contains sign + * @param width Size of type in bytes + * @param endian Byte order used to represent the number + */ case class IntMultiType(signed: Boolean, width: IntWidth, endian: Option[FixedEndian]) extends IntType with ReadableType { override def apiCall(defEndian: Option[FixedEndian]): String = { val ch1 = if (signed) 's' else 'u' @@ -50,11 +94,34 @@ object DataType { s"$ch1${width.width}${finalEnd.map(_.toSuffix).getOrElse("")}" } } + /** + * A boolean type that occupies one bit in the stream. + * + * Parameters have this type when it's declared with `type: b1`. + */ case class BitsType1(bitEndian: BitEndianness) extends BooleanType + /** + * An integer number type that occupies some predefined size in _bits_ in the stream. + * + * Parameters have this type when it's declared with `type: bX`. + * + * @param width Size of type in bits + * @param bitEndian Bit order inside of byte used to represent the number + */ case class BitsType(width: Int, bitEndian: BitEndianness) extends IntType + /** A generic floating-point number type. */ abstract sealed class FloatType extends NumericType + /** A floating-point number type that occupies undecided number of bytes in the stream. */ case object CalcFloatType extends FloatType + /** + * A floating-point number type that occupies some predefined size in bytes in the stream. + * + * Parameters have this type when it's declared with `type: fX`. + * + * @param width Size of type in bytes + * @param endian Byte order used to represent the number + */ case class FloatMultiType(width: IntWidth, endian: Option[FixedEndian]) extends FloatType with ReadableType { override def apiCall(defEndian: Option[FixedEndian]): String = { val finalEnd = endian.orElse(defEndian) @@ -66,7 +133,13 @@ object DataType { def process: Option[ProcessExpr] } + /** A generic raw bytes type. */ abstract sealed class BytesType extends DataType with Processing + /** + * A raw bytes type that occupies undecided number of bytes in the stream. + * + * Parameters have this type when it's declared with `type: bytes`. + */ case object CalcBytesType extends BytesType { override def process = None } @@ -91,7 +164,13 @@ object DataType { override val process: Option[ProcessExpr] ) extends BytesType + /** A generic string type. */ abstract sealed class StrType extends DataType + /** + * A pure string type that occupies undecided number of bytes in the stream. + * + * Parameters have this type when it's declared with `type: str`. + */ case object CalcStrType extends StrType /** * A type that have the `str` and `strz` built-in Kaitai types. @@ -112,6 +191,11 @@ object DataType { isEncodingDerived: Boolean, ) extends StrType + /** + * A boolean type that occupies undecided number of bytes in the stream. + * + * Parameters have this type when it's declared with `type: bool`. + */ case object CalcBooleanType extends BooleanType /** @@ -178,6 +262,7 @@ object DataType { cs.meta.isOpaque } } + /** User type which isn't restricted in size (i.e. without `size`, `terminator`, or `size-eos`). */ case class UserTypeInstream( _name: List[String], _forcedParent: Option[Ast.expr], @@ -190,6 +275,7 @@ object DataType { r } } + /** User type which is restricted in size either via `size`, `terminator`, or `size-eos`. */ case class UserTypeFromBytes( _name: List[String], _forcedParent: Option[Ast.expr], @@ -204,6 +290,12 @@ object DataType { r } } + /** + * Reference to the user type which isn't restricted in size (i.e. without + * `size`, `terminator`, or `size-eos`). + * + * Parameters have this type when it's declared with any non-built-in type. + */ case class CalcUserType( _name: List[String], _forcedParent: Option[Ast.expr], @@ -212,6 +304,10 @@ object DataType { ) extends UserType(_name, _forcedParent, _args) { override def isOwning = false } + /** + * Reference to the user type which restricted in size either via `size`, + * `terminator`, or `size-eos`. + */ case class CalcUserTypeFromBytes( _name: List[String], _forcedParent: Option[Ast.expr], @@ -223,12 +319,29 @@ object DataType { override def isOwning = false } + /** + * A generic collection type. + * + * @param elType Type of elements in the collection + */ abstract sealed class ArrayType(val elType: DataType) extends ComplexDataType - + /** + * An owned slice of array type. This type is used for holding data in attributes + * with `repeat` key. Number of elements in that type is unknown + * + * @param _elType Type of elements in the slice + */ case class ArrayTypeInStream(_elType: DataType) extends ArrayType(_elType) { override def isOwning: Boolean = true override def asNonOwning(isOwningInExpr: Boolean = false): CalcArrayType = CalcArrayType(elType, isOwningInExpr) } + /** + * A borrowed slice of an array. This type is used when array is passed as a + * parameter to the user type (parameter have type `bytes` or haven't any + * explicitly defined type). Number of elements in that type is unknown + * + * @param _elType Type of elements in the slice + */ case class CalcArrayType(_elType: DataType, override val isOwningInExpr: Boolean = false) extends ArrayType(_elType) { override def isOwning: Boolean = false } @@ -236,18 +349,46 @@ object DataType { /** Represents `_parent: false` expression which means that type explicitly has no parent. */ val USER_TYPE_NO_PARENT = Ast.expr.Bool(false) + /** + * A very generic type that can hold any other type. Used when type of expression + * is completely unknown to the compiler. + * + * Parameters have this type when it's declared with `type: any`. + */ case object AnyType extends DataType + + /** + * A type that can hold any Kaitai generated type. Can be used as common ancestor + * of `switch-on` types, when all alternative types is owned. + */ case object KaitaiStructType extends StructType { def isOwning = true override def asNonOwning(isOwningInExpr: Boolean = false): DataType = CalcKaitaiStructType(isOwningInExpr) } + /** + * A type that can hold any Kaitai generated type. Can be used as common ancestor + * of `switch-on` types, when at least one of the alternative types is borrowed. + * + * Parameters have this type when it's declared with `type: struct`. + */ case class CalcKaitaiStructType(override val isOwningInExpr: Boolean = false) extends StructType { def isOwning = false } + /** + * A type that hold and own Kaitai stream object. This type is used when a new IO object + * is allocated (i.e. when new sub-stream is created for types with `size`, `terminator`, + * or `size-eos: true` attributes). + */ case object OwnedKaitaiStreamType extends ComplexDataType { def isOwning = true override def asNonOwning(isOwningInExpr: Boolean = false): DataType = KaitaiStreamType } + /** + * A type that hold and borrow Kaitai stream object. This type is used + * when an IO object is passed as parameter to the user type. + * + * Parameters have this type when it's declared with `type: io`. + */ case object KaitaiStreamType extends ComplexDataType { def isOwning = false } @@ -459,6 +600,8 @@ object DataType { StrFromBytesType(bat, enc, arg.encoding.isEmpty) case _ => val typeWithArgs = Expressions.parseTypeRef(dt) + // if `size`, `terminator` and `size-eos: true` isn't defined, + // user type uses parent stream, otherwise creates an own stream if (arg.size.isEmpty && !arg.sizeEos && arg.terminator.isEmpty) { if (arg.process.isDefined) throw KSYParseError(s"user type '$dt': need 'size' / 'size-eos' / 'terminator' if 'process' is used", path).toException From 279ec246ad2ef0f54f1bc7306ff11d1aca83d882 Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 27 Nov 2020 23:55:27 +0500 Subject: [PATCH 4/4] Remove redundant dataTypeByteSize and fix non-exhaustive warnings in CalculateSeqSize --- .../struct/precompile/CalculateSeqSizes.scala | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala index 4d28c263b..89bc570c4 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala @@ -80,43 +80,46 @@ object CalculateSeqSizes { * Determines how many bits occupies given data type. * * @param dataType data type to analyze - * @return number of bits or None, if it's impossible to determine a priori + * @return number of bits or [[DynamicSized]], if it's impossible to determine a priori */ def dataTypeBitsSize(dataType: DataType): Sized = { dataType match { case BitsType1(_) => FixedSized(1) - case BitsType(width, _) => FixedSized(width) + case CalcBooleanType => DynamicSized + case EnumType(_, basedOn) => dataTypeBitsSize(basedOn) + case ut: UserTypeInstream => getSeqSize(ut.classSpec.get) - case _ => - dataTypeByteSize(dataType) match { - case FixedSized(x) => FixedSized(x * 8) - case otherSize => otherSize - } - } - } + case ut: UserTypeFromBytes => dataTypeBitsSize(ut.bytes) + case ut: CalcUserTypeFromBytes => dataTypeBitsSize(ut.bytes) + case _: StructType => DynamicSized + + case BitsType(width, _) => FixedSized(width) + case Int1Type(_) => FixedSized(8) + case IntMultiType(_, width, _) => FixedSized(width.width * 8) + case CalcIntType => DynamicSized + + case FloatMultiType(width, _) => FixedSized(width.width * 8) + case CalcFloatType => DynamicSized - /** - * Determines how many bytes occupies a given data type. - * - * @param dataType data type to analyze - * @return number of bytes or None, if it's impossible to determine a priori - */ - def dataTypeByteSize(dataType: DataType): Sized = { - dataType match { - case _: Int1Type => FixedSized(1) - case IntMultiType(_, width, _) => FixedSized(width.width) - case FloatMultiType(width, _) => FixedSized(width.width) case _: BytesEosType => DynamicSized case blt: BytesLimitType => blt.size.evaluateIntConst match { - case Some(x) => FixedSized(x.toInt) + case Some(x) => FixedSized(x.toInt * 8) case None => DynamicSized } case _: BytesTerminatedType => DynamicSized - case StrFromBytesType(basedOn, _, _) => dataTypeByteSize(basedOn) - case utb: UserTypeFromBytes => dataTypeByteSize(utb.bytes) - case cutb: CalcUserTypeFromBytes => dataTypeByteSize(cutb.bytes) + case CalcBytesType => DynamicSized + + case StrFromBytesType(basedOn, _, _) => dataTypeBitsSize(basedOn) + case CalcStrType => DynamicSized + case st: SwitchType => DynamicSized // FIXME: it's really possible get size if st.hasSize + + case OwnedKaitaiStreamType | KaitaiStreamType => DynamicSized + + // TODO: Add special type or attribute to ArrayType for arrays of known size + case _: ArrayType => DynamicSized + case AnyType => DynamicSized } } }