Merge branch 'master' into serialization

shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala

@@ -270,13 +270,9 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condIfFooter: Unit = fileFooter(null)
 
-  override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = {
+  override def condRepeatInitAttr(id: Identifier, dataType: DataType): Unit = {
     importList.add("System.Collections.Generic")
 
-    if (needRaw.level >= 1)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = new List<byte[]>();")
-    if (needRaw.level >= 2)
-      out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = new List<byte[]>();")
     out.puts(s"${privateMemberName(id)} = new ${kaitaiType2NativeType(ArrayTypeInStream(dataType))}();")
   }
 

shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala

@@ -386,87 +386,43 @@ class CppCompiler(
       outSrc.inc
     }
 
-    val needRaw = this.needRaw(attr.dataType)
-    val innerType = attr.dataType match {
-      case st: SwitchType => st.combinedType
-      case t => t
+    (ExtraAttrs.forAttr(attr, this) ++ List(attr)).foreach { (attr) =>
+      val innerType = attr.dataType match {
+        case st: SwitchType => combineSwitchType(st)
+        case t => t
+      }
+      destructMember(attr.id, innerType, attr.isArray)
     }
 
-    destructMember(id, innerType, attr.isArray, needRaw)
-
     if (checks.nonEmpty) {
       outSrc.dec
       outSrc.puts("}")
     }
   }
 
-  def destructMember(id: Identifier, innerType: DataType, isArray: Boolean, needRaw: NeedRaw): Unit = {
-    def destructWithSafeguardHeader(ptr: String): Unit = {
-      outSrc.puts(s"if ($ptr) {")
-      outSrc.inc
-    }
-    def destructWithSafeguardFooter(ptr: String): Unit = {
-      outSrc.puts(s"delete $ptr; $ptr = $nullPtr;")
-      outSrc.dec
-      outSrc.puts("}")
-    }
-    def destructWithSafeguardSimple(ptr: String): Unit = {
-      destructWithSafeguardHeader(ptr)
-      destructWithSafeguardFooter(ptr)
-    }
-    if (config.cppConfig.pointers == CppRuntimeConfig.RawPointers) {
-      if (isArray) {
-        // raw is std::vector<string>*, no need to delete its contents, but we
-        // need to clean up the vector pointer itself
-        if (needRaw.level >= 1) {
-          destructWithSafeguardSimple(privateMemberName(RawIdentifier(id)))
-
-          // IO is std::vector<kstream*>*, needs destruction of both members
-          // and the vector pointer itself
-          if (needRaw.hasIO) {
-            val ioVar = privateMemberName(IoStorageIdentifier(RawIdentifier(id)))
-            destructWithSafeguardHeader(ioVar)
-            destructVector(s"$kstreamName*", ioVar)
-            destructWithSafeguardFooter(ioVar)
-          }
-        }
-        if (needRaw.level >= 2) {
-          // m__raw__raw_* is also std::vector<string>*, we just clean up the vector pointer
-          destructWithSafeguardSimple(privateMemberName(RawIdentifier(RawIdentifier(id))))
-        }
-
-        val arrVar = privateMemberName(id)
-        destructWithSafeguardHeader(arrVar)
+  def destructMember(id: Identifier, innerType: DataType, isArray: Boolean): Unit = {
+    if (config.cppConfig.pointers != CppRuntimeConfig.RawPointers)
+      return
 
-        // main member contents
-        if (needsDestruction(innerType)) {
-          // C++ specific substitution: AnyType results from generic struct + raw bytes
-          // so we would assume that only generic struct needs to be cleaned up
-          val realType = innerType match {
-            case AnyType => KaitaiStructType
-            case _ => innerType
-          }
+    val ptr = privateMemberName(id)
+    val innerNeedsDestruct = needsDestruction(innerType)
 
-          destructVector(kaitaiType2NativeType(realType), arrVar)
-        }
+    if (!isArray && !innerNeedsDestruct)
+      return
 
-        // main member is a std::vector of something, always needs destruction
-        destructWithSafeguardFooter(arrVar)
-      } else {
-        // raw is just a string, no need to cleanup => we ignore `needRaw.hasRaw`
+    outSrc.puts(s"if ($ptr) {")
+    outSrc.inc
 
-        // but needRaw.hasIO is important
-        if (needRaw.hasIO)
-          destructWithSafeguardSimple(privateMemberName(IoStorageIdentifier(RawIdentifier(id))))
+    if (isArray && innerNeedsDestruct)
+      destructVector(kaitaiType2NativeType(innerType), ptr)
 
-        if (needsDestruction(innerType))
-          destructWithSafeguardSimple(privateMemberName(id))
-      }
-    }
+    outSrc.puts(s"delete $ptr; $ptr = $nullPtr;")
+    outSrc.dec
+    outSrc.puts("}")
   }
 
   def needsDestruction(t: DataType): Boolean = t match {
-    case _: UserType | _: ArrayTypeInStream | KaitaiStructType | AnyType => true
+    case _: UserType | _: ArrayTypeInStream | KaitaiStructType | AnyType | OwnedKaitaiStreamType => true
     case _ => false
   }
 
@@ -615,18 +571,9 @@ class CppCompiler(
     outSrc.puts("}")
   }
 
-  override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = {
+  override def condRepeatInitAttr(id: Identifier, dataType: DataType): Unit = {
     importListHdr.addSystem("vector")
 
-    if (needRaw.level >= 1) {
-      outSrc.puts(s"${privateMemberName(RawIdentifier(id))} = ${newVector(CalcBytesType)};")
-      if (needRaw.hasIO) {
-        outSrc.puts(s"${privateMemberName(IoStorageIdentifier(RawIdentifier(id)))} = ${newVector(OwnedKaitaiStreamType)};")
-      }
-    }
-    if (needRaw.level >= 2) {
-      outSrc.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = ${newVector(CalcBytesType)};")
-    }
     outSrc.puts(s"${privateMemberName(id)} = ${newVector(dataType)};")
   }
 
@@ -908,14 +855,24 @@ class CppCompiler(
     outSrc.puts(s"return ${nonOwningPointer(instName, attrType)};")
 
   override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = {
-    if (config.readStoresPos)
+    if (attrDebugNeeded(instName))
       attrDebugStart(instName, dataType, None, NoRepeat)
     val valExpr = expression(value)
     val isOwningInExpr = dataType match {
       case ct: ComplexDataType => ct.isOwningInExpr
       case _ => false
     }
-    handleAssignmentSimple(instName, if (isOwningInExpr) s"$valExpr.get()" else valExpr)
+    val valExprConverted = if (isOwningInExpr) {
+      config.cppConfig.pointers match {
+        case RawPointers =>
+          valExpr
+        case UniqueAndRawPointers =>
+          s"$valExpr.get()"
+      }
+    } else {
+      valExpr
+    }
+    handleAssignmentSimple(instName, valExprConverted)
   }
 
   override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = {

shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala

@@ -274,7 +274,7 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.inc
   }
 
-  override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = {
+  override def condRepeatInitAttr(id: Identifier, dataType: DataType): Unit = {
     // slices don't have to be manually initialized in Go: the built-in append()
     // function works even on `nil` slices (https://go.dev/tour/moretypes/15)
   }

shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala

@@ -501,10 +501,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def attrDebugStart(attrId: Identifier, attrType: DataType, ios: Option[String], rep: RepeatSpec): Unit = {
     ios.foreach { (io) =>
-      val name = attrId match {
-        case _: RawIdentifier | _: SpecialIdentifier => return
-        case _ => idToStr(attrId)
-      }
+      val name = idToStr(attrId)
       rep match {
         case NoRepeat =>
           out.puts("_attrStart.put(\"" + name + "\", " + io + ".pos());")
@@ -514,11 +511,12 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     }
   }
 
+  override def attrDebugArrInit(attrName: Identifier, attrType: DataType): Unit = {
+    // no _debug[$name]['arr'] initialization needed in Java
+  }
+
   override def attrDebugEnd(attrId: Identifier, attrType: DataType, io: String, rep: RepeatSpec): Unit = {
-    val name = attrId match {
-      case _: RawIdentifier | _: SpecialIdentifier => return
-      case _ => idToStr(attrId)
-    }
+    val name = idToStr(attrId)
     rep match {
       case NoRepeat =>
         out.puts("_attrEnd.put(\"" + name + "\", " + io + ".pos());")
@@ -547,30 +545,8 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.inc
   }
 
-  override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = {
-    if (needRaw.level >= 1)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>();")
-    if (needRaw.level >= 2)
-      out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = new ArrayList<byte[]>();")
-    if (config.readWrite) {
-      dataType match {
-        case utb: UserTypeFromBytes =>
-          if (writeNeedsOuterSize(utb.bytes))
-            out.puts(s"${privateMemberName(OuterSizeIdentifier(id))} = new ArrayList<Integer>();")
-          if (writeNeedsInnerSize(utb.bytes))
-            out.puts(s"${privateMemberName(InnerSizeIdentifier(id))} = new ArrayList<Integer>();")
-        case _ => // do nothing
-      }
-    }
+  override def condRepeatInitAttr(id: Identifier, dataType: DataType): Unit =
     out.puts(s"${privateMemberName(id)} = new ${kaitaiType2JavaType(ArrayTypeInStream(dataType))}();")
-  }
-
-  override def condRepeatCommonWriteInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = {
-    if (needRaw.level >= 1)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>();")
-    if (needRaw.level >= 2)
-      out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = new ArrayList<byte[]>();")
-  }
 
   override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = {
     out.puts("{")
@@ -688,9 +664,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     castIfNeeded(expr, dataType, assignType)
   }
 
-  override def createSubstreamFixedSize(id: Identifier, sizeExpr: Ast.expr, io: String): String = {
+  override def createSubstreamFixedSize(id: Identifier, blt: BytesLimitType, io: String, rep: RepeatSpec, defEndian: Option[FixedEndian]): String = {
     val ioName = idToStr(IoStorageIdentifier(id))
-    handleAssignmentTempVar(KaitaiStreamType, ioName, s"$io.substream(${translator.translate(sizeExpr)});")
+    handleAssignmentTempVar(KaitaiStreamType, ioName, s"$io.substream(${translator.translate(blt.size)})")
     ioName
   }
 

shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala

@@ -254,9 +254,6 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts(s"$io.alignToByte();")
 
   override def attrDebugStart(attrId: Identifier, attrType: DataType, io: Option[String], rep: RepeatSpec): Unit = {
-    if (!attrDebugNeeded(attrId))
-      return
-
     val debugName = attrDebugName(attrId, rep, false)
 
     val ioProps = io match {
@@ -272,9 +269,10 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts(s"$debugName = { $ioProps${if (ioProps != "" && enumNameProps != "") ", " else ""}$enumNameProps };")
   }
 
+  override def attrDebugArrInit(id: Identifier, attrType: DataType): Unit =
+    out.puts(s"this._debug.${idToStr(id)}.arr = [];")
+
   override def attrDebugEnd(attrId: Identifier, attrType: DataType, io: String, rep: RepeatSpec): Unit = {
-    if (!attrDebugNeeded(attrId))
-      return
     val debugName = attrDebugName(attrId, rep, true)
 
     out.puts(s"$debugName.end = $io.pos;")
@@ -291,15 +289,8 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts("}")
   }
 
-  override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = {
-    if (needRaw.level >= 1)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = [];")
-    if (needRaw.level >= 2)
-      out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = [];")
+  override def condRepeatInitAttr(id: Identifier, dataType: DataType): Unit =
     out.puts(s"${privateMemberName(id)} = [];")
-    if (config.readStoresPos)
-      out.puts(s"this._debug.${idToStr(id)}.arr = [];")
-  }
 
   override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = {
     out.puts("var i = 0;")
@@ -578,12 +569,6 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts("}")
   }
 
-  private
-  def attrDebugNeeded(attrId: Identifier) = attrId match {
-    case _: NamedIdentifier | _: NumberedIdentifier | _: InstanceIdentifier => true
-    case _: RawIdentifier | _: SpecialIdentifier => false
-  }
-
   def attrDebugName(attrId: Identifier, rep: RepeatSpec, end: Boolean) = {
     val arrIndexExpr = rep match {
       case NoRepeat => ""

shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala

@@ -1,7 +1,7 @@
 package io.kaitai.struct.languages
 
 import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils}
-import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError, ValidationNotEqualError, NeedRaw}
+import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError, ValidationNotEqualError}
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format._
@@ -158,11 +158,7 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts("end")
   }
 
-  override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = {
-    if (needRaw.level >= 1)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = {}")
-    if (needRaw.level >= 2)
-      out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = {}")
+  override def condRepeatInitAttr(id: Identifier, dataType: DataType): Unit = {
     out.puts(s"${privateMemberName(id)} = {}")
   }
 

shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala

@@ -149,7 +149,7 @@ class NimCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     universalFooter
   }
 
-  override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = {
+  override def condRepeatInitAttr(id: Identifier, dataType: DataType): Unit = {
     // sequences don't have to be manually initialized in Nim - they're automatically initialized as
     // empty sequences (see https://narimiran.github.io/nim-basics/#_result_variable)
   }

shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala

@@ -1,7 +1,7 @@
 package io.kaitai.struct.languages
 
 import io.kaitai.struct.datatype.DataType._
-import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian, KSError, NeedRaw, UndecidedEndiannessError}
+import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian, KSError, UndecidedEndiannessError}
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format.{NoRepeat, RepeatEos, RepeatExpr, RepeatSpec, _}
 import io.kaitai.struct.languages.components._
@@ -263,13 +263,8 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.inc
   }
 
-  override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = {
-    if (needRaw.level >= 1)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = [];")
-    if (needRaw.level >= 2)
-      out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = [];")
+  override def condRepeatInitAttr(id: Identifier, dataType: DataType): Unit =
     out.puts(s"${privateMemberName(id)} = [];")
-  }
 
   override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = {
     out.puts("$i = 0;")