Serialization

jvm/src/main/scala/io/kaitai/struct/JavaMain.scala

@@ -57,6 +57,10 @@ object JavaMain {
         }
       }
 
+      opt[Unit]('w', "read-write") action { (x, c) =>
+        c.copy(runtime = c.runtime.copy(readWrite = true, autoRead = false))
+      } text("generate read-write support in classes (implies `--no-auto-read --zero-copy-substream false`, Java and Python only, default: read-only)")
+
       opt[File]('d', "outdir") valueName("<directory>") action { (x, c) =>
         c.copy(outDir = x)
       } text("output directory (filenames will be auto-generated); on Unix-like shells, the short form `-d` requires arguments to be preceded by `--`")
@@ -172,7 +176,13 @@ object JavaMain {
       version("version") text("output version information and exit")
     }
 
-    parser.parse(args, CLIConfig())
+    parser.parse(args, CLIConfig()).map { c =>
+      if (c.runtime.readWrite) {
+        c.copy(runtime = c.runtime.copy(zeroCopySubstream = false))
+      } else {
+        c
+      }
+    }
   }
 
   /**

jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala

@@ -693,7 +693,7 @@ class TranslatorSpec extends AnyFunSuite {
       CppCompiler -> new CppTranslator(tp, new CppImportList(), new CppImportList(), RuntimeConfig()),
       CSharpCompiler -> new CSharpTranslator(tp, new ImportList()),
       GoCompiler -> new GoTranslator(goOutput, tp, new ImportList()),
-      JavaCompiler -> new JavaTranslator(tp, new ImportList()),
+      JavaCompiler -> new JavaTranslator(tp, new ImportList(), RuntimeConfig()),
       JavaScriptCompiler -> new JavaScriptTranslator(tp),
       LuaCompiler -> new LuaTranslator(tp, new ImportList()),
       PerlCompiler -> new PerlTranslator(tp, new ImportList()),

shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala

@@ -78,6 +78,15 @@ class ClassCompiler(
     // Read method(s)
     compileEagerRead(curClass.seq, curClass.meta.endian)
 
+    compileFetchInstancesProc(curClass.seq ++ curClass.instances.values.collect {
+      case inst: AttrLikeSpec => inst
+    })
+
+    if (config.readWrite) {
+      compileWrite(curClass.seq, curClass.instances, curClass.meta.endian)
+      compileCheck(curClass.seq)
+    }
+
     // Destructor
     compileDestructor(curClass)
 
@@ -132,6 +141,15 @@ class ClassCompiler(
     )
     compileInit(curClass)
     curClass.instances.foreach { case (instName, _) => lang.instanceClear(instName) }
+    if (config.readWrite) {
+      curClass.instances.foreach { case (instName, instSpec) =>
+        instSpec match {
+          case _: ParseInstanceSpec =>
+            lang.instanceWriteFlagInit(instName)
+          case _: ValueInstanceSpec => // do nothing
+        }
+      }
+    }
     if (lang.config.autoRead)
       lang.runRead(curClass.name)
     lang.classConstructorFooter
@@ -217,6 +235,9 @@ class ClassCompiler(
         attr.isNullable
       }
       lang.attributeReader(attr.id, attr.dataTypeComposite, isNullable)
+      if (config.readWrite) {
+        lang.attributeSetter(attr.id, attr.dataTypeComposite, isNullable)
+      }
     }
   }
 
@@ -236,25 +257,45 @@ class ClassCompiler(
   def compileEagerRead(seq: List[AttrSpec], endian: Option[Endianness]): Unit = {
     endian match {
       case None | Some(_: FixedEndian) =>
-        compileSeqProc(seq, None)
+        compileSeqReadProc(seq, None)
       case Some(ce: CalcEndian) =>
         lang.readHeader(None, false)
         compileCalcEndian(ce)
         lang.runReadCalc()
         lang.readFooter()
 
-        compileSeqProc(seq, Some(LittleEndian))
-        compileSeqProc(seq, Some(BigEndian))
+        compileSeqReadProc(seq, Some(LittleEndian))
+        compileSeqReadProc(seq, Some(BigEndian))
       case Some(InheritedEndian) =>
         lang.readHeader(None, false)
         lang.runReadCalc()
         lang.readFooter()
 
-        compileSeqProc(seq, Some(LittleEndian))
-        compileSeqProc(seq, Some(BigEndian))
+        compileSeqReadProc(seq, Some(LittleEndian))
+        compileSeqReadProc(seq, Some(BigEndian))
     }
   }
 
+  def compileWrite(seq: List[AttrSpec], instances: Map[InstanceIdentifier, InstanceSpec], endian: Option[Endianness]): Unit = {
+    endian match {
+      case None | Some(_: FixedEndian) =>
+        compileSeqWriteProc(seq, instances, None)
+      case Some(CalcEndian(_, _)) | Some(InheritedEndian) =>
+        lang.writeHeader(None, false)
+        lang.runWriteCalc()
+        lang.writeFooter()
+
+        compileSeqWriteProc(seq, instances, Some(LittleEndian))
+        compileSeqWriteProc(seq, instances, Some(BigEndian))
+    }
+  }
+
+  def compileCheck(seq: List[AttrSpec]): Unit = {
+    lang.checkHeader()
+    compileSeqCheck(seq)
+    lang.checkFooter()
+  }
+
   val IS_LE_ID = SpecialIdentifier("_is_le")
 
   /**
@@ -276,18 +317,31 @@ class ClassCompiler(
     * @param seq sequence of attributes
     * @param defEndian default endianness
     */
-  def compileSeqProc(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = {
+  def compileSeqReadProc(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = {
     lang.readHeader(defEndian, seq.isEmpty)
-    compileSeq(seq, defEndian)
+    compileSeqRead(seq, defEndian)
     lang.readFooter()
   }
 
+  def compileFetchInstancesProc(attrs: List[AttrLikeSpec]) = {
+    lang.fetchInstancesHeader()
+    compileFetchInstances(attrs)
+    lang.fetchInstancesFooter()
+  }
+
+  def compileSeqWriteProc(seq: List[AttrSpec], instances: Map[InstanceIdentifier, InstanceSpec], defEndian: Option[FixedEndian]) = {
+    lang.writeHeader(defEndian, !instances.values.exists(i => i.isInstanceOf[ParseInstanceSpec]) && seq.isEmpty)
+    compileSetInstanceWriteFlags(instances)
+    compileSeqWrite(seq, defEndian)
+    lang.writeFooter()
+  }
+
   /**
     * Compiles seq reading method body (only reading statements).
     * @param seq sequence of attributes
     * @param defEndian default endianness
     */
-  def compileSeq(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = {
+  def compileSeqRead(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = {
     var wasUnaligned = false
     seq.foreach { (attr) =>
       val nowUnaligned = isUnalignedBits(attr.dataType)
@@ -298,6 +352,39 @@ class ClassCompiler(
     }
   }
 
+  def compileFetchInstances(attrs: List[AttrLikeSpec]): Unit = {
+    attrs.foreach { (attr) =>
+      lang.attrFetchInstances(attr, attr.id)
+    }
+  }
+
+  def compileSetInstanceWriteFlags(instances: Map[InstanceIdentifier, InstanceSpec]) = {
+    instances.foreach { case (instName, instSpec) =>
+      instSpec match {
+        case _: ParseInstanceSpec =>
+          lang.instanceSetWriteFlag(instName)
+        case _: ValueInstanceSpec => // do nothing
+      }
+    }
+  }
+
+  def compileSeqWrite(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = {
+    var wasUnaligned = false
+    seq.foreach { (attr) =>
+      val nowUnaligned = isUnalignedBits(attr.dataType)
+      if (wasUnaligned && !nowUnaligned)
+        lang.alignToByte(lang.normalIO)
+      lang.attrWrite(attr, attr.id, defEndian)
+      wasUnaligned = nowUnaligned
+    }
+  }
+
+  def compileSeqCheck(seq: List[AttrSpec]) = {
+    seq.foreach { (attr) =>
+      lang.attrCheck(attr, attr.id)
+    }
+  }
+
   /**
     * Compiles all enums specifications for a given type.
     * @param curClass current type to generate code for
@@ -329,6 +416,12 @@ class ClassCompiler(
     lang.instanceHeader(className, instName, dataType, instSpec.isNullable)
     if (lang.innerDocstrings)
       compileInstanceDoc(instName, instSpec)
+    if (config.readWrite)
+      instSpec match {
+        case _: ParseInstanceSpec =>
+          lang.instanceCheckWriteFlagAndWrite(instName)
+        case _: ValueInstanceSpec => // do nothing
+      }
     lang.instanceCheckCacheAndReturn(instName, dataType)
 
     instSpec match {
@@ -343,6 +436,22 @@ class ClassCompiler(
 
     lang.instanceReturn(instName, dataType)
     lang.instanceFooter
+
+    if (config.readWrite)
+      instSpec match {
+        case pi: ParseInstanceSpec =>
+          lang.attributeSetter(instName, dataType, instSpec.isNullable)
+          lang.instanceToWriteSetter(instName)
+          lang.writeInstanceHeader(instName)
+          lang.attrWrite(pi, instName, endian)
+          lang.writeInstanceFooter
+
+          lang.checkInstanceHeader(instName)
+          lang.attrCheck(pi, instName)
+          lang.checkInstanceFooter
+        case _: ValueInstanceSpec =>
+          lang.instanceInvalidate(instName)
+      }
   }
 
   def compileInstanceDeclaration(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
@@ -355,6 +464,12 @@ class ClassCompiler(
       instSpec.isNullable
     }
     lang.instanceDeclaration(instName, instSpec.dataTypeComposite, isNullable)
+    if (config.readWrite)
+      instSpec match {
+        case _: ParseInstanceSpec =>
+          lang.instanceWriteFlagDeclaration(instName)
+        case _: ValueInstanceSpec => // do nothing
+      }
   }
 
   def compileEnum(curClass: ClassSpec, enumColl: EnumSpec): Unit =

shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala

@@ -57,6 +57,33 @@ class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends
       case NamedIdentifier(name) => return determineType(inClass, name)
       case InstanceIdentifier(name) => return determineType(inClass, name)
       case SpecialIdentifier(name) => return determineType(inClass, name)
+      case RawIdentifier(innerId) => {
+        val innerType = determineType(innerId)
+        val (isArray, itemType: DataType) = innerType match {
+          case at: ArrayType => (true, at.elType)
+          case t => (false, t)
+        }
+        val singleType: DataType = itemType match {
+          case st: SwitchType => st.cases.collectFirst {
+            case (_, caseType)
+              if caseType.isInstanceOf[BytesType]
+              || caseType.isInstanceOf[UserTypeFromBytes] => caseType
+          }.get
+          case t => t
+        }
+        /** see [[languages.components.ExtraAttrs$]] for possible types */
+        val bytesType = singleType match {
+          case bt: BytesType => bt
+          case utb: UserTypeFromBytes => utb.bytes
+        }
+        return if (isArray) ArrayTypeInStream(bytesType) else bytesType
+      }
+      case OuterSizeIdentifier(innerId) =>
+        val singleType = CalcIntType
+        return if (determineType(innerId).isInstanceOf[ArrayType]) ArrayTypeInStream(singleType) else singleType
+      case InnerSizeIdentifier(innerId) =>
+        val singleType = CalcIntType
+        return if (determineType(innerId).isInstanceOf[ArrayType]) ArrayTypeInStream(singleType) else singleType
       case _ => // do nothing
     }
     throw new FieldNotFoundError(attrId.humanReadable, inClass)

shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala

@@ -37,7 +37,7 @@ abstract class DocClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) ext
     classHeader(curClass)
 
     // Sequence
-    compileSeq(curClass)
+    compileSeqRead(curClass)
 
     // Instances
     curClass.instances.foreach { case (_, instSpec) =>
@@ -58,7 +58,7 @@ abstract class DocClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) ext
     classFooter(curClass)
   }
 
-  def compileSeq(curClass: ClassSpec): Unit = {
+  def compileSeqRead(curClass: ClassSpec): Unit = {
     seqHeader(curClass)
 
     CalculateSeqSizes.forEachSeqAttr(curClass, (attr, seqPos, sizeElement, sizeContainer) => {

shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala

@@ -59,7 +59,7 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends
     out.puts
 
     // Sequence
-    compileSeq(className, curClass)
+    compileSeqRead(className, curClass)
 
     curClass.instances.foreach { case (instName, instSpec) =>
       instSpec match {
@@ -84,7 +84,7 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends
     out.puts("}")
   }
 
-  def compileSeq(className: List[String], curClass: ClassSpec): Unit = {
+  def compileSeqRead(className: List[String], curClass: ClassSpec): Unit = {
     tableStart(className, "seq")
 
     CalculateSeqSizes.forEachSeqAttr(curClass, (attr, seqPos, _, _) => {

shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala

@@ -51,25 +51,25 @@ class NimClassCompiler(
   override def compileEagerRead(seq: List[AttrSpec], endian: Option[Endianness]): Unit = {
     endian match {
       case None | Some(_: FixedEndian) =>
-        compileSeqProc(seq, None)
+        compileSeqReadProc(seq, None)
       case Some(ce: CalcEndian) =>
-        compileSeqProc(seq, Some(LittleEndian))
-        compileSeqProc(seq, Some(BigEndian))
+        compileSeqReadProc(seq, Some(LittleEndian))
+        compileSeqReadProc(seq, Some(BigEndian))
         lang.readHeader(None, false)
         compileCalcEndian(ce)
         lang.runReadCalc()
         lang.readFooter()
       case Some(InheritedEndian) =>
-        compileSeqProc(seq, Some(LittleEndian))
-        compileSeqProc(seq, Some(BigEndian))
+        compileSeqReadProc(seq, Some(LittleEndian))
+        compileSeqReadProc(seq, Some(BigEndian))
         lang.readHeader(None, false)
         lang.runReadCalc()
         lang.readFooter()
     }
   }
 
   // Must override just to add attribute docstrings
-  override def compileSeq(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = {
+  override def compileSeqRead(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = {
     var wasUnaligned = false
     seq.foreach { (attr) =>
       val nowUnaligned = isUnalignedBits(attr.dataType)
@@ -153,7 +153,7 @@ class NimClassCompiler(
   def compileTypesRec(curClass: ClassSpec): Unit = {
     curClass.types.foreach { case (_, subClass) => compileTypes(subClass) }
   }
-  
+
 //  def compileEnumConstants(curClass: ClassSpec): Unit = {
 //    provider.nowClass = curClass
 //    curClass.enums.foreach { case(_, enumColl) => {
@@ -203,4 +203,3 @@ class NimClassCompiler(
   }
 
 }
-

shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala

@@ -99,6 +99,7 @@ case class RuntimeConfig(
   readStoresPos: Boolean = false,
   opaqueTypes: Boolean = false,
   zeroCopySubstream: Boolean = true,
+  readWrite: Boolean = false,
   cppConfig: CppRuntimeConfig = CppRuntimeConfig(),
   goPackage: String = "",
   java: JavaRuntimeConfig = JavaRuntimeConfig(),

shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala

@@ -66,7 +66,7 @@ class RustClassCompiler(
 
     lang.readHeader(defEndian, false)
 
-    compileSeq(curClass.seq, defEndian)
+    compileSeqRead(curClass.seq, defEndian)
     lang.classConstructorFooter
   }
 

shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala

@@ -57,7 +57,7 @@ object Endianness {
   def fromString(s: Option[String], defaultEndian: Option[Endianness], dt: String, path: List[String]): Option[FixedEndian] = s match {
     case Some("le") => Some(LittleEndian)
     case Some("be") => Some(BigEndian)
-    case None =>
+    case _ =>
       defaultEndian match {
         case Some(e: FixedEndian) => Some(e)
         case Some(_: CalcEndian) | Some(InheritedEndian) => None // to be overridden during compile