From 7d4769d2632e874bf55436067ca9a278d7760170 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 6 Jul 2022 09:06:28 +0800 Subject: [PATCH 001/153] kaitai + rust = latest development from bspeice + rebase + small changes https://github.com/kaitai-io/kaitai_struct_compiler/pull/164/files --- .../io/kaitai/struct/RustClassCompiler.scala | 1 + .../struct/languages/RustCompiler.scala | 1001 +++++++++-------- .../struct/translators/RustTranslator.scala | 64 +- 3 files changed, 598 insertions(+), 468 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala index a148abc43..2b0fb7773 100644 --- a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala @@ -75,6 +75,7 @@ class RustClassCompiler( curClass.instances.foreach { case (instName, instSpec) => compileInstance(curClass.name, instName, instSpec, curClass.meta.endian) } + lang.instanceFooter } override def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, endian: Option[Endianness]): Unit = { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 2a460879a..069f1ed1a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -1,614 +1,725 @@ package io.kaitai.struct.languages +import io.kaitai.struct._ import io.kaitai.struct.datatype.DataType._ -import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError, NeedRaw} +import io.kaitai.struct.datatype._ import io.kaitai.struct.exprlang.Ast -import io.kaitai.struct.format.{NoRepeat, RepeatEos, RepeatExpr, RepeatSpec, _} +import io.kaitai.struct.format._ import io.kaitai.struct.languages.components._ import io.kaitai.struct.translators.RustTranslator -import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig} class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) + with AllocateIOLocalVar + with EveryReadIsExpression + with FixedContentsUsingArrayByteLiteral with ObjectOrientedLanguage - with UpperCamelCaseClasses with SingleOutputFile - with AllocateIOLocalVar + with UpperCamelCaseClasses with UniversalFooter - with UniversalDoc - with FixedContentsUsingArrayByteLiteral - with EveryReadIsExpression { + with UniversalDoc { import RustCompiler._ + override val translator: RustTranslator = + new RustTranslator(typeProvider, config) + override def innerClasses = false override def innerEnums = false - override val translator: RustTranslator = new RustTranslator(typeProvider, config) - - override def universalFooter: Unit = { - out.dec - out.puts("}") - } - - override def outImports(topClass: ClassSpec) = - importList.toList.map((x) => s"use $x;").mkString("", "\n", "\n") - override def indent: String = " " + override def outFileName(topClassName: String): String = s"$topClassName.rs" + override def outImports(topClass: ClassSpec): String = + importList.toList + .map(i => s"#[allow(unused_imports)]\nuse $i;") + .mkString("", "\n", "\n") + override def fileHeader(topClassName: String): Unit = { outHeader.puts(s"// $headerComment") outHeader.puts - importList.add("std::option::Option") - importList.add("std::boxed::Box") - importList.add("std::io::Result") - importList.add("std::io::Cursor") - importList.add("std::vec::Vec") - importList.add("std::default::Default") - importList.add("kaitai_struct::KaitaiStream") - importList.add("kaitai_struct::KaitaiStruct") - - out.puts + importList.add( + "kaitai::{BytesReader, KError, KResult, KStream, KStruct, KStructUnit, TypedStack}" + ) + importList.add("kaitai::{kf32_max, kf64_max, kf32_min, kf64_min}") + importList.add("std::convert::{TryFrom, TryInto}") } - override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = { - val name = type2class(classSpec.name.last) - val pkg = type2classAbs(classSpec.name) + override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = + importList.add( + s"crate::${classSpec.name.last}::${type2class(classSpec.name.last)}" + ) - importList.add(s"$pkg::$name") - } + override def classHeader(name: List[String]): Unit = { + out.puts + out.puts("#[allow(non_camel_case_types)]") + out.puts("#[derive(Default, Debug, PartialEq)]") + out.puts(s"pub struct ${classTypeName(typeProvider.nowClass)} {") + out.inc - override def classHeader(name: List[String]): Unit = - classHeader(name, Some(kstructName)) + // Because we can't predict whether opaque types will need lifetimes as a type parameter, + // everyone gets a phantom data marker + //out.puts(s"_phantom: std::marker::PhantomData<&$streamLife ()>,") - def classHeader(name: List[String], parentClass: Option[String]): Unit = { - out.puts("#[derive(Default)]") - out.puts(s"pub struct ${type2class(name)} {") - } + typeProvider.nowClass.params.foreach { p => + // Make sure the parameter is imported if necessary + p.dataType match { + case u: UserType => if (u.isOpaque && u.classSpec.isDefined) opaqueClassDeclaration(u.classSpec.get) + case _ => () + } - override def classFooter(name: List[String]): Unit = universalFooter + // Declare parameters as if they were attributes + attributeDeclaration(p.id, p.dataType, isNullable = false) + } + } - override def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { - out.puts("}") - out.puts + // Intentional no-op; Rust has already ended the struct definition by the time we reach this + override def classFooter(name: List[String]): Unit = {} - out.puts(s"impl KaitaiStruct for ${type2class(name)} {") - out.inc + override def classConstructorHeader(name: List[String], + parentType: DataType, + rootClassName: List[String], + isHybrid: Boolean, + params: List[ParamDefSpec]): Unit = { - // Parameter names - val pIo = paramName(IoIdentifier) - val pParent = paramName(ParentIdentifier) - val pRoot = paramName(RootIdentifier) + // Unlike other OOP languages, implementing an interface happens outside the struct declaration. + universalFooter - // Types - val tIo = kstreamName - val tParent = kaitaiType2NativeType(parentType) + // If there are any switch types in the struct definition, create the enums for them + typeProvider.nowClass.seq.foreach( + a => + a.dataType match { + case st: SwitchType => switchTypeEnum(a.id, st) + case _ => () + } + ) + typeProvider.nowClass.instances.foreach( + i => + i._2.dataTypeComposite match { + case st: SwitchType => switchTypeEnum(i._1, st) + case _ => () + } + ) - out.puts(s"fn new(stream: &mut S,") - out.puts(s" _parent: &Option>,") - out.puts(s" _root: &Option>)") - out.puts(s" -> Result") + out.puts( + s"impl<$readLife, $streamLife: $readLife> $kstructName<$readLife, $streamLife> for ${classTypeName(typeProvider.nowClass)} {" + ) out.inc - out.puts(s"where Self: Sized {") - - out.puts(s"let mut s: Self = Default::default();") + out.puts(s"type Root = ${rootClassTypeName(typeProvider.nowClass)};") + out.puts( + s"type ParentStack = ${parentStackTypeName(typeProvider.nowClass)};" + ) out.puts + } - out.puts(s"s.stream = stream;") + override def runRead(name: List[String]): Unit = out.puts(s"// runRead($name)") - out.puts(s"s.read(stream, _parent, _root)?;") - out.puts + override def runReadCalc(): Unit = out.puts(s"// runReadCalc()") - out.puts("Ok(s)") + override def readHeader(endian: Option[FixedEndian], + isEmpty: Boolean): Unit = { + out.puts(s"fn read(") + out.inc + out.puts(s"&mut self,") + out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S,") + out.puts( + s"${privateMemberName(RootIdentifier)}: Option<&$readLife Self::Root>," + ) + out.puts( + s"${privateMemberName(ParentIdentifier)}: TypedStack" + ) out.dec - out.puts("}") - out.puts + out.puts(s") -> KResult<$streamLife, ()> {") + out.inc + + // If there aren't any attributes to parse, we need to end the read implementation here + if (typeProvider.nowClass.seq.isEmpty) + endRead() } - override def runRead(name: List[String]): Unit = { + override def readFooter(): Unit = out.puts(s"// readFooter()") + + override def attributeDeclaration(attrName: Identifier, + attrType: DataType, + isNullable: Boolean): Unit = { + val typeName = attrName match { + // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct + case IoIdentifier | RootIdentifier | ParentIdentifier => return + case _ => + kaitaiTypeToNativeType(attrName, typeProvider.nowClass, attrType) + } + out.puts(s"pub ${idToStr(attrName)}: $typeName,") } - override def runReadCalc(): Unit = { + // Intentional no-op; Rust handles ownership, so don't worry about reader methods + override def attributeReader(attrName: Identifier, + attrType: DataType, + isNullable: Boolean): Unit = {} - } + override def attrParse(attr: AttrLikeSpec, + id: Identifier, + defEndian: Option[Endianness]): Unit = { + super.attrParse(attr, id, defEndian) - override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = { - out.puts - out.puts(s"fn read(&mut self,") - out.puts(s" stream: &mut S,") - out.puts(s" _parent: &Option>,") - out.puts(s" _root: &Option>)") - out.puts(s" -> Result<()>") - out.inc - out.puts(s"where Self: Sized {") + // Detect if this is the last attribute parse and finish the read method + if (typeProvider.nowClass.seq.nonEmpty && typeProvider.nowClass.seq.last.id == id) + endRead() } - override def readFooter(): Unit = { - out.puts + def endRead(): Unit = { out.puts("Ok(())") out.dec out.puts("}") } - override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { - attrName match { - case ParentIdentifier | RootIdentifier | IoIdentifier => - // just ignore it for now - case IoIdentifier => - out.puts(s" stream: ${kaitaiType2NativeType(attrType)},") - case _ => - out.puts(s" pub ${idToStr(attrName)}: ${kaitaiType2NativeType(attrType)},") - } - } - - override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = + out.puts(s"// attrParseHybrid(${leProc()}, ${beProc()})") - } + override def condIfHeader(expr: Ast.expr): Unit = { + // TODO: Actual implementation, this is a shim to enable compiling + out.puts("{") + out.inc - override def universalDoc(doc: DocSpec): Unit = { - if (doc.summary.isDefined) { - out.puts - out.puts("/*") - doc.summary.foreach((summary) => out.putsLines(" * ", summary)) - out.puts(" */") - } + out.puts(s"// condIfHeader($expr)") } - override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { - out.puts("if ($this->_m__is_le) {") + override def condRepeatEosHeader(id: Identifier, + io: String, + dataType: DataType): Unit = { + // TODO: Actual implementation, this is a shim to enable compiling + out.puts("{") out.inc - leProc() - out.dec - out.puts("} else {") - out.inc - beProc() - out.dec - out.puts("}") - } - - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = - out.puts(s"${privateMemberName(attrName)} = $normalIO.ensureFixedContents($contents);") - override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { - val srcExpr = getRawIdExpr(varSrc, rep) + out.puts(s"// condRepeatEosHeader($id, $io, $dataType)") + } - val expr = proc match { - case ProcessXor(xorValue) => - val procName = translator.detectType(xorValue) match { - case _: IntType => "processXorOne" - case _: BytesType => "processXorMany" - } - s"$kstreamName::$procName($srcExpr, ${expression(xorValue)})" - case ProcessZlib => - s"$kstreamName::processZlib($srcExpr);" - case ProcessRotate(isLeft, rotValue) => - val expr = if (isLeft) { - expression(rotValue) - } else { - s"8 - (${expression(rotValue)})" - } - s"$kstreamName::processRotateLeft($srcExpr, $expr, 1)" - case ProcessCustom(name, args) => - val procClass = if (name.length == 1) { - val onlyName = name.head - val className = type2class(onlyName) - importList.add(s"$onlyName::$className") - className - } else { - val pkgName = type2classAbs(name.init) - val className = type2class(name.last) - importList.add(s"$pkgName::$className") - s"$pkgName::$className" - } + override def condRepeatExprHeader(id: Identifier, + io: String, + dataType: DataType, + repeatExpr: Ast.expr): Unit = { + // TODO: Actual implementation, this is a shim to enable compiling + out.puts("{") + out.inc - out.puts(s"let _process = $procClass::new(${args.map(expression).mkString(", ")});") - s"_process.decode($srcExpr)" - } - handleAssignment(varDest, expr, rep, false) + out.puts( + s"// condRepeatExprHeader($id, $io, $dataType, $repeatExpr)" + ) } - override def allocateIO(id: Identifier, rep: RepeatSpec): String = { - val memberName = privateMemberName(id) - - val args = rep match { - case RepeatUntil(_) => translator.doLocalName(Identifier.ITERATOR2) - case _ => getRawIdExpr(id, rep) - } + override def condRepeatUntilHeader(id: Identifier, + io: String, + dataType: DataType, + repeatExpr: Ast.expr): Unit = { + // TODO: Actual implementation, this is a shim to enable compiling + out.puts("{") + out.inc - out.puts(s"let mut io = Cursor::new($args);") - "io" + out.puts( + s"// condRepeatUntilHeader($id, $io, $dataType, $repeatExpr)" + ) } - def getRawIdExpr(varName: Identifier, rep: RepeatSpec): String = { - val memberName = privateMemberName(varName) - rep match { - case NoRepeat => memberName - case _ => s"$memberName.last()" - } + override def condRepeatUntilFooter(id: Identifier, + io: String, + dataType: DataType, + repeatExpr: Ast.expr): Unit = { + out.puts( + s"// condRepeatUntilFooter($id, $io, $dataType, $repeatExpr)" + ) + out.dec + out.puts("} {}") } - override def useIO(ioEx: Ast.expr): String = { - out.puts(s"let mut io = ${expression(ioEx)};") - "io" - } + override def attrProcess(proc: ProcessExpr, + varSrc: Identifier, + varDest: Identifier, + rep: RepeatSpec): Unit = + out.puts(s"// attrProcess($proc, $varSrc, $varDest, $rep)") + + override def useIO(ioEx: Ast.expr): String = s"// useIO($ioEx)" - override def pushPos(io: String): Unit = - out.puts(s"let _pos = $io.pos();") + override def pushPos(io: String): Unit = out.puts(s"// pushPos($io)") override def seek(io: String, pos: Ast.expr): Unit = - out.puts(s"$io.seek(${expression(pos)});") + out.puts(s"// seek($io, $pos)") - override def popPos(io: String): Unit = - out.puts(s"$io.seek(_pos);") + override def popPos(io: String): Unit = out.puts(s"// popPos($io)") override def alignToByte(io: String): Unit = - out.puts(s"$io.alignToByte();") + out.puts(s"${privateMemberName(IoIdentifier)}.align_to_byte()?;") - override def condIfHeader(expr: Ast.expr): Unit = { - out.puts(s"if ${expression(expr)} {") + override def privateMemberName(id: Identifier): String = + RustCompiler.privateMemberName(id) + + override def instanceDeclHeader(className: List[String]): Unit = { + out.puts( + s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {" + ) out.inc } - override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { - if (needRaw.level >= 1) - out.puts(s"${privateMemberName(RawIdentifier(id))} = vec!();") - if (needRaw.level >= 2) - out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = vec!();") - out.puts(s"${privateMemberName(id)} = vec!();") + override def universalFooter: Unit = { + out.dec + out.puts("}") } - override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = { - out.puts(s"while !$io.isEof() {") - out.inc + override def instanceDeclaration(attrName: InstanceIdentifier, + attrType: DataType, + isNullable: Boolean): Unit = { + val typeName = kaitaiTypeToNativeType( + attrName, + typeProvider.nowClass, + attrType, + excludeOptionWrapper = true + ) + attrType match { + case _: ArrayType => out.puts(s"pub ${idToStr(attrName)}: $typeName,") + case _ => out.puts(s"pub ${idToStr(attrName)}: Option<$typeName>,") + } } - override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = { - out.puts(s"${privateMemberName(id)}.append($expr);") - } + override def idToStr(id: Identifier): String = RustCompiler.idToStr(id) - override def condRepeatEosFooter: Unit = { - super.condRepeatEosFooter - } + override def instanceHeader(className: List[String], + instName: InstanceIdentifier, + dataType: DataType, + isNullable: Boolean): Unit = { - override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, repeatExpr: Ast.expr): Unit = { - out.puts(s"for i in 0..${expression(repeatExpr)} {") + out.puts(s"fn ${idToStr(instName)}(") + out.inc + out.puts("&mut self,") + out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S,") + out.puts( + s"${privateMemberName(RootIdentifier)}: Option<&$readLife ${rootClassTypeName(typeProvider.nowClass)}>," + ) + out.puts( + s"${privateMemberName(ParentIdentifier)}: TypedStack<${parentStackTypeName(typeProvider.nowClass)}>" + ) + out.dec + val typeName = kaitaiTypeToNativeType( + instName, + typeProvider.nowClass, + dataType, + excludeOptionWrapper = true + ) + out.puts(s") -> KResult<$streamLife, $typeName> {") out.inc } - override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = - handleAssignmentRepeatEos(id, expr) + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, + dataType: DataType): Unit = + out.puts(s"// instanceCheckCacheAndReturn($instName, $dataType)") - override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, untilExpr: Ast.expr): Unit = { - out.puts("while {") - out.inc + override def instanceReturn(instName: InstanceIdentifier, + attrType: DataType): Unit = { + out.puts("panic!(\"Instance calculation not yet supported.\");") + out.puts(s"// instanceReturn($instName, $attrType)") } - override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = { - val tempVar = if (isRaw) { - translator.doLocalName(Identifier.ITERATOR2) - } else { - translator.doLocalName(Identifier.ITERATOR) + override def enumDeclaration(curClass: List[String], + enumName: String, + enumColl: Seq[(Long, EnumValueSpec)]): Unit = { + + val enumClass = types2class(curClass ::: List(enumName)) + + // Set up the actual enum definition + out.puts(s"#[allow(non_camel_case_types)]") + out.puts(s"#[derive(Debug, PartialEq)]") + out.puts(s"pub enum $enumClass {") + out.inc + + enumColl.foreach { + case (_, label) => + if (label.doc.summary.isDefined) + universalDoc(label.doc) + + out.puts(s"${type2class(label.name)},") } - out.puts(s"let $tempVar = $expr;") - out.puts(s"${privateMemberName(id)}.append($tempVar);") - } - override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, untilExpr: Ast.expr): Unit = { - typeProvider._currentIteratorType = Some(dataType) - out.puts(s"!(${expression(untilExpr)})") out.dec - out.puts("} { }") - } + out.puts("}") - override def handleAssignmentSimple(id: Identifier, expr: String): Unit = { - out.puts(s"${privateMemberName(id)} = $expr;") - } + // Set up parsing enums from the underlying value + out.puts(s"impl TryFrom for $enumClass {") - override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { - dataType match { - case t: ReadableType => - s"$io.read_${t.apiCall(defEndian)}()?" - case blt: BytesLimitType => - s"$io.read_bytes(${expression(blt.size)})?" - case _: BytesEosType => - s"$io.read_bytes_full()?" - case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io.read_bytes_term($terminator, $include, $consume, $eosError)?" - case BitsType1(bitEndian) => - s"$io.read_bits_int(1)? != 0" - case BitsType(width: Int, bitEndian) => - s"$io.read_bits_int($width)?" - case t: UserType => - val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") - val addArgs = if (t.isOpaque) { - "" - } else { - val parent = t.forcedParent match { - case Some(USER_TYPE_NO_PARENT) => "null" - case Some(fp) => translator.translate(fp) - case None => "self" - } - val addEndian = t.classSpec.get.meta.endian match { - case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}" - case _ => "" - } - s", $parent, ${privateMemberName(RootIdentifier)}$addEndian" - } + out.inc + // We typically need the lifetime in KError for returning byte slices from stream; + // because we can only return `UnknownVariant` which contains a Copy type, it's safe + // to declare that the error type is `'static` + out.puts(s"type Error = KError<'static>;") + out.puts(s"fn try_from(flag: i64) -> KResult<'static, $enumClass> {") - s"Box::new(${translator.types2classAbs(t.classSpec.get.name)}::new(self.stream, self, _root)?)" + out.inc + out.puts(s"match flag {") + + out.inc + enumColl.foreach { + case (value, label) => + out.puts(s"$value => Ok($enumClass::${type2class(label.name)}),") } + out.puts("_ => Err(KError::UnknownVariant(flag)),") + out.dec + + out.puts(s"}") + out.dec + out.puts(s"}") + out.dec + out.puts(s"}") + out.puts } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { - val expr1 = padRight match { - case Some(padByte) => s"$kstreamName::bytesStripRight($expr0, $padByte)" - case None => expr0 - } - val expr2 = terminator match { - case Some(term) => s"$kstreamName::bytesTerminate($expr1, $term, $include)" - case None => expr1 - } - expr2 + override def universalDoc(doc: DocSpec): Unit = { + out.puts(s"// universalDoc()") } - var switchIfs = false - val NAME_SWITCH_ON = Ast.expr.Name(Ast.identifier(Identifier.SWITCH_ON)) + override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = + out.puts(s"// handleAssignmentRepeatEos($id, $expr)") - override def switchStart(id: Identifier, on: Ast.expr): Unit = { - val onType = translator.detectType(on) + override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = + out.puts(s"// handleAssignmentRepeatExpr($id, $expr)") - switchIfs = onType match { - case _: ArrayTypeInStream | _: BytesType => true - case _ => false - } + override def handleAssignmentRepeatUntil(id: Identifier, + expr: String, + isRaw: Boolean): Unit = + out.puts(s"// handleAssignmentRepeatUntil($id, $expr, $isRaw)") - if (!switchIfs) { - out.puts(s"match ${expression(on)} {") - out.inc + override def handleAssignmentSimple(id: Identifier, expr: String): Unit = { + val seqId = typeProvider.nowClass.seq.find(s => s.id == id) + + if (seqId.isDefined) seqId.get.dataType match { + case _: EnumType => + out.puts( + s"${privateMemberName(id)} = Some(($expr as i64).try_into()?);" + ) + case _: SwitchType | _: BytesLimitType => + out.puts(s"// handleAssignmentSimple($id, $expr)") + case t: UserType => + out.puts(s"{"); + out.inc + val tempVarName = localTemporaryName(id) + out.puts(s"let mut $tempVarName = ${kaitaiTypeToNativeType(id, typeProvider.nowClass, t, excludeOptionWrapper = true)}::default();") + out.puts(s"$tempVarName.read(_io, Some(self), _parent.push(self))?;") + out.puts(s"${privateMemberName(id)} = Some($tempVarName);"); + out.dec + out.puts(s"}") + case _ => { + out.puts(s"${privateMemberName(id)} = $expr;") + } } } - def switchCmpExpr(condition: Ast.expr): String = - expression( - Ast.expr.Compare( - NAME_SWITCH_ON, - Ast.cmpop.Eq, - condition - ) - ) - - override def switchCaseFirstStart(condition: Ast.expr): Unit = { - if (switchIfs) { - out.puts(s"if ${switchCmpExpr(condition)} {") - out.inc - } else { - switchCaseStart(condition) + override def parseExpr(dataType: DataType, + assignType: DataType, + io: String, + defEndian: Option[FixedEndian]): String = + dataType match { + case IntMultiType(_, _, None) => "panic!(\"Unable to parse unknown-endian integers\")" + case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?" + case _: BytesEosType => s"$io.read_bytes_full()?" + case b: BytesTerminatedType => + s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?" + case b: BytesLimitType => s"$io.read_bytes(${expression(b.size)} as usize)?" + case BitsType1(bitEndian) => s"$io.read_bits_int(1)? != 0" + case BitsType(width, bitEndian) => s"$io.read_bits_int($width)?" + case _ => s"// parseExpr($dataType, $assignType, $io, $defEndian)" } - } - override def switchCaseStart(condition: Ast.expr): Unit = { - if (switchIfs) { - out.puts(s"else if ${switchCmpExpr(condition)} {") - out.inc - } else { - out.puts(s"${expression(condition)} => {") - out.inc + override def bytesPadTermExpr(expr0: String, + padRight: Option[Int], + terminator: Option[Int], + include: Boolean): String = { + val ioId = privateMemberName(IoIdentifier) + val expr = padRight match { + case Some(p) => s"$ioId.bytes_strip_right($expr0, $p)" + case None => expr0 } - } - override def switchCaseEnd(): Unit = { - if (switchIfs) { - out.dec - out.puts("}") - } else { - out.dec - out.puts("},") + terminator match { + case Some(term) => s"$ioId.bytes_terminate($expr, $term, $include)" + case None => expr } } - override def switchElseStart(): Unit = { - if (switchIfs) { - out.puts("else {") - out.inc - } else { - out.puts("_ => {") - out.inc - } - } + override def attrFixedContentsParse(attrName: Identifier, + contents: String): Unit = + out.puts(s"// attrFixedContentsParse($attrName, $contents)") - override def switchElseEnd(): Unit = { - out.dec - out.puts("}") - } + override def publicMemberName(id: Identifier): String = + s"// publicMemberName($id)" - override def switchEnd(): Unit = universalFooter + override def localTemporaryName(id: Identifier): String = + s"_t_${idToStr(id)}" - override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = { - out.puts(s" pub ${idToStr(attrName)}: Option<${kaitaiType2NativeType(attrType)}>,") - } + override def switchStart(id: Identifier, on: Ast.expr): Unit = + out.puts(s"// switchStart($id, $on)") - override def instanceDeclHeader(className: List[String]): Unit = { - out.dec - out.puts("}") - out.puts + override def switchCaseStart(condition: Ast.expr): Unit = + out.puts(s"// switchCaseStart($condition)") - out.puts(s"impl ${type2class(className)} {") - out.inc - } + override def switchCaseEnd(): Unit = out.puts(s"// switchCaseEnd()") - override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { - out.puts(s"fn ${idToStr(instName)}(&mut self) -> ${kaitaiType2NativeType(dataType)} {") - out.inc - } + override def switchElseStart(): Unit = out.puts(s"// switchElseStart()") - override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { - out.puts(s"if let Some(x) = ${privateMemberName(instName)} {") - out.inc - out.puts("return x;") - out.dec - out.puts("}") - out.puts - } + override def switchEnd(): Unit = out.puts(s"// switchEnd()") - override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { - out.puts(s"return ${privateMemberName(instName)};") + override def extraAttrForIO(id: Identifier, + rep: RepeatSpec): List[AttrSpec] = { + out.puts(s"// extraAttrForIO($id, $rep)") + Nil } - override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { - val enumClass = type2class(curClass ::: List(enumName)) + override def allocateIO(varName: Identifier, rep: RepeatSpec): String = + s"// allocateIO($varName, $rep)" - out.puts(s"enum $enumClass {") + def switchTypeEnum(id: Identifier, st: SwitchType): Unit = { + // Because Rust can't handle `AnyType` in the type hierarchy, + // we generate an enum with all possible variations + val typeName = kaitaiTypeToNativeType( + id, + typeProvider.nowClass, + st, + excludeOptionWrapper = true + ) + out.puts("#[allow(non_camel_case_types)]") + out.puts("#[derive(Debug, PartialEq)]") + out.puts(s"pub enum $typeName {") out.inc - enumColl.foreach { case (id, label) => - universalDoc(label.doc) - out.puts(s"${value2Const(label.name)},") - } + val types = st.cases.values.toSet + types.foreach(t => { + // Because this switch type will itself be in an option, we can exclude it from user types + val variantName = switchVariantName(id, t) + val typeName = kaitaiTypeToNativeType( + id, + typeProvider.nowClass, + t, + excludeOptionWrapper = true + ) + out.puts(s"$variantName($typeName),") + }) out.dec out.puts("}") } - def value2Const(label: String) = Utils.upperUnderscoreCase(label) + def switchVariantName(id: Identifier, attrType: DataType): String = + attrType match { + // TODO: Not exhaustive + case Int1Type(false) => "U1" + case IntMultiType(false, Width2, _) => "U2" + case IntMultiType(false, Width4, _) => "U4" + case IntMultiType(false, Width8, _) => "U8" + + case Int1Type(true) => "S1" + case IntMultiType(true, Width2, _) => "S2" + case IntMultiType(true, Width4, _) => "S4" + case IntMultiType(true, Width8, _) => "S8" + + case FloatMultiType(Width4, _) => "F4" + case FloatMultiType(Width8, _) => "F8" + + case BitsType(_,_) => "Bits" + case _: BooleanType => "Boolean" + case CalcIntType => "Int" + case CalcFloatType => "Float" + case _: StrType => "String" + case _: BytesType => "Bytes" - def idToStr(id: Identifier): String = { - id match { - case SpecialIdentifier(name) => name - case NamedIdentifier(name) => Utils.lowerCamelCase(name) - case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" - case InstanceIdentifier(name) => Utils.lowerCamelCase(name) - case RawIdentifier(innerId) => "_raw_" + idToStr(innerId) + case t: UserType => + kaitaiTypeToNativeType( + id, + typeProvider.nowClass, + t, + excludeOptionWrapper = true, + excludeLifetime = true, + excludeBox = true + ) + case t: EnumType => + kaitaiTypeToNativeType( + id, + typeProvider.nowClass, + t, + excludeOptionWrapper = true + ) + case t: ArrayType => s"Arr${switchVariantName(id, t.elType)}" } - } - override def privateMemberName(id: Identifier): String = { - id match { - case IoIdentifier => s"self.stream" - case RootIdentifier => s"_root" - case ParentIdentifier => s"_parent" - case _ => s"self.${idToStr(id)}" - } + override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { + out.puts(s"// condRepeatCommonInit($id, $dataType, $needRaw)") } - override def publicMemberName(id: Identifier) = idToStr(id) - - override def localTemporaryName(id: Identifier): String = s"$$_t_${idToStr(id)}" + override def ksErrorName(err: io.kaitai.struct.datatype.KSError): String = + s"KaitaiStream.$err" +} - override def paramName(id: Identifier): String = s"${idToStr(id)}" +object RustCompiler + extends LanguageCompilerStatic + with StreamStructNames + with UpperCamelCaseClasses { + override def getCompiler(tp: ClassTypeProvider, + config: RuntimeConfig): LanguageCompiler = + new RustCompiler(tp, config) - def kaitaiType2NativeType(attrType: DataType): String = { - attrType match { - case Int1Type(false) => "u8" - case IntMultiType(false, Width2, _) => "u16" - case IntMultiType(false, Width4, _) => "u32" - case IntMultiType(false, Width8, _) => "u64" + override def kstreamName = "KStream" - case Int1Type(true) => "i8" - case IntMultiType(true, Width2, _) => "i16" - case IntMultiType(true, Width4, _) => "i32" - case IntMultiType(true, Width8, _) => "i64" + def privateMemberName(id: Identifier): String = id match { + case IoIdentifier => "_io" + case RootIdentifier => "_root" + case ParentIdentifier => "_parent" + case _ => s"self.${idToStr(id)}" + } - case FloatMultiType(Width4, _) => "f32" - case FloatMultiType(Width8, _) => "f64" + def idToStr(id: Identifier): String = id match { + case SpecialIdentifier(n) => n + case NamedIdentifier(n) => n + case InstanceIdentifier(n) => n + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case RawIdentifier(inner) => s"raw_${idToStr(inner)}" + } - case BitsType(_, _) => "u64" + def rootClassTypeName(c: ClassSpec, isRecurse: Boolean = false): String = { + if (!isRecurse && c.isTopLevel) + "Self" + else if (c.isTopLevel) + classTypeName(c) + else + rootClassTypeName(c.upClass.get, isRecurse = true) + } - case _: BooleanType => "bool" - case CalcIntType => "i32" - case CalcFloatType => "f64" + def parentStackTypeName(c: ClassSpec): String = { + if (c.isTopLevel) + s"$kstructUnitName" + else + s"(&$readLife ${classTypeName(c.upClass.get)}, <${classTypeName(c.upClass.get)} as $kstructName<$readLife, $streamLife>>::ParentStack)" + } - case _: StrType => "String" - case _: BytesType => "Vec" + override def kstructName = s"KStruct" - case t: UserType => t.classSpec match { - case Some(cs) => s"Box<${type2class(cs.name)}>" - case None => s"Box<${type2class(t.name)}>" - } + def readLife = "'r" - case t: EnumType => t.enumSpec match { - case Some(cs) => s"Box<${type2class(cs.name)}>" - case None => s"Box<${type2class(t.name)}>" - } + def kstructUnitName = "KStructUnit" - case at: ArrayType => s"Vec<${kaitaiType2NativeType(at.elType)}>" + def classTypeName(c: ClassSpec): String = + s"${types2class(c.name)}" - case KaitaiStreamType | OwnedKaitaiStreamType => s"Option>" - case KaitaiStructType | CalcKaitaiStructType => s"Option>" + def streamLife = "'s" - case st: SwitchType => kaitaiType2NativeType(st.combinedType) - } - } + def types2class(names: List[String]): String = + // TODO: Use `mod` to scope types instead of weird names + names.map(x => type2class(x)).mkString("_") - def kaitaiType2Default(attrType: DataType): String = { - attrType match { - case Int1Type(false) => "0" - case IntMultiType(false, Width2, _) => "0" - case IntMultiType(false, Width4, _) => "0" - case IntMultiType(false, Width8, _) => "0" + def lifetimeParam(d: DataType): String = + if (containsReferences(d)) s"<$streamLife>" else "" - case Int1Type(true) => "0" - case IntMultiType(true, Width2, _) => "0" - case IntMultiType(true, Width4, _) => "0" - case IntMultiType(true, Width8, _) => "0" + def containsReferences(d: DataType): Boolean = containsReferences(d, None) - case FloatMultiType(Width4, _) => "0" - case FloatMultiType(Width8, _) => "0" + def containsReferences(c: ClassSpec, + originating: Option[ClassSpec]): Boolean = + c.seq.exists(t => containsReferences(t.dataType, originating)) || + c.instances.exists( + i => containsReferences(i._2.dataTypeComposite, originating) + ) - case BitsType(_, _) => "0" + def containsReferences(d: DataType, originating: Option[ClassSpec]): Boolean = + d match { + case _: BytesType | _: StrType => true + case t: UserType => true + /* + t.classSpec match { + // Recursive types may need references, but the recursion itself + // will be handled by `Box<>`, so doesn't need a reference + case Some(inner) if originating.contains(inner) => false + case Some(inner) => containsReferences(inner, originating.orElse(Some(inner))) + case None => false + } + */ + case t: ArrayType => containsReferences(t.elType, originating) + case st: SwitchType => + st.cases.values.exists(t => containsReferences(t, originating)) + case _ => false + } - case _: BooleanType => "false" - case CalcIntType => "0" - case CalcFloatType => "0" + def kaitaiTypeToNativeType(id: Identifier, + cs: ClassSpec, + attrType: DataType, + excludeOptionWrapper: Boolean = false, + excludeLifetime: Boolean = false, + excludeBox: Boolean = false): String = + attrType match { + // TODO: Not exhaustive + case _: NumericType => kaitaiPrimitiveToNativeType(attrType) + case _: BooleanType => kaitaiPrimitiveToNativeType(attrType) + case _: StrType => kaitaiPrimitiveToNativeType(attrType) + case _: BytesType => kaitaiPrimitiveToNativeType(attrType) - case _: StrType => "\"\"" - case _: BytesType => "vec!()" + case t: UserType => + val baseName = t.classSpec match { + case Some(spec) => types2class(spec.name) + case None => types2class(t.name) + } + //val lifetime = if (!excludeLifetime) s"<$streamLife>" else "" + + // Because we can't predict if opaque types will recurse, we have to box them + val typeName = + if (!excludeBox && t.isOpaque) s"Box<$baseName>" + else s"$baseName" + if (excludeOptionWrapper) typeName else s"Option<$typeName>" + + case t: EnumType => + val typeName = t.enumSpec match { + case Some(spec) => s"${types2class(spec.name)}" + case None => s"${types2class(t.name)}" + } + if (excludeOptionWrapper) typeName else s"Option<$typeName>" + + case t: ArrayType => + s"Vec<${kaitaiTypeToNativeType(id, cs, t.elType, excludeOptionWrapper = true, excludeLifetime = excludeLifetime)}>" + + case st: SwitchType => + val types = st.cases.values.toSet + val lifetime = + if (!excludeLifetime && types.exists(containsReferences)) + s"<$streamLife>" + else "" + val typeName = id match { + case name: NamedIdentifier => + s"${types2class(cs.name ::: List(name.name))}$lifetime" + case name: InstanceIdentifier => + s"${types2class(cs.name ::: List(name.name))}$lifetime" + case _ => kstructUnitName + } - case t: UserType => "Default::default()" - case t: EnumType => "Default::default()" + if (excludeOptionWrapper) typeName else s"Option<$typeName>" - case ArrayTypeInStream(inType) => "vec!()" + case KaitaiStreamType => kstreamName + } - case KaitaiStreamType | OwnedKaitaiStreamType => "None" - case KaitaiStructType => "None" + def kaitaiPrimitiveToNativeType(attrType: DataType): String = attrType match { + case Int1Type(false) => "u8" + case IntMultiType(false, Width2, _) => "u16" + case IntMultiType(false, Width4, _) => "u32" + case IntMultiType(false, Width8, _) => "u64" - case _: SwitchType => "" - // TODO - } - } + case Int1Type(true) => "i8" + case IntMultiType(true, Width2, _) => "i16" + case IntMultiType(true, Width4, _) => "i32" + case IntMultiType(true, Width8, _) => "i64" - def type2class(names: List[String]) = types2classRel(names) + case FloatMultiType(Width4, _) => "f32" + case FloatMultiType(Width8, _) => "f64" - def type2classAbs(names: List[String]) = - names.mkString("::") + case BitsType(_,_) => "u64" - override def ksErrorName(err: KSError): String = RustCompiler.ksErrorName(err) -} + case _: BooleanType => "bool" + case CalcIntType => "i32" + case CalcFloatType => "f64" -object RustCompiler extends LanguageCompilerStatic - with StreamStructNames - with UpperCamelCaseClasses - with ExceptionNames { - override def getCompiler( - tp: ClassTypeProvider, - config: RuntimeConfig - ): LanguageCompiler = new RustCompiler(tp, config) - - override def kstructName = "&Option>" - override def kstreamName = "&mut S" - override def ksErrorName(err: KSError): String = ??? - - def types2class(typeName: Ast.typeId) = { - typeName.names.map(type2class).mkString( - if (typeName.absolute) "__" else "", - "__", - "" - ) + case _: StrType => s"&$streamLife str" + case _: BytesType => s"&$streamLife [u8]" } - - def types2classRel(names: List[String]) = - names.map(type2class).mkString("__") } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 252f7cf08..d1e49707f 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -3,15 +3,17 @@ package io.kaitai.struct.translators import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr -import io.kaitai.struct.format.Identifier +import io.kaitai.struct.format.{Identifier, InstanceIdentifier, IoIdentifier, NamedIdentifier, ParentIdentifier, RootIdentifier} import io.kaitai.struct.languages.RustCompiler import io.kaitai.struct.{RuntimeConfig, Utils} -class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseTranslator(provider) { +class RustTranslator(provider: TypeProvider, config: RuntimeConfig) + extends BaseTranslator(provider) { + + import RustCompiler._ + override def doByteArrayLiteral(arr: Seq[Byte]): String = - "vec!([" + arr.map((x) => - "%0#2x".format(x & 0xff) - ).mkString(", ") + "])" + "&[" + arr.map(x => "%0#2x".format(x & 0xff)).mkString(", ") + "]" override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = s"pack('C*', ${elts.map(translate).mkString(", ")})" @@ -26,7 +28,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base override def strLiteralUnicode(code: Char): String = "\\u{%x}".format(code.toInt) - override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = { + override def numericBinOp(left: Ast.expr, + op: Ast.operator, + right: Ast.expr) = { (detectType(left), detectType(right), op) match { case (_: IntType, _: IntType, Ast.operator.Div) => s"${translate(left)} / ${translate(right)}" @@ -42,26 +46,41 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base case Identifier.ITERATOR => "tmpa" case Identifier.ITERATOR2 => "tmpb" case Identifier.INDEX => "i" - case _ => s"self.${doName(s)}" + case Identifier.IO => s"${RustCompiler.privateMemberName(IoIdentifier)}" + case Identifier.ROOT => s"${RustCompiler.privateMemberName(RootIdentifier)}.ok_or(KError::MissingRoot)?" + case Identifier.PARENT => + // TODO: How to handle _parent._parent? + s"${RustCompiler.privateMemberName(ParentIdentifier)}.peek()" + case _ => + if (provider.nowClass.seq.exists(a => a.id != IoIdentifier && a.id == NamedIdentifier(s))) { + // If the name is part of the `seq` parse list, it's safe to return as-is + s"self.${doName(s)}" + } else if (provider.nowClass.instances.contains(InstanceIdentifier(s))) { + // It's an instance, we need to safely handle lookup + s"self.${doName(s)}(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)}, ${privateMemberName(ParentIdentifier)})?" + } else { + // TODO: Is it possible to reach this block? RawIdentifier? + s"self.${doName(s)}" + } } } override def doName(s: String) = s - override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = { - val enumClass = types2classAbs(enumTypeAbs) - s"$enumClass::${Utils.upperUnderscoreCase(label)}" - } + override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = + s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" + override def doEnumById(enumTypeAbs: List[String], id: String) = // Just an integer, without any casts / resolutions - one would have to look up constants manually id override def arraySubscript(container: expr, idx: expr): String = - s"${translate(container)}[${translate(idx)}]" + s"${translate(container)}[${translate(idx)} as usize]" + override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = "if " + translate(condition) + - " { " + translate(ifTrue) + " } else { " + - translate(ifFalse) + "}" + " { " + translate(ifTrue) + " } else { " + + translate(ifFalse) + "}" // Predefined methods of various types override def strConcat(left: Ast.expr, right: Ast.expr): String = @@ -72,7 +91,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base case "10" => s"${translate(s)}.parse().unwrap()" case _ => - "panic!(\"Converting from string to int in base {} is unimplemented\", " + translate(base) + ")" + "panic!(\"Converting from string to int in base {} is unimplemented\"" + translate( + base + ) + ")" } override def enumToInt(v: expr, et: EnumType): String = @@ -96,10 +117,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = translate(encoding) match { case "\"ASCII\"" => - s"String::from_utf8_lossy($bytesExpr)" + // Currently has issues because the `&str` created doesn't outlive the function, + // will likely need to decode *as* as string or handle specially elsewhere + // s"&String::from_utf8_lossy($bytesExpr)" + "panic!(\"Unresolved lifetime issues with string parsing\")" case _ => "panic!(\"Unimplemented encoding for bytesToStr: {}\", " + - translate(encoding) + ")" + translate(encoding) + ")" } override def bytesLength(b: Ast.expr): String = s"${translate(b)}.len()" @@ -120,10 +144,4 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base s"${translate(a)}.iter().min()" override def arrayMax(a: Ast.expr): String = s"${translate(a)}.iter().max()" - - def types2classAbs(names: List[String]) = - names match { - case List("kaitai_struct") => RustCompiler.kstructName - case _ => RustCompiler.types2classRel(names) - } } From be78dcc51a726081fee6429a80870a3dc0b6d2e5 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 6 Jul 2022 17:26:01 +0800 Subject: [PATCH 002/153] handle strings in ASCII, UTF-8, UTF-16 encodings. --- .../io/kaitai/struct/translators/RustTranslator.scala | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index d1e49707f..85017fa9d 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -116,14 +116,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = translate(encoding) match { - case "\"ASCII\"" => - // Currently has issues because the `&str` created doesn't outlive the function, - // will likely need to decode *as* as string or handle specially elsewhere - // s"&String::from_utf8_lossy($bytesExpr)" - "panic!(\"Unresolved lifetime issues with string parsing\")" - case _ => - "panic!(\"Unimplemented encoding for bytesToStr: {}\", " + - translate(encoding) + ")" + case "\"ASCII\"" | "\"UTF-8\"" => s"String::from_utf8_lossy($bytesExpr).to_string()" + case "\"UTF-16LE\"" => s"to_utf16_string($bytesExpr)?" + case _ => "panic!(\"Unimplemented encoding for bytesToStr: {}\", " + translate(encoding) + ")" } override def bytesLength(b: Ast.expr): String = s"${translate(b)}.len()" From a5e8066bdff4119e52fc7768eae4a7f2b456dd58 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 6 Jul 2022 17:27:14 +0800 Subject: [PATCH 003/153] instance generation fixed. --- .../struct/languages/RustCompiler.scala | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 069f1ed1a..4e5bff04e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -43,9 +43,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) outHeader.puts importList.add( - "kaitai::{BytesReader, KError, KResult, KStream, KStruct, KStructUnit, TypedStack}" + "kaitai::*" ) - importList.add("kaitai::{kf32_max, kf64_max, kf32_min, kf64_min}") importList.add("std::convert::{TryFrom, TryInto}") } @@ -307,18 +306,28 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) dataType, excludeOptionWrapper = true ) - out.puts(s") -> KResult<$streamLife, $typeName> {") + out.puts(s") -> KResult<&$typeName> {") out.inc } override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, - dataType: DataType): Unit = - out.puts(s"// instanceCheckCacheAndReturn($instName, $dataType)") + dataType: DataType): Unit = { + out.puts(s"if ${privateMemberName(instName)}.is_some() {") + out.inc + instanceReturn(instName, dataType) + out.dec + out.puts(s"}") + } + + override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { + val primType = kaitaiPrimitiveToNativeType(dataType) + out.puts(s"${privateMemberName(instName)} = Some(${expression(value)} as $primType);") + //handleAssignmentSimple(instName, s"${privateMemberName(instName)} = ${expression(value)}") + } override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { - out.puts("panic!(\"Instance calculation not yet supported.\");") - out.puts(s"// instanceReturn($instName, $attrType)") + out.puts(s"return Ok(${privateMemberName(instName)}.as_ref().unwrap());") } override def enumDeclaration(curClass: List[String], @@ -719,7 +728,7 @@ object RustCompiler case CalcIntType => "i32" case CalcFloatType => "f64" - case _: StrType => s"&$streamLife str" - case _: BytesType => s"&$streamLife [u8]" + case _: StrType => s"String" + case _: BytesType => s"[u8]" } } From 0768146c9d6f615cd3afb7b796b3347fce893c1c Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 7 Jul 2022 18:03:58 +0800 Subject: [PATCH 004/153] decode almost any string encoding. --- .../io/kaitai/struct/translators/RustTranslator.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 85017fa9d..8dcab8bcb 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -115,11 +115,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = - translate(encoding) match { - case "\"ASCII\"" | "\"UTF-8\"" => s"String::from_utf8_lossy($bytesExpr).to_string()" - case "\"UTF-16LE\"" => s"to_utf16_string($bytesExpr)?" - case _ => "panic!(\"Unimplemented encoding for bytesToStr: {}\", " + translate(encoding) + ")" - } + s"decode_string($bytesExpr, ${translate(encoding)})?" + override def bytesLength(b: Ast.expr): String = s"${translate(b)}.len()" override def strLength(s: expr): String = From 20fbd40c065cc29e87a7e962e0d882de15160a0d Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 11 Jul 2022 13:55:12 +0800 Subject: [PATCH 005/153] handle string literals. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 5 ++++- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 4e5bff04e..5c83f1aca 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -321,7 +321,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { val primType = kaitaiPrimitiveToNativeType(dataType) - out.puts(s"${privateMemberName(instName)} = Some(${expression(value)} as $primType);") + val converted = dataType match { + case _: StrType => out.puts(s"${privateMemberName(instName)} = Some(${expression(value)}.to_string());") + case _ => out.puts(s"${privateMemberName(instName)} = Some(${expression(value)} as $primType);") + } //handleAssignmentSimple(instName, s"${privateMemberName(instName)} = ${expression(value)}") } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 8dcab8bcb..db9eabd7a 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -25,6 +25,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) '\\' -> "\\\\" ) + override def strLiteralGenericCC(code: Char): String = + strLiteralUnicode(code) + override def strLiteralUnicode(code: Char): String = "\\u{%x}".format(code.toInt) From a61807ab62b4213411bad6faac727947a7735b03 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Mon, 11 Jul 2022 20:49:15 +0200 Subject: [PATCH 006/153] BytesType => String instead of [u8] --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 4e5bff04e..e5e44d629 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -729,6 +729,6 @@ object RustCompiler case CalcFloatType => "f64" case _: StrType => s"String" - case _: BytesType => s"[u8]" + case _: BytesType => s"String" //s"[u8]" } } From 1a63ea8ed9bd4e59dea12a3116cbd8a81b0d9bc0 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 13 Jul 2022 13:36:24 +0800 Subject: [PATCH 007/153] updated KResult type. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 5c83f1aca..f3912149f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -132,7 +132,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"${privateMemberName(ParentIdentifier)}: TypedStack" ) out.dec - out.puts(s") -> KResult<$streamLife, ()> {") + out.puts(s") -> KResult<()> {") out.inc // If there aren't any attributes to parse, we need to end the read implementation here @@ -732,6 +732,6 @@ object RustCompiler case CalcFloatType => "f64" case _: StrType => s"String" - case _: BytesType => s"[u8]" + case _: BytesType => s"Vec" } } From dddf0dcd4c4cdeaef5272ab1ac388ac370c41c91 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Wed, 13 Jul 2022 12:36:06 +0200 Subject: [PATCH 008/153] BytesType => Vec, remove lifetime --- .../io/kaitai/struct/languages/RustCompiler.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index e5e44d629..80f87cc14 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -132,7 +132,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"${privateMemberName(ParentIdentifier)}: TypedStack" ) out.dec - out.puts(s") -> KResult<$streamLife, ()> {") + out.puts(s") -> KResult<()> {") out.inc // If there aren't any attributes to parse, we need to end the read implementation here @@ -360,8 +360,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // We typically need the lifetime in KError for returning byte slices from stream; // because we can only return `UnknownVariant` which contains a Copy type, it's safe // to declare that the error type is `'static` - out.puts(s"type Error = KError<'static>;") - out.puts(s"fn try_from(flag: i64) -> KResult<'static, $enumClass> {") + out.puts(s"type Error = KError;") + out.puts(s"fn try_from(flag: i64) -> KResult<$enumClass> {") out.inc out.puts(s"match flag {") @@ -406,7 +406,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"${privateMemberName(id)} = Some(($expr as i64).try_into()?);" ) case _: SwitchType | _: BytesLimitType => - out.puts(s"// handleAssignmentSimple($id, $expr)") + out.puts(s"${privateMemberName(id)} = $expr.to_vec();") case t: UserType => out.puts(s"{"); out.inc @@ -729,6 +729,6 @@ object RustCompiler case CalcFloatType => "f64" case _: StrType => s"String" - case _: BytesType => s"String" //s"[u8]" + case _: BytesType => s"Vec" //s"[u8]" } } From 0c7c0b2753e68a43b465b907ffcb94ae1bbbc6fd Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 13 Jul 2022 19:51:52 +0800 Subject: [PATCH 009/153] debug output added. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index ee1a07165..266f4f86e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -416,7 +416,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val tempVarName = localTemporaryName(id) out.puts(s"let mut $tempVarName = ${kaitaiTypeToNativeType(id, typeProvider.nowClass, t, excludeOptionWrapper = true)}::default();") out.puts(s"$tempVarName.read(_io, Some(self), _parent.push(self))?;") - out.puts(s"${privateMemberName(id)} = Some($tempVarName);"); + out.puts(s"${privateMemberName(id)} = Some($tempVarName); // $expr"); out.dec out.puts(s"}") case _ => { From ff45079678462db0fb0cea007b914676692f9a45 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Wed, 13 Jul 2022 21:56:53 +0200 Subject: [PATCH 010/153] implementing UserTypeFromBytes --- .../io/kaitai/struct/languages/RustCompiler.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 266f4f86e..a8a67ce08 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -415,8 +415,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc val tempVarName = localTemporaryName(id) out.puts(s"let mut $tempVarName = ${kaitaiTypeToNativeType(id, typeProvider.nowClass, t, excludeOptionWrapper = true)}::default();") - out.puts(s"$tempVarName.read(_io, Some(self), _parent.push(self))?;") - out.puts(s"${privateMemberName(id)} = Some($tempVarName); // $expr"); + out.puts(s"$tempVarName.read($expr, Some(self), _parent.push(self))?;") + out.puts(s"${privateMemberName(id)} = Some($tempVarName);"); out.dec out.puts(s"}") case _ => { @@ -434,10 +434,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?" case _: BytesEosType => s"$io.read_bytes_full()?" case b: BytesTerminatedType => - s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?" + s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?" case b: BytesLimitType => s"$io.read_bytes(${expression(b.size)} as usize)?" case BitsType1(bitEndian) => s"$io.read_bits_int(1)? != 0" case BitsType(width, bitEndian) => s"$io.read_bits_int($width)?" + case utfb: UserTypeFromBytes => s"&BytesReader::new(&_io" + + parseExpr(utfb.bytes.asInstanceOf[BytesLimitType], assignType, io, defEndian) + + ")" case _ => s"// parseExpr($dataType, $assignType, $io, $defEndian)" } @@ -485,8 +488,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) Nil } - override def allocateIO(varName: Identifier, rep: RepeatSpec): String = - s"// allocateIO($varName, $rep)" + override def allocateIO(varName: Identifier, rep: RepeatSpec): String = "" + // s"// allocateIO($varName, $rep)" def switchTypeEnum(id: Identifier, st: SwitchType): Unit = { // Because Rust can't handle `AnyType` in the type hierarchy, From 585973c9dce82b5d379d142744570e1567dccdef Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Thu, 14 Jul 2022 20:10:00 +0200 Subject: [PATCH 011/153] clippy doesn't like borrowing of io.read_bytes(...), use allocateIO(...) --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index a8a67ce08..fcbddf70a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -438,7 +438,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case b: BytesLimitType => s"$io.read_bytes(${expression(b.size)} as usize)?" case BitsType1(bitEndian) => s"$io.read_bits_int(1)? != 0" case BitsType(width, bitEndian) => s"$io.read_bits_int($width)?" - case utfb: UserTypeFromBytes => s"&BytesReader::new(&_io" + + case utfb: UserTypeFromBytes => s"&BytesReader::new(" + parseExpr(utfb.bytes.asInstanceOf[BytesLimitType], assignType, io, defEndian) + ")" case _ => s"// parseExpr($dataType, $assignType, $io, $defEndian)" @@ -488,8 +488,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) Nil } - override def allocateIO(varName: Identifier, rep: RepeatSpec): String = "" - // s"// allocateIO($varName, $rep)" + override def allocateIO(varName: Identifier, rep: RepeatSpec): String = privateMemberName(IoIdentifier) def switchTypeEnum(id: Identifier, st: SwitchType): Unit = { // Because Rust can't handle `AnyType` in the type hierarchy, From ba4a3c246f08e314837ebb0eb7b881fe3130914b Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sun, 17 Jul 2022 14:47:12 +0800 Subject: [PATCH 012/153] SwitchType implemented. --- .../struct/languages/RustCompiler.scala | 190 ++++++++++++++---- 1 file changed, 151 insertions(+), 39 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index a8a67ce08..4bd90e08c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -18,6 +18,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with SingleOutputFile with UpperCamelCaseClasses with UniversalFooter + with SwitchIfOps with UniversalDoc { import RustCompiler._ @@ -35,13 +36,18 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def outImports(topClass: ClassSpec): String = importList.toList - .map(i => s"#[allow(unused_imports)]\nuse $i;") + .map(i => s"use $i;") .mkString("", "\n", "\n") override def fileHeader(topClassName: String): Unit = { outHeader.puts(s"// $headerComment") outHeader.puts + outHeader.puts("#![allow(unused_imports)]") + outHeader.puts("#![allow(non_snake_case)]") + outHeader.puts("#![allow(non_camel_case_types)]") + outHeader.puts + importList.add( "kaitai::*" ) @@ -55,7 +61,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def classHeader(name: List[String]): Unit = { out.puts - out.puts("#[allow(non_camel_case_types)]") out.puts("#[derive(Default, Debug, PartialEq)]") out.puts(s"pub struct ${classTypeName(typeProvider.nowClass)} {") out.inc @@ -187,14 +192,29 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"// condIfHeader($expr)") } + override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { + out.puts(s"${privateMemberName(id)} = Vec::new();") + } + override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = { - // TODO: Actual implementation, this is a shim to enable compiling out.puts("{") out.inc + out.puts(s"type ArrayElement = ${kaitaiTypeToNativeType(id, typeProvider.nowClass, dataType, excludeOptionWrapper = true)};") + out.puts(s"while !_io.is_eof() {") + out.inc + } + + override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = { + out.puts(s"${privateMemberName(id)}.push(Self::read_into::(_io, _root, _parent.push(self))?);"); + } - out.puts(s"// condRepeatEosHeader($id, $io, $dataType)") + override def condRepeatEosFooter: Unit = { + out.dec + out.puts("}") + out.dec + out.puts("}") } override def condRepeatExprHeader(id: Identifier, @@ -340,7 +360,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val enumClass = types2class(curClass ::: List(enumName)) // Set up the actual enum definition - out.puts(s"#[allow(non_camel_case_types)]") out.puts(s"#[derive(Debug, PartialEq)]") out.puts(s"pub enum $enumClass {") out.inc @@ -389,9 +408,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"// universalDoc()") } - override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = - out.puts(s"// handleAssignmentRepeatEos($id, $expr)") - override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = out.puts(s"// handleAssignmentRepeatExpr($id, $expr)") @@ -408,17 +424,18 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts( s"${privateMemberName(id)} = Some(($expr as i64).try_into()?);" ) - case _: SwitchType | _: BytesLimitType => + case _: BytesLimitType => out.puts(s"${privateMemberName(id)} = $expr.to_vec();") + case t: SwitchType => + out.puts(s"${privateMemberName(id)} = Some($expr);"); case t: UserType => - out.puts(s"{"); - out.inc - val tempVarName = localTemporaryName(id) - out.puts(s"let mut $tempVarName = ${kaitaiTypeToNativeType(id, typeProvider.nowClass, t, excludeOptionWrapper = true)}::default();") - out.puts(s"$tempVarName.read($expr, Some(self), _parent.push(self))?;") - out.puts(s"${privateMemberName(id)} = Some($tempVarName);"); - out.dec - out.puts(s"}") + val e = s"$expr" + val streamType = if (e == privateMemberName(IoIdentifier)) { + "S" + } else { + "BytesReader" + } + out.puts(s"${privateMemberName(id)} = Some(Self::read_into::<$streamType, ${kaitaiTypeToNativeType(id, typeProvider.nowClass, t, excludeOptionWrapper = true)}>($expr, Some(self), _parent.push(self))?);"); case _ => { out.puts(s"${privateMemberName(id)} = $expr;") } @@ -441,6 +458,36 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case utfb: UserTypeFromBytes => s"&BytesReader::new(&_io" + parseExpr(utfb.bytes.asInstanceOf[BytesLimitType], assignType, io, defEndian) + ")" + case t: UserType => + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") + val addArgs = if (t.isOpaque) { + "" + } else { + val parent = t.forcedParent match { + case Some(USER_TYPE_NO_PARENT) => "KStructUnit::parent_stack()" + case Some(fp) => translator.translate(fp) + case None => s"${privateMemberName(ParentIdentifier)}.push(self)" + } + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}" + case _ => "" + } + s", ${privateMemberName(RootIdentifier)}, $parent$addEndian" + } + val streamType = if (io == privateMemberName(IoIdentifier)) { + "S" + } else { + "BytesReader" + } + val userType = t match { + case t: UserType => + val baseName = t.classSpec match { + case Some(spec) => types2class(spec.name) + case None => types2class(t.name) + } + s"$baseName" + } + s"Self::read_into::<$streamType, $userType>($addParams$io$addArgs)?.into()" case _ => s"// parseExpr($dataType, $assignType, $io, $defEndian)" } @@ -470,17 +517,66 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" - override def switchStart(id: Identifier, on: Ast.expr): Unit = - out.puts(s"// switchStart($id, $on)") + override def switchRequiresIfs(onType: DataType): Boolean = onType match { + case _: IntType | _: EnumType => false + case _ => true + } + + override def switchStart(id: Identifier, on: Ast.expr): Unit = { + out.puts(s"match ${expression(on)} {") + out.inc + } + + override def switchCaseStart(condition: Ast.expr): Unit = { + out.puts(s"${expression(condition)} => {") + out.inc + } + + override def switchCaseEnd(): Unit = { + out.dec + out.puts("}") + } + + override def switchElseStart(): Unit = { + out.puts("// switchElseStart()") + } + + override def switchEnd(): Unit = { + out.puts("_ => panic!(\"unhandled value\")") + out.dec + out.puts("}") + } + + override def switchIfStart(id: Identifier, on: Ast.expr, onType: DataType): Unit = { + out.puts("{") + out.inc + out.puts(s"let on = &${expression(on)};") + } - override def switchCaseStart(condition: Ast.expr): Unit = - out.puts(s"// switchCaseStart($condition)") + override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { + out.puts(s"if on == ${expression(condition)} {") + out.inc + } - override def switchCaseEnd(): Unit = out.puts(s"// switchCaseEnd()") + override def switchIfCaseStart(condition: Ast.expr): Unit = { + out.puts(s"else if on == ${expression(condition)} {") + out.inc + } - override def switchElseStart(): Unit = out.puts(s"// switchElseStart()") + override def switchIfCaseEnd(): Unit = { + out.dec + out.puts("}") + } - override def switchEnd(): Unit = out.puts(s"// switchEnd()") + override def switchIfElseStart(): Unit = { + out.puts("else {") + out.inc + } + + override def switchIfEnd(): Unit = { + out.dec + out.puts("}") + } override def extraAttrForIO(id: Identifier, rep: RepeatSpec): List[AttrSpec] = { @@ -489,20 +585,18 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def allocateIO(varName: Identifier, rep: RepeatSpec): String = "" - // s"// allocateIO($varName, $rep)" def switchTypeEnum(id: Identifier, st: SwitchType): Unit = { // Because Rust can't handle `AnyType` in the type hierarchy, // we generate an enum with all possible variations - val typeName = kaitaiTypeToNativeType( + val enum_typeName = kaitaiTypeToNativeType( id, typeProvider.nowClass, st, excludeOptionWrapper = true ) - out.puts("#[allow(non_camel_case_types)]") out.puts("#[derive(Debug, PartialEq)]") - out.puts(s"pub enum $typeName {") + out.puts(s"pub enum $enum_typeName {") out.inc val types = st.cases.values.toSet @@ -520,6 +614,28 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec out.puts("}") + + // add helper methods From + types.foreach(t => { + // Because this switch type will itself be in an option, we can exclude it from user types + val variantName = switchVariantName(id, t) + val typeName = kaitaiTypeToNativeType( + id, + typeProvider.nowClass, + t, + excludeOptionWrapper = true + ) + out.puts(s"impl From<$typeName> for $enum_typeName {") + out.inc + out.puts(s"fn from(v: $typeName) -> Self {") + out.inc + out.puts(s"Self::$variantName(v)") + out.dec + out.puts("}") + out.dec + out.puts("}") + }) + out.puts } def switchVariantName(id: Identifier, attrType: DataType): String = @@ -564,10 +680,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case t: ArrayType => s"Arr${switchVariantName(id, t.elType)}" } - override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { - out.puts(s"// condRepeatCommonInit($id, $dataType, $needRaw)") - } - override def ksErrorName(err: io.kaitai.struct.datatype.KSError): String = s"KaitaiStream.$err" } @@ -696,16 +808,16 @@ object RustCompiler s"Vec<${kaitaiTypeToNativeType(id, cs, t.elType, excludeOptionWrapper = true, excludeLifetime = excludeLifetime)}>" case st: SwitchType => - val types = st.cases.values.toSet - val lifetime = - if (!excludeLifetime && types.exists(containsReferences)) - s"<$streamLife>" - else "" + // val types = st.cases.values.toSet + // val lifetime = + // if (!excludeLifetime && types.exists(containsReferences)) + // s"<$streamLife>" + // else "" val typeName = id match { case name: NamedIdentifier => - s"${types2class(cs.name ::: List(name.name))}$lifetime" + s"${types2class(cs.name ::: List(name.name))}" case name: InstanceIdentifier => - s"${types2class(cs.name ::: List(name.name))}$lifetime" + s"${types2class(cs.name ::: List(name.name))}" case _ => kstructUnitName } From b93ede151b05512413f1490bcdc7b550f2272d59 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 18 Jul 2022 12:49:02 +0800 Subject: [PATCH 013/153] cast type into directly (help with enums). --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 4bd90e08c..96c7c57db 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -448,7 +448,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) defEndian: Option[FixedEndian]): String = dataType match { case IntMultiType(_, _, None) => "panic!(\"Unable to parse unknown-endian integers\")" - case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?" + case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?.into()" case _: BytesEosType => s"$io.read_bytes_full()?" case b: BytesTerminatedType => s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?" From 4a31b6cce39d57c4270d55338aaca84ae4b8e8ce Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Mon, 18 Jul 2022 18:27:21 +0200 Subject: [PATCH 014/153] kaitaiPrimitiveToNativeType(...) processes EnumType (not checked yet) --- .../src/main/scala/io/kaitai/struct/languages/RustCompiler.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index fcbddf70a..a661d522a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -732,6 +732,7 @@ object RustCompiler case _: BooleanType => "bool" case CalcIntType => "i32" case CalcFloatType => "f64" + case EnumType(_, basedOn) => kaitaiPrimitiveToNativeType(basedOn) //??? case _: StrType => s"String" case _: BytesType => s"Vec" From 11fd4d191ce2629c9d22136143d6d89613ce2c8d Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 19 Jul 2022 21:56:10 +0800 Subject: [PATCH 015/153] generate helper function for SwitchEnum type. --- .../struct/languages/RustCompiler.scala | 67 ++++++++++++++++++- .../struct/translators/RustTranslator.scala | 4 +- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 96c7c57db..3e12a67d1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -160,10 +160,42 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"pub ${idToStr(attrName)}: $typeName,") } - // Intentional no-op; Rust handles ownership, so don't worry about reader methods override def attributeReader(attrName: Identifier, attrType: DataType, - isNullable: Boolean): Unit = {} + isNullable: Boolean): Unit = { + val typeName = attrName match { + // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct + case IoIdentifier | RootIdentifier | ParentIdentifier => return + case _ => + kaitaiTypeToNativeType(attrName, typeProvider.nowClass, attrType) + } + val typeNameEx = kaitaiTypeToNativeType(attrName, typeProvider.nowClass, attrType, excludeOptionWrapper = true) + out.puts( + s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") + out.inc + val enum_typename = attrType match { + case _: EnumType => true + case _: SwitchType => true + case _ => false + } + if (enum_typename) { + out.puts(s"fn ${idToStr(attrName)}(&self) -> usize {") + out.inc + out.puts(s"self.${idToStr(attrName)}.as_ref().unwrap().into()") + } else { + out.puts(s"fn ${idToStr(attrName)}(&self) -> &$typeNameEx {") + out.inc + if (typeName != typeNameEx) { + out.puts(s"self.${idToStr(attrName)}.as_ref().unwrap()") + } else { + out.puts(s"&self.${idToStr(attrName)}") + } + } + out.dec + out.puts("}") + out.dec + out.puts("}") + } override def attrParse(attr: AttrLikeSpec, id: Identifier, @@ -634,7 +666,38 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") out.dec out.puts("}") + // + out.puts(s"impl From<&$enum_typeName> for $typeName {") + out.inc + out.puts(s"fn from(e: &$enum_typeName) -> Self {") + out.inc + out.puts(s"if let $enum_typeName::$variantName(v) = e {") + out.inc + out.puts(s"return *v") + out.dec + out.puts("}") + out.puts(s"""panic!(\"trying to convert from enum $enum_typeName::$variantName to $typeName, enum value {:?}\", e)""") + out.dec + out.puts("}") + out.dec + out.puts("}") }) + out.puts(s"impl From<&$enum_typeName> for usize {") + out.inc + out.puts(s"fn from(e: &$enum_typeName) -> Self {") + out.inc + out.puts(s"match e {") + out.inc + types.foreach(t => { + val variantName = switchVariantName(id, t) + out.puts(s"$enum_typeName::$variantName(v) => *v as usize,") + }) + out.dec + out.puts("}") + out.dec + out.puts("}") + out.dec + out.puts("}") out.puts } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index db9eabd7a..093c31866 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -57,7 +57,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _ => if (provider.nowClass.seq.exists(a => a.id != IoIdentifier && a.id == NamedIdentifier(s))) { // If the name is part of the `seq` parse list, it's safe to return as-is - s"self.${doName(s)}" + s"self.${doName(s)}()" } else if (provider.nowClass.instances.contains(InstanceIdentifier(s))) { // It's an instance, we need to safely handle lookup s"self.${doName(s)}(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)}, ${privateMemberName(ParentIdentifier)})?" @@ -100,7 +100,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } override def enumToInt(v: expr, et: EnumType): String = - translate(v) + s"usize::from(${translate(v)})" override def boolToInt(v: expr): String = s"${translate(v)} as i32" From 06148b5abee36ea26bf809408a3b7da0b0aa5008 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 20 Jul 2022 09:45:29 +0800 Subject: [PATCH 016/153] condIfHeader implemented. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 3e12a67d1..970862a2c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -217,11 +217,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"// attrParseHybrid(${leProc()}, ${beProc()})") override def condIfHeader(expr: Ast.expr): Unit = { - // TODO: Actual implementation, this is a shim to enable compiling - out.puts("{") + out.puts(s"if ${expression(expr)} {") out.inc - - out.puts(s"// condIfHeader($expr)") } override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { From f866710da64f1b27304674719c094763515b3572 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 20 Jul 2022 10:54:09 +0800 Subject: [PATCH 017/153] generate SwitchEnum -> usize helper func only if enum is based on numeric types. --- .../struct/languages/RustCompiler.scala | 61 +++++++++++++------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 970862a2c..a22be52b3 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -173,12 +173,26 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts( s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") out.inc - val enum_typename = attrType match { - case _: EnumType => true - case _: SwitchType => true + + var types : Set[DataType] = Set() + var enum_typename = false + attrType match { + //TODO check + //case _: EnumType => enum_typename = true + case st: SwitchType => { + types = st.cases.values.toSet + enum_typename = true + } case _ => false } - if (enum_typename) { + var enum_only_numeric = true; + types.foreach(t => { + t match { + case _: NumericType => // leave unchanged + case _ => enum_only_numeric = false + } + }) + if (enum_typename && enum_only_numeric) { out.puts(s"fn ${idToStr(attrName)}(&self) -> usize {") out.inc out.puts(s"self.${idToStr(attrName)}.as_ref().unwrap().into()") @@ -679,23 +693,32 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec out.puts("}") }) - out.puts(s"impl From<&$enum_typeName> for usize {") - out.inc - out.puts(s"fn from(e: &$enum_typeName) -> Self {") - out.inc - out.puts(s"match e {") - out.inc + var enum_only_numeric = true; types.foreach(t => { - val variantName = switchVariantName(id, t) - out.puts(s"$enum_typeName::$variantName(v) => *v as usize,") + t match { + case _: NumericType => // leave true + case _ => enum_only_numeric = false + } }) - out.dec - out.puts("}") - out.dec - out.puts("}") - out.dec - out.puts("}") - out.puts + if (enum_only_numeric) { + out.puts(s"impl From<&$enum_typeName> for usize {") + out.inc + out.puts(s"fn from(e: &$enum_typeName) -> Self {") + out.inc + out.puts(s"match e {") + out.inc + types.foreach(t => { + val variantName = switchVariantName(id, t) + out.puts(s"$enum_typeName::$variantName(v) => *v as usize,") + }) + out.dec + out.puts("}") + out.dec + out.puts("}") + out.dec + out.puts("}") + out.puts + } } def switchVariantName(id: Identifier, attrType: DataType): String = From 7dcf82e4659edbce64e4abce26e0872c423309c9 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 20 Jul 2022 11:22:47 +0800 Subject: [PATCH 018/153] restrict helper func from into int for numeric types only. --- .../struct/languages/RustCompiler.scala | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index a22be52b3..64a0bb678 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -658,6 +658,14 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec out.puts("}") + var enum_only_numeric = true; + types.foreach(t => { + t match { + case _: NumericType => // leave true + case _ => enum_only_numeric = false + } + }) + // add helper methods From types.foreach(t => { // Because this switch type will itself be in an option, we can exclude it from user types @@ -677,27 +685,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") out.dec out.puts("}") - // - out.puts(s"impl From<&$enum_typeName> for $typeName {") - out.inc - out.puts(s"fn from(e: &$enum_typeName) -> Self {") - out.inc - out.puts(s"if let $enum_typeName::$variantName(v) = e {") - out.inc - out.puts(s"return *v") - out.dec - out.puts("}") - out.puts(s"""panic!(\"trying to convert from enum $enum_typeName::$variantName to $typeName, enum value {:?}\", e)""") - out.dec - out.puts("}") - out.dec - out.puts("}") - }) - var enum_only_numeric = true; - types.foreach(t => { - t match { - case _: NumericType => // leave true - case _ => enum_only_numeric = false + if (enum_only_numeric) { + out.puts(s"impl From<&$enum_typeName> for $typeName {") + out.inc + out.puts(s"fn from(e: &$enum_typeName) -> Self {") + out.inc + out.puts(s"if let $enum_typeName::$variantName(v) = e {") + out.inc + out.puts(s"return *v") + out.dec + out.puts("}") + out.puts(s"""panic!(\"trying to convert from enum $enum_typeName::$variantName to $typeName, enum value {:?}\", e)""") + out.dec + out.puts("}") + out.dec + out.puts("}") } }) if (enum_only_numeric) { From b61b4f7165b74bfb543772a0e1685a9d6d3f9560 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Wed, 20 Jul 2022 21:36:34 +0200 Subject: [PATCH 019/153] correct reading enums, Option> --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 3b3b4d9bd..30a084123 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -134,7 +134,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"${privateMemberName(RootIdentifier)}: Option<&$readLife Self::Root>," ) out.puts( - s"${privateMemberName(ParentIdentifier)}: TypedStack" + s"${privateMemberName(ParentIdentifier)}: Option>" ) out.dec out.puts(s") -> KResult<()> {") @@ -317,7 +317,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"${privateMemberName(RootIdentifier)}: Option<&$readLife ${rootClassTypeName(typeProvider.nowClass)}>," ) out.puts( - s"${privateMemberName(ParentIdentifier)}: TypedStack<${parentStackTypeName(typeProvider.nowClass)}>" + s"${privateMemberName(ParentIdentifier)}: Option>" ) out.dec val typeName = kaitaiTypeToNativeType( @@ -448,7 +448,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) defEndian: Option[FixedEndian]): String = dataType match { case IntMultiType(_, _, None) => "panic!(\"Unable to parse unknown-endian integers\")" - case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?.into()" + case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?" case _: BytesEosType => s"$io.read_bytes_full()?" case b: BytesTerminatedType => s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?" From 3ebcf838407dc33e0cc6d9549fed6a49bcde36be Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 21 Jul 2022 09:30:14 +0800 Subject: [PATCH 020/153] Switch implemented. --- .../struct/languages/RustCompiler.scala | 78 +++++++++++++------ .../struct/translators/RustTranslator.scala | 2 +- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 64a0bb678..0d0e86d8c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -46,6 +46,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) outHeader.puts("#![allow(unused_imports)]") outHeader.puts("#![allow(non_snake_case)]") outHeader.puts("#![allow(non_camel_case_types)]") + outHeader.puts("#![allow(irrefutable_let_patterns)]") outHeader.puts importList.add( @@ -176,14 +177,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) var types : Set[DataType] = Set() var enum_typename = false - attrType match { - //TODO check - //case _: EnumType => enum_typename = true + var simple_typename = attrType match { + case _: UserType => false + case _: BytesType => false + case _: StrType => false + case _: ArrayType => false case st: SwitchType => { types = st.cases.values.toSet enum_typename = true + false } - case _ => false + case _: EnumType => { + // TODO? enum_typename = true + false + } + case _ => true } var enum_only_numeric = true; types.foreach(t => { @@ -193,16 +201,24 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } }) if (enum_typename && enum_only_numeric) { - out.puts(s"fn ${idToStr(attrName)}(&self) -> usize {") + out.puts(s"pub fn ${idToStr(attrName)}(&self) -> usize {") out.inc out.puts(s"self.${idToStr(attrName)}.as_ref().unwrap().into()") } else { - out.puts(s"fn ${idToStr(attrName)}(&self) -> &$typeNameEx {") + if (simple_typename) { + out.puts(s"pub fn ${idToStr(attrName)}(&self) -> $typeNameEx {") + } else { + out.puts(s"pub fn ${idToStr(attrName)}(&self) -> &$typeNameEx {") + } out.inc if (typeName != typeNameEx) { out.puts(s"self.${idToStr(attrName)}.as_ref().unwrap()") } else { - out.puts(s"&self.${idToStr(attrName)}") + if (simple_typename) { + out.puts(s"self.${idToStr(attrName)}") + } else { + out.puts(s"&self.${idToStr(attrName)}") + } } } out.dec @@ -352,7 +368,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) dataType: DataType, isNullable: Boolean): Unit = { - out.puts(s"fn ${idToStr(instName)}(") + out.puts(s"pub fn ${idToStr(instName)}(") out.inc out.puts("&mut self,") out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S,") @@ -379,7 +395,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc instanceReturn(instName, dataType) out.dec - out.puts(s"}") + out.puts("}") } override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { @@ -439,11 +455,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("_ => Err(KError::UnknownVariant(flag)),") out.dec - out.puts(s"}") + out.puts("}") out.dec - out.puts(s"}") + out.puts("}") out.dec - out.puts(s"}") + out.puts("}") out.puts } @@ -503,13 +519,28 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ")" case t: UserType => val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") + val userType = t match { + case t: UserType => + val baseName = t.classSpec match { + case Some(spec) => types2class(spec.name) + case None => types2class(t.name) + } + s"$baseName" + } val addArgs = if (t.isOpaque) { "" } else { + val currentType = classTypeName(typeProvider.nowClass); val parent = t.forcedParent match { case Some(USER_TYPE_NO_PARENT) => "KStructUnit::parent_stack()" case Some(fp) => translator.translate(fp) - case None => s"${privateMemberName(ParentIdentifier)}.push(self)" + case None => { + if (userType contains currentType) { + s"${privateMemberName(ParentIdentifier)}.push(self)" + } else { + s"${privateMemberName(ParentIdentifier)}" + } + } } val addEndian = t.classSpec.get.meta.endian match { case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}" @@ -522,14 +553,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } else { "BytesReader" } - val userType = t match { - case t: UserType => - val baseName = t.classSpec match { - case Some(spec) => types2class(spec.name) - case None => types2class(t.name) - } - s"$baseName" - } s"Self::read_into::<$streamType, $userType>($addParams$io$addArgs)?.into()" case _ => s"// parseExpr($dataType, $assignType, $io, $defEndian)" } @@ -566,6 +589,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def switchStart(id: Identifier, on: Ast.expr): Unit = { + switch_else_exist = false out.puts(s"match ${expression(on)} {") out.inc } @@ -580,12 +604,18 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } + var switch_else_exist = false + override def switchElseStart(): Unit = { - out.puts("// switchElseStart()") + switch_else_exist = true + out.puts("_ => {") + out.inc } override def switchEnd(): Unit = { - out.puts("_ => panic!(\"unhandled value\")") + if (!switch_else_exist) { + out.puts("_ => {}") + } out.dec out.puts("}") } @@ -593,7 +623,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def switchIfStart(id: Identifier, on: Ast.expr, onType: DataType): Unit = { out.puts("{") out.inc - out.puts(s"let on = &${expression(on)};") + out.puts(s"let on = ${expression(on)};") } override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 093c31866..4f0a5adc5 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -60,7 +60,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"self.${doName(s)}()" } else if (provider.nowClass.instances.contains(InstanceIdentifier(s))) { // It's an instance, we need to safely handle lookup - s"self.${doName(s)}(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)}, ${privateMemberName(ParentIdentifier)})?" + s"*self.${doName(s)}(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)}, ${privateMemberName(ParentIdentifier)})?" } else { // TODO: Is it possible to reach this block? RawIdentifier? s"self.${doName(s)}" From 5bcc7d6218bcae6ae3a4c575c1eadfe46f1d8dcf Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 21 Jul 2022 10:38:08 +0800 Subject: [PATCH 021/153] repeat-eos fixed. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 0d0e86d8c..53281ff03 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -260,13 +260,12 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) dataType: DataType): Unit = { out.puts("{") out.inc - out.puts(s"type ArrayElement = ${kaitaiTypeToNativeType(id, typeProvider.nowClass, dataType, excludeOptionWrapper = true)};") out.puts(s"while !_io.is_eof() {") out.inc } override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = { - out.puts(s"${privateMemberName(id)}.push(Self::read_into::(_io, _root, _parent.push(self))?);"); + out.puts(s"${privateMemberName(id)}.push($expr);"); } override def condRepeatEosFooter: Unit = { From 98f0a0fe92a9942b8acf526a48eb92a7939846be Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Thu, 21 Jul 2022 22:00:20 +0200 Subject: [PATCH 022/153] doInternalName --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index db9eabd7a..8d63c1f0f 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -70,6 +70,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doName(s: String) = s + override def doInternalName(id: Identifier): String = + s"${idToStr(id)}()" + override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" From 42eb073a692133766328c0e9bb045e071386315e Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Thu, 21 Jul 2022 22:00:46 +0200 Subject: [PATCH 023/153] generate `from_file(path)` --- .../struct/languages/RustCompiler.scala | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 30a084123..70739a64e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -51,7 +51,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add( "kaitai::*" ) - importList.add("std::convert::{TryFrom, TryInto}") + importList.add("std::{fs, path::PathBuf, convert::{TryFrom, TryInto}}") } override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = @@ -276,9 +276,22 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) RustCompiler.privateMemberName(id) override def instanceDeclHeader(className: List[String]): Unit = { - out.puts( - s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {" - ) + val code = + s"""impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} { + | pub fn from_file(path: &str) -> Self { + | let bytes = fs::read(path).unwrap(); + | let reader = BytesReader::new(&bytes); + | let mut obj = ${classTypeName(typeProvider.nowClass)}::default(); + | + | if let Err(err) = obj.read(&reader, None, None) { + | panic!("error '{:?}' reading from file '{}'", err, path); + | } + | + | obj + | } + |""".stripMargin + + out.puts(code) out.inc } @@ -442,6 +455,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } + override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = + out.puts(s"${kaitaiTypeToNativeType(NamedIdentifier(id), typeProvider.nowClass, dataType)} $id = $expr;") + override def parseExpr(dataType: DataType, assignType: DataType, io: String, @@ -846,6 +862,7 @@ object RustCompiler case CalcIntType => "i32" case CalcFloatType => "f64" case EnumType(_, basedOn) => kaitaiPrimitiveToNativeType(basedOn) //??? + case t: UserType => types2class(t.name) case _: StrType => s"String" case _: BytesType => s"Vec" From 2b86db0216d1c49abea1400fce90b3a0d3d8307b Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 22 Jul 2022 14:33:43 +0800 Subject: [PATCH 024/153] switch_manual_int_size.ksy implemented. --- .../struct/languages/RustCompiler.scala | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 53281ff03..3adabb56a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -482,18 +482,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts( s"${privateMemberName(id)} = Some(($expr as i64).try_into()?);" ) - case _: BytesLimitType => - out.puts(s"${privateMemberName(id)} = $expr.to_vec();") - case t: SwitchType => - out.puts(s"${privateMemberName(id)} = Some($expr);"); - case t: UserType => - val e = s"$expr" - val streamType = if (e == privateMemberName(IoIdentifier)) { - "S" - } else { - "BytesReader" - } - out.puts(s"${privateMemberName(id)} = Some(Self::read_into::<$streamType, ${kaitaiTypeToNativeType(id, typeProvider.nowClass, t, excludeOptionWrapper = true)}>($expr, Some(self), _parent.push(self))?);"); + case st: SwitchType => + out.puts(s"${privateMemberName(id)} = Some($expr);") case _ => { out.puts(s"${privateMemberName(id)} = $expr;") } @@ -510,12 +500,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesEosType => s"$io.read_bytes_full()?" case b: BytesTerminatedType => s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?" - case b: BytesLimitType => s"$io.read_bytes(${expression(b.size)} as usize)?" + case b: BytesLimitType => s"$io.read_bytes(${expression(b.size)} as usize)?.into()" case BitsType1(bitEndian) => s"$io.read_bits_int(1)? != 0" case BitsType(width, bitEndian) => s"$io.read_bits_int($width)?" - case utfb: UserTypeFromBytes => s"&BytesReader::new(&_io" + + case utfb: UserTypeFromBytes => + val userType = utfb match { + case t: UserType => + val baseName = t.classSpec match { + case Some(spec) => types2class(spec.name) + case None => types2class(t.name) + } + s"$baseName" + } + s"Self::read_into::(&BytesReader::new(" + parseExpr(utfb.bytes.asInstanceOf[BytesLimitType], assignType, io, defEndian) + - ")" + "), _root, _parent.push(self))?.into()" case t: UserType => val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") val userType = t match { @@ -650,13 +649,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def extraAttrForIO(id: Identifier, - rep: RepeatSpec): List[AttrSpec] = { - out.puts(s"// extraAttrForIO($id, $rep)") - Nil - } - - override def allocateIO(varName: Identifier, rep: RepeatSpec): String = "" + override def allocateIO(varName: Identifier, rep: RepeatSpec): String = privateMemberName(IoIdentifier) def switchTypeEnum(id: Identifier, st: SwitchType): Unit = { // Because Rust can't handle `AnyType` in the type hierarchy, @@ -699,17 +692,23 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) types.foreach(t => { // Because this switch type will itself be in an option, we can exclude it from user types val variantName = switchVariantName(id, t) - val typeName = kaitaiTypeToNativeType( - id, - typeProvider.nowClass, - t, - excludeOptionWrapper = true - ) + var v = "v" + val typeName = t match { + case _ : BytesType => { + v = "v.to_vec()" + s"&[u8]" // special case for Bytes(Vec[u8]) (else switch) + } + case _ => kaitaiTypeToNativeType( + id, + typeProvider.nowClass, + t, + excludeOptionWrapper = true) + } out.puts(s"impl From<$typeName> for $enum_typeName {") out.inc out.puts(s"fn from(v: $typeName) -> Self {") out.inc - out.puts(s"Self::$variantName(v)") + out.puts(s"Self::$variantName($v)") out.dec out.puts("}") out.dec From 052d61a361e7efe5b3be7d03f65da1829be0ca95 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 22 Jul 2022 16:22:46 +0800 Subject: [PATCH 025/153] small bug fixed. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 3adabb56a..4892337e9 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -497,7 +497,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) dataType match { case IntMultiType(_, _, None) => "panic!(\"Unable to parse unknown-endian integers\")" case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?.into()" - case _: BytesEosType => s"$io.read_bytes_full()?" + case _: BytesEosType => s"$io.read_bytes_full()?.into()" case b: BytesTerminatedType => s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?" case b: BytesLimitType => s"$io.read_bytes(${expression(b.size)} as usize)?.into()" From e7f856785f3b88a4d5ede69ece98e155b1c8a99f Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 22 Jul 2022 17:46:57 +0800 Subject: [PATCH 026/153] handle UserTypeFromBytes[BytesEosType]. --- .../io/kaitai/struct/languages/RustCompiler.scala | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 4892337e9..543345161 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -512,9 +512,14 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } s"$baseName" } - s"Self::read_into::(&BytesReader::new(" + - parseExpr(utfb.bytes.asInstanceOf[BytesLimitType], assignType, io, defEndian) + - "), _root, _parent.push(self))?.into()" + val expr = if (utfb.bytes.isInstanceOf[BytesLimitType]) { + parseExpr(utfb.bytes.asInstanceOf[BytesLimitType], assignType, io, defEndian) + } else if (utfb.bytes.isInstanceOf[BytesEosType]) { + parseExpr(utfb.bytes.asInstanceOf[BytesEosType], assignType, io, defEndian) + } else { + s"TODO: impl UserTypeFromBytes.asInstanceOf $utfb" + } + s"Self::read_into::(&BytesReader::new(" + expr + "), _root, _parent.push(self))?.into()" case t: UserType => val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") val userType = t match { From 407207b5bbfc336144aa910c8e4920a75ddcd1eb Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 22 Jul 2022 21:55:38 +0800 Subject: [PATCH 027/153] read_bytes_term().into() --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 543345161..ee5790af9 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -499,7 +499,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?.into()" case _: BytesEosType => s"$io.read_bytes_full()?.into()" case b: BytesTerminatedType => - s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?" + s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?.into()" case b: BytesLimitType => s"$io.read_bytes(${expression(b.size)} as usize)?.into()" case BitsType1(bitEndian) => s"$io.read_bits_int(1)? != 0" case BitsType(width, bitEndian) => s"$io.read_bits_int($width)?" From 0edd9c05a92e4b218c7540490ce096f8b7619b01 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Fri, 22 Jul 2022 21:29:46 +0200 Subject: [PATCH 028/153] out full path of generated file --- jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala index bd33da06e..541ff7981 100644 --- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala +++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala @@ -360,7 +360,7 @@ class JavaMain(config: CLIConfig) { ): SpecSuccess = { val res = Main.compile(specs, spec, lang, runtime) res.files.foreach { (file) => - Log.fileOps.info(() => s".... writing ${file.fileName}") + Log.fileOps.info(() => s".... writing $outDir/${file.fileName}") val outPath = new File(outDir + "/" + file.fileName) From d19d9dda7f9d0851195f9f551ffc14fb8bd03773 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Fri, 22 Jul 2022 21:31:12 +0200 Subject: [PATCH 029/153] reformat `handleAssignmentSimple` for `UserType` --- .../io/kaitai/struct/languages/RustCompiler.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 70739a64e..9c25121c5 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -448,7 +448,15 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } else { "BytesReader" } - out.puts(s"${privateMemberName(id)} = Some(Self::read_into::<$streamType, ${kaitaiTypeToNativeType(id, typeProvider.nowClass, t, excludeOptionWrapper = true)}>($expr, Some(self), _parent.push(self))?);"); + val code = + s""" + | ${privateMemberName(id)} = Some( + | Self::read_into::<$streamType, ${kaitaiTypeToNativeType(id, typeProvider.nowClass, t, excludeOptionWrapper = true)}> + | ($expr, + | Some(self), + | Some(_parent).push(self))?); + |""".stripMargin + out.puts(code); case _ => { out.puts(s"${privateMemberName(id)} = $expr;") } From d90a712163d100bf52550f8369e4aa776084b388 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 23 Jul 2022 08:15:38 +0800 Subject: [PATCH 030/153] universalDoc implemented. --- .../io/kaitai/struct/languages/RustCompiler.scala | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index ee5790af9..18898ea2f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -463,7 +463,19 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def universalDoc(doc: DocSpec): Unit = { - out.puts(s"// universalDoc()") + out.puts + out.puts( "/**") + + doc.summary.foreach(docStr => out.putsLines(" * ", docStr)) + + doc.ref.foreach { + case TextRef(text) => + out.putsLines(" * ", s"\\sa $text") + case UrlRef(url, text) => + out.putsLines(" * ", s"\\sa $url $text") + } + + out.puts( " */") } override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = From a7b98d807453f62f28bb233ac2e32ef9603359e5 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sun, 24 Jul 2022 13:30:49 +0800 Subject: [PATCH 031/153] instances usage fixed. --- .../struct/languages/RustCompiler.scala | 91 +++++++++++++------ .../struct/translators/RustTranslator.scala | 11 ++- 2 files changed, 71 insertions(+), 31 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 18898ea2f..d24547aa8 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -356,6 +356,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) attrType match { case _: ArrayType => out.puts(s"pub ${idToStr(attrName)}: $typeName,") + case _: UserType => // do nothing (instance return &UserType) case _ => out.puts(s"pub ${idToStr(attrName)}: Option<$typeName>,") } } @@ -390,6 +391,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { + val userType = dataType match { + case _: UserType => true + case _ => false + } + if (userType) { + return + } out.puts(s"if ${privateMemberName(instName)}.is_some() {") out.inc instanceReturn(instName, dataType) @@ -398,17 +406,29 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { - val primType = kaitaiPrimitiveToNativeType(dataType) - val converted = dataType match { - case _: StrType => out.puts(s"${privateMemberName(instName)} = Some(${expression(value)}.to_string());") - case _ => out.puts(s"${privateMemberName(instName)} = Some(${expression(value)} as $primType);") + dataType match { + case _: UserType => { + out.puts(s"Ok(${expression(value)})") + } + case _: StrType => { + out.puts(s"${privateMemberName(instName)} = Some(${expression(value)}.to_string());") + } + case _ => { + val primType = kaitaiPrimitiveToNativeType(dataType) + out.puts(s"${privateMemberName(instName)} = Some(${expression(value)} as $primType);") + } } - //handleAssignmentSimple(instName, s"${privateMemberName(instName)} = ${expression(value)}") } override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { - out.puts(s"return Ok(${privateMemberName(instName)}.as_ref().unwrap());") + val userType = attrType match { + case _: UserType => true + case _ => false + } + if (!userType) { + out.puts(s"return Ok(${privateMemberName(instName)}.as_ref().unwrap());") + } } override def enumDeclaration(curClass: List[String], @@ -488,17 +508,20 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def handleAssignmentSimple(id: Identifier, expr: String): Unit = { val seqId = typeProvider.nowClass.seq.find(s => s.id == id) - + var done = false; if (seqId.isDefined) seqId.get.dataType match { - case _: EnumType => + case et: EnumType => + done = true; out.puts( s"${privateMemberName(id)} = Some(($expr as i64).try_into()?);" ) case st: SwitchType => + done = true; out.puts(s"${privateMemberName(id)} = Some($expr);") - case _ => { - out.puts(s"${privateMemberName(id)} = $expr;") - } + case _ => done = false; + } + if (!done) { + out.puts(s"${privateMemberName(id)} = $expr;") } } @@ -515,23 +538,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case b: BytesLimitType => s"$io.read_bytes(${expression(b.size)} as usize)?.into()" case BitsType1(bitEndian) => s"$io.read_bits_int(1)? != 0" case BitsType(width, bitEndian) => s"$io.read_bits_int($width)?" - case utfb: UserTypeFromBytes => - val userType = utfb match { - case t: UserType => - val baseName = t.classSpec match { - case Some(spec) => types2class(spec.name) - case None => types2class(t.name) - } - s"$baseName" - } - val expr = if (utfb.bytes.isInstanceOf[BytesLimitType]) { - parseExpr(utfb.bytes.asInstanceOf[BytesLimitType], assignType, io, defEndian) - } else if (utfb.bytes.isInstanceOf[BytesEosType]) { - parseExpr(utfb.bytes.asInstanceOf[BytesEosType], assignType, io, defEndian) - } else { - s"TODO: impl UserTypeFromBytes.asInstanceOf $utfb" - } - s"Self::read_into::(&BytesReader::new(" + expr + "), _root, _parent.push(self))?.into()" case t: UserType => val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") val userType = t match { @@ -666,7 +672,33 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def allocateIO(varName: Identifier, rep: RepeatSpec): String = privateMemberName(IoIdentifier) + override def allocateIO(id: Identifier, rep: RepeatSpec): String = {//= privateMemberName(IoIdentifier) + val memberName = privateMemberName(id) + val ioId = IoStorageIdentifier(id) + + val args = rep match { + case RepeatUntil(_) => translator.doName(Identifier.ITERATOR2) + case _ => privateMemberName(id) + } + + val newStreamRaw = s"${memberName}" + val ioName = rep match { + case NoRepeat => + val newStream = newStreamRaw + val localIO = localTemporaryName(ioId) + out.puts(s"let ${localIO} = BytesReader::new(&$newStream);") + s"&$localIO" + case _ => + newStreamRaw + // TODO + //val localIO = s"io_${idToStr(id)}" + // out.puts(s"$kstreamName* $localIO = $newStreamRaw;") + // out.puts(s"${privateMemberName(ioId)}->push_back($localIO);") + //localIO + } + + ioName + } def switchTypeEnum(id: Identifier, st: SwitchType): Unit = { // Because Rust can't handle `AnyType` in the type hierarchy, @@ -837,6 +869,7 @@ object RustCompiler case InstanceIdentifier(n) => n case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" case RawIdentifier(inner) => s"raw_${idToStr(inner)}" + case IoStorageIdentifier(inner) => s"io_${idToStr(inner)}" } def rootClassTypeName(c: ClassSpec, isRecurse: Boolean = false): String = { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 4f0a5adc5..b9d15dd3f 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -59,8 +59,15 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) // If the name is part of the `seq` parse list, it's safe to return as-is s"self.${doName(s)}()" } else if (provider.nowClass.instances.contains(InstanceIdentifier(s))) { - // It's an instance, we need to safely handle lookup - s"*self.${doName(s)}(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)}, ${privateMemberName(ParentIdentifier)})?" + val userType = provider.nowClass.instances.get(InstanceIdentifier(s)).get.dataTypeComposite match { + case _: UserType => true + case _ => false + } + if (userType) { + s"self.${doName(s)}(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)}, ${privateMemberName(ParentIdentifier)})?" + } else { + s"*self.${doName(s)}(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)}, ${privateMemberName(ParentIdentifier)})?" + } } else { // TODO: Is it possible to reach this block? RawIdentifier? s"self.${doName(s)}" From 66bd5dbac3551510e899885d0592939b97d3e7ad Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 25 Jul 2022 13:02:00 +0800 Subject: [PATCH 032/153] attrProcess implemented. --- .../struct/languages/RustCompiler.scala | 60 +++++++++++++++---- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index d24547aa8..34bb3ef1d 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -279,13 +279,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) io: String, dataType: DataType, repeatExpr: Ast.expr): Unit = { - // TODO: Actual implementation, this is a shim to enable compiling - out.puts("{") + val lenVar = s"l_${idToStr(id)}" + out.puts(s"let $lenVar = ${expression(repeatExpr)};") + out.puts(s"for _i in 0..$lenVar {") out.inc - - out.puts( - s"// condRepeatExprHeader($id, $io, $dataType, $repeatExpr)" - ) } override def condRepeatUntilHeader(id: Identifier, @@ -312,11 +309,46 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("} {}") } - override def attrProcess(proc: ProcessExpr, - varSrc: Identifier, - varDest: Identifier, - rep: RepeatSpec): Unit = - out.puts(s"// attrProcess($proc, $varSrc, $varDest, $rep)") + def getRawIdExpr(varName: Identifier, rep: RepeatSpec): String = { + val memberName = privateMemberName(varName) + rep match { + case NoRepeat => memberName + case _ => s"// TODO $memberName->at($memberName->size() - 1)" + } + } + + override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { + val srcExpr = getRawIdExpr(varSrc, rep) + + val expr = proc match { + case ProcessXor(xorValue) => + val procName = translator.detectType(xorValue) match { + case _: IntType => "process_xor_one" + case _: BytesType => "process_xor_many" + } + s"S::$procName(&$srcExpr, ${expression(xorValue)})" + case ProcessZlib => + s"S::process_zlib(&$srcExpr)" + case ProcessRotate(isLeft, rotValue) => + val expr = if (isLeft) { + expression(rotValue) + } else { + s"8 - (${expression(rotValue)})" + } + s"S::process_rotate_left(&$srcExpr, $expr)" + case ProcessCustom(name, args) => + val procClass = name.map((x) => type2class(x)).mkString("::") + val procName = s"_process_${idToStr(varSrc)}" + + //importListSrc.addLocal(outFileNameHeader(name.last)) + + val argList = args.map(expression).mkString(", ") + var argListInParens = if (argList.nonEmpty) s"($argList)" else "" + out.puts(s"$procClass $procName$argListInParens;") + s"$procName.decode(&$srcExpr)" + } + handleAssignment(varDest, expr, rep, false) + } override def useIO(ioEx: Ast.expr): String = s"// useIO($ioEx)" @@ -413,6 +445,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: StrType => { out.puts(s"${privateMemberName(instName)} = Some(${expression(value)}.to_string());") } + case _: BytesType => { + out.puts(s"let _t = ${expression(value)};") + out.puts(s"${privateMemberName(instName)} = Some(_t.to_vec());") + } case _ => { val primType = kaitaiPrimitiveToNativeType(dataType) out.puts(s"${privateMemberName(instName)} = Some(${expression(value)} as $primType);") @@ -499,7 +535,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = - out.puts(s"// handleAssignmentRepeatExpr($id, $expr)") + handleAssignmentRepeatEos(id, expr) override def handleAssignmentRepeatUntil(id: Identifier, expr: String, From 362a0f50edc6a3e8a9e4804835d54a4111cc4ea4 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 25 Jul 2022 13:39:39 +0800 Subject: [PATCH 033/153] root generation value fixed. --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 34bb3ef1d..6ac79f250 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -588,6 +588,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) "" } else { val currentType = classTypeName(typeProvider.nowClass); + val root = if (typeProvider.nowClass.isTopLevel) { + "Some(self)" + } else { + privateMemberName(RootIdentifier) + } val parent = t.forcedParent match { case Some(USER_TYPE_NO_PARENT) => "KStructUnit::parent_stack()" case Some(fp) => translator.translate(fp) @@ -603,7 +608,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}" case _ => "" } - s", ${privateMemberName(RootIdentifier)}, $parent$addEndian" + s", $root, $parent$addEndian" } val streamType = if (io == privateMemberName(IoIdentifier)) { "S" From b481020d1ce63d8e8dbb878cd7890ffe3c7d5591 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 25 Jul 2022 22:44:44 +0800 Subject: [PATCH 034/153] valid: & pos: implemented. --- .../struct/languages/RustCompiler.scala | 31 +++++++++++++------ .../struct/translators/RustTranslator.scala | 9 ++++-- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 6ac79f250..c331d8ec9 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -352,12 +352,14 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def useIO(ioEx: Ast.expr): String = s"// useIO($ioEx)" - override def pushPos(io: String): Unit = out.puts(s"// pushPos($io)") + override def pushPos(io: String): Unit = + out.puts(s"let _pos = $io.pos();") override def seek(io: String, pos: Ast.expr): Unit = - out.puts(s"// seek($io, $pos)") + out.puts(s"$io.seek(${expression(pos)})?;") - override def popPos(io: String): Unit = out.puts(s"// popPos($io)") + override def popPos(io: String): Unit = + out.puts(s"$io.seek(_pos)?;") override def alignToByte(io: String): Unit = out.puts(s"${privateMemberName(IoIdentifier)}.align_to_byte()?;") @@ -403,13 +405,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"pub fn ${idToStr(instName)}(") out.inc out.puts("&mut self,") - out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S,") - out.puts( - s"${privateMemberName(RootIdentifier)}: Option<&$readLife ${rootClassTypeName(typeProvider.nowClass)}>," - ) - out.puts( - s"${privateMemberName(ParentIdentifier)}: TypedStack<${parentStackTypeName(typeProvider.nowClass)}>" - ) + out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S") out.dec val typeName = kaitaiTypeToNativeType( instName, @@ -885,6 +881,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: io.kaitai.struct.datatype.KSError): String = s"KaitaiStream.$err" + + override def attrValidateExpr( + attrId: Identifier, + attrType: DataType, + checkExpr: Ast.expr, + err: KSError, + errArgs: List[Ast.expr] + ): Unit = { + val errArgsStr = errArgs.map(translator.translate).mkString(", ") + out.puts(s"if !(${expression(checkExpr)}) {") + out.inc + out.puts(s"""return Err(KError::ValidationNotEqual(r#"$errArgsStr"#.to_string()));""") + out.dec + out.puts("}") + } } object RustCompiler diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index b9d15dd3f..72a30ba7c 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -13,7 +13,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) import RustCompiler._ override def doByteArrayLiteral(arr: Seq[Byte]): String = - "&[" + arr.map(x => "%0#2x".format(x & 0xff)).mkString(", ") + "]" + "&[" + arr.map(x => "%0#2xu8".format(x & 0xff)).mkString(", ") + "].to_vec()" override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = s"pack('C*', ${elts.map(translate).mkString(", ")})" @@ -64,9 +64,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _ => false } if (userType) { - s"self.${doName(s)}(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)}, ${privateMemberName(ParentIdentifier)})?" + s"self.${doName(s)}(${privateMemberName(IoIdentifier)})?" } else { - s"*self.${doName(s)}(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)}, ${privateMemberName(ParentIdentifier)})?" + s"*self.${doName(s)}(${privateMemberName(IoIdentifier)})?" } } else { // TODO: Is it possible to reach this block? RawIdentifier? @@ -77,6 +77,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doName(s: String) = s + override def doInternalName(id: Identifier): String = + s"${doLocalName(idToStr(id))}" + override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" From ffed05609539a2af49d0145270818d2d0e48ab03 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 27 Jul 2022 20:46:54 +0800 Subject: [PATCH 035/153] switchenum fixed handle same typename (in case of different endianness). --- .../struct/languages/RustCompiler.scala | 127 ++++++++++-------- 1 file changed, 74 insertions(+), 53 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index c331d8ec9..07deaf5f4 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -47,7 +47,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) outHeader.puts("#![allow(non_snake_case)]") outHeader.puts("#![allow(non_camel_case_types)]") outHeader.puts("#![allow(irrefutable_let_patterns)]") + outHeader.puts("#![allow(unused_comparisons)]") outHeader.puts + outHeader.puts("extern crate kaitai;") importList.add( "kaitai::*" @@ -751,17 +753,25 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc val types = st.cases.values.toSet - types.foreach(t => { - // Because this switch type will itself be in an option, we can exclude it from user types - val variantName = switchVariantName(id, t) - val typeName = kaitaiTypeToNativeType( - id, - typeProvider.nowClass, - t, - excludeOptionWrapper = true - ) - out.puts(s"$variantName($typeName),") - }) + + { + var types_set = scala.collection.mutable.Set[String]() + types.foreach(t => { + // Because this switch type will itself be in an option, we can exclude it from user types + val variantName = switchVariantName(id, t) + val typeName = kaitaiTypeToNativeType( + id, + typeProvider.nowClass, + t, + excludeOptionWrapper = true + ) + val new_typename = types_set.add(typeName) + // same typename could be in case of different endianness + if (new_typename) { + out.puts(s"$variantName($typeName),") + } + }) + } out.dec out.puts("}") @@ -774,48 +784,55 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } }) - // add helper methods From - types.foreach(t => { - // Because this switch type will itself be in an option, we can exclude it from user types - val variantName = switchVariantName(id, t) - var v = "v" - val typeName = t match { - case _ : BytesType => { - v = "v.to_vec()" - s"&[u8]" // special case for Bytes(Vec[u8]) (else switch) + // generate only if switch types are different + { + var types_set = scala.collection.mutable.Set[String]() + // add helper methods From + types.foreach(t => { + // Because this switch type will itself be in an option, we can exclude it from user types + val variantName = switchVariantName(id, t) + var v = "v" + val typeName = t match { + case _ : BytesType => { + v = "v.to_vec()" + s"&[u8]" // special case for Bytes(Vec[u8]) (else switch) + } + case _ => kaitaiTypeToNativeType( + id, + typeProvider.nowClass, + t, + excludeOptionWrapper = true) } - case _ => kaitaiTypeToNativeType( - id, - typeProvider.nowClass, - t, - excludeOptionWrapper = true) - } - out.puts(s"impl From<$typeName> for $enum_typeName {") - out.inc - out.puts(s"fn from(v: $typeName) -> Self {") - out.inc - out.puts(s"Self::$variantName($v)") - out.dec - out.puts("}") - out.dec - out.puts("}") - if (enum_only_numeric) { - out.puts(s"impl From<&$enum_typeName> for $typeName {") - out.inc - out.puts(s"fn from(e: &$enum_typeName) -> Self {") - out.inc - out.puts(s"if let $enum_typeName::$variantName(v) = e {") - out.inc - out.puts(s"return *v") - out.dec - out.puts("}") - out.puts(s"""panic!(\"trying to convert from enum $enum_typeName::$variantName to $typeName, enum value {:?}\", e)""") - out.dec - out.puts("}") - out.dec - out.puts("}") - } - }) + val new_typename = types_set.add(typeName) + if (new_typename) { + out.puts(s"impl From<$typeName> for $enum_typeName {") + out.inc + out.puts(s"fn from(v: $typeName) -> Self {") + out.inc + out.puts(s"Self::$variantName($v)") + out.dec + out.puts("}") + out.dec + out.puts("}") + if (enum_only_numeric) { + out.puts(s"impl From<&$enum_typeName> for $typeName {") + out.inc + out.puts(s"fn from(e: &$enum_typeName) -> Self {") + out.inc + out.puts(s"if let $enum_typeName::$variantName(v) = e {") + out.inc + out.puts(s"return *v") + out.dec + out.puts("}") + out.puts(s"""panic!(\"trying to convert from enum $enum_typeName::$variantName to $typeName, enum value {:?}\", e)""") + out.dec + out.puts("}") + out.dec + out.puts("}") + } + } + }) + } if (enum_only_numeric) { out.puts(s"impl From<&$enum_typeName> for usize {") out.inc @@ -823,9 +840,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc out.puts(s"match e {") out.inc + var variants_set = scala.collection.mutable.Set[String]() types.foreach(t => { val variantName = switchVariantName(id, t) - out.puts(s"$enum_typeName::$variantName(v) => *v as usize,") + val new_typename = variants_set.add(variantName) + if (new_typename) { + out.puts(s"$enum_typeName::$variantName(v) => *v as usize,") + } }) out.dec out.puts("}") From 5f301fd670fa029714999d10ddb6267d9ac5d250 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 28 Jul 2022 14:38:48 +0800 Subject: [PATCH 036/153] Report details if NotImplementedError. --- .../main/scala/io/kaitai/struct/Main.scala | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index 47bf1013b..d178b184a 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -81,24 +81,30 @@ object Main { */ def compile(specs: ClassSpecs, spec: ClassSpec, lang: LanguageCompilerStatic, conf: RuntimeConfig): CompileLog.SpecSuccess = { val config = updateConfig(conf, spec) - - val cc = lang match { - case GraphvizClassCompiler => - new GraphvizClassCompiler(specs, spec) - case GoCompiler => - new GoClassCompiler(specs, spec, config) - case RustCompiler => - new RustClassCompiler(specs, spec, config) - case ConstructClassCompiler => - new ConstructClassCompiler(specs, spec) - case NimCompiler => - new NimClassCompiler(specs, spec, config) - case HtmlClassCompiler => - new HtmlClassCompiler(specs, spec) - case _ => - new ClassCompiler(specs, spec, config, lang) + try { + val cc = lang match { + case GraphvizClassCompiler => + new GraphvizClassCompiler(specs, spec) + case GoCompiler => + new GoClassCompiler(specs, spec, config) + case RustCompiler => + new RustClassCompiler(specs, spec, config) + case ConstructClassCompiler => + new ConstructClassCompiler(specs, spec) + case NimCompiler => + new NimClassCompiler(specs, spec, config) + case HtmlClassCompiler => + new HtmlClassCompiler(specs, spec) + case _ => + new ClassCompiler(specs, spec, config, lang) + } + cc.compile + } catch { + case e: NotImplementedError => { + e.printStackTrace + throw e + } } - cc.compile } /** From c476a6491c52b6ba94908ee16a807d9377b49169 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 28 Jul 2022 14:42:09 +0800 Subject: [PATCH 037/153] unnamed var generation and usage fixed. --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 5 ++++- .../io/kaitai/struct/translators/RustTranslator.scala | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 07deaf5f4..6fbdffc9c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -559,6 +559,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } + override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = + out.puts(s"let $id = $expr;") + override def parseExpr(dataType: DataType, assignType: DataType, io: String, @@ -940,7 +943,7 @@ object RustCompiler case SpecialIdentifier(n) => n case NamedIdentifier(n) => n case InstanceIdentifier(n) => n - case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case NumberedIdentifier(idx) => s"${NumberedIdentifier.TEMPLATE}$idx" case RawIdentifier(inner) => s"raw_${idToStr(inner)}" case IoStorageIdentifier(inner) => s"io_${idToStr(inner)}" } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 72a30ba7c..c9ae825be 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -69,8 +69,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"*self.${doName(s)}(${privateMemberName(IoIdentifier)})?" } } else { - // TODO: Is it possible to reach this block? RawIdentifier? - s"self.${doName(s)}" + // unnamed identifier + s"self.${doName(s)}()" } } } @@ -83,6 +83,10 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" + override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr) = { + s"${translate(left)}.as_str() ${cmpOp(op)} ${translate(right)}" + } + override def doEnumById(enumTypeAbs: List[String], id: String) = // Just an integer, without any casts / resolutions - one would have to look up constants manually id From bb816c8790170c689a8e5be2361f455fb648c816 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Sat, 30 Jul 2022 20:56:32 +0200 Subject: [PATCH 038/153] up to date --- .../struct/languages/RustCompiler.scala | 118 ++++++++++++------ .../struct/translators/RustTranslator.scala | 22 +++- 2 files changed, 96 insertions(+), 44 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 26db3cc36..ad097115b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -279,13 +279,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) io: String, dataType: DataType, repeatExpr: Ast.expr): Unit = { - // TODO: Actual implementation, this is a shim to enable compiling - out.puts("{") + val lenVar = s"l_${idToStr(id)}" + out.puts(s"let $lenVar = ${expression(repeatExpr)};") + out.puts(s"for _i in 0..$lenVar {") out.inc - - out.puts( - s"// TODO: condRepeatExprHeader($id, $io, $dataType, $repeatExpr)" - ) } override def condRepeatUntilHeader(id: Identifier, @@ -312,20 +309,57 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("} {}") } - override def attrProcess(proc: ProcessExpr, - varSrc: Identifier, - varDest: Identifier, - rep: RepeatSpec): Unit = - out.puts(s"// TODO: attrProcess($proc, $varSrc, $varDest, $rep)") + def getRawIdExpr(varName: Identifier, rep: RepeatSpec): String = { + val memberName = privateMemberName(varName) + rep match { + case NoRepeat => memberName + case _ => s"// TODO $memberName->at($memberName->size() - 1)" + } + } + + override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { + val srcExpr = getRawIdExpr(varSrc, rep) + + val expr = proc match { + case ProcessXor(xorValue) => + val procName = translator.detectType(xorValue) match { + case _: IntType => "process_xor_one" + case _: BytesType => "process_xor_many" + } + s"S::$procName(&$srcExpr, ${expression(xorValue)})" + case ProcessZlib => + s"S::process_zlib(&$srcExpr)" + case ProcessRotate(isLeft, rotValue) => + val expr = if (isLeft) { + expression(rotValue) + } else { + s"8 - (${expression(rotValue)})" + } + s"S::process_rotate_left(&$srcExpr, $expr)" + case ProcessCustom(name, args) => + val procClass = name.map((x) => type2class(x)).mkString("::") + val procName = s"_process_${idToStr(varSrc)}" + + //importListSrc.addLocal(outFileNameHeader(name.last)) + + val argList = args.map(expression).mkString(", ") + var argListInParens = if (argList.nonEmpty) s"($argList)" else "" + out.puts(s"$procClass $procName$argListInParens;") + s"$procName.decode(&$srcExpr)" + } + handleAssignment(varDest, expr, rep, false) + } override def useIO(ioEx: Ast.expr): String = s"// TODO: useIO($ioEx)" - override def pushPos(io: String): Unit = out.puts(s"// TODO: pushPos($io)") + override def pushPos(io: String): Unit = + out.puts(s"let _pos = $io.pos();") override def seek(io: String, pos: Ast.expr): Unit = - out.puts(s"// TODO: seek($io, $pos)") + out.puts(s"$io.seek(${expression(pos)})?;") - override def popPos(io: String): Unit = out.puts(s"// TODO: popPos($io)") + override def popPos(io: String): Unit = + out.puts(s"$io.seek(_pos)?;") override def alignToByte(io: String): Unit = out.puts(s"${privateMemberName(IoIdentifier)}.align_to_byte()?;") @@ -334,22 +368,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) RustCompiler.privateMemberName(id) override def instanceDeclHeader(className: List[String]): Unit = { - val code = - s"""impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} { - | pub fn from_file(path: &str) -> Self { - | let bytes = fs::read(path).unwrap(); - | let reader = BytesReader::new(&bytes); - | let mut obj = ${classTypeName(typeProvider.nowClass)}::default(); - | - | if let Err(err) = obj.read(&reader, None, KStructUnit::parent_stack()) { - | panic!("error '{:?}' reading from file '{}'", err, path); - | } - | - | obj - | } - |""".stripMargin - - out.puts(code) + out.puts( + s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {" + ) out.inc } @@ -384,13 +405,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"pub fn ${idToStr(instName)}(") out.inc out.puts("&mut self,") - out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S,") - out.puts( - s"${privateMemberName(RootIdentifier)}: Option<&$readLife ${rootClassTypeName(typeProvider.nowClass)}>," - ) - out.puts( - s"${privateMemberName(ParentIdentifier)}: TypedStack<${parentStackTypeName(typeProvider.nowClass)}>" - ) + out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S") out.dec val typeName = kaitaiTypeToNativeType( instName, @@ -426,6 +441,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: StrType => { out.puts(s"${privateMemberName(instName)} = Some(${expression(value)}.to_string());") } + case _: BytesType => { + out.puts(s"let _t = ${expression(value)};") + out.puts(s"${privateMemberName(instName)} = Some(_t.to_vec());") + } case _ => { val primType = kaitaiPrimitiveToNativeType(dataType) out.puts(s"${privateMemberName(instName)} = Some(${expression(value)} as $primType);") @@ -512,7 +531,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = - out.puts(s"// TODO: handleAssignmentRepeatExpr($id, $expr)") + handleAssignmentRepeatEos(id, expr) override def handleAssignmentRepeatUntil(id: Identifier, expr: String, @@ -529,6 +548,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"${privateMemberName(id)} = Some(($expr as i64).try_into()?);" ) case _: BytesLimitType => + done = true; out.puts(s"${privateMemberName(id)} = $expr.to_vec();") case st: SwitchType => done = true; @@ -570,6 +590,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) "" } else { val currentType = classTypeName(typeProvider.nowClass); + val root = if (typeProvider.nowClass.isTopLevel) { + "Some(self)" + } else { + privateMemberName(RootIdentifier) + } val parent = t.forcedParent match { case Some(USER_TYPE_NO_PARENT) => "KStructUnit::parent_stack()" case Some(fp) => translator.translate(fp) @@ -585,7 +610,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}" case _ => "" } - s", ${privateMemberName(RootIdentifier)}, $parent$addEndian" + s", $root, $parent$addEndian" } val streamType = if (io == privateMemberName(IoIdentifier)) { "S" @@ -862,6 +887,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: io.kaitai.struct.datatype.KSError): String = s"KaitaiStream.$err" + + override def attrValidateExpr( + attrId: Identifier, + attrType: DataType, + checkExpr: Ast.expr, + err: KSError, + errArgs: List[Ast.expr] + ): Unit = { + val errArgsStr = errArgs.map(translator.translate).mkString(", ") + out.puts(s"if !(${expression(checkExpr)}) {") + out.inc + out.puts(s"""return Err(KError::ValidationNotEqual(r#"$errArgsStr"#.to_string()));""") + out.dec + out.puts("}") + } } object RustCompiler diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index a29ec5743..d14ecee2c 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -13,7 +13,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) import RustCompiler._ override def doByteArrayLiteral(arr: Seq[Byte]): String = - "&[" + arr.map(x => "%0#2x".format(x & 0xff)).mkString(", ") + "]" + "&[" + arr.map(x => "%0#2xu8".format(x & 0xff)).mkString(", ") + "].to_vec()" override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = s"pack('C*', ${elts.map(translate).mkString(", ")})" @@ -59,8 +59,15 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) // If the name is part of the `seq` parse list, it's safe to return as-is s"self.${doName(s)}()" } else if (provider.nowClass.instances.contains(InstanceIdentifier(s))) { - // It's an instance, we need to safely handle lookup - s"*self.${doName(s)}(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)}, ${privateMemberName(ParentIdentifier)})?" + val userType = provider.nowClass.instances.get(InstanceIdentifier(s)).get.dataTypeComposite match { + case _: UserType => true + case _ => false + } + if (userType) { + s"self.${doName(s)}(${privateMemberName(IoIdentifier)})?" + } else { + s"*self.${doName(s)}(${privateMemberName(IoIdentifier)})?" + } } else { // TODO: Is it possible to reach this block? RawIdentifier? s"self.${doName(s)}" @@ -71,10 +78,10 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doName(s: String) = s override def doInternalName(id: Identifier): String = - s"${idToStr(id)}()" + s"${doLocalName(idToStr(id))}" override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = - s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" + s"&${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" override def doEnumById(enumTypeAbs: List[String], id: String) = // Just an integer, without any casts / resolutions - one would have to look up constants manually @@ -142,4 +149,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${translate(a)}.iter().min()" override def arrayMax(a: Ast.expr): String = s"${translate(a)}.iter().max()" + + override def userTypeField(userType: UserType, value: Ast.expr, attrName: String): String = { + val code = s"${anyField(value, attrName)}()" + code + } } From b857343cec17e04a6613d09bb939c1b5407818b9 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Mon, 1 Aug 2022 23:50:00 +0200 Subject: [PATCH 039/153] override `attrParse2(...)` to process `EnumType` --- .../struct/languages/RustCompiler.scala | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 6fbdffc9c..d7c661564 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -1,7 +1,7 @@ package io.kaitai.struct.languages import io.kaitai.struct._ -import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype.DataType.{ReadableType, _} import io.kaitai.struct.datatype._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ @@ -920,6 +920,30 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec out.puts("}") } + + override def attrParse2( + id: Identifier, + dataType: DataType, + io: String, + rep: RepeatSpec, + isRaw: Boolean, + defEndian: Option[FixedEndian], + assignTypeOpt: Option[DataType] = None + ): Unit = { + dataType match { + case t: EnumType => + if(t.basedOn.isInstanceOf[ReadableType]) { + val expr = s"$io.read_${t.basedOn.asInstanceOf[ReadableType].apiCall(defEndian)}()?" + handleAssignment(id, expr, rep, isRaw) + } else { + out.puts( + "unimplemented!();" + ) + } + case _ => + super.attrParse2(id, dataType, io, rep, isRaw, defEndian, assignTypeOpt) + } + } } object RustCompiler From 948773eb2359259142d7be5286f7c7bd930b2dce Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 2 Aug 2022 15:57:21 +0800 Subject: [PATCH 040/153] correctly handle calling of instances. --- .../struct/translators/RustTranslator.scala | 86 +++++++++++++------ 1 file changed, 59 insertions(+), 27 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index c9ae825be..455585079 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -1,5 +1,8 @@ package io.kaitai.struct.translators +import io.kaitai.struct.format._ +import io.kaitai.struct.datatype._ + import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr @@ -44,38 +47,67 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } - override def doLocalName(s: String) = { - s match { - case Identifier.ITERATOR => "tmpa" - case Identifier.ITERATOR2 => "tmpb" - case Identifier.INDEX => "i" - case Identifier.IO => s"${RustCompiler.privateMemberName(IoIdentifier)}" - case Identifier.ROOT => s"${RustCompiler.privateMemberName(RootIdentifier)}.ok_or(KError::MissingRoot)?" - case Identifier.PARENT => - // TODO: How to handle _parent._parent? - s"${RustCompiler.privateMemberName(ParentIdentifier)}.peek()" - case _ => - if (provider.nowClass.seq.exists(a => a.id != IoIdentifier && a.id == NamedIdentifier(s))) { - // If the name is part of the `seq` parse list, it's safe to return as-is - s"self.${doName(s)}()" - } else if (provider.nowClass.instances.contains(InstanceIdentifier(s))) { - val userType = provider.nowClass.instances.get(InstanceIdentifier(s)).get.dataTypeComposite match { - case _: UserType => true - case _ => false - } - if (userType) { - s"self.${doName(s)}(${privateMemberName(IoIdentifier)})?" - } else { - s"*self.${doName(s)}(${privateMemberName(IoIdentifier)})?" - } + override def doName(s: String) = s match { + case _ => + val found = get_instance(get_top_class(provider.nowClass), InstanceIdentifier(s)) + if (found.isDefined) { + val userType = found.get.dataTypeComposite match { + case _: UserType => true + case _ => false + } + if (userType) { + s"$s(${privateMemberName(IoIdentifier)})?" } else { - // unnamed identifier - s"self.${doName(s)}()" + s"$s(${privateMemberName(IoIdentifier)})?" + } + } else { + s"$s()" + } + } + + def get_top_class(c: ClassSpec): ClassSpec = { + if (c.isTopLevel) { + return c + } + get_top_class(c.upClass.get) + } + + def get_instance(cs: ClassSpec, id: Identifier): Option[InstanceSpec] = { + var found : Option[InstanceSpec] = None; + // look for instance + cs.instances.foreach { case (instName, instSpec) => + if (instName == id) { + found = Some(instSpec); + } + } + // look deeper + if (found.isEmpty) { + cs.types.foreach { + case (_, typeSpec) => { + found = get_instance(typeSpec, id) + if (found.isDefined) { + return found; + } } + } } + return found; } - override def doName(s: String) = s + override def anyField(value: expr, attrName: String): String = + s"${translate(value)}.${doName(attrName)}" + + override def doLocalName(s: String) = s match { + case Identifier.ITERATOR => "tmpa" + case Identifier.ITERATOR2 => "tmpb" + case Identifier.INDEX => "i" + case Identifier.IO => s"${RustCompiler.privateMemberName(IoIdentifier)}" + case Identifier.ROOT => s"${RustCompiler.privateMemberName(RootIdentifier)}.ok_or(KError::MissingRoot)?" + case Identifier.PARENT => + // TODO: How to handle _parent._parent? + s"${RustCompiler.privateMemberName(ParentIdentifier)}.as_ref().unwrap().peek()" + case _ => s"self.${doName(s)}" + } override def doInternalName(id: Identifier): String = s"${doLocalName(idToStr(id))}" From 632a6ff1afee4829f2fc1e66a2f87cc1737ba98c Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 2 Aug 2022 15:58:01 +0800 Subject: [PATCH 041/153] instances generation and usage reviewd. --- .../struct/languages/RustCompiler.scala | 136 ++++++++++++++---- 1 file changed, 105 insertions(+), 31 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 6fbdffc9c..b166be227 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -137,7 +137,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"${privateMemberName(RootIdentifier)}: Option<&$readLife Self::Root>," ) out.puts( - s"${privateMemberName(ParentIdentifier)}: TypedStack" + s"${privateMemberName(ParentIdentifier)}: Option>" ) out.dec out.puts(s") -> KResult<()> {") @@ -179,11 +179,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) var types : Set[DataType] = Set() var enum_typename = false + var as_ref = false var simple_typename = attrType match { case _: UserType => false - case _: BytesType => false - case _: StrType => false - case _: ArrayType => false + case _: BytesType => { + as_ref = true + true + } + case _: StrType => { + as_ref = true + true + } + case _: ArrayType => { + as_ref = true + true + } case st: SwitchType => { types = st.cases.values.toSet enum_typename = true @@ -207,19 +217,23 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc out.puts(s"self.${idToStr(attrName)}.as_ref().unwrap().into()") } else { - if (simple_typename) { + if (as_ref) { + out.puts(s"pub fn ${idToStr(attrName)}(&self) -> &$typeNameEx {") + } else if (simple_typename) { out.puts(s"pub fn ${idToStr(attrName)}(&self) -> $typeNameEx {") } else { - out.puts(s"pub fn ${idToStr(attrName)}(&self) -> &$typeNameEx {") + out.puts(s"pub fn ${idToStr(attrName)}(&mut self) -> &mut $typeNameEx {") } out.inc if (typeName != typeNameEx) { - out.puts(s"self.${idToStr(attrName)}.as_ref().unwrap()") + out.puts(s"self.${idToStr(attrName)}.as_mut().unwrap()") } else { - if (simple_typename) { + if (as_ref) { + out.puts(s"&self.${idToStr(attrName)}") + } else if (simple_typename) { out.puts(s"self.${idToStr(attrName)}") } else { - out.puts(s"&self.${idToStr(attrName)}") + out.puts(s"&mut self.${idToStr(attrName)}") } } } @@ -392,18 +406,30 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) attrType match { case _: ArrayType => out.puts(s"pub ${idToStr(attrName)}: $typeName,") - case _: UserType => // do nothing (instance return &UserType) case _ => out.puts(s"pub ${idToStr(attrName)}: Option<$typeName>,") } } override def idToStr(id: Identifier): String = RustCompiler.idToStr(id) + def is_copy_type(dataType: DataType): Boolean = dataType match { + case _: UserType => false + case _: BytesType => false + case _: StrType => false + case _: ArrayType => false + case _ => true + } + + def is_mut_needed(dataType: DataType): Boolean = dataType match { + case _: UserType => true + case _ => false + } + override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { - + in_instance = true out.puts(s"pub fn ${idToStr(instName)}(") out.inc out.puts("&mut self,") @@ -415,30 +441,42 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) dataType, excludeOptionWrapper = true ) - out.puts(s") -> KResult<&$typeName> {") + if (is_copy_type(dataType)) { + out.puts(s") -> KResult<$typeName> {") + } else { + if (is_mut_needed(dataType)) { + out.puts(s") -> KResult<&mut $typeName> {") + } else { + out.puts(s") -> KResult<&$typeName> {") + } + } out.inc + out.puts(s"let _root : Option<&${classTypeName(typeProvider.topClass)}> = None;") } override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { - val userType = dataType match { - case _: UserType => true - case _ => false - } - if (userType) { - return - } out.puts(s"if ${privateMemberName(instName)}.is_some() {") out.inc - instanceReturn(instName, dataType) + if (is_copy_type(dataType)) { + out.puts(s"return Ok(${privateMemberName(instName)}.unwrap());") + } else { + if (is_mut_needed(dataType)) { + out.puts(s"return Ok(${privateMemberName(instName)}.as_mut().unwrap());") + } else { + out.puts(s"return Ok(${privateMemberName(instName)}.as_ref().unwrap());") + } + } out.dec out.puts("}") } + var in_instance = false + override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { dataType match { case _: UserType => { - out.puts(s"Ok(${expression(value)})") + out.puts(s"let ret = ${expression(value)};") } case _: StrType => { out.puts(s"${privateMemberName(instName)} = Some(${expression(value)}.to_string());") @@ -449,20 +487,30 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } case _ => { val primType = kaitaiPrimitiveToNativeType(dataType) - out.puts(s"${privateMemberName(instName)} = Some(${expression(value)} as $primType);") + out.puts(s"${privateMemberName(instName)} = Some((${expression(value)}) as $primType);") } } } override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { - val userType = attrType match { - case _: UserType => true - case _ => false - } - if (!userType) { - out.puts(s"return Ok(${privateMemberName(instName)}.as_ref().unwrap());") + attrType match { + case _: UserType => { + out.puts(s"Ok(ret)") + } + case _ => { + if (is_copy_type(attrType)) { + out.puts(s"Ok(${privateMemberName(instName)}.unwrap())") + } else { + if (is_mut_needed(attrType)) { + out.puts(s"Ok(${privateMemberName(instName)}.as_mut().unwrap())") + } else { + out.puts(s"Ok(${privateMemberName(instName)}.as_ref().unwrap())") + } + } + } } + in_instance = false } override def enumDeclaration(curClass: List[String], @@ -554,6 +602,28 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(id)} = Some($expr);") case _ => done = false; } + if (!done) { + id match { + case _: InstanceIdentifier => { + done = true; + out.puts(s"${privateMemberName(id)} = Some($expr);") + + if (in_instance) { + val found = translator.get_instance(translator.get_top_class(typeProvider.nowClass), id) + if (found.isDefined) { + val userType = found.get.dataTypeComposite match { + case _: UserType => true + case _ => false + } + if (userType) { + out.puts(s"let ret = ${privateMemberName(id)}.as_mut().unwrap();") + } + } + } + } + case _ => false + } + } if (!done) { out.puts(s"${privateMemberName(id)} = $expr;") } @@ -598,10 +668,14 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(USER_TYPE_NO_PARENT) => "KStructUnit::parent_stack()" case Some(fp) => translator.translate(fp) case None => { - if (userType contains currentType) { - s"${privateMemberName(ParentIdentifier)}.push(self)" + if ((userType contains currentType) && !in_instance) { + s"Some(${privateMemberName(ParentIdentifier)}.unwrap().push(self))" } else { - s"${privateMemberName(ParentIdentifier)}" + if(in_instance) { + s"None" + } else { + s"${privateMemberName(ParentIdentifier)}" + } } } } From bece5c1da8d57c7e7e7ecee1b5334c55c8ddb452 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 2 Aug 2022 16:58:58 +0800 Subject: [PATCH 042/153] useless code removed (base method is the same). --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 97616e493..93815cff8 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -185,9 +185,4 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${translate(a)}.iter().min()" override def arrayMax(a: Ast.expr): String = s"${translate(a)}.iter().max()" - - override def userTypeField(userType: UserType, value: Ast.expr, attrName: String): String = { - val code = s"${anyField(value, attrName)}()" - code - } } From 9f63df1b96a97effaec37e344f97454586777de6 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Tue, 2 Aug 2022 19:19:53 +0200 Subject: [PATCH 043/153] process `BitsType` --- .../kaitai/struct/languages/RustCompiler.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index fe78f4f4c..4a4edc4c2 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -1006,14 +1006,14 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ): Unit = { dataType match { case t: EnumType => - if(t.basedOn.isInstanceOf[ReadableType]) { - val expr = s"$io.read_${t.basedOn.asInstanceOf[ReadableType].apiCall(defEndian)}()?" - handleAssignment(id, expr, rep, isRaw) - } else { - out.puts( - "unimplemented!();" - ) - } + val expr = + t.basedOn match { + case inst: ReadableType => + s"$io.read_${inst.apiCall(defEndian)}()?" + case BitsType(width: Int, _) => + s"$io.read_bits_int($width)?" + } + handleAssignment(id, expr, rep, isRaw) case _ => super.attrParse2(id, dataType, io, rep, isRaw, defEndian, assignTypeOpt) } From 905073d38a50dcbbe80de0e326821d183b95dbb4 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 5 Aug 2022 16:43:04 +0800 Subject: [PATCH 044/153] instances refactoring (using RefCell) --- .../struct/languages/RustCompiler.scala | 176 +++++++++--------- .../struct/translators/RustTranslator.scala | 77 ++++++-- 2 files changed, 150 insertions(+), 103 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index fe78f4f4c..c56f918a2 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -55,6 +55,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) "kaitai::*" ) importList.add("std::convert::{TryFrom, TryInto}") + importList.add("std::cell::{Ref, Cell, RefCell}") } override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = @@ -160,7 +161,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) kaitaiTypeToNativeType(attrName, typeProvider.nowClass, attrType) } - out.puts(s"pub ${idToStr(attrName)}: $typeName,") + out.puts(s"${idToStr(attrName)}: $typeName,") } override def attributeReader(attrName: Identifier, @@ -179,31 +180,15 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) var types : Set[DataType] = Set() var enum_typename = false - var as_ref = false - var simple_typename = attrType match { - case _: UserType => false - case _: BytesType => { - as_ref = true - true - } - case _: StrType => { - as_ref = true - true - } - case _: ArrayType => { - as_ref = true - true - } + attrType match { case st: SwitchType => { types = st.cases.values.toSet enum_typename = true - false } case _: EnumType => { // TODO? enum_typename = true - false } - case _ => true + case _ => } var enum_only_numeric = true; types.foreach(t => { @@ -212,29 +197,30 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _ => enum_only_numeric = false } }) + var fn = idToStr(attrName) if (enum_typename && enum_only_numeric) { - out.puts(s"pub fn ${idToStr(attrName)}(&self) -> usize {") + out.puts(s"pub fn $fn(&self) -> usize {") out.inc out.puts(s"self.${idToStr(attrName)}.as_ref().unwrap().into()") - } else { - if (as_ref) { - out.puts(s"pub fn ${idToStr(attrName)}(&self) -> &$typeNameEx {") - } else if (simple_typename) { - out.puts(s"pub fn ${idToStr(attrName)}(&self) -> $typeNameEx {") + out.dec + out.puts("}") + fn = s"${fn}_enum" + } + { + if (typeName.startsWith("RefCell")) { + out.puts(s"pub fn $fn(&self) -> Ref<$typeNameEx> {") } else { - out.puts(s"pub fn ${idToStr(attrName)}(&mut self) -> &mut $typeNameEx {") + out.puts(s"pub fn $fn(&self) -> &$typeNameEx {") } out.inc if (typeName != typeNameEx) { - out.puts(s"self.${idToStr(attrName)}.as_mut().unwrap()") - } else { - if (as_ref) { - out.puts(s"&self.${idToStr(attrName)}") - } else if (simple_typename) { - out.puts(s"self.${idToStr(attrName)}") + if (typeName.startsWith("RefCell")) { + out.puts(s"self.${idToStr(attrName)}.borrow()") } else { - out.puts(s"&mut self.${idToStr(attrName)}") + out.puts(s"self.${idToStr(attrName)}.as_ref().unwrap()") } + } else { + out.puts(s"&self.${idToStr(attrName)}") } } out.dec @@ -338,11 +324,14 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val expr = proc match { case ProcessXor(xorValue) => - val procName = translator.detectType(xorValue) match { - case _: IntType => "process_xor_one" - case _: BytesType => "process_xor_many" + translator.detectType(xorValue) match { + case _: IntType => { + s"S::process_xor_one(&$srcExpr, ${expression(xorValue)})" + } + case _: BytesType => { + s"S::process_xor_many(&$srcExpr, &${translator.remove_deref(expression(xorValue))})" + } } - s"S::$procName(&$srcExpr, ${expression(xorValue)})" case ProcessZlib => s"S::process_zlib(&$srcExpr)" case ProcessRotate(isLeft, rotValue) => @@ -404,9 +393,38 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) attrType, excludeOptionWrapper = true ) - attrType match { - case _: ArrayType => out.puts(s"pub ${idToStr(attrName)}: $typeName,") - case _ => out.puts(s"pub ${idToStr(attrName)}: Option<$typeName>,") + out.puts(s"${calculatedFlagForName(attrName)}: Cell,") + out.puts(s"${idToStr(attrName)}: RefCell<$typeName>,") + } + + def calculatedFlagForName(ksName: Identifier) = + s"f_${idToStr(ksName)}" + + override def instanceClear(instName: InstanceIdentifier): Unit = { + var set = false; + val ins = translator.get_instance(typeProvider.nowClass, instName) + if (ins.isDefined) { + set = ins.get.dataTypeComposite match { + case _: UserType => true + case _ => false + } + } + if (!set) { + out.puts(s"self.${calculatedFlagForName(instName)}.set(false);") + } + } + + override def instanceSetCalculated(instName: InstanceIdentifier): Unit = { + var set = false; + val ins = translator.get_instance(typeProvider.nowClass, instName) + if (ins.isDefined) { + set = ins.get.dataTypeComposite match { + case _: UserType => true + case _ => false + } + } + if (!set) { + out.puts(s"self.${calculatedFlagForName(instName)}.set(true);") } } @@ -432,7 +450,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) in_instance = true out.puts(s"pub fn ${idToStr(instName)}(") out.inc - out.puts("&mut self,") + out.puts("&self,") out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S") out.dec val typeName = kaitaiTypeToNativeType( @@ -441,32 +459,16 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) dataType, excludeOptionWrapper = true ) - if (is_copy_type(dataType)) { - out.puts(s") -> KResult<$typeName> {") - } else { - if (is_mut_needed(dataType)) { - out.puts(s") -> KResult<&mut $typeName> {") - } else { - out.puts(s") -> KResult<&$typeName> {") - } - } + out.puts(s") -> KResult> {") out.inc out.puts(s"let _root : Option<&${classTypeName(typeProvider.topClass)}> = None;") } override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { - out.puts(s"if ${privateMemberName(instName)}.is_some() {") + out.puts(s"if self.${calculatedFlagForName(instName)}.get() {") out.inc - if (is_copy_type(dataType)) { - out.puts(s"return Ok(${privateMemberName(instName)}.unwrap());") - } else { - if (is_mut_needed(dataType)) { - out.puts(s"return Ok(${privateMemberName(instName)}.as_mut().unwrap());") - } else { - out.puts(s"return Ok(${privateMemberName(instName)}.as_ref().unwrap());") - } - } + out.puts(s"return Ok(${privateMemberName(instName)}.borrow());") out.dec out.puts("}") } @@ -476,39 +478,31 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { dataType match { case _: UserType => { - out.puts(s"let ret = ${expression(value)};") + out.puts(s"let ret = ${translator.remove_deref(expression(value))};") } case _: StrType => { - out.puts(s"${privateMemberName(instName)} = Some(${expression(value)}.to_string());") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.to_string();") } case _: BytesType => { - out.puts(s"let _t = ${expression(value)};") - out.puts(s"${privateMemberName(instName)} = Some(_t.to_vec());") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = (${translator.remove_deref(expression(value))}).to_vec().clone();") } case _ => { val primType = kaitaiPrimitiveToNativeType(dataType) - out.puts(s"${privateMemberName(instName)} = Some((${expression(value)}) as $primType);") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = (${expression(value)}) as $primType;") } } } override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { - attrType match { - case _: UserType => { - out.puts(s"Ok(ret)") - } - case _ => { - if (is_copy_type(attrType)) { - out.puts(s"Ok(${privateMemberName(instName)}.unwrap())") - } else { - if (is_mut_needed(attrType)) { - out.puts(s"Ok(${privateMemberName(instName)}.as_mut().unwrap())") - } else { - out.puts(s"Ok(${privateMemberName(instName)}.as_ref().unwrap())") - } - } - } + val userType = attrType match { + case _: UserType => true + case _ => false + } + if (userType) { + out.puts(s"Ok(ret)") + } else { + out.puts(s"Ok(${privateMemberName(instName)}.borrow())") } in_instance = false } @@ -606,7 +600,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) id match { case _: InstanceIdentifier => { done = true; - out.puts(s"${privateMemberName(id)} = Some($expr);") + out.puts(s"*${privateMemberName(id)}.borrow_mut() = $expr;") if (in_instance) { val found = translator.get_instance(translator.get_top_class(typeProvider.nowClass), id) @@ -616,7 +610,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _ => false } if (userType) { - out.puts(s"let ret = ${privateMemberName(id)}.as_mut().unwrap();") + out.puts(s"let ret = ${privateMemberName(id)}.borrow();") } } } @@ -635,8 +629,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def parseExpr(dataType: DataType, assignType: DataType, io: String, - defEndian: Option[FixedEndian]): String = - dataType match { + defEndian: Option[FixedEndian]): String = { + val expr = dataType match { case IntMultiType(_, _, None) => "panic!(\"Unable to parse unknown-endian integers\")" case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?.into()" case _: BytesEosType => s"$io.read_bytes_full()?.into()" @@ -693,6 +687,12 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"Self::read_into::<$streamType, $userType>($addParams$io$addArgs)?.into()" case _ => s"// parseExpr($dataType, $assignType, $io, $defEndian)" } + // in expr_2.ksy we got into rustc bug + // https://github.com/rust-lang/rust/issues/82656 + // workaround: + out.puts(s"let t = $expr;") + s"t" + } override def bytesPadTermExpr(expr0: String, padRight: Option[Int], @@ -760,16 +760,16 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def switchIfStart(id: Identifier, on: Ast.expr, onType: DataType): Unit = { out.puts("{") out.inc - out.puts(s"let on = ${expression(on)};") + out.puts(s"let on = &${expression(on)};") } override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { - out.puts(s"if on == ${expression(condition)} {") + out.puts(s"if *on == ${expression(condition)} {") out.inc } override def switchIfCaseStart(condition: Ast.expr): Unit = { - out.puts(s"else if on == ${expression(condition)} {") + out.puts(s"else if *on == ${expression(condition)} {") out.inc } @@ -1132,7 +1132,7 @@ object RustCompiler val typeName = if (!excludeBox && t.isOpaque) s"Box<$baseName>" else s"$baseName" - if (excludeOptionWrapper) typeName else s"Option<$typeName>" + if (excludeOptionWrapper) typeName else s"RefCell<$typeName>" case t: EnumType => val typeName = t.enumSpec match { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 93815cff8..644df0745 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -51,15 +51,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _ => val found = get_instance(get_top_class(provider.nowClass), InstanceIdentifier(s)) if (found.isDefined) { - val userType = found.get.dataTypeComposite match { - case _: UserType => true - case _ => false - } - if (userType) { - s"$s(${privateMemberName(IoIdentifier)})?" - } else { - s"$s(${privateMemberName(IoIdentifier)})?" - } + s"$s(${privateMemberName(IoIdentifier)})?" } else { s"$s()" } @@ -94,8 +86,45 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) return found; } - override def anyField(value: expr, attrName: String): String = - s"${translate(value)}.${doName(attrName)}" + override def anyField(value: expr, attrName: String): String = { + val t = translate(value) + var r = "" + if (t.charAt(0) == '*') { + r = s"$t.${doName(attrName)}" + } else { + r = s"*$t.${doName(attrName)}" + } + r + } + + def remove_deref(s: String): String = { + if (s.charAt(0) == '*') { + s.substring(1, s.length()) + } else { + s + } + } + + def get_attr(cs: ClassSpec, id: String): Option[MemberSpec] = { + var found : Option[MemberSpec] = None; + cs.seq.foreach { el => + if (idToStr(el.id) == id) { + found = Some(el); + } + } + // look deeper + if (found.isEmpty) { + cs.types.foreach { + case (_, typeSpec) => { + found = get_attr(typeSpec, id) + if (found.isDefined) { + return found; + } + } + } + } + return found; + } override def doLocalName(s: String) = s match { case Identifier.ITERATOR => "tmpa" @@ -106,7 +135,25 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case Identifier.PARENT => // TODO: How to handle _parent._parent? s"${RustCompiler.privateMemberName(ParentIdentifier)}.as_ref().unwrap().peek()" - case _ => s"self.${doName(s)}" + case _ => { + val n = doName(s) + var deref = true + val found = get_attr(get_top_class(provider.nowClass), s) + if (found.isDefined) { + deref = found.get.dataTypeComposite match { + case _: SwitchType => false + case _: UserType => false + case _: BytesType => false + case _: ArrayType => false + case _ => true + } + } + if (deref) { + s"*self.$n" + } else { + s"self.$n" + } + } } override def doInternalName(id: Identifier): String = @@ -116,7 +163,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"&${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr) = { - s"${translate(left)}.as_str() ${cmpOp(op)} ${translate(right)}" + s"${remove_deref(translate(left))}.as_str() ${cmpOp(op)} ${remove_deref(translate(right))}" } override def doEnumById(enumTypeAbs: List[String], id: String) = @@ -169,7 +216,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def bytesLength(b: Ast.expr): String = s"${translate(b)}.len()" override def strLength(s: expr): String = - s"${translate(s)}.len()" + s"${remove_deref(translate(s))}.len()" override def strReverse(s: expr): String = s"${translate(s)}.graphemes(true).rev().flat_map(|g| g.chars()).collect()" override def strSubstring(s: expr, from: expr, to: expr): String = @@ -180,7 +227,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def arrayLast(a: expr): String = s"${translate(a)}.last()" override def arraySize(a: expr): String = - s"${translate(a)}.len()" + s"${remove_deref(translate(a))}.len()" override def arrayMin(a: Ast.expr): String = s"${translate(a)}.iter().min()" override def arrayMax(a: Ast.expr): String = From 0d5aa350d08cb980d363bfb1a8d7dedb65d83ae6 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 5 Aug 2022 22:36:30 +0800 Subject: [PATCH 045/153] Clone added to workaround complex issues with instances calling other instaince returning ref type (like expr_2.str1_tuple5). --- .../struct/languages/RustCompiler.scala | 32 ++++--------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index c56f918a2..7a1293c1e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -65,7 +65,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def classHeader(name: List[String]): Unit = { out.puts - out.puts("#[derive(Default, Debug, PartialEq)]") + out.puts("#[derive(Default, Debug, PartialEq, Clone)]") out.puts(s"pub struct ${classTypeName(typeProvider.nowClass)} {") out.inc @@ -478,7 +478,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { dataType match { case _: UserType => { - out.puts(s"let ret = ${translator.remove_deref(expression(value))};") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.clone();") } case _: StrType => { out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.to_string();") @@ -495,15 +495,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { - val userType = attrType match { - case _: UserType => true - case _ => false - } - if (userType) { - out.puts(s"Ok(ret)") - } else { - out.puts(s"Ok(${privateMemberName(instName)}.borrow())") - } + out.puts(s"Ok(${privateMemberName(instName)}.borrow())") in_instance = false } @@ -514,7 +506,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val enumClass = types2class(curClass ::: List(enumName)) // Set up the actual enum definition - out.puts(s"#[derive(Debug, PartialEq)]") + out.puts(s"#[derive(Debug, PartialEq, Clone)]") out.puts(s"pub enum $enumClass {") out.inc @@ -601,19 +593,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: InstanceIdentifier => { done = true; out.puts(s"*${privateMemberName(id)}.borrow_mut() = $expr;") - - if (in_instance) { - val found = translator.get_instance(translator.get_top_class(typeProvider.nowClass), id) - if (found.isDefined) { - val userType = found.get.dataTypeComposite match { - case _: UserType => true - case _ => false - } - if (userType) { - out.puts(s"let ret = ${privateMemberName(id)}.borrow();") - } - } - } } case _ => false } @@ -689,6 +668,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } // in expr_2.ksy we got into rustc bug // https://github.com/rust-lang/rust/issues/82656 + // https://github.com/rust-lang/rust/issues/70919 // workaround: out.puts(s"let t = $expr;") s"t" @@ -825,7 +805,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) st, excludeOptionWrapper = true ) - out.puts("#[derive(Debug, PartialEq)]") + out.puts("#[derive(Debug, PartialEq, Clone)]") out.puts(s"pub enum $enum_typeName {") out.inc From db4ed94a2908140a0dc286df2b178f73a6319ef7 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 8 Aug 2022 09:53:40 +0800 Subject: [PATCH 046/153] correctly handle min/max for float type. --- .../struct/translators/RustTranslator.scala | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 644df0745..5d69193cc 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -223,13 +223,37 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${translate(s)}.substring(${translate(from)}, ${translate(to)})" override def arrayFirst(a: expr): String = - s"${translate(a)}.first()" + s"*${translate(a)}.first().ok_or(KError::EmptyIterator)?" override def arrayLast(a: expr): String = - s"${translate(a)}.last()" + s"*${translate(a)}.last().ok_or(KError::EmptyIterator)?" override def arraySize(a: expr): String = s"${remove_deref(translate(a))}.len()" - override def arrayMin(a: Ast.expr): String = - s"${translate(a)}.iter().min()" - override def arrayMax(a: Ast.expr): String = - s"${translate(a)}.iter().max()" + + def is_float_type(a: Ast.expr): Boolean = { + detectType(a) match { + case t: ArrayType => { + t.elType match { + case f: FloatMultiType => true + case _ => false + } + } + case _ => false + } + } + + override def arrayMin(a: Ast.expr): String = { + if (is_float_type(a)) { + s"*${translate(a)}.iter().reduce(|a, b| if (a.min(*b)) == *b {b} else {a}).ok_or(KError::EmptyIterator)?" + } else { + s"*${translate(a)}.iter().min().ok_or(KError::EmptyIterator)?" + } + } + + override def arrayMax(a: Ast.expr): String = { + if (is_float_type(a)) { + s"*${translate(a)}.iter().reduce(|a, b| if (a.max(*b)) == *b {b} else {a}).ok_or(KError::EmptyIterator)?" + } else { + s"*${translate(a)}.iter().max().ok_or(KError::EmptyIterator)?" + } + } } From c35652e36ddf934d48ad6a4d79baa527480dc61a Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 9 Aug 2022 16:53:07 +0800 Subject: [PATCH 047/153] handle _parent._parent. --- .../struct/languages/RustCompiler.scala | 6 ++- .../struct/translators/RustTranslator.scala | 41 +++++++++++-------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 7a1293c1e..96d2c14c7 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -402,7 +402,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceClear(instName: InstanceIdentifier): Unit = { var set = false; - val ins = translator.get_instance(typeProvider.nowClass, instName) + val ins = translator.get_instance(typeProvider.nowClass, idToStr(instName)) if (ins.isDefined) { set = ins.get.dataTypeComposite match { case _: UserType => true @@ -416,7 +416,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceSetCalculated(instName: InstanceIdentifier): Unit = { var set = false; - val ins = translator.get_instance(typeProvider.nowClass, instName) + val ins = translator.get_instance(typeProvider.nowClass, idToStr(instName)) if (ins.isDefined) { set = ins.get.dataTypeComposite match { case _: UserType => true @@ -487,8 +487,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"*${privateMemberName(instName)}.borrow_mut() = (${translator.remove_deref(expression(value))}).to_vec().clone();") } case _ => { + translator.in_instance_need_deref_attr = true val primType = kaitaiPrimitiveToNativeType(dataType) out.puts(s"*${privateMemberName(instName)}.borrow_mut() = (${expression(value)}) as $primType;") + translator.in_instance_need_deref_attr = false } } } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 5d69193cc..e0336d894 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -48,8 +48,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } override def doName(s: String) = s match { + case Identifier.PARENT => s case _ => - val found = get_instance(get_top_class(provider.nowClass), InstanceIdentifier(s)) + val found = get_instance(get_top_class(provider.nowClass), s) if (found.isDefined) { s"$s(${privateMemberName(IoIdentifier)})?" } else { @@ -64,11 +65,11 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) get_top_class(c.upClass.get) } - def get_instance(cs: ClassSpec, id: Identifier): Option[InstanceSpec] = { + def get_instance(cs: ClassSpec, s: String): Option[InstanceSpec] = { var found : Option[InstanceSpec] = None; // look for instance cs.instances.foreach { case (instName, instSpec) => - if (instName == id) { + if (idToStr(instName) == s) { found = Some(instSpec); } } @@ -76,7 +77,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) if (found.isEmpty) { cs.types.foreach { case (_, typeSpec) => { - found = get_instance(typeSpec, id) + found = get_instance(typeSpec, s) if (found.isDefined) { return found; } @@ -88,11 +89,19 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def anyField(value: expr, attrName: String): String = { val t = translate(value) + val a = doName(attrName) var r = "" if (t.charAt(0) == '*') { - r = s"$t.${doName(attrName)}" + r = s"$t.$a" } else { - r = s"*$t.${doName(attrName)}" + r = s"*$t.$a" + } + attrName match { + case Identifier.PARENT => { + // handle _parent._parent + r = r.replace(".peek()._parent", ".pop().peek()") + } + case _ => } r } @@ -126,15 +135,15 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) return found; } + var in_instance_need_deref_attr = false + override def doLocalName(s: String) = s match { case Identifier.ITERATOR => "tmpa" case Identifier.ITERATOR2 => "tmpb" case Identifier.INDEX => "i" case Identifier.IO => s"${RustCompiler.privateMemberName(IoIdentifier)}" case Identifier.ROOT => s"${RustCompiler.privateMemberName(RootIdentifier)}.ok_or(KError::MissingRoot)?" - case Identifier.PARENT => - // TODO: How to handle _parent._parent? - s"${RustCompiler.privateMemberName(ParentIdentifier)}.as_ref().unwrap().peek()" + case Identifier.PARENT => s"${RustCompiler.privateMemberName(ParentIdentifier)}.as_ref().unwrap().peek()" case _ => { val n = doName(s) var deref = true @@ -148,7 +157,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _ => true } } - if (deref) { + if (in_instance_need_deref_attr || deref) { s"*self.$n" } else { s"self.$n" @@ -223,9 +232,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${translate(s)}.substring(${translate(from)}, ${translate(to)})" override def arrayFirst(a: expr): String = - s"*${translate(a)}.first().ok_or(KError::EmptyIterator)?" + s"${translate(a)}.first().ok_or(KError::EmptyIterator)?" override def arrayLast(a: expr): String = - s"*${translate(a)}.last().ok_or(KError::EmptyIterator)?" + s"${translate(a)}.last().ok_or(KError::EmptyIterator)?" override def arraySize(a: expr): String = s"${remove_deref(translate(a))}.len()" @@ -243,17 +252,17 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def arrayMin(a: Ast.expr): String = { if (is_float_type(a)) { - s"*${translate(a)}.iter().reduce(|a, b| if (a.min(*b)) == *b {b} else {a}).ok_or(KError::EmptyIterator)?" + s"${translate(a)}.iter().reduce(|a, b| if (a.min(*b)) == *b {b} else {a}).ok_or(KError::EmptyIterator)?" } else { - s"*${translate(a)}.iter().min().ok_or(KError::EmptyIterator)?" + s"${translate(a)}.iter().min().ok_or(KError::EmptyIterator)?" } } override def arrayMax(a: Ast.expr): String = { if (is_float_type(a)) { - s"*${translate(a)}.iter().reduce(|a, b| if (a.max(*b)) == *b {b} else {a}).ok_or(KError::EmptyIterator)?" + s"${translate(a)}.iter().reduce(|a, b| if (a.max(*b)) == *b {b} else {a}).ok_or(KError::EmptyIterator)?" } else { - s"*${translate(a)}.iter().max().ok_or(KError::EmptyIterator)?" + s"${translate(a)}.iter().max().ok_or(KError::EmptyIterator)?" } } } From 28a8f2aaa256fd3d2b3c008d7c98fd018a749c25 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 9 Aug 2022 19:18:21 +0800 Subject: [PATCH 048/153] doByteArrayNonLiteral implemented. --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index e0336d894..bf7bd872d 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -18,7 +18,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doByteArrayLiteral(arr: Seq[Byte]): String = "&[" + arr.map(x => "%0#2xu8".format(x & 0xff)).mkString(", ") + "].to_vec()" override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = - s"pack('C*', ${elts.map(translate).mkString(", ")})" + "&[" + elts.map(translate).mkString(", ") + "]" override val asciiCharQuoteMap: Map[Char, String] = Map( '\t' -> "\\t", From 0415b1149ced291194685251db52c247378891c6 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Tue, 9 Aug 2022 21:21:50 +0200 Subject: [PATCH 049/153] #[derive(..., Copy)] for enums --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 1e9a14c9b..602e483f5 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -65,7 +65,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def classHeader(name: List[String]): Unit = { out.puts - out.puts("#[derive(Default, Debug, PartialEq, Clone)]") + out.puts("#[derive(Default, Debug, PartialEq, Clone, Copy)]") out.puts(s"pub struct ${classTypeName(typeProvider.nowClass)} {") out.inc @@ -506,7 +506,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val enumClass = types2class(curClass ::: List(enumName)) // Set up the actual enum definition - out.puts(s"#[derive(Debug, PartialEq, Clone)]") + out.puts(s"#[derive(Debug, PartialEq, Clone, Copy)]") out.puts(s"pub enum $enumClass {") out.inc From ecd764cf59bd5393937c9f34271c87ac13184d91 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Tue, 9 Aug 2022 21:23:54 +0200 Subject: [PATCH 050/153] no `&` in doEnumByLabel(...) --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 5d69193cc..b3e33e280 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -160,7 +160,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${doLocalName(idToStr(id))}" override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = - s"&${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" + s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr) = { s"${remove_deref(translate(left))}.as_str() ${cmpOp(op)} ${remove_deref(translate(right))}" From e37a99fbeebc68d1ca5db28fe35efd1a5812189c Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 11 Aug 2022 19:24:26 +0800 Subject: [PATCH 051/153] handle arrays correctly. --- .../struct/languages/RustCompiler.scala | 2 ++ .../struct/translators/RustTranslator.scala | 21 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 96d2c14c7..f10402fca 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -1167,5 +1167,7 @@ object RustCompiler case _: StrType => s"String" case _: BytesType => s"Vec" + + case ArrayTypeInStream(inType) => s"Vec<${kaitaiPrimitiveToNativeType(inType)}>" } } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index bf7bd872d..e2cfb8137 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -18,7 +18,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doByteArrayLiteral(arr: Seq[Byte]): String = "&[" + arr.map(x => "%0#2xu8".format(x & 0xff)).mkString(", ") + "].to_vec()" override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = - "&[" + elts.map(translate).mkString(", ") + "]" + "&[" + elts.map(translate).mkString(", ") + "].to_vec()" + override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = { + t match { + case CalcStrType => "vec![" + value.map((v) => translate(v)).mkString(".to_string(), ") + ".to_string()]" + case _ => "vec![" + value.map((v) => translate(v)).mkString(", ") + "]" + } + } override val asciiCharQuoteMap: Map[Char, String] = Map( '\t' -> "\\t", @@ -153,7 +159,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _: SwitchType => false case _: UserType => false case _: BytesType => false - case _: ArrayType => false + //case _: ArrayType => false case _ => true } } @@ -180,7 +186,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) id override def arraySubscript(container: expr, idx: expr): String = - s"${translate(container)}[${translate(idx)} as usize]" + s"${remove_deref(translate(container))}[${translate(idx)} as usize]" override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = "if " + translate(condition) + @@ -223,7 +229,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"decode_string($bytesExpr, ${translate(encoding)})?" override def bytesLength(b: Ast.expr): String = - s"${translate(b)}.len()" + s"${remove_deref(translate(b))}.len()" override def strLength(s: expr): String = s"${remove_deref(translate(s))}.len()" override def strReverse(s: expr): String = @@ -240,6 +246,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) def is_float_type(a: Ast.expr): Boolean = { detectType(a) match { + case t: CalcArrayType => { + t.elType match { + case f: FloatMultiType => true + case CalcFloatType => true + case _ => false + } + } case t: ArrayType => { t.elType match { case f: FloatMultiType => true From 087ac24df5e2a83e350ba68ac184f399fc535e6d Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 11 Aug 2022 19:33:38 +0800 Subject: [PATCH 052/153] to_string from int fixed. --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index e2cfb8137..dee807cc8 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -220,7 +220,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) val baseStr = translate(base) baseStr match { case "10" => - s"${translate(i)}.to_string()" + s"${remove_deref(translate(i))}.to_string()" case _ => s"base_convert(strval(${translate(i)}), 10, $baseStr)" } From c681bfc743df2d8b3c40eacfb7af3f8a72b15950 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Thu, 11 Aug 2022 13:58:30 +0200 Subject: [PATCH 053/153] remove Copy --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index e5b454bc5..8c5b73b6b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -65,7 +65,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def classHeader(name: List[String]): Unit = { out.puts - out.puts("#[derive(Default, Debug, PartialEq, Clone, Copy)]") + out.puts("#[derive(Default, Debug, PartialEq, Clone)]") out.puts(s"pub struct ${classTypeName(typeProvider.nowClass)} {") out.inc @@ -508,7 +508,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val enumClass = types2class(curClass ::: List(enumName)) // Set up the actual enum definition - out.puts(s"#[derive(Debug, PartialEq, Clone, Copy)]") + out.puts(s"#[derive(Debug, PartialEq, Clone)]") out.puts(s"pub enum $enumClass {") out.inc From 426d3b28b2d6fb1c69c717e71a49984ae3f03552 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Thu, 11 Aug 2022 21:25:43 +0200 Subject: [PATCH 054/153] add field `allClasses` to publish `classSpecs` --- .../src/main/scala/io/kaitai/struct/ClassTypeProvider.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala index abed472ff..5c9a153b6 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala @@ -8,7 +8,8 @@ import io.kaitai.struct.precompile.{EnumNotFoundError, FieldNotFoundError, TypeN import io.kaitai.struct.translators.TypeProvider class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends TypeProvider { - var nowClass = topClass + var nowClass: ClassSpec = topClass + val allClasses: ClassSpecs = classSpecs var _currentIteratorType: Option[DataType] = None var _currentSwitchType: Option[DataType] = None @@ -67,7 +68,6 @@ class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends * @param inClass type specification to search member in * @param attrName name of a member to search for * @return member spec if found, or throws an exception - * @throws format.InvalidIdentifier if attribute name is not a valid name for a member * @throws precompile.FieldNotFoundError if attribute with such name is not found */ def resolveMember(inClass: ClassSpec, attrName: String): MemberSpec = { From 7e39daafa7e9c69961b3d285f9f454a9cf83ad7c Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Thu, 11 Aug 2022 21:35:56 +0200 Subject: [PATCH 055/153] add to `importList` all external types, remove some warnings --- .../struct/languages/RustCompiler.scala | 96 +++++++++---------- 1 file changed, 44 insertions(+), 52 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index de23404e0..4336a7b46 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -9,6 +9,8 @@ import io.kaitai.struct.languages.components._ import io.kaitai.struct.translators.RustTranslator import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig} +import scala.annotation.tailrec + class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) with AllocateIOLocalVar @@ -56,6 +58,12 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) importList.add("std::convert::{TryFrom, TryInto}") importList.add("std::cell::{Ref, Cell, RefCell}") + + typeProvider.allClasses.foreach{ + case (name, _) => + if(name != topClassName) //TODO: do not add to imported + importList.add(s"super::$name::*") + } } override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = @@ -181,22 +189,17 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) var types : Set[DataType] = Set() var enum_typename = false attrType match { - case st: SwitchType => { + case st: SwitchType => types = st.cases.values.toSet enum_typename = true - } - case _: EnumType => { - // TODO? enum_typename = true - } + case _: EnumType => // TODO? enum_typename = true case _ => } - var enum_only_numeric = true; - types.foreach(t => { - t match { - case _: NumericType => // leave unchanged - case _ => enum_only_numeric = false - } - }) + var enum_only_numeric = true + types.foreach { + case _: NumericType => // leave unchanged + case _ => enum_only_numeric = false + } var fn = idToStr(attrName) if (enum_typename && enum_only_numeric) { out.puts(s"pub fn $fn(&self) -> usize {") @@ -267,7 +270,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = { - out.puts(s"${privateMemberName(id)}.push($expr);"); + out.puts(s"${privateMemberName(id)}.push($expr);") } override def condRepeatEosFooter: Unit = { @@ -325,12 +328,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val expr = proc match { case ProcessXor(xorValue) => translator.detectType(xorValue) match { - case _: IntType => { + case _: IntType => s"S::process_xor_one(&$srcExpr, ${expression(xorValue)})" - } - case _: BytesType => { + case _: BytesType => s"S::process_xor_many(&$srcExpr, &${translator.remove_deref(expression(xorValue))})" - } } case ProcessZlib => s"S::process_zlib(&$srcExpr)" @@ -342,7 +343,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } s"S::process_rotate_left(&$srcExpr, $expr)" case ProcessCustom(name, args) => - val procClass = name.map((x) => type2class(x)).mkString("::") + val procClass = name.map(x => type2class(x)).mkString("::") val procName = s"_process_${idToStr(varSrc)}" //importListSrc.addLocal(outFileNameHeader(name.last)) @@ -352,7 +353,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"$procClass $procName$argListInParens;") s"$procName.decode(&$srcExpr)" } - handleAssignment(varDest, expr, rep, false) + handleAssignment(varDest, expr, rep, isRaw = false) } override def useIO(ioEx: Ast.expr): String = s"// useIO($ioEx)" @@ -401,7 +402,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"f_${idToStr(ksName)}" override def instanceClear(instName: InstanceIdentifier): Unit = { - var set = false; + var set = false val ins = translator.get_instance(typeProvider.nowClass, idToStr(instName)) if (ins.isDefined) { set = ins.get.dataTypeComposite match { @@ -415,7 +416,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def instanceSetCalculated(instName: InstanceIdentifier): Unit = { - var set = false; + var set = false val ins = translator.get_instance(typeProvider.nowClass, idToStr(instName)) if (ins.isDefined) { set = ins.get.dataTypeComposite match { @@ -477,21 +478,17 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { dataType match { - case _: UserType => { + case _: UserType => out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.clone();") - } - case _: StrType => { + case _: StrType => out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.to_string();") - } - case _: BytesType => { + case _: BytesType => out.puts(s"*${privateMemberName(instName)}.borrow_mut() = (${translator.remove_deref(expression(value))}).to_vec().clone();") - } - case _ => { + case _ => translator.in_instance_need_deref_attr = true val primType = kaitaiPrimitiveToNativeType(dataType) out.puts(s"*${privateMemberName(instName)}.borrow_mut() = (${expression(value)}) as $primType;") translator.in_instance_need_deref_attr = false - } } } @@ -578,30 +575,28 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def handleAssignmentSimple(id: Identifier, expr: String): Unit = { val seqId = typeProvider.nowClass.seq.find(s => s.id == id) - var done = false; + var done = false if (seqId.isDefined) seqId.get.dataType match { case et: EnumType => - done = true; + done = true out.puts( s"${privateMemberName(id)} = Some(($expr as i64).try_into()?);" ) case st: SwitchType => - done = true; + done = true out.puts(s"${privateMemberName(id)} = Some($expr);") case _ => done = false; } if (!done) { id match { - case _: InstanceIdentifier => { - done = true; + case _: InstanceIdentifier => + done = true out.puts(s"*${privateMemberName(id)}.borrow_mut() = $expr;") - } case _ => false } } - if (!done) { + if (!done) out.puts(s"${privateMemberName(id)} = $expr;") - } } override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = @@ -621,7 +616,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case BitsType1(bitEndian) => s"$io.read_bits_int(1)? != 0" case BitsType(width, bitEndian) => s"$io.read_bits_int($width)?" case t: UserType => - val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") + val addParams = Utils.join(t.args.map(a => translator.translate(a)), "", ", ", ", ") val userType = t match { case t: UserType => val baseName = t.classSpec match { @@ -633,7 +628,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val addArgs = if (t.isOpaque) { "" } else { - val currentType = classTypeName(typeProvider.nowClass); + val currentType = classTypeName(typeProvider.nowClass) val root = if (typeProvider.nowClass.isTopLevel) { "Some(self)" } else { @@ -642,7 +637,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val parent = t.forcedParent match { case Some(USER_TYPE_NO_PARENT) => "KStructUnit::parent_stack()" case Some(fp) => translator.translate(fp) - case None => { + case None => if ((userType contains currentType) && !in_instance) { s"Some(${privateMemberName(ParentIdentifier)}.unwrap().push(self))" } else { @@ -652,7 +647,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"${privateMemberName(ParentIdentifier)}" } } - } } val addEndian = t.classSpec.get.meta.endian match { case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}" @@ -779,12 +773,12 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _ => privateMemberName(id) } - val newStreamRaw = s"${memberName}" + val newStreamRaw = s"$memberName" val ioName = rep match { case NoRepeat => val newStream = newStreamRaw val localIO = localTemporaryName(ioId) - out.puts(s"let ${localIO} = BytesReader::new(&$newStream);") + out.puts(s"let $localIO = BytesReader::new(&$newStream);") s"&$localIO" case _ => newStreamRaw @@ -835,13 +829,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec out.puts("}") - var enum_only_numeric = true; - types.foreach(t => { - t match { - case _: NumericType => // leave true - case _ => enum_only_numeric = false - } - }) + var enum_only_numeric = true + types.foreach { + case _: NumericType => // leave true + case _ => enum_only_numeric = false + } // generate only if switch types are different { @@ -852,10 +844,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val variantName = switchVariantName(id, t) var v = "v" val typeName = t match { - case _ : BytesType => { + case _ : BytesType => v = "v.to_vec()" s"&[u8]" // special case for Bytes(Vec[u8]) (else switch) - } case _ => kaitaiTypeToNativeType( id, typeProvider.nowClass, @@ -1028,6 +1019,7 @@ object RustCompiler case IoStorageIdentifier(inner) => s"io_${idToStr(inner)}" } + @tailrec def rootClassTypeName(c: ClassSpec, isRecurse: Boolean = false): String = { if (!isRecurse && c.isTopLevel) "Self" From 2cd28b1f475a9618d3f0ff7f695ae92c83d84b06 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 12 Aug 2022 11:57:36 +0800 Subject: [PATCH 056/153] expr ref/deref finally fixed? --- .../struct/translators/RustTranslator.scala | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index dee807cc8..407d464ce 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -97,10 +97,18 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) val t = translate(value) val a = doName(attrName) var r = "" - if (t.charAt(0) == '*') { - r = s"$t.$a" + if (need_deref(attrName)) { + if (t.charAt(0) == '*') { + r = s"$t.$a" + } else { + r = s"*$t.$a" + } } else { - r = s"*$t.$a" + if (t.charAt(0) == '*') { + r = s"${t.substring(1)}.$a" + } else { + r = s"$t.$a" + } } attrName match { case Identifier.PARENT => { @@ -143,6 +151,25 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) var in_instance_need_deref_attr = false + def need_deref(s: String) = { + var deref = true + val found_attr = get_attr(get_top_class(provider.nowClass), s) + if (found_attr.isDefined ) { + deref = found_attr.get.dataTypeComposite match { + case _: SwitchType => false + case _: UserType => false + case _: BytesType => false + //case _: ArrayType => false + case _ => true + } + } else if (get_instance(get_top_class(provider.nowClass), s).isDefined) { + deref = true + } else { + deref = false + } + deref + } + override def doLocalName(s: String) = s match { case Identifier.ITERATOR => "tmpa" case Identifier.ITERATOR2 => "tmpb" @@ -152,17 +179,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case Identifier.PARENT => s"${RustCompiler.privateMemberName(ParentIdentifier)}.as_ref().unwrap().peek()" case _ => { val n = doName(s) - var deref = true - val found = get_attr(get_top_class(provider.nowClass), s) - if (found.isDefined) { - deref = found.get.dataTypeComposite match { - case _: SwitchType => false - case _: UserType => false - case _: BytesType => false - //case _: ArrayType => false - case _ => true - } - } + var deref = need_deref(s) if (in_instance_need_deref_attr || deref) { s"*self.$n" } else { From 395ebadd7e4ea60d51ed219cc3cbf89c32633ddb Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 12 Aug 2022 15:13:14 +0800 Subject: [PATCH 057/153] no parent case fixed. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index f10402fca..778bc8831 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -640,7 +640,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) privateMemberName(RootIdentifier) } val parent = t.forcedParent match { - case Some(USER_TYPE_NO_PARENT) => "KStructUnit::parent_stack()" + case Some(USER_TYPE_NO_PARENT) => "None" case Some(fp) => translator.translate(fp) case None => { if ((userType contains currentType) && !in_instance) { From 7ea6d393ff86fe0cee268b1e36aa190712c5ce45 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 12 Aug 2022 22:24:12 +0800 Subject: [PATCH 058/153] enum ref fixed. --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 3e0445d1c..407d464ce 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -192,7 +192,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${doLocalName(idToStr(id))}" override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = - s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" + s"&${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr) = { s"${remove_deref(translate(left))}.as_str() ${cmpOp(op)} ${remove_deref(translate(right))}" From 974c4ab2709613cfb713256e6e560f8e895644cd Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Wed, 17 Aug 2022 21:24:03 +0200 Subject: [PATCH 059/153] `instanceCalculate` processes `EnumType`, add impl Default for $enumClass --- .../io/kaitai/struct/languages/RustCompiler.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 4336a7b46..c8156dfc8 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -484,6 +484,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.to_string();") case _: BytesType => out.puts(s"*${privateMemberName(instName)}.borrow_mut() = (${translator.remove_deref(expression(value))}).to_vec().clone();") + case e: EnumType => + val expr = expression(value) + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))};") case _ => translator.in_instance_need_deref_attr = true val primType = kaitaiPrimitiveToNativeType(dataType) @@ -547,6 +550,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec out.puts("}") out.puts + + out.puts(s"impl Default for $enumClass {") + out.inc + //FIXME: what is default? + out.puts(s"fn default() -> Self { $enumClass::${type2class(enumColl(0)._2.name)} }") + out.puts("}") + out.puts } override def universalDoc(doc: DocSpec): Unit = { @@ -1157,8 +1167,8 @@ object RustCompiler case CalcIntType => "i32" case CalcFloatType => "f64" - case _: StrType => s"String" - case _: BytesType => s"Vec" + case _: StrType => "String" + case _: BytesType => "Vec" case ArrayTypeInStream(inType) => s"Vec<${kaitaiPrimitiveToNativeType(inType)}>" } From 5a68cf21af7da39a6c1f0e71650a63f0ccfc9704 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Wed, 17 Aug 2022 21:30:37 +0200 Subject: [PATCH 060/153] `translate` for `EnumType` may use `try_from` --- .../struct/translators/RustTranslator.scala | 71 +++++++++++-------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index ba355f84f..a4d774ad4 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -2,7 +2,6 @@ package io.kaitai.struct.translators import io.kaitai.struct.format._ import io.kaitai.struct.datatype._ - import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr @@ -21,8 +20,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) "&[" + elts.map(translate).mkString(", ") + "].to_vec()" override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = { t match { - case CalcStrType => "vec![" + value.map((v) => translate(v)).mkString(".to_string(), ") + ".to_string()]" - case _ => "vec![" + value.map((v) => translate(v)).mkString(", ") + "]" + case CalcStrType => "vec![" + value.map(v => translate(v)).mkString(".to_string(), ") + ".to_string()]" + case _ => "vec![" + value.map(v => translate(v)).mkString(", ") + "]" } } @@ -42,7 +41,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def numericBinOp(left: Ast.expr, op: Ast.operator, - right: Ast.expr) = { + right: Ast.expr): String = { (detectType(left), detectType(right), op) match { case (_: IntType, _: IntType, Ast.operator.Div) => s"${translate(left)} / ${translate(right)}" @@ -53,7 +52,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } - override def doName(s: String) = s match { + override def doName(s: String): String = s match { case Identifier.PARENT => s case _ => val found = get_instance(get_top_class(provider.nowClass), s) @@ -72,25 +71,24 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } def get_instance(cs: ClassSpec, s: String): Option[InstanceSpec] = { - var found : Option[InstanceSpec] = None; + var found : Option[InstanceSpec] = None // look for instance cs.instances.foreach { case (instName, instSpec) => if (idToStr(instName) == s) { - found = Some(instSpec); + found = Some(instSpec) } } // look deeper if (found.isEmpty) { cs.types.foreach { - case (_, typeSpec) => { + case (_, typeSpec) => found = get_instance(typeSpec, s) if (found.isDefined) { - return found; + return found } } - } } - return found; + found } override def anyField(value: expr, attrName: String): String = { @@ -103,10 +101,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) r = s"*$t.$a" } attrName match { - case Identifier.PARENT => { + case Identifier.PARENT => // handle _parent._parent r = r.replace(".peek()._parent", ".pop().peek()") - } case _ => } r @@ -121,36 +118,35 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } def get_attr(cs: ClassSpec, id: String): Option[MemberSpec] = { - var found : Option[MemberSpec] = None; + var found : Option[MemberSpec] = None cs.seq.foreach { el => if (idToStr(el.id) == id) { - found = Some(el); + found = Some(el) } } // look deeper if (found.isEmpty) { cs.types.foreach { - case (_, typeSpec) => { + case (_, typeSpec) => found = get_attr(typeSpec, id) if (found.isDefined) { - return found; + return found } } - } } - return found; + found } var in_instance_need_deref_attr = false - override def doLocalName(s: String) = s match { + override def doLocalName(s: String): String = s match { case Identifier.ITERATOR => "tmpa" case Identifier.ITERATOR2 => "tmpb" case Identifier.INDEX => "i" case Identifier.IO => s"${RustCompiler.privateMemberName(IoIdentifier)}" case Identifier.ROOT => s"${RustCompiler.privateMemberName(RootIdentifier)}.ok_or(KError::MissingRoot)?" case Identifier.PARENT => s"${RustCompiler.privateMemberName(ParentIdentifier)}.as_ref().unwrap().peek()" - case _ => { + case _ => val n = doName(s) var deref = true val found = get_attr(get_top_class(provider.nowClass), s) @@ -168,7 +164,6 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } else { s"self.$n" } - } } override def doInternalName(id: Identifier): String = @@ -177,17 +172,39 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" - override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr) = { + override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { s"${remove_deref(translate(left))}.as_str() ${cmpOp(op)} ${remove_deref(translate(right))}" } - override def doEnumById(enumTypeAbs: List[String], id: String) = + override def doEnumById(enumTypeAbs: List[String], id: String): String = // Just an integer, without any casts / resolutions - one would have to look up constants manually id override def arraySubscript(container: expr, idx: expr): String = s"${remove_deref(translate(container))}[${translate(idx)} as usize]" + override def translate(v: Ast.expr): String = { + v match { + case Ast.expr.EnumById(enumType, id, inType) => + id match { + case ifExp: Ast.expr.IfExp => + val enumSpec = provider.resolveEnum(inType, enumType.name) + val enumName = RustCompiler.types2class(enumSpec.name) + def toStr(ex: Ast.expr) = ex match { + case Ast.expr.IntNum(n) => s"$enumName::try_from($n)?" + case _ => super.translate(ex) + } + val ifTrue = toStr(ifExp.ifTrue) + val ifFalse = toStr(ifExp.ifFalse) + + "if " + translate(ifExp.condition) + s" { $ifTrue } else { $ifFalse }" + case _ => super.translate(v) + } + case _ => + super.translate(v) + } + } + override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = "if " + translate(condition) + " { " + translate(ifTrue) + " } else { " + @@ -246,19 +263,17 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) def is_float_type(a: Ast.expr): Boolean = { detectType(a) match { - case t: CalcArrayType => { + case t: CalcArrayType => t.elType match { case f: FloatMultiType => true case CalcFloatType => true case _ => false } - } - case t: ArrayType => { + case t: ArrayType => t.elType match { case f: FloatMultiType => true case _ => false } - } case _ => false } } From b943048ba76796c3eab1e6f3e910f66e0690a79b Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 18 Aug 2022 10:27:41 +0200 Subject: [PATCH 061/153] default to not deref. --- .../struct/languages/RustCompiler.scala | 74 +++++++-------- .../struct/translators/RustTranslator.scala | 89 +++++++++++++------ 2 files changed, 101 insertions(+), 62 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index c16c39fc4..c75e99135 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -166,7 +166,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct case IoIdentifier | RootIdentifier | ParentIdentifier => return case _ => - kaitaiTypeToNativeType(attrName, typeProvider.nowClass, attrType) + kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType) } out.puts(s"${idToStr(attrName)}: $typeName,") @@ -179,9 +179,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct case IoIdentifier | RootIdentifier | ParentIdentifier => return case _ => - kaitaiTypeToNativeType(attrName, typeProvider.nowClass, attrType) + kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType) } - val typeNameEx = kaitaiTypeToNativeType(attrName, typeProvider.nowClass, attrType, excludeOptionWrapper = true) + val typeNameEx = kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType, excludeOptionWrapper = true) out.puts( s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") out.inc @@ -389,7 +389,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) attrType: DataType, isNullable: Boolean): Unit = { val typeName = kaitaiTypeToNativeType( - attrName, + Some(attrName), typeProvider.nowClass, attrType, excludeOptionWrapper = true @@ -431,19 +431,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def idToStr(id: Identifier): String = RustCompiler.idToStr(id) - def is_copy_type(dataType: DataType): Boolean = dataType match { - case _: UserType => false - case _: BytesType => false - case _: StrType => false - case _: ArrayType => false - case _ => true - } - - def is_mut_needed(dataType: DataType): Boolean = dataType match { - case _: UserType => true - case _ => false - } - override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, @@ -455,7 +442,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S") out.dec val typeName = kaitaiTypeToNativeType( - instName, + Some(instName), typeProvider.nowClass, dataType, excludeOptionWrapper = true @@ -483,7 +470,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: StrType => out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.to_string();") case _: BytesType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = (${translator.remove_deref(expression(value))}).to_vec().clone();") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") case _ => translator.in_instance_need_deref_attr = true val primType = kaitaiPrimitiveToNativeType(dataType) @@ -576,16 +563,31 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def handleAssignmentSimple(id: Identifier, expr: String): Unit = { val seqId = typeProvider.nowClass.seq.find(s => s.id == id) var done = false - if (seqId.isDefined) seqId.get.dataType match { - case et: EnumType => - done = true - out.puts( - s"${privateMemberName(id)} = Some(($expr as i64).try_into()?);" - ) - case st: SwitchType => - done = true - out.puts(s"${privateMemberName(id)} = Some($expr);") - case _ => done = false; + var refcell = false + if (seqId.isDefined) { + val idType = seqId.get.dataType + idType match { + case _: UserType => refcell = true + case _: BytesType => refcell = true + case _: ArrayType => refcell = true + case _: StrType => refcell = true + case et: EnumType => + done = true + out.puts( + s"${privateMemberName(id)} = Some(($expr as i64).try_into()?);" + ) + case st: SwitchType => + done = true + out.puts(s"${privateMemberName(id)} = Some($expr);") + case _ => + } + if (refcell) { + val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, idType) + if (typeName.startsWith("RefCell")) { + out.puts(s"${privateMemberName(id)} = RefCell::new($expr);") + done = true + } + } } if (!done) { id match { @@ -796,7 +798,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // Because Rust can't handle `AnyType` in the type hierarchy, // we generate an enum with all possible variations val enum_typeName = kaitaiTypeToNativeType( - id, + Some(id), typeProvider.nowClass, st, excludeOptionWrapper = true @@ -813,7 +815,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // Because this switch type will itself be in an option, we can exclude it from user types val variantName = switchVariantName(id, t) val typeName = kaitaiTypeToNativeType( - id, + Some(id), typeProvider.nowClass, t, excludeOptionWrapper = true @@ -848,7 +850,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) v = "v.to_vec()" s"&[u8]" // special case for Bytes(Vec[u8]) (else switch) case _ => kaitaiTypeToNativeType( - id, + Some(id), typeProvider.nowClass, t, excludeOptionWrapper = true) @@ -933,7 +935,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case t: UserType => kaitaiTypeToNativeType( - id, + Some(id), typeProvider.nowClass, t, excludeOptionWrapper = true, @@ -942,7 +944,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) case t: EnumType => kaitaiTypeToNativeType( - id, + Some(id), typeProvider.nowClass, t, excludeOptionWrapper = true @@ -1082,7 +1084,7 @@ object RustCompiler case _ => false } - def kaitaiTypeToNativeType(id: Identifier, + def kaitaiTypeToNativeType(id: Option[Identifier], cs: ClassSpec, attrType: DataType, excludeOptionWrapper: Boolean = false, @@ -1124,7 +1126,7 @@ object RustCompiler // if (!excludeLifetime && types.exists(containsReferences)) // s"<$streamLife>" // else "" - val typeName = id match { + val typeName = id.get match { case name: NamedIdentifier => s"${types2class(cs.name ::: List(name.name))}" case name: InstanceIdentifier => diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 407d464ce..1537f0c49 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -16,9 +16,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) import RustCompiler._ override def doByteArrayLiteral(arr: Seq[Byte]): String = - "&[" + arr.map(x => "%0#2xu8".format(x & 0xff)).mkString(", ") + "].to_vec()" + "&vec![" + arr.map(x => "%0#2xu8".format(x & 0xff)).mkString(", ") + "]" override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = - "&[" + elts.map(translate).mkString(", ") + "].to_vec()" + "&vec![" + elts.map(translate).mkString(", ") + "]" override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = { t match { case CalcStrType => "vec![" + value.map((v) => translate(v)).mkString(".to_string(), ") + ".to_string()]" @@ -120,9 +120,25 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) r } + def rem_vec_amp(s: String): String = { + if (s.startsWith("&vec!")) { + s.substring(1) + } else { + s + } + } + def remove_deref(s: String): String = { if (s.charAt(0) == '*') { - s.substring(1, s.length()) + s.substring(1) + } else { + s + } + } + + def ensure_deref(s: String): String = { + if (s.startsWith("self")) { + s"*$s" } else { s } @@ -151,21 +167,27 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) var in_instance_need_deref_attr = false + def is_copy_type(dataType: DataType): Boolean = dataType match { + case _: SwitchType => false + case _: UserType => false + case _: BytesType => false + case _: ArrayType => false + case _: StrType => false + case _ => true + } + def need_deref(s: String) = { - var deref = true + var deref = false val found_attr = get_attr(get_top_class(provider.nowClass), s) if (found_attr.isDefined ) { - deref = found_attr.get.dataTypeComposite match { - case _: SwitchType => false - case _: UserType => false - case _: BytesType => false - //case _: ArrayType => false - case _ => true - } - } else if (get_instance(get_top_class(provider.nowClass), s).isDefined) { - deref = true + deref = is_copy_type(found_attr.get.dataTypeComposite) } else { - deref = false + val found_inst = get_instance(get_top_class(provider.nowClass), s) + if (found_inst.isDefined) { + deref = true //is_copy_type(found_inst.get.dataTypeComposite) + } else { + deref = false + } } deref } @@ -205,10 +227,20 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def arraySubscript(container: expr, idx: expr): String = s"${remove_deref(translate(container))}[${translate(idx)} as usize]" - override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = - "if " + translate(condition) + - " { " + translate(ifTrue) + " } else { " + - translate(ifFalse) + "}" + override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = { + var to_type = "" + detectType(ifTrue) match { + case _: UserType => to_type = ".clone()" + case _: StrType => to_type = ".to_string()" + case _: BytesType => to_type = ".to_vec()" + case _ => + } + if (to_type.isEmpty) { + s"if ${translate(condition)} { ${translate(ifTrue)} } else { ${translate(ifFalse)} }" + } else { + s"if ${translate(condition)} { ${remove_deref(translate(ifTrue))}$to_type } else { ${remove_deref(translate(ifFalse))}$to_type }" + } + } // Predefined methods of various types override def strConcat(left: Ast.expr, right: Ast.expr): String = @@ -242,8 +274,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"base_convert(strval(${translate(i)}), 10, $baseStr)" } } - override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = - s"decode_string($bytesExpr, ${translate(encoding)})?" + override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = { + if (bytesExpr.charAt(0) == '*') { + s"decode_string(&$bytesExpr, &${translate(encoding)})?" + } else { + s"decode_string($bytesExpr, &${translate(encoding)})?" + } + } override def bytesLength(b: Ast.expr): String = s"${remove_deref(translate(b))}.len()" @@ -255,9 +292,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${translate(s)}.substring(${translate(from)}, ${translate(to)})" override def arrayFirst(a: expr): String = - s"${translate(a)}.first().ok_or(KError::EmptyIterator)?" + s"${ensure_deref(translate(a))}.first().ok_or(KError::EmptyIterator)?" override def arrayLast(a: expr): String = - s"${translate(a)}.last().ok_or(KError::EmptyIterator)?" + s"${ensure_deref(translate(a))}.last().ok_or(KError::EmptyIterator)?" override def arraySize(a: expr): String = s"${remove_deref(translate(a))}.len()" @@ -282,17 +319,17 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def arrayMin(a: Ast.expr): String = { if (is_float_type(a)) { - s"${translate(a)}.iter().reduce(|a, b| if (a.min(*b)) == *b {b} else {a}).ok_or(KError::EmptyIterator)?" + s"${ensure_deref(translate(a))}.iter().reduce(|a, b| if (a.min(*b)) == *b {b} else {a}).ok_or(KError::EmptyIterator)?" } else { - s"${translate(a)}.iter().min().ok_or(KError::EmptyIterator)?" + s"${ensure_deref(translate(a))}.iter().min().ok_or(KError::EmptyIterator)?" } } override def arrayMax(a: Ast.expr): String = { if (is_float_type(a)) { - s"${translate(a)}.iter().reduce(|a, b| if (a.max(*b)) == *b {b} else {a}).ok_or(KError::EmptyIterator)?" + s"${ensure_deref(translate(a))}.iter().reduce(|a, b| if (a.max(*b)) == *b {b} else {a}).ok_or(KError::EmptyIterator)?" } else { - s"${translate(a)}.iter().max().ok_or(KError::EmptyIterator)?" + s"${ensure_deref(translate(a))}.iter().max().ok_or(KError::EmptyIterator)?" } } } From 7968a80eeb565fd9a447ed1b8527b640124d2c32 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Tue, 23 Aug 2022 18:37:39 +0200 Subject: [PATCH 062/153] `enumToInt` uses `i64::from` (enum_to_i.ksy) --- .../kaitai/struct/translators/RustTranslator.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 6f053c777..0cd47d1e9 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -5,7 +5,7 @@ import io.kaitai.struct.datatype._ import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr -import io.kaitai.struct.format.{Identifier, InstanceIdentifier, IoIdentifier, NamedIdentifier, ParentIdentifier, RootIdentifier} +import io.kaitai.struct.format.{Identifier, IoIdentifier, ParentIdentifier, RootIdentifier} import io.kaitai.struct.languages.RustCompiler import io.kaitai.struct.{RuntimeConfig, Utils} @@ -172,10 +172,10 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _ => true } - def need_deref(s: String) = { + def need_deref(s: String): Boolean = { var deref = false val found_attr = get_attr(get_top_class(provider.nowClass), s) - if (found_attr.isDefined ) { + if (found_attr.isDefined) { deref = is_copy_type(found_attr.get.dataTypeComposite) } else { val found_inst = get_instance(get_top_class(provider.nowClass), s) @@ -197,7 +197,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case Identifier.PARENT => s"${RustCompiler.privateMemberName(ParentIdentifier)}.as_ref().unwrap().peek()" case _ => val n = doName(s) - var deref = need_deref(s) + val deref = need_deref(s) if (in_instance_need_deref_attr || deref) { s"*self.$n" } else { @@ -273,7 +273,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } override def enumToInt(v: expr, et: EnumType): String = - s"usize::from(${translate(v)})" + s"i64::from(${remove_deref(translate(v))})" override def boolToInt(v: expr): String = s"${translate(v)} as i32" @@ -318,13 +318,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) detectType(a) match { case t: CalcArrayType => t.elType match { - case f: FloatMultiType => true + //case f: FloatMultiType => true case CalcFloatType => true case _ => false } case t: ArrayType => t.elType match { - case f: FloatMultiType => true + //case f: FloatMultiType => true case _ => false } case _ => false From 4844f52b5b9b4f518c923687c33f4ef04c4be3a7 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Tue, 23 Aug 2022 18:45:20 +0200 Subject: [PATCH 063/153] `impl From<&$enumClass> for i64` (enum_to_i.ksy) --- .../struct/languages/RustCompiler.scala | 67 ++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index bdf7a0b94..09737185d 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -349,7 +349,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) //importListSrc.addLocal(outFileNameHeader(name.last)) val argList = args.map(expression).mkString(", ") - var argListInParens = if (argList.nonEmpty) s"($argList)" else "" + val argListInParens = if (argList.nonEmpty) s"($argList)" else "" out.puts(s"$procClass $procName$argListInParens;") s"$procName.decode(&$srcExpr)" } @@ -464,20 +464,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) var in_instance = false override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { + val expr = expression(value) + dataType match { case _: UserType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.clone();") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expr)}.clone();") case _: StrType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.to_string();") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expr)}.to_string();") case _: BytesType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") - case e: EnumType => - val expr = expression(value) - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))};") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.rem_vec_amp(translator.remove_deref(expr))}.to_vec();") + case _: EnumType => + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expr)};") case _ => translator.in_instance_need_deref_attr = true val primType = kaitaiPrimitiveToNativeType(dataType) - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = (${expression(value)}) as $primType;") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ($expr) as $primType;") translator.in_instance_need_deref_attr = false } } @@ -509,6 +510,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec out.puts("}") + out.puts // Set up parsing enums from the underlying value out.puts(s"impl TryFrom for $enumClass {") @@ -538,10 +540,29 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") out.puts + out.puts(s"impl From<&$enumClass> for i64 {") + out.inc + out.puts(s"fn from(val: &$enumClass) -> Self {") + out.inc + out.puts(s"match val {") + out.inc + enumColl.foreach { + case (value, label) => + out.puts(s"$enumClass::${type2class(label.name)} => $value,") + } + out.dec + out.puts("}") + out.dec + out.puts("}") + out.dec + out.puts("}") + out.puts + out.puts(s"impl Default for $enumClass {") out.inc //FIXME: what is default? - out.puts(s"fn default() -> Self { $enumClass::${type2class(enumColl(0)._2.name)} }") + out.puts(s"fn default() -> Self { $enumClass::${type2class(enumColl.head._2.name)} }") + out.dec out.puts("}") out.puts } @@ -581,12 +602,12 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesType => refcell = true case _: ArrayType => refcell = true case _: StrType => refcell = true - case et: EnumType => + case _: EnumType => done = true out.puts( s"${privateMemberName(id)} = Some(($expr as i64).try_into()?);" ) - case st: SwitchType => + case _: SwitchType => done = true out.puts(s"${privateMemberName(id)} = Some($expr);") case _ => @@ -604,7 +625,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: InstanceIdentifier => done = true out.puts(s"*${privateMemberName(id)}.borrow_mut() = $expr;") - case _ => false + case _ => } } if (!done) @@ -625,8 +646,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case b: BytesTerminatedType => s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?.into()" case b: BytesLimitType => s"$io.read_bytes(${expression(b.size)} as usize)?.into()" - case BitsType1(bitEndian) => s"$io.read_bits_int(1)? != 0" - case BitsType(width, bitEndian) => s"$io.read_bits_int($width)?" + case _: BitsType1 => s"$io.read_bits_int(1)? != 0" + case BitsType(width, _) => s"$io.read_bits_int($width)?" case t: UserType => val addParams = Utils.join(t.args.map(a => translator.translate(a)), "", ", ", ", ") val userType = t match { @@ -780,10 +801,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val memberName = privateMemberName(id) val ioId = IoStorageIdentifier(id) - val args = rep match { - case RepeatUntil(_) => translator.doName(Identifier.ITERATOR2) - case _ => privateMemberName(id) - } +// val args = rep match { +// case RepeatUntil(_) => translator.doName(Identifier.ITERATOR2) +// case _ => privateMemberName(id) +// } val newStreamRaw = s"$memberName" val ioName = rep match { @@ -820,7 +841,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val types = st.cases.values.toSet { - var types_set = scala.collection.mutable.Set[String]() + val types_set = scala.collection.mutable.Set[String]() types.foreach(t => { // Because this switch type will itself be in an option, we can exclude it from user types val variantName = switchVariantName(id, t) @@ -849,7 +870,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // generate only if switch types are different { - var types_set = scala.collection.mutable.Set[String]() + val types_set = scala.collection.mutable.Set[String]() // add helper methods From types.foreach(t => { // Because this switch type will itself be in an option, we can exclude it from user types @@ -902,7 +923,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc out.puts(s"match e {") out.inc - var variants_set = scala.collection.mutable.Set[String]() + val variants_set = scala.collection.mutable.Set[String]() types.foreach(t => { val variantName = switchVariantName(id, t) val new_typename = variants_set.add(variantName) @@ -1078,7 +1099,7 @@ object RustCompiler def containsReferences(d: DataType, originating: Option[ClassSpec]): Boolean = d match { case _: BytesType | _: StrType => true - case t: UserType => true + case _: UserType => true /* t.classSpec match { // Recursive types may need references, but the recursion itself @@ -1130,7 +1151,7 @@ object RustCompiler case t: ArrayType => s"Vec<${kaitaiTypeToNativeType(id, cs, t.elType, excludeOptionWrapper = true, excludeLifetime = excludeLifetime)}>" - case st: SwitchType => + case _: SwitchType => // val types = st.cases.values.toSet // val lifetime = // if (!excludeLifetime && types.exists(containsReferences)) From 528886e7fd34b5d191a2084000999b3de8d05763 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 24 Aug 2022 17:12:15 +0200 Subject: [PATCH 064/153] handle params. --- .../struct/languages/RustCompiler.scala | 82 +++++++++++++++++-- .../struct/translators/RustTranslator.scala | 42 ++++++++-- 2 files changed, 110 insertions(+), 14 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index bdf7a0b94..fab987f07 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -374,9 +374,29 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) RustCompiler.privateMemberName(id) override def instanceDeclHeader(className: List[String]): Unit = { - out.puts( - s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {" - ) + if (!typeProvider.nowClass.params.isEmpty) { + val paramsArg = Utils.join(typeProvider.nowClass.params.map{ case p => + val n = paramName(p.id) + val t = kaitaiTypeToNativeType(Some(p.id), typeProvider.nowClass, p.dataType, excludeOptionWrapper = true) + var byref = "" + if (!translator.is_copy_type(p.dataType)) + byref = "&" + // generate param access helper + attributeReader(p.id, p.dataType, false) + s"$n: $byref$t" + }, "", ", ", "") + + out.puts(s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") + out.inc + out.puts(s"pub fn set_params(&mut self, $paramsArg) {") + out.inc + typeProvider.nowClass.params.foreach((p) => handleAssignmentParams(p.id, paramName(p.id))) + out.dec + out.puts("}") + out.dec + out.puts("}") + } + out.puts(s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") out.inc } @@ -471,6 +491,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.to_string();") case _: BytesType => out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") + case _: ArrayType => + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") case e: EnumType => val expr = expression(value) out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))};") @@ -542,8 +564,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc //FIXME: what is default? out.puts(s"fn default() -> Self { $enumClass::${type2class(enumColl(0)._2.name)} }") + out.dec out.puts("}") - out.puts } override def universalDoc(doc: DocSpec): Unit = { @@ -570,6 +592,28 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) isRaw: Boolean): Unit = out.puts(s"// handleAssignmentRepeatUntil($id, $expr, $isRaw)") + def handleAssignmentParams(id: Identifier, expr: String): Unit = { + val paramId = typeProvider.nowClass.params.find(s => s.id == id) + var need_clone = false + if (paramId.isDefined) { + need_clone = !translator.is_copy_type(paramId.get.dataType) + } + val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, paramId.get.dataType) + if (typeName.startsWith("RefCell")) + out.puts(s"${privateMemberName(id)} = RefCell::new($expr.clone());") + else { + paramId.get.dataType match { + case et: EnumType => + out.puts(s"${privateMemberName(id)} = Some($expr.clone());") + case _ => + if (need_clone) + out.puts(s"${privateMemberName(id)} = ${expr}.clone();") + else + out.puts(s"${privateMemberName(id)} = $expr;") + } + } + } + override def handleAssignmentSimple(id: Identifier, expr: String): Unit = { val seqId = typeProvider.nowClass.seq.find(s => s.id == id) var done = false @@ -618,6 +662,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { + var addParams = "" val expr = dataType match { case IntMultiType(_, _, None) => "panic!(\"Unable to parse unknown-endian integers\")" case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?.into()" @@ -628,7 +673,22 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case BitsType1(bitEndian) => s"$io.read_bits_int(1)? != 0" case BitsType(width, bitEndian) => s"$io.read_bits_int($width)?" case t: UserType => - val addParams = Utils.join(t.args.map(a => translator.translate(a)), "", ", ", ", ") + addParams = Utils.join(t.args.map{ a => + val typ = translator.detectType(a) + var byref = "" + // exclude enums + typ match { + case _: EnumType => + case _ => + if (!translator.is_copy_type(typ)) + byref = "&" + } + val t = kaitaiTypeToNativeType(None, typeProvider.nowClass, typ) + var need_deref = "" + if (t.startsWith("RefCell")) + need_deref = "&*" + s"$byref$need_deref${translator.translate(a)}" + }, "", ", ", "") val userType = t match { case t: UserType => val baseName = t.classSpec match { @@ -671,7 +731,15 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } else { "BytesReader" } - s"Self::read_into::<$streamType, $userType>($addParams$io$addArgs)?.into()" + if (addParams.isEmpty) { + out.puts(s"let t = Self::read_into::<$streamType, $userType>($io$addArgs)?.into();") + } else { + //val at = kaitaiTypeToNativeType(None, typeProvider.nowClass, assignType, excludeOptionWrapper = true) + out.puts(s"let mut t = $userType::default();") + out.puts(s"t.set_params($addParams);") + out.puts(s"t.read::<$streamType>($io$addArgs)?;"); + } + return s"t" case _ => s"// parseExpr($dataType, $assignType, $io, $defEndian)" } // in expr_2.ksy we got into rustc bug @@ -1147,6 +1215,8 @@ object RustCompiler if (excludeOptionWrapper) typeName else s"Option<$typeName>" case KaitaiStreamType => kstreamName + + //case CalcKaitaiStructType => } def kaitaiPrimitiveToNativeType(attrType: DataType): String = attrType match { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 6f053c777..b0363ed5e 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -161,6 +161,26 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) found } + def get_param(cs: ClassSpec, id: String): Option[MemberSpec] = { + var found : Option[MemberSpec] = None + cs.params.foreach { el => + if (idToStr(el.id) == id) { + found = Some(el) + } + } + // look deeper + if (found.isEmpty) { + cs.types.foreach { + case (_, typeSpec) => + found = get_param(typeSpec, id) + if (found.isDefined) { + return found + } + } + } + found + } + var in_instance_need_deref_attr = false def is_copy_type(dataType: DataType): Boolean = dataType match { @@ -169,20 +189,26 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _: BytesType => false case _: ArrayType => false case _: StrType => false + case _: EnumType => false case _ => true } def need_deref(s: String) = { var deref = false - val found_attr = get_attr(get_top_class(provider.nowClass), s) - if (found_attr.isDefined ) { - deref = is_copy_type(found_attr.get.dataTypeComposite) + var found = get_attr(get_top_class(provider.nowClass), s) + if (found.isDefined ) { + deref = is_copy_type(found.get.dataTypeComposite) } else { - val found_inst = get_instance(get_top_class(provider.nowClass), s) - if (found_inst.isDefined) { - deref = true //is_copy_type(found_inst.get.dataTypeComposite) + found = get_instance(get_top_class(provider.nowClass), s) + if (found.isDefined) { + deref = true //is_copy_type(found.get.dataTypeComposite) } else { - deref = false + found = get_param(get_top_class(provider.nowClass), s) + if (found.isDefined) { + deref = true + } else { + deref = false + } } } deref @@ -209,7 +235,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${doLocalName(idToStr(id))}" override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = - s"&${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" + s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { s"${remove_deref(translate(left))}.as_str() ${cmpOp(op)} ${remove_deref(translate(right))}" From 3550542aabf4a3d3902b4adaccce259b617af9d0 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 25 Aug 2022 08:42:12 +0200 Subject: [PATCH 065/153] cast to usize. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index fab987f07..096cd442f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -362,7 +362,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"let _pos = $io.pos();") override def seek(io: String, pos: Ast.expr): Unit = - out.puts(s"$io.seek(${expression(pos)})?;") + out.puts(s"$io.seek(${expression(pos)} as usize)?;") override def popPos(io: String): Unit = out.puts(s"$io.seek(_pos)?;") From 4184139ebbb09c5082872d3adb1721b72c6e1b0c Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 25 Aug 2022 08:47:37 +0200 Subject: [PATCH 066/153] .into() added. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 096cd442f..a8af5821a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -756,12 +756,12 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) include: Boolean): String = { val ioId = privateMemberName(IoIdentifier) val expr = padRight match { - case Some(p) => s"$ioId.bytes_strip_right($expr0, $p)" + case Some(p) => s"$ioId.bytes_strip_right($expr0, $p).into()" case None => expr0 } terminator match { - case Some(term) => s"$ioId.bytes_terminate($expr, $term, $include)" + case Some(term) => s"$ioId.bytes_terminate($expr, $term, $include).into()" case None => expr } } From d3e8ee650a4ddc510663fb742e2617da2e748200 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Fri, 26 Aug 2022 22:02:54 +0200 Subject: [PATCH 067/153] use `super::` instead of `crate::` --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 396f8660a..3753ff7aa 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -68,7 +68,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = importList.add( - s"crate::${classSpec.name.last}::${type2class(classSpec.name.last)}" + s"super::${classSpec.name.last}::${type2class(classSpec.name.last)}" ) override def classHeader(name: List[String]): Unit = { From 6fb9fe76c2b47533efe0a2c2143a6b40b10ed5e2 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Fri, 26 Aug 2022 22:03:51 +0200 Subject: [PATCH 068/153] `doName` checks for param --- .../kaitai/struct/translators/RustTranslator.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 5b5c41e15..4f7b50b38 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -55,11 +55,17 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doName(s: String): String = s match { case Identifier.PARENT => s case _ => - val found = get_instance(get_top_class(provider.nowClass), s) - if (found.isDefined) { + val topClass = get_top_class(provider.nowClass) + val instance_found = get_instance(topClass, s) + if (instance_found.isDefined) { s"$s(${privateMemberName(IoIdentifier)})?" } else { - s"$s()" + val param_found = get_param(topClass, s) + if(param_found.isDefined) { + s + } else { + s"$s()" + } } } From ac8ee1bfc2007666930cd37d8a3c4b90599e6eda Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 27 Aug 2022 08:44:05 +0200 Subject: [PATCH 069/153] code restored. --- .../struct/languages/RustCompiler.scala | 44 +++++++++++++------ .../struct/translators/RustTranslator.scala | 11 ++--- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 3753ff7aa..b1d96a44c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -374,9 +374,29 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) RustCompiler.privateMemberName(id) override def instanceDeclHeader(className: List[String]): Unit = { - out.puts( - s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {" - ) + if (!typeProvider.nowClass.params.isEmpty) { + val paramsArg = Utils.join(typeProvider.nowClass.params.map{ case p => + val n = paramName(p.id) + val t = kaitaiTypeToNativeType(Some(p.id), typeProvider.nowClass, p.dataType, excludeOptionWrapper = true) + var byref = "" + if (!translator.is_copy_type(p.dataType)) + byref = "&" + // generate param access helper + attributeReader(p.id, p.dataType, false) + s"$n: $byref$t" + }, "", ", ", "") + + out.puts(s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") + out.inc + out.puts(s"pub fn set_params(&mut self, $paramsArg) {") + out.inc + typeProvider.nowClass.params.foreach((p) => handleAssignmentParams(p.id, paramName(p.id))) + out.dec + out.puts("}") + out.dec + out.puts("}") + } + out.puts(s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") out.inc } @@ -464,25 +484,23 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) var in_instance = false override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { - translator.context_need_deref_attr = true - val expr = expression(value) - dataType match { case _: UserType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expr)}.clone();") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.clone();") case _: StrType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expr)}.to_string();") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.to_string();") case _: BytesType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.rem_vec_amp(translator.remove_deref(expr))}.to_vec();") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") case _: ArrayType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.rem_vec_amp(translator.remove_deref(expr))}.to_vec();") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") case _: EnumType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expr)};") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))};") case _ => + translator.context_need_deref_attr = true val primType = kaitaiPrimitiveToNativeType(dataType) - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ($expr) as $primType;") + out.puts(s"*${privateMemberName(instName)}.borrow_mut() = (${expression(value)}) as $primType;") + translator.context_need_deref_attr = false } - translator.context_need_deref_attr = false } override def instanceReturn(instName: InstanceIdentifier, diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 4f7b50b38..913a6604d 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -60,12 +60,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) if (instance_found.isDefined) { s"$s(${privateMemberName(IoIdentifier)})?" } else { - val param_found = get_param(topClass, s) - if(param_found.isDefined) { - s - } else { - s"$s()" - } + s"$s()" } } @@ -356,13 +351,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) detectType(a) match { case t: CalcArrayType => t.elType match { - //case f: FloatMultiType => true + case f: FloatMultiType => true case CalcFloatType => true case _ => false } case t: ArrayType => t.elType match { - //case f: FloatMultiType => true + case f: FloatMultiType => true case _ => false } case _ => false From 1653d52eabae6330e0c3a5ddc9baa633c656700b Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 1 Sep 2022 16:04:55 +0200 Subject: [PATCH 070/153] strings ops, numeric ops. --- .../struct/translators/RustTranslator.scala | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 913a6604d..c63162f48 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -39,16 +39,28 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def strLiteralUnicode(code: Char): String = "\\u{%x}".format(code.toInt) + def isSignedIntType(dt: DataType): Boolean = dt match { + case Int1Type(true) => true + case IntMultiType(true, _, _) => true + case CalcIntType => true + case _ => false + } + override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr): String = { - (detectType(left), detectType(right), op) match { - case (_: IntType, _: IntType, Ast.operator.Div) => - s"${translate(left)} / ${translate(right)}" - case (_: IntType, _: IntType, Ast.operator.Mod) => - s"${translate(left)} % ${translate(right)}" - case _ => + val lt = detectType(left) + val rt = detectType(right) + + if (isSignedIntType(lt) && isSignedIntType(rt) && op == Ast.operator.Mod) + s"modulo(${translate(left)} as i64, ${translate(right)} as i64)" + else { + if (lt != rt) { + val ct = RustCompiler.kaitaiPrimitiveToNativeType(TypeDetector.combineTypes(lt, rt)) + s"(${translate(left)} as $ct ${binOp(op)} ${translate(right)} as $ct)" + } else { super.numericBinOp(left, op, right) + } } } @@ -298,7 +310,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def strToInt(s: expr, base: expr): String = translate(base) match { case "10" => - s"${translate(s)}.parse().unwrap()" + s"${translate(s)}.parse::().unwrap()" case _ => "panic!(\"Converting from string to int in base {} is unimplemented\"" + translate( base @@ -309,7 +321,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"i64::from(${remove_deref(translate(v))})" override def boolToInt(v: expr): String = - s"${translate(v)} as i32" + s"(${translate(v)}) as i32" override def floatToInt(v: expr): String = s"${translate(v)} as i32" @@ -318,7 +330,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) val baseStr = translate(base) baseStr match { case "10" => - s"${remove_deref(translate(i))}.to_string()" + s"${remove_deref(translate(i))}.to_string()" case _ => s"base_convert(strval(${translate(i)}), 10, $baseStr)" } @@ -335,10 +347,15 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${remove_deref(translate(b))}.len()" override def strLength(s: expr): String = s"${remove_deref(translate(s))}.len()" - override def strReverse(s: expr): String = - s"${translate(s)}.graphemes(true).rev().flat_map(|g| g.chars()).collect()" + override def strReverse(s: expr): String = { + val e = translate(s) + if (e.charAt(0) == '*') + s"reverse_string(&$e)?" + else + s"reverse_string($e)?" + } override def strSubstring(s: expr, from: expr, to: expr): String = - s"${translate(s)}.substring(${translate(from)}, ${translate(to)})" + s"${translate(s)}[${translate(from)}..${translate(to)}]" override def arrayFirst(a: expr): String = s"${ensure_deref(translate(a))}.first().ok_or(KError::EmptyIterator)?" From 95ac64957dc7b2e78b2bf577cb5c40d2fd7c73cc Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 2 Sep 2022 09:56:08 +0200 Subject: [PATCH 071/153] int to enum. --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index c63162f48..b403ad8dd 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -261,8 +261,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } override def doEnumById(enumTypeAbs: List[String], id: String): String = - // Just an integer, without any casts / resolutions - one would have to look up constants manually - id + s"($id as i64).try_into()?" override def arraySubscript(container: expr, idx: expr): String = s"${remove_deref(translate(container))}[${translate(idx)} as usize]" From 4f6daf569b683f0652327a16762658b35d1faae1 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 2 Sep 2022 21:10:15 +0200 Subject: [PATCH 072/153] strToInt with any base. --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index b403ad8dd..3dd7ae0a5 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -304,16 +304,14 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) // Predefined methods of various types override def strConcat(left: Ast.expr, right: Ast.expr): String = - "format!(\"{}{}\", " + translate(left) + ", " + translate(right) + ")" + s"""format!("{}{}", ${translate(left)}, ${translate(right)})""" override def strToInt(s: expr, base: expr): String = translate(base) match { case "10" => s"${translate(s)}.parse::().unwrap()" case _ => - "panic!(\"Converting from string to int in base {} is unimplemented\"" + translate( - base - ) + ")" + s"i32::from_str_radix(${translate(s)}, ${translate(base)}).unwrap()" } override def enumToInt(v: expr, et: EnumType): String = From 0aca2f1685209c433c530f59546ff9c9bba990e3 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 3 Sep 2022 13:31:05 +0200 Subject: [PATCH 073/153] string compare fixed. --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 3dd7ae0a5..1fe60249f 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -257,7 +257,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { - s"${remove_deref(translate(left))}.as_str() ${cmpOp(op)} ${remove_deref(translate(right))}" + s"${ensure_deref(translate(left))} ${cmpOp(op)} ${remove_deref(translate(right))}.to_string()" } override def doEnumById(enumTypeAbs: List[String], id: String): String = From f3384ff49f1ed2564aacf108ae63ff03fcb80c3c Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sun, 4 Sep 2022 13:28:23 +0200 Subject: [PATCH 074/153] repeatUntil initial version. --- .../struct/languages/RustCompiler.scala | 33 ++++++++++--------- .../struct/translators/RustTranslator.scala | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index b1d96a44c..0a859dba3 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -265,6 +265,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) dataType: DataType): Unit = { out.puts("{") out.inc + out.puts(s"let mut _i = 0;") out.puts(s"while !_io.is_eof() {") out.inc } @@ -274,6 +275,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def condRepeatEosFooter: Unit = { + out.puts("_i += 1;") out.dec out.puts("}") out.dec @@ -294,24 +296,28 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) io: String, dataType: DataType, repeatExpr: Ast.expr): Unit = { - // TODO: Actual implementation, this is a shim to enable compiling out.puts("{") out.inc - - out.puts( - s"// condRepeatUntilHeader($id, $io, $dataType, $repeatExpr)" - ) + out.puts("let mut _i = 0;") + out.puts("while {") + out.inc } + override def handleAssignmentRepeatUntil(id: Identifier, + expr: String, + isRaw: Boolean): Unit = + out.puts(s"${privateMemberName(id)}.push($expr);") + override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, repeatExpr: Ast.expr): Unit = { - out.puts( - s"// condRepeatUntilFooter($id, $io, $dataType, $repeatExpr)" - ) + out.puts("_i += 1;") + out.puts(s"!(${expression(repeatExpr)})") out.dec out.puts("} {}") + out.dec + out.puts("}") } def getRawIdExpr(varName: Identifier, rep: RepeatSpec): String = { @@ -606,11 +612,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = handleAssignmentRepeatEos(id, expr) - override def handleAssignmentRepeatUntil(id: Identifier, - expr: String, - isRaw: Boolean): Unit = - out.puts(s"// handleAssignmentRepeatUntil($id, $expr, $isRaw)") - def handleAssignmentParams(id: Identifier, expr: String): Unit = { val paramId = typeProvider.nowClass.params.find(s => s.id == id) var need_clone = false @@ -695,18 +696,20 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) addParams = Utils.join(t.args.map{ a => val typ = translator.detectType(a) var byref = "" + val t = kaitaiTypeToNativeType(None, typeProvider.nowClass, typ) + var try_into = "" // exclude enums typ match { + case _: NumericType => try_into = s".try_into().map_err(|_| KError::CastError)?" case _: EnumType => case _ => if (!translator.is_copy_type(typ)) byref = "&" } - val t = kaitaiTypeToNativeType(None, typeProvider.nowClass, typ) var need_deref = "" if (t.startsWith("RefCell")) need_deref = "&*" - s"$byref$need_deref${translator.translate(a)}" + s"$byref$need_deref${translator.translate(a)}$try_into" }, "", ", ", "") val userType = t match { case t: UserType => diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 1fe60249f..6abaeece6 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -230,7 +230,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doLocalName(s: String): String = s match { case Identifier.ITERATOR => "tmpa" case Identifier.ITERATOR2 => "tmpb" - case Identifier.INDEX => "i" + case Identifier.INDEX => "_i" case Identifier.IO => s"${RustCompiler.privateMemberName(IoIdentifier)}" case Identifier.ROOT => s"${RustCompiler.privateMemberName(RootIdentifier)}.ok_or(KError::MissingRoot)?" case Identifier.PARENT => s"${RustCompiler.privateMemberName(ParentIdentifier)}.as_ref().unwrap().peek()" From 2e76c36435b76313b18e078ad1236538dab75dc0 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 9 Sep 2022 14:51:22 +0200 Subject: [PATCH 075/153] always report stacktrace at exception. --- shared/src/main/scala/io/kaitai/struct/Main.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index d178b184a..8559e4c52 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -104,6 +104,10 @@ object Main { e.printStackTrace throw e } + case e: Throwable => { + e.printStackTrace + throw e + } } } From fbded32c5255c33d1943cc64ef8d9b1f2648e4ab Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 9 Sep 2022 14:52:27 +0200 Subject: [PATCH 076/153] condRepeatUntil implemented. --- .../struct/languages/RustCompiler.scala | 28 ++++++++++--------- .../struct/translators/RustTranslator.scala | 4 +-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 0a859dba3..3eebde1f1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -256,7 +256,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc } - override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { + override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { + // this line required for handleAssignmentRepeatUntil + typeProvider._currentIteratorType = Some(dataType) out.puts(s"${privateMemberName(id)} = Vec::new();") } @@ -305,13 +307,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def handleAssignmentRepeatUntil(id: Identifier, expr: String, - isRaw: Boolean): Unit = + isRaw: Boolean): Unit = { out.puts(s"${privateMemberName(id)}.push($expr);") + var copy_type = "" + if (typeProvider._currentIteratorType.isDefined && translator.is_copy_type(typeProvider._currentIteratorType.get)) { + copy_type = "*" + } + out.puts(s"let ${translator.doLocalName(Identifier.ITERATOR)} = $copy_type${privateMemberName(id)}.last().unwrap();") + } override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, repeatExpr: Ast.expr): Unit = { + // this line required by kaitai code + typeProvider._currentIteratorType = Some(dataType) out.puts("_i += 1;") out.puts(s"!(${expression(repeatExpr)})") out.dec @@ -870,11 +880,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val memberName = privateMemberName(id) val ioId = IoStorageIdentifier(id) -// val args = rep match { -// case RepeatUntil(_) => translator.doName(Identifier.ITERATOR2) -// case _ => privateMemberName(id) -// } - val newStreamRaw = s"$memberName" val ioName = rep match { case NoRepeat => @@ -883,12 +888,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"let $localIO = BytesReader::new(&$newStream);") s"&$localIO" case _ => - newStreamRaw - // TODO - //val localIO = s"io_${idToStr(id)}" - // out.puts(s"$kstreamName* $localIO = $newStreamRaw;") - // out.puts(s"${privateMemberName(ioId)}->push_back($localIO);") - //localIO + val localIO = s"io_${idToStr(id)}" + out.puts(s"let $localIO = BytesReader::new(&$newStreamRaw.last().unwrap());") + s"&$localIO" } ioName diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 6abaeece6..b4895dacc 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -228,8 +228,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } override def doLocalName(s: String): String = s match { - case Identifier.ITERATOR => "tmpa" - case Identifier.ITERATOR2 => "tmpb" + case Identifier.ITERATOR => "_tmpa" + case Identifier.ITERATOR2 => "_tmpb" case Identifier.INDEX => "_i" case Identifier.IO => s"${RustCompiler.privateMemberName(IoIdentifier)}" case Identifier.ROOT => s"${RustCompiler.privateMemberName(RootIdentifier)}.ok_or(KError::MissingRoot)?" From 23437619ab10dc1edca57f1378293c4a8618ac66 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 15 Sep 2022 20:40:31 +0200 Subject: [PATCH 077/153] instance and repeat until. --- .../io/kaitai/struct/languages/RustCompiler.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 3eebde1f1..a13dcff38 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -259,7 +259,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { // this line required for handleAssignmentRepeatUntil typeProvider._currentIteratorType = Some(dataType) - out.puts(s"${privateMemberName(id)} = Vec::new();") + if (in_instance) { + out.puts(s"*${privateMemberName(id)}.borrow_mut() = Vec::new();") + } else { + out.puts(s"${privateMemberName(id)} = Vec::new();") + } } override def condRepeatEosHeader(id: Identifier, @@ -273,7 +277,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = { - out.puts(s"${privateMemberName(id)}.push($expr);") + if (in_instance) { + out.puts(s"${privateMemberName(id)}.borrow_mut().push($expr);") + } else { + out.puts(s"${privateMemberName(id)}.push($expr);") + } } override def condRepeatEosFooter: Unit = { From 2f71ce232fc72830f79fe7a5f350813b5412eb69 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 16 Sep 2022 09:30:55 +0200 Subject: [PATCH 078/153] append RefCell to raw_ ids if they using in instance. --- .../struct/languages/RustCompiler.scala | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index a13dcff38..4317b8f75 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -166,12 +166,27 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct case IoIdentifier | RootIdentifier | ParentIdentifier => return case _ => - kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType) + kaitaiTypeToNativeTypeWrapper(Some(attrName), attrType) } out.puts(s"${idToStr(attrName)}: $typeName,") } + def kaitaiTypeToNativeTypeWrapper(id: Option[Identifier], + attrType: DataType): String = { + if (id.isDefined) id.get match { + case RawIdentifier(inner) => { + val found = translator.get_instance(typeProvider.nowClass, idToStr(inner)) + if (found.isDefined) { + // raw id for instance should have RefCell wrapper + return s"RefCell<${kaitaiTypeToNativeType(id, typeProvider.nowClass, attrType)}>" + } + } + case _ => + } + kaitaiTypeToNativeType(id, typeProvider.nowClass, attrType) + } + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { @@ -179,7 +194,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct case IoIdentifier | RootIdentifier | ParentIdentifier => return case _ => - kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType) + kaitaiTypeToNativeTypeWrapper(Some(attrName), attrType) } val typeNameEx = kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType, excludeOptionWrapper = true) out.puts( @@ -888,7 +903,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val memberName = privateMemberName(id) val ioId = IoStorageIdentifier(id) - val newStreamRaw = s"$memberName" + var newStreamRaw = s"$memberName" val ioName = rep match { case NoRepeat => val newStream = newStreamRaw @@ -896,7 +911,12 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"let $localIO = BytesReader::new(&$newStream);") s"&$localIO" case _ => - val localIO = s"io_${idToStr(id)}" + val ids = idToStr(id) + val localIO = s"io_$ids" + if (in_instance) { + out.puts(s"let $ids = $newStreamRaw.borrow();") + newStreamRaw = ids + } out.puts(s"let $localIO = BytesReader::new(&$newStreamRaw.last().unwrap());") s"&$localIO" } From bbe279a5608d71ab3d0bca475bdd42dbc58e7791 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sun, 18 Sep 2022 10:56:48 +0200 Subject: [PATCH 079/153] root arg restored for instance methods. --- .../io/kaitai/struct/languages/RustCompiler.scala | 13 ++++++++----- .../kaitai/struct/translators/RustTranslator.scala | 4 +++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 4317b8f75..adcd096c7 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -395,7 +395,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) handleAssignment(varDest, expr, rep, isRaw = false) } - override def useIO(ioEx: Ast.expr): String = s"// useIO($ioEx)" + override def useIO(ioEx: Ast.expr): String = { + out.puts(s"let io = BytesReader::new(&${expression(ioEx)});") + "io" + } override def pushPos(io: String): Unit = out.puts(s"let _pos = $io.pos();") @@ -498,7 +501,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"pub fn ${idToStr(instName)}(") out.inc out.puts("&self,") - out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S") + out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S,") + out.puts(s"${privateMemberName(RootIdentifier)}: Option<&${classTypeName(typeProvider.topClass)}>") out.dec val typeName = kaitaiTypeToNativeType( Some(instName), @@ -508,7 +512,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) out.puts(s") -> KResult> {") out.inc - out.puts(s"let _root : Option<&${classTypeName(typeProvider.topClass)}> = None;") } override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, @@ -1147,8 +1150,8 @@ object RustCompiler case NamedIdentifier(n) => n case InstanceIdentifier(n) => n case NumberedIdentifier(idx) => s"${NumberedIdentifier.TEMPLATE}$idx" - case RawIdentifier(inner) => s"raw_${idToStr(inner)}" - case IoStorageIdentifier(inner) => s"io_${idToStr(inner)}" + case RawIdentifier(inner) => s"${idToStr(inner)}_raw" // use suffix naming, easy to replace, like in anyField() + case IoStorageIdentifier(inner) => s"${idToStr(inner)}_io" // same here } @tailrec diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index b4895dacc..81194d967 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -70,7 +70,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) val topClass = get_top_class(provider.nowClass) val instance_found = get_instance(topClass, s) if (instance_found.isDefined) { - s"$s(${privateMemberName(IoIdentifier)})?" + s"$s(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)})?" } else { s"$s()" } @@ -122,6 +122,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } attrName match { + case Identifier.IO => + r = r.replace("()._io()", "_raw()") case Identifier.PARENT => // handle _parent._parent r = r.replace(".peek()._parent", ".pop().peek()") From 61d02fa49be2911453371165ef8b6e032cef517a Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sun, 18 Sep 2022 17:48:44 +0200 Subject: [PATCH 080/153] detect if in_instance. --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index adcd096c7..c745cc052 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -909,8 +909,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) var newStreamRaw = s"$memberName" val ioName = rep match { case NoRepeat => - val newStream = newStreamRaw + var newStream = newStreamRaw val localIO = localTemporaryName(ioId) + if (in_instance) { + val ids = idToStr(id) + out.puts(s"let $ids = $newStream.borrow();") + newStream = ids + } out.puts(s"let $localIO = BytesReader::new(&$newStream);") s"&$localIO" case _ => From a7f5e16efd38245ca71b271bef66ddee33465148 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 22 Sep 2022 20:40:01 +0200 Subject: [PATCH 081/153] handle RawIdentifier(InstanceIdentifier()) --- .../io/kaitai/struct/languages/RustCompiler.scala | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index c745cc052..6c3bccc40 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -700,12 +700,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } if (!done) { + var inst = false id match { case _: InstanceIdentifier => - done = true - out.puts(s"*${privateMemberName(id)}.borrow_mut() = $expr;") + inst = true + case RawIdentifier(inner) => inner match { + case _: InstanceIdentifier => + inst = true + case _ => + } case _ => } + if (inst) { + done = true + out.puts(s"*${privateMemberName(id)}.borrow_mut() = $expr;") + } } if (!done) out.puts(s"${privateMemberName(id)} = $expr;") From 888ce28a148c0ed449d98d0c7d3df9eec6fadbea Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 24 Sep 2022 15:45:04 +0200 Subject: [PATCH 082/153] handle opaque types. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 6c3bccc40..283905954 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -765,7 +765,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$baseName" } val addArgs = if (t.isOpaque) { - "" + ", None, None" } else { val currentType = classTypeName(typeProvider.nowClass) val root = if (typeProvider.nowClass.isTopLevel) { From 7565497a6185794d3201481c2b69d6af5453293f Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 24 Sep 2022 15:45:21 +0200 Subject: [PATCH 083/153] getTopClass rewritted. --- .../io/kaitai/struct/translators/RustTranslator.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 81194d967..e32c6a807 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -76,11 +76,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } - def get_top_class(c: ClassSpec): ClassSpec = { - if (c.isTopLevel) { - return c - } - get_top_class(c.upClass.get) + def get_top_class(c: ClassSpec): ClassSpec = c.upClass match { + case Some(upClass) => get_top_class(upClass) + case None => c } def get_instance(cs: ClassSpec, s: String): Option[InstanceSpec] = { From a8a27b877cba44b0a5e8e2e30c8f805f91f494e8 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 24 Sep 2022 20:56:48 +0300 Subject: [PATCH 084/153] handle opaque types. --- .../io/kaitai/struct/languages/RustCompiler.scala | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 283905954..048b7dbcc 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -674,10 +674,15 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val seqId = typeProvider.nowClass.seq.find(s => s.id == id) var done = false var refcell = false + var opaque = false if (seqId.isDefined) { val idType = seqId.get.dataType idType match { - case _: UserType => refcell = true + case t: UserType => + refcell = true + if (t.isOpaque) { + opaque = true + } case _: BytesType => refcell = true case _: ArrayType => refcell = true case _: StrType => refcell = true @@ -694,7 +699,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (refcell) { val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, idType) if (typeName.startsWith("RefCell")) { - out.puts(s"${privateMemberName(id)} = RefCell::new($expr);") + if (opaque) { + out.puts(s"${privateMemberName(id)} = RefCell::new(Some(Box::new($expr)));") + } else { + out.puts(s"${privateMemberName(id)} = RefCell::new($expr);") + } done = true } } @@ -1253,7 +1262,7 @@ object RustCompiler // Because we can't predict if opaque types will recurse, we have to box them val typeName = - if (!excludeBox && t.isOpaque) s"Box<$baseName>" + if (!excludeBox && t.isOpaque) s"Option>" else s"$baseName" if (excludeOptionWrapper) typeName else s"RefCell<$typeName>" From 0b31ec45b298686dfe9a5f8071a62b09fd643412 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 1 Oct 2022 16:15:33 +0200 Subject: [PATCH 085/153] handle bitEndian flag. --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 048b7dbcc..a33ad4ee9 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -738,14 +738,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) defEndian: Option[FixedEndian]): String = { var addParams = "" val expr = dataType match { - case IntMultiType(_, _, None) => "panic!(\"Unable to parse unknown-endian integers\")" case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?.into()" case _: BytesEosType => s"$io.read_bytes_full()?.into()" case b: BytesTerminatedType => s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?.into()" case b: BytesLimitType => s"$io.read_bytes(${expression(b.size)} as usize)?.into()" - case _: BitsType1 => s"$io.read_bits_int(1)? != 0" - case BitsType(width, _) => s"$io.read_bits_int($width)?" + case BitsType1(bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}(1)? != 0" + case BitsType(width: Int, bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}($width)?" case t: UserType => addParams = Utils.join(t.args.map{ a => val typ = translator.detectType(a) @@ -1141,8 +1140,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) t.basedOn match { case inst: ReadableType => s"$io.read_${inst.apiCall(defEndian)}()?" - case BitsType(width: Int, _) => - s"$io.read_bits_int($width)?" + case BitsType(width: Int, bitEndian) => + s"$io.read_bits_int_${bitEndian.toSuffix}($width)?" } handleAssignment(id, expr, rep, isRaw) case _ => From 5ddee6dad17ca40f7d2eb773df53d5251549ac6c Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 5 Oct 2022 22:39:35 +0200 Subject: [PATCH 086/153] handle endian. --- .../io/kaitai/struct/RustClassCompiler.scala | 32 +++++++--- .../struct/languages/RustCompiler.scala | 62 +++++++++++++++---- 2 files changed, 75 insertions(+), 19 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala index 2b0fb7773..cbc765cf3 100644 --- a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala @@ -1,7 +1,8 @@ package io.kaitai.struct -import io.kaitai.struct.datatype.DataType.{KaitaiStreamType, UserTypeInstream} -import io.kaitai.struct.datatype.{Endianness, FixedEndian, InheritedEndian} +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype._ +import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ import io.kaitai.struct.languages.RustCompiler import io.kaitai.struct.languages.components.ExtraAttrs @@ -34,10 +35,10 @@ class RustClassCompiler( curClass.instances.foreach { case (instName, instSpec) => compileInstanceDeclaration(instName, instSpec) } - + // Constructor = Read() function compileReadFunction(curClass) - + compileInstances(curClass) compileAttrReaders(curClass.seq ++ extraAttrs) @@ -58,18 +59,35 @@ class RustClassCompiler( curClass.params ) - // FIXME val defEndian = curClass.meta.endian match { case Some(fe: FixedEndian) => Some(fe) case _ => None } - + lang.readHeader(defEndian, false) - + + curClass.meta.endian match { + case Some(ce: CalcEndian) => compileCalcEndian(ce) + case Some(_) => // Nothing to generate + case None => // Same here + } + compileSeq(curClass.seq, defEndian) lang.classConstructorFooter } + override def compileCalcEndian(ce: CalcEndian): Unit = { + def renderProc(result: FixedEndian): Unit = { + val v = result match { + case LittleEndian => Ast.expr.IntNum(1) + case BigEndian => Ast.expr.IntNum(2) + } + lang.instanceCalculate(IS_LE_ID, CalcIntType, v) + } + lang.switchCases[FixedEndian](IS_LE_ID, ce.on, ce.cases, renderProc, renderProc) + lang.runReadCalc() + } + override def compileInstances(curClass: ClassSpec) = { lang.instanceDeclHeader(curClass.name) curClass.instances.foreach { case (instName, instSpec) => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index a33ad4ee9..ffccbba41 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -101,6 +101,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + typeProvider.nowClass.meta.endian match { + case Some(_: CalcEndian) | Some(InheritedEndian) => + attributeDeclaration(EndianIdentifier, IntMultiType(true, Width4, None), false) + case _ => + } // Unlike other OOP languages, implementing an interface happens outside the struct declaration. universalFooter @@ -132,9 +137,15 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts } - override def runRead(name: List[String]): Unit = out.puts(s"// runRead($name)") + override def runRead(name: List[String]): Unit = {} - override def runReadCalc(): Unit = out.puts(s"// runReadCalc()") + override def runReadCalc(): Unit = { + out.puts(s"if *${privateMemberName(EndianIdentifier)}.borrow() == 0 {") + out.inc + out.puts(s"""return Err(KError::UndecidedEndiannessError("${typeProvider.nowClass.path.mkString("/", "/", "")}".to_string()));""") + out.dec + out.puts("}") + } override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { @@ -146,7 +157,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"${privateMemberName(RootIdentifier)}: Option<&$readLife Self::Root>," ) out.puts( - s"${privateMemberName(ParentIdentifier)}: Option>" + s"${privateMemberName(ParentIdentifier)}: Option>," ) out.dec out.puts(s") -> KResult<()> {") @@ -165,6 +176,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val typeName = attrName match { // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct case IoIdentifier | RootIdentifier | ParentIdentifier => return + case EndianIdentifier => s"RefCell<${kaitaiTypeToNativeTypeWrapper(Some(attrName), attrType)}>" case _ => kaitaiTypeToNativeTypeWrapper(Some(attrName), attrType) } @@ -263,8 +275,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = - out.puts(s"// attrParseHybrid(${leProc()}, ${beProc()})") + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = {} override def condIfHeader(expr: Ast.expr): Unit = { out.puts(s"if ${expression(expr)} {") @@ -438,6 +449,20 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec out.puts("}") } + typeProvider.nowClass.meta.endian match { + case Some(_: CalcEndian) | Some(InheritedEndian) => + out.puts(s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") + out.inc + val t = kaitaiTypeToNativeType(Some(EndianIdentifier), typeProvider.nowClass, IntMultiType(true, Width4, None), excludeOptionWrapper = true) + out.puts(s"pub fn set_endian(&mut self, ${idToStr(EndianIdentifier)}: $t) {") + out.inc + handleAssignmentSimple(EndianIdentifier, s"${idToStr(EndianIdentifier)}") + out.dec + out.puts("}") + out.dec + out.puts("}") + case _ => + } out.puts(s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") out.inc } @@ -718,6 +743,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) inst = true case _ => } + case EndianIdentifier => + inst = true case _ => } if (inst) { @@ -738,7 +765,15 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) defEndian: Option[FixedEndian]): String = { var addParams = "" val expr = dataType match { - case t: ReadableType => s"$io.read_${t.apiCall(defEndian)}()?.into()" + case t: ReadableType => + t match { + case IntMultiType(_, _, None) => + s"if *${privateMemberName(EndianIdentifier)}.borrow() == 1 { $io.read_${t.apiCall(Some(LittleEndian))}()?.into() } else { $io.read_${t.apiCall(Some(BigEndian))}()?.into() }" + case IntMultiType(_, _, Some(e)) => + s"$io.read_${t.apiCall(Some(e))}()?.into()" + case _ => + s"$io.read_${t.apiCall(defEndian)}()?.into()" + } case _: BytesEosType => s"$io.read_bytes_full()?.into()" case b: BytesTerminatedType => s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?.into()" @@ -795,11 +830,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } } - val addEndian = t.classSpec.get.meta.endian match { - case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}" - case _ => "" - } - s", $root, $parent$addEndian" + s", $root, $parent" } val streamType = if (io == privateMemberName(IoIdentifier)) { "S" @@ -807,7 +838,14 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) "BytesReader" } if (addParams.isEmpty) { - out.puts(s"let t = Self::read_into::<$streamType, $userType>($io$addArgs)?.into();") + if (t.classSpec.isDefined) t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => + out.puts(s"let mut t = $userType::default();") + out.puts(s"t.set_endian(*${privateMemberName(EndianIdentifier)}.borrow());") + out.puts(s"t.read::<$streamType>($io$addArgs)?;") + case _ => + out.puts(s"let t = Self::read_into::<$streamType, $userType>($io$addArgs)?.into();") + } } else { //val at = kaitaiTypeToNativeType(None, typeProvider.nowClass, assignType, excludeOptionWrapper = true) out.puts(s"let mut t = $userType::default();") From d75de69d4a5c1ca6da92d6d06088e55b135f216f Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 6 Oct 2022 20:56:20 +0200 Subject: [PATCH 087/153] enum into array fixed. --- .../io/kaitai/struct/languages/RustCompiler.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index ffccbba41..a3e3c4765 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -714,7 +714,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: EnumType => done = true out.puts( - s"${privateMemberName(id)} = Some(($expr as i64).try_into()?);" + s"${privateMemberName(id)} = Some($expr);" ) case _: SwitchType => done = true @@ -889,6 +889,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" + override def userTypeDebugRead(id: String, dataType: DataType, assignType: DataType): Unit = { + // we already have splitted construction of object and read method + } + override def switchRequiresIfs(onType: DataType): Boolean = onType match { case _: IntType | _: EnumType => false case _ => true @@ -1177,9 +1181,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val expr = t.basedOn match { case inst: ReadableType => - s"$io.read_${inst.apiCall(defEndian)}()?" + s"($io.read_${inst.apiCall(defEndian)}()? as i64).try_into()?" case BitsType(width: Int, bitEndian) => - s"$io.read_bits_int_${bitEndian.toSuffix}($width)?" + s"($io.read_bits_int_${bitEndian.toSuffix}($width)? as i64).try_into()?" } handleAssignment(id, expr, rep, isRaw) case _ => From 6118dc830d175573b4e9d4a265c136e3d4f1ff41 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 8 Oct 2022 11:42:28 +0200 Subject: [PATCH 088/153] small fix of numericOp. --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index e32c6a807..37ef5e7cc 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -57,7 +57,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) else { if (lt != rt) { val ct = RustCompiler.kaitaiPrimitiveToNativeType(TypeDetector.combineTypes(lt, rt)) - s"(${translate(left)} as $ct ${binOp(op)} ${translate(right)} as $ct)" + s"((${translate(left)} as $ct) ${binOp(op)} (${translate(right)} as $ct))" } else { super.numericBinOp(left, op, right) } From be059060d30684d95650601e53ba7078aa6ac7ab Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 11 Oct 2022 14:38:18 +0200 Subject: [PATCH 089/153] handle ProcessCustom. --- .../struct/languages/RustCompiler.scala | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index a3e3c4765..4e07d32b0 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -38,7 +38,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def outImports(topClass: ClassSpec): String = importList.toList - .map(i => s"use $i;") .mkString("", "\n", "\n") override def fileHeader(topClassName: String): Unit = { @@ -54,21 +53,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) outHeader.puts("extern crate kaitai;") importList.add( - "kaitai::*" + "use kaitai::*;" ) - importList.add("std::convert::{TryFrom, TryInto}") - importList.add("std::cell::{Ref, Cell, RefCell}") + importList.add("use std::convert::{TryFrom, TryInto};") + importList.add("use std::cell::{Ref, Cell, RefCell};") typeProvider.allClasses.foreach{ case (name, _) => if(name != topClassName) //TODO: do not add to imported - importList.add(s"super::$name::*") + importList.add(s"use super::$name::*;") } } override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = importList.add( - s"super::${classSpec.name.last}::${type2class(classSpec.name.last)}" + s"use super::${classSpec.name.last}::${type2class(classSpec.name.last)};" ) override def classHeader(name: List[String]): Unit = { @@ -396,11 +395,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val procClass = name.map(x => type2class(x)).mkString("::") val procName = s"_process_${idToStr(varSrc)}" - //importListSrc.addLocal(outFileNameHeader(name.last)) + val mod_name = name.last; + importList.add(s"""#[path = "$mod_name.rs"] mod $mod_name;""") + importList.add(s"use self::$mod_name::*;") val argList = args.map(expression).mkString(", ") - val argListInParens = if (argList.nonEmpty) s"($argList)" else "" - out.puts(s"$procClass $procName$argListInParens;") + val argListInParens = s"($argList)" + out.puts(s"let $procName = $procClass::new$argListInParens;") s"$procName.decode(&$srcExpr)" } handleAssignment(varDest, expr, rep, isRaw = false) From 072746cc3aedbcb273b23494b12e0a7a3ff43a34 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 11 Oct 2022 15:26:07 +0200 Subject: [PATCH 090/153] process_xor fixed. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 4e07d32b0..b0dc31be4 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -367,7 +367,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val memberName = privateMemberName(varName) rep match { case NoRepeat => memberName - case _ => s"// TODO $memberName->at($memberName->size() - 1)" + case _ => s"$memberName[$memberName.len() - 1]" } } From 25b8a2f0cf8dff59679601c250b0d63ac2dbd794 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 12 Oct 2022 19:56:19 +0200 Subject: [PATCH 091/153] closer behaviour for enums to C++ (enums to i64). --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index b0dc31be4..c11c949c8 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -595,6 +595,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${type2class(label.name)},") } + out.puts("Unknown(i64),") out.dec out.puts("}") @@ -618,7 +619,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case (value, label) => out.puts(s"$value => Ok($enumClass::${type2class(label.name)}),") } - out.puts("_ => Err(KError::UnknownVariant(flag)),") + out.puts(s"_ => Ok($enumClass::Unknown(flag)),") out.dec out.puts("}") @@ -638,6 +639,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case (value, label) => out.puts(s"$enumClass::${type2class(label.name)} => $value,") } + out.puts(s"$enumClass::Unknown(v) => *v") out.dec out.puts("}") out.dec @@ -648,8 +650,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"impl Default for $enumClass {") out.inc - //FIXME: what is default? - out.puts(s"fn default() -> Self { $enumClass::${type2class(enumColl.head._2.name)} }") + out.puts(s"fn default() -> Self { $enumClass::Unknown(0) }") out.dec out.puts("}") out.puts From 08b367a37458a3784d40c8140c85f8e976fb91b5 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 13 Oct 2022 15:02:40 +0200 Subject: [PATCH 092/153] provide default parent if not provided (only for root). --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index c11c949c8..9e50d8630 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -823,7 +823,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(fp) => translator.translate(fp) case None => if ((userType contains currentType) && !in_instance) { - s"Some(${privateMemberName(ParentIdentifier)}.unwrap().push(self))" + if (typeProvider.nowClass.isTopLevel) { + s"Some(${privateMemberName(ParentIdentifier)}.unwrap_or(KStructUnit::parent_stack()).push(self))" + } else { + s"Some(${privateMemberName(ParentIdentifier)}.unwrap().push(self))" + } } else { if(in_instance) { s"None" From 938b8a8b29c8b72a4766ce7b6b77eaf1ce56c433 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 21 Oct 2022 10:56:02 +0200 Subject: [PATCH 093/153] classToString implemented. --- .../scala/io/kaitai/struct/RustClassCompiler.scala | 1 + .../io/kaitai/struct/languages/RustCompiler.scala | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala index cbc765cf3..771c90d7d 100644 --- a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala @@ -42,6 +42,7 @@ class RustClassCompiler( compileInstances(curClass) compileAttrReaders(curClass.seq ++ extraAttrs) + curClass.toStringExpr.foreach(expr => lang.classToString(expr)) lang.classFooter(curClass.name) compileEnums(curClass) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 9e50d8630..659fb9c69 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -1196,6 +1196,19 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) super.attrParse2(id, dataType, io, rep, isRaw, defEndian, assignTypeOpt) } } + + override def classToString(toStringExpr: Ast.expr): Unit = { + importList.add("use std::fmt;") + out.puts(s"impl fmt::Display for ${classTypeName(typeProvider.nowClass)} {") + out.inc + out.puts(s"fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {") + out.inc + out.puts(s"""write!(f, "{}", ${translator.translate(toStringExpr)})""") + out.dec + out.puts("}") + out.dec + out.puts("}") + } } object RustCompiler From 792d349ddae6e097e0b2a17b03f0aa7632805d98 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 15 Nov 2022 08:33:40 +0800 Subject: [PATCH 094/153] parent type generation reviewed. --- .../struct/languages/RustCompiler.scala | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 659fb9c69..c044ac7d5 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -130,8 +130,17 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) out.inc out.puts(s"type Root = ${rootClassTypeName(typeProvider.nowClass)};") + + // generate parent stack type + val parentStack = if (typeProvider.nowClass.isTopLevel) + s"$kstructUnitName" + else { + val parentType = kaitaiTypeToNativeType(None, typeProvider.nowClass, typeProvider.nowClass.parentType, excludeOptionWrapper = true, excludeBox = true) + s"(&$readLife $parentType, <$parentType as $kstructName<$readLife, $streamLife>>::ParentStack)" + } + out.puts( - s"type ParentStack = ${parentStackTypeName(typeProvider.nowClass)};" + s"type ParentStack = $parentStack;" ) out.puts } @@ -818,24 +827,24 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } else { privateMemberName(RootIdentifier) } - val parent = t.forcedParent match { + var parent = t.forcedParent match { case Some(USER_TYPE_NO_PARENT) => "None" case Some(fp) => translator.translate(fp) case None => - if ((userType contains currentType) && !in_instance) { + if (!in_instance) { if (typeProvider.nowClass.isTopLevel) { s"Some(${privateMemberName(ParentIdentifier)}.unwrap_or(KStructUnit::parent_stack()).push(self))" } else { s"Some(${privateMemberName(ParentIdentifier)}.unwrap().push(self))" } } else { - if(in_instance) { - s"None" - } else { - s"${privateMemberName(ParentIdentifier)}" - } + s"None" } } + t.classSpec.get.parentType match { + case CalcKaitaiStructType => parent = "None" + case _ => + } s", $root, $parent" } val streamType = if (io == privateMemberName(IoIdentifier)) { @@ -1247,13 +1256,6 @@ object RustCompiler rootClassTypeName(c.upClass.get, isRecurse = true) } - def parentStackTypeName(c: ClassSpec): String = { - if (c.isTopLevel) - s"$kstructUnitName" - else - s"(&$readLife ${classTypeName(c.upClass.get)}, <${classTypeName(c.upClass.get)} as $kstructName<$readLife, $streamLife>>::ParentStack)" - } - override def kstructName = s"KStruct" def readLife = "'r" @@ -1353,6 +1355,7 @@ object RustCompiler if (excludeOptionWrapper) typeName else s"Option<$typeName>" case KaitaiStreamType => kstreamName + case CalcKaitaiStructType => kstructUnitName } def kaitaiPrimitiveToNativeType(attrType: DataType): String = attrType match { From 30f4a9969f2f452f009bf95eafb15909bdc32b7f Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 17 Nov 2022 09:58:26 +0800 Subject: [PATCH 095/153] EnumType need to clone. --- .../main/scala/io/kaitai/struct/translators/RustTranslator.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 37ef5e7cc..6cfd5717b 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -270,6 +270,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) var to_type = "" detectType(ifTrue) match { case _: UserType => to_type = ".clone()" + case _: EnumType => to_type = ".clone()" case _: StrType => to_type = ".to_string()" case _: BytesType => to_type = ".to_vec()" case _ => From a7f48ae104629d03d1da4cf259f07fd9b3c498a1 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 28 Nov 2022 10:43:25 +0800 Subject: [PATCH 096/153] always use buffered substreams. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index c044ac7d5..6ba7d51b4 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -347,6 +347,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc } + override def createSubstream(id: Identifier, byteType: BytesType, io: String, rep: RepeatSpec, defEndian: Option[FixedEndian]): String = { + createSubstreamBuffered(id, byteType, io, rep, defEndian) + } + override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = { From 282ca92085d744392196666b2fef4d6d89197a6c Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 26 Dec 2022 20:42:08 +0800 Subject: [PATCH 097/153] RefCell for all. Root, parent as SharedType<> --- project/build.properties | 2 +- .../struct/languages/RustCompiler.scala | 253 +++++++++--------- .../struct/translators/RustTranslator.scala | 20 +- 3 files changed, 139 insertions(+), 136 deletions(-) diff --git a/project/build.properties b/project/build.properties index d738b858c..9a19778c3 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.7.1 +sbt.version = 1.8.0 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 6ba7d51b4..d33b2dba0 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -57,6 +57,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) importList.add("use std::convert::{TryFrom, TryInto};") importList.add("use std::cell::{Ref, Cell, RefCell};") + importList.add("use std::rc::{Rc, Weak};") typeProvider.allClasses.foreach{ case (name, _) => @@ -72,7 +73,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def classHeader(name: List[String]): Unit = { out.puts - out.puts("#[derive(Default, Debug, PartialEq, Clone)]") + out.puts("#[derive(Default, Debug, Clone)]") out.puts(s"pub struct ${classTypeName(typeProvider.nowClass)} {") out.inc @@ -80,6 +81,16 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // everyone gets a phantom data marker //out.puts(s"_phantom: std::marker::PhantomData<&$streamLife ()>,") + val root = types2class(name.slice(0, 1)) + out.puts(s"pub ${privateMemberName(RootIdentifier)}: SharedType<$root>,") + + val parent = if (typeProvider.nowClass.isTopLevel) + root + else { + kaitaiTypeToNativeType(None, typeProvider.nowClass, typeProvider.nowClass.parentType, cleanTypename = true) + } + out.puts(s"pub ${privateMemberName(ParentIdentifier)}: SharedType<$parent>,") + typeProvider.nowClass.params.foreach { p => // Make sure the parameter is imported if necessary p.dataType match { @@ -129,18 +140,17 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"impl<$readLife, $streamLife: $readLife> $kstructName<$readLife, $streamLife> for ${classTypeName(typeProvider.nowClass)} {" ) out.inc - out.puts(s"type Root = ${rootClassTypeName(typeProvider.nowClass)};") + val root = classTypeName(typeProvider.topClass) + out.puts(s"type Root = $root;") - // generate parent stack type - val parentStack = if (typeProvider.nowClass.isTopLevel) - s"$kstructUnitName" + val parent = if (typeProvider.nowClass.isTopLevel) + root else { - val parentType = kaitaiTypeToNativeType(None, typeProvider.nowClass, typeProvider.nowClass.parentType, excludeOptionWrapper = true, excludeBox = true) - s"(&$readLife $parentType, <$parentType as $kstructName<$readLife, $streamLife>>::ParentStack)" + kaitaiTypeToNativeType(None, typeProvider.nowClass, typeProvider.nowClass.parentType, cleanTypename = true) } out.puts( - s"type ParentStack = $parentStack;" + s"type Parent = $parent;" ) out.puts } @@ -148,7 +158,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def runRead(name: List[String]): Unit = {} override def runReadCalc(): Unit = { - out.puts(s"if *${privateMemberName(EndianIdentifier)}.borrow() == 0 {") + out.puts(s"if *${privateMemberName(EndianIdentifier)} == 0 {") out.inc out.puts(s"""return Err(KError::UndecidedEndiannessError("${typeProvider.nowClass.path.mkString("/", "/", "")}".to_string()));""") out.dec @@ -157,20 +167,26 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { + RustCompiler.in_reader = true + val root = privateMemberName(RootIdentifier) out.puts(s"fn read(") out.inc - out.puts(s"&mut self,") + out.puts(s"self_rc: &Rc,") out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S,") out.puts( - s"${privateMemberName(RootIdentifier)}: Option<&$readLife Self::Root>," + s"$root: SharedType," ) out.puts( - s"${privateMemberName(ParentIdentifier)}: Option>," + s"${privateMemberName(ParentIdentifier)}: SharedType," ) out.dec out.puts(s") -> KResult<()> {") out.inc + out.puts(s"self_rc._root.set(_root.get());") + out.puts(s"self_rc._parent.set(_parent.get());") + out.puts(s"let _new_parent = SharedType::::new(self_rc.clone());") + // If there aren't any attributes to parse, we need to end the read implementation here if (typeProvider.nowClass.seq.isEmpty) endRead() @@ -184,26 +200,25 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val typeName = attrName match { // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct case IoIdentifier | RootIdentifier | ParentIdentifier => return - case EndianIdentifier => s"RefCell<${kaitaiTypeToNativeTypeWrapper(Some(attrName), attrType)}>" case _ => kaitaiTypeToNativeTypeWrapper(Some(attrName), attrType) } - out.puts(s"${idToStr(attrName)}: $typeName,") + out.puts(s"${idToStr(attrName)}: RefCell<$typeName>,") } def kaitaiTypeToNativeTypeWrapper(id: Option[Identifier], attrType: DataType): String = { - if (id.isDefined) id.get match { - case RawIdentifier(inner) => { - val found = translator.get_instance(typeProvider.nowClass, idToStr(inner)) - if (found.isDefined) { - // raw id for instance should have RefCell wrapper - return s"RefCell<${kaitaiTypeToNativeType(id, typeProvider.nowClass, attrType)}>" - } - } - case _ => - } + // if (id.isDefined) id.get match { + // case RawIdentifier(inner) => { + // val found = translator.get_instance(typeProvider.nowClass, idToStr(inner)) + // if (found.isDefined) { + // // raw id for instance should have RefCell wrapper + // return s"${kaitaiTypeToNativeType(id, typeProvider.nowClass, attrType)}" + // } + // } + // case _ => + // } kaitaiTypeToNativeType(id, typeProvider.nowClass, attrType) } @@ -227,10 +242,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case st: SwitchType => types = st.cases.values.toSet enum_typename = true - case _: EnumType => // TODO? enum_typename = true + case _: EnumType => enum_typename = true case _ => } - var enum_only_numeric = true + var enum_only_numeric = false//true types.foreach { case _: NumericType => // leave unchanged case _ => enum_only_numeric = false @@ -239,27 +254,19 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (enum_typename && enum_only_numeric) { out.puts(s"pub fn $fn(&self) -> usize {") out.inc - out.puts(s"self.${idToStr(attrName)}.as_ref().unwrap().into()") + out.puts(s"self.${idToStr(attrName)}.borrow().as_ref().unwrap().into()") out.dec out.puts("}") fn = s"${fn}_enum" } - { - if (typeName.startsWith("RefCell")) { - out.puts(s"pub fn $fn(&self) -> Ref<$typeNameEx> {") - } else { - out.puts(s"pub fn $fn(&self) -> &$typeNameEx {") - } + if (enum_typename) { + out.puts(s"pub fn $fn(&self) -> &$typeNameEx {") out.inc - if (typeName != typeNameEx) { - if (typeName.startsWith("RefCell")) { - out.puts(s"self.${idToStr(attrName)}.borrow()") - } else { - out.puts(s"self.${idToStr(attrName)}.as_ref().unwrap()") - } - } else { - out.puts(s"&self.${idToStr(attrName)}") - } + out.puts(s"self.${idToStr(attrName)}.borrow().as_ref().unwrap()") + } else { + out.puts(s"pub fn $fn(&self) -> Ref<$typeName> {") + out.inc + out.puts(s"self.${idToStr(attrName)}.borrow()") } out.dec out.puts("}") @@ -281,6 +288,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("Ok(())") out.dec out.puts("}") + RustCompiler.in_reader = false } override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = {} @@ -293,11 +301,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { // this line required for handleAssignmentRepeatUntil typeProvider._currentIteratorType = Some(dataType) - if (in_instance) { - out.puts(s"*${privateMemberName(id)}.borrow_mut() = Vec::new();") - } else { - out.puts(s"${privateMemberName(id)} = Vec::new();") - } + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = Vec::new();") } override def condRepeatEosHeader(id: Identifier, @@ -311,11 +315,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = { - if (in_instance) { - out.puts(s"${privateMemberName(id)}.borrow_mut().push($expr);") - } else { - out.puts(s"${privateMemberName(id)}.push($expr);") - } + out.puts(s"${RustCompiler.privateMemberName(id, true)}.push($expr);") } override def condRepeatEosFooter: Unit = { @@ -354,7 +354,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = { - out.puts(s"${privateMemberName(id)}.push($expr);") + out.puts(s"${RustCompiler.privateMemberName(id, true)}.push($expr);") var copy_type = "" if (typeProvider._currentIteratorType.isDefined && translator.is_copy_type(typeProvider._currentIteratorType.get)) { copy_type = "*" @@ -421,7 +421,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def useIO(ioEx: Ast.expr): String = { - out.puts(s"let io = BytesReader::new(&${expression(ioEx)});") + out.puts(s"let io = BytesReader::new(&*${expression(ioEx)});") "io" } @@ -540,8 +540,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"pub fn ${idToStr(instName)}(") out.inc out.puts("&self,") - out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S,") - out.puts(s"${privateMemberName(RootIdentifier)}: Option<&${classTypeName(typeProvider.topClass)}>") + out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S") out.dec val typeName = kaitaiTypeToNativeType( Some(instName), @@ -557,7 +556,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) dataType: DataType): Unit = { out.puts(s"if self.${calculatedFlagForName(instName)}.get() {") out.inc - out.puts(s"return Ok(${privateMemberName(instName)}.borrow());") + out.puts(s"return Ok(${privateMemberName(instName)});") out.dec out.puts("}") } @@ -567,26 +566,26 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { dataType match { case _: UserType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.clone();") + out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = ${translator.remove_deref(expression(value))}.clone();") case _: StrType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))}.to_string();") + out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = ${translator.remove_deref(expression(value))}.to_string();") case _: BytesType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") + out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") case _: ArrayType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") + out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") case _: EnumType => - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = ${translator.remove_deref(expression(value))};") + out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = ${translator.remove_deref(expression(value))};") case _ => translator.context_need_deref_attr = true val primType = kaitaiPrimitiveToNativeType(dataType) - out.puts(s"*${privateMemberName(instName)}.borrow_mut() = (${expression(value)}) as $primType;") + out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = (${expression(value)}) as $primType;") translator.context_need_deref_attr = false } } override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { - out.puts(s"Ok(${privateMemberName(instName)}.borrow())") + out.puts(s"Ok(${privateMemberName(instName)})") in_instance = false } @@ -695,18 +694,14 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) need_clone = !translator.is_copy_type(paramId.get.dataType) } val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, paramId.get.dataType) - if (typeName.startsWith("RefCell")) - out.puts(s"${privateMemberName(id)} = RefCell::new($expr.clone());") - else { - paramId.get.dataType match { - case et: EnumType => - out.puts(s"${privateMemberName(id)} = Some($expr.clone());") - case _ => - if (need_clone) - out.puts(s"${privateMemberName(id)} = $expr.clone();") - else - out.puts(s"${privateMemberName(id)} = $expr;") - } + paramId.get.dataType match { + case et: EnumType => + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = Some($expr.clone());") + case _ => + if (need_clone) + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = RefCell::new($expr.clone());") + else + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = RefCell::new($expr);") } } @@ -729,23 +724,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: EnumType => done = true out.puts( - s"${privateMemberName(id)} = Some($expr);" + s"*${RustCompiler.privateMemberName(id, true)} = Some($expr);" ) case _: SwitchType => done = true - out.puts(s"${privateMemberName(id)} = Some($expr);") + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = Some($expr);") case _ => } if (refcell) { val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, idType) - if (typeName.startsWith("RefCell")) { - if (opaque) { - out.puts(s"${privateMemberName(id)} = RefCell::new(Some(Box::new($expr)));") - } else { - out.puts(s"${privateMemberName(id)} = RefCell::new($expr);") - } - done = true + if (opaque) { + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr;") + } else { + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr;") } + done = true } } if (!done) { @@ -764,11 +757,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } if (inst) { done = true - out.puts(s"*${privateMemberName(id)}.borrow_mut() = $expr;") + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr;") } } if (!done) - out.puts(s"${privateMemberName(id)} = $expr;") + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr;") } override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = @@ -783,7 +776,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case t: ReadableType => t match { case IntMultiType(_, _, None) => - s"if *${privateMemberName(EndianIdentifier)}.borrow() == 1 { $io.read_${t.apiCall(Some(LittleEndian))}()?.into() } else { $io.read_${t.apiCall(Some(BigEndian))}()?.into() }" + s"if *${privateMemberName(EndianIdentifier)} == 1 { $io.read_${t.apiCall(Some(LittleEndian))}()?.into() } else { $io.read_${t.apiCall(Some(BigEndian))}()?.into() }" case IntMultiType(_, _, Some(e)) => s"$io.read_${t.apiCall(Some(e))}()?.into()" case _ => @@ -822,27 +815,25 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } s"$baseName" } + val root = s"Some(${self_name()}.${privateMemberName(RootIdentifier)}.clone())" val addArgs = if (t.isOpaque) { ", None, None" } else { val currentType = classTypeName(typeProvider.nowClass) - val root = if (typeProvider.nowClass.isTopLevel) { - "Some(self)" - } else { - privateMemberName(RootIdentifier) - } var parent = t.forcedParent match { case Some(USER_TYPE_NO_PARENT) => "None" case Some(fp) => translator.translate(fp) case None => if (!in_instance) { - if (typeProvider.nowClass.isTopLevel) { - s"Some(${privateMemberName(ParentIdentifier)}.unwrap_or(KStructUnit::parent_stack()).push(self))" - } else { - s"Some(${privateMemberName(ParentIdentifier)}.unwrap().push(self))" - } + s"Some(_new_parent.clone())" + // if (typeProvider.nowClass.isTopLevel) { + // s"Some(${privateMemberName(ParentIdentifier)}.unwrap_or(KStructUnit::parent_stack()).push(self))" + // } else { + // s"Some(${privateMemberName(ParentIdentifier)}.unwrap().push(self))" + // } } else { - s"None" + //s"Some(${self_name()}.${privateMemberName(ParentIdentifier)}.clone())" + "None" } } t.classSpec.get.parentType match { @@ -860,7 +851,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (t.classSpec.isDefined) t.classSpec.get.meta.endian match { case Some(InheritedEndian) => out.puts(s"let mut t = $userType::default();") - out.puts(s"t.set_endian(*${privateMemberName(EndianIdentifier)}.borrow());") + out.puts(s"t.set_endian(*${privateMemberName(EndianIdentifier)});") out.puts(s"t.read::<$streamType>($io$addArgs)?;") case _ => out.puts(s"let t = Self::read_into::<$streamType, $userType>($io$addArgs)?.into();") @@ -952,7 +943,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def switchIfStart(id: Identifier, on: Ast.expr, onType: DataType): Unit = { out.puts("{") out.inc - out.puts(s"let on = &${expression(on)};") + out.puts(s"let on = ${expression(on)};") } override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { @@ -989,20 +980,20 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case NoRepeat => var newStream = newStreamRaw val localIO = localTemporaryName(ioId) - if (in_instance) { + //if (in_instance) { val ids = idToStr(id) - out.puts(s"let $ids = $newStream.borrow();") + out.puts(s"let $ids = $newStream;") newStream = ids - } + //} out.puts(s"let $localIO = BytesReader::new(&$newStream);") s"&$localIO" case _ => val ids = idToStr(id) val localIO = s"io_$ids" - if (in_instance) { - out.puts(s"let $ids = $newStreamRaw.borrow();") + //if (in_instance) { + out.puts(s"let $ids = $newStreamRaw;") newStreamRaw = ids - } + //} out.puts(s"let $localIO = BytesReader::new(&$newStreamRaw.last().unwrap());") s"&$localIO" } @@ -1019,7 +1010,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) st, excludeOptionWrapper = true ) - out.puts("#[derive(Debug, PartialEq, Clone)]") + out.puts("#[derive(Debug, Clone)]") out.puts(s"pub enum $enum_typeName {") out.inc @@ -1154,9 +1145,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) Some(id), typeProvider.nowClass, t, - excludeOptionWrapper = true, - excludeLifetime = true, - excludeBox = true + cleanTypename = true ) case t: EnumType => kaitaiTypeToNativeType( @@ -1234,11 +1223,23 @@ object RustCompiler override def kstreamName = "KStream" - def privateMemberName(id: Identifier): String = id match { + var in_reader = false + + def self_name(): String = { + if (in_reader) "self_rc" else "self" + } + + def privateMemberName(id: Identifier, writeAccess: Boolean = false): String = id match { case IoIdentifier => "_io" case RootIdentifier => "_root" case ParentIdentifier => "_parent" - case _ => s"self.${idToStr(id)}" + case _ => { + val n = s"${self_name()}.${idToStr(id)}" + if (writeAccess) + s"$n.borrow_mut()" + else + s"$n.borrow()" + } } def idToStr(id: Identifier): String = id match { @@ -1310,7 +1311,7 @@ object RustCompiler cs: ClassSpec, attrType: DataType, excludeOptionWrapper: Boolean = false, - excludeLifetime: Boolean = false, + cleanTypename: Boolean = false, excludeBox: Boolean = false): String = attrType match { // TODO: Not exhaustive @@ -1324,30 +1325,32 @@ object RustCompiler case Some(spec) => types2class(spec.name) case None => types2class(t.name) } - //val lifetime = if (!excludeLifetime) s"<$streamLife>" else "" + + if (cleanTypename) { + return baseName + } // Because we can't predict if opaque types will recurse, we have to box them val typeName = - if (!excludeBox && t.isOpaque) s"Option>" + if (!excludeBox && t.isOpaque) s"$baseName" else s"$baseName" - if (excludeOptionWrapper) typeName else s"RefCell<$typeName>" + //if (excludeOptionWrapper) typeName else s"Rc<$typeName>" + s"Rc<$typeName>" case t: EnumType => - val typeName = t.enumSpec match { + val baseName = t.enumSpec match { case Some(spec) => s"${types2class(spec.name)}" case None => s"${types2class(t.name)}" } - if (excludeOptionWrapper) typeName else s"Option<$typeName>" + if (cleanTypename) { + return baseName + } + if (excludeOptionWrapper) baseName else s"Option<$baseName>" case t: ArrayType => - s"Vec<${kaitaiTypeToNativeType(id, cs, t.elType, excludeOptionWrapper = true, excludeLifetime = excludeLifetime)}>" + s"Vec<${kaitaiTypeToNativeType(id, cs, t.elType, excludeOptionWrapper = true)}>" case _: SwitchType => - // val types = st.cases.values.toSet - // val lifetime = - // if (!excludeLifetime && types.exists(containsReferences)) - // s"<$streamLife>" - // else "" val typeName = id.get match { case name: NamedIdentifier => s"${types2class(cs.name ::: List(name.name))}" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 6cfd5717b..fd81fdda4 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -15,9 +15,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) import RustCompiler._ override def doByteArrayLiteral(arr: Seq[Byte]): String = - "&vec![" + arr.map(x => "%0#2xu8".format(x & 0xff)).mkString(", ") + "]" + "vec![" + arr.map(x => "%0#2xu8".format(x & 0xff)).mkString(", ") + "].as_slice()" override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = - "&vec![" + elts.map(translate).mkString(", ") + "]" + "vec![" + elts.map(translate).mkString(", ") + "].as_slice()" override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = { t match { case CalcStrType => "vec![" + value.map(v => translate(v)).mkString(".to_string(), ") + ".to_string()]" @@ -70,7 +70,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) val topClass = get_top_class(provider.nowClass) val instance_found = get_instance(topClass, s) if (instance_found.isDefined) { - s"$s(${privateMemberName(IoIdentifier)}, ${privateMemberName(RootIdentifier)})?" + s"$s(${privateMemberName(IoIdentifier)})?" } else { s"$s()" } @@ -124,7 +124,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) r = r.replace("()._io()", "_raw()") case Identifier.PARENT => // handle _parent._parent - r = r.replace(".peek()._parent", ".pop().peek()") + r = r.replace(".get().as_ref()._parent", ".get().as_ref()._parent.get().as_ref()") case _ => } r @@ -147,7 +147,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } def ensure_deref(s: String): String = { - if (s.startsWith("self")) { + if (s.startsWith(self_name())) { s"*$s" } else { s @@ -210,7 +210,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) var deref = false var found = get_attr(get_top_class(provider.nowClass), s) if (found.isDefined ) { - deref = is_copy_type(found.get.dataTypeComposite) + deref = true//is_copy_type(found.get.dataTypeComposite) } else { found = get_instance(get_top_class(provider.nowClass), s) if (found.isDefined) { @@ -232,15 +232,15 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case Identifier.ITERATOR2 => "_tmpb" case Identifier.INDEX => "_i" case Identifier.IO => s"${RustCompiler.privateMemberName(IoIdentifier)}" - case Identifier.ROOT => s"${RustCompiler.privateMemberName(RootIdentifier)}.ok_or(KError::MissingRoot)?" - case Identifier.PARENT => s"${RustCompiler.privateMemberName(ParentIdentifier)}.as_ref().unwrap().peek()" + case Identifier.ROOT => s"${self_name()}.${RustCompiler.privateMemberName(RootIdentifier)}.get().as_ref()" + case Identifier.PARENT => s"${RustCompiler.privateMemberName(ParentIdentifier)}.get().as_ref()" case _ => val n = doName(s) val deref = need_deref(s) if (context_need_deref_attr || deref) { - s"*self.$n" + s"*${self_name()}.$n" } else { - s"self.$n" + s"${self_name()}.$n" } } override def doEnumCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { From c83e1314d6521eb51e8e0ddc68262be241da6f80 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 27 Dec 2022 21:24:06 +0800 Subject: [PATCH 098/153] Rust code generation updated. --- .../struct/languages/RustCompiler.scala | 57 ++++++++++--------- .../struct/translators/RustTranslator.scala | 51 +++++++++++++---- 2 files changed, 69 insertions(+), 39 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index d33b2dba0..140054d1f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -238,20 +238,21 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) var types : Set[DataType] = Set() var enum_typename = false + var switch_typename = false attrType match { case st: SwitchType => types = st.cases.values.toSet - enum_typename = true + switch_typename = true case _: EnumType => enum_typename = true case _ => } - var enum_only_numeric = false//true + var enum_only_numeric = true types.foreach { case _: NumericType => // leave unchanged case _ => enum_only_numeric = false } var fn = idToStr(attrName) - if (enum_typename && enum_only_numeric) { + if (switch_typename && enum_only_numeric) { out.puts(s"pub fn $fn(&self) -> usize {") out.inc out.puts(s"self.${idToStr(attrName)}.borrow().as_ref().unwrap().into()") @@ -259,10 +260,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") fn = s"${fn}_enum" } - if (enum_typename) { - out.puts(s"pub fn $fn(&self) -> &$typeNameEx {") + if (switch_typename || enum_typename) { + out.puts(s"pub fn $fn(&self) -> $typeNameEx {") out.inc - out.puts(s"self.${idToStr(attrName)}.borrow().as_ref().unwrap()") + out.puts(s"self.${idToStr(attrName)}.borrow().as_ref().unwrap().clone()") } else { out.puts(s"pub fn $fn(&self) -> Ref<$typeName> {") out.inc @@ -576,10 +577,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: EnumType => out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = ${translator.remove_deref(expression(value))};") case _ => - translator.context_need_deref_attr = true + //translator.context_need_deref_attr = true val primType = kaitaiPrimitiveToNativeType(dataType) out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = (${expression(value)}) as $primType;") - translator.context_need_deref_attr = false + //translator.context_need_deref_attr = false } } @@ -641,9 +642,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") out.puts - out.puts(s"impl From<&$enumClass> for i64 {") + out.puts(s"impl From<$enumClass> for i64 {") out.inc - out.puts(s"fn from(val: &$enumClass) -> Self {") + out.puts(s"fn from(val: $enumClass) -> Self {") out.inc out.puts(s"match val {") out.inc @@ -651,7 +652,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case (value, label) => out.puts(s"$enumClass::${type2class(label.name)} => $value,") } - out.puts(s"$enumClass::Unknown(v) => *v") + out.puts(s"$enumClass::Unknown(v) => v") out.dec out.puts("}") out.dec @@ -699,9 +700,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"*${RustCompiler.privateMemberName(id, true)} = Some($expr.clone());") case _ => if (need_clone) - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = RefCell::new($expr.clone());") + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr.clone();") else - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = RefCell::new($expr);") + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr;") } } @@ -734,7 +735,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (refcell) { val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, idType) if (opaque) { - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr;") + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = Some($expr);") } else { out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr;") } @@ -850,17 +851,17 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (addParams.isEmpty) { if (t.classSpec.isDefined) t.classSpec.get.meta.endian match { case Some(InheritedEndian) => - out.puts(s"let mut t = $userType::default();") - out.puts(s"t.set_endian(*${privateMemberName(EndianIdentifier)});") - out.puts(s"t.read::<$streamType>($io$addArgs)?;") + // out.puts(s"let mut t = $userType::default();") + // out.puts(s"t.set_endian(*${privateMemberName(EndianIdentifier)});") + // out.puts(s"t.read::<$streamType>($io$addArgs)?;") + out.puts(s"let f = |t : &mut $userType| Ok(t.set_endian(*${privateMemberName(EndianIdentifier)}));") + out.puts(s"let t = Self::read_into_with_init::<$streamType, $userType>($io$addArgs, &f)?.into();") case _ => out.puts(s"let t = Self::read_into::<$streamType, $userType>($io$addArgs)?.into();") } } else { - //val at = kaitaiTypeToNativeType(None, typeProvider.nowClass, assignType, excludeOptionWrapper = true) - out.puts(s"let mut t = $userType::default();") - out.puts(s"t.set_params($addParams);") - out.puts(s"t.read::<$streamType>($io$addArgs)?;") + out.puts(s"let f = |t : &mut $userType| Ok(t.set_params($addParams));") + out.puts(s"let t = Self::read_into_with_init::<$streamType, $userType>($io$addArgs, &f)?.into();") } return s"t" case _ => s"// parseExpr($dataType, $assignType, $io, $defEndian)" @@ -947,12 +948,12 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { - out.puts(s"if *on == ${expression(condition)} {") + out.puts(s"if on == ${expression(condition)} {") out.inc } override def switchIfCaseStart(condition: Ast.expr): Unit = { - out.puts(s"else if *on == ${expression(condition)} {") + out.puts(s"else if on == ${expression(condition)} {") out.inc } @@ -1330,12 +1331,12 @@ object RustCompiler return baseName } - // Because we can't predict if opaque types will recurse, we have to box them - val typeName = - if (!excludeBox && t.isOpaque) s"$baseName" - else s"$baseName" //if (excludeOptionWrapper) typeName else s"Rc<$typeName>" - s"Rc<$typeName>" + if (t.isOpaque) s"Option>" else s"Rc<$baseName>" + // Because we can't predict if opaque types will recurse, we have to box them + // val typeName = + // if (!excludeBox && t.isOpaque) s"$baseName" + // else s"$baseName" case t: EnumType => val baseName = t.enumSpec match { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index fd81fdda4..94f64fc4c 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -124,7 +124,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) r = r.replace("()._io()", "_raw()") case Identifier.PARENT => // handle _parent._parent - r = r.replace(".get().as_ref()._parent", ".get().as_ref()._parent.get().as_ref()") + var builder = new StringBuilder() + val expandStr = ".get().as_ref()._parent" + val start = r.lastIndexOf(expandStr) + builder.append(r.substring(0, start)) + builder.append(".get().as_ref()._parent.get().as_ref()") + builder.append(r.substring(start + expandStr.length())) + r = builder.toString() case _ => } r @@ -196,13 +202,32 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) var context_need_deref_attr = false + def enum_numeric_only(dataType: DataType): Boolean = { + var types : Set[DataType] = Set() + var enum_typename = false + dataType match { + case st: SwitchType => + types = st.cases.values.toSet + enum_typename = true + case _: EnumType => return true + case _ => return false + } + var enum_only_numeric = true + types.foreach { + case _: NumericType => // leave unchanged + case _ => enum_only_numeric = false + } + //println(s"$dataType - $enum_only_numeric") + enum_only_numeric + } + def is_copy_type(dataType: DataType): Boolean = dataType match { - case _: SwitchType => false - case _: UserType => false - case _: BytesType => false - case _: ArrayType => false - case _: StrType => false - case _: EnumType => false + // case _: SwitchType => false + // case _: UserType => false + // case _: BytesType => false + // case _: ArrayType => false + // case _: StrType => false + // case _: EnumType => false case _ => true } @@ -210,20 +235,23 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) var deref = false var found = get_attr(get_top_class(provider.nowClass), s) if (found.isDefined ) { - deref = true//is_copy_type(found.get.dataTypeComposite) + deref = !enum_numeric_only(found.get.dataTypeComposite) //is_copy_type(found.get.dataTypeComposite) + //println(s"need_deref get_attr $s $deref ${found.get.dataTypeComposite}") } else { found = get_instance(get_top_class(provider.nowClass), s) if (found.isDefined) { + //println(s"need_deref get_instance $s $deref ${found.get.dataTypeComposite}") deref = true //is_copy_type(found.get.dataTypeComposite) } else { found = get_param(get_top_class(provider.nowClass), s) if (found.isDefined) { - deref = true + deref = !enum_numeric_only(found.get.dataTypeComposite) } else { deref = false } } } + //println(s"need_deref $s $deref") deref } @@ -238,15 +266,16 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) val n = doName(s) val deref = need_deref(s) if (context_need_deref_attr || deref) { + //println(s"$context_need_deref_attr, $deref") s"*${self_name()}.$n" } else { s"${self_name()}.$n" } } override def doEnumCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { - context_need_deref_attr = true + //context_need_deref_attr = true val code = s"${translate(left)} ${cmpOp(op)} ${translate(right)}" - context_need_deref_attr = false + //context_need_deref_attr = false code } From 8dd7d1d77976493ec9a3030f19d7486647882ad4 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Tue, 27 Dec 2022 21:38:17 +0100 Subject: [PATCH 099/153] generate delegates for enums' variants --- .../struct/languages/RustCompiler.scala | 50 +++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index d33b2dba0..6375ff7d7 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -52,9 +52,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) outHeader.puts outHeader.puts("extern crate kaitai;") - importList.add( - "use kaitai::*;" - ) + importList.add("use kaitai::*;") importList.add("use std::convert::{TryFrom, TryInto};") importList.add("use std::cell::{Ref, Cell, RefCell};") importList.add("use std::rc::{Rc, Weak};") @@ -113,7 +111,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) params: List[ParamDefSpec]): Unit = { typeProvider.nowClass.meta.endian match { case Some(_: CalcEndian) | Some(InheritedEndian) => - attributeDeclaration(EndianIdentifier, IntMultiType(true, Width4, None), false) + attributeDeclaration(EndianIdentifier, IntMultiType(signed = true, Width4, None), isNullable = false) case _ => } @@ -1115,6 +1113,42 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") out.puts } + + { + val types_set = scala.collection.mutable.Set[String]() + val attrs_set = scala.collection.mutable.Set[String]() + types.foreach(t => { + val variantName = switchVariantName(id, t) + val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, t, excludeBox = true) + if (types_set.add(typeName)) { + t.asInstanceOf[UserType].classSpec.get.seq.foreach( + attr => { + val attrName = attr.id + if (attrs_set.add(idToStr(attrName))) { + out.puts(s"impl $enum_typeName {") + out.inc + val fn = idToStr(attrName) + val nativeType = kaitaiTypeToNativeTypeWrapper(Some(attrName), attr.dataTypeComposite) + //out.puts(nativeType.attrGetterDcl(fn)) + out.puts(s"pub fn $fn(&self) -> Ref<$nativeType> {") + out.inc + out.puts("match self {") + out.inc + out.puts(s"$enum_typeName::$typeName(x) => x.$fn.borrow(),") + out.puts("_ => panic!(\"wrong variant: {:?}\", self),") + out.dec + out.puts("}") + out.dec + out.puts("}") + out.dec + out.puts("}") + } + } + ) + } + }) + } + } def switchVariantName(id: Identifier, attrType: DataType): String = @@ -1330,12 +1364,8 @@ object RustCompiler return baseName } - // Because we can't predict if opaque types will recurse, we have to box them - val typeName = - if (!excludeBox && t.isOpaque) s"$baseName" - else s"$baseName" - //if (excludeOptionWrapper) typeName else s"Rc<$typeName>" - s"Rc<$typeName>" + if (excludeBox) s"$baseName" + else s"Rc<$baseName>" case t: EnumType => val baseName = t.enumSpec match { From 711022d516c1008b8960e27bb142f6c1c3075778 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Wed, 28 Dec 2022 20:17:40 +0100 Subject: [PATCH 100/153] `attributeReader` always returns `Ref<$typeName>` (`test_nav_parent_switch`) --- .../io/kaitai/struct/languages/RustCompiler.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 6375ff7d7..bbbb28d6e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -257,15 +257,15 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") fn = s"${fn}_enum" } - if (enum_typename) { - out.puts(s"pub fn $fn(&self) -> &$typeNameEx {") - out.inc - out.puts(s"self.${idToStr(attrName)}.borrow().as_ref().unwrap()") - } else { +// if (enum_typename) { +// out.puts(s"pub fn $fn(&self) -> &$typeNameEx {") +// out.inc +// out.puts(s"self.${idToStr(attrName)}.borrow().as_ref().unwrap()") +// } else { out.puts(s"pub fn $fn(&self) -> Ref<$typeName> {") out.inc out.puts(s"self.${idToStr(attrName)}.borrow()") - } +// } out.dec out.puts("}") out.dec From 040929fe85065f62a765031330ecd0a2b8028951 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Thu, 29 Dec 2022 19:46:00 +0100 Subject: [PATCH 101/153] use `cleanTypename` in generating delegates --- .../struct/languages/RustCompiler.scala | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 04cfeb146..ae5fb4726 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -266,7 +266,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"pub fn $fn(&self) -> Ref<$typeName> {") out.inc out.puts(s"self.${idToStr(attrName)}.borrow()") -// } + } out.dec out.puts("}") out.dec @@ -1120,7 +1120,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val attrs_set = scala.collection.mutable.Set[String]() types.foreach(t => { val variantName = switchVariantName(id, t) - val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, t, excludeBox = true) + val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, t, cleanTypename = true) if (types_set.add(typeName)) { t.asInstanceOf[UserType].classSpec.get.seq.foreach( attr => { @@ -1346,8 +1346,7 @@ object RustCompiler cs: ClassSpec, attrType: DataType, excludeOptionWrapper: Boolean = false, - cleanTypename: Boolean = false, - excludeBox: Boolean = false): String = + cleanTypename: Boolean = false): String = attrType match { // TODO: Not exhaustive case _: NumericType => kaitaiPrimitiveToNativeType(attrType) @@ -1361,17 +1360,12 @@ object RustCompiler case None => types2class(t.name) } - if (cleanTypename) { - return baseName + (t.isOpaque, cleanTypename) match { + case (_, true) => baseName + case (true, _) => s"Option>" + case (false, _) => s"Rc<$baseName>" } - //if (excludeOptionWrapper) typeName else s"Rc<$typeName>" - if (t.isOpaque) s"Option>" else s"Rc<$baseName>" - // Because we can't predict if opaque types will recurse, we have to box them - // val typeName = - // if (!excludeBox && t.isOpaque) s"$baseName" - // else s"$baseName" - case t: EnumType => val baseName = t.enumSpec match { case Some(spec) => s"${types2class(spec.name)}" From 56205f3f22f4ef0f2b7863c70bdbefe3f9d458f8 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Fri, 30 Dec 2022 21:34:31 +0100 Subject: [PATCH 102/153] generate delegates only for `UserType` --- .../struct/languages/RustCompiler.scala | 53 ++++++++++--------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index ae5fb4726..69e723b1f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -1119,33 +1119,36 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val types_set = scala.collection.mutable.Set[String]() val attrs_set = scala.collection.mutable.Set[String]() types.foreach(t => { - val variantName = switchVariantName(id, t) val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, t, cleanTypename = true) if (types_set.add(typeName)) { - t.asInstanceOf[UserType].classSpec.get.seq.foreach( - attr => { - val attrName = attr.id - if (attrs_set.add(idToStr(attrName))) { - out.puts(s"impl $enum_typeName {") - out.inc - val fn = idToStr(attrName) - val nativeType = kaitaiTypeToNativeTypeWrapper(Some(attrName), attr.dataTypeComposite) - //out.puts(nativeType.attrGetterDcl(fn)) - out.puts(s"pub fn $fn(&self) -> Ref<$nativeType> {") - out.inc - out.puts("match self {") - out.inc - out.puts(s"$enum_typeName::$typeName(x) => x.$fn.borrow(),") - out.puts("_ => panic!(\"wrong variant: {:?}\", self),") - out.dec - out.puts("}") - out.dec - out.puts("}") - out.dec - out.puts("}") - } - } - ) + t match { + case ut: UserType => + ut.classSpec.get.seq.foreach( + attr => { + val attrName = attr.id + if (attrs_set.add(idToStr(attrName))) { + out.puts(s"impl $enum_typeName {") + out.inc + val fn = idToStr(attrName) + val nativeType = kaitaiTypeToNativeTypeWrapper(Some(attrName), attr.dataTypeComposite) + //out.puts(nativeType.attrGetterDcl(fn)) + out.puts(s"pub fn $fn(&self) -> Ref<$nativeType> {") + out.inc + out.puts("match self {") + out.inc + out.puts(s"$enum_typeName::$typeName(x) => x.$fn.borrow(),") + out.puts("_ => panic!(\"wrong variant: {:?}\", self),") + out.dec + out.puts("}") + out.dec + out.puts("}") + out.dec + out.puts("}") + } + } + ) + case _ => + } } }) } From 61c2dcc61705d169f7192b404eb3b27f74c16b7f Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 31 Dec 2022 16:57:06 +0800 Subject: [PATCH 103/153] latest version of Rust code generation. --- .../struct/languages/RustCompiler.scala | 72 ++++---- .../struct/translators/RustTranslator.scala | 159 ++++++++++++++---- 2 files changed, 164 insertions(+), 67 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 140054d1f..484904bdb 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -87,7 +87,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val parent = if (typeProvider.nowClass.isTopLevel) root else { - kaitaiTypeToNativeType(None, typeProvider.nowClass, typeProvider.nowClass.parentType, cleanTypename = true) + kaitaiTypeToNativeType(None, typeProvider.nowClass, typeProvider.nowClass.parentType, cleanType = true) } out.puts(s"pub ${privateMemberName(ParentIdentifier)}: SharedType<$parent>,") @@ -146,7 +146,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val parent = if (typeProvider.nowClass.isTopLevel) root else { - kaitaiTypeToNativeType(None, typeProvider.nowClass, typeProvider.nowClass.parentType, cleanTypename = true) + kaitaiTypeToNativeType(None, typeProvider.nowClass, typeProvider.nowClass.parentType, cleanType = true) } out.puts( @@ -186,6 +186,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"self_rc._root.set(_root.get());") out.puts(s"self_rc._parent.set(_parent.get());") out.puts(s"let _new_parent = SharedType::::new(self_rc.clone());") + out.puts(s"let _rrc = self_rc._root.get();") + out.puts(s"let _prc = self_rc._parent.get();") + out.puts(s"let _r = _rrc.as_ref();") + out.puts(s"let _p = _prc.as_ref();") // If there aren't any attributes to parse, we need to end the read implementation here if (typeProvider.nowClass.seq.isEmpty) @@ -260,11 +264,12 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") fn = s"${fn}_enum" } - if (switch_typename || enum_typename) { - out.puts(s"pub fn $fn(&self) -> $typeNameEx {") - out.inc - out.puts(s"self.${idToStr(attrName)}.borrow().as_ref().unwrap().clone()") - } else { + // if (switch_typename || enum_typename) { + // out.puts(s"pub fn $fn(&self) -> RefEnum<$typeNameEx> {") + // out.inc + // out.puts(s"RefEnum::new(self.${idToStr(attrName)}.borrow())") + // } else + { out.puts(s"pub fn $fn(&self) -> Ref<$typeName> {") out.inc out.puts(s"self.${idToStr(attrName)}.borrow()") @@ -360,7 +365,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (typeProvider._currentIteratorType.isDefined && translator.is_copy_type(typeProvider._currentIteratorType.get)) { copy_type = "*" } - out.puts(s"let ${translator.doLocalName(Identifier.ITERATOR)} = $copy_type${privateMemberName(id)}.last().unwrap();") + val t = localTemporaryName(id) + out.puts(s"let $t = ${privateMemberName(id)};") + out.puts(s"let ${translator.doLocalName(Identifier.ITERATOR)} = $copy_type$t.last().unwrap();") } override def condRepeatUntilFooter(id: Identifier, @@ -370,7 +377,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // this line required by kaitai code typeProvider._currentIteratorType = Some(dataType) out.puts("_i += 1;") - out.puts(s"!(${expression(repeatExpr)})") + out.puts(s"let x = !(${expression(repeatExpr)});") + out.puts("x") out.dec out.puts("} {}") out.dec @@ -422,7 +430,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def useIO(ioEx: Ast.expr): String = { - out.puts(s"let io = BytesReader::new(&*${expression(ioEx)});") + out.puts(s"let t = ${expression(ioEx)};") + out.puts(s"let io = BytesReader::new(&*t);") "io" } @@ -551,6 +560,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) out.puts(s") -> KResult> {") out.inc + out.puts(s"let _rrc = self._root.get();") + out.puts(s"let _prc = self._parent.get();") + out.puts(s"let _r = _rrc.as_ref();") + out.puts(s"let _p = _prc.as_ref();") } override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, @@ -642,11 +655,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") out.puts - out.puts(s"impl From<$enumClass> for i64 {") + out.puts(s"impl From<&$enumClass> for i64 {") out.inc - out.puts(s"fn from(val: $enumClass) -> Self {") + out.puts(s"fn from(v: &$enumClass) -> Self {") out.inc - out.puts(s"match val {") + out.puts(s"match *v {") out.inc enumColl.foreach { case (value, label) => @@ -697,7 +710,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, paramId.get.dataType) paramId.get.dataType match { case et: EnumType => - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = Some($expr.clone());") + out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr.clone();") case _ => if (need_clone) out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr.clone();") @@ -725,7 +738,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: EnumType => done = true out.puts( - s"*${RustCompiler.privateMemberName(id, true)} = Some($expr);" + s"*${RustCompiler.privateMemberName(id, true)} = $expr;" ) case _: SwitchType => done = true @@ -798,15 +811,14 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // exclude enums typ match { case _: NumericType => try_into = s".try_into().map_err(|_| KError::CastError)?" - case _: EnumType => case _ => if (!translator.is_copy_type(typ)) byref = "&" } - var need_deref = "" - if (t.startsWith("RefCell")) - need_deref = "&*" - s"$byref$need_deref${translator.translate(a)}$try_into" + //var need_deref = "*" + //if (t.startsWith("RefCell")) + // need_deref = "&*" + s"$byref${translator.translate(a)}$try_into" }, "", ", ", "") val userType = t match { case t: UserType => @@ -851,9 +863,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (addParams.isEmpty) { if (t.classSpec.isDefined) t.classSpec.get.meta.endian match { case Some(InheritedEndian) => - // out.puts(s"let mut t = $userType::default();") - // out.puts(s"t.set_endian(*${privateMemberName(EndianIdentifier)});") - // out.puts(s"t.read::<$streamType>($io$addArgs)?;") out.puts(s"let f = |t : &mut $userType| Ok(t.set_endian(*${privateMemberName(EndianIdentifier)}));") out.puts(s"let t = Self::read_into_with_init::<$streamType, $userType>($io$addArgs, &f)?.into();") case _ => @@ -944,16 +953,16 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def switchIfStart(id: Identifier, on: Ast.expr, onType: DataType): Unit = { out.puts("{") out.inc - out.puts(s"let on = ${expression(on)};") + out.puts(s"let on = ${translator.remove_deref(expression(on))};") } override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { - out.puts(s"if on == ${expression(condition)} {") + out.puts(s"if *on == ${expression(condition)} {") out.inc } override def switchIfCaseStart(condition: Ast.expr): Unit = { - out.puts(s"else if on == ${expression(condition)} {") + out.puts(s"else if *on == ${expression(condition)} {") out.inc } @@ -1146,7 +1155,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) Some(id), typeProvider.nowClass, t, - cleanTypename = true + cleanType = true ) case t: EnumType => kaitaiTypeToNativeType( @@ -1312,7 +1321,7 @@ object RustCompiler cs: ClassSpec, attrType: DataType, excludeOptionWrapper: Boolean = false, - cleanTypename: Boolean = false, + cleanType: Boolean = false, excludeBox: Boolean = false): String = attrType match { // TODO: Not exhaustive @@ -1327,7 +1336,7 @@ object RustCompiler case None => types2class(t.name) } - if (cleanTypename) { + if (cleanType) { return baseName } @@ -1343,10 +1352,10 @@ object RustCompiler case Some(spec) => s"${types2class(spec.name)}" case None => s"${types2class(t.name)}" } - if (cleanTypename) { + if (cleanType) { return baseName } - if (excludeOptionWrapper) baseName else s"Option<$baseName>" + if (excludeOptionWrapper) baseName else s"$baseName" case t: ArrayType => s"Vec<${kaitaiTypeToNativeType(id, cs, t.elType, excludeOptionWrapper = true)}>" @@ -1390,5 +1399,6 @@ object RustCompiler case _: BytesType => "Vec" case ArrayTypeInStream(inType) => s"Vec<${kaitaiPrimitiveToNativeType(inType)}>" + case _ => "kaitaiPrimitiveToNativeType ???" } } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 94f64fc4c..d6b68b98e 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -7,7 +7,7 @@ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format.{Identifier, IoIdentifier, ParentIdentifier, RootIdentifier} import io.kaitai.struct.languages.RustCompiler -import io.kaitai.struct.{RuntimeConfig, Utils} +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseTranslator(provider) { @@ -15,9 +15,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) import RustCompiler._ override def doByteArrayLiteral(arr: Seq[Byte]): String = - "vec![" + arr.map(x => "%0#2xu8".format(x & 0xff)).mkString(", ") + "].as_slice()" + "vec![" + arr.map(x => "%0#2xu8".format(x & 0xff)).mkString(", ") + "]" override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = - "vec![" + elts.map(translate).mkString(", ") + "].as_slice()" + "vec![" + elts.map(translate).mkString(", ") + "]" override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = { t match { case CalcStrType => "vec![" + value.map(v => translate(v)).mkString(".to_string(), ") + ".to_string()]" @@ -67,15 +67,93 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doName(s: String): String = s match { case Identifier.PARENT => s case _ => - val topClass = get_top_class(provider.nowClass) - val instance_found = get_instance(topClass, s) - if (instance_found.isDefined) { - s"$s(${privateMemberName(IoIdentifier)})?" - } else { + val memberFound = findMember(s) + if (memberFound.isDefined) + memberFound.get match { + case vis: ValueInstanceSpec => + val call = s"$s(${privateMemberName(IoIdentifier)})?" + vis.dataTypeOpt match { + case Some(dt) => + dt match { + //case _: StrType => s"$call.as_str()" + //case _: BytesType => s"$call.as_slice()" + case _ => call + } + case None => call + } + case as: AttrSpec => + val code = s"$s()" + val aType = RustCompiler.kaitaiPrimitiveToNativeType(as.dataTypeComposite) + aType match { + //case "String" => s"$code.as_str()" + //case "Vec" => s"$code.as_slice()" + //case "SwitchType" => s"$code.as_ref().unwrap()" + case _ => code + } + case pis: ParseInstanceSpec => + pis.dataType match { + case _: NumericType | _: StrType => + s"$s(${privateMemberName(IoIdentifier)})?" + case t: UserTypeInstream => + if (t.isOpaque) + s"$s(${privateMemberName(IoIdentifier)})?.as_ref().unwrap()" + else + s"$s(${privateMemberName(IoIdentifier)})?" + case _ => + s"$s(${privateMemberName(IoIdentifier)})?.as_ref().unwrap()" + } + case _ => + s"$s()" + } + else { s"$s()" } } + def findMember(attrName: String): Option[MemberSpec] = { + def findInClass(inClass: ClassSpec): Option[MemberSpec] = { + + inClass.seq.foreach { el => + if (idToStr(el.id) == attrName) { + return Some(el) + } + } + + inClass.params.foreach { el => + if (idToStr(el.id) == attrName) { + return Some(el) + } + } + + inClass.instances.foreach { case (instName, instSpec) => + if (idToStr(instName) == attrName) { + return Some(instSpec) + } + } + + inClass.types.foreach{ t => + for { found <- findInClass(t._2) } + return Some(found) + } + None + } + + val attr = attrName match { + case Identifier.PARENT | Identifier.IO => + None + case _ => + for { ms <- findInClass(provider.nowClass) } + return Some(ms) + + provider.asInstanceOf[ClassTypeProvider].allClasses.foreach { cls => + for { ms <- findInClass(cls._2) } + return Some(ms) + } + None + } + attr + } + def get_top_class(c: ClassSpec): ClassSpec = c.upClass match { case Some(upClass) => get_top_class(upClass) case None => c @@ -104,7 +182,11 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def anyField(value: expr, attrName: String): String = { val t = translate(value) - val a = doName(attrName) + var a = doName(attrName) + attrName match { + case Identifier.PARENT => a = a + ".get().as_ref()" + case _ => + } var r = "" if (need_deref(attrName)) { if (t.charAt(0) == '*') { @@ -122,15 +204,6 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) attrName match { case Identifier.IO => r = r.replace("()._io()", "_raw()") - case Identifier.PARENT => - // handle _parent._parent - var builder = new StringBuilder() - val expandStr = ".get().as_ref()._parent" - val start = r.lastIndexOf(expandStr) - builder.append(r.substring(0, start)) - builder.append(".get().as_ref()._parent.get().as_ref()") - builder.append(r.substring(start + expandStr.length())) - r = builder.toString() case _ => } r @@ -144,6 +217,14 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } + def ensure_vec_amp(s: String): String = { + if (s.startsWith("vec!")) { + s"&$s" + } else { + s + } + } + def remove_deref(s: String): String = { if (s.charAt(0) == '*') { s.substring(1) @@ -209,7 +290,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case st: SwitchType => types = st.cases.values.toSet enum_typename = true - case _: EnumType => return true + //case _: EnumType => return true case _ => return false } var enum_only_numeric = true @@ -222,12 +303,12 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } def is_copy_type(dataType: DataType): Boolean = dataType match { - // case _: SwitchType => false - // case _: UserType => false - // case _: BytesType => false - // case _: ArrayType => false - // case _: StrType => false - // case _: EnumType => false + case _: SwitchType => false + case _: UserType => false + case _: BytesType => false + case _: ArrayType => false + case _: StrType => false + case _: EnumType => false case _ => true } @@ -235,13 +316,15 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) var deref = false var found = get_attr(get_top_class(provider.nowClass), s) if (found.isDefined ) { - deref = !enum_numeric_only(found.get.dataTypeComposite) //is_copy_type(found.get.dataTypeComposite) - //println(s"need_deref get_attr $s $deref ${found.get.dataTypeComposite}") + val cleanType = RustCompiler.kaitaiTypeToNativeType(Some(NamedIdentifier(s)), provider.nowClass, found.get.dataType, cleanType = true) + //if (cleanType != "Vec") + deref = !enum_numeric_only(found.get.dataTypeComposite) //is_copy_type(found.get.dataTypeComposite) } else { found = get_instance(get_top_class(provider.nowClass), s) if (found.isDefined) { - //println(s"need_deref get_instance $s $deref ${found.get.dataTypeComposite}") - deref = true //is_copy_type(found.get.dataTypeComposite) + val cleanType = RustCompiler.kaitaiTypeToNativeType(Some(NamedIdentifier(s)), provider.nowClass, found.get.dataType, cleanType = true) + //if (cleanType != "Vec") + deref = true //is_copy_type(found.get.dataTypeComposite) } else { found = get_param(get_top_class(provider.nowClass), s) if (found.isDefined) { @@ -251,7 +334,6 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } } - //println(s"need_deref $s $deref") deref } @@ -260,13 +342,12 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case Identifier.ITERATOR2 => "_tmpb" case Identifier.INDEX => "_i" case Identifier.IO => s"${RustCompiler.privateMemberName(IoIdentifier)}" - case Identifier.ROOT => s"${self_name()}.${RustCompiler.privateMemberName(RootIdentifier)}.get().as_ref()" - case Identifier.PARENT => s"${RustCompiler.privateMemberName(ParentIdentifier)}.get().as_ref()" + case Identifier.ROOT => s"_r" + case Identifier.PARENT => s"_p" case _ => val n = doName(s) - val deref = need_deref(s) + val deref = !n.endsWith(".as_str()") && !n.endsWith(".as_slice()") && need_deref(s) if (context_need_deref_attr || deref) { - //println(s"$context_need_deref_attr, $deref") s"*${self_name()}.$n" } else { s"${self_name()}.$n" @@ -286,6 +367,10 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { + // val l = translate(left) + // val r = translate(right) + // val asStr = if (l.endsWith(".as_str()") && r.endsWith(")?")) ".as_str()" else "" + // s"$l ${cmpOp(op)} $r$asStr" s"${ensure_deref(translate(left))} ${cmpOp(op)} ${remove_deref(translate(right))}.to_string()" } @@ -302,8 +387,10 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _: EnumType => to_type = ".clone()" case _: StrType => to_type = ".to_string()" case _: BytesType => to_type = ".to_vec()" + case _: CalcArrayType => to_type = ".clone()" case _ => } + println(s"$ifTrue, $ifFalse - ${detectType(ifTrue)}") if (to_type.isEmpty) { s"if ${translate(condition)} { ${translate(ifTrue)} } else { ${translate(ifFalse)} }" } else { @@ -345,7 +432,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } override def enumToInt(v: expr, et: EnumType): String = - s"i64::from(${remove_deref(translate(v))})" + s"i64::from(&${translate(v)})" override def boolToInt(v: expr): String = s"(${translate(v)}) as i32" @@ -366,7 +453,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) if (bytesExpr.charAt(0) == '*') { s"decode_string(&$bytesExpr, &${translate(encoding)})?" } else { - s"decode_string($bytesExpr, &${translate(encoding)})?" + s"decode_string(${ensure_vec_amp(bytesExpr)}, &${translate(encoding)})?" } } From 6170f33c13a587f698b6fd3f3955fbfa4dc76371 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Tue, 3 Jan 2023 13:58:19 +0100 Subject: [PATCH 104/153] clean warnings --- .../struct/languages/RustCompiler.scala | 65 +++++++++---------- .../struct/translators/RustTranslator.scala | 4 +- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 3197557c5..8bd5410dc 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -465,7 +465,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc out.puts(s"pub fn set_params(&mut self, $paramsArg) {") out.inc - typeProvider.nowClass.params.foreach((p) => handleAssignmentParams(p.id, paramName(p.id))) + typeProvider.nowClass.params.foreach(p => handleAssignmentParams(p.id, paramName(p.id))) out.dec out.puts("}") out.dec @@ -475,7 +475,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(_: CalcEndian) | Some(InheritedEndian) => out.puts(s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") out.inc - val t = kaitaiTypeToNativeType(Some(EndianIdentifier), typeProvider.nowClass, IntMultiType(true, Width4, None), excludeOptionWrapper = true) + val t = kaitaiTypeToNativeType(Some(EndianIdentifier), typeProvider.nowClass, IntMultiType(signed = true, Width4, None), excludeOptionWrapper = true) out.puts(s"pub fn set_endian(&mut self, ${idToStr(EndianIdentifier)}: $t) {") out.inc handleAssignmentSimple(EndianIdentifier, s"${idToStr(EndianIdentifier)}") @@ -578,19 +578,19 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { dataType match { case _: UserType => - out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = ${translator.remove_deref(expression(value))}.clone();") + out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = ${translator.remove_deref(expression(value))}.clone();") case _: StrType => - out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = ${translator.remove_deref(expression(value))}.to_string();") + out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = ${translator.remove_deref(expression(value))}.to_string();") case _: BytesType => - out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") + out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") case _: ArrayType => - out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") + out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") case _: EnumType => - out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = ${translator.remove_deref(expression(value))};") + out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = ${translator.remove_deref(expression(value))};") case _ => //translator.context_need_deref_attr = true val primType = kaitaiPrimitiveToNativeType(dataType) - out.puts(s"*${RustCompiler.privateMemberName(instName, true)} = (${expression(value)}) as $primType;") + out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = (${expression(value)}) as $primType;") //translator.context_need_deref_attr = false } } @@ -705,15 +705,15 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (paramId.isDefined) { need_clone = !translator.is_copy_type(paramId.get.dataType) } - val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, paramId.get.dataType) + //val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, paramId.get.dataType) paramId.get.dataType match { - case et: EnumType => - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr.clone();") + case _: EnumType => + out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = $expr.clone();") case _ => if (need_clone) - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr.clone();") + out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = $expr.clone();") else - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr;") + out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = $expr;") } } @@ -736,19 +736,19 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: EnumType => done = true out.puts( - s"*${RustCompiler.privateMemberName(id, true)} = $expr;" + s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = $expr;" ) case _: SwitchType => done = true - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = Some($expr);") + out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = Some($expr);") case _ => } if (refcell) { - val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, idType) + //val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, idType) if (opaque) { - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = Some($expr);") + out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = Some($expr);") } else { - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr;") + out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = $expr;") } done = true } @@ -769,11 +769,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } if (inst) { done = true - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr;") + out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = $expr;") } } if (!done) - out.puts(s"*${RustCompiler.privateMemberName(id, true)} = $expr;") + out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = $expr;") } override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = @@ -804,7 +804,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) addParams = Utils.join(t.args.map{ a => val typ = translator.detectType(a) var byref = "" - val t = kaitaiTypeToNativeType(None, typeProvider.nowClass, typ) + //val t = kaitaiTypeToNativeType(None, typeProvider.nowClass, typ) var try_into = "" // exclude enums typ match { @@ -830,7 +830,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val addArgs = if (t.isOpaque) { ", None, None" } else { - val currentType = classTypeName(typeProvider.nowClass) + //val currentType = classTypeName(typeProvider.nowClass) var parent = t.forcedParent match { case Some(USER_TYPE_NO_PARENT) => "None" case Some(fp) => translator.translate(fp) @@ -1280,13 +1280,12 @@ object RustCompiler case IoIdentifier => "_io" case RootIdentifier => "_root" case ParentIdentifier => "_parent" - case _ => { + case _ => val n = s"${self_name()}.${idToStr(id)}" if (writeAccess) s"$n.borrow_mut()" else s"$n.borrow()" - } } def idToStr(id: Identifier): String = id match { @@ -1323,17 +1322,17 @@ object RustCompiler // TODO: Use `mod` to scope types instead of weird names names.map(x => type2class(x)).mkString("_") - def lifetimeParam(d: DataType): String = - if (containsReferences(d)) s"<$streamLife>" else "" +// def lifetimeParam(d: DataType): String = +// if (containsReferences(d)) s"<$streamLife>" else "" - def containsReferences(d: DataType): Boolean = containsReferences(d, None) +// def containsReferences(d: DataType): Boolean = containsReferences(d, None) - def containsReferences(c: ClassSpec, - originating: Option[ClassSpec]): Boolean = - c.seq.exists(t => containsReferences(t.dataType, originating)) || - c.instances.exists( - i => containsReferences(i._2.dataTypeComposite, originating) - ) +// def containsReferences(c: ClassSpec, +// originating: Option[ClassSpec]): Boolean = +// c.seq.exists(t => containsReferences(t.dataType, originating)) || +// c.instances.exists( +// i => containsReferences(i._2.dataTypeComposite, originating) +// ) def containsReferences(d: DataType, originating: Option[ClassSpec]): Boolean = d match { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index d6b68b98e..2328efa51 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -316,13 +316,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) var deref = false var found = get_attr(get_top_class(provider.nowClass), s) if (found.isDefined ) { - val cleanType = RustCompiler.kaitaiTypeToNativeType(Some(NamedIdentifier(s)), provider.nowClass, found.get.dataType, cleanType = true) + val cleanType = RustCompiler.kaitaiTypeToNativeType(Some(NamedIdentifier(s)), provider.nowClass, found.get.dataType, cleanTypename = true) //if (cleanType != "Vec") deref = !enum_numeric_only(found.get.dataTypeComposite) //is_copy_type(found.get.dataTypeComposite) } else { found = get_instance(get_top_class(provider.nowClass), s) if (found.isDefined) { - val cleanType = RustCompiler.kaitaiTypeToNativeType(Some(NamedIdentifier(s)), provider.nowClass, found.get.dataType, cleanType = true) + val cleanType = RustCompiler.kaitaiTypeToNativeType(Some(NamedIdentifier(s)), provider.nowClass, found.get.dataType, cleanTypename = true) //if (cleanType != "Vec") deref = true //is_copy_type(found.get.dataTypeComposite) } else { From c74309fdd84e0a2c6211073607714a730696defa Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Tue, 3 Jan 2023 21:10:27 +0100 Subject: [PATCH 105/153] add `RefCell<...>` in `kaitaiTypeToNativeType`, remove `kaitaiTypeToNativeTypeWrapper` (autogenerated `test_debug_switch_user` passed) --- .../struct/languages/RustCompiler.scala | 48 ++++++++----------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 8bd5410dc..0ee9633ce 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -203,36 +203,22 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct case IoIdentifier | RootIdentifier | ParentIdentifier => return case _ => - kaitaiTypeToNativeTypeWrapper(Some(attrName), attrType) + kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType) } - out.puts(s"${idToStr(attrName)}: RefCell<$typeName>,") - } - - def kaitaiTypeToNativeTypeWrapper(id: Option[Identifier], - attrType: DataType): String = { - // if (id.isDefined) id.get match { - // case RawIdentifier(inner) => { - // val found = translator.get_instance(typeProvider.nowClass, idToStr(inner)) - // if (found.isDefined) { - // // raw id for instance should have RefCell wrapper - // return s"${kaitaiTypeToNativeType(id, typeProvider.nowClass, attrType)}" - // } - // } - // case _ => - // } - kaitaiTypeToNativeType(id, typeProvider.nowClass, attrType) + out.puts(s"${idToStr(attrName)}: $typeName,") } override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { - val typeName = attrName match { + var typeName = attrName match { // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct - case IoIdentifier | RootIdentifier | ParentIdentifier => return - case _ => - kaitaiTypeToNativeTypeWrapper(Some(attrName), attrType) + case IoIdentifier | RootIdentifier | ParentIdentifier => return + case _ => + kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType) } + //val typeNameEx = kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType, excludeOptionWrapper = true) out.puts( s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") @@ -268,7 +254,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // out.puts(s"RefEnum::new(self.${idToStr(attrName)}.borrow())") // } else { - out.puts(s"pub fn $fn(&self) -> Ref<$typeName> {") + if (!typeName.startsWith("Ref<")) + typeName = typeName.replace("RefCell<", "Ref<") + + out.puts(s"pub fn $fn(&self) -> $typeName {") out.inc out.puts(s"self.${idToStr(attrName)}.borrow()") } @@ -1139,7 +1128,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"impl $enum_typeName {") out.inc val fn = idToStr(attrName) - val nativeType = kaitaiTypeToNativeTypeWrapper(Some(attrName), attr.dataTypeComposite) + val nativeType = kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attr.dataTypeComposite, cleanTypename = true) //out.puts(nativeType.attrGetterDcl(fn)) out.puts(s"pub fn $fn(&self) -> Ref<$nativeType> {") out.inc @@ -1360,10 +1349,11 @@ object RustCompiler cleanTypename: Boolean = false): String = attrType match { // TODO: Not exhaustive - case _: NumericType => kaitaiPrimitiveToNativeType(attrType) - case _: BooleanType => kaitaiPrimitiveToNativeType(attrType) - case _: StrType => kaitaiPrimitiveToNativeType(attrType) - case _: BytesType => kaitaiPrimitiveToNativeType(attrType) + case _: NumericType | _: BooleanType | _: StrType | _: BytesType => + if (cleanTypename) + kaitaiPrimitiveToNativeType(attrType) + else + s"RefCell<${kaitaiPrimitiveToNativeType(attrType)}>" case t: UserType => val baseName = t.classSpec match { @@ -1399,7 +1389,7 @@ object RustCompiler case _ => kstructUnitName } - if (excludeOptionWrapper) typeName else s"Option<$typeName>" + if (excludeOptionWrapper) typeName else s"RefCell>" case KaitaiStreamType => kstreamName case CalcKaitaiStructType => kstructUnitName @@ -1429,6 +1419,6 @@ object RustCompiler case _: BytesType => "Vec" case ArrayTypeInStream(inType) => s"Vec<${kaitaiPrimitiveToNativeType(inType)}>" - case _ => "kaitaiPrimitiveToNativeType ???" + case _ => s"kaitaiPrimitiveToNativeType '${attrType.toString}' ???" } } From fc1699d01c25cadaae6ef3a428efee0c7c109dc4 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Tue, 3 Jan 2023 21:21:20 +0100 Subject: [PATCH 106/153] '.as_ref().unwrap()' for `RefCell val code = s"$s()" - val aType = RustCompiler.kaitaiPrimitiveToNativeType(as.dataTypeComposite) + val aType = RustCompiler.kaitaiTypeToNativeType(Some(as.id), provider.nowClass, as.dataTypeComposite) + val refOpt = "RefCell]+>>".r aType match { //case "String" => s"$code.as_str()" //case "Vec" => s"$code.as_slice()" - //case "SwitchType" => s"$code.as_ref().unwrap()" + case refOpt() => s"$code.as_ref().unwrap()" case _ => code } case pis: ParseInstanceSpec => From 37bcb02f30500d12f427877ad5b50aa270aace83 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 4 Jan 2023 13:12:26 +0800 Subject: [PATCH 107/153] parent access fixed + translate_args added. --- .../struct/languages/RustCompiler.scala | 49 ++++++++++--------- .../struct/translators/RustTranslator.scala | 4 +- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 484904bdb..66f662bc0 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -186,10 +186,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"self_rc._root.set(_root.get());") out.puts(s"self_rc._parent.set(_parent.get());") out.puts(s"let _new_parent = SharedType::::new(self_rc.clone());") - out.puts(s"let _rrc = self_rc._root.get();") - out.puts(s"let _prc = self_rc._parent.get();") + out.puts(s"let _rrc = self_rc._root.get()?;") + out.puts(s"let _prc = self_rc._parent.get_value().borrow();") out.puts(s"let _r = _rrc.as_ref();") - out.puts(s"let _p = _prc.as_ref();") // If there aren't any attributes to parse, we need to end the read implementation here if (typeProvider.nowClass.seq.isEmpty) @@ -421,7 +420,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add(s"""#[path = "$mod_name.rs"] mod $mod_name;""") importList.add(s"use self::$mod_name::*;") - val argList = args.map(expression).mkString(", ") + val argList = translate_args(args, false) val argListInParens = s"($argList)" out.puts(s"let $procName = $procClass::new$argListInParens;") s"$procName.decode(&$srcExpr)" @@ -560,10 +559,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) out.puts(s") -> KResult> {") out.inc - out.puts(s"let _rrc = self._root.get();") - out.puts(s"let _prc = self._parent.get();") + out.puts(s"let _rrc = self._root.get()?;") + out.puts(s"let _prc = self._parent.get_value().borrow();") out.puts(s"let _r = _rrc.as_ref();") - out.puts(s"let _p = _prc.as_ref();") } override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, @@ -781,6 +779,25 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = out.puts(s"let $id = $expr;") + def translate_args(args: Seq[Ast.expr], into: Boolean): String = { + Utils.join(args.map { a => + val typ = translator.detectType(a) + var byref = "" + val t = kaitaiTypeToNativeType(None, typeProvider.nowClass, typ) + var try_into = "" + typ match { + case _: NumericType => + if (into) { + try_into = s".try_into().map_err(|_| KError::CastError)?" + } + case _ => + if (!translator.is_copy_type(typ)) + byref = "&" + } + s"$byref${translator.translate(a)}$try_into" + }, "", ", ", "") + } + override def parseExpr(dataType: DataType, assignType: DataType, io: String, @@ -803,23 +820,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case BitsType1(bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}(1)? != 0" case BitsType(width: Int, bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}($width)?" case t: UserType => - addParams = Utils.join(t.args.map{ a => - val typ = translator.detectType(a) - var byref = "" - val t = kaitaiTypeToNativeType(None, typeProvider.nowClass, typ) - var try_into = "" - // exclude enums - typ match { - case _: NumericType => try_into = s".try_into().map_err(|_| KError::CastError)?" - case _ => - if (!translator.is_copy_type(typ)) - byref = "&" - } - //var need_deref = "*" - //if (t.startsWith("RefCell")) - // need_deref = "&*" - s"$byref${translator.translate(a)}$try_into" - }, "", ", ", "") + addParams = translate_args(t.args, true) val userType = t match { case t: UserType => val baseName = t.classSpec match { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index d6b68b98e..909f3fb70 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -184,7 +184,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) val t = translate(value) var a = doName(attrName) attrName match { - case Identifier.PARENT => a = a + ".get().as_ref()" + case Identifier.PARENT => a = a + ".get_value().borrow().as_ref().unwrap()" case _ => } var r = "" @@ -343,7 +343,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case Identifier.INDEX => "_i" case Identifier.IO => s"${RustCompiler.privateMemberName(IoIdentifier)}" case Identifier.ROOT => s"_r" - case Identifier.PARENT => s"_p" + case Identifier.PARENT => s"_prc.as_ref().unwrap()" case _ => val n = doName(s) val deref = !n.endsWith(".as_str()") && !n.endsWith(".as_slice()") && need_deref(s) From b09b820d26fc35eb6b5252fac6d3e633816980bc Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 4 Jan 2023 18:28:43 +0800 Subject: [PATCH 108/153] need_deref extended with classspec + code cleanup. --- .../struct/translators/RustTranslator.scala | 51 ++++--------------- 1 file changed, 10 insertions(+), 41 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 909f3fb70..6df726626 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -71,37 +71,11 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) if (memberFound.isDefined) memberFound.get match { case vis: ValueInstanceSpec => - val call = s"$s(${privateMemberName(IoIdentifier)})?" - vis.dataTypeOpt match { - case Some(dt) => - dt match { - //case _: StrType => s"$call.as_str()" - //case _: BytesType => s"$call.as_slice()" - case _ => call - } - case None => call - } + s"$s(${privateMemberName(IoIdentifier)})?" case as: AttrSpec => - val code = s"$s()" - val aType = RustCompiler.kaitaiPrimitiveToNativeType(as.dataTypeComposite) - aType match { - //case "String" => s"$code.as_str()" - //case "Vec" => s"$code.as_slice()" - //case "SwitchType" => s"$code.as_ref().unwrap()" - case _ => code - } + s"$s()" case pis: ParseInstanceSpec => - pis.dataType match { - case _: NumericType | _: StrType => - s"$s(${privateMemberName(IoIdentifier)})?" - case t: UserTypeInstream => - if (t.isOpaque) - s"$s(${privateMemberName(IoIdentifier)})?.as_ref().unwrap()" - else - s"$s(${privateMemberName(IoIdentifier)})?" - case _ => - s"$s(${privateMemberName(IoIdentifier)})?.as_ref().unwrap()" - } + s"$s(${privateMemberName(IoIdentifier)})?" case _ => s"$s()" } @@ -298,7 +272,6 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _: NumericType => // leave unchanged case _ => enum_only_numeric = false } - //println(s"$dataType - $enum_only_numeric") enum_only_numeric } @@ -312,21 +285,18 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _ => true } - def need_deref(s: String): Boolean = { + def need_deref(s: String, c: ClassSpec = provider.nowClass): Boolean = { var deref = false - var found = get_attr(get_top_class(provider.nowClass), s) + var tc = get_top_class(c) + var found = get_attr(tc, s) if (found.isDefined ) { - val cleanType = RustCompiler.kaitaiTypeToNativeType(Some(NamedIdentifier(s)), provider.nowClass, found.get.dataType, cleanType = true) - //if (cleanType != "Vec") - deref = !enum_numeric_only(found.get.dataTypeComposite) //is_copy_type(found.get.dataTypeComposite) + deref = !enum_numeric_only(found.get.dataTypeComposite) } else { - found = get_instance(get_top_class(provider.nowClass), s) + found = get_instance(tc, s) if (found.isDefined) { - val cleanType = RustCompiler.kaitaiTypeToNativeType(Some(NamedIdentifier(s)), provider.nowClass, found.get.dataType, cleanType = true) - //if (cleanType != "Vec") - deref = true //is_copy_type(found.get.dataTypeComposite) + deref = true } else { - found = get_param(get_top_class(provider.nowClass), s) + found = get_param(tc, s) if (found.isDefined) { deref = !enum_numeric_only(found.get.dataTypeComposite) } else { @@ -390,7 +360,6 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _: CalcArrayType => to_type = ".clone()" case _ => } - println(s"$ifTrue, $ifFalse - ${detectType(ifTrue)}") if (to_type.isEmpty) { s"if ${translate(condition)} { ${translate(ifTrue)} } else { ${translate(ifFalse)} }" } else { From 704befaace91d8bf0ad5442c74cc93df29179921 Mon Sep 17 00:00:00 2001 From: Vitaly Reshetyuk Date: Thu, 5 Jan 2023 21:34:55 +0100 Subject: [PATCH 109/153] restore '.as_ref().unwrap()' for `RefCell s"$s(${privateMemberName(IoIdentifier)})?" case as: AttrSpec => - s"$s()" + val code = s"$s()" + val aType = RustCompiler.kaitaiTypeToNativeType(Some(as.id), provider.nowClass, as.dataTypeComposite) + val refOpt = "RefCell]+>>".r + aType match { + //case "String" => s"$code.as_str()" + //case "Vec" => s"$code.as_slice()" + case refOpt() => s"$code.as_ref().unwrap()" + case _ => code + } case pis: ParseInstanceSpec => s"$s(${privateMemberName(IoIdentifier)})?" case _ => From 389ca1505c4ca2e453a1f53d06a8974f8dfbaf02 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 7 Jan 2023 15:32:57 +0800 Subject: [PATCH 110/153] refactoring. --- .../struct/languages/RustCompiler.scala | 72 ++++--------------- .../struct/translators/RustTranslator.scala | 7 +- 2 files changed, 19 insertions(+), 60 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index f24d5f02e..33381bcd1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -199,20 +199,18 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) attrType: DataType, isNullable: Boolean): Unit = { val typeName = attrName match { - // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct case IoIdentifier | RootIdentifier | ParentIdentifier => return case _ => kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType) } - out.puts(s"${idToStr(attrName)}: $typeName,") + out.puts(s"${idToStr(attrName)}: RefCell<$typeName>,") } override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { var typeName = attrName match { - // For keeping lifetimes simple, we don't store _io, _root, or _parent with the struct case IoIdentifier | RootIdentifier | ParentIdentifier => return case _ => kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType) @@ -247,16 +245,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") fn = s"${fn}_enum" } - // if (switch_typename || enum_typename) { - // out.puts(s"pub fn $fn(&self) -> RefEnum<$typeNameEx> {") - // out.inc - // out.puts(s"RefEnum::new(self.${idToStr(attrName)}.borrow())") - // } else { - if (!typeName.startsWith("Ref<")) - typeName = typeName.replace("RefCell<", "Ref<") - - out.puts(s"pub fn $fn(&self) -> $typeName {") + out.puts(s"pub fn $fn(&self) -> Ref<$typeName> {") out.inc out.puts(s"self.${idToStr(attrName)}.borrow()") } @@ -820,20 +810,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val addArgs = if (t.isOpaque) { ", None, None" } else { - //val currentType = classTypeName(typeProvider.nowClass) var parent = t.forcedParent match { case Some(USER_TYPE_NO_PARENT) => "None" case Some(fp) => translator.translate(fp) case None => if (!in_instance) { s"Some(_new_parent.clone())" - // if (typeProvider.nowClass.isTopLevel) { - // s"Some(${privateMemberName(ParentIdentifier)}.unwrap_or(KStructUnit::parent_stack()).push(self))" - // } else { - // s"Some(${privateMemberName(ParentIdentifier)}.unwrap().push(self))" - // } } else { - //s"Some(${self_name()}.${privateMemberName(ParentIdentifier)}.clone())" "None" } } @@ -1129,13 +1112,20 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"impl $enum_typeName {") out.inc val fn = idToStr(attrName) - val nativeType = kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attr.dataTypeComposite, cleanTypename = true) - //out.puts(nativeType.attrGetterDcl(fn)) - out.puts(s"pub fn $fn(&self) -> Ref<$nativeType> {") + var nativeType = kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attr.dataTypeComposite, cleanTypename = true) + var nativeTypeEx = kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attr.dataTypeComposite) + var clone = "" + val rc_typename = nativeTypeEx.startsWith("Rc<") + if (rc_typename) { + nativeType = s"$nativeTypeEx" + clone = ".clone()" + } else + nativeType = s"Ref<$nativeType>" + out.puts(s"pub fn $fn(&self) -> $nativeType {") out.inc out.puts("match self {") out.inc - out.puts(s"$enum_typeName::$typeName(x) => x.$fn.borrow(),") + out.puts(s"$enum_typeName::$typeName(x) => x.$fn.borrow()$clone,") out.puts("_ => panic!(\"wrong variant: {:?}\", self),") out.dec out.puts("}") @@ -1312,37 +1302,6 @@ object RustCompiler // TODO: Use `mod` to scope types instead of weird names names.map(x => type2class(x)).mkString("_") -// def lifetimeParam(d: DataType): String = -// if (containsReferences(d)) s"<$streamLife>" else "" - -// def containsReferences(d: DataType): Boolean = containsReferences(d, None) - -// def containsReferences(c: ClassSpec, -// originating: Option[ClassSpec]): Boolean = -// c.seq.exists(t => containsReferences(t.dataType, originating)) || -// c.instances.exists( -// i => containsReferences(i._2.dataTypeComposite, originating) -// ) - - def containsReferences(d: DataType, originating: Option[ClassSpec]): Boolean = - d match { - case _: BytesType | _: StrType => true - case _: UserType => true - /* - t.classSpec match { - // Recursive types may need references, but the recursion itself - // will be handled by `Box<>`, so doesn't need a reference - case Some(inner) if originating.contains(inner) => false - case Some(inner) => containsReferences(inner, originating.orElse(Some(inner))) - case None => false - } - */ - case t: ArrayType => containsReferences(t.elType, originating) - case st: SwitchType => - st.cases.values.exists(t => containsReferences(t, originating)) - case _ => false - } - def kaitaiTypeToNativeType(id: Option[Identifier], cs: ClassSpec, attrType: DataType, @@ -1351,10 +1310,7 @@ object RustCompiler attrType match { // TODO: Not exhaustive case _: NumericType | _: BooleanType | _: StrType | _: BytesType => - if (cleanTypename) kaitaiPrimitiveToNativeType(attrType) - else - s"RefCell<${kaitaiPrimitiveToNativeType(attrType)}>" case t: UserType => val baseName = t.classSpec match { @@ -1390,7 +1346,7 @@ object RustCompiler case _ => kstructUnitName } - if (excludeOptionWrapper) typeName else s"RefCell>" + if (excludeOptionWrapper) typeName else s"Option<$typeName>" case KaitaiStreamType => kstreamName case CalcKaitaiStructType => kstructUnitName diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 63667d4e5..aa23f4a0c 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -75,11 +75,14 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case as: AttrSpec => val code = s"$s()" val aType = RustCompiler.kaitaiTypeToNativeType(Some(as.id), provider.nowClass, as.dataTypeComposite) - val refOpt = "RefCell]+>>".r + val refOpt = "Option<[^>]+>$".r aType match { //case "String" => s"$code.as_str()" //case "Vec" => s"$code.as_slice()" - case refOpt() => s"$code.as_ref().unwrap()" + case refOpt() => + if (!enum_numeric_only(as.dataTypeComposite)) { + s"$code.as_ref().unwrap()" + } else code case _ => code } case pis: ParseInstanceSpec => From 39b6635ebeaa29c2a7f63e01a9d0b9186eba4c00 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 19 Jan 2023 16:10:26 +0800 Subject: [PATCH 111/153] translate_args: catch root and replace with *_rrc. --- .../io/kaitai/struct/languages/RustCompiler.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 33381bcd1..acf6866ec 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -184,9 +184,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"self_rc._root.set(_root.get());") out.puts(s"self_rc._parent.set(_parent.get());") out.puts(s"let _new_parent = SharedType::::new(self_rc.clone());") - out.puts(s"let _rrc = self_rc._root.get()?;") + out.puts(s"let _rrc = self_rc._root.get_value().borrow();") out.puts(s"let _prc = self_rc._parent.get_value().borrow();") - out.puts(s"let _r = _rrc.as_ref();") + out.puts(s"let _r = _rrc.as_ref().unwrap();") // If there aren't any attributes to parse, we need to end the read implementation here if (typeProvider.nowClass.seq.isEmpty) @@ -536,9 +536,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) out.puts(s") -> KResult> {") out.inc - out.puts(s"let _rrc = self._root.get()?;") + out.puts(s"let _rrc = self._root.get_value().borrow();") out.puts(s"let _prc = self._parent.get_value().borrow();") - out.puts(s"let _r = _rrc.as_ref();") + out.puts(s"let _r = _rrc.as_ref().unwrap();") } override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, @@ -771,7 +771,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (!translator.is_copy_type(typ)) byref = "&" } - s"$byref${translator.translate(a)}$try_into" + var translated = translator.translate(a) + if (translated == "_r") // _root + translated = "*_rrc" + s"$byref$translated$try_into" }, "", ", ", "") } From 628e3a508ababa749216b05731c96b31f94042e7 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 19 Jan 2023 16:11:32 +0800 Subject: [PATCH 112/153] unwrap Option. --- .../struct/translators/RustTranslator.scala | 88 +++++++------------ 1 file changed, 30 insertions(+), 58 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index aa23f4a0c..d5198f217 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -71,7 +71,12 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) if (memberFound.isDefined) memberFound.get match { case vis: ValueInstanceSpec => - s"$s(${privateMemberName(IoIdentifier)})?" + val aType = RustCompiler.kaitaiTypeToNativeType(Some(vis.id), provider.nowClass, vis.dataTypeComposite) + val refOpt = "^Option<.*".r + aType match { + case refOpt() => s"$s(${privateMemberName(IoIdentifier)})?.as_ref().unwrap()" + case _ => s"$s(${privateMemberName(IoIdentifier)})?" + } case as: AttrSpec => val code = s"$s()" val aType = RustCompiler.kaitaiTypeToNativeType(Some(as.id), provider.nowClass, as.dataTypeComposite) @@ -85,8 +90,21 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } else code case _ => code } + case pd: ParamDefSpec => + val aType = RustCompiler.kaitaiTypeToNativeType(Some(pd.id), provider.nowClass, pd.dataTypeComposite) + val refOpt = "^Option<.*".r + aType match { + case refOpt() => s"$s().as_ref().unwrap()" + case _ => s"$s()" + } case pis: ParseInstanceSpec => - s"$s(${privateMemberName(IoIdentifier)})?" + //s"$s(${privateMemberName(IoIdentifier)})?" + val aType = RustCompiler.kaitaiTypeToNativeType(Some(pis.id), provider.nowClass, pis.dataTypeComposite) + val refOpt = "^Option<.*".r + aType match { + case refOpt() => s"$s(${privateMemberName(IoIdentifier)})?.as_ref().unwrap()" + case _ => s"$s(${privateMemberName(IoIdentifier)})?" + } case _ => s"$s()" } @@ -95,7 +113,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } - def findMember(attrName: String): Option[MemberSpec] = { + def findMember(attrName: String, c: ClassSpec = provider.nowClass): Option[MemberSpec] = { def findInClass(inClass: ClassSpec): Option[MemberSpec] = { inClass.seq.foreach { el => @@ -127,7 +145,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case Identifier.PARENT | Identifier.IO => None case _ => - for { ms <- findInClass(provider.nowClass) } + for { ms <- findInClass(get_top_class(c)) } return Some(ms) provider.asInstanceOf[ClassTypeProvider].allClasses.foreach { cls => @@ -226,46 +244,6 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } - def get_attr(cs: ClassSpec, id: String): Option[MemberSpec] = { - var found : Option[MemberSpec] = None - cs.seq.foreach { el => - if (idToStr(el.id) == id) { - found = Some(el) - } - } - // look deeper - if (found.isEmpty) { - cs.types.foreach { - case (_, typeSpec) => - found = get_attr(typeSpec, id) - if (found.isDefined) { - return found - } - } - } - found - } - - def get_param(cs: ClassSpec, id: String): Option[MemberSpec] = { - var found : Option[MemberSpec] = None - cs.params.foreach { el => - if (idToStr(el.id) == id) { - found = Some(el) - } - } - // look deeper - if (found.isEmpty) { - cs.types.foreach { - case (_, typeSpec) => - found = get_param(typeSpec, id) - if (found.isDefined) { - return found - } - } - } - found - } - var context_need_deref_attr = false def enum_numeric_only(dataType: DataType): Boolean = { @@ -298,21 +276,15 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) def need_deref(s: String, c: ClassSpec = provider.nowClass): Boolean = { var deref = false - var tc = get_top_class(c) - var found = get_attr(tc, s) - if (found.isDefined ) { - deref = !enum_numeric_only(found.get.dataTypeComposite) - } else { - found = get_instance(tc, s) - if (found.isDefined) { + val memberFound = findMember(s, c) + if (memberFound.isDefined ) { + val spec = memberFound.get + spec match { + case _: AttrSpec | _: ParamDefSpec => + deref = !enum_numeric_only(spec.dataTypeComposite) + case _: ValueInstanceSpec | _: ParseInstanceSpec => deref = true - } else { - found = get_param(tc, s) - if (found.isDefined) { - deref = !enum_numeric_only(found.get.dataTypeComposite) - } else { - deref = false - } + case _ => } } deref From 94dde6d8e5b49a52225010682648f75ae92c7ee1 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 24 Jan 2023 15:17:53 +0800 Subject: [PATCH 113/153] instanceCalculate use handleAssignmentSimple (fixed). --- .../struct/languages/RustCompiler.scala | 22 +++++++++---------- .../struct/translators/RustTranslator.scala | 5 +++-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index acf6866ec..9fd51bf6f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -49,6 +49,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) outHeader.puts("#![allow(non_camel_case_types)]") outHeader.puts("#![allow(irrefutable_let_patterns)]") outHeader.puts("#![allow(unused_comparisons)]") + outHeader.puts("#![allow(arithmetic_overflow)]") outHeader.puts outHeader.puts("extern crate kaitai;") @@ -555,20 +556,17 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { dataType match { case _: UserType => - out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = ${translator.remove_deref(expression(value))}.clone();") + handleAssignmentSimple(instName, s"${translator.remove_deref(expression(value))}.clone()") case _: StrType => - out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = ${translator.remove_deref(expression(value))}.to_string();") + handleAssignmentSimple(instName, s"${translator.remove_deref(expression(value))}.to_string()") case _: BytesType => - out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") + handleAssignmentSimple(instName, s"${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec()") case _: ArrayType => - out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = ${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec();") + handleAssignmentSimple(instName, s"${translator.rem_vec_amp(translator.remove_deref(expression(value)))}.to_vec()") case _: EnumType => - out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = ${translator.remove_deref(expression(value))};") + handleAssignmentSimple(instName, s"${translator.remove_deref(expression(value))}") case _ => - //translator.context_need_deref_attr = true - val primType = kaitaiPrimitiveToNativeType(dataType) - out.puts(s"*${RustCompiler.privateMemberName(instName, writeAccess = true)} = (${expression(value)}) as $primType;") - //translator.context_need_deref_attr = false + handleAssignmentSimple(instName, s"(${expression(value)}) as ${kaitaiPrimitiveToNativeType(dataType)}") } } @@ -695,7 +693,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def handleAssignmentSimple(id: Identifier, expr: String): Unit = { - val seqId = typeProvider.nowClass.seq.find(s => s.id == id) + val seqId = translator.findMember(idToStr(id)) var done = false var refcell = false var opaque = false @@ -721,8 +719,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _ => } if (refcell) { - //val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, idType) - if (opaque) { + val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, idType) + if (opaque || typeName.startsWith("Option<")) { out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = Some($expr);") } else { out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = $expr;") diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index d5198f217..ea1d1402f 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -80,7 +80,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case as: AttrSpec => val code = s"$s()" val aType = RustCompiler.kaitaiTypeToNativeType(Some(as.id), provider.nowClass, as.dataTypeComposite) - val refOpt = "Option<[^>]+>$".r + //val refOpt = "Option<[^>]+>$".r + val refOpt = "^Option<.*".r aType match { //case "String" => s"$code.as_str()" //case "Vec" => s"$code.as_slice()" @@ -145,7 +146,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case Identifier.PARENT | Identifier.IO => None case _ => - for { ms <- findInClass(get_top_class(c)) } + for { ms <- findInClass(c) } return Some(ms) provider.asInstanceOf[ClassTypeProvider].allClasses.foreach { cls => From 8238825a91c8901a9b99a43c81c89f8e0adaafdb Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 25 Jan 2023 18:12:11 +0800 Subject: [PATCH 114/153] catch opaque types (with Option wrapper). --- .../struct/languages/RustCompiler.scala | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 9fd51bf6f..55cec8acc 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -983,6 +983,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ioName } + def if_opaque(dt: DataType): Boolean = dt match { + case t: UserType => if (t.isOpaque) true else false + case _ => false + } + def switchTypeEnum(id: Identifier, st: SwitchType): Unit = { // Because Rust can't handle `AnyType` in the type hierarchy, // we generate an enum with all possible variations @@ -1042,7 +1047,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) Some(id), typeProvider.nowClass, t, - excludeOptionWrapper = true) + excludeOptionWrapperAlways = true) } val new_typename = types_set.add(typeName) if (new_typename) { @@ -1050,7 +1055,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc out.puts(s"fn from(v: $typeName) -> Self {") out.inc - out.puts(s"Self::$variantName($v)") + if (if_opaque(t)) { + out.puts(s"Self::$variantName(Some($v))") + } else { + out.puts(s"Self::$variantName($v)") + } out.dec out.puts("}") out.dec @@ -1115,6 +1124,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val fn = idToStr(attrName) var nativeType = kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attr.dataTypeComposite, cleanTypename = true) var nativeTypeEx = kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attr.dataTypeComposite) + val typeNameEx = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, t) + val x = if (typeNameEx.startsWith("Option<")) "x.as_ref().unwrap()" else "x" var clone = "" val rc_typename = nativeTypeEx.startsWith("Rc<") if (rc_typename) { @@ -1126,7 +1137,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc out.puts("match self {") out.inc - out.puts(s"$enum_typeName::$typeName(x) => x.$fn.borrow()$clone,") + out.puts(s"$enum_typeName::$typeName(x) => $x.$fn.borrow()$clone,") out.puts("_ => panic!(\"wrong variant: {:?}\", self),") out.dec out.puts("}") @@ -1307,6 +1318,7 @@ object RustCompiler cs: ClassSpec, attrType: DataType, excludeOptionWrapper: Boolean = false, + excludeOptionWrapperAlways: Boolean = false, cleanTypename: Boolean = false): String = attrType match { // TODO: Not exhaustive @@ -1321,7 +1333,7 @@ object RustCompiler (t.isOpaque, cleanTypename) match { case (_, true) => baseName - case (true, _) => s"Option>" + case (true, _) => if (excludeOptionWrapperAlways) s"Rc<$baseName>" else s"Option>" case (false, _) => s"Rc<$baseName>" } @@ -1333,7 +1345,7 @@ object RustCompiler if (cleanTypename) { return baseName } - if (excludeOptionWrapper) baseName else s"$baseName" + if (excludeOptionWrapper || excludeOptionWrapperAlways) baseName else s"$baseName" case t: ArrayType => s"Vec<${kaitaiTypeToNativeType(id, cs, t.elType, excludeOptionWrapper = true)}>" @@ -1347,7 +1359,7 @@ object RustCompiler case _ => kstructUnitName } - if (excludeOptionWrapper) typeName else s"Option<$typeName>" + if (excludeOptionWrapper || excludeOptionWrapperAlways) typeName else s"Option<$typeName>" case KaitaiStreamType => kstreamName case CalcKaitaiStructType => kstructUnitName From 3784ad2560d58facb4f6f751cc9c94204e391cc8 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 26 Jan 2023 14:01:51 +0800 Subject: [PATCH 115/153] allow(overflowing_literals) --- .../src/main/scala/io/kaitai/struct/languages/RustCompiler.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 55cec8acc..cf64f6b82 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -50,6 +50,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) outHeader.puts("#![allow(irrefutable_let_patterns)]") outHeader.puts("#![allow(unused_comparisons)]") outHeader.puts("#![allow(arithmetic_overflow)]") + outHeader.puts("#![allow(overflowing_literals)]") outHeader.puts outHeader.puts("extern crate kaitai;") From e9d4555cfd65b34361f3879bc1aae3c5d887d286 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 30 Jan 2023 16:18:55 +0800 Subject: [PATCH 116/153] store IO (as BytesReader) and use. --- .../struct/languages/RustCompiler.scala | 43 ++++++++++--------- .../struct/translators/RustTranslator.scala | 15 +++---- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index cf64f6b82..b4c7df353 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -183,6 +183,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s") -> KResult<()> {") out.inc + out.puts(s"*self_rc._io.borrow_mut() = _io.clone();") out.puts(s"self_rc._root.set(_root.get());") out.puts(s"self_rc._parent.set(_parent.get());") out.puts(s"let _new_parent = SharedType::::new(self_rc.clone());") @@ -201,7 +202,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) attrType: DataType, isNullable: Boolean): Unit = { val typeName = attrName match { - case IoIdentifier | RootIdentifier | ParentIdentifier => return + case RootIdentifier | ParentIdentifier => return case _ => kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType) } @@ -213,7 +214,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) attrType: DataType, isNullable: Boolean): Unit = { var typeName = attrName match { - case IoIdentifier | RootIdentifier | ParentIdentifier => return + case RootIdentifier | ParentIdentifier => return case _ => kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType) } @@ -408,8 +409,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def useIO(ioEx: Ast.expr): String = { - out.puts(s"let t = ${expression(ioEx)};") - out.puts(s"let io = BytesReader::new(&*t);") + out.puts(s"let io = Clone::clone(&*${expression(ioEx)});") "io" } @@ -480,8 +480,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val typeName = kaitaiTypeToNativeType( Some(attrName), typeProvider.nowClass, - attrType, - excludeOptionWrapper = true + attrType ) out.puts(s"${calculatedFlagForName(attrName)}: Cell,") out.puts(s"${idToStr(attrName)}: RefCell<$typeName>,") @@ -525,19 +524,18 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) dataType: DataType, isNullable: Boolean): Unit = { in_instance = true - out.puts(s"pub fn ${idToStr(instName)}(") + out.puts(s"pub fn ${idToStr(instName)}(") out.inc - out.puts("&self,") - out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S") + out.puts("&self") out.dec val typeName = kaitaiTypeToNativeType( Some(instName), typeProvider.nowClass, - dataType, - excludeOptionWrapper = true + dataType ) out.puts(s") -> KResult> {") out.inc + out.puts(s"let _io = self._io.borrow();") out.puts(s"let _rrc = self._root.get_value().borrow();") out.puts(s"let _prc = self._parent.get_value().borrow();") out.puts(s"let _r = _rrc.as_ref().unwrap();") @@ -828,22 +826,26 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } s", $root, $parent" } - val streamType = if (io == privateMemberName(IoIdentifier)) { - "S" + var io2 = "" + var streamType = "" + if (io == privateMemberName(IoIdentifier)) { + io2 = s"&*$io" + streamType = "_" } else { - "BytesReader" + io2 = s"$io" + streamType = "BytesReader" } if (addParams.isEmpty) { if (t.classSpec.isDefined) t.classSpec.get.meta.endian match { case Some(InheritedEndian) => out.puts(s"let f = |t : &mut $userType| Ok(t.set_endian(*${privateMemberName(EndianIdentifier)}));") - out.puts(s"let t = Self::read_into_with_init::<$streamType, $userType>($io$addArgs, &f)?.into();") + out.puts(s"let t = Self::read_into_with_init::<$streamType, $userType>($io2$addArgs, &f)?.into();") case _ => - out.puts(s"let t = Self::read_into::<$streamType, $userType>($io$addArgs)?.into();") + out.puts(s"let t = Self::read_into::<$streamType, $userType>($io2$addArgs)?.into();") } } else { out.puts(s"let f = |t : &mut $userType| Ok(t.set_params($addParams));") - out.puts(s"let t = Self::read_into_with_init::<$streamType, $userType>($io$addArgs, &f)?.into();") + out.puts(s"let t = Self::read_into_with_init::<$streamType, $userType>($io2$addArgs, &f)?.into();") } return s"t" case _ => s"// parseExpr($dataType, $assignType, $io, $defEndian)" @@ -968,7 +970,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"let $ids = $newStream;") newStream = ids //} - out.puts(s"let $localIO = BytesReader::new(&$newStream);") + out.puts(s"let $localIO = BytesReader::from($newStream.clone());") s"&$localIO" case _ => val ids = idToStr(id) @@ -977,7 +979,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"let $ids = $newStreamRaw;") newStreamRaw = ids //} - out.puts(s"let $localIO = BytesReader::new(&$newStreamRaw.last().unwrap());") + out.puts(s"let $localIO = BytesReader::from($newStreamRaw.last().unwrap().clone());") s"&$localIO" } @@ -1362,8 +1364,9 @@ object RustCompiler if (excludeOptionWrapper || excludeOptionWrapperAlways) typeName else s"Option<$typeName>" - case KaitaiStreamType => kstreamName + case KaitaiStreamType => "BytesReader" case CalcKaitaiStructType => kstructUnitName + case _ => s"kaitaiTypeToNativeType'${attrType.toString}'???" } def kaitaiPrimitiveToNativeType(attrType: DataType): String = attrType match { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index ea1d1402f..565a224a1 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -74,8 +74,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) val aType = RustCompiler.kaitaiTypeToNativeType(Some(vis.id), provider.nowClass, vis.dataTypeComposite) val refOpt = "^Option<.*".r aType match { - case refOpt() => s"$s(${privateMemberName(IoIdentifier)})?.as_ref().unwrap()" - case _ => s"$s(${privateMemberName(IoIdentifier)})?" + case refOpt() => s"$s()?.as_ref().unwrap()" + case _ => s"$s()?" } case as: AttrSpec => val code = s"$s()" @@ -99,12 +99,12 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _ => s"$s()" } case pis: ParseInstanceSpec => - //s"$s(${privateMemberName(IoIdentifier)})?" + //s"$s()?" val aType = RustCompiler.kaitaiTypeToNativeType(Some(pis.id), provider.nowClass, pis.dataTypeComposite) val refOpt = "^Option<.*".r aType match { - case refOpt() => s"$s(${privateMemberName(IoIdentifier)})?.as_ref().unwrap()" - case _ => s"$s(${privateMemberName(IoIdentifier)})?" + case refOpt() => s"$s()?.as_ref().unwrap()" + case _ => s"$s()?" } case _ => s"$s()" @@ -205,11 +205,6 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) r = s"$t.$a" } } - attrName match { - case Identifier.IO => - r = r.replace("()._io()", "_raw()") - case _ => - } r } From 2f03869843e41714ad980a18e01fb4c4260cf95b Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 31 Jan 2023 12:14:32 +0800 Subject: [PATCH 117/153] code cleanup. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index b4c7df353..e6ab04a25 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -1345,10 +1345,7 @@ object RustCompiler case Some(spec) => s"${types2class(spec.name)}" case None => s"${types2class(t.name)}" } - if (cleanTypename) { - return baseName - } - if (excludeOptionWrapper || excludeOptionWrapperAlways) baseName else s"$baseName" + baseName case t: ArrayType => s"Vec<${kaitaiTypeToNativeType(id, cs, t.elType, excludeOptionWrapper = true)}>" From 9359e8419052825affe93b01ae9f4acdd8fec3c8 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 1 Feb 2023 16:10:08 +0800 Subject: [PATCH 118/153] support .as (doCast) for SwitchType enum (enum <-> variant). --- .../struct/languages/RustCompiler.scala | 38 ++++++++++++++++--- .../struct/translators/RustTranslator.scala | 12 ++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index e6ab04a25..c50be3d50 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -1042,18 +1042,44 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // Because this switch type will itself be in an option, we can exclude it from user types val variantName = switchVariantName(id, t) var v = "v" - val typeName = t match { - case _ : BytesType => - v = "v.to_vec()" - s"&[u8]" // special case for Bytes(Vec[u8]) (else switch) - case _ => kaitaiTypeToNativeType( + var typeName = kaitaiTypeToNativeType( Some(id), typeProvider.nowClass, t, excludeOptionWrapperAlways = true) - } + val new_typename = types_set.add(typeName) if (new_typename) { + // generate helpers to convert enum into variant (let x : Rc = enum1.into()) + if (!enum_only_numeric) { + val asOption = "^Option<.*".r + val suffix = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, t) match { + case asOption() => s".as_ref().unwrap()" + case _ => "" + } + out.puts(s"impl From<&$enum_typeName> for $typeName {") + out.inc + out.puts(s"fn from(v: &$enum_typeName) -> Self {") + out.inc + out.puts(s"if let $enum_typeName::$variantName(x) = v {") + out.inc + out.puts(s"return x$suffix.clone();") + out.dec + out.puts("}") + out.puts(s"""panic!("expected $enum_typeName::$variantName, got {:?}", v)""") + out.dec + out.puts("}") + out.dec + out.puts("}") + } + // special case for Bytes(Vec[u8]) (else switch) + t match { + case _ : BytesType => + v = "v.to_vec()" + typeName = s"&[u8]" + case _ => + } + // generate helpers to create enum from variant (let enum1 = Var1.into()) out.puts(s"impl From<$typeName> for $enum_typeName {") out.inc out.puts(s"fn from(v: $typeName) -> Self {") diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 565a224a1..b2f706b17 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -345,6 +345,18 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"if ${translate(condition)} { ${remove_deref(translate(ifTrue))}$to_type } else { ${remove_deref(translate(ifFalse))}$to_type }" } } + + override def doCast(value: Ast.expr, typeName: DataType): String = { + val t1_type = detectType(value) + val t1 = RustCompiler.kaitaiTypeToNativeType(None, provider.nowClass, t1_type) + val t2 = RustCompiler.kaitaiTypeToNativeType(None, provider.nowClass, typeName) + if (t1 != t2 && t1_type != KaitaiStructType && t1_type != CalcFloatType && t1_type != CalcIntType) { + s"Into::<$t2>::into(&${translate(value)})" + } else { + translate(value) + } + } + override def translate(v: Ast.expr): String = { v match { case Ast.expr.EnumById(enumType, id, inType) => From 2e8fb7b84778ee31f719c3cb00df59bce08a4fca Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 1 Feb 2023 20:02:59 +0800 Subject: [PATCH 119/153] generate helper method with name from variant type only if there is only single variant. --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index c50be3d50..db71d3f15 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -1136,7 +1136,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts } - { + // generate helper method with name from variant type (to convert enum into variant and call variant method inside) + // only if there is only single variant + // if more than 1 - Kaitai will do casting + if (types.size == 1) { val types_set = scala.collection.mutable.Set[String]() val attrs_set = scala.collection.mutable.Set[String]() types.foreach(t => { @@ -1167,7 +1170,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("match self {") out.inc out.puts(s"$enum_typeName::$typeName(x) => $x.$fn.borrow()$clone,") - out.puts("_ => panic!(\"wrong variant: {:?}\", self),") + //out.puts("_ => panic!(\"wrong variant: {:?}\", self),") out.dec out.puts("}") out.dec From ed2019e21aa48f6e92b360eb7ba5811763194274 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 1 Feb 2023 20:03:42 +0800 Subject: [PATCH 120/153] doCast: check types without option always. --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index b2f706b17..72fc826ac 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -348,9 +348,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doCast(value: Ast.expr, typeName: DataType): String = { val t1_type = detectType(value) - val t1 = RustCompiler.kaitaiTypeToNativeType(None, provider.nowClass, t1_type) - val t2 = RustCompiler.kaitaiTypeToNativeType(None, provider.nowClass, typeName) - if (t1 != t2 && t1_type != KaitaiStructType && t1_type != CalcFloatType && t1_type != CalcIntType) { + val t1 = RustCompiler.kaitaiTypeToNativeType(None, provider.nowClass, t1_type, excludeOptionWrapperAlways = true) + val t2 = RustCompiler.kaitaiTypeToNativeType(None, provider.nowClass, typeName, excludeOptionWrapperAlways = true) + if (t1 != t2 && t1_type != CalcFloatType && t1_type != CalcIntType) { s"Into::<$t2>::into(&${translate(value)})" } else { translate(value) From 16da2bf5c6b447b0093955564011d61bcde41389 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Wed, 1 Feb 2023 21:06:13 +0800 Subject: [PATCH 121/153] ensure amp (&) for _io. --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- .../io/kaitai/struct/translators/RustTranslator.scala | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index db71d3f15..0c708f2ff 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -832,7 +832,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) io2 = s"&*$io" streamType = "_" } else { - io2 = s"$io" + io2 = translator.ensure_amp(io) streamType = "BytesReader" } if (addParams.isEmpty) { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 72fc826ac..0261f5b55 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -224,6 +224,14 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } + def ensure_amp(s: String): String = { + if (s.charAt(0) == '&') { + s + } else { + s"&$s" + } + } + def remove_deref(s: String): String = { if (s.charAt(0) == '*') { s.substring(1) From aea84d382a18711b27aac5ac1b60979dad0cf993 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 2 Feb 2023 10:14:20 +0800 Subject: [PATCH 122/153] pub _self introduced. --- .../struct/languages/RustCompiler.scala | 21 ++++++++----------- .../struct/translators/RustTranslator.scala | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 0c708f2ff..bf188fe6d 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -90,6 +90,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) kaitaiTypeToNativeType(None, typeProvider.nowClass, typeProvider.nowClass.parentType, cleanTypename = true) } out.puts(s"pub ${privateMemberName(ParentIdentifier)}: SharedType<$parent>,") + out.puts(s"pub _self: SharedType,") typeProvider.nowClass.params.foreach { p => // Make sure the parameter is imported if necessary @@ -186,9 +187,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"*self_rc._io.borrow_mut() = _io.clone();") out.puts(s"self_rc._root.set(_root.get());") out.puts(s"self_rc._parent.set(_parent.get());") - out.puts(s"let _new_parent = SharedType::::new(self_rc.clone());") - out.puts(s"let _rrc = self_rc._root.get_value().borrow();") - out.puts(s"let _prc = self_rc._parent.get_value().borrow();") + out.puts(s"self_rc._self.set(Ok(self_rc.clone()));"); + + out.puts(s"let _rrc = self_rc._root.get_value().borrow().upgrade();") + out.puts(s"let _prc = self_rc._parent.get_value().borrow().upgrade();") out.puts(s"let _r = _rrc.as_ref().unwrap();") // If there aren't any attributes to parse, we need to end the read implementation here @@ -536,8 +538,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s") -> KResult> {") out.inc out.puts(s"let _io = self._io.borrow();") - out.puts(s"let _rrc = self._root.get_value().borrow();") - out.puts(s"let _prc = self._parent.get_value().borrow();") + out.puts(s"let _rrc = self._root.get_value().borrow().upgrade();") + out.puts(s"let _prc = self._parent.get_value().borrow().upgrade();") out.puts(s"let _r = _rrc.as_ref().unwrap();") } @@ -770,7 +772,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } var translated = translator.translate(a) if (translated == "_r") // _root - translated = "*_rrc" + translated = "_rrc" s"$byref$translated$try_into" }, "", ", ", "") } @@ -813,12 +815,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) var parent = t.forcedParent match { case Some(USER_TYPE_NO_PARENT) => "None" case Some(fp) => translator.translate(fp) - case None => - if (!in_instance) { - s"Some(_new_parent.clone())" - } else { - "None" - } + case None => s"Some(${self_name()}._self.clone())" } t.classSpec.get.parentType match { case CalcKaitaiStructType => parent = "None" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 0261f5b55..b236c0f31 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -188,7 +188,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) val t = translate(value) var a = doName(attrName) attrName match { - case Identifier.PARENT => a = a + ".get_value().borrow().as_ref().unwrap()" + case Identifier.PARENT => a = a + ".get_value().borrow().upgrade().as_ref().unwrap()" case _ => } var r = "" From e140e7e66199b642f1ea083f6e26cac6ea313b15 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 2 Feb 2023 13:56:21 +0800 Subject: [PATCH 123/153] correctly wrap parent type (nav_parent_override.ksy). --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index bf188fe6d..58452bfa7 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -814,7 +814,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } else { var parent = t.forcedParent match { case Some(USER_TYPE_NO_PARENT) => "None" - case Some(fp) => translator.translate(fp) + case Some(fp) => s"Some(SharedType::new(${translator.translate(fp)}.clone()))" case None => s"Some(${self_name()}._self.clone())" } t.classSpec.get.parentType match { From 58f792db983604969de35208d87816c45c190cdb Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 4 Feb 2023 12:57:42 +0800 Subject: [PATCH 124/153] cast bugs fixed. --- .../struct/languages/RustCompiler.scala | 5 ++- .../struct/translators/RustTranslator.scala | 40 +++++++++++++------ 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 58452bfa7..5bc177347 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -773,7 +773,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) var translated = translator.translate(a) if (translated == "_r") // _root translated = "_rrc" - s"$byref$translated$try_into" + if (try_into.nonEmpty) + s"$byref($translated)$try_into" + else + s"$byref$translated" }, "", ", ", "") } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index b236c0f31..a14e76426 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -55,12 +55,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) if (isSignedIntType(lt) && isSignedIntType(rt) && op == Ast.operator.Mod) s"modulo(${translate(left)} as i64, ${translate(right)} as i64)" else { - if (lt != rt) { - val ct = RustCompiler.kaitaiPrimitiveToNativeType(TypeDetector.combineTypes(lt, rt)) - s"((${translate(left)} as $ct) ${binOp(op)} (${translate(right)} as $ct))" - } else { - super.numericBinOp(left, op, right) - } + val ct = RustCompiler.kaitaiPrimitiveToNativeType(TypeDetector.combineTypes(lt, rt)) + s"((${translate(left)} as $ct) ${binOp(op)} (${translate(right)} as $ct))" } } @@ -323,6 +319,17 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = s"${RustCompiler.types2class(enumTypeAbs)}::${Utils.upperCamelCase(label)}" + override def doNumericCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { + val lt = detectType(left) + val rt = detectType(right) + if (lt != rt) { + val ct = RustCompiler.kaitaiPrimitiveToNativeType(TypeDetector.combineTypes(lt, rt)) + s"((${translate(left)} as $ct) ${cmpOp(op)} (${translate(right)} as $ct))" + } else { + s"${translate(left)} ${cmpOp(op)} ${translate(right)}" + } + } + override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { // val l = translate(left) // val r = translate(right) @@ -354,14 +361,21 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } - override def doCast(value: Ast.expr, typeName: DataType): String = { - val t1_type = detectType(value) - val t1 = RustCompiler.kaitaiTypeToNativeType(None, provider.nowClass, t1_type, excludeOptionWrapperAlways = true) - val t2 = RustCompiler.kaitaiTypeToNativeType(None, provider.nowClass, typeName, excludeOptionWrapperAlways = true) - if (t1 != t2 && t1_type != CalcFloatType && t1_type != CalcIntType) { - s"Into::<$t2>::into(&${translate(value)})" + override def doCast(value: Ast.expr, castTypeName: DataType): String = { + val value_type = detectType(value) + if(castTypeName == value_type) + return translate(value) + + val ct = RustCompiler.kaitaiTypeToNativeType(None, provider.nowClass, castTypeName, excludeOptionWrapperAlways = true) + var into = false + castTypeName match { + case _: UserType => into = true; + case _ => + } + if (into) { + s"Into::<$ct>::into(&${translate(value)})" } else { - translate(value) + s"(${translate(value)} as $ct)" } } From 4986ddefdc13f270b33b95bf468be6a42005ed93 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 4 Feb 2023 21:52:43 +0800 Subject: [PATCH 125/153] let rust decide final type for simple integers. --- .../struct/translators/RustTranslator.scala | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index a14e76426..4b89dd699 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -46,17 +46,26 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _ => false } + def isAllDigits(x: String) = x forall Character.isDigit + override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr): String = { val lt = detectType(left) val rt = detectType(right) + val tl = translate(left) + val tr = translate(right) if (isSignedIntType(lt) && isSignedIntType(rt) && op == Ast.operator.Mod) - s"modulo(${translate(left)} as i64, ${translate(right)} as i64)" + s"modulo($tl as i64, $tr as i64)" else { - val ct = RustCompiler.kaitaiPrimitiveToNativeType(TypeDetector.combineTypes(lt, rt)) - s"((${translate(left)} as $ct) ${binOp(op)} (${translate(right)} as $ct))" + if (lt == rt && isAllDigits(tl) && isAllDigits(tr)) { + // let rust decide final type + s"($tl ${binOp(op)} $tr)" + } else { + val ct = RustCompiler.kaitaiPrimitiveToNativeType(TypeDetector.combineTypes(lt, rt)) + s"(($tl as $ct) ${binOp(op)} ($tr as $ct))" + } } } @@ -370,6 +379,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) var into = false castTypeName match { case _: UserType => into = true; + case CalcBytesType => into = true; case _ => } if (into) { From a11a38b1e266a1fbebe5e40c3b20ec3d8c3285ec Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sun, 5 Feb 2023 17:17:09 +0800 Subject: [PATCH 126/153] Do logical right shift instead of arithmetic right shift. --- .../io/kaitai/struct/translators/RustTranslator.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 4b89dd699..c033cdd3e 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -56,9 +56,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) val tl = translate(left) val tr = translate(right) - if (isSignedIntType(lt) && isSignedIntType(rt) && op == Ast.operator.Mod) + if (isSignedIntType(lt) && isSignedIntType(rt) && op == Ast.operator.Mod) { s"modulo($tl as i64, $tr as i64)" - else { + } else if (isSignedIntType(lt) && isSignedIntType(rt) && op == Ast.operator.RShift) { + // Arithmetic right shift on signed integer types, logical right shift on unsigned integer types + val ct = RustCompiler.kaitaiPrimitiveToNativeType(TypeDetector.combineTypes(lt, rt)) + s"((($tl as u64) >> $tr) as $ct)" + } else { if (lt == rt && isAllDigits(tl) && isAllDigits(tr)) { // let rust decide final type s"($tl ${binOp(op)} $tr)" From 86071c8f685706c7379d01c3236086bc9e747bf4 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sun, 5 Feb 2023 18:02:21 +0800 Subject: [PATCH 127/153] debug info removed. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 5bc177347..13f996706 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -1392,7 +1392,6 @@ object RustCompiler case KaitaiStreamType => "BytesReader" case CalcKaitaiStructType => kstructUnitName - case _ => s"kaitaiTypeToNativeType'${attrType.toString}'???" } def kaitaiPrimitiveToNativeType(attrType: DataType): String = attrType match { @@ -1419,6 +1418,5 @@ object RustCompiler case _: BytesType => "Vec" case ArrayTypeInStream(inType) => s"Vec<${kaitaiPrimitiveToNativeType(inType)}>" - case _ => s"kaitaiPrimitiveToNativeType '${attrType.toString}' ???" } } From c71120af9dbeda419a955384807fb34a8b2f1f89 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 6 Feb 2023 22:27:18 +0800 Subject: [PATCH 128/153] life params cleaned up. --- .../struct/languages/RustCompiler.scala | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 13f996706..d11f9d438 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -77,10 +77,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"pub struct ${classTypeName(typeProvider.nowClass)} {") out.inc - // Because we can't predict whether opaque types will need lifetimes as a type parameter, - // everyone gets a phantom data marker - //out.puts(s"_phantom: std::marker::PhantomData<&$streamLife ()>,") - val root = types2class(name.slice(0, 1)) out.puts(s"pub ${privateMemberName(RootIdentifier)}: SharedType<$root>,") @@ -138,7 +134,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) out.puts( - s"impl<$readLife, $streamLife: $readLife> $kstructName<$readLife, $streamLife> for ${classTypeName(typeProvider.nowClass)} {" + s"impl $kstructName for ${classTypeName(typeProvider.nowClass)} {" ) out.inc val root = classTypeName(typeProvider.topClass) @@ -173,7 +169,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"fn read(") out.inc out.puts(s"self_rc: &Rc,") - out.puts(s"${privateMemberName(IoIdentifier)}: &$streamLife S,") + out.puts(s"${privateMemberName(IoIdentifier)}: &S,") out.puts( s"$root: SharedType," ) @@ -223,7 +219,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) //val typeNameEx = kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType, excludeOptionWrapper = true) out.puts( - s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") + s"impl ${classTypeName(typeProvider.nowClass)} {") out.inc var types : Set[DataType] = Set() @@ -443,7 +439,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$n: $byref$t" }, "", ", ", "") - out.puts(s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") + out.puts(s"impl ${classTypeName(typeProvider.nowClass)} {") out.inc out.puts(s"pub fn set_params(&mut self, $paramsArg) {") out.inc @@ -455,7 +451,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } typeProvider.nowClass.meta.endian match { case Some(_: CalcEndian) | Some(InheritedEndian) => - out.puts(s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") + out.puts(s"impl ${classTypeName(typeProvider.nowClass)} {") out.inc val t = kaitaiTypeToNativeType(Some(EndianIdentifier), typeProvider.nowClass, IntMultiType(signed = true, Width4, None), excludeOptionWrapper = true) out.puts(s"pub fn set_endian(&mut self, ${idToStr(EndianIdentifier)}: $t) {") @@ -467,7 +463,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") case _ => } - out.puts(s"impl<$readLife, $streamLife: $readLife> ${classTypeName(typeProvider.nowClass)} {") + out.puts(s"impl ${classTypeName(typeProvider.nowClass)} {") out.inc } @@ -1333,15 +1329,11 @@ object RustCompiler override def kstructName = s"KStruct" - def readLife = "'r" - def kstructUnitName = "KStructUnit" def classTypeName(c: ClassSpec): String = s"${types2class(c.name)}" - def streamLife = "'s" - def types2class(names: List[String]): String = // TODO: Use `mod` to scope types instead of weird names names.map(x => type2class(x)).mkString("_") From ed64b29d42f9ce26b48953a7d5e63b020d85bcee Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Thu, 9 Feb 2023 16:09:13 +0800 Subject: [PATCH 129/153] generate code that use OptRc. --- .../struct/languages/RustCompiler.scala | 39 ++++++------------- .../struct/translators/RustTranslator.scala | 2 +- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index d11f9d438..13f62ae16 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -168,7 +168,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val root = privateMemberName(RootIdentifier) out.puts(s"fn read(") out.inc - out.puts(s"self_rc: &Rc,") + out.puts(s"self_rc: &OptRc,") out.puts(s"${privateMemberName(IoIdentifier)}: &S,") out.puts( s"$root: SharedType," @@ -693,15 +693,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val seqId = translator.findMember(idToStr(id)) var done = false var refcell = false - var opaque = false if (seqId.isDefined) { val idType = seqId.get.dataType idType match { case t: UserType => refcell = true - if (t.isOpaque) { - opaque = true - } case _: BytesType => refcell = true case _: ArrayType => refcell = true case _: StrType => refcell = true @@ -717,7 +713,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } if (refcell) { val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, idType) - if (opaque || typeName.startsWith("Option<")) { + if (typeName.startsWith("Option<")) { out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = Some($expr);") } else { out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = $expr;") @@ -768,7 +764,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } var translated = translator.translate(a) if (translated == "_r") // _root - translated = "_rrc" + translated = "OptRc::new(&_rrc)" if (try_into.nonEmpty) s"$byref($translated)$try_into" else @@ -982,11 +978,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ioName } - def if_opaque(dt: DataType): Boolean = dt match { - case t: UserType => if (t.isOpaque) true else false - case _ => false - } - def switchTypeEnum(id: Identifier, st: SwitchType): Unit = { // Because Rust can't handle `AnyType` in the type hierarchy, // we generate an enum with all possible variations @@ -1042,7 +1033,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) Some(id), typeProvider.nowClass, t, - excludeOptionWrapperAlways = true) + excludeOptionWrapper = true) val new_typename = types_set.add(typeName) if (new_typename) { @@ -1080,11 +1071,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc out.puts(s"fn from(v: $typeName) -> Self {") out.inc - if (if_opaque(t)) { - out.puts(s"Self::$variantName(Some($v))") - } else { - out.puts(s"Self::$variantName($v)") - } + out.puts(s"Self::$variantName($v)") out.dec out.puts("}") out.dec @@ -1155,8 +1142,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val typeNameEx = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, t) val x = if (typeNameEx.startsWith("Option<")) "x.as_ref().unwrap()" else "x" var clone = "" - val rc_typename = nativeTypeEx.startsWith("Rc<") - if (rc_typename) { + if (nativeTypeEx.startsWith("OptRc<")) { nativeType = s"$nativeTypeEx" clone = ".clone()" } else @@ -1342,7 +1328,6 @@ object RustCompiler cs: ClassSpec, attrType: DataType, excludeOptionWrapper: Boolean = false, - excludeOptionWrapperAlways: Boolean = false, cleanTypename: Boolean = false): String = attrType match { // TODO: Not exhaustive @@ -1354,12 +1339,10 @@ object RustCompiler case Some(spec) => types2class(spec.name) case None => types2class(t.name) } - - (t.isOpaque, cleanTypename) match { - case (_, true) => baseName - case (true, _) => if (excludeOptionWrapperAlways) s"Rc<$baseName>" else s"Option>" - case (false, _) => s"Rc<$baseName>" - } + if (cleanTypename) + baseName + else + s"OptRc<$baseName>" case t: EnumType => val baseName = t.enumSpec match { @@ -1380,7 +1363,7 @@ object RustCompiler case _ => kstructUnitName } - if (excludeOptionWrapper || excludeOptionWrapperAlways) typeName else s"Option<$typeName>" + if (excludeOptionWrapper) typeName else s"Option<$typeName>" case KaitaiStreamType => "BytesReader" case CalcKaitaiStructType => kstructUnitName diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index c033cdd3e..f80605050 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -379,7 +379,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) if(castTypeName == value_type) return translate(value) - val ct = RustCompiler.kaitaiTypeToNativeType(None, provider.nowClass, castTypeName, excludeOptionWrapperAlways = true) + val ct = RustCompiler.kaitaiTypeToNativeType(None, provider.nowClass, castTypeName, excludeOptionWrapper = true) var into = false castTypeName match { case _: UserType => into = true; From f15094c47e7c4a9cc06d7b7f2d359d75dcff5394 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 6 Mar 2023 09:55:03 +0700 Subject: [PATCH 130/153] code cleanup. --- .../io/kaitai/struct/RustClassCompiler.scala | 6 +-- .../struct/languages/RustCompiler.scala | 6 +-- .../struct/translators/RustTranslator.scala | 48 +++++++------------ 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala index 771c90d7d..2e3b8b128 100644 --- a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala @@ -51,7 +51,7 @@ class RustClassCompiler( compileSubclasses(curClass) } - def compileReadFunction(curClass: ClassSpec) = { + def compileReadFunction(curClass: ClassSpec): Unit = { lang.classConstructorHeader( curClass.name, curClass.parentType, @@ -65,7 +65,7 @@ class RustClassCompiler( case _ => None } - lang.readHeader(defEndian, false) + lang.readHeader(defEndian, isEmpty = false) curClass.meta.endian match { case Some(ce: CalcEndian) => compileCalcEndian(ce) @@ -89,7 +89,7 @@ class RustClassCompiler( lang.runReadCalc() } - override def compileInstances(curClass: ClassSpec) = { + override def compileInstances(curClass: ClassSpec): Unit = { lang.instanceDeclHeader(curClass.name) curClass.instances.foreach { case (instName, instSpec) => compileInstance(curClass.name, instName, instSpec, curClass.meta.endian) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 13f62ae16..4a92e657a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -183,7 +183,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"*self_rc._io.borrow_mut() = _io.clone();") out.puts(s"self_rc._root.set(_root.get());") out.puts(s"self_rc._parent.set(_parent.get());") - out.puts(s"self_rc._self.set(Ok(self_rc.clone()));"); + out.puts(s"self_rc._self.set(Ok(self_rc.clone()));") out.puts(s"let _rrc = self_rc._root.get_value().borrow().upgrade();") out.puts(s"let _prc = self_rc._parent.get_value().borrow().upgrade();") @@ -398,7 +398,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add(s"""#[path = "$mod_name.rs"] mod $mod_name;""") importList.add(s"use self::$mod_name::*;") - val argList = translate_args(args, false) + val argList = translate_args(args, into = false) val argListInParens = s"($argList)" out.puts(s"let $procName = $procClass::new$argListInParens;") s"$procName.decode(&$srcExpr)" @@ -794,7 +794,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case BitsType1(bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}(1)? != 0" case BitsType(width: Int, bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}($width)?" case t: UserType => - addParams = translate_args(t.args, true) + addParams = translate_args(t.args, into = true) val userType = t match { case t: UserType => val baseName = t.classSpec match { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index f80605050..42407a87f 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -73,46 +73,41 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } + def unwrap(s: String): String = s + ".as_ref().unwrap()" + override def doName(s: String): String = s match { case Identifier.PARENT => s case _ => + val refOpt = "^Option<.*".r val memberFound = findMember(s) if (memberFound.isDefined) memberFound.get match { case vis: ValueInstanceSpec => val aType = RustCompiler.kaitaiTypeToNativeType(Some(vis.id), provider.nowClass, vis.dataTypeComposite) - val refOpt = "^Option<.*".r aType match { - case refOpt() => s"$s()?.as_ref().unwrap()" + case refOpt() => unwrap(s"$s()?") case _ => s"$s()?" } case as: AttrSpec => val code = s"$s()" val aType = RustCompiler.kaitaiTypeToNativeType(Some(as.id), provider.nowClass, as.dataTypeComposite) - //val refOpt = "Option<[^>]+>$".r - val refOpt = "^Option<.*".r aType match { - //case "String" => s"$code.as_str()" - //case "Vec" => s"$code.as_slice()" case refOpt() => if (!enum_numeric_only(as.dataTypeComposite)) { - s"$code.as_ref().unwrap()" + unwrap(code) } else code case _ => code } case pd: ParamDefSpec => val aType = RustCompiler.kaitaiTypeToNativeType(Some(pd.id), provider.nowClass, pd.dataTypeComposite) - val refOpt = "^Option<.*".r aType match { - case refOpt() => s"$s().as_ref().unwrap()" + case refOpt() => unwrap(s"$s()") case _ => s"$s()" } case pis: ParseInstanceSpec => - //s"$s()?" val aType = RustCompiler.kaitaiTypeToNativeType(Some(pis.id), provider.nowClass, pis.dataTypeComposite) - val refOpt = "^Option<.*".r aType match { - case refOpt() => s"$s()?.as_ref().unwrap()" + case refOpt() => unwrap(s"$s()?") case _ => s"$s()?" } case _ => @@ -197,7 +192,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) val t = translate(value) var a = doName(attrName) attrName match { - case Identifier.PARENT => a = a + ".get_value().borrow().upgrade().as_ref().unwrap()" + case Identifier.PARENT => a = a + unwrap(".get_value().borrow().upgrade()") case _ => } var r = "" @@ -257,8 +252,6 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } - var context_need_deref_attr = false - def enum_numeric_only(dataType: DataType): Boolean = { var types : Set[DataType] = Set() var enum_typename = false @@ -308,23 +301,19 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case Identifier.ITERATOR2 => "_tmpb" case Identifier.INDEX => "_i" case Identifier.IO => s"${RustCompiler.privateMemberName(IoIdentifier)}" - case Identifier.ROOT => s"_r" - case Identifier.PARENT => s"_prc.as_ref().unwrap()" + case Identifier.ROOT => "_r" + case Identifier.PARENT => unwrap("_prc") case _ => val n = doName(s) val deref = !n.endsWith(".as_str()") && !n.endsWith(".as_slice()") && need_deref(s) - if (context_need_deref_attr || deref) { + if (deref) { s"*${self_name()}.$n" } else { s"${self_name()}.$n" } } - override def doEnumCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { - //context_need_deref_attr = true - val code = s"${translate(left)} ${cmpOp(op)} ${translate(right)}" - //context_need_deref_attr = false - code - } + override def doEnumCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = + s"${translate(left)} ${cmpOp(op)} ${translate(right)}" override def doInternalName(id: Identifier): String = s"${doLocalName(idToStr(id))}" @@ -343,13 +332,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } - override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { - // val l = translate(left) - // val r = translate(right) - // val asStr = if (l.endsWith(".as_str()") && r.endsWith(")?")) ".as_str()" else "" - // s"$l ${cmpOp(op)} $r$asStr" + override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = s"${ensure_deref(translate(left))} ${cmpOp(op)} ${remove_deref(translate(right))}.to_string()" - } override def doEnumById(enumTypeAbs: List[String], id: String): String = s"($id as i64).try_into()?" @@ -478,13 +462,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) detectType(a) match { case t: CalcArrayType => t.elType match { - case f: FloatMultiType => true + case _: FloatMultiType => true case CalcFloatType => true case _ => false } case t: ArrayType => t.elType match { - case f: FloatMultiType => true + case _: FloatMultiType => true case _ => false } case _ => false From 05082224e6b6c0dfa1d035506a4ce6f7b5df6a8d Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 10 Mar 2023 10:24:48 +0700 Subject: [PATCH 131/153] unused code cleanup. --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 8 -------- 1 file changed, 8 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 4a92e657a..46eae3390 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -521,7 +521,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { - in_instance = true out.puts(s"pub fn ${idToStr(instName)}(") out.inc out.puts("&self") @@ -548,8 +547,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - var in_instance = false - override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { dataType match { case _: UserType => @@ -570,7 +567,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { out.puts(s"Ok(${privateMemberName(instName)})") - in_instance = false } override def enumDeclaration(curClass: List[String], @@ -957,20 +953,16 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case NoRepeat => var newStream = newStreamRaw val localIO = localTemporaryName(ioId) - //if (in_instance) { val ids = idToStr(id) out.puts(s"let $ids = $newStream;") newStream = ids - //} out.puts(s"let $localIO = BytesReader::from($newStream.clone());") s"&$localIO" case _ => val ids = idToStr(id) val localIO = s"io_$ids" - //if (in_instance) { out.puts(s"let $ids = $newStreamRaw;") newStreamRaw = ids - //} out.puts(s"let $localIO = BytesReader::from($newStreamRaw.last().unwrap().clone());") s"&$localIO" } From 27d84ea1993a83ced039bf22467d0bee5f764192 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 10 Mar 2023 13:06:55 +0700 Subject: [PATCH 132/153] no more rustc bug here. Gen better for reading code. --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 46eae3390..73a61dc2c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -773,7 +773,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) io: String, defEndian: Option[FixedEndian]): String = { var addParams = "" - val expr = dataType match { + dataType match { case t: ReadableType => t match { case IntMultiType(_, _, None) => @@ -838,12 +838,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) return s"t" case _ => s"// parseExpr($dataType, $assignType, $io, $defEndian)" } - // in expr_2.ksy we got into rustc bug - // https://github.com/rust-lang/rust/issues/82656 - // https://github.com/rust-lang/rust/issues/70919 - // workaround: - out.puts(s"let t = $expr;") - s"t" } override def bytesPadTermExpr(expr0: String, From 31c904c97e76cde8915297cf5ff120ec463b4c37 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 10 Mar 2023 13:07:21 +0700 Subject: [PATCH 133/153] code cleanup. --- .../struct/translators/RustTranslator.scala | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 42407a87f..a560769dd 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -80,43 +80,44 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _ => val refOpt = "^Option<.*".r val memberFound = findMember(s) - if (memberFound.isDefined) + if (memberFound.isDefined) { + val f = s"$s()" memberFound.get match { case vis: ValueInstanceSpec => val aType = RustCompiler.kaitaiTypeToNativeType(Some(vis.id), provider.nowClass, vis.dataTypeComposite) aType match { - case refOpt() => unwrap(s"$s()?") - case _ => s"$s()?" + case refOpt() => unwrap(s"$f?") + case _ => s"$f?" } case as: AttrSpec => - val code = s"$s()" val aType = RustCompiler.kaitaiTypeToNativeType(Some(as.id), provider.nowClass, as.dataTypeComposite) aType match { case refOpt() => if (!enum_numeric_only(as.dataTypeComposite)) { - unwrap(code) - } else code - case _ => code + unwrap(f) + } else f + case _ => f } case pd: ParamDefSpec => val aType = RustCompiler.kaitaiTypeToNativeType(Some(pd.id), provider.nowClass, pd.dataTypeComposite) aType match { - case refOpt() => unwrap(s"$s()") - case _ => s"$s()" + case refOpt() => unwrap(s"$f") + case _ => s"$f" } case pis: ParseInstanceSpec => val aType = RustCompiler.kaitaiTypeToNativeType(Some(pis.id), provider.nowClass, pis.dataTypeComposite) aType match { - case refOpt() => unwrap(s"$s()?") - case _ => s"$s()?" + case refOpt() => unwrap(s"$f?") + case _ => s"$f?" } case _ => - s"$s()" + s"$f" } + } else { s"$s()" } - } + } def findMember(attrName: String, c: ClassSpec = provider.nowClass): Option[MemberSpec] = { def findInClass(inClass: ClassSpec): Option[MemberSpec] = { @@ -146,9 +147,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) None } - val attr = attrName match { + attrName match { case Identifier.PARENT | Identifier.IO => - None + return None case _ => for { ms <- findInClass(c) } return Some(ms) @@ -157,9 +158,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) for { ms <- findInClass(cls._2) } return Some(ms) } - None } - attr + None } def get_top_class(c: ClassSpec): ClassSpec = c.upClass match { From a043923459611d5daee632941cdb2063525f1111 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 10 Mar 2023 14:08:22 +0700 Subject: [PATCH 134/153] code cleanup. --- .../io/kaitai/struct/translators/RustTranslator.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index a560769dd..b7101f3cf 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -80,8 +80,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _ => val refOpt = "^Option<.*".r val memberFound = findMember(s) + val f = s"$s()" if (memberFound.isDefined) { - val f = s"$s()" memberFound.get match { case vis: ValueInstanceSpec => val aType = RustCompiler.kaitaiTypeToNativeType(Some(vis.id), provider.nowClass, vis.dataTypeComposite) @@ -101,8 +101,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case pd: ParamDefSpec => val aType = RustCompiler.kaitaiTypeToNativeType(Some(pd.id), provider.nowClass, pd.dataTypeComposite) aType match { - case refOpt() => unwrap(s"$f") - case _ => s"$f" + case refOpt() => unwrap(f) + case _ => f } case pis: ParseInstanceSpec => val aType = RustCompiler.kaitaiTypeToNativeType(Some(pis.id), provider.nowClass, pis.dataTypeComposite) @@ -111,11 +111,11 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case _ => s"$f?" } case _ => - s"$f" + f } } else { - s"$s()" + f } } From 6eca29923a9faca6cd2dfdc49a0c596f8079902e Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 10 Mar 2023 15:26:30 +0700 Subject: [PATCH 135/153] dead code removed. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 73a61dc2c..a565ec304 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -217,7 +217,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType) } - //val typeNameEx = kaitaiTypeToNativeType(Some(attrName), typeProvider.nowClass, attrType, excludeOptionWrapper = true) out.puts( s"impl ${classTypeName(typeProvider.nowClass)} {") out.inc @@ -673,7 +672,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (paramId.isDefined) { need_clone = !translator.is_copy_type(paramId.get.dataType) } - //val typeName = kaitaiTypeToNativeType(Some(id), typeProvider.nowClass, paramId.get.dataType) paramId.get.dataType match { case _: EnumType => out.puts(s"*${RustCompiler.privateMemberName(id, writeAccess = true)} = $expr.clone();") From 8400eb877a8dd4636bb785f70031357ad6988c79 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 10 Mar 2023 15:28:41 +0700 Subject: [PATCH 136/153] useless comment removed. --- .../main/scala/io/kaitai/struct/translators/RustTranslator.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index b7101f3cf..63976c6d8 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -399,7 +399,6 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } - // Predefined methods of various types override def strConcat(left: Ast.expr, right: Ast.expr): String = s"""format!("{}{}", ${translate(left)}, ${translate(right)})""" From 5e342f0503426ce3954add87882b7ac9229bf850 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 11 Mar 2023 12:39:06 +0700 Subject: [PATCH 137/153] unused func removed. --- .../scala/io/kaitai/struct/translators/RustTranslator.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 63976c6d8..27f19f31d 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -162,11 +162,6 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) None } - def get_top_class(c: ClassSpec): ClassSpec = c.upClass match { - case Some(upClass) => get_top_class(upClass) - case None => c - } - def get_instance(cs: ClassSpec, s: String): Option[InstanceSpec] = { var found : Option[InstanceSpec] = None // look for instance From 75683ce60589f996644cc3a2d4ef4e4f80265e93 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 11 Mar 2023 13:28:25 +0700 Subject: [PATCH 138/153] Revert "out full path of generated file" This reverts commit 0edd9c05a92e4b218c7540490ce096f8b7619b01. --- jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala index 393e2f7b0..210b556ac 100644 --- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala +++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala @@ -364,7 +364,7 @@ class JavaMain(config: CLIConfig) { ): SpecSuccess = { val res = Main.compile(specs, spec, lang, runtime) res.files.foreach { (file) => - Log.fileOps.info(() => s".... writing $outDir/${file.fileName}") + Log.fileOps.info(() => s".... writing ${file.fileName}") val outPath = new File(outDir + "/" + file.fileName) From 619f380dbc7fd82587bf97cd5a8d753679c8eb85 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 11 Mar 2023 13:31:47 +0700 Subject: [PATCH 139/153] reverted back to 1.7.1 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 9a19778c3..d738b858c 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.8.0 +sbt.version = 1.7.1 From 8e230f33938e2ab3e2412125fd61b3372306fb99 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 11 Mar 2023 13:34:01 +0700 Subject: [PATCH 140/153] Revert "add field `allClasses` to publish `classSpecs`" This reverts commit 426d3b28b2d6fb1c69c717e71a49984ae3f03552. --- .../src/main/scala/io/kaitai/struct/ClassTypeProvider.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala index 5c9a153b6..abed472ff 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala @@ -8,8 +8,7 @@ import io.kaitai.struct.precompile.{EnumNotFoundError, FieldNotFoundError, TypeN import io.kaitai.struct.translators.TypeProvider class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends TypeProvider { - var nowClass: ClassSpec = topClass - val allClasses: ClassSpecs = classSpecs + var nowClass = topClass var _currentIteratorType: Option[DataType] = None var _currentSwitchType: Option[DataType] = None @@ -68,6 +67,7 @@ class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends * @param inClass type specification to search member in * @param attrName name of a member to search for * @return member spec if found, or throws an exception + * @throws format.InvalidIdentifier if attribute name is not a valid name for a member * @throws precompile.FieldNotFoundError if attribute with such name is not found */ def resolveMember(inClass: ClassSpec, attrName: String): MemberSpec = { From 581dcf7ca630b3d61f03b1ba86d493912b43471f Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 11 Mar 2023 13:40:02 +0700 Subject: [PATCH 141/153] Revert "Report details if NotImplementedError." This reverts commit 5f301fd670fa029714999d10ddb6267d9ac5d250. --- .../main/scala/io/kaitai/struct/Main.scala | 46 ++++++++----------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index 511b8003a..02393be76 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -88,35 +88,25 @@ object Main { * @return a container that contains all compiled files and results */ def compile(specs: ClassSpecs, spec: ClassSpec, lang: LanguageCompilerStatic, conf: RuntimeConfig): CompileLog.SpecSuccess = { - val config = updateConfigFromMeta(conf, spec.meta) - try { - val cc = lang match { - case GraphvizClassCompiler => - new GraphvizClassCompiler(specs, spec) - case GoCompiler => - new GoClassCompiler(specs, spec, config) - case RustCompiler => - new RustClassCompiler(specs, spec, config) - case ConstructClassCompiler => - new ConstructClassCompiler(specs, spec) - case NimCompiler => - new NimClassCompiler(specs, spec, config) - case HtmlClassCompiler => - new HtmlClassCompiler(specs, spec) - case _ => - new ClassCompiler(specs, spec, config, lang) - } - cc.compile - } catch { - case e: NotImplementedError => { - e.printStackTrace - throw e - } - case e: Throwable => { - e.printStackTrace - throw e - } + val config = updateConfig(conf, spec) + + val cc = lang match { + case GraphvizClassCompiler => + new GraphvizClassCompiler(specs, spec) + case GoCompiler => + new GoClassCompiler(specs, spec, config) + case RustCompiler => + new RustClassCompiler(specs, spec, config) + case ConstructClassCompiler => + new ConstructClassCompiler(specs, spec) + case NimCompiler => + new NimClassCompiler(specs, spec, config) + case HtmlClassCompiler => + new HtmlClassCompiler(specs, spec) + case _ => + new ClassCompiler(specs, spec, config, lang) } + cc.compile } /** From 30f552f98ae7a676293986529f65bbd9f0c0e97f Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 11 Mar 2023 13:52:11 +0700 Subject: [PATCH 142/153] code reverted. --- shared/src/main/scala/io/kaitai/struct/Main.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index 02393be76..d1e30294d 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -88,7 +88,7 @@ object Main { * @return a container that contains all compiled files and results */ def compile(specs: ClassSpecs, spec: ClassSpec, lang: LanguageCompilerStatic, conf: RuntimeConfig): CompileLog.SpecSuccess = { - val config = updateConfig(conf, spec) + val config = updateConfigFromMeta(conf, spec.meta) val cc = lang match { case GraphvizClassCompiler => From 6b9aba63338ba59a6beba9af7c41097ef1ccf551 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sat, 11 Mar 2023 15:43:39 +0700 Subject: [PATCH 143/153] added new field `allClasses` to publish classSpecs Using in RustCompiler & RustTranslator. --- shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala index abed472ff..0dee55827 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala @@ -9,6 +9,7 @@ import io.kaitai.struct.translators.TypeProvider class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends TypeProvider { var nowClass = topClass + val allClasses: ClassSpecs = classSpecs var _currentIteratorType: Option[DataType] = None var _currentSwitchType: Option[DataType] = None From cd83719fef3e73fae557a3b8117346e2e93a4ea7 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 13 Mar 2023 11:20:53 +0700 Subject: [PATCH 144/153] space removed. --- shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala index 2e3b8b128..8be1aec4b 100644 --- a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala @@ -30,7 +30,7 @@ class RustClassCompiler( // Basic struct declaration lang.classHeader(curClass.name) - + compileAttrDeclarations(curClass.seq ++ extraAttrs) curClass.instances.foreach { case (instName, instSpec) => compileInstanceDeclaration(instName, instSpec) From b56c33d80623635a538b9e1c1c9d209b56d4d171 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 13 Mar 2023 13:57:58 +0700 Subject: [PATCH 145/153] Rust lang added. --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 35f1f5bae..812695f7b 100644 --- a/build.sbt +++ b/build.sbt @@ -9,7 +9,7 @@ resolvers ++= Resolver.sonatypeOssRepos("public") val NAME = "kaitai-struct-compiler" val VERSION = "0.11-SNAPSHOT" -val TARGET_LANGS = "C++/STL, C#, Go, Java, JavaScript, Lua, Nim, Perl, PHP, Python, Ruby" +val TARGET_LANGS = "C++/STL, C#, Go, Java, JavaScript, Lua, Nim, Perl, PHP, Python, Ruby, Rust" val UTF8 = Charset.forName("UTF-8") lazy val root = project.in(file(".")). From c85ec1e08cb95b6200bbbabdd23291a66b703189 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Sun, 26 Mar 2023 09:18:13 +0700 Subject: [PATCH 146/153] sync changes with runtime/rust. --- .../kaitai/struct/languages/RustCompiler.scala | 18 ++++++++---------- .../struct/translators/RustTranslator.scala | 12 +++++------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index a565ec304..014378c1a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -376,19 +376,19 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case ProcessXor(xorValue) => translator.detectType(xorValue) match { case _: IntType => - s"S::process_xor_one(&$srcExpr, ${expression(xorValue)})" + s"BytesReader::process_xor_one(&$srcExpr, ${expression(xorValue)})" case _: BytesType => - s"S::process_xor_many(&$srcExpr, &${translator.remove_deref(expression(xorValue))})" + s"BytesReader::process_xor_many(&$srcExpr, &${translator.remove_deref(expression(xorValue))})" } case ProcessZlib => - s"S::process_zlib(&$srcExpr)" + s"BytesReader::process_zlib(&$srcExpr)" case ProcessRotate(isLeft, rotValue) => val expr = if (isLeft) { expression(rotValue) } else { s"8 - (${expression(rotValue)})" } - s"S::process_rotate_left(&$srcExpr, $expr)" + s"BytesReader::process_rotate_left(&$srcExpr, $expr)" case ProcessCustom(name, args) => val procClass = name.map(x => type2class(x)).mkString("::") val procName = s"_process_${idToStr(varSrc)}" @@ -844,12 +844,12 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) include: Boolean): String = { val ioId = privateMemberName(IoIdentifier) val expr = padRight match { - case Some(p) => s"$ioId.bytes_strip_right($expr0, $p).into()" + case Some(p) => s"BytesReader::bytes_strip_right(&$expr0, $p).into()" case None => expr0 } terminator match { - case Some(term) => s"$ioId.bytes_terminate($expr, $term, $include).into()" + case Some(term) => s"BytesReader::bytes_terminate(&$expr, $term, $include).into()" case None => expr } } @@ -1012,7 +1012,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) types.foreach(t => { // Because this switch type will itself be in an option, we can exclude it from user types val variantName = switchVariantName(id, t) - var v = "v" var typeName = kaitaiTypeToNativeType( Some(id), typeProvider.nowClass, @@ -1046,8 +1045,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // special case for Bytes(Vec[u8]) (else switch) t match { case _ : BytesType => - v = "v.to_vec()" - typeName = s"&[u8]" + typeName = s"Vec" case _ => } // generate helpers to create enum from variant (let enum1 = Var1.into()) @@ -1055,7 +1053,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc out.puts(s"fn from(v: $typeName) -> Self {") out.inc - out.puts(s"Self::$variantName($v)") + out.puts(s"Self::$variantName(v)") out.dec out.puts("}") out.dec diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 27f19f31d..3095b1f2e 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -423,18 +423,16 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"base_convert(strval(${translate(i)}), 10, $baseStr)" } } - override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = { - if (bytesExpr.charAt(0) == '*') { - s"decode_string(&$bytesExpr, &${translate(encoding)})?" - } else { - s"decode_string(${ensure_vec_amp(bytesExpr)}, &${translate(encoding)})?" - } - } + + override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = + s"decode_string(&$bytesExpr, &${translate(encoding)})?" override def bytesLength(b: Ast.expr): String = s"${remove_deref(translate(b))}.len()" + override def strLength(s: expr): String = s"${remove_deref(translate(s))}.len()" + override def strReverse(s: expr): String = { val e = translate(s) if (e.charAt(0) == '*') From 2bef376284a73419d927c90e994efad39ad0da22 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Tue, 19 Dec 2023 18:46:34 +0700 Subject: [PATCH 147/153] generate clean use-declaration. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 014378c1a..d1a0467a4 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -394,8 +394,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val procName = s"_process_${idToStr(varSrc)}" val mod_name = name.last - importList.add(s"""#[path = "$mod_name.rs"] mod $mod_name;""") - importList.add(s"use self::$mod_name::*;") + importList.add(s"use crate::$mod_name::*;") val argList = translate_args(args, into = false) val argListInParens = s"($argList)" From 9eb7efa1ba664284e43eccc517184ccad6749be9 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Fri, 16 Feb 2024 13:07:05 +0800 Subject: [PATCH 148/153] context-oriented method for searching members. sample: self.groups().value() # look for "value" method inside of class (type) "groups" self.value() # look for "value" in current class --- .../struct/translators/RustTranslator.scala | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 19ecffc82..24d0c2bf4 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -14,6 +14,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) import RustCompiler._ + var lastFoundMemberClass: ClassSpec = provider.nowClass + override def doByteArrayLiteral(arr: Seq[Byte]): String = "vec![" + arr.map(x => "%0#2xu8".format(x & 0xff)).mkString(", ") + "]" override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = @@ -119,22 +121,41 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } } - def findMember(attrName: String, c: ClassSpec = provider.nowClass): Option[MemberSpec] = { + def updateLastFoundMemberClass(dt: DataType) { + if (dt.isInstanceOf[UserType]) { + val s = dt.asInstanceOf[UserType] + if (s.classSpec.isDefined) { + lastFoundMemberClass = s.classSpec.get + } + } + } + + def resetLastFoundMemberClass() { + lastFoundMemberClass = provider.nowClass + } + + def findMember(attrName: String, c: ClassSpec = lastFoundMemberClass): Option[MemberSpec] = { def findInClass(inClass: ClassSpec): Option[MemberSpec] = { inClass.seq.foreach { el => - if (idToStr(el.id) == attrName) + if (idToStr(el.id) == attrName) { + updateLastFoundMemberClass(el.dataType) return Some(el) + } } inClass.params.foreach { el => - if (idToStr(el.id) == attrName) + if (idToStr(el.id) == attrName) { + updateLastFoundMemberClass(el.dataType) return Some(el) + } } inClass.instances.foreach { case (instName, instSpec) => - if (idToStr(instName) == attrName) + if (idToStr(instName) == attrName) { + updateLastFoundMemberClass(instSpec.dataType) return Some(instSpec) + } } inClass.types.foreach{ t => @@ -181,6 +202,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) } override def anyField(value: expr, attrName: String): String = { + resetLastFoundMemberClass() val t = translate(value) var a = doName(attrName) attrName match { @@ -296,6 +318,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) case Identifier.ROOT => "_r" case Identifier.PARENT => unwrap("_prc") case _ => + // reset "looking for variable" context + resetLastFoundMemberClass() val n = doName(s) val deref = !n.endsWith(".as_str()") && !n.endsWith(".as_slice()") && need_deref(s) if (deref) { From f1c697da23f2a7198d92b9e31be1a842cb3e29b1 Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 24 Jun 2024 11:40:57 +0200 Subject: [PATCH 149/153] external type declaration fix. --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index cb5be820b..705786929 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -69,7 +69,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def externalTypeDeclaration(extType: ExternalType): Unit = importList.add( - s"use super::${extType.name.last}::${type2class(extType.name.last)};" + s"use super::${extType.name.head}::${type2class(extType.name.head)};" ) override def classHeader(name: List[String]): Unit = { From 23328785803ba93a8b3bfc33e1e0b998a34df01f Mon Sep 17 00:00:00 2001 From: Oleg Dolgov Date: Mon, 24 Jun 2024 11:41:40 +0200 Subject: [PATCH 150/153] new feature interpolated strings implemented. --- .../kaitai/struct/translators/RustTranslator.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index 75dbc0050..f30fe27af 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -419,6 +419,18 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) override def strConcat(left: Ast.expr, right: Ast.expr, extPrec: Int): String = s"""format!("{}{}", ${translate(left)}, ${translate(right)})""" + override def doInterpolatedStringLiteral(exprs: Seq[Ast.expr]): String = + if (exprs.isEmpty) { + doStringLiteral("") + } else { // format!("{expr1}{expr2}{expr3}") + var s = "format!(\"" + exprs.foreach(i => { s+= "{}" }) + s += "\", " + s += exprs.map(translate).mkString(", ") + s += ")" + s + } + override def strToInt(s: expr, base: expr): String = translate(base) match { case "10" => From 24f04d9e0bdf1fe345e173382224c3737d397cad Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Thu, 9 May 2024 23:15:44 +0200 Subject: [PATCH 151/153] Rust: fix ill-conceived insertion of `use` declarations --- .../scala/io/kaitai/struct/languages/RustCompiler.scala | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 705786929..52f9ff8e4 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -59,17 +59,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add("use std::convert::{TryFrom, TryInto};") importList.add("use std::cell::{Ref, Cell, RefCell};") importList.add("use std::rc::{Rc, Weak};") - - typeProvider.allClasses.foreach{ - case (name, _) => - if(name != topClassName) //TODO: do not add to imported - importList.add(s"use super::$name::*;") - } } override def externalTypeDeclaration(extType: ExternalType): Unit = importList.add( - s"use super::${extType.name.head}::${type2class(extType.name.head)};" + s"use super::${extType.name.head}::${types2class(extType.name)};" ) override def classHeader(name: List[String]): Unit = { From d6793db5f171658ffeeda8b08a1b6519e2566edf Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 1 Sep 2024 20:00:00 +0200 Subject: [PATCH 152/153] Rust: adapt to recent runtime library changes --- .../struct/languages/RustCompiler.scala | 56 ++++++++++++------- .../struct/translators/RustTranslator.scala | 2 +- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index d8b2e7867..d3f5d5cf8 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -152,7 +152,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def runReadCalc(): Unit = { out.puts(s"if *${privateMemberName(EndianIdentifier)} == 0 {") out.inc - out.puts(s"""return Err(KError::UndecidedEndiannessError("${typeProvider.nowClass.path.mkString("/", "/", "")}".to_string()));""") + out.puts(s"""return Err(${ksErrorName(UndecidedEndiannessError)} { src_path: "${typeProvider.nowClass.path.mkString("/", "/", "")}".to_string() });""") out.dec out.puts("}") } @@ -371,19 +371,19 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case ProcessXor(xorValue) => translator.detectType(xorValue) match { case _: IntType => - s"BytesReader::process_xor_one(&$srcExpr, ${expression(xorValue)})" + s"process_xor_one(&$srcExpr, ${expression(xorValue)})" case _: BytesType => - s"BytesReader::process_xor_many(&$srcExpr, &${translator.remove_deref(expression(xorValue))})" + s"process_xor_many(&$srcExpr, &${translator.remove_deref(expression(xorValue))})" } case ProcessZlib => - s"BytesReader::process_zlib(&$srcExpr)" + s"process_zlib(&$srcExpr).map_err(|msg| KError::BytesDecodingError { msg })?" case ProcessRotate(isLeft, rotValue) => val expr = if (isLeft) { expression(rotValue) } else { s"8 - (${expression(rotValue)})" } - s"BytesReader::process_rotate_left(&$srcExpr, $expr)" + s"process_rotate_left(&$srcExpr, $expr)" case ProcessCustom(name, args) => val procClass = name.map(x => type2class(x)).mkString("::") val procName = s"_process_${idToStr(varSrc)}" @@ -394,7 +394,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val argList = translate_args(args, into = false) val argListInParens = s"($argList)" out.puts(s"let $procName = $procClass::new$argListInParens;") - s"$procName.decode(&$srcExpr)" + s"$procName.decode(&$srcExpr).map_err(|msg| KError::BytesDecodingError { msg })?" } handleAssignment(varDest, expr, rep, isRaw = false) } @@ -836,13 +836,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean): String = { val ioId = privateMemberName(IoIdentifier) val expr1 = padRight match { - case Some(padByte) => s"BytesReader::bytes_strip_right(&$expr0, $padByte).into()" + case Some(padByte) => s"bytes_strip_right(&$expr0, $padByte).into()" case None => expr0 } val expr2 = terminator match { case Some(term) => val t = term.head & 0xff - s"BytesReader::bytes_terminate(&$expr1, $t, $include).into()" + s"bytes_terminate(&$expr1, $t, $include).into()" case None => expr1 } expr2 @@ -1186,13 +1186,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case t: ArrayType => s"Arr${switchVariantName(id, t.elType)}" } - override def ksErrorName(err: KSError): String = err match { - case EndOfStreamError => "KError::EncounteredEOF" - case UndecidedEndiannessError => "KError::UndecidedEndiannessError" - case ConversionError => "KError::CastError" - case _: ValidationError => s"KError::ValidationNotEqual" - } - + override def ksErrorName(err: KSError): String = RustCompiler.ksErrorName(err) override def attrValidateExpr( attr: AttrLikeSpec, @@ -1200,10 +1194,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) err: KSError, errArgs: List[Ast.expr] ): Unit = { - val errArgsStr = errArgs.map(translator.translate).mkString(", ") + val srcPathStr = translator.translate(Ast.expr.Str(attr.path.mkString("/", "/", ""))) + val validationKind = RustCompiler.validationErrorKind(err.asInstanceOf[ValidationError]) out.puts(s"if !(${expression(checkExpr)}) {") out.inc - out.puts(s"""return Err(KError::ValidationNotEqual(r#"$errArgsStr"#.to_string()));""") + out.puts(s"""return Err(${ksErrorName(err)}(ValidationFailedError { kind: $validationKind, src_path: $srcPathStr.to_string() }));""") out.dec out.puts("}") } @@ -1249,13 +1244,12 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) object RustCompiler extends LanguageCompilerStatic with StreamStructNames - with UpperCamelCaseClasses { + with UpperCamelCaseClasses + with ExceptionNames { override def getCompiler(tp: ClassTypeProvider, config: RuntimeConfig): LanguageCompiler = new RustCompiler(tp, config) - override def kstreamName = "KStream" - var in_reader = false def self_name(): String = { @@ -1293,10 +1287,30 @@ object RustCompiler rootClassTypeName(c.upClass.get, isRecurse = true) } - override def kstructName = s"KStruct" + override def kstreamName = "KStream" + override def kstructName = "KStruct" def kstructUnitName = "KStructUnit" + override def ksErrorName(err: KSError): String = err match { + case EndOfStreamError => "KError::Eof" + case UndecidedEndiannessError => "KError::UndecidedEndianness" + case ConversionError => "KError::CastError" + case _: ValidationError => "KError::ValidationFailed" + } + + def validationErrorKind(err: ValidationError): String = { + val kind = err match { + case _: ValidationNotEqualError => "NotEqual" + case _: ValidationLessThanError => "LessThan" + case _: ValidationGreaterThanError => "GreaterThan" + case _: ValidationNotAnyOfError => "NotAnyOf" + case _: ValidationNotInEnumError => "NotInEnum" + case _: ValidationExprError => "Expr" + } + s"ValidationKind::$kind" + } + def classTypeName(c: ClassSpec): String = s"${types2class(c.name)}" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala index f30fe27af..788d49182 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -452,7 +452,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) s"${remove_deref(translate(i))}.to_string()" override def bytesToStr(bytesExpr: String, encoding: String): String = - s"""decode_string(&$bytesExpr, &"$encoding")?""" + s"""bytes_to_str(&$bytesExpr, "$encoding")?""" override def bytesLength(b: Ast.expr): String = s"${remove_deref(translate(b))}.len()" From d7c95164b5a4997985e04e31f3840760d80fb6a1 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Tue, 3 Sep 2024 01:05:53 +0200 Subject: [PATCH 153/153] Rust: deny `arithmetic_overflow` and `overflowing_literals` --- .../main/scala/io/kaitai/struct/languages/RustCompiler.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index d3f5d5cf8..b0e885e7b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -50,8 +50,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) outHeader.puts("#![allow(non_camel_case_types)]") outHeader.puts("#![allow(irrefutable_let_patterns)]") outHeader.puts("#![allow(unused_comparisons)]") - outHeader.puts("#![allow(arithmetic_overflow)]") - outHeader.puts("#![allow(overflowing_literals)]") outHeader.puts outHeader.puts("extern crate kaitai;")