From ca305eb1997b85ce682ea56c59adc91cce86952d Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Sun, 2 Jul 2023 16:57:28 +0200 Subject: [PATCH 01/37] Copy Ruby classes to Julia{Compiler,Translator} --- .../struct/languages/JuliaCompiler.scala | 505 ++++++++++++++++++ .../struct/translators/JuliaTranslator.scala | 131 +++++ 2 files changed, 636 insertions(+) create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala new file mode 100644 index 000000000..8473411c5 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -0,0 +1,505 @@ +package io.kaitai.struct.languages + +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.exprlang.Ast.expr +import io.kaitai.struct.format._ +import io.kaitai.struct.languages.components._ +import io.kaitai.struct.translators.JuliaTranslator +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} + +class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) + extends LanguageCompiler(typeProvider, config) + with ObjectOrientedLanguage + with SingleOutputFile + with UniversalFooter + with UniversalDoc + with UpperCamelCaseClasses + with AllocateIOLocalVar + with EveryReadIsExpression + with FixedContentsUsingArrayByteLiteral + with NoNeedForFullClassPath { + + import JuliaCompiler._ + + val translator = new JuliaTranslator(typeProvider) + + override def universalFooter: Unit = { + out.dec + out.puts("end") + } + + override def outFileName(topClassName: String): String = s"$topClassName.rb" + override def indent: String = " " + + override def outImports(topClass: ClassSpec) = + importList.toList.map((x) => s"require '$x'").mkString("\n") + "\n" + + override def fileHeader(topClassName: String): Unit = { + outHeader.puts(s"# $headerComment") + outHeader.puts + + importList.add("kaitai/struct/struct") + + out.puts + + // API compatibility check + out.puts( + "unless Gem::Version.new(Kaitai::Struct::VERSION) >= Gem::Version.new('" + + KSVersion.minimalRuntime + + "')" + ) + out.inc + out.puts( + "raise \"Incompatible Kaitai Struct Ruby API: " + + KSVersion.minimalRuntime + + " or later is required, but you have #{Kaitai::Struct::VERSION}\"" + ) + out.dec + out.puts("end") + out.puts + } + + override def classHeader(name: String): Unit = { + out.puts(s"class ${type2class(name)} < $kstructName") + out.inc + if (config.readStoresPos) + out.puts("attr_reader :_debug") + } + + override def classConstructorHeader(name: String, parentType: DataType, rootClassName: String, isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + val endianSuffix = if (isHybrid) { + ", _is_le = nil" + } else { + "" + } + + val paramsList = Utils.join(params.map((p) => paramName(p.id)), ", ", ", ", "") + + out.puts(s"def initialize(_io, _parent = nil, _root = self$endianSuffix$paramsList)") + out.inc + out.puts("super(_io, _parent, _root)") + + if (isHybrid) { + out.puts("@_is_le = _is_le") + } + + // Store parameters passed to us + params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id))) + + if (config.readStoresPos) { + out.puts("@_debug = {}") + } + } + + override def runRead(name: List[String]): Unit = { + out.puts("_read") + } + + override def runReadCalc(): Unit = { + out.puts + out.puts(s"if @_is_le == true") + out.inc + out.puts("_read_le") + out.dec + out.puts("elsif @_is_le == false") + out.inc + out.puts("_read_be") + out.dec + out.puts("else") + out.inc + out.puts(s"raise ${ksErrorName(UndecidedEndiannessError)}.new(" + "\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\")") + out.dec + out.puts("end") + } + + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = { + val suffix = endian match { + case Some(e) => s"_${e.toSuffix}" + case None => "" + } + out.puts + out.puts(s"def _read$suffix") + out.inc + } + + override def readFooter() = { + // This is required for debug mode to be able to do stuff like: + // + // obj = Obj.new(...)._read + // + // i.e. drop-in replacement of non-debug mode invocation: + // + // obj = Obj.new(...) + out.puts("self") + + universalFooter + } + + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} + + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + attrName match { + case RootIdentifier | ParentIdentifier => + // ignore, they are already added in Kaitai::Struct::Struct + case _ => + out.puts(s"attr_reader :${publicMemberName(attrName)}") + } + } + + override def universalDoc(doc: DocSpec): Unit = { + out.puts + out.puts("##") + + doc.summary.foreach(summary => out.putsLines("# ", summary)) + + doc.ref.foreach { + case TextRef(text) => + out.putsLines("# ", s"@see '' $text", " ") + case UrlRef(url, text) => + out.putsLines("# ", s"@see $url $text", " ") + } + } + + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + out.puts("if @_is_le") + out.inc + leProc() + out.dec + out.puts("else") + out.inc + beProc() + out.dec + out.puts("end") + } + + override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = + out.puts(s"${privateMemberName(attrName)} = $normalIO.ensure_fixed_contents($contents)") + + override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { + val srcExpr = getRawIdExpr(varSrc, rep) + + val expr = proc match { + case ProcessXor(xorValue) => + val procName = translator.detectType(xorValue) match { + case _: IntType => "process_xor_one" + case _: BytesType => "process_xor_many" + } + s"$kstreamName::$procName($srcExpr, ${expression(xorValue)})" + case ProcessZlib => + importList.add("zlib") + s"Zlib::Inflate.inflate($srcExpr)" + case ProcessRotate(isLeft, rotValue) => + val expr = if (isLeft) { + expression(rotValue) + } else { + s"8 - (${expression(rotValue)})" + } + s"$kstreamName::process_rotate_left($srcExpr, $expr, 1)" + case ProcessCustom(name, args) => + val procClass = name.map((x) => type2class(x)).mkString("::") + out.puts(s"_process = $procClass.new(${args.map(expression).mkString(", ")})") + s"_process.decode($srcExpr)" + } + handleAssignment(varDest, expr, rep, false) + } + + override def allocateIO(id: Identifier, rep: RepeatSpec): String = { + val memberName = privateMemberName(id) + val ioName = s"_io_${idToStr(id)}" + + val args = getRawIdExpr(id, rep) + + out.puts(s"$ioName = $kstreamName.new($args)") + ioName + } + + def getRawIdExpr(varName: Identifier, rep: RepeatSpec): String = { + val memberName = privateMemberName(varName) + rep match { + case NoRepeat => memberName + case RepeatExpr(_) => s"$memberName[i]" + case _ => s"$memberName.last" + } + } + + override def useIO(ioEx: expr): String = { + out.puts(s"io = ${expression(ioEx)}") + "io" + } + + override def pushPos(io: String): Unit = + out.puts(s"_pos = $io.pos") + + override def seek(io: String, pos: Ast.expr): Unit = + out.puts(s"$io.seek(${expression(pos)})") + + override def popPos(io: String): Unit = + out.puts(s"$io.seek(_pos)") + + override def alignToByte(io: String): Unit = + out.puts(s"$io.align_to_byte") + + override def attrDebugStart(attrId: Identifier, attrType: DataType, ios: Option[String], rep: RepeatSpec): Unit = { + ios.foreach { (io) => + val name = attrId match { + case _: RawIdentifier | _: SpecialIdentifier => return + case _ => idToStr(attrId) + } + rep match { + case NoRepeat => + out.puts(s"(@_debug['$name'] ||= {})[:start] = $io.pos") + case _: RepeatExpr => + out.puts(s"(@_debug['$name'][:arr] ||= [])[i] = {:start => $io.pos}") + case RepeatEos | _: RepeatUntil => + out.puts(s"(@_debug['$name'][:arr] ||= [])[${privateMemberName(attrId)}.size] = {:start => $io.pos}") + } + } + } + + override def attrDebugEnd(attrId: Identifier, attrType: DataType, io: String, rep: RepeatSpec): Unit = { + val name = attrId match { + case _: RawIdentifier | _: SpecialIdentifier => return + case _ => idToStr(attrId) + } + rep match { + case NoRepeat => + out.puts(s"(@_debug['$name'] ||= {})[:end] = $io.pos") + case _: RepeatExpr => + out.puts(s"@_debug['$name'][:arr][i][:end] = $io.pos") + case RepeatEos | _: RepeatUntil => + out.puts(s"@_debug['$name'][:arr][${privateMemberName(attrId)}.size - 1][:end] = $io.pos") + } + } + + override def condIfHeader(expr: Ast.expr): Unit = { + out.puts(s"if ${expression(expr)}") + out.inc + } + + override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { + if (needRaw.level >= 1) + out.puts(s"${privateMemberName(RawIdentifier(id))} = []") + if (needRaw.level >= 2) + out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = []") + out.puts(s"${privateMemberName(id)} = []") + } + + override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = { + out.puts("i = 0") + out.puts(s"while not $io.eof?") + out.inc + } + override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = + out.puts(s"${privateMemberName(id)} << $expr") + + override def condRepeatEosFooter: Unit = { + out.puts("i += 1") + super.condRepeatEosFooter + } + + override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, repeatExpr: expr): Unit = { + out.puts(s"(${expression(repeatExpr)}).times { |i|") + out.inc + } + override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = + handleAssignmentRepeatEos(id, expr) + + override def condRepeatExprFooter: Unit = { + out.dec + out.puts("}") + } + + override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, untilExpr: expr): Unit = { + out.puts("i = 0") + out.puts("begin") + out.inc + } + + override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = { + val tmpName = translator.doName(if (isRaw) Identifier.ITERATOR2 else Identifier.ITERATOR) + out.puts(s"$tmpName = $expr") + out.puts(s"${privateMemberName(id)} << $tmpName") + } + + override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, untilExpr: expr): Unit = { + typeProvider._currentIteratorType = Some(dataType) + out.puts("i += 1") + out.dec + out.puts(s"end until ${expression(untilExpr)}") + } + + override def handleAssignmentSimple(id: Identifier, expr: String): Unit = + out.puts(s"${privateMemberName(id)} = $expr") + + override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = + out.puts(s"$id = $expr") + + override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { + dataType match { + case t: ReadableType => + s"$io.read_${t.apiCall(defEndian)}" + case blt: BytesLimitType => + s"$io.read_bytes(${expression(blt.size)})" + case _: BytesEosType => + s"$io.read_bytes_full" + case BytesTerminatedType(terminator, include, consume, eosError, _) => + s"$io.read_bytes_term($terminator, $include, $consume, $eosError)" + case BitsType1(bitEndian) => + s"$io.read_bits_int_${bitEndian.toSuffix}(1) != 0" + case BitsType(width: Int, bitEndian) => + s"$io.read_bits_int_${bitEndian.toSuffix}($width)" + case t: UserType => + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), ", ", ", ", "") + val addArgs = if (t.isOpaque) { + "" + } else { + val parent = t.forcedParent match { + case Some(USER_TYPE_NO_PARENT) => "nil" + case Some(fp) => translator.translate(fp) + case None => "self" + } + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => ", @_is_le" + case _ => "" + } + s", $parent, @_root$addEndian" + } + s"${types2class(t.name)}.new($io$addArgs$addParams)" + } + } + + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + val expr1 = padRight match { + case Some(padByte) => s"$kstreamName::bytes_strip_right($expr0, $padByte)" + case None => expr0 + } + val expr2 = terminator match { + case Some(term) => s"$kstreamName::bytes_terminate($expr1, $term, $include)" + case None => expr1 + } + expr2 + } + + override def userTypeDebugRead(id: String, dataType: DataType, assignType: DataType): Unit = + out.puts(s"$id._read") + + override def switchStart(id: Identifier, on: Ast.expr): Unit = + out.puts(s"case ${expression(on)}") + + override def switchCaseStart(condition: Ast.expr): Unit = { + out.puts(s"when ${expression(condition)}") + out.inc + } + + override def switchCaseEnd(): Unit = + out.dec + + override def switchElseStart(): Unit = { + out.puts("else") + out.inc + } + + override def switchEnd(): Unit = + out.puts("end") + + override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { + out.puts(s"def ${instName.name}") + out.inc + } + + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { + out.puts(s"return ${privateMemberName(instName)} unless ${privateMemberName(instName)}.nil?") + } + + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { + out.puts(privateMemberName(instName)) + } + + override def enumDeclaration(curClass: String, enumName: String, enumColl: Seq[(Long, String)]): Unit = { + val enumConst = value2Const(enumName) + + out.puts + out.puts(s"$enumConst = {") + out.inc + enumColl.foreach { case (id, label) => + out.puts(s"${translator.doIntLiteral(id)} => ${enumValue(enumName, label)},") + } + out.dec + out.puts("}") + + // Generate inverse hash + out.puts(s"${inverseEnumName(enumConst)} = $enumConst.invert") + } + + def enumValue(enumName: String, enumLabel: String) = translator.doEnumByLabel(List(enumName), enumLabel) + + def value2Const(s: String) = Utils.upperUnderscoreCase(s) + + override def debugClassSequence(seq: List[AttrSpec]) = { + val seqStr = seq.map((attr) => "\"" + idToStr(attr.id) + "\"").mkString(", ") + out.puts(s"SEQ_FIELDS = [$seqStr]") + } + + override def classToString(toStringExpr: Ast.expr): Unit = { + out.puts + out.puts("def inspect") + out.inc + out.puts(translator.translate(toStringExpr)) + out.dec + out.puts("end") + } + + override def idToStr(id: Identifier): String = JuliaCompiler.idToStr(id) + + override def publicMemberName(id: Identifier): String = JuliaCompiler.publicMemberName(id) + + override def privateMemberName(id: Identifier): String = s"@${idToStr(id)}" + + override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" + + override def ksErrorName(err: KSError): String = JuliaCompiler.ksErrorName(err) + + override def attrValidateExpr( + attrId: Identifier, + attrType: DataType, + checkExpr: Ast.expr, + err: KSError, + errArgs: List[Ast.expr] + ): Unit = { + val errArgsStr = errArgs.map(translator.translate).mkString(", ") + out.puts(s"raise ${ksErrorName(err)}.new($errArgsStr) if not ${translator.translate(checkExpr)}") + } + + def types2class(names: List[String]) = names.map(type2class).mkString("::") +} + +object JuliaCompiler extends LanguageCompilerStatic + with StreamStructNames + with ExceptionNames { + override def getCompiler( + tp: ClassTypeProvider, + config: RuntimeConfig + ): LanguageCompiler = new JuliaCompiler(tp, config) + + def idToStr(id: Identifier): String = + id match { + case SpecialIdentifier(name) => name + case NamedIdentifier(name) => Utils.lowerUnderscoreCase(name) + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case InstanceIdentifier(name) => Utils.lowerUnderscoreCase(name) + case RawIdentifier(inner) => s"_raw_${idToStr(inner)}" + } + + def publicMemberName(id: Identifier) = idToStr(id) + + override def kstreamName: String = "Kaitai::Struct::Stream" + override def kstructName: String = "Kaitai::Struct::Struct" + override def ksErrorName(err: KSError): String = err match { + case EndOfStreamError => "EOFError" + case _ => s"Kaitai::Struct::${err.name}" + } + + def inverseEnumName(enumName: String) = s"I__$enumName" +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala new file mode 100644 index 000000000..5a046cc96 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -0,0 +1,131 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.Utils +import io.kaitai.struct.datatype.DataType.EnumType +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.Identifier +import io.kaitai.struct.languages.JuliaCompiler + +class JuliaTranslator(provider: TypeProvider) extends BaseTranslator(provider) + with ByteArraysAsTrueArrays[String] { + override def doByteArrayLiteral(arr: Seq[Byte]): String = + s"${super.doByteArrayLiteral(arr)}.pack('C*')" + override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = + s"[${elts.map(translate).mkString(", ")}].pack('C*')" + + // https://github.com/ruby/ruby/blob/trunk/doc/syntax/literals.rdoc#strings + // https://github.com/ruby/ruby/blob/trunk/string.c - see "rb_str_inspect" + override val asciiCharQuoteMap: Map[Char, String] = Map( + '\t' -> "\\t", + '\n' -> "\\n", + '\r' -> "\\r", + '"' -> "\\\"", + '\\' -> "\\\\", + + '#' -> "\\#", + '\u0007' -> "\\a", + '\f' -> "\\f", + '\u000b' -> "\\v", + '\u001b' -> "\\e", + '\b' -> "\\b" + ) + + override def doName(s: String) = { + s match { + case Identifier.INDEX => "i" // FIXME: probably would clash with attribute named "i" + case _ => s + } + } + + override def doInternalName(id: Identifier): String = + JuliaCompiler.publicMemberName(id) + + override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = + s":${enumTypeAbs.last}_$label" + override def doEnumById(enumType: List[String], id: String): String = + s"${JuliaCompiler.kstreamName}::resolve_enum(${enumDirectMap(enumType)}, $id)" + + def enumDirectMap(enumTypeAndName: List[String]): String = { + val enumTypeAbs = enumTypeAndName.dropRight(1) + val enumTypeName = Utils.upperUnderscoreCase(enumTypeAndName.last) + + val enumTypeRel = Utils.relClass(enumTypeAbs, provider.nowClass.name) + + if (enumTypeRel.nonEmpty) { + (enumTypeRel.map((x) => Utils.upperCamelCase(x)) ++ List(enumTypeName)).mkString("::") + } else { + enumTypeName + } + } + + def enumInverseMap(et: EnumType): String = { + val enumTypeAndName = et.enumSpec.get.name + val enumDirectMap = this.enumDirectMap(enumTypeAndName) + val enumNameDirect = Utils.upperUnderscoreCase(enumTypeAndName.last) + val enumNameInverse = JuliaCompiler.inverseEnumName(enumNameDirect) + + enumDirectMap.replace(enumNameDirect, enumNameInverse) + } + + override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = + s"${translate(container)}[${translate(idx)}]" + override def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String = + s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})" + + // Predefined methods of various types + override def strToInt(s: Ast.expr, base: Ast.expr): String = { + val baseStr = translate(base) + translate(s) + ".to_i" + (baseStr match { + case "10" => "" + case _ => s"($baseStr)" + }) + } + override def enumToInt(v: Ast.expr, et: EnumType): String = + s"${enumInverseMap(et)}[${translate(v)}]" + override def floatToInt(v: Ast.expr): String = + s"(${translate(v)}).to_i" + override def intToStr(i: Ast.expr, base: Ast.expr): String = + translate(i) + s".to_s(${translate(base)})" + + override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = + s"($bytesExpr).force_encoding(${translate(encoding)})" + override def bytesLength(b: Ast.expr): String = + s"${translate(b)}.size" + /** + * Alternatives considered: + * + * * value[0].ord => 6341 => winner by performance + * * value.bytes[0] => 8303 + */ + override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = + s"${translate(container)}[${translate(idx)}].ord" + override def bytesFirst(b: Ast.expr): String = + s"${translate(b)}[0].ord" + override def bytesLast(b: Ast.expr): String = + s"${translate(b)}[-1].ord" + override def bytesMin(b: Ast.expr): String = + s"${translate(b)}.bytes.min" + override def bytesMax(b: Ast.expr): String = + s"${translate(b)}.bytes.max" + + override def strLength(s: Ast.expr): String = + s"${translate(s)}.size" + override def strReverse(s: Ast.expr): String = + s"${translate(s)}.reverse" + override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = + s"${translate(s)}[${translate(from)}..(${translate(to)} - 1)]" + + override def arrayFirst(a: Ast.expr): String = + s"${translate(a)}.first" + override def arrayLast(a: Ast.expr): String = + s"${translate(a)}.last" + override def arraySize(a: Ast.expr): String = + s"${translate(a)}.length" + override def arrayMin(a: Ast.expr): String = + s"${translate(a)}.min" + override def arrayMax(a: Ast.expr): String = + s"${translate(a)}.max" + + override def kaitaiStreamEof(value: Ast.expr): String = + s"${translate(value)}.eof?" +} From 8324e62b9ca93d642323eb03e0622342c60314e3 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Mon, 31 Jul 2023 12:51:29 +0200 Subject: [PATCH 02/37] WIP: handle Julia class compilation --- .../io/kaitai/struct/JuliaClassCompiler.scala | 96 +++++ .../struct/languages/JuliaCompiler.scala | 396 ++++++++++-------- .../struct/translators/JuliaTranslator.scala | 163 +++---- 3 files changed, 409 insertions(+), 246 deletions(-) create mode 100644 shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala diff --git a/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala new file mode 100644 index 000000000..368a291d1 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala @@ -0,0 +1,96 @@ +package io.kaitai.struct + +import io.kaitai.struct.datatype.DataType.{CalcIntType, KaitaiStreamType, UserTypeInstream} +import io.kaitai.struct.datatype.{BigEndian, CalcEndian, Endianness, FixedEndian, InheritedEndian, LittleEndian} +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ +import io.kaitai.struct.languages.JuliaCompiler +import io.kaitai.struct.languages.components.ExtraAttrs + +class JuliaClassCompiler( + classSpecs: ClassSpecs, + override val topClass: ClassSpec, + config: RuntimeConfig +) extends ClassCompiler(classSpecs, topClass, config, JuliaCompiler) { + + override def compileClass(curClass: ClassSpec): Unit = { + provider.nowClass = curClass + + val extraAttrs = List( + AttrSpec(List(), IoIdentifier, KaitaiStreamType), + AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None)), + AttrSpec(List(), ParentIdentifier, curClass.parentType) + ) ++ ExtraAttrs.forClassSpec(curClass, lang) + + if (!curClass.doc.isEmpty) + lang.classDoc(curClass.name, curClass.doc) + + // Enums declaration defines types, so they need to go first + compileEnums(curClass) + + // Basic struct declaration + lang.classHeader(curClass.name) + compileAttrDeclarations(curClass.seq ++ curClass.params ++ extraAttrs) + curClass.instances.foreach { case (instName, instSpec) => + compileInstanceDeclaration(instName, instSpec) + } + compileConstructor(curClass) + lang.classFooter(curClass.name) + + // Constructor = Read() function + compileEagerRead(curClass.seq, curClass.meta.endian) + + compileInstances(curClass) + + compileAttrReaders(curClass.seq ++ extraAttrs) + + // Recursive types + compileSubclasses(curClass) + } + + def compileReadFunction(curClass: ClassSpec) = { + lang.classConstructorHeader( + curClass.name, + curClass.parentType, + topClassName, + curClass.meta.endian.contains(InheritedEndian), + curClass.params + ) + compileEagerRead(curClass.seq, curClass.meta.endian) + lang.classConstructorFooter + } + + override def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, endian: Option[Endianness]): Unit = { + // Determine datatype + val dataType = instSpec.dataTypeComposite + + if (!instSpec.doc.isEmpty) + lang.attributeDoc(instName, instSpec.doc) + lang.instanceHeader(className, instName, dataType, instSpec.isNullable) + lang.instanceCheckCacheAndReturn(instName, dataType) + + instSpec match { + case vi: ValueInstanceSpec => + lang.attrParseIfHeader(instName, vi.ifExpr) + lang.instanceCalculate(instName, dataType, vi.value) + lang.attrParseIfFooter(vi.ifExpr) + case i: ParseInstanceSpec => + lang.attrParse(i, instName, endian) + } + + lang.instanceSetCalculated(instName) + lang.instanceReturn(instName, dataType) + lang.instanceFooter + } + + override def compileCalcEndian(ce: CalcEndian): Unit = { + def renderProc(result: FixedEndian): Unit = { + val v = result match { + case LittleEndian => Ast.expr.IntNum(1) + case BigEndian => Ast.expr.IntNum(0) + } + lang.instanceCalculate(IS_LE_ID, CalcIntType, v) + } + lang.switchCases[FixedEndian](IS_LE_ID, ce.on, ce.cases, renderProc, renderProc) + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index 8473411c5..02c75a9ae 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -1,182 +1,194 @@ package io.kaitai.struct.languages import io.kaitai.struct.datatype.DataType._ -import io.kaitai.struct.datatype._ +import io.kaitai.struct.datatype.{DataType, EndOfStreamError, FixedEndian, InheritedEndian, KSError, UndecidedEndiannessError, NeedRaw} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ import io.kaitai.struct.languages.components._ import io.kaitai.struct.translators.JuliaTranslator -import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, StringLanguageOutputWriter, Utils} class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) with ObjectOrientedLanguage + with UpperCamelCaseClasses with SingleOutputFile with UniversalFooter - with UniversalDoc - with UpperCamelCaseClasses - with AllocateIOLocalVar with EveryReadIsExpression + with AllocateIOLocalVar with FixedContentsUsingArrayByteLiteral + with UniversalDoc + with SwitchIfOps with NoNeedForFullClassPath { import JuliaCompiler._ - val translator = new JuliaTranslator(typeProvider) + override val translator = new JuliaTranslator(typeProvider, importList) + + override def innerDocstrings = true override def universalFooter: Unit = { out.dec out.puts("end") } - override def outFileName(topClassName: String): String = s"$topClassName.rb" - override def indent: String = " " + override def indent: String = " " + override def outFileName(topClassName: String): String = s"$topClassName.jl" override def outImports(topClass: ClassSpec) = - importList.toList.map((x) => s"require '$x'").mkString("\n") + "\n" + importList.toList.mkString("", "\n", "\n") override def fileHeader(topClassName: String): Unit = { outHeader.puts(s"# $headerComment") outHeader.puts - importList.add("kaitai/struct/struct") + importList.add("using kaitaistruct") out.puts + // out.puts + + // // API compatibility check + // out.puts( + // // The API_VERSION tuple attribute was introduced in version 0.10 of the + // // Python runtime. Runtime version 0.9 and older only have a __version__ + // // attribute that stores the version in string form. + // // We don't need to include any complex handling for runtimes that don't + // // have the API_VERSION attribute - we know that such a runtime must have + // // version 0.9 or older, which means that it is incompatible with code + // // generated by newer compiler versions. + // "if getattr(kaitaistruct, 'API_VERSION', (0, 9)) < " + + // KSVersion.minimalRuntime.toPythonTuple + + // ":" + // ) + // out.inc + // out.puts( + // "raise Exception(\"Incompatible Kaitai Struct Python API: " + + // KSVersion.minimalRuntime + + // " or later is required, but you have %s\" % (kaitaistruct.__version__))" + // ) + // out.dec + out.puts + } - // API compatibility check - out.puts( - "unless Gem::Version.new(Kaitai::Struct::VERSION) >= Gem::Version.new('" + - KSVersion.minimalRuntime + - "')" - ) - out.inc + override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = { + val name = classSpec.name.head out.puts( - "raise \"Incompatible Kaitai Struct Ruby API: " + - KSVersion.minimalRuntime + - " or later is required, but you have #{Kaitai::Struct::VERSION}\"" + if (config.pythonPackage.nonEmpty) { + s"from ${config.pythonPackage} import $name" + } else { + s"import $name" + } ) - out.dec - out.puts("end") - out.puts } override def classHeader(name: String): Unit = { - out.puts(s"class ${type2class(name)} < $kstructName") + out.puts(s"mutable struct ${type2class(name)}") out.inc - if (config.readStoresPos) - out.puts("attr_reader :_debug") } override def classConstructorHeader(name: String, parentType: DataType, rootClassName: String, isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { - val endianSuffix = if (isHybrid) { - ", _is_le = nil" - } else { - "" - } - + val endianAdd = if (isHybrid) ", _is_le=None" else "" val paramsList = Utils.join(params.map((p) => paramName(p.id)), ", ", ", ", "") - out.puts(s"def initialize(_io, _parent = nil, _root = self$endianSuffix$paramsList)") + out.puts(s"${type2class(name)}(_io, _parent = nothing, _root = nothing$endianAdd, $paramsList) = (") out.inc - out.puts("super(_io, _parent, _root)") - - if (isHybrid) { - out.puts("@_is_le = _is_le") - } // Store parameters passed to us params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id))) - - if (config.readStoresPos) { - out.puts("@_debug = {}") - } } override def runRead(name: List[String]): Unit = { - out.puts("_read") + out.puts("x = new();") + out.puts("x._io = _io;") + out.puts("x._parent = _parent;") + out.puts("x._root = _root === nothing ? x : _root;") + out.puts("_read(x);") + out.puts("x") + } + + override def classConstructorFooter(): Unit = { + out.dec + out.puts(")") } override def runReadCalc(): Unit = { - out.puts - out.puts(s"if @_is_le == true") + out.puts(s"if not hasattr(self, '_is_le'):") out.inc - out.puts("_read_le") + out.puts(s"raise ${ksErrorName(UndecidedEndiannessError)}(" + "\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\")") out.dec - out.puts("elsif @_is_le == false") + out.puts(s"elif self._is_le == True:") out.inc - out.puts("_read_be") + out.puts("self._read_le()") out.dec - out.puts("else") + out.puts("elif self._is_le == False:") out.inc - out.puts(s"raise ${ksErrorName(UndecidedEndiannessError)}.new(" + "\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\")") + out.puts("self._read_be()") out.dec - out.puts("end") } - override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = { + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { val suffix = endian match { case Some(e) => s"_${e.toSuffix}" case None => "" } - out.puts - out.puts(s"def _read$suffix") + out.puts(s"function _read$suffix(this::${type2class(typeProvider.nowClass.name.last)})") out.inc } override def readFooter() = { - // This is required for debug mode to be able to do stuff like: - // - // obj = Obj.new(...)._read - // - // i.e. drop-in replacement of non-debug mode invocation: - // - // obj = Obj.new(...) - out.puts("self") - + out.puts("nothing") universalFooter } - override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} - - override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { - attrName match { - case RootIdentifier | ParentIdentifier => - // ignore, they are already added in Kaitai::Struct::Struct - case _ => - out.puts(s"attr_reader :${publicMemberName(attrName)}") - } + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + out.puts(s"${idToStr(attrName)}::$attrType") } - override def universalDoc(doc: DocSpec): Unit = { - out.puts - out.puts("##") + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} - doc.summary.foreach(summary => out.putsLines("# ", summary)) + override def universalDoc(doc: DocSpec): Unit = { + val docStr = doc.summary match { + case Some(summary) => + val lastChar = summary.last + if (lastChar == '.' || lastChar == '\n') { + summary + } else { + summary + "." + } + case None => + "" + } - doc.ref.foreach { + val extraNewline = if (docStr.isEmpty || docStr.last == '\n') "" else "\n" + val refStr = doc.ref.map { case TextRef(text) => - out.putsLines("# ", s"@see '' $text", " ") - case UrlRef(url, text) => - out.putsLines("# ", s"@see $url $text", " ") - } + val seeAlso = new StringLanguageOutputWriter("") + seeAlso.putsLines(" ", text) + s"$extraNewline\n.. seealso::\n${seeAlso.result}" + case ref: UrlRef => + val seeAlso = new StringLanguageOutputWriter("") + seeAlso.putsLines(" ", s"${ref.text} - ${ref.url}") + s"$extraNewline\n.. seealso::\n${seeAlso.result}" + }.mkString("\n") + + out.putsLines("", "\"\"\"" + docStr + refStr + "\"\"\"") } + override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = + out.puts(s"${privateMemberName(attrName)} = self._io.ensure_fixed_contents($contents)") + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { - out.puts("if @_is_le") + out.puts("if self._is_le:") out.inc leProc() out.dec - out.puts("else") + out.puts("else:") out.inc beProc() out.dec - out.puts("end") } - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = - out.puts(s"${privateMemberName(attrName)} = $normalIO.ensure_fixed_contents($contents)") - override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { val srcExpr = getRawIdExpr(varSrc, rep) @@ -186,32 +198,44 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: IntType => "process_xor_one" case _: BytesType => "process_xor_many" } - s"$kstreamName::$procName($srcExpr, ${expression(xorValue)})" + s"$kstreamName.$procName($srcExpr, ${expression(xorValue)})" case ProcessZlib => - importList.add("zlib") - s"Zlib::Inflate.inflate($srcExpr)" + importList.add("import zlib") + s"zlib.decompress($srcExpr)" case ProcessRotate(isLeft, rotValue) => val expr = if (isLeft) { expression(rotValue) } else { s"8 - (${expression(rotValue)})" } - s"$kstreamName::process_rotate_left($srcExpr, $expr, 1)" + s"$kstreamName.process_rotate_left($srcExpr, $expr, 1)" case ProcessCustom(name, args) => - val procClass = name.map((x) => type2class(x)).mkString("::") - out.puts(s"_process = $procClass.new(${args.map(expression).mkString(", ")})") + val procClass = if (name.length == 1) { + val onlyName = name.head + val className = type2class(onlyName) + importList.add(s"from $onlyName import $className") + className + } else { + val pkgName = name.init.mkString(".") + importList.add(s"import $pkgName") + s"$pkgName.${type2class(name.last)}" + } + + out.puts(s"_process = $procClass(${args.map(expression).mkString(", ")})") s"_process.decode($srcExpr)" } handleAssignment(varDest, expr, rep, false) } - override def allocateIO(id: Identifier, rep: RepeatSpec): String = { - val memberName = privateMemberName(id) - val ioName = s"_io_${idToStr(id)}" + override def normalIO: String = "this._io" + + override def allocateIO(varName: Identifier, rep: RepeatSpec): String = { + val varStr = privateMemberName(varName) + val ioName = s"_io_${idToStr(varName)}" - val args = getRawIdExpr(id, rep) + val args = getRawIdExpr(varName, rep) - out.puts(s"$ioName = $kstreamName.new($args)") + out.puts(s"$ioName = $kstreamName(BytesIO($args))") ioName } @@ -220,7 +244,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) rep match { case NoRepeat => memberName case RepeatExpr(_) => s"$memberName[i]" - case _ => s"$memberName.last" + case _ => s"$memberName[-1]" } } @@ -230,7 +254,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def pushPos(io: String): Unit = - out.puts(s"_pos = $io.pos") + out.puts(s"_pos = $io.pos()") override def seek(io: String, pos: Ast.expr): Unit = out.puts(s"$io.seek(${expression(pos)})") @@ -239,7 +263,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"$io.seek(_pos)") override def alignToByte(io: String): Unit = - out.puts(s"$io.align_to_byte") + out.puts(s"$io.align_to_byte()") override def attrDebugStart(attrId: Identifier, attrType: DataType, ios: Option[String], rep: RepeatSpec): Unit = { ios.foreach { (io) => @@ -249,11 +273,15 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } rep match { case NoRepeat => - out.puts(s"(@_debug['$name'] ||= {})[:start] = $io.pos") - case _: RepeatExpr => - out.puts(s"(@_debug['$name'][:arr] ||= [])[i] = {:start => $io.pos}") - case RepeatEos | _: RepeatUntil => - out.puts(s"(@_debug['$name'][:arr] ||= [])[${privateMemberName(attrId)}.size] = {:start => $io.pos}") + out.puts(s"self._debug['$name']['start'] = $io.pos()") + case _: RepeatExpr | RepeatEos | _: RepeatUntil => + /** TODO: move array initialization to [[condRepeatCommonInit]] - see + * [[JavaScriptCompiler.condRepeatCommonInit]] for inspiration */ + out.puts(s"if not 'arr' in self._debug['$name']:") + out.inc + out.puts(s"self._debug['$name']['arr'] = []") + out.dec + out.puts(s"self._debug['$name']['arr'].append({'start': $io.pos()})") } } } @@ -265,16 +293,16 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } rep match { case NoRepeat => - out.puts(s"(@_debug['$name'] ||= {})[:end] = $io.pos") + out.puts(s"self._debug['$name']['end'] = $io.pos()") case _: RepeatExpr => - out.puts(s"@_debug['$name'][:arr][i][:end] = $io.pos") + out.puts(s"self._debug['$name']['arr'][i]['end'] = $io.pos()") case RepeatEos | _: RepeatUntil => - out.puts(s"@_debug['$name'][:arr][${privateMemberName(attrId)}.size - 1][:end] = $io.pos") + out.puts(s"self._debug['$name']['arr'][len(${privateMemberName(attrId)}) - 1]['end'] = $io.pos()") } } override def condIfHeader(expr: Ast.expr): Unit = { - out.puts(s"if ${expression(expr)}") + out.puts(s"if ${expression(expr)}:") out.inc } @@ -288,46 +316,44 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = { out.puts("i = 0") - out.puts(s"while not $io.eof?") + out.puts(s"while not $io.is_eof():") out.inc } override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = - out.puts(s"${privateMemberName(id)} << $expr") + out.puts(s"${privateMemberName(id)}.append($expr)") override def condRepeatEosFooter: Unit = { out.puts("i += 1") - super.condRepeatEosFooter + universalFooter } override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, repeatExpr: expr): Unit = { - out.puts(s"(${expression(repeatExpr)}).times { |i|") + out.puts(s"for i in range(${expression(repeatExpr)}):") out.inc } override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = handleAssignmentRepeatEos(id, expr) - override def condRepeatExprFooter: Unit = { - out.dec - out.puts("}") - } - override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, untilExpr: expr): Unit = { out.puts("i = 0") - out.puts("begin") + out.puts("while True:") out.inc } override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = { val tmpName = translator.doName(if (isRaw) Identifier.ITERATOR2 else Identifier.ITERATOR) out.puts(s"$tmpName = $expr") - out.puts(s"${privateMemberName(id)} << $tmpName") + out.puts(s"${privateMemberName(id)}.append($tmpName)") } override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, untilExpr: expr): Unit = { typeProvider._currentIteratorType = Some(dataType) + out.puts(s"if ${expression(untilExpr)}:") + out.inc + out.puts("break") + out.dec out.puts("i += 1") out.dec - out.puts(s"end until ${expression(untilExpr)}") } override def handleAssignmentSimple(id: Identifier, expr: String): Unit = @@ -339,104 +365,112 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { dataType match { case t: ReadableType => - s"$io.read_${t.apiCall(defEndian)}" + s"$io.read_${t.apiCall(defEndian)}()" case blt: BytesLimitType => s"$io.read_bytes(${expression(blt.size)})" case _: BytesEosType => - s"$io.read_bytes_full" + s"$io.read_bytes_full()" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io.read_bytes_term($terminator, $include, $consume, $eosError)" + s"$io.read_bytes_term($terminator, ${bool2Py(include)}, ${bool2Py(consume)}, ${bool2Py(eosError)})" case BitsType1(bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}(1) != 0" case BitsType(width: Int, bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}($width)" case t: UserType => - val addParams = Utils.join(t.args.map((a) => translator.translate(a)), ", ", ", ", "") + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") val addArgs = if (t.isOpaque) { "" } else { val parent = t.forcedParent match { - case Some(USER_TYPE_NO_PARENT) => "nil" + case Some(USER_TYPE_NO_PARENT) => "None" case Some(fp) => translator.translate(fp) case None => "self" } val addEndian = t.classSpec.get.meta.endian match { - case Some(InheritedEndian) => ", @_is_le" + case Some(InheritedEndian) => ", self._is_le" case _ => "" } - s", $parent, @_root$addEndian" + s", $parent, self._root$addEndian" } - s"${types2class(t.name)}.new($io$addArgs$addParams)" + s"${userType2class(t)}($addParams$io$addArgs)" } } override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { val expr1 = padRight match { - case Some(padByte) => s"$kstreamName::bytes_strip_right($expr0, $padByte)" + case Some(padByte) => s"$kstreamName.bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName::bytes_terminate($expr1, $term, $include)" + case Some(term) => s"$kstreamName.bytes_terminate($expr1, $term, ${bool2Py(include)})" case None => expr1 } expr2 } override def userTypeDebugRead(id: String, dataType: DataType, assignType: DataType): Unit = - out.puts(s"$id._read") + out.puts(s"$id._read()") + + override def switchStart(id: Identifier, on: Ast.expr): Unit = {} + override def switchCaseStart(condition: Ast.expr): Unit = {} + override def switchCaseEnd(): Unit = {} + override def switchElseStart(): Unit = {} + override def switchEnd(): Unit = {} - override def switchStart(id: Identifier, on: Ast.expr): Unit = - out.puts(s"case ${expression(on)}") + override def switchRequiresIfs(onType: DataType): Boolean = true + override def switchIfStart(id: Identifier, on: Ast.expr, onType: DataType): Unit = { + out.puts(s"_on = ${expression(on)}") + } - override def switchCaseStart(condition: Ast.expr): Unit = { - out.puts(s"when ${expression(condition)}") + override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { + out.puts(s"if _on == ${expression(condition)}:") out.inc } - override def switchCaseEnd(): Unit = + override def switchIfCaseStart(condition: Ast.expr): Unit = { + out.puts(s"elif _on == ${expression(condition)}:") + out.inc + } + + override def switchIfCaseEnd(): Unit = out.dec - override def switchElseStart(): Unit = { - out.puts("else") + override def switchIfElseStart(): Unit = { + out.puts(s"else:") out.inc } - override def switchEnd(): Unit = - out.puts("end") + override def switchIfEnd(): Unit = {} override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { - out.puts(s"def ${instName.name}") + out.puts("@property") + out.puts(s"def ${publicMemberName(instName)}(self):") out.inc } override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { - out.puts(s"return ${privateMemberName(instName)} unless ${privateMemberName(instName)}.nil?") + out.puts(s"if hasattr(self, '${idToStr(instName)}'):") + out.inc + out.puts(s"return ${privateMemberName(instName)}") + out.dec + out.puts } override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { - out.puts(privateMemberName(instName)) + // workaround to avoid Python raising an "AttributeError: instance has no attribute" + out.puts(s"return getattr(self, '${idToStr(instName)}', None)") } override def enumDeclaration(curClass: String, enumName: String, enumColl: Seq[(Long, String)]): Unit = { - val enumConst = value2Const(enumName) + importList.add("from enum import Enum") out.puts - out.puts(s"$enumConst = {") + out.puts(s"class ${type2class(enumName)}(Enum):") out.inc - enumColl.foreach { case (id, label) => - out.puts(s"${translator.doIntLiteral(id)} => ${enumValue(enumName, label)},") - } + enumColl.foreach { case (id: Long, label: String) => out.puts(s"$label = ${translator.doIntLiteral(id)}") } out.dec - out.puts("}") - - // Generate inverse hash - out.puts(s"${inverseEnumName(enumConst)} = $enumConst.invert") } - def enumValue(enumName: String, enumLabel: String) = translator.doEnumByLabel(List(enumName), enumLabel) - - def value2Const(s: String) = Utils.upperUnderscoreCase(s) - override def debugClassSequence(seq: List[AttrSpec]) = { val seqStr = seq.map((attr) => "\"" + idToStr(attr.id) + "\"").mkString(", ") out.puts(s"SEQ_FIELDS = [$seqStr]") @@ -444,18 +478,19 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def classToString(toStringExpr: Ast.expr): Unit = { out.puts - out.puts("def inspect") + out.puts("def __repr__(self):") out.inc - out.puts(translator.translate(toStringExpr)) + out.puts(s"return ${translator.translate(toStringExpr)}") out.dec - out.puts("end") } + def bool2Py(b: Boolean): String = if (b) { "True" } else { "False" } + override def idToStr(id: Identifier): String = JuliaCompiler.idToStr(id) override def publicMemberName(id: Identifier): String = JuliaCompiler.publicMemberName(id) - override def privateMemberName(id: Identifier): String = s"@${idToStr(id)}" + override def privateMemberName(id: Identifier): String = s"this.${idToStr(id)}" override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" @@ -469,13 +504,26 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) errArgs: List[Ast.expr] ): Unit = { val errArgsStr = errArgs.map(translator.translate).mkString(", ") - out.puts(s"raise ${ksErrorName(err)}.new($errArgsStr) if not ${translator.translate(checkExpr)}") + out.puts(s"if not ${translator.translate(checkExpr)}:") + out.inc + out.puts(s"raise ${ksErrorName(err)}($errArgsStr)") + out.dec } - def types2class(names: List[String]) = names.map(type2class).mkString("::") + def userType2class(t: UserType): String = { + val name = t.classSpec.get.name + val firstName = name.head + val prefix = if (t.isOpaque && firstName != translator.provider.nowClass.name.head) { + s"$firstName." + } else { + "" + } + s"$prefix${types2class(name)}" + } } object JuliaCompiler extends LanguageCompilerStatic + with UpperCamelCaseClasses with StreamStructNames with ExceptionNames { override def getCompiler( @@ -486,20 +534,24 @@ object JuliaCompiler extends LanguageCompilerStatic def idToStr(id: Identifier): String = id match { case SpecialIdentifier(name) => name - case NamedIdentifier(name) => Utils.lowerUnderscoreCase(name) + case NamedIdentifier(name) => name case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" - case InstanceIdentifier(name) => Utils.lowerUnderscoreCase(name) - case RawIdentifier(inner) => s"_raw_${idToStr(inner)}" + case InstanceIdentifier(name) => s"_m_$name" + case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" } - def publicMemberName(id: Identifier) = idToStr(id) + def publicMemberName(id: Identifier): String = + id match { + case InstanceIdentifier(name) => name + case _ => idToStr(id) + } - override def kstreamName: String = "Kaitai::Struct::Stream" - override def kstructName: String = "Kaitai::Struct::Struct" + override def kstreamName: String = "KaitaiStream" + override def kstructName: String = "KaitaiStruct" override def ksErrorName(err: KSError): String = err match { case EndOfStreamError => "EOFError" - case _ => s"Kaitai::Struct::${err.name}" + case _ => s"kaitaistruct.${err.name}" } - def inverseEnumName(enumName: String) = s"I__$enumName" + def types2class(name: List[String]): String = name.map(x => type2class(x)).mkString(".") } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 5a046cc96..d7371b180 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -1,20 +1,28 @@ package io.kaitai.struct.translators -import io.kaitai.struct.Utils -import io.kaitai.struct.datatype.DataType.EnumType +import io.kaitai.struct.{ImportList, Utils} +import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format.Identifier -import io.kaitai.struct.languages.JuliaCompiler +import io.kaitai.struct.languages.{JuliaCompiler, RubyCompiler} + +class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { + override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = { + (detectType(left), detectType(right), op) match { + case (_: IntType, _: IntType, Ast.operator.Div) => + s"${translate(left)} // ${translate(right)}" + case _ => + super.numericBinOp(left, op, right) + } + } -class JuliaTranslator(provider: TypeProvider) extends BaseTranslator(provider) - with ByteArraysAsTrueArrays[String] { - override def doByteArrayLiteral(arr: Seq[Byte]): String = - s"${super.doByteArrayLiteral(arr)}.pack('C*')" - override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = - s"[${elts.map(translate).mkString(", ")}].pack('C*')" + override def doStringLiteral(s: String): String = "u" + super.doStringLiteral(s) + override def doBoolLiteral(n: Boolean): String = if (n) "True" else "False" - // https://github.com/ruby/ruby/blob/trunk/doc/syntax/literals.rdoc#strings - // https://github.com/ruby/ruby/blob/trunk/string.c - see "rb_str_inspect" + /** + * https://docs.Julia.org/2.7/reference/lexical_analysis.html#string-literals + * https://docs.Julia.org/3.6/reference/lexical_analysis.html#string-and-bytes-literals + */ override val asciiCharQuoteMap: Map[Char, String] = Map( '\t' -> "\\t", '\n' -> "\\n", @@ -22,110 +30,117 @@ class JuliaTranslator(provider: TypeProvider) extends BaseTranslator(provider) '"' -> "\\\"", '\\' -> "\\\\", - '#' -> "\\#", '\u0007' -> "\\a", '\f' -> "\\f", '\u000b' -> "\\v", - '\u001b' -> "\\e", '\b' -> "\\b" ) - override def doName(s: String) = { + override def doByteArrayLiteral(arr: Seq[Byte]): String = + "b\"" + Utils.hexEscapeByteArray(arr) + "\"" + override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = { + importList.add("import struct") + s"struct.pack('${elts.length}b', ${elts.map(translate).mkString(", ")})" + } + + override def doLocalName(s: String) = { s match { - case Identifier.INDEX => "i" // FIXME: probably would clash with attribute named "i" - case _ => s + case Identifier.ITERATOR => "_" + case Identifier.INDEX => "i" + case _ => s"self.${doName(s)}" } } - + override def doName(s: String) = + s override def doInternalName(id: Identifier): String = - JuliaCompiler.publicMemberName(id) + s"self.${JuliaCompiler.publicMemberName(id)}" override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = - s":${enumTypeAbs.last}_$label" - override def doEnumById(enumType: List[String], id: String): String = - s"${JuliaCompiler.kstreamName}::resolve_enum(${enumDirectMap(enumType)}, $id)" - - def enumDirectMap(enumTypeAndName: List[String]): String = { - val enumTypeAbs = enumTypeAndName.dropRight(1) - val enumTypeName = Utils.upperUnderscoreCase(enumTypeAndName.last) - - val enumTypeRel = Utils.relClass(enumTypeAbs, provider.nowClass.name) + s"${JuliaCompiler.types2class(enumTypeAbs)}.$label" + override def doEnumById(enumTypeAbs: List[String], id: String): String = + s"${JuliaCompiler.kstreamName}.resolve_enum(${JuliaCompiler.types2class(enumTypeAbs)}, $id)" - if (enumTypeRel.nonEmpty) { - (enumTypeRel.map((x) => Utils.upperCamelCase(x)) ++ List(enumTypeName)).mkString("::") - } else { - enumTypeName - } + override def booleanOp(op: Ast.boolop) = op match { + case Ast.boolop.Or => "or" + case Ast.boolop.And => "and" } - def enumInverseMap(et: EnumType): String = { - val enumTypeAndName = et.enumSpec.get.name - val enumDirectMap = this.enumDirectMap(enumTypeAndName) - val enumNameDirect = Utils.upperUnderscoreCase(enumTypeAndName.last) - val enumNameInverse = JuliaCompiler.inverseEnumName(enumNameDirect) - - enumDirectMap.replace(enumNameDirect, enumNameInverse) + override def unaryOp(op: Ast.unaryop) = op match { + case Ast.unaryop.Not => "not " + case _ => super.unaryOp(op) } override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = s"${translate(container)}[${translate(idx)}]" override def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String = - s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})" + s"(${translate(ifTrue)} if ${translate(condition)} else ${translate(ifFalse)})" // Predefined methods of various types override def strToInt(s: Ast.expr, base: Ast.expr): String = { val baseStr = translate(base) - translate(s) + ".to_i" + (baseStr match { + val add = baseStr match { case "10" => "" - case _ => s"($baseStr)" - }) + case _ => s", $baseStr" + } + s"int(${translate(s)}$add)" } override def enumToInt(v: Ast.expr, et: EnumType): String = - s"${enumInverseMap(et)}[${translate(v)}]" + s"${translate(v)}.value" + override def boolToInt(v: Ast.expr): String = + s"int(${translate(v)})" override def floatToInt(v: Ast.expr): String = - s"(${translate(v)}).to_i" - override def intToStr(i: Ast.expr, base: Ast.expr): String = - translate(i) + s".to_s(${translate(base)})" + s"int(${translate(v)})" + override def intToStr(i: Ast.expr, base: Ast.expr): String = { + val baseStr = translate(base) + val func = baseStr match { + case "2" => "bin" + case "8" => "oct" + case "10" => "str" + case "16" => "hex" + case _ => throw new UnsupportedOperationException(baseStr) + } + s"$func(${translate(i)})" + } override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = - s"($bytesExpr).force_encoding(${translate(encoding)})" - override def bytesLength(b: Ast.expr): String = - s"${translate(b)}.size" - /** - * Alternatives considered: - * - * * value[0].ord => 6341 => winner by performance - * * value.bytes[0] => 8303 - */ + s"($bytesExpr).decode(${translate(encoding)})" + + override def bytesLength(value: Ast.expr): String = + s"len(${translate(value)})" override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = - s"${translate(container)}[${translate(idx)}].ord" - override def bytesFirst(b: Ast.expr): String = - s"${translate(b)}[0].ord" - override def bytesLast(b: Ast.expr): String = - s"${translate(b)}[-1].ord" + s"${JuliaCompiler.kstreamName}.byte_array_index(${translate(container)}, ${translate(idx)})" + override def bytesFirst(a: Ast.expr): String = + bytesSubscript(a, Ast.expr.IntNum(0)) + override def bytesLast(a: Ast.expr): String = + bytesSubscript(a, Ast.expr.IntNum(-1)) override def bytesMin(b: Ast.expr): String = - s"${translate(b)}.bytes.min" + s"${JuliaCompiler.kstreamName}.byte_array_min(${translate(b)})" override def bytesMax(b: Ast.expr): String = - s"${translate(b)}.bytes.max" + s"${JuliaCompiler.kstreamName}.byte_array_max(${translate(b)})" + - override def strLength(s: Ast.expr): String = - s"${translate(s)}.size" - override def strReverse(s: Ast.expr): String = - s"${translate(s)}.reverse" + override def strLength(value: Ast.expr): String = + s"len(${translate(value)})" + override def strReverse(value: Ast.expr): String = + s"(${translate(value)})[::-1]" override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = - s"${translate(s)}[${translate(from)}..(${translate(to)} - 1)]" + s"(${translate(s)})[${translate(from)}:${translate(to)}]" override def arrayFirst(a: Ast.expr): String = - s"${translate(a)}.first" + s"${translate(a)}[0]" override def arrayLast(a: Ast.expr): String = - s"${translate(a)}.last" + s"${translate(a)}[-1]" override def arraySize(a: Ast.expr): String = - s"${translate(a)}.length" + s"len(${translate(a)})" override def arrayMin(a: Ast.expr): String = - s"${translate(a)}.min" + s"min(${translate(a)})" override def arrayMax(a: Ast.expr): String = - s"${translate(a)}.max" + s"max(${translate(a)})" + override def kaitaiStreamSize(value: Ast.expr): String = + s"${translate(value)}.size()" override def kaitaiStreamEof(value: Ast.expr): String = - s"${translate(value)}.eof?" + s"${translate(value)}.is_eof()" + override def kaitaiStreamPos(value: Ast.expr): String = + s"${translate(value)}.pos()" } From c66eaf401186b3b2866b972ba4d5c1eb8520fa18 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Tue, 8 Aug 2023 14:35:57 +0200 Subject: [PATCH 03/37] add Julia configuration to Main --- shared/src/main/scala/io/kaitai/struct/Main.scala | 4 +++- .../struct/languages/components/LanguageCompilerStatic.scala | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index 7ac34ef6e..cae03f47a 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -1,7 +1,7 @@ package io.kaitai.struct import io.kaitai.struct.format.{ClassSpec, ClassSpecs, MetaSpec} -import io.kaitai.struct.languages.{GoCompiler, NimCompiler, RustCompiler} +import io.kaitai.struct.languages.{GoCompiler, NimCompiler, RustCompiler, JuliaCompiler} import io.kaitai.struct.languages.components.LanguageCompilerStatic import io.kaitai.struct.precompile._ import io.kaitai.struct.problems.CompilationProblem @@ -80,6 +80,8 @@ object Main { new GraphvizClassCompiler(specs, spec) case GoCompiler => new GoClassCompiler(specs, spec, config) + case JuliaCompiler => + new JuliaClassCompiler(specs, spec, config) case RustCompiler => new RustClassCompiler(specs, spec, config) case ConstructClassCompiler => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala index 99f42503e..b026adf28 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala @@ -23,6 +23,7 @@ object LanguageCompilerStatic { "php" -> PHPCompiler, "python" -> PythonCompiler, "ruby" -> RubyCompiler, + "julia" -> JuliaCompiler, "rust" -> RustCompiler ) From abc3221fafefca50460487c3e5273a063e4a7ec2 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Tue, 8 Aug 2023 14:39:45 +0200 Subject: [PATCH 04/37] JuliaTranslator can handle basic expressions --- .../struct/translators/JuliaTranslator.scala | 97 ++++++++++--------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index d7371b180..b704f4b33 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -7,17 +7,32 @@ import io.kaitai.struct.format.Identifier import io.kaitai.struct.languages.{JuliaCompiler, RubyCompiler} class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { - override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = { + override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr): String = { (detectType(left), detectType(right), op) match { case (_: IntType, _: IntType, Ast.operator.Div) => - s"${translate(left)} // ${translate(right)}" + s"${translate(left)} ÷ ${translate(right)}" case _ => super.numericBinOp(left, op, right) } } - override def doStringLiteral(s: String): String = "u" + super.doStringLiteral(s) - override def doBoolLiteral(n: Boolean): String = if (n) "True" else "False" + override def binOp(op: Ast.operator): String = { + op match { + case Ast.operator.Add => "+" + case Ast.operator.Sub => "-" + case Ast.operator.Mult => "*" + case Ast.operator.Div => "/" + case Ast.operator.Mod => "%" + case Ast.operator.BitAnd => "&" + case Ast.operator.BitOr => "|" + case Ast.operator.BitXor => "⊻" + case Ast.operator.LShift => "<<" + case Ast.operator.RShift => ">>" + } + } + +// override def doStringLiteral(s: String): String = super.doStringLiteral(s) +// override def doBoolLiteral(n: Boolean): String = if (n) "True" else "False" /** * https://docs.Julia.org/2.7/reference/lexical_analysis.html#string-literals @@ -37,43 +52,39 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba ) override def doByteArrayLiteral(arr: Seq[Byte]): String = - "b\"" + Utils.hexEscapeByteArray(arr) + "\"" + s"Vector{UInt8}(${super.doByteArrayLiteral(arr)})" override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = { - importList.add("import struct") - s"struct.pack('${elts.length}b', ${elts.map(translate).mkString(", ")})" +// importList.add("import struct") +// s"struct.pack('${elts.length}b', ${elts.map(translate).mkString(", ")})" + s"Vector{UInt8}([${elts.map(translate).mkString(", ")}])" } - override def doLocalName(s: String) = { + override def doLocalName(s: String): String = { s match { - case Identifier.ITERATOR => "_" + case Identifier.ITERATOR => "_it" case Identifier.INDEX => "i" - case _ => s"self.${doName(s)}" + case _ => s"this.${doName(s)}" + } + } + override def doName(s: String): String = { + s match { + case Identifier.ITERATOR => "_it" + case _ => s } } - override def doName(s: String) = - s + override def doInternalName(id: Identifier): String = - s"self.${JuliaCompiler.publicMemberName(id)}" + s"this.${JuliaCompiler.publicMemberName(id)}" override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = - s"${JuliaCompiler.types2class(enumTypeAbs)}.$label" + s"${JuliaCompiler.types2class(enumTypeAbs)}_$label" override def doEnumById(enumTypeAbs: List[String], id: String): String = - s"${JuliaCompiler.kstreamName}.resolve_enum(${JuliaCompiler.types2class(enumTypeAbs)}, $id)" - - override def booleanOp(op: Ast.boolop) = op match { - case Ast.boolop.Or => "or" - case Ast.boolop.And => "and" - } - - override def unaryOp(op: Ast.unaryop) = op match { - case Ast.unaryop.Not => "not " - case _ => super.unaryOp(op) - } + s"${JuliaCompiler.types2class(enumTypeAbs)}($id)" override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = s"${translate(container)}[${translate(idx)}]" override def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String = - s"(${translate(ifTrue)} if ${translate(condition)} else ${translate(ifFalse)})" + s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})" // Predefined methods of various types override def strToInt(s: Ast.expr, base: Ast.expr): String = { @@ -92,21 +103,17 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba s"int(${translate(v)})" override def intToStr(i: Ast.expr, base: Ast.expr): String = { val baseStr = translate(base) - val func = baseStr match { - case "2" => "bin" - case "8" => "oct" - case "10" => "str" - case "16" => "hex" - case _ => throw new UnsupportedOperationException(baseStr) - } - - s"$func(${translate(i)})" + s"string(${translate(i)}, base = $baseStr)" + } + override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = { + importList.add("using StringEncodings") + s"decode(($bytesExpr), ${translate(encoding)})" } - override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = - s"($bytesExpr).decode(${translate(encoding)})" + + override def strConcat(left: Ast.expr, right: Ast.expr): String = s"${translate(left)} * ${translate(right)}" override def bytesLength(value: Ast.expr): String = - s"len(${translate(value)})" + s"length(${translate(value)})" override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = s"${JuliaCompiler.kstreamName}.byte_array_index(${translate(container)}, ${translate(idx)})" override def bytesFirst(a: Ast.expr): String = @@ -120,22 +127,22 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def strLength(value: Ast.expr): String = - s"len(${translate(value)})" + s"length(${translate(value)})" override def strReverse(value: Ast.expr): String = - s"(${translate(value)})[::-1]" + s"reverse(${translate(value)})" override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = s"(${translate(s)})[${translate(from)}:${translate(to)}]" override def arrayFirst(a: Ast.expr): String = - s"${translate(a)}[0]" + s"${translate(a)}[1]" override def arrayLast(a: Ast.expr): String = - s"${translate(a)}[-1]" + s"${translate(a)}[end]" override def arraySize(a: Ast.expr): String = - s"len(${translate(a)})" + s"size(${translate(a)}, 1)" override def arrayMin(a: Ast.expr): String = - s"min(${translate(a)})" + s"minimum(${translate(a)})" override def arrayMax(a: Ast.expr): String = - s"max(${translate(a)})" + s"maximum(${translate(a)})" override def kaitaiStreamSize(value: Ast.expr): String = s"${translate(value)}.size()" From c6ce2896d5055d122c626a63fa0378e9c13b4425 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Tue, 8 Aug 2023 14:40:38 +0200 Subject: [PATCH 05/37] Add fromfile functionality to JuliaClassCompiler --- .../io/kaitai/struct/JuliaClassCompiler.scala | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala index 368a291d1..fe2e77193 100644 --- a/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala @@ -1,7 +1,7 @@ package io.kaitai.struct -import io.kaitai.struct.datatype.DataType.{CalcIntType, KaitaiStreamType, UserTypeInstream} -import io.kaitai.struct.datatype.{BigEndian, CalcEndian, Endianness, FixedEndian, InheritedEndian, LittleEndian} +import io.kaitai.struct.datatype.DataType.{CalcIntType, KaitaiStreamType, UserType, UserTypeInstream} +import io.kaitai.struct.datatype.{BigEndian, CalcEndian, Endianness, FixedEndian, LittleEndian} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ import io.kaitai.struct.languages.JuliaCompiler @@ -13,6 +13,8 @@ class JuliaClassCompiler( config: RuntimeConfig ) extends ClassCompiler(classSpecs, topClass, config, JuliaCompiler) { + private val julialang = lang.asInstanceOf[JuliaCompiler] + override def compileClass(curClass: ClassSpec): Unit = { provider.nowClass = curClass @@ -22,6 +24,8 @@ class JuliaClassCompiler( AttrSpec(List(), ParentIdentifier, curClass.parentType) ) ++ ExtraAttrs.forClassSpec(curClass, lang) + curClass.types.foreach { case (typeName, _) => lang.classForwardDeclaration(curClass.name ++ List(typeName)) } + if (!curClass.doc.isEmpty) lang.classDoc(curClass.name, curClass.doc) @@ -38,8 +42,12 @@ class JuliaClassCompiler( lang.classFooter(curClass.name) // Constructor = Read() function + if (curClass.isTopLevel) + julialang.fromFile(curClass.name) + compileEagerRead(curClass.seq, curClass.meta.endian) + julialang.overrideGetProperty(curClass.name, curClass.instances) compileInstances(curClass) compileAttrReaders(curClass.seq ++ extraAttrs) @@ -48,18 +56,6 @@ class JuliaClassCompiler( compileSubclasses(curClass) } - def compileReadFunction(curClass: ClassSpec) = { - lang.classConstructorHeader( - curClass.name, - curClass.parentType, - topClassName, - curClass.meta.endian.contains(InheritedEndian), - curClass.params - ) - compileEagerRead(curClass.seq, curClass.meta.endian) - lang.classConstructorFooter - } - override def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, endian: Option[Endianness]): Unit = { // Determine datatype val dataType = instSpec.dataTypeComposite @@ -86,8 +82,8 @@ class JuliaClassCompiler( override def compileCalcEndian(ce: CalcEndian): Unit = { def renderProc(result: FixedEndian): Unit = { val v = result match { - case LittleEndian => Ast.expr.IntNum(1) - case BigEndian => Ast.expr.IntNum(0) + case LittleEndian => Ast.expr.Bool(true) + case BigEndian => Ast.expr.Bool(false) } lang.instanceCalculate(IS_LE_ID, CalcIntType, v) } From baeb13d274b023affd6fa1fa76b392f877979cf7 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Tue, 8 Aug 2023 14:47:20 +0200 Subject: [PATCH 06/37] Fix getproperty and stream methods --- .../struct/languages/JuliaCompiler.scala | 279 ++++++++++++------ 1 file changed, 185 insertions(+), 94 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index 02c75a9ae..969fd8484 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -1,7 +1,7 @@ package io.kaitai.struct.languages -import io.kaitai.struct.datatype.DataType._ -import io.kaitai.struct.datatype.{DataType, EndOfStreamError, FixedEndian, InheritedEndian, KSError, UndecidedEndiannessError, NeedRaw} +import io.kaitai.struct.datatype.DataType.{BooleanType, _} +import io.kaitai.struct.datatype.{CalcEndian, DataType, EndOfStreamError, FixedEndian, InheritedEndian, KSError, NeedRaw, UndecidedEndiannessError} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ @@ -19,31 +19,43 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with AllocateIOLocalVar with FixedContentsUsingArrayByteLiteral with UniversalDoc - with SwitchIfOps - with NoNeedForFullClassPath { + with SwitchIfOps { import JuliaCompiler._ override val translator = new JuliaTranslator(typeProvider, importList) + private val abstractTypes = new StringLanguageOutputWriter(indent) override def innerDocstrings = true + override def innerClasses: Boolean = false + override def universalFooter: Unit = { out.dec out.puts("end") + out.puts + } + + override def results(topClass: ClassSpec): Map[String, String] = + Map(outFileName(topClass.nameAsStr) -> + (outHeader.result + outImports(topClass) + abstractTypes.result + out.result) + ) + + override def classForwardDeclaration(name: List[String]): Unit = { + abstractTypes.puts(s"abstract type ${types2class("Abstract" :: name)} end") } override def indent: String = " " override def outFileName(topClassName: String): String = s"$topClassName.jl" - override def outImports(topClass: ClassSpec) = + override def outImports(topClass: ClassSpec): String = importList.toList.mkString("", "\n", "\n") override def fileHeader(topClassName: String): Unit = { outHeader.puts(s"# $headerComment") outHeader.puts - importList.add("using kaitaistruct") + importList.add("include(\"../../../runtime/julia/kaitaistruct.jl\")") out.puts // out.puts @@ -82,49 +94,56 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) } - override def classHeader(name: String): Unit = { - out.puts(s"mutable struct ${type2class(name)}") + override def classHeader(name: List[String]): Unit = { + val subtype = if (typeProvider.nowClass.isTopLevel) "" else s" <: ${types2class("Abstract" :: name)}" + out.puts(s"mutable struct ${types2class(name)}$subtype") out.inc + typeProvider.nowClass.meta.endian match { + case Some(_: CalcEndian) | Some(InheritedEndian) => + out.puts(s"_is_le::Bool") + case _ => + // no _is_le variable + } } - override def classConstructorHeader(name: String, parentType: DataType, rootClassName: String, isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + override def classFooter(name: List[String]): Unit = { + universalFooter + } + + override def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { val endianAdd = if (isHybrid) ", _is_le=None" else "" - val paramsList = Utils.join(params.map((p) => paramName(p.id)), ", ", ", ", "") + val paramsList = Utils.join(params.map(p => paramName(p.id)), "", ", ", ",") - out.puts(s"${type2class(name)}(_io, _parent = nothing, _root = nothing$endianAdd, $paramsList) = (") + out.puts(s"function ${types2class(name)}(${paramsList}_io, _parent = nothing, _root = nothing$endianAdd)") out.inc + out.puts("this = new()") // Store parameters passed to us - params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id))) + params.foreach(p => handleAssignmentSimple(p.id, paramName(p.id))) } override def runRead(name: List[String]): Unit = { - out.puts("x = new();") - out.puts("x._io = _io;") - out.puts("x._parent = _parent;") - out.puts("x._root = _root === nothing ? x : _root;") - out.puts("_read(x);") - out.puts("x") - } - - override def classConstructorFooter(): Unit = { - out.dec - out.puts(")") + typeProvider.nowClass.instances.keys.foreach(instanceIdentifier => out.puts(s"this.${idToStr(instanceIdentifier)} = nothing")) + out.puts("this._io = _io") + out.puts("this._parent = _parent") + out.puts("this._root = _root === nothing ? this : _root") + out.puts("_read(this)") + out.puts("this") } override def runReadCalc(): Unit = { - out.puts(s"if not hasattr(self, '_is_le'):") + out.puts(s"if !hasfield(${types2class(typeProvider.nowClass.name)}, :_is_le)") out.inc - out.puts(s"raise ${ksErrorName(UndecidedEndiannessError)}(" + "\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\")") + out.puts(s"throw(${ksErrorName(UndecidedEndiannessError)}(" + "\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\"))") out.dec - out.puts(s"elif self._is_le == True:") + out.puts(s"elseif this._is_le == true") out.inc - out.puts("self._read_le()") + out.puts("_read_le(this)") out.dec - out.puts("elif self._is_le == False:") + out.puts("elseif this._is_le == false") out.inc - out.puts("self._read_be()") - out.dec + out.puts("_read_be(this)") + universalFooter } override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { @@ -132,17 +151,30 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(e) => s"_${e.toSuffix}" case None => "" } - out.puts(s"function _read$suffix(this::${type2class(typeProvider.nowClass.name.last)})") + out.puts(s"function _read$suffix(this::${types2class(typeProvider.nowClass.name)})") out.inc } - override def readFooter() = { + override def readFooter(): Unit = { out.puts("nothing") universalFooter } override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { - out.puts(s"${idToStr(attrName)}::$attrType") + out.puts(s"${idToStr(attrName)}::${kaitaiType2NativeType(attrType)}") + } + + override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = { + out.puts(s"${idToStr(attrName)}::Union{Nothing, ${kaitaiType2NativeType(attrType)}}") + } + + def fromFile(name: List[String]): Unit = { + if (typeProvider.nowClass.params.isEmpty) { + out.puts(s"function from_file(filename::String)::${types2class(name)}") + out.inc + out.puts(s"${types2class(name)}($kstreamName(open(filename, ${'"'}r${'"'})))") + universalFooter + } } override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} @@ -200,8 +232,8 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } s"$kstreamName.$procName($srcExpr, ${expression(xorValue)})" case ProcessZlib => - importList.add("import zlib") - s"zlib.decompress($srcExpr)" + importList.add("using Zlib") + s"decompress($srcExpr)" case ProcessRotate(isLeft, rotValue) => val expr = if (isLeft) { expression(rotValue) @@ -224,18 +256,17 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"_process = $procClass(${args.map(expression).mkString(", ")})") s"_process.decode($srcExpr)" } - handleAssignment(varDest, expr, rep, false) + handleAssignment(varDest, expr, rep, isRaw = false) } override def normalIO: String = "this._io" override def allocateIO(varName: Identifier, rep: RepeatSpec): String = { - val varStr = privateMemberName(varName) val ioName = s"_io_${idToStr(varName)}" val args = getRawIdExpr(varName, rep) - out.puts(s"$ioName = $kstreamName(BytesIO($args))") + out.puts(s"$ioName = $kstreamName(IOBuffer($args))") ioName } @@ -244,7 +275,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) rep match { case NoRepeat => memberName case RepeatExpr(_) => s"$memberName[i]" - case _ => s"$memberName[-1]" + case _ => s"$memberName[end]" } } @@ -254,19 +285,19 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def pushPos(io: String): Unit = - out.puts(s"_pos = $io.pos()") + out.puts(s"_pos = pos($io)") override def seek(io: String, pos: Ast.expr): Unit = - out.puts(s"$io.seek(${expression(pos)})") + out.puts(s"seek($io, ${expression(pos)})") override def popPos(io: String): Unit = - out.puts(s"$io.seek(_pos)") + out.puts(s"seek($io, _pos)") override def alignToByte(io: String): Unit = - out.puts(s"$io.align_to_byte()") + out.puts(s"align_to_byte($io)") override def attrDebugStart(attrId: Identifier, attrType: DataType, ios: Option[String], rep: RepeatSpec): Unit = { - ios.foreach { (io) => + ios.foreach { io => val name = attrId match { case _: RawIdentifier | _: SpecialIdentifier => return case _ => idToStr(attrId) @@ -302,7 +333,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def condIfHeader(expr: Ast.expr): Unit = { - out.puts(s"if ${expression(expr)}:") + out.puts(s"if ${expression(expr)}") out.inc } @@ -316,11 +347,11 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = { out.puts("i = 0") - out.puts(s"while not $io.is_eof():") + out.puts(s"while !iseof($io)") out.inc } override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = - out.puts(s"${privateMemberName(id)}.append($expr)") + out.puts(s"push!(${privateMemberName(id)}, $expr)") override def condRepeatEosFooter: Unit = { out.puts("i += 1") @@ -328,7 +359,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, repeatExpr: expr): Unit = { - out.puts(s"for i in range(${expression(repeatExpr)}):") + out.puts(s"for i in 0:${expression(repeatExpr)}-1") out.inc } override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = @@ -336,24 +367,24 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, untilExpr: expr): Unit = { out.puts("i = 0") - out.puts("while True:") + out.puts("while true") out.inc } override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = { val tmpName = translator.doName(if (isRaw) Identifier.ITERATOR2 else Identifier.ITERATOR) out.puts(s"$tmpName = $expr") - out.puts(s"${privateMemberName(id)}.append($tmpName)") + out.puts(s"push!(${privateMemberName(id)}, $tmpName)") } override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, untilExpr: expr): Unit = { typeProvider._currentIteratorType = Some(dataType) - out.puts(s"if ${expression(untilExpr)}:") + out.puts(s"if ${expression(untilExpr)}") out.inc out.puts("break") - out.dec + universalFooter out.puts("i += 1") - out.dec + universalFooter } override def handleAssignmentSimple(id: Identifier, expr: String): Unit = @@ -365,44 +396,44 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { dataType match { case t: ReadableType => - s"$io.read_${t.apiCall(defEndian)}()" + s"read${Utils.capitalize(t.apiCall(defEndian))}($io)" case blt: BytesLimitType => - s"$io.read_bytes(${expression(blt.size)})" + s"read_bytes($io, convert(UInt, ${expression(blt.size)}))" case _: BytesEosType => - s"$io.read_bytes_full()" + s"read_bytes_full($io)" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io.read_bytes_term($terminator, ${bool2Py(include)}, ${bool2Py(consume)}, ${bool2Py(eosError)})" + s"read_bytes_term($io, convert(UInt8, $terminator), $include, $consume, $eosError)" case BitsType1(bitEndian) => - s"$io.read_bits_int_${bitEndian.toSuffix}(1) != 0" + s"read_bits_int_${bitEndian.toSuffix}($io, 1) != 0" case BitsType(width: Int, bitEndian) => - s"$io.read_bits_int_${bitEndian.toSuffix}($width)" + s"read_bits_int_${bitEndian.toSuffix}($io, $width)" case t: UserType => - val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") + val addParams = Utils.join(t.args.map(a => translator.translate(a)), "", ", ", ", ") val addArgs = if (t.isOpaque) { "" } else { val parent = t.forcedParent match { case Some(USER_TYPE_NO_PARENT) => "None" case Some(fp) => translator.translate(fp) - case None => "self" + case None => "this" } val addEndian = t.classSpec.get.meta.endian match { - case Some(InheritedEndian) => ", self._is_le" + case Some(InheritedEndian) => ", this._is_le" case _ => "" } - s", $parent, self._root$addEndian" + s", $parent, this._root$addEndian" } s"${userType2class(t)}($addParams$io$addArgs)" } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName.bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.bytes_terminate($expr1, $term, ${bool2Py(include)})" + case Some(term) => s"$kstreamName.bytes_terminate($expr1, $term, ${include})" case None => expr1 } expr2 @@ -423,56 +454,75 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { - out.puts(s"if _on == ${expression(condition)}:") + out.puts(s"if _on == ${expression(condition)}") out.inc } override def switchIfCaseStart(condition: Ast.expr): Unit = { - out.puts(s"elif _on == ${expression(condition)}:") + out.puts(s"elseif _on == ${expression(condition)}") out.inc } - override def switchIfCaseEnd(): Unit = + override def switchIfCaseEnd(): Unit = { out.dec + } override def switchIfElseStart(): Unit = { - out.puts(s"else:") + out.puts(s"else") out.inc } - override def switchIfEnd(): Unit = {} + override def switchIfEnd(): Unit = { + out.puts("end") + } + + override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { + out.puts(s"function _get_${publicMemberName(instName)}(this::${types2class(className)})") + out.inc + } - override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { - out.puts("@property") - out.puts(s"def ${publicMemberName(instName)}(self):") + def overrideGetProperty(className: List[String], instances: Map[InstanceIdentifier, InstanceSpec]): Unit = { + if (instances.isEmpty) + return + out.puts(s"function Base.getproperty(obj::${types2class(className)}, sym::Symbol)") out.inc + var c = "if" + instances.keys.foreach(instName => { + out.puts(s"$c sym === :${publicMemberName(instName)}") + c = "elseif" + out.inc + out.puts(s"return _get_${publicMemberName(instName)}(obj)") + out.dec + }) + out.puts("else") + out.inc + out.puts("return getfield(obj, sym)") + universalFooter + universalFooter } override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { - out.puts(s"if hasattr(self, '${idToStr(instName)}'):") + out.puts(s"if ${privateMemberName(instName)} !== nothing") out.inc out.puts(s"return ${privateMemberName(instName)}") - out.dec - out.puts + universalFooter } override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { // workaround to avoid Python raising an "AttributeError: instance has no attribute" - out.puts(s"return getattr(self, '${idToStr(instName)}', None)") + out.puts(s"${privateMemberName(instName)}") } - override def enumDeclaration(curClass: String, enumName: String, enumColl: Seq[(Long, String)]): Unit = { - importList.add("from enum import Enum") - - out.puts - out.puts(s"class ${type2class(enumName)}(Enum):") + override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { + val fullEnumName: List[String] = curClass ++ List(enumName) + out.puts(s"@enum ${types2class(fullEnumName)} begin") out.inc - enumColl.foreach { case (id: Long, label: String) => out.puts(s"$label = ${translator.doIntLiteral(id)}") } - out.dec + enumColl.foreach { case (id: Long, label: EnumValueSpec) => out.puts(s"${enumToStr(fullEnumName, label.name)} = ${translator.doIntLiteral(id)}") } + universalFooter } - override def debugClassSequence(seq: List[AttrSpec]) = { - val seqStr = seq.map((attr) => "\"" + idToStr(attr.id) + "\"").mkString(", ") + override def debugClassSequence(seq: List[AttrSpec]): Unit = { + val seqStr = seq.map(attr => "\"" + idToStr(attr.id) + "\"").mkString(", ") out.puts(s"SEQ_FIELDS = [$seqStr]") } @@ -484,8 +534,6 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec } - def bool2Py(b: Boolean): String = if (b) { "True" } else { "False" } - override def idToStr(id: Identifier): String = JuliaCompiler.idToStr(id) override def publicMemberName(id: Identifier): String = JuliaCompiler.publicMemberName(id) @@ -504,10 +552,10 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) errArgs: List[Ast.expr] ): Unit = { val errArgsStr = errArgs.map(translator.translate).mkString(", ") - out.puts(s"if not ${translator.translate(checkExpr)}:") + out.puts(s"if !(${translator.translate(checkExpr)})") out.inc - out.puts(s"raise ${ksErrorName(err)}($errArgsStr)") - out.dec + out.puts(s"throw(${ksErrorName(err)}($errArgsStr))") + universalFooter } def userType2class(t: UserType): String = { @@ -547,11 +595,54 @@ object JuliaCompiler extends LanguageCompilerStatic } override def kstreamName: String = "KaitaiStream" - override def kstructName: String = "KaitaiStruct" + override def kstructName: String = "Any" override def ksErrorName(err: KSError): String = err match { case EndOfStreamError => "EOFError" - case _ => s"kaitaistruct.${err.name}" + case _ => s"${err.name}" } - def types2class(name: List[String]): String = name.map(x => type2class(x)).mkString(".") + def types2class(name: List[String]): String = name.map(x => type2class(x)).mkString("_") + + def kaitaiType2NativeType(attrType: DataType): String = { + attrType match { + case Int1Type(false) => "UInt8" + case IntMultiType(false, Width2, _) => "UInt16" + case IntMultiType(false, Width4, _) => "UInt32" + case IntMultiType(false, Width8, _) => "UInt64" + + case Int1Type(true) => "Int8" + case IntMultiType(true, Width2, _) => "Int16" + case IntMultiType(true, Width4, _) => "Int32" + case IntMultiType(true, Width8, _) => "Int64" + + case FloatMultiType(Width4, _) => "Float32" + case FloatMultiType(Width8, _) => "Float64" + + case BitsType(_, _) => "UInt64" + + case _: BooleanType => "Bool" + case CalcIntType => "Int" + case CalcFloatType => "Float64" + + case _: StrType => "String" + case _: BytesType => "Vector{UInt8}" + + case AnyType => "Any" + case KaitaiStreamType | OwnedKaitaiStreamType => kstreamName + case KaitaiStructType | CalcKaitaiStructType => kstructName + case t: UserType => types2class(t.classSpec match { + case Some(cs) => if (cs.isTopLevel) cs.name else "Abstract" :: cs.name + case None => t.name + }) + + case t: EnumType => types2class(t.enumSpec.get.name) + + case at: ArrayType => s"Vector{${kaitaiType2NativeType(at.elType)}}" + + case st: SwitchType => kaitaiType2NativeType(st.combinedType) + } + + } + def enumToStr(typeName: List[String], enumName: String): String = + typeName.mkString("_") + "__" + enumName } From 6707effa1f76ab9e2621fcbbaa72a180eb9811f9 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Tue, 8 Aug 2023 15:09:49 +0200 Subject: [PATCH 07/37] Use correct conversion --- .../io/kaitai/struct/translators/JuliaTranslator.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index b704f4b33..16c1398bb 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -96,11 +96,11 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba s"int(${translate(s)}$add)" } override def enumToInt(v: Ast.expr, et: EnumType): String = - s"${translate(v)}.value" + s"Int(${translate(v)})" override def boolToInt(v: Ast.expr): String = - s"int(${translate(v)})" + s"Int(${translate(v)})" override def floatToInt(v: Ast.expr): String = - s"int(${translate(v)})" + s"trunc(${translate(v)})" override def intToStr(i: Ast.expr, base: Ast.expr): String = { val baseStr = translate(base) s"string(${translate(i)}, base = $baseStr)" From 61d7cf37d9b92341b293df9cbdfc8929e02b72d1 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Tue, 8 Aug 2023 15:13:24 +0200 Subject: [PATCH 08/37] add string to int parsing --- .../scala/io/kaitai/struct/translators/JuliaTranslator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 16c1398bb..06e4cc073 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -93,7 +93,7 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba case "10" => "" case _ => s", $baseStr" } - s"int(${translate(s)}$add)" + s"parse(Int64, ${translate(s)}, $add)" } override def enumToInt(v: Ast.expr, et: EnumType): String = s"Int(${translate(v)})" From 4f670f3e2fab48db040317d570dd1cd411411c30 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Tue, 8 Aug 2023 15:39:20 +0200 Subject: [PATCH 09/37] fix kaitaistream method calls --- .../io/kaitai/struct/translators/JuliaTranslator.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 06e4cc073..62f436d83 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -77,7 +77,7 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba s"this.${JuliaCompiler.publicMemberName(id)}" override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = - s"${JuliaCompiler.types2class(enumTypeAbs)}_$label" + s"${JuliaCompiler.enumToStr(enumTypeAbs, label)}" override def doEnumById(enumTypeAbs: List[String], id: String): String = s"${JuliaCompiler.types2class(enumTypeAbs)}($id)" @@ -145,9 +145,9 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba s"maximum(${translate(a)})" override def kaitaiStreamSize(value: Ast.expr): String = - s"${translate(value)}.size()" + s"size(${translate(value)})" override def kaitaiStreamEof(value: Ast.expr): String = - s"${translate(value)}.is_eof()" + s"iseof(${translate(value)})" override def kaitaiStreamPos(value: Ast.expr): String = - s"${translate(value)}.pos()" + s"pos(${translate(value)})" } From ca5604f37c17266d2e920e5e1eb0a73f5b6ddb6e Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Wed, 9 Aug 2023 16:17:08 +0200 Subject: [PATCH 10/37] Fix enum translation to string --- .../scala/io/kaitai/struct/languages/JuliaCompiler.scala | 2 +- .../io/kaitai/struct/translators/JuliaTranslator.scala | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index 969fd8484..d36f9fb9d 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -514,7 +514,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { - val fullEnumName: List[String] = curClass ++ List(enumName) + val fullEnumName: List[String] = curClass :+ enumName out.puts(s"@enum ${types2class(fullEnumName)} begin") out.inc enumColl.foreach { case (id: Long, label: EnumValueSpec) => out.puts(s"${enumToStr(fullEnumName, label.name)} = ${translator.doIntLiteral(id)}") } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 62f436d83..c3bf6fd81 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -89,11 +89,7 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba // Predefined methods of various types override def strToInt(s: Ast.expr, base: Ast.expr): String = { val baseStr = translate(base) - val add = baseStr match { - case "10" => "" - case _ => s", $baseStr" - } - s"parse(Int64, ${translate(s)}, $add)" + s"parse(Int64, ${translate(s)}, $baseStr)" } override def enumToInt(v: Ast.expr, et: EnumType): String = s"Int(${translate(v)})" From 44b8c4677837176547529374daf2cb55ed811f52 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Thu, 10 Aug 2023 12:34:37 +0200 Subject: [PATCH 11/37] fix julia indexing --- .../io/kaitai/struct/translators/JuliaTranslator.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index c3bf6fd81..f2d387b43 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -5,6 +5,7 @@ import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format.Identifier import io.kaitai.struct.languages.{JuliaCompiler, RubyCompiler} +import io.kaitai.struct.exprlang.ConstEvaluator class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr): String = { @@ -82,7 +83,9 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba s"${JuliaCompiler.types2class(enumTypeAbs)}($id)" override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = - s"${translate(container)}[${translate(idx)}]" + s"${translate(container)}[${translateIndex(idx)}]" + def translateIndex(idx: Ast.expr): String = + (ConstEvaluator.evaluateIntConst(idx).get + 1).toString override def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String = s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})" @@ -127,10 +130,10 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def strReverse(value: Ast.expr): String = s"reverse(${translate(value)})" override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = - s"(${translate(s)})[${translate(from)}:${translate(to)}]" + s"${translate(s)}[${translate(from)}:${translate(to)}]" override def arrayFirst(a: Ast.expr): String = - s"${translate(a)}[1]" + s"${translate(a)}[begin]" override def arrayLast(a: Ast.expr): String = s"${translate(a)}[end]" override def arraySize(a: Ast.expr): String = From 5867b5f4b8aa6988693282985748ac0946e4255e Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Thu, 10 Aug 2023 12:35:10 +0200 Subject: [PATCH 12/37] fix julia functions calls from runtime lib --- .../main/scala/io/kaitai/struct/languages/JuliaCompiler.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index d36f9fb9d..28a45338e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -429,11 +429,11 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { val expr1 = padRight match { - case Some(padByte) => s"$kstreamName.bytes_strip_right($expr0, $padByte)" + case Some(padByte) => s"bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.bytes_terminate($expr1, $term, ${include})" + case Some(term) => s"bytes_terminate($expr1, $term, ${include})" case None => expr1 } expr2 From dc92064ea89136aefbbed38a02e6c4c56748ab61 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Thu, 10 Aug 2023 13:08:08 +0200 Subject: [PATCH 13/37] fix byte array indexing --- .../scala/io/kaitai/struct/translators/JuliaTranslator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index f2d387b43..8df2b6eea 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -114,7 +114,7 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def bytesLength(value: Ast.expr): String = s"length(${translate(value)})" override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = - s"${JuliaCompiler.kstreamName}.byte_array_index(${translate(container)}, ${translate(idx)})" + s"${translate(container)}[${translateIndex(idx)}]" override def bytesFirst(a: Ast.expr): String = bytesSubscript(a, Ast.expr.IntNum(0)) override def bytesLast(a: Ast.expr): String = From b1ee9188c0c5ad440c2b80a805d53f7b14d2e5c5 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Thu, 10 Aug 2023 13:08:21 +0200 Subject: [PATCH 14/37] fix opaque class declataion --- .../kaitai/struct/languages/JuliaCompiler.scala | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index 28a45338e..717f27bf5 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -85,13 +85,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = { val name = classSpec.name.head - out.puts( - if (config.pythonPackage.nonEmpty) { - s"from ${config.pythonPackage} import $name" - } else { - s"import $name" - } - ) + out.puts(s"include(${'"'}$name.jl${'"'})") } override def classHeader(name: List[String]): Unit = { @@ -211,14 +205,14 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(attrName)} = self._io.ensure_fixed_contents($contents)") override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { - out.puts("if self._is_le:") + out.puts("if this._is_le") out.inc leProc() out.dec - out.puts("else:") + out.puts("else") out.inc beProc() - out.dec + universalFooter } override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { @@ -566,7 +560,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } else { "" } - s"$prefix${types2class(name)}" + s"${types2class(name)}" } } From 21c75939b177f4225f469a59606e1e20bd1b186b Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Thu, 10 Aug 2023 13:58:21 +0200 Subject: [PATCH 15/37] move enum declaration to file header --- .../struct/languages/JuliaCompiler.scala | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index 717f27bf5..ff7338078 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -24,7 +24,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) import JuliaCompiler._ override val translator = new JuliaTranslator(typeProvider, importList) - private val abstractTypes = new StringLanguageOutputWriter(indent) + private val abstractTypesAndEnums = new StringLanguageOutputWriter(indent) override def innerDocstrings = true @@ -38,11 +38,11 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def results(topClass: ClassSpec): Map[String, String] = Map(outFileName(topClass.nameAsStr) -> - (outHeader.result + outImports(topClass) + abstractTypes.result + out.result) + (outHeader.result + outImports(topClass) + abstractTypesAndEnums.result + out.result) ) override def classForwardDeclaration(name: List[String]): Unit = { - abstractTypes.puts(s"abstract type ${types2class("Abstract" :: name)} end") + abstractTypesAndEnums.puts(s"abstract type ${types2class("Abstract" :: name)} end") } override def indent: String = " " @@ -111,6 +111,8 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"function ${types2class(name)}(${paramsList}_io, _parent = nothing, _root = nothing$endianAdd)") out.inc out.puts("this = new()") + if (isHybrid) + out.puts("this._is_le = _is_le") // Store parameters passed to us params.foreach(p => handleAssignmentSimple(p.id, paramName(p.id))) @@ -509,10 +511,12 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { val fullEnumName: List[String] = curClass :+ enumName - out.puts(s"@enum ${types2class(fullEnumName)} begin") - out.inc - enumColl.foreach { case (id: Long, label: EnumValueSpec) => out.puts(s"${enumToStr(fullEnumName, label.name)} = ${translator.doIntLiteral(id)}") } - universalFooter + abstractTypesAndEnums.puts(s"@enum ${types2class(fullEnumName)} begin") + abstractTypesAndEnums.inc + enumColl.foreach { case (id: Long, label: EnumValueSpec) => abstractTypesAndEnums.puts(s"${enumToStr(fullEnumName, label.name)} = ${translator.doIntLiteral(id)}") } + abstractTypesAndEnums.dec + abstractTypesAndEnums.puts("end") + abstractTypesAndEnums.puts } override def debugClassSequence(seq: List[AttrSpec]): Unit = { From 1fcea5a667a0a9346f46cd2f4e676b585da31438 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Fri, 11 Aug 2023 15:23:01 +0200 Subject: [PATCH 16/37] fix enum, strToInt and method calls --- .../io/kaitai/struct/translators/JuliaTranslator.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 8df2b6eea..64ddc8df7 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -80,7 +80,7 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = s"${JuliaCompiler.enumToStr(enumTypeAbs, label)}" override def doEnumById(enumTypeAbs: List[String], id: String): String = - s"${JuliaCompiler.types2class(enumTypeAbs)}($id)" + s"resolve_enum(${JuliaCompiler.types2class(enumTypeAbs)}, $id)" override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = s"${translate(container)}[${translateIndex(idx)}]" @@ -92,7 +92,7 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba // Predefined methods of various types override def strToInt(s: Ast.expr, base: Ast.expr): String = { val baseStr = translate(base) - s"parse(Int64, ${translate(s)}, $baseStr)" + s"parse(Int64, ${translate(s)}, base=$baseStr)" } override def enumToInt(v: Ast.expr, et: EnumType): String = s"Int(${translate(v)})" @@ -120,9 +120,9 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def bytesLast(a: Ast.expr): String = bytesSubscript(a, Ast.expr.IntNum(-1)) override def bytesMin(b: Ast.expr): String = - s"${JuliaCompiler.kstreamName}.byte_array_min(${translate(b)})" + s"byte_array_min(${JuliaCompiler.kstreamName}, ${translate(b)})" override def bytesMax(b: Ast.expr): String = - s"${JuliaCompiler.kstreamName}.byte_array_max(${translate(b)})" + s"byte_array_max(${JuliaCompiler.kstreamName}, ${translate(b)})" override def strLength(value: Ast.expr): String = @@ -137,7 +137,7 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def arrayLast(a: Ast.expr): String = s"${translate(a)}[end]" override def arraySize(a: Ast.expr): String = - s"size(${translate(a)}, 1)" + s"Base.size(${translate(a)}, 1)" override def arrayMin(a: Ast.expr): String = s"minimum(${translate(a)})" override def arrayMax(a: Ast.expr): String = From 89dc9f53769130a56c9e1b8db2c1aab65b757712 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Fri, 11 Aug 2023 15:23:11 +0200 Subject: [PATCH 17/37] introduce tree value logic --- .../kaitai/struct/languages/JuliaCompiler.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index ff7338078..43ee58bab 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -94,7 +94,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc typeProvider.nowClass.meta.endian match { case Some(_: CalcEndian) | Some(InheritedEndian) => - out.puts(s"_is_le::Bool") + out.puts(s"_is_le::Union{Bool, Nothing}") case _ => // no _is_le variable } @@ -128,17 +128,17 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def runReadCalc(): Unit = { - out.puts(s"if !hasfield(${types2class(typeProvider.nowClass.name)}, :_is_le)") - out.inc - out.puts(s"throw(${ksErrorName(UndecidedEndiannessError)}(" + "\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\"))") - out.dec - out.puts(s"elseif this._is_le == true") + out.puts(s"if this._is_le == true") out.inc out.puts("_read_le(this)") out.dec out.puts("elseif this._is_le == false") out.inc out.puts("_read_be(this)") + out.dec + out.puts(s"else") + out.inc + out.puts(s"throw(${ksErrorName(UndecidedEndiannessError)}(" + "\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\"))") universalFooter } @@ -511,7 +511,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { val fullEnumName: List[String] = curClass :+ enumName - abstractTypesAndEnums.puts(s"@enum ${types2class(fullEnumName)} begin") + abstractTypesAndEnums.puts(s"@enum ${types2class(fullEnumName)}::Int64 begin") abstractTypesAndEnums.inc enumColl.foreach { case (id: Long, label: EnumValueSpec) => abstractTypesAndEnums.puts(s"${enumToStr(fullEnumName, label.name)} = ${translator.doIntLiteral(id)}") } abstractTypesAndEnums.dec @@ -595,7 +595,7 @@ object JuliaCompiler extends LanguageCompilerStatic override def kstreamName: String = "KaitaiStream" override def kstructName: String = "Any" override def ksErrorName(err: KSError): String = err match { - case EndOfStreamError => "EOFError" + case EndOfStreamError => "ErrorException" case _ => s"${err.name}" } From d8169ee733385bb8096ca243b33b15fe17714ff1 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Fri, 11 Aug 2023 16:49:18 +0200 Subject: [PATCH 18/37] fix indexing and invalid enums --- .../struct/languages/JuliaCompiler.scala | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index 43ee58bab..b20e876f1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -105,7 +105,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { - val endianAdd = if (isHybrid) ", _is_le=None" else "" + val endianAdd = if (isHybrid) ", _is_le=nothing" else "" val paramsList = Utils.join(params.map(p => paramName(p.id)), "", ", ", ",") out.puts(s"function ${types2class(name)}(${paramsList}_io, _parent = nothing, _root = nothing$endianAdd)") @@ -222,11 +222,11 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val expr = proc match { case ProcessXor(xorValue) => - val procName = translator.detectType(xorValue) match { - case _: IntType => "process_xor_one" - case _: BytesType => "process_xor_many" + val xorValueStr = translator.detectType(xorValue) match { + case _: IntType => translator.doCast(xorValue, Int1Type(true)) + case _ => expression(xorValue) } - s"$kstreamName.$procName($srcExpr, ${expression(xorValue)})" + s"process_xor($srcExpr, UInt8($xorValueStr))" case ProcessZlib => importList.add("using Zlib") s"decompress($srcExpr)" @@ -236,7 +236,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } else { s"8 - (${expression(rotValue)})" } - s"$kstreamName.process_rotate_left($srcExpr, $expr, 1)" + s"process_rotate_left($srcExpr, $expr, 1)" case ProcessCustom(name, args) => val procClass = if (name.length == 1) { val onlyName = name.head @@ -355,7 +355,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, repeatExpr: expr): Unit = { - out.puts(s"for i in 0:${expression(repeatExpr)}-1") + out.puts(s"for i in 1:${expression(repeatExpr)}") out.inc } override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = @@ -409,7 +409,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) "" } else { val parent = t.forcedParent match { - case Some(USER_TYPE_NO_PARENT) => "None" + case Some(USER_TYPE_NO_PARENT) => "nothing" case Some(fp) => translator.translate(fp) case None => "this" } @@ -633,7 +633,7 @@ object JuliaCompiler extends LanguageCompilerStatic case None => t.name }) - case t: EnumType => types2class(t.enumSpec.get.name) + case t: EnumType => s"Union{${types2class(t.enumSpec.get.name)}, Integer}" case at: ArrayType => s"Vector{${kaitaiType2NativeType(at.elType)}}" From c77aebee1b06242f7f6821ac9ebd841b4c3aabf1 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Fri, 11 Aug 2023 16:49:48 +0200 Subject: [PATCH 19/37] fix bytes operations and floor division --- .../struct/translators/JuliaTranslator.scala | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 64ddc8df7..38139adf3 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -11,7 +11,9 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr): String = { (detectType(left), detectType(right), op) match { case (_: IntType, _: IntType, Ast.operator.Div) => - s"${translate(left)} ÷ ${translate(right)}" + s"fld(${translate(left)}, ${translate(right)})" + case (_: IntType, _: IntType, Ast.operator.Mod) => + s"mod(${translate(left)}, ${translate(right)})" case _ => super.numericBinOp(left, op, right) } @@ -115,14 +117,14 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba s"length(${translate(value)})" override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = s"${translate(container)}[${translateIndex(idx)}]" - override def bytesFirst(a: Ast.expr): String = - bytesSubscript(a, Ast.expr.IntNum(0)) - override def bytesLast(a: Ast.expr): String = - bytesSubscript(a, Ast.expr.IntNum(-1)) + override def bytesFirst(container: Ast.expr): String = + s"${translate(container)}[begin]" + override def bytesLast(container: Ast.expr): String = + s"${translate(container)}[end]" override def bytesMin(b: Ast.expr): String = - s"byte_array_min(${JuliaCompiler.kstreamName}, ${translate(b)})" + s"minimum(${translate(b)})" override def bytesMax(b: Ast.expr): String = - s"byte_array_max(${JuliaCompiler.kstreamName}, ${translate(b)})" + s"maximum(${translate(b)})" override def strLength(value: Ast.expr): String = @@ -130,7 +132,7 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def strReverse(value: Ast.expr): String = s"reverse(${translate(value)})" override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = - s"${translate(s)}[${translate(from)}:${translate(to)}]" + s"(${translate(s)})[${translateIndex(from)}:${translate(to)}]" override def arrayFirst(a: Ast.expr): String = s"${translate(a)}[begin]" From c9cf3ac7ff340ad0b95390d4ac7decd95f0a21dc Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Sun, 8 Oct 2023 18:23:04 +0200 Subject: [PATCH 20/37] escape dollar sign --- .../io/kaitai/struct/translators/JuliaTranslator.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 38139adf3..346c23479 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -34,11 +34,11 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba } } -// override def doStringLiteral(s: String): String = super.doStringLiteral(s) +// override def doStringLiteral(s: String): String = "raw" + super.doStringLiteral(s) // override def doBoolLiteral(n: Boolean): String = if (n) "True" else "False" /** - * https://docs.Julia.org/2.7/reference/lexical_analysis.html#string-literals + * https://docs.python.org/2.7/reference/lexical_analysis.html#string-literals * https://docs.Julia.org/3.6/reference/lexical_analysis.html#string-and-bytes-literals */ override val asciiCharQuoteMap: Map[Char, String] = Map( @@ -47,11 +47,11 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba '\r' -> "\\r", '"' -> "\\\"", '\\' -> "\\\\", - '\u0007' -> "\\a", '\f' -> "\\f", '\u000b' -> "\\v", - '\b' -> "\\b" + '\b' -> "\\b", + '$' -> "\\$" ) override def doByteArrayLiteral(arr: Seq[Byte]): String = From b60ba6d93653554b68720d18f6f0328ba8e07726 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Sun, 8 Oct 2023 18:23:39 +0200 Subject: [PATCH 21/37] introduce debug sequence support --- .../io/kaitai/struct/JuliaClassCompiler.scala | 3 + .../struct/languages/JuliaCompiler.scala | 115 ++++++++++++------ 2 files changed, 80 insertions(+), 38 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala index fe2e77193..611ed6234 100644 --- a/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala @@ -32,6 +32,9 @@ class JuliaClassCompiler( // Enums declaration defines types, so they need to go first compileEnums(curClass) + if (lang.config.readStoresPos) + lang.debugClassSequence(curClass.seq) + // Basic struct declaration lang.classHeader(curClass.name) compileAttrDeclarations(curClass.seq ++ curClass.params ++ extraAttrs) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index b20e876f1..b3340e86c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -55,7 +55,8 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) outHeader.puts(s"# $headerComment") outHeader.puts - importList.add("include(\"../../../runtime/julia/kaitaistruct.jl\")") + importList.add("include(\"../../../runtime/julia/KaitaiStruct/src/KaitaiStruct.jl\")") + importList.add("using .KaitaiStruct") out.puts // out.puts @@ -98,6 +99,12 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _ => // no _is_le variable } + if (config.readStoresPos) { + out.puts("_attrStart::Dict") + out.puts("_attrEnd::Dict") + out.puts("_arrStart::Dict") + out.puts("_arrEnd::Dict") + } } override def classFooter(name: List[String]): Unit = { @@ -116,15 +123,32 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // Store parameters passed to us params.foreach(p => handleAssignmentSimple(p.id, paramName(p.id))) - } - override def runRead(name: List[String]): Unit = { - typeProvider.nowClass.instances.keys.foreach(instanceIdentifier => out.puts(s"this.${idToStr(instanceIdentifier)} = nothing")) out.puts("this._io = _io") out.puts("this._parent = _parent") out.puts("this._root = _root === nothing ? this : _root") - out.puts("_read(this)") + + if (config.readStoresPos) { + out.puts("this._attrStart = Dict()") + out.puts("this._attrEnd = Dict()") + out.puts("this._arrStart = Dict()") + out.puts("this._arrEnd = Dict()") + } + } + + override def attrInit(attr: AttrLikeSpec): Unit = { + if (attr.isNullable) + out.puts(s"this.${idToStr(attr.id)} = nothing") + } + + override def classConstructorFooter: Unit = { out.puts("this") + universalFooter + } + + override def runRead(name: List[String]): Unit = { + typeProvider.nowClass.instances.keys.foreach(instanceIdentifier => out.puts(s"this.${idToStr(instanceIdentifier)} = nothing")) + out.puts("_read(this)") } override def runReadCalc(): Unit = { @@ -157,7 +181,11 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { - out.puts(s"${idToStr(attrName)}::${kaitaiType2NativeType(attrType)}") + if (isNullable) { + out.puts(s"${idToStr(attrName)}::Union{Nothing, ${kaitaiType2NativeType(attrType)}}") + } else { + out.puts(s"${idToStr(attrName)}::${kaitaiType2NativeType(attrType)}") + } } override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = { @@ -204,7 +232,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = - out.puts(s"${privateMemberName(attrName)} = self._io.ensure_fixed_contents($contents)") + out.puts(s"#ensure_fixed_content") override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { out.puts("if this._is_le") @@ -214,7 +242,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("else") out.inc beProc() - universalFooter + blockScopeFooter } override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { @@ -223,13 +251,13 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val expr = proc match { case ProcessXor(xorValue) => val xorValueStr = translator.detectType(xorValue) match { - case _: IntType => translator.doCast(xorValue, Int1Type(true)) + case _: IntType => s"UInt8(${translator.doCast(xorValue, Int1Type(true))})" case _ => expression(xorValue) } - s"process_xor($srcExpr, UInt8($xorValueStr))" + s"process_xor($srcExpr, $xorValueStr)" case ProcessZlib => - importList.add("using Zlib") - s"decompress($srcExpr)" + importList.add("using CodecZlib") + s"transcode(GzipDecompressor, $srcExpr)" case ProcessRotate(isLeft, rotValue) => val expr = if (isLeft) { expression(rotValue) @@ -241,16 +269,14 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val procClass = if (name.length == 1) { val onlyName = name.head val className = type2class(onlyName) - importList.add(s"from $onlyName import $className") className } else { val pkgName = name.init.mkString(".") - importList.add(s"import $pkgName") s"$pkgName.${type2class(name.last)}" } - out.puts(s"_process = $procClass(${args.map(expression).mkString(", ")})") - s"_process.decode($srcExpr)" + // out.puts(s"_process = $procClass(${args.map(expression).mkString(", ")})") + s"$procClass.decode(${args.map(expression).mkString(", ")}$srcExpr)" } handleAssignment(varDest, expr, rep, isRaw = false) } @@ -293,22 +319,16 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"align_to_byte($io)") override def attrDebugStart(attrId: Identifier, attrType: DataType, ios: Option[String], rep: RepeatSpec): Unit = { - ios.foreach { io => + ios.foreach { (io) => val name = attrId match { case _: RawIdentifier | _: SpecialIdentifier => return case _ => idToStr(attrId) } rep match { case NoRepeat => - out.puts(s"self._debug['$name']['start'] = $io.pos()") + out.puts("this._attrStart[\"" + name + "\"] = pos(" + io + ")") case _: RepeatExpr | RepeatEos | _: RepeatUntil => - /** TODO: move array initialization to [[condRepeatCommonInit]] - see - * [[JavaScriptCompiler.condRepeatCommonInit]] for inspiration */ - out.puts(s"if not 'arr' in self._debug['$name']:") - out.inc - out.puts(s"self._debug['$name']['arr'] = []") - out.dec - out.puts(s"self._debug['$name']['arr'].append({'start': $io.pos()})") + getOrCreatePosList("_arrStart", name, io) } } } @@ -320,14 +340,34 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } rep match { case NoRepeat => - out.puts(s"self._debug['$name']['end'] = $io.pos()") - case _: RepeatExpr => - out.puts(s"self._debug['$name']['arr'][i]['end'] = $io.pos()") - case RepeatEos | _: RepeatUntil => - out.puts(s"self._debug['$name']['arr'][len(${privateMemberName(attrId)}) - 1]['end'] = $io.pos()") + out.puts("this._attrEnd[\"" + name + "\"] = pos(" + io + ")") + case _: RepeatExpr | RepeatEos | _: RepeatUntil => + getOrCreatePosList("_arrEnd", name, io) } } + override def blockScopeHeader: Unit = { + out.puts("begin") + out.inc + } + + override def blockScopeFooter: Unit = { + out.dec + out.puts("end") + } + + def getOrCreatePosList(listName: String, varName: String, io: String): Unit = { + blockScopeHeader + out.puts("_posList = get(" + listName + ", \"" + varName + "\", nothing)") + out.puts("if _posList === nothing ") + out.inc + out.puts("_posList = Vector{Integer}()") + out.puts(listName + "[\"" + varName + "\"] = _posList") + blockScopeFooter + out.puts(s"push!(_posList, pos($io))") + blockScopeFooter + } + override def condIfHeader(expr: Ast.expr): Unit = { out.puts(s"if ${expression(expr)}") out.inc @@ -351,7 +391,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatEosFooter: Unit = { out.puts("i += 1") - universalFooter + blockScopeFooter } override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, repeatExpr: expr): Unit = { @@ -378,9 +418,9 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"if ${expression(untilExpr)}") out.inc out.puts("break") - universalFooter + blockScopeFooter out.puts("i += 1") - universalFooter + blockScopeFooter } override def handleAssignmentSimple(id: Identifier, expr: String): Unit = @@ -436,7 +476,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def userTypeDebugRead(id: String, dataType: DataType, assignType: DataType): Unit = - out.puts(s"$id._read()") + out.puts(s"_read($id)") override def switchStart(id: Identifier, on: Ast.expr): Unit = {} override def switchCaseStart(condition: Ast.expr): Unit = {} @@ -619,7 +659,7 @@ object JuliaCompiler extends LanguageCompilerStatic case BitsType(_, _) => "UInt64" case _: BooleanType => "Bool" - case CalcIntType => "Int" + case CalcIntType => "Int64" case CalcFloatType => "Float64" case _: StrType => "String" @@ -628,10 +668,10 @@ object JuliaCompiler extends LanguageCompilerStatic case AnyType => "Any" case KaitaiStreamType | OwnedKaitaiStreamType => kstreamName case KaitaiStructType | CalcKaitaiStructType => kstructName - case t: UserType => types2class(t.classSpec match { + case t: UserType => "Union{" + types2class(t.classSpec match { case Some(cs) => if (cs.isTopLevel) cs.name else "Abstract" :: cs.name case None => t.name - }) + }) + ", Nothing}" case t: EnumType => s"Union{${types2class(t.enumSpec.get.name)}, Integer}" @@ -639,7 +679,6 @@ object JuliaCompiler extends LanguageCompilerStatic case st: SwitchType => kaitaiType2NativeType(st.combinedType) } - } def enumToStr(typeName: List[String], enumName: String): String = typeName.mkString("_") + "__" + enumName From dcea394c3657f03b8c56808a267f244041bf168f Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Sun, 8 Oct 2023 18:27:37 +0200 Subject: [PATCH 22/37] use KaitaiStruct functions explicitly --- .../struct/languages/JuliaCompiler.scala | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index b3340e86c..0e146ee15 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -254,7 +254,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: IntType => s"UInt8(${translator.doCast(xorValue, Int1Type(true))})" case _ => expression(xorValue) } - s"process_xor($srcExpr, $xorValueStr)" + s"KaitaiStruct.process_xor($srcExpr, $xorValueStr)" case ProcessZlib => importList.add("using CodecZlib") s"transcode(GzipDecompressor, $srcExpr)" @@ -264,7 +264,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } else { s"8 - (${expression(rotValue)})" } - s"process_rotate_left($srcExpr, $expr, 1)" + s"KaitaiStruct.process_rotate_left($srcExpr, $expr, 1)" case ProcessCustom(name, args) => val procClass = if (name.length == 1) { val onlyName = name.head @@ -307,16 +307,16 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def pushPos(io: String): Unit = - out.puts(s"_pos = pos($io)") + out.puts(s"_pos = KaitaiStruct.pos($io)") override def seek(io: String, pos: Ast.expr): Unit = - out.puts(s"seek($io, ${expression(pos)})") + out.puts(s"KaitaiStruct.seek($io, ${expression(pos)})") override def popPos(io: String): Unit = - out.puts(s"seek($io, _pos)") + out.puts(s"KaitaiStruct.seek($io, _pos)") override def alignToByte(io: String): Unit = - out.puts(s"align_to_byte($io)") + out.puts(s"KaitaiStruct.align_to_byte($io)") override def attrDebugStart(attrId: Identifier, attrType: DataType, ios: Option[String], rep: RepeatSpec): Unit = { ios.foreach { (io) => @@ -432,17 +432,17 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { dataType match { case t: ReadableType => - s"read${Utils.capitalize(t.apiCall(defEndian))}($io)" + s"KaitaiStruct.read${Utils.capitalize(t.apiCall(defEndian))}($io)" case blt: BytesLimitType => - s"read_bytes($io, convert(UInt, ${expression(blt.size)}))" + s"KaitaiStruct.read_bytes($io, convert(UInt, ${expression(blt.size)}))" case _: BytesEosType => - s"read_bytes_full($io)" + s"KaitaiStruct.read_bytes_full($io)" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"read_bytes_term($io, convert(UInt8, $terminator), $include, $consume, $eosError)" + s"KaitaiStruct.read_bytes_term($io, convert(UInt8, $terminator), $include, $consume, $eosError)" case BitsType1(bitEndian) => - s"read_bits_int_${bitEndian.toSuffix}($io, 1) != 0" + s"KaitaiStruct.read_bits_int_${bitEndian.toSuffix}($io, 1) != 0" case BitsType(width: Int, bitEndian) => - s"read_bits_int_${bitEndian.toSuffix}($io, $width)" + s"KaitaiStruct.read_bits_int_${bitEndian.toSuffix}($io, $width)" case t: UserType => val addParams = Utils.join(t.args.map(a => translator.translate(a)), "", ", ", ", ") val addArgs = if (t.isOpaque) { @@ -465,11 +465,11 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { val expr1 = padRight match { - case Some(padByte) => s"bytes_strip_right($expr0, $padByte)" + case Some(padByte) => s"KaitaiStruct.bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"bytes_terminate($expr1, $term, ${include})" + case Some(term) => s"KaitaiStruct.bytes_terminate($expr1, $term, ${include})" case None => expr1 } expr2 @@ -632,7 +632,7 @@ object JuliaCompiler extends LanguageCompilerStatic case _ => idToStr(id) } - override def kstreamName: String = "KaitaiStream" + override def kstreamName: String = "KaitaiStruct.KaitaiStream" override def kstructName: String = "Any" override def ksErrorName(err: KSError): String = err match { case EndOfStreamError => "ErrorException" From 396c68b75cf53cad94e4c48f4dad36c8398795df Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Sun, 8 Oct 2023 18:46:15 +0200 Subject: [PATCH 23/37] bugfixing use KaiTaiStruct functions explicitly --- .../io/kaitai/struct/languages/JuliaCompiler.scala | 10 +++++----- .../io/kaitai/struct/translators/JuliaTranslator.scala | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index 0e146ee15..0b7fd24b1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -326,7 +326,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } rep match { case NoRepeat => - out.puts("this._attrStart[\"" + name + "\"] = pos(" + io + ")") + out.puts("this._attrStart[\"" + name + "\"] = KaitaiStruct.pos(" + io + ")") case _: RepeatExpr | RepeatEos | _: RepeatUntil => getOrCreatePosList("_arrStart", name, io) } @@ -340,7 +340,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } rep match { case NoRepeat => - out.puts("this._attrEnd[\"" + name + "\"] = pos(" + io + ")") + out.puts("this._attrEnd[\"" + name + "\"] = KaitaiStruct.pos(" + io + ")") case _: RepeatExpr | RepeatEos | _: RepeatUntil => getOrCreatePosList("_arrEnd", name, io) } @@ -364,7 +364,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("_posList = Vector{Integer}()") out.puts(listName + "[\"" + varName + "\"] = _posList") blockScopeFooter - out.puts(s"push!(_posList, pos($io))") + out.puts(s"push!(_posList, KaitaiStruct.pos($io))") blockScopeFooter } @@ -383,7 +383,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = { out.puts("i = 0") - out.puts(s"while !iseof($io)") + out.puts(s"while !KaitaiStruct.iseof($io)") out.inc } override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = @@ -636,7 +636,7 @@ object JuliaCompiler extends LanguageCompilerStatic override def kstructName: String = "Any" override def ksErrorName(err: KSError): String = err match { case EndOfStreamError => "ErrorException" - case _ => s"${err.name}" + case _ => s"KaitaiStruct.${err.name}" } def types2class(name: List[String]): String = name.map(x => type2class(x)).mkString("_") diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 346c23479..79fdc69e0 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -13,7 +13,7 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba case (_: IntType, _: IntType, Ast.operator.Div) => s"fld(${translate(left)}, ${translate(right)})" case (_: IntType, _: IntType, Ast.operator.Mod) => - s"mod(${translate(left)}, ${translate(right)})" + s"KaitaiStruct.mod(${translate(left)}, ${translate(right)})" case _ => super.numericBinOp(left, op, right) } @@ -82,7 +82,7 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = s"${JuliaCompiler.enumToStr(enumTypeAbs, label)}" override def doEnumById(enumTypeAbs: List[String], id: String): String = - s"resolve_enum(${JuliaCompiler.types2class(enumTypeAbs)}, $id)" + s"KaitaiStruct.resolve_enum(${JuliaCompiler.types2class(enumTypeAbs)}, $id)" override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = s"${translate(container)}[${translateIndex(idx)}]" @@ -146,9 +146,9 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba s"maximum(${translate(a)})" override def kaitaiStreamSize(value: Ast.expr): String = - s"size(${translate(value)})" + s"KaitaiStruct.size(${translate(value)})" override def kaitaiStreamEof(value: Ast.expr): String = - s"iseof(${translate(value)})" + s"KaitaiStruct.iseof(${translate(value)})" override def kaitaiStreamPos(value: Ast.expr): String = - s"pos(${translate(value)})" + s"KaitaiStruct.pos(${translate(value)})" } From 13705772b32a9854e6548b6a81ca5e7f48daaf57 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Mon, 9 Oct 2023 19:24:22 +0200 Subject: [PATCH 24/37] add modules support --- .../kaitai/struct/languages/JuliaCompiler.scala | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index 0b7fd24b1..b9f9bf292 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -38,7 +38,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def results(topClass: ClassSpec): Map[String, String] = Map(outFileName(topClass.nameAsStr) -> - (outHeader.result + outImports(topClass) + abstractTypesAndEnums.result + out.result) + (s"module ${types2class(topClass.name)}Module\n" + outHeader.result + outImports(topClass) + abstractTypesAndEnums.result + out.result + "end\n") ) override def classForwardDeclaration(name: List[String]): Unit = { @@ -55,8 +55,9 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) outHeader.puts(s"# $headerComment") outHeader.puts - importList.add("include(\"../../../runtime/julia/KaitaiStruct/src/KaitaiStruct.jl\")") - importList.add("using .KaitaiStruct") + // importList.add("include(\"../../../runtime/julia/KaitaiStruct/src/KaitaiStruct.jl\")") + importList.add("export all") + importList.add("using ..KaitaiStruct") out.puts // out.puts @@ -85,8 +86,8 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = { - val name = classSpec.name.head - out.puts(s"include(${'"'}$name.jl${'"'})") + // val name = classSpec.name.head + out.puts(s"using ..${types2class(classSpec.name)}Module: ${types2class(classSpec.name)}") } override def classHeader(name: List[String]): Unit = { @@ -680,6 +681,8 @@ object JuliaCompiler extends LanguageCompilerStatic case st: SwitchType => kaitaiType2NativeType(st.combinedType) } } - def enumToStr(typeName: List[String], enumName: String): String = + + def enumToStr(typeName: List[String], enumName: String): String = { typeName.mkString("_") + "__" + enumName + } } From a086343284d98d2728a5b4957d60bb5d4163779f Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Mon, 9 Oct 2023 19:24:36 +0200 Subject: [PATCH 25/37] fix enum imports --- .../struct/translators/JuliaTranslator.scala | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 79fdc69e0..2e40de815 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -6,8 +6,11 @@ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format.Identifier import io.kaitai.struct.languages.{JuliaCompiler, RubyCompiler} import io.kaitai.struct.exprlang.ConstEvaluator +import scala.collection.immutable.HashSet class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { + var usedForeignImports: HashSet[String] = HashSet.empty[String] + override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr): String = { (detectType(left), detectType(right), op) match { case (_: IntType, _: IntType, Ast.operator.Div) => @@ -79,10 +82,24 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def doInternalName(id: Identifier): String = s"this.${JuliaCompiler.publicMemberName(id)}" - override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = + override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = { + if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head)) + && !usedForeignImports.contains(JuliaCompiler.type2class(enumTypeAbs.head))) { + usedForeignImports += JuliaCompiler.type2class(enumTypeAbs.head) + importList.add(s"using ..${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") + importList.add(s"#${JuliaCompiler.types2class(provider.nowClass.name)} ${JuliaCompiler.type2class(enumTypeAbs.head)}") + } s"${JuliaCompiler.enumToStr(enumTypeAbs, label)}" - override def doEnumById(enumTypeAbs: List[String], id: String): String = + } + override def doEnumById(enumTypeAbs: List[String], id: String): String = { + if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head)) + && !usedForeignImports.contains(JuliaCompiler.type2class(enumTypeAbs.head))) { + usedForeignImports += JuliaCompiler.type2class(enumTypeAbs.head) + importList.add(s"using ..${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") + importList.add(s"#${JuliaCompiler.types2class(provider.nowClass.name)} ${JuliaCompiler.type2class(enumTypeAbs.head)}") + } s"KaitaiStruct.resolve_enum(${JuliaCompiler.types2class(enumTypeAbs)}, $id)" + } override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = s"${translate(container)}[${translateIndex(idx)}]" From c60a6832c1c36932398ab195bd5de9b0a9e8761b Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Tue, 14 Nov 2023 12:18:26 +0100 Subject: [PATCH 26/37] refactor enum imports --- .../struct/translators/JuliaTranslator.scala | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 2e40de815..12dede003 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -83,20 +83,26 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba s"this.${JuliaCompiler.publicMemberName(id)}" override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = { - if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head)) - && !usedForeignImports.contains(JuliaCompiler.type2class(enumTypeAbs.head))) { - usedForeignImports += JuliaCompiler.type2class(enumTypeAbs.head) - importList.add(s"using ..${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") - importList.add(s"#${JuliaCompiler.types2class(provider.nowClass.name)} ${JuliaCompiler.type2class(enumTypeAbs.head)}") + if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head))){ + importList.add(s"include(${'"'}../../compiled/julia/${enumTypeAbs.head}.jl${'"'})") + importList.add(s"using .${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") + importList.add(s"#my code here 1") } - s"${JuliaCompiler.enumToStr(enumTypeAbs, label)}" + + s"${JuliaCompiler.type2class(enumTypeAbs.head)}Module.${JuliaCompiler.enumToStr(enumTypeAbs, label)}" } + override def doEnumById(enumTypeAbs: List[String], id: String): String = { - if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head)) - && !usedForeignImports.contains(JuliaCompiler.type2class(enumTypeAbs.head))) { - usedForeignImports += JuliaCompiler.type2class(enumTypeAbs.head) - importList.add(s"using ..${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") - importList.add(s"#${JuliaCompiler.types2class(provider.nowClass.name)} ${JuliaCompiler.type2class(enumTypeAbs.head)}") + // if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head)) + // && !usedForeignImports.contains(JuliaCompiler.type2class(enumTypeAbs.head))) { + // usedForeignImports += JuliaCompiler.type2class(enumTypeAbs.head) + // importList.add(s"using ..${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") + // importList.add(s"#${JuliaCompiler.types2class(provider.nowClass.name)} ${JuliaCompiler.type2class(enumTypeAbs.head)}") + // } + if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head))){ + importList.add(s"include(${'"'}../../compiled/julia/${enumTypeAbs.head}.jl${'"'})") + importList.add(s"using .${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") + importList.add(s"#my code here 2") } s"KaitaiStruct.resolve_enum(${JuliaCompiler.types2class(enumTypeAbs)}, $id)" } From 788ae6c50a335c82972541427dd7480f5f43c4fe Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Tue, 14 Nov 2023 12:19:54 +0100 Subject: [PATCH 27/37] refactor imports, housekeeping --- .../kaitai/struct/languages/JuliaCompiler.scala | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index b9f9bf292..dc9715a50 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -54,12 +54,8 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def fileHeader(topClassName: String): Unit = { outHeader.puts(s"# $headerComment") outHeader.puts - - // importList.add("include(\"../../../runtime/julia/KaitaiStruct/src/KaitaiStruct.jl\")") importList.add("export all") importList.add("using ..KaitaiStruct") - - out.puts // out.puts // // API compatibility check @@ -87,7 +83,8 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = { // val name = classSpec.name.head - out.puts(s"using ..${types2class(classSpec.name)}Module: ${types2class(classSpec.name)}") + importList.add(s"include(${'"'}../../compiled/julia/${classSpec.name.head}.jl${'"'})") + importList.add(s"using .${types2class(classSpec.name)}Module: ${types2class(classSpec.name)}") } override def classHeader(name: List[String]): Unit = { @@ -450,7 +447,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) "" } else { val parent = t.forcedParent match { - case Some(USER_TYPE_NO_PARENT) => "nothing" + case Some(USER_TYPE_NO_PARENT) => "Any" case Some(fp) => translator.translate(fp) case None => "this" } @@ -567,9 +564,9 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def classToString(toStringExpr: Ast.expr): Unit = { out.puts - out.puts("def __repr__(self):") - out.inc - out.puts(s"return ${translator.translate(toStringExpr)}") + // out.puts("def __repr__(self):") + // out.inc + // out.puts(s"return ${translator.translate(toStringExpr)}") out.dec } From 61194de9828c309bd497435146d0ef5895d71063 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Mon, 8 Apr 2024 14:07:37 +0200 Subject: [PATCH 28/37] use package loading insted of code inclusion, use abstract UserType --- .../struct/languages/JuliaCompiler.scala | 100 +++++++----------- 1 file changed, 37 insertions(+), 63 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index dc9715a50..f27564f2e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -36,17 +36,18 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts } - override def results(topClass: ClassSpec): Map[String, String] = + override def results(topClass: ClassSpec): Map[String, String] = { Map(outFileName(topClass.nameAsStr) -> (s"module ${types2class(topClass.name)}Module\n" + outHeader.result + outImports(topClass) + abstractTypesAndEnums.result + out.result + "end\n") ) - - override def classForwardDeclaration(name: List[String]): Unit = { - abstractTypesAndEnums.puts(s"abstract type ${types2class("Abstract" :: name)} end") } + // override def classForwardDeclaration(name: List[String]): Unit = { + // abstractTypesAndEnums.puts(s"abstract type ${types2class("Abstract" :: name)} end") + // } + override def indent: String = " " - override def outFileName(topClassName: String): String = s"$topClassName.jl" + override def outFileName(topClassName: String): String = s"${type2class(topClassName)}Module.jl" override def outImports(topClass: ClassSpec): String = importList.toList.mkString("", "\n", "\n") @@ -54,46 +55,26 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def fileHeader(topClassName: String): Unit = { outHeader.puts(s"# $headerComment") outHeader.puts - importList.add("export all") - importList.add("using ..KaitaiStruct") - // out.puts - - // // API compatibility check - // out.puts( - // // The API_VERSION tuple attribute was introduced in version 0.10 of the - // // Python runtime. Runtime version 0.9 and older only have a __version__ - // // attribute that stores the version in string form. - // // We don't need to include any complex handling for runtimes that don't - // // have the API_VERSION attribute - we know that such a runtime must have - // // version 0.9 or older, which means that it is incompatible with code - // // generated by newer compiler versions. - // "if getattr(kaitaistruct, 'API_VERSION', (0, 9)) < " + - // KSVersion.minimalRuntime.toPythonTuple + - // ":" - // ) - // out.inc - // out.puts( - // "raise Exception(\"Incompatible Kaitai Struct Python API: " + - // KSVersion.minimalRuntime + - // " or later is required, but you have %s\" % (kaitaistruct.__version__))" - // ) - // out.dec + importList.add("export from_file") + importList.add("using KaitaiStruct") out.puts } override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = { // val name = classSpec.name.head - importList.add(s"include(${'"'}../../compiled/julia/${classSpec.name.head}.jl${'"'})") - importList.add(s"using .${types2class(classSpec.name)}Module: ${types2class(classSpec.name)}") + // importList.add(s"include(${'"'}../../compiled/julia/${classSpec.name.head}.jl${'"'})") + // importList.add(s"using .${types2class(classSpec.name)}Module: ${types2class(classSpec.name)}") + importList.add(s"import ${types2class(classSpec.name)}Module: ${types2class(classSpec.name)}") } override def classHeader(name: List[String]): Unit = { - val subtype = if (typeProvider.nowClass.isTopLevel) "" else s" <: ${types2class("Abstract" :: name)}" - out.puts(s"mutable struct ${types2class(name)}$subtype") + // val subtype = if (typeProvider.nowClass.isTopLevel) "" else s" <: ${types2class("Abstract" :: name)}" + // out.puts(s"mutable struct ${types2class(name)}$subtype") + out.puts(s"mutable struct ${types2class(name)} <: KaitaiStruct.UserType") out.inc typeProvider.nowClass.meta.endian match { case Some(_: CalcEndian) | Some(InheritedEndian) => - out.puts(s"_is_le::Union{Bool, Nothing}") + out.puts(s"_is_le::Union{Bool,Nothing}") case _ => // no _is_le variable } @@ -113,7 +94,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val endianAdd = if (isHybrid) ", _is_le=nothing" else "" val paramsList = Utils.join(params.map(p => paramName(p.id)), "", ", ", ",") - out.puts(s"function ${types2class(name)}(${paramsList}_io, _parent = nothing, _root = nothing$endianAdd)") + out.puts(s"function ${types2class(name)}(${paramsList}_io, _parent=nothing, _root=nothing$endianAdd)") out.inc out.puts("this = new()") if (isHybrid) @@ -180,14 +161,14 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { if (isNullable) { - out.puts(s"${idToStr(attrName)}::Union{Nothing, ${kaitaiType2NativeType(attrType)}}") + out.puts(s"${idToStr(attrName)}::Union{Nothing,${kaitaiType2NativeType(attrType)}}") } else { out.puts(s"${idToStr(attrName)}::${kaitaiType2NativeType(attrType)}") } } override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = { - out.puts(s"${idToStr(attrName)}::Union{Nothing, ${kaitaiType2NativeType(attrType)}}") + out.puts(s"${idToStr(attrName)}::Union{Nothing,${kaitaiType2NativeType(attrType)}}") } def fromFile(name: List[String]): Unit = { @@ -264,17 +245,9 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } s"KaitaiStruct.process_rotate_left($srcExpr, $expr, 1)" case ProcessCustom(name, args) => - val procClass = if (name.length == 1) { - val onlyName = name.head - val className = type2class(onlyName) - className - } else { - val pkgName = name.init.mkString(".") - s"$pkgName.${type2class(name.last)}" - } - - // out.puts(s"_process = $procClass(${args.map(expression).mkString(", ")})") - s"$procClass.decode(${args.map(expression).mkString(", ")}$srcExpr)" + val procClass = type2class(name.last) + importList.add(s"using ${name.map(x => type2class(x)).mkString(".")}") + s"$procClass.decode(${(args.map(expression) :+ srcExpr).mkString(", ")})" } handleAssignment(varDest, expr, rep, isRaw = false) } @@ -432,11 +405,11 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case t: ReadableType => s"KaitaiStruct.read${Utils.capitalize(t.apiCall(defEndian))}($io)" case blt: BytesLimitType => - s"KaitaiStruct.read_bytes($io, convert(UInt, ${expression(blt.size)}))" + s"KaitaiStruct.read_bytes($io, UInt(${expression(blt.size)}))" case _: BytesEosType => s"KaitaiStruct.read_bytes_full($io)" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"KaitaiStruct.read_bytes_term($io, convert(UInt8, $terminator), $include, $consume, $eosError)" + s"KaitaiStruct.read_bytes_term($io, UInt8($terminator), $include, $consume, $eosError)" case BitsType1(bitEndian) => s"KaitaiStruct.read_bits_int_${bitEndian.toSuffix}($io, 1) != 0" case BitsType(width: Int, bitEndian) => @@ -447,7 +420,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) "" } else { val parent = t.forcedParent match { - case Some(USER_TYPE_NO_PARENT) => "Any" + case Some(USER_TYPE_NO_PARENT) => "nothing" case Some(fp) => translator.translate(fp) case None => "this" } @@ -543,7 +516,6 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { - // workaround to avoid Python raising an "AttributeError: instance has no attribute" out.puts(s"${privateMemberName(instName)}") } @@ -596,12 +568,13 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) def userType2class(t: UserType): String = { val name = t.classSpec.get.name - val firstName = name.head - val prefix = if (t.isOpaque && firstName != translator.provider.nowClass.name.head) { - s"$firstName." - } else { - "" - } + // val firstName = name.head + // val prefix = if (t.isOpaque && firstName != translator.provider.nowClass.name.head) { + // s"${type2class(firstName)}Module." + // } else { + // "" + // } + // s"$prefix${types2class(name)}" s"${types2class(name)}" } } @@ -666,12 +639,13 @@ object JuliaCompiler extends LanguageCompilerStatic case AnyType => "Any" case KaitaiStreamType | OwnedKaitaiStreamType => kstreamName case KaitaiStructType | CalcKaitaiStructType => kstructName - case t: UserType => "Union{" + types2class(t.classSpec match { - case Some(cs) => if (cs.isTopLevel) cs.name else "Abstract" :: cs.name - case None => t.name - }) + ", Nothing}" + // case t: UserType => types2class(t.classSpec match { + // case Some(cs) => if (cs.isTopLevel) cs.name else "Abstract" :: cs.name + // case None => t.name + // }) + case t: UserType => "KaitaiStruct.UserType" - case t: EnumType => s"Union{${types2class(t.enumSpec.get.name)}, Integer}" + case t: EnumType => s"Union{${types2class(t.enumSpec.get.name)},Integer}" case at: ArrayType => s"Vector{${kaitaiType2NativeType(at.elType)}}" From 8b889a5667a195b7484242d62a1e288ec67ab27a Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Mon, 8 Apr 2024 14:08:18 +0200 Subject: [PATCH 29/37] Make UserType nullable --- .../scala/io/kaitai/struct/JuliaClassCompiler.scala | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala index 611ed6234..f45942ea5 100644 --- a/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala @@ -92,4 +92,15 @@ class JuliaClassCompiler( } lang.switchCases[FixedEndian](IS_LE_ID, ce.on, ce.cases, renderProc, renderProc) } + + override def compileAttrDeclarations(attrs: List[MemberSpec]): Unit = { + attrs.foreach { (attr) => + val isNullable = if (lang.switchBytesOnlyAsRaw) { + attr.isNullableSwitchRaw + } else { + attr.isNullable || attr.dataType.isInstanceOf[UserType] + } + lang.attributeDeclaration(attr.id, attr.dataTypeComposite, isNullable) + } + } } From 210a8feb890f745a0913bfc04665e5e04420a906 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Mon, 8 Apr 2024 14:09:18 +0200 Subject: [PATCH 30/37] use package loading --- .../struct/translators/JuliaTranslator.scala | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 12dede003..8ec3b6b9e 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -9,7 +9,6 @@ import io.kaitai.struct.exprlang.ConstEvaluator import scala.collection.immutable.HashSet class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { - var usedForeignImports: HashSet[String] = HashSet.empty[String] override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr): String = { (detectType(left), detectType(right), op) match { @@ -37,9 +36,6 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba } } -// override def doStringLiteral(s: String): String = "raw" + super.doStringLiteral(s) -// override def doBoolLiteral(n: Boolean): String = if (n) "True" else "False" - /** * https://docs.python.org/2.7/reference/lexical_analysis.html#string-literals * https://docs.Julia.org/3.6/reference/lexical_analysis.html#string-and-bytes-literals @@ -59,9 +55,8 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def doByteArrayLiteral(arr: Seq[Byte]): String = s"Vector{UInt8}(${super.doByteArrayLiteral(arr)})" +// "b\"" + Utils.hexEscapeByteArray(arr) + "\"" override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = { -// importList.add("import struct") -// s"struct.pack('${elts.length}b', ${elts.map(translate).mkString(", ")})" s"Vector{UInt8}([${elts.map(translate).mkString(", ")}])" } @@ -84,25 +79,15 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = { if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head))){ - importList.add(s"include(${'"'}../../compiled/julia/${enumTypeAbs.head}.jl${'"'})") - importList.add(s"using .${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") - importList.add(s"#my code here 1") + importList.add(s"using ${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") } s"${JuliaCompiler.type2class(enumTypeAbs.head)}Module.${JuliaCompiler.enumToStr(enumTypeAbs, label)}" } override def doEnumById(enumTypeAbs: List[String], id: String): String = { - // if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head)) - // && !usedForeignImports.contains(JuliaCompiler.type2class(enumTypeAbs.head))) { - // usedForeignImports += JuliaCompiler.type2class(enumTypeAbs.head) - // importList.add(s"using ..${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") - // importList.add(s"#${JuliaCompiler.types2class(provider.nowClass.name)} ${JuliaCompiler.type2class(enumTypeAbs.head)}") - // } if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head))){ - importList.add(s"include(${'"'}../../compiled/julia/${enumTypeAbs.head}.jl${'"'})") - importList.add(s"using .${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") - importList.add(s"#my code here 2") + importList.add(s"using ${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") } s"KaitaiStruct.resolve_enum(${JuliaCompiler.types2class(enumTypeAbs)}, $id)" } From 40f2f300a48b38183b8c67c473810cb86585ca49 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Mon, 8 Apr 2024 15:33:39 +0200 Subject: [PATCH 31/37] remove unnecessay arguments from to_s methods --- .../struct/languages/JuliaCompiler.scala | 34 ++++++++----------- .../struct/translators/JuliaTranslator.scala | 21 ++++++------ 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index f27564f2e..0b980bfb2 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -1,7 +1,7 @@ package io.kaitai.struct.languages import io.kaitai.struct.datatype.DataType.{BooleanType, _} -import io.kaitai.struct.datatype.{CalcEndian, DataType, EndOfStreamError, FixedEndian, InheritedEndian, KSError, NeedRaw, UndecidedEndiannessError} +import io.kaitai.struct.datatype._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ @@ -24,7 +24,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) import JuliaCompiler._ override val translator = new JuliaTranslator(typeProvider, importList) - private val abstractTypesAndEnums = new StringLanguageOutputWriter(indent) + private val enums = new StringLanguageOutputWriter(indent) override def innerDocstrings = true @@ -38,7 +38,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def results(topClass: ClassSpec): Map[String, String] = { Map(outFileName(topClass.nameAsStr) -> - (s"module ${types2class(topClass.name)}Module\n" + outHeader.result + outImports(topClass) + abstractTypesAndEnums.result + out.result + "end\n") + (s"module ${types2class(topClass.name)}Module\n" + outHeader.result + outImports(topClass) + enums.result + out.result + "end\n") ) } @@ -60,7 +60,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts } - override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = { + override def externalClassDeclaration(classSpec: ClassSpec): Unit = { // val name = classSpec.name.head // importList.add(s"include(${'"'}../../compiled/julia/${classSpec.name.head}.jl${'"'})") // importList.add(s"using .${types2class(classSpec.name)}Module: ${types2class(classSpec.name)}") @@ -290,7 +290,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"KaitaiStruct.align_to_byte($io)") override def attrDebugStart(attrId: Identifier, attrType: DataType, ios: Option[String], rep: RepeatSpec): Unit = { - ios.foreach { (io) => + ios.foreach { io => val name = attrId match { case _: RawIdentifier | _: SpecialIdentifier => return case _ => idToStr(attrId) @@ -344,11 +344,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc } - override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { - if (needRaw.level >= 1) - out.puts(s"${privateMemberName(RawIdentifier(id))} = []") - if (needRaw.level >= 2) - out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = []") + override def condRepeatInitAttr(id: Identifier, dataType: DataType): Unit = { out.puts(s"${privateMemberName(id)} = []") } @@ -440,7 +436,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case None => expr0 } val expr2 = terminator match { - case Some(term) => s"KaitaiStruct.bytes_terminate($expr1, $term, ${include})" + case Some(term) => s"KaitaiStruct.bytes_terminate($expr1, $term, $include)" case None => expr1 } expr2 @@ -521,12 +517,12 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { val fullEnumName: List[String] = curClass :+ enumName - abstractTypesAndEnums.puts(s"@enum ${types2class(fullEnumName)}::Int64 begin") - abstractTypesAndEnums.inc - enumColl.foreach { case (id: Long, label: EnumValueSpec) => abstractTypesAndEnums.puts(s"${enumToStr(fullEnumName, label.name)} = ${translator.doIntLiteral(id)}") } - abstractTypesAndEnums.dec - abstractTypesAndEnums.puts("end") - abstractTypesAndEnums.puts + enums.puts(s"@enum ${types2class(fullEnumName)}::Int64 begin") + enums.inc + enumColl.foreach { case (id: Long, label: EnumValueSpec) => enums.puts(s"${enumToStr(fullEnumName, label.name)} = ${translator.doIntLiteral(id)}") } + enums.dec + enums.puts("end") + enums.puts } override def debugClassSequence(seq: List[AttrSpec]): Unit = { @@ -638,12 +634,12 @@ object JuliaCompiler extends LanguageCompilerStatic case AnyType => "Any" case KaitaiStreamType | OwnedKaitaiStreamType => kstreamName - case KaitaiStructType | CalcKaitaiStructType => kstructName + case KaitaiStructType | CalcKaitaiStructType(_) => kstructName // case t: UserType => types2class(t.classSpec match { // case Some(cs) => if (cs.isTopLevel) cs.name else "Abstract" :: cs.name // case None => t.name // }) - case t: UserType => "KaitaiStruct.UserType" + case _: UserType => "KaitaiStruct.UserType" case t: EnumType => s"Union{${types2class(t.enumSpec.get.name)},Integer}" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 8ec3b6b9e..7c44f80ee 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -1,23 +1,22 @@ package io.kaitai.struct.translators -import io.kaitai.struct.{ImportList, Utils} +import io.kaitai.struct.{ImportList} import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format.Identifier -import io.kaitai.struct.languages.{JuliaCompiler, RubyCompiler} +import io.kaitai.struct.languages.{JuliaCompiler} import io.kaitai.struct.exprlang.ConstEvaluator -import scala.collection.immutable.HashSet class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { - override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr): String = { + override def genericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr, extPrec: Int): String = { (detectType(left), detectType(right), op) match { case (_: IntType, _: IntType, Ast.operator.Div) => s"fld(${translate(left)}, ${translate(right)})" case (_: IntType, _: IntType, Ast.operator.Mod) => s"KaitaiStruct.mod(${translate(left)}, ${translate(right)})" case _ => - super.numericBinOp(left, op, right) + super.genericBinOp(left, op, right, extPrec) } } @@ -110,16 +109,16 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba s"Int(${translate(v)})" override def floatToInt(v: Ast.expr): String = s"trunc(${translate(v)})" - override def intToStr(i: Ast.expr, base: Ast.expr): String = { - val baseStr = translate(base) - s"string(${translate(i)}, base = $baseStr)" + override def intToStr(i: Ast.expr): String = { + s"string(${translate(i)})" } - override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = { + override def bytesToStr(bytesExpr: String, encoding: String): String = { importList.add("using StringEncodings") - s"decode(($bytesExpr), ${translate(encoding)})" + s"""decode(($bytesExpr), "$encoding")""" } - override def strConcat(left: Ast.expr, right: Ast.expr): String = s"${translate(left)} * ${translate(right)}" + override def strConcat(left: Ast.expr, right: Ast.expr, extPrec: Int): String = + s"${translate(left)} * ${translate(right)}" override def bytesLength(value: Ast.expr): String = s"length(${translate(value)})" From dcef9bb9e2f0467ed48821121a0667511ec7a79c Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Tue, 9 Apr 2024 16:12:28 +0200 Subject: [PATCH 32/37] add custom toString support for julia --- .../io/kaitai/struct/JuliaClassCompiler.scala | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala index f45942ea5..9345e72d6 100644 --- a/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/JuliaClassCompiler.scala @@ -24,7 +24,7 @@ class JuliaClassCompiler( AttrSpec(List(), ParentIdentifier, curClass.parentType) ) ++ ExtraAttrs.forClassSpec(curClass, lang) - curClass.types.foreach { case (typeName, _) => lang.classForwardDeclaration(curClass.name ++ List(typeName)) } + // curClass.types.foreach { case (typeName, _) => lang.classForwardDeclaration(curClass.name ++ List(typeName)) } if (!curClass.doc.isEmpty) lang.classDoc(curClass.name, curClass.doc) @@ -44,9 +44,8 @@ class JuliaClassCompiler( compileConstructor(curClass) lang.classFooter(curClass.name) - // Constructor = Read() function - if (curClass.isTopLevel) - julialang.fromFile(curClass.name) + // if (curClass.isTopLevel) + // julialang.fromFile(curClass.name) compileEagerRead(curClass.seq, curClass.meta.endian) @@ -55,6 +54,7 @@ class JuliaClassCompiler( compileAttrReaders(curClass.seq ++ extraAttrs) + curClass.toStringExpr.foreach(expr => lang.classToString(expr)) // Recursive types compileSubclasses(curClass) } @@ -63,9 +63,13 @@ class JuliaClassCompiler( // Determine datatype val dataType = instSpec.dataTypeComposite - if (!instSpec.doc.isEmpty) - lang.attributeDoc(instName, instSpec.doc) + // compileInstanceDeclaration(instName, instSpec) + + if (!lang.innerDocstrings) + compileInstanceDoc(instName, instSpec) lang.instanceHeader(className, instName, dataType, instSpec.isNullable) + if (lang.innerDocstrings) + compileInstanceDoc(instName, instSpec) lang.instanceCheckCacheAndReturn(instName, dataType) instSpec match { @@ -73,25 +77,25 @@ class JuliaClassCompiler( lang.attrParseIfHeader(instName, vi.ifExpr) lang.instanceCalculate(instName, dataType, vi.value) lang.attrParseIfFooter(vi.ifExpr) - case i: ParseInstanceSpec => - lang.attrParse(i, instName, endian) + lang.instanceSetCalculated(instName) + case pi: ParseInstanceSpec => + lang.attrParse(pi, instName, endian) } - lang.instanceSetCalculated(instName) lang.instanceReturn(instName, dataType) lang.instanceFooter } - override def compileCalcEndian(ce: CalcEndian): Unit = { - def renderProc(result: FixedEndian): Unit = { - val v = result match { - case LittleEndian => Ast.expr.Bool(true) - case BigEndian => Ast.expr.Bool(false) - } - lang.instanceCalculate(IS_LE_ID, CalcIntType, v) - } - lang.switchCases[FixedEndian](IS_LE_ID, ce.on, ce.cases, renderProc, renderProc) - } + // override def compileCalcEndian(ce: CalcEndian): Unit = { + // def renderProc(result: FixedEndian): Unit = { + // val v = result match { + // case LittleEndian => Ast.expr.Bool(true) + // case BigEndian => Ast.expr.Bool(false) + // } + // lang.instanceCalculate(IS_LE_ID, CalcIntType, v) + // } + // lang.switchCases[FixedEndian](IS_LE_ID, ce.on, ce.cases, renderProc, renderProc) + // } override def compileAttrDeclarations(attrs: List[MemberSpec]): Unit = { attrs.foreach { (attr) => From 27140e7020a9d697a1d676d0f4d37909daa88abd Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Tue, 9 Apr 2024 16:13:59 +0200 Subject: [PATCH 33/37] fix imports, assign _root directly to non root types --- .../struct/languages/JuliaCompiler.scala | 51 +++++++++++-------- .../struct/translators/JuliaTranslator.scala | 13 +++-- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index 0b980bfb2..56ecc48e0 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -38,7 +38,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def results(topClass: ClassSpec): Map[String, String] = { Map(outFileName(topClass.nameAsStr) -> - (s"module ${types2class(topClass.name)}Module\n" + outHeader.result + outImports(topClass) + enums.result + out.result + "end\n") + (outHeader.result + outImports(topClass) + enums.result + out.result) ) } @@ -53,6 +53,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.toList.mkString("", "\n", "\n") override def fileHeader(topClassName: String): Unit = { + outHeader.puts(s"module ${type2class(topClassName)}Module") outHeader.puts(s"# $headerComment") outHeader.puts importList.add("export from_file") @@ -60,11 +61,16 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts } + override def fileFooter(topClassName: String): Unit = { + fromFile(topClassName) + out.puts("end") + } + override def externalClassDeclaration(classSpec: ClassSpec): Unit = { // val name = classSpec.name.head // importList.add(s"include(${'"'}../../compiled/julia/${classSpec.name.head}.jl${'"'})") // importList.add(s"using .${types2class(classSpec.name)}Module: ${types2class(classSpec.name)}") - importList.add(s"import ${types2class(classSpec.name)}Module: ${types2class(classSpec.name)}") + importList.add(s"import ${type2class(classSpec.name.head)}Module: ${types2class(classSpec.name)}") } override def classHeader(name: List[String]): Unit = { @@ -105,8 +111,12 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("this._io = _io") out.puts("this._parent = _parent") - out.puts("this._root = _root === nothing ? this : _root") - + if (name == rootClassName) { + out.puts("this._root = _root === nothing ? this : _root") + } + else { + out.puts("this._root = _root") + } if (config.readStoresPos) { out.puts("this._attrStart = Dict()") out.puts("this._attrEnd = Dict()") @@ -171,13 +181,15 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${idToStr(attrName)}::Union{Nothing,${kaitaiType2NativeType(attrType)}}") } - def fromFile(name: List[String]): Unit = { - if (typeProvider.nowClass.params.isEmpty) { - out.puts(s"function from_file(filename::String)::${types2class(name)}") - out.inc - out.puts(s"${types2class(name)}($kstreamName(open(filename, ${'"'}r${'"'})))") - universalFooter - } + + + def fromFile(name: String): Unit = { + // if (typeProvider.nowClass.params.isEmpty) { + out.puts(s"function from_file(filename::String)::${type2class(name)}") + out.inc + out.puts(s"${type2class(name)}($kstreamName(open(filename, ${'"'}r${'"'})))") + universalFooter + // } } override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} @@ -200,11 +212,11 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case TextRef(text) => val seeAlso = new StringLanguageOutputWriter("") seeAlso.putsLines(" ", text) - s"$extraNewline\n.. seealso::\n${seeAlso.result}" + s"$extraNewline\nSee also ${seeAlso.result}" case ref: UrlRef => val seeAlso = new StringLanguageOutputWriter("") seeAlso.putsLines(" ", s"${ref.text} - ${ref.url}") - s"$extraNewline\n.. seealso::\n${seeAlso.result}" + s"$extraNewline\nSee also ${seeAlso.result}" }.mkString("\n") out.putsLines("", "\"\"\"" + docStr + refStr + "\"\"\"") @@ -412,7 +424,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"KaitaiStruct.read_bits_int_${bitEndian.toSuffix}($io, $width)" case t: UserType => val addParams = Utils.join(t.args.map(a => translator.translate(a)), "", ", ", ", ") - val addArgs = if (t.isOpaque) { + val addArgs = if (t.isExternal(typeProvider.nowClass)) { "" } else { val parent = t.forcedParent match { @@ -531,11 +543,8 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def classToString(toStringExpr: Ast.expr): Unit = { + out.puts(s"Base.show(io::IO, this::${types2class(typeProvider.nowClass.name)}) = print(io, ${translator.translate(toStringExpr)})") out.puts - // out.puts("def __repr__(self):") - // out.inc - // out.puts(s"return ${translator.translate(toStringExpr)}") - out.dec } override def idToStr(id: Identifier): String = JuliaCompiler.idToStr(id) @@ -603,6 +612,7 @@ object JuliaCompiler extends LanguageCompilerStatic override def kstructName: String = "Any" override def ksErrorName(err: KSError): String = err match { case EndOfStreamError => "ErrorException" + case ConversionError => "ArgumentError" case _ => s"KaitaiStruct.${err.name}" } @@ -626,7 +636,7 @@ object JuliaCompiler extends LanguageCompilerStatic case BitsType(_, _) => "UInt64" case _: BooleanType => "Bool" - case CalcIntType => "Int64" + case CalcIntType => "Integer" case CalcFloatType => "Float64" case _: StrType => "String" @@ -641,7 +651,8 @@ object JuliaCompiler extends LanguageCompilerStatic // }) case _: UserType => "KaitaiStruct.UserType" - case t: EnumType => s"Union{${types2class(t.enumSpec.get.name)},Integer}" + // case t: EnumType => s"Union{${types2class(t.enumSpec.get.name)},Integer}" + case t: EnumType => s"Union{Enum,Integer}" case at: ArrayType => s"Vector{${kaitaiType2NativeType(at.elType)}}" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 7c44f80ee..75921aafe 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -78,7 +78,7 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = { if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head))){ - importList.add(s"using ${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") + importList.add(s"import ${JuliaCompiler.type2class(enumTypeAbs.head)}Module") } s"${JuliaCompiler.type2class(enumTypeAbs.head)}Module.${JuliaCompiler.enumToStr(enumTypeAbs, label)}" @@ -86,9 +86,9 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def doEnumById(enumTypeAbs: List[String], id: String): String = { if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head))){ - importList.add(s"using ${JuliaCompiler.type2class(enumTypeAbs.head)}Module: ${JuliaCompiler.types2class(enumTypeAbs)}") + importList.add(s"import ${JuliaCompiler.type2class(enumTypeAbs.head)}Module") } - s"KaitaiStruct.resolve_enum(${JuliaCompiler.types2class(enumTypeAbs)}, $id)" + s"KaitaiStruct.resolve_enum(${JuliaCompiler.type2class(enumTypeAbs.head)}Module.${JuliaCompiler.types2class(enumTypeAbs)}, $id)" } override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = @@ -158,4 +158,11 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba s"KaitaiStruct.iseof(${translate(value)})" override def kaitaiStreamPos(value: Ast.expr): String = s"KaitaiStruct.pos(${translate(value)})" + + override def doInterpolatedStringLiteral(exprs: Seq[Ast.expr]): String = + if (exprs.isEmpty) { + doStringLiteral("") + } else { + exprs.map(anyToStr).mkString(" * ") + } } From eef7faffcf28dafd259f622c1e9b186be8dc0d8b Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Fri, 26 Apr 2024 14:53:47 +0200 Subject: [PATCH 34/37] use ExternalType for external types --- .../struct/languages/JuliaCompiler.scala | 6 +++--- .../struct/translators/JuliaTranslator.scala | 20 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index 56ecc48e0..745bd2da1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -7,7 +7,7 @@ import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ import io.kaitai.struct.languages.components._ import io.kaitai.struct.translators.JuliaTranslator -import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, StringLanguageOutputWriter, Utils} +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, StringLanguageOutputWriter, Utils, ExternalType} class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) @@ -66,11 +66,11 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("end") } - override def externalClassDeclaration(classSpec: ClassSpec): Unit = { + override def externalTypeDeclaration(extType: ExternalType): Unit = { // val name = classSpec.name.head // importList.add(s"include(${'"'}../../compiled/julia/${classSpec.name.head}.jl${'"'})") // importList.add(s"using .${types2class(classSpec.name)}Module: ${types2class(classSpec.name)}") - importList.add(s"import ${type2class(classSpec.name.head)}Module: ${types2class(classSpec.name)}") + importList.add(s"import ${type2class(extType.name.head)}Module: ${types2class(extType.name)}") } override def classHeader(name: List[String]): Unit = { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 75921aafe..48ae14d38 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -3,7 +3,7 @@ package io.kaitai.struct.translators import io.kaitai.struct.{ImportList} import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast -import io.kaitai.struct.format.Identifier +import io.kaitai.struct.format.{EnumSpec, Identifier} import io.kaitai.struct.languages.{JuliaCompiler} import io.kaitai.struct.exprlang.ConstEvaluator @@ -76,19 +76,21 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def doInternalName(id: Identifier): String = s"this.${JuliaCompiler.publicMemberName(id)}" - override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = { - if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head))){ - importList.add(s"import ${JuliaCompiler.type2class(enumTypeAbs.head)}Module") + override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = { + val isExternal = enumSpec.isExternal(provider.nowClass) + if (isExternal) { + importList.add(s"import ${JuliaCompiler.type2class(enumSpec.name.head)}Module") } - s"${JuliaCompiler.type2class(enumTypeAbs.head)}Module.${JuliaCompiler.enumToStr(enumTypeAbs, label)}" + s"${JuliaCompiler.type2class(enumSpec.name.head)}Module.${JuliaCompiler.enumToStr(enumSpec.name, label)}" } - override def doEnumById(enumTypeAbs: List[String], id: String): String = { - if (!JuliaCompiler.type2class(enumTypeAbs.head).equals(JuliaCompiler.type2class(provider.nowClass.name.head))){ - importList.add(s"import ${JuliaCompiler.type2class(enumTypeAbs.head)}Module") + override def doEnumById(enumSpec: EnumSpec, id: String): String = { + val isExternal = enumSpec.isExternal(provider.nowClass) + if (isExternal) { + importList.add(s"import ${JuliaCompiler.type2class(enumSpec.name.head)}Module") } - s"KaitaiStruct.resolve_enum(${JuliaCompiler.type2class(enumTypeAbs.head)}Module.${JuliaCompiler.types2class(enumTypeAbs)}, $id)" + s"KaitaiStruct.resolve_enum(${JuliaCompiler.type2class(enumSpec.name.head)}Module.${JuliaCompiler.types2class(enumSpec.name)}, $id)" } override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = From bb39c1cdd4633074de4f6fe6465a0a34ddc6c8c8 Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Mon, 29 Apr 2024 11:29:08 +0200 Subject: [PATCH 35/37] delete commented code --- .../struct/languages/JuliaCompiler.scala | 23 ------------------- .../struct/translators/JuliaTranslator.scala | 4 ---- 2 files changed, 27 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index 745bd2da1..c184b7655 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -42,10 +42,6 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) } - // override def classForwardDeclaration(name: List[String]): Unit = { - // abstractTypesAndEnums.puts(s"abstract type ${types2class("Abstract" :: name)} end") - // } - override def indent: String = " " override def outFileName(topClassName: String): String = s"${type2class(topClassName)}Module.jl" @@ -67,15 +63,10 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def externalTypeDeclaration(extType: ExternalType): Unit = { - // val name = classSpec.name.head - // importList.add(s"include(${'"'}../../compiled/julia/${classSpec.name.head}.jl${'"'})") - // importList.add(s"using .${types2class(classSpec.name)}Module: ${types2class(classSpec.name)}") importList.add(s"import ${type2class(extType.name.head)}Module: ${types2class(extType.name)}") } override def classHeader(name: List[String]): Unit = { - // val subtype = if (typeProvider.nowClass.isTopLevel) "" else s" <: ${types2class("Abstract" :: name)}" - // out.puts(s"mutable struct ${types2class(name)}$subtype") out.puts(s"mutable struct ${types2class(name)} <: KaitaiStruct.UserType") out.inc typeProvider.nowClass.meta.endian match { @@ -184,12 +175,10 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) def fromFile(name: String): Unit = { - // if (typeProvider.nowClass.params.isEmpty) { out.puts(s"function from_file(filename::String)::${type2class(name)}") out.inc out.puts(s"${type2class(name)}($kstreamName(open(filename, ${'"'}r${'"'})))") universalFooter - // } } override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} @@ -573,13 +562,6 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) def userType2class(t: UserType): String = { val name = t.classSpec.get.name - // val firstName = name.head - // val prefix = if (t.isOpaque && firstName != translator.provider.nowClass.name.head) { - // s"${type2class(firstName)}Module." - // } else { - // "" - // } - // s"$prefix${types2class(name)}" s"${types2class(name)}" } } @@ -645,13 +627,8 @@ object JuliaCompiler extends LanguageCompilerStatic case AnyType => "Any" case KaitaiStreamType | OwnedKaitaiStreamType => kstreamName case KaitaiStructType | CalcKaitaiStructType(_) => kstructName - // case t: UserType => types2class(t.classSpec match { - // case Some(cs) => if (cs.isTopLevel) cs.name else "Abstract" :: cs.name - // case None => t.name - // }) case _: UserType => "KaitaiStruct.UserType" - // case t: EnumType => s"Union{${types2class(t.enumSpec.get.name)},Integer}" case t: EnumType => s"Union{Enum,Integer}" case at: ArrayType => s"Vector{${kaitaiType2NativeType(at.elType)}}" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 48ae14d38..301de9854 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -35,10 +35,6 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba } } - /** - * https://docs.python.org/2.7/reference/lexical_analysis.html#string-literals - * https://docs.Julia.org/3.6/reference/lexical_analysis.html#string-and-bytes-literals - */ override val asciiCharQuoteMap: Map[Char, String] = Map( '\t' -> "\\t", '\n' -> "\\n", From 479d14a544e8e5d9864dfb843b154d28e21936ef Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Wed, 1 May 2024 11:35:23 +0200 Subject: [PATCH 36/37] Specify data type of elements in condRepeatInitAttr --- .../main/scala/io/kaitai/struct/languages/JuliaCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index c184b7655..f43777076 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -346,7 +346,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def condRepeatInitAttr(id: Identifier, dataType: DataType): Unit = { - out.puts(s"${privateMemberName(id)} = []") + out.puts(s"${privateMemberName(id)} = Vector{${kaitaiType2NativeType(dataType)}}()") } override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = { From 2354168f88c9c50ead7bb1f68cb38cbb59c5232f Mon Sep 17 00:00:00 2001 From: Dias Rystin Date: Wed, 1 May 2024 11:45:07 +0200 Subject: [PATCH 37/37] Introduce JulaCompiler.type2module to reduce code repetition --- .../scala/io/kaitai/struct/languages/JuliaCompiler.scala | 7 ++++--- .../io/kaitai/struct/translators/JuliaTranslator.scala | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala index f43777076..598870e8b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JuliaCompiler.scala @@ -43,13 +43,13 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def indent: String = " " - override def outFileName(topClassName: String): String = s"${type2class(topClassName)}Module.jl" + override def outFileName(topClassName: String): String = s"${type2module(topClassName)}.jl" override def outImports(topClass: ClassSpec): String = importList.toList.mkString("", "\n", "\n") override def fileHeader(topClassName: String): Unit = { - outHeader.puts(s"module ${type2class(topClassName)}Module") + outHeader.puts(s"module ${type2module(topClassName)}") outHeader.puts(s"# $headerComment") outHeader.puts importList.add("export from_file") @@ -63,7 +63,7 @@ class JuliaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def externalTypeDeclaration(extType: ExternalType): Unit = { - importList.add(s"import ${type2class(extType.name.head)}Module: ${types2class(extType.name)}") + importList.add(s"import ${type2module(extType.name.head)}: ${types2class(extType.name)}") } override def classHeader(name: List[String]): Unit = { @@ -599,6 +599,7 @@ object JuliaCompiler extends LanguageCompilerStatic } def types2class(name: List[String]): String = name.map(x => type2class(x)).mkString("_") + def type2module(name: String): String = s"${type2class(name)}Module" def kaitaiType2NativeType(attrType: DataType): String = { attrType match { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala index 301de9854..a0c0456d2 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JuliaTranslator.scala @@ -75,18 +75,18 @@ class JuliaTranslator(provider: TypeProvider, importList: ImportList) extends Ba override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = { val isExternal = enumSpec.isExternal(provider.nowClass) if (isExternal) { - importList.add(s"import ${JuliaCompiler.type2class(enumSpec.name.head)}Module") + importList.add(s"import ${JuliaCompiler.type2module(enumSpec.name.head)}") } - s"${JuliaCompiler.type2class(enumSpec.name.head)}Module.${JuliaCompiler.enumToStr(enumSpec.name, label)}" + s"${JuliaCompiler.type2module(enumSpec.name.head)}.${JuliaCompiler.enumToStr(enumSpec.name, label)}" } override def doEnumById(enumSpec: EnumSpec, id: String): String = { val isExternal = enumSpec.isExternal(provider.nowClass) if (isExternal) { - importList.add(s"import ${JuliaCompiler.type2class(enumSpec.name.head)}Module") + importList.add(s"import ${JuliaCompiler.type2module(enumSpec.name.head)}") } - s"KaitaiStruct.resolve_enum(${JuliaCompiler.type2class(enumSpec.name.head)}Module.${JuliaCompiler.types2class(enumSpec.name)}, $id)" + s"KaitaiStruct.resolve_enum(${JuliaCompiler.type2module(enumSpec.name.head)}.${JuliaCompiler.types2class(enumSpec.name)}, $id)" } override def arraySubscript(container: Ast.expr, idx: Ast.expr): String =