diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 0766a42b4..33d3963fc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -71,7 +71,7 @@ * Implement parsed data validations using `valid` key ([#435](https://github.com/kaitai-io/kaitai_struct/issues/435)) * Implement compile-time `sizeof` and `bitsizeof` operators ([#84](https://github.com/kaitai-io/kaitai_struct/issues/84)) * Type-based: `sizeof`, `bitsizeof`, `sizeof` - * Value-based: `file_header._sizeof`, `flags._bitsizeof` (`file_header`, `flags` are fields defined in the current type) + * Value-based: `file_header._sizeof` (`file_header` is a field defined in the current type) * Implement little-endian bit-sized integers ([docs](https://doc.kaitai.io/user_guide.html#bit-ints-le)) * Support choosing endianness using `le` / `be` suffix: `type: b12le`, `type: b1be` * Add `meta/bit-endian` key for selecting default bit endianness (`le` / `be`) diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala index efd3b3bdb..902d08174 100644 --- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala +++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala @@ -210,35 +210,71 @@ object JavaMain { // Windows, custom install path with spaces and non-latin chars: // /G:/%d0%b3%d0%b4%d0%b5-%d1%82%d0%be%20%d1%82%d0%b0%d0%bc/lib/io.kaitai.kaitai-struct-compiler-0.10-SNAPSHOT.jar - val fStr = classOf[JavaMain].getProtectionDomain.getCodeSource.getLocation.getPath - Log.importOps.info(() => s"home path: location = $fStr") - - if (fStr.endsWith(".jar")) { - val fDec = URLDecoder.decode(fStr, "UTF-8") - Log.importOps.info(() => s"... URL-decoded = $fDec") - - val homeFile = new File(fDec).getParentFile.getParentFile - Log.importOps.info(() => s"... home = $homeFile") - - if (homeFile.exists) { - val homeFormat = new File(homeFile, "formats") - Log.importOps.info(() => s"... formats = $homeFormat") - if (homeFormat.exists) { - Some(homeFormat.toString) + try { + optionOrLog( + classOf[JavaMain].getProtectionDomain.getCodeSource, + "home path: unable to run getCodeSource(), got null" + ).flatMap(sourceCode => optionOrLog( + sourceCode.getLocation, + "home path: unable to run getLocation(), got null" + )).flatMap(location => optionOrLog( + location.getPath, + "home path: unable to run getPath(), got null" + )).flatMap(fStr => { + Log.importOps.info(() => s"home path: location = $fStr") + + if (fStr.endsWith(".jar")) { + val fDec = URLDecoder.decode(fStr, "UTF-8") + Log.importOps.info(() => s"... URL-decoded = $fDec") + + val homeFile = new File(fDec).getParentFile.getParentFile + Log.importOps.info(() => s"... home = $homeFile") + + if (homeFile.exists) { + val homeFormat = new File(homeFile, "formats") + Log.importOps.info(() => s"... formats = $homeFormat") + if (homeFormat.exists) { + Some(homeFormat.toString) + } else { + Log.importOps.info(() => "... home formats dir doesn't exist => fail") + None + } + } else { + Log.importOps.info(() => s"... home doesn't exist => no home import paths") + None + } } else { - Log.importOps.info(() => "... home formats dir doesn't exist => fail") + Log.importOps.info(() => s"... not a jar, we're not running a packaged app => no home") None } - } else { - Log.importOps.info(() => s"... home doesn't exist => no home import paths") + }) + } catch { + case se: SecurityException => + Log.importOps.info(() => s"home path: unable to run getProtectionDomain(), got SecurityException $se") None - } - } else { - Log.importOps.info(() => s"... not a jar, we're not running a packaged app => no home") - None } } + /** + * Helper method to wrap nullable value (coming from Java API) into Option. + * If it's null, we will bail out and won't process any longer due to a chain + * of flatMap(), but if we use this method, we'll also note in our logging which + * step failed, making it easier to diagnose. + * @param nullableValue value which is potentially null + * @param errMsg error message to show in case if it's null + * @tparam T type of potentially nullable value + * @return option-wrapped value + * @see [[scala.Option.apply()]] + */ + private def optionOrLog[T](nullableValue: T, errMsg: String): Option[T] = + Option(nullableValue) match { + case None => + Log.importOps.info(() => errMsg) + None + case someValue => + someValue + } + private def envPaths: List[String] = sys.env.get("KSPATH").toList.flatMap((x) => x.split(File.pathSeparatorChar)) diff --git a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala index 488c81861..309630fac 100644 --- a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala +++ b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala @@ -404,6 +404,21 @@ class ExpressionsSpec extends AnyFunSpec { Expressions.parse("foo.bar") should be (Attribute(Name(identifier("foo")),identifier("bar"))) } + describe("strings") { + it("single-quoted") { + // \" -> \" + // \\ -> \\ + Expressions.parse(""" ' \" \\ ' """) should be(Str(" \\\" \\\\ ")) + Expressions.parse(""" 'ASCII\\x' """) should be(Str("ASCII\\\\x")) + } + it("double-quoted") { + // \" -> " + // \\ -> \ + Expressions.parse(""" " \" \\ " """) should be(Str(" \" \\ ")) + Expressions.parse(""" "ASCII\\'x" """) should be(Str("ASCII\\'x")) + } + } + describe("f-strings") { it("parses f-string with just a string") { Expressions.parse("f\"abc\"") should be(InterpolatedStr(Seq( diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala index 6581e383f..8db8d279a 100644 --- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala +++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala @@ -187,7 +187,7 @@ class TranslatorSpec extends AnyFunSpec { full("42.to_s", CalcIntType, CalcStrType, ResultMap( CppCompiler -> "kaitai::kstream::to_string(42)", CSharpCompiler -> "42.ToString()", - GoCompiler -> "strconv.Itoa(int64(42))", + GoCompiler -> "strconv.FormatInt(int64(42), 10)", JavaCompiler -> "Long.toString(42)", JavaScriptCompiler -> "(42).toString()", LuaCompiler -> "tostring(42)", @@ -200,7 +200,7 @@ class TranslatorSpec extends AnyFunSpec { full("(a + 42).to_s", CalcIntType, CalcStrType, ResultMap( CppCompiler -> "kaitai::kstream::to_string(a() + 42)", CSharpCompiler -> "(A + 42).ToString()", - GoCompiler -> "strconv.Itoa(int64(this.A + 42))", + GoCompiler -> "strconv.FormatInt(int64(this.A + 42), 10)", JavaCompiler -> "Long.toString(a() + 42)", JavaScriptCompiler -> "(this.a + 42).toString()", LuaCompiler -> "tostring(self.a + 42)", @@ -213,7 +213,7 @@ class TranslatorSpec extends AnyFunSpec { full("a + 42.to_s", CalcStrType, CalcStrType, ResultMap( CppCompiler -> "a() + kaitai::kstream::to_string(42)", CSharpCompiler -> "A + 42.ToString()", - GoCompiler -> "this.A + strconv.Itoa(int64(42))", + GoCompiler -> "this.A + strconv.FormatInt(int64(42), 10)", JavaCompiler -> "a() + Long.toString(42)", JavaScriptCompiler -> "this.a + (42).toString()", LuaCompiler -> "self.a .. tostring(42)", @@ -701,7 +701,7 @@ class TranslatorSpec extends AnyFunSpec { PerlCompiler -> "substr(\"foobar\", 2, 4 - 2)", PHPCompiler -> "\\Kaitai\\Struct\\Stream::substring(\"foobar\", 2, 4)", PythonCompiler -> "u\"foobar\"[2:4]", - RubyCompiler -> "\"foobar\"[2..4 - 1]" + RubyCompiler -> "\"foobar\"[2...4]" )) // substring() call on concatenation of strings: for some languages, concatenation needs to be @@ -716,7 +716,7 @@ class TranslatorSpec extends AnyFunSpec { PerlCompiler -> "substr($self->foo() . $self->bar(), 2, 4 - 2)", PHPCompiler -> "\\Kaitai\\Struct\\Stream::substring($this->foo() . $this->bar(), 2, 4)", PythonCompiler -> "(self.foo + self.bar)[2:4]", - RubyCompiler -> "(foo + bar)[2..4 - 1]" + RubyCompiler -> "(foo + bar)[2...4]" )) // substring() call with non-left-associative "from" and "to": for languages where subtraction @@ -731,7 +731,7 @@ class TranslatorSpec extends AnyFunSpec { PerlCompiler -> "substr($self->foo(), 10 - 7, (10 - 3) - (10 - 7))", // TODO: PerlCompiler -> "substr($self->foo(), 10 - 7, 10 - 3 - (10 - 7))", PHPCompiler -> "\\Kaitai\\Struct\\Stream::substring($this->foo(), 10 - 7, 10 - 3)", PythonCompiler -> "self.foo[10 - 7:10 - 3]", - RubyCompiler -> "foo[10 - 7..(10 - 3) - 1]" // TODO: RubyCompiler -> "foo[10 - 7..10 - 3 - 1]" + RubyCompiler -> "foo[10 - 7...10 - 3]" )) // substring() call with "to" using `<<` which is lower precedence than `+` or `-`: if such @@ -746,7 +746,7 @@ class TranslatorSpec extends AnyFunSpec { PerlCompiler -> "substr($self->foo(), 10 - 7, (10 << 2) - (10 - 7))", PHPCompiler -> "\\Kaitai\\Struct\\Stream::substring($this->foo(), 10 - 7, 10 << 2)", PythonCompiler -> "self.foo[10 - 7:10 << 2]", - RubyCompiler -> "foo[10 - 7..(10 << 2) - 1]" + RubyCompiler -> "foo[10 - 7...10 << 2]" )) // substring() call with "from" using `<<` which is lower precedence than `+` or `-`: if such @@ -761,7 +761,7 @@ class TranslatorSpec extends AnyFunSpec { PerlCompiler -> "substr($self->foo(), 10 << 1, 42 - (10 << 1))", PHPCompiler -> "\\Kaitai\\Struct\\Stream::substring($this->foo(), 10 << 1, 42)", PythonCompiler -> "self.foo[10 << 1:42]", - RubyCompiler -> "foo[10 << 1..42 - 1]" + RubyCompiler -> "foo[10 << 1...42]" )) } } diff --git a/project/build.properties b/project/build.properties index b19d4e1ed..be54e7763 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.9.7 +sbt.version = 1.10.0 diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index 45a6ef58a..35d32d178 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -316,12 +316,12 @@ class ClassCompiler( compileInstanceDoc(instName, instSpec) lang.instanceCheckCacheAndReturn(instName, dataType) + lang.instanceSetCalculated(instName) instSpec match { case vi: ValueInstanceSpec => lang.attrParseIfHeader(instName, vi.ifExpr) lang.instanceCalculate(instName, dataType, vi.value) lang.attrParseIfFooter(vi.ifExpr) - lang.instanceSetCalculated(instName) case pi: ParseInstanceSpec => lang.attrParse(pi, instName, endian) } diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala index 192259fff..4637773bd 100644 --- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala @@ -163,8 +163,7 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend val subcon2 = blt.terminator match { case None => subcon case Some(term) => - val termStr = "\\x%02X".format(term & 0xff) - s"NullTerminated($subcon, term=b'$termStr', include=${translator.doBoolLiteral(blt.include)})" + s"NullTerminated($subcon, term=${translator.doByteArrayLiteral(term)}, include=${translator.doBoolLiteral(blt.include)})" } val subcon3 = blt.padRight match { case None => subcon2 @@ -176,9 +175,8 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend } def attrBytesTerminatedType(btt: BytesTerminatedType, subcon: String): String = { - val termStr = "\\x%02X".format(btt.terminator & 0xff) s"NullTerminated($subcon, " + - s"term=b'$termStr', " + + s"term=${translator.doByteArrayLiteral(btt.terminator)}, " + s"include=${translator.doBoolLiteral(btt.include)}, " + s"consume=${translator.doBoolLiteral(btt.consume)})" } diff --git a/shared/src/main/scala/io/kaitai/struct/EncodingList.scala b/shared/src/main/scala/io/kaitai/struct/EncodingList.scala index 87645d45f..f1091632d 100644 --- a/shared/src/main/scala/io/kaitai/struct/EncodingList.scala +++ b/shared/src/main/scala/io/kaitai/struct/EncodingList.scala @@ -14,37 +14,40 @@ object EncodingList { * generate a KSC warning no matter which form was used. */ val canonicalToAlias = Map( - "ASCII" -> Set("US-ASCII"), - "UTF-8" -> Set("UTF8", "ISO-10646/UTF-8", "ISO-10646/UTF8"), - "UTF-16LE" -> Set("UTF16LE"), - "UTF-16BE" -> Set("UTF16BE"), - "UTF-32LE" -> Set("UTF32LE"), - "UTF-32BE" -> Set("UTF32BE"), - "ISO-8859-1" -> Set("ISO8859-1", "ISO_8859-1", "ISO88591", "8859_1", "8859-1", "88591", "LATIN1", "IBM819", "CP819", "csISOLatin1", "iso-ir-100", "windows-28591", "WE8ISO8859P1"), - "ISO-8859-2" -> Set("ISO8859-2", "ISO_8859-2", "ISO88592", "8859_2", "8859-2", "88592", "LATIN2", "IBM1111", "CP1111", "csISOLatin2", "iso-ir-101", "windows-28592"), - "ISO-8859-3" -> Set("ISO8859-3", "ISO_8859-3", "ISO88593", "8859_3", "8859-3", "88593", "LATIN3", "csISOLatin3", "iso-ir-109", "windows-28593"), - "ISO-8859-4" -> Set("ISO8859-4", "ISO_8859-4", "ISO88594", "8859_4", "8859-4", "88594", "LATIN4", "csISOLatin4", "iso-ir-110", "windows-28594"), - "ISO-8859-5" -> Set("ISO8859-5", "ISO_8859-5", "ISO88595", "8859_5", "8859-5", "88595", "csISOLatinCyrillic", "ISO-IR-144", "windows-28595"), - "ISO-8859-6" -> Set("ISO8859-6", "ISO_8859-6", "ISO88596", "8859_6", "8859-6", "88596", "iso-ir-127", "ECMA-114", "ASMO-708", "csISOLatinArabic", "windows-28596"), - "ISO-8859-7" -> Set("ISO8859-7", "ISO_8859-7", "ISO88597", "8859_7", "8859-7", "88597"), - "ISO-8859-8" -> Set("ISO8859-8", "ISO_8859-8", "ISO88598", "8859_8", "8859-8", "88598"), - "ISO-8859-9" -> Set("ISO8859-9", "ISO_8859-9", "ISO88599", "8859_9", "8859-9", "88599"), - "ISO-8859-10" -> Set("ISO8859-10", "ISO_8859-10", "ISO885910", "8859_10", "8859-10", "885910"), - "ISO-8859-11" -> Set("ISO8859-11", "ISO_8859-11", "ISO885911", "8859_11", "8859-11", "885911"), - "ISO-8859-13" -> Set("ISO8859-13", "ISO_8859-13", "ISO885913", "8859_13", "8859-13", "885913"), - "ISO-8859-14" -> Set("ISO8859-14", "ISO_8859-14", "ISO885914", "8859_14", "8859-14", "885914"), - "ISO-8859-15" -> Set("ISO8859-15", "ISO_8859-15", "ISO885915", "8859_15", "8859-15", "885915"), - "ISO-8859-16" -> Set("ISO8859-16", "ISO_8859-16", "ISO885916", "8859_16", "8859-16", "885916"), - "windows-1250" -> Set("CP1250"), - "windows-1251" -> Set("CP1251"), - "windows-1252" -> Set("CP1252"), - "windows-1253" -> Set("CP1253"), - "windows-1254" -> Set("CP1254"), - "windows-1255" -> Set("CP1255"), - "windows-1256" -> Set("CP1256"), - "windows-1257" -> Set("CP1257"), - "windows-1258" -> Set("CP1258"), - "IBM437" -> Set("cp437", "437", "csibm437"), - "IBM866" -> Set("cp866", "866", "csibm866"), + "ASCII" -> Set("US-ASCII", "US_ASCII", "IBM367", "cp367", "csASCII", "iso-ir-6"), + "UTF-8" -> Set("UTF8", "UTF_8", "ISO-10646/UTF-8", "ISO-10646/UTF8", "cp65001", "csUTF8", "unicode-1-1-utf-8", "unicode-2-0-utf-8"), + "UTF-16BE" -> Set("UTF16BE", "UTF16-BE", "UTF-16-BE", "UTF_16BE", "UTF16_BE", "UTF_16_BE", "csUTF16BE"), + "UTF-16LE" -> Set("UTF16LE", "UTF16-LE", "UTF-16-LE", "UTF_16LE", "UTF16_LE", "UTF_16_LE", "csUTF16LE"), + "UTF-32BE" -> Set("UTF32BE", "UTF32-BE", "UTF-32-BE", "UTF_32BE", "UTF32_BE", "UTF_32_BE", "csUTF32BE"), + "UTF-32LE" -> Set("UTF32LE", "UTF32-LE", "UTF-32-LE", "UTF_32LE", "UTF32_LE", "UTF_32_LE", "csUTF32LE"), + "ISO-8859-1" -> Set("ISO8859-1", "ISO_8859-1", "ISO88591", "ISO_8859_1", "ISO8859_1", "8859_1", "8859-1", "88591", "latin1", "L1", "csISOLatin1", "iso-ir-100", "IBM819", "cp819", "windows-28591"), + "ISO-8859-2" -> Set("ISO8859-2", "ISO_8859-2", "ISO88592", "ISO_8859_2", "ISO8859_2", "8859_2", "8859-2", "88592", "latin2", "L2", "csISOLatin2", "iso-ir-101", "IBM1111", "windows-28592"), + "ISO-8859-3" -> Set("ISO8859-3", "ISO_8859-3", "ISO88593", "ISO_8859_3", "ISO8859_3", "8859_3", "8859-3", "88593", "latin3", "L3", "csISOLatin3", "iso-ir-109", "windows-28593"), + "ISO-8859-4" -> Set("ISO8859-4", "ISO_8859-4", "ISO88594", "ISO_8859_4", "ISO8859_4", "8859_4", "8859-4", "88594", "latin4", "L4", "csISOLatin4", "iso-ir-110", "windows-28594"), + "ISO-8859-5" -> Set("ISO8859-5", "ISO_8859-5", "ISO88595", "ISO_8859_5", "ISO8859_5", "8859_5", "8859-5", "88595", "cyrillic", "csISOLatinCyrillic", "iso-ir-144", "windows-28595"), + "ISO-8859-6" -> Set("ISO8859-6", "ISO_8859-6", "ISO88596", "ISO_8859_6", "ISO8859_6", "8859_6", "8859-6", "88596", "arabic", "csISOLatinArabic", "iso-ir-127", "windows-28596", "ECMA-114", "ASMO-708"), + "ISO-8859-7" -> Set("ISO8859-7", "ISO_8859-7", "ISO88597", "ISO_8859_7", "ISO8859_7", "8859_7", "8859-7", "88597", "greek", "greek8", "csISOLatinGreek", "iso-ir-126", "windows-28597", "ECMA-118", "ELOT_928"), + "ISO-8859-8" -> Set("ISO8859-8", "ISO_8859-8", "ISO88598", "ISO_8859_8", "ISO8859_8", "8859_8", "8859-8", "88598", "hebrew", "csISOLatinHebrew", "iso-ir-138", "windows-28598"), + "ISO-8859-9" -> Set("ISO8859-9", "ISO_8859-9", "ISO88599", "ISO_8859_9", "ISO8859_9", "8859_9", "8859-9", "88599", "latin5", "L5", "csISOLatin5", "iso-ir-148", "windows-28599"), + "ISO-8859-10" -> Set("ISO8859-10", "ISO_8859-10", "ISO885910", "ISO_8859_10", "ISO8859_10", "8859_10", "8859-10", "885910", "latin6", "L6", "csISOLatin6", "iso-ir-157"), + "ISO-8859-11" -> Set("ISO8859-11", "ISO_8859-11", "ISO885911", "ISO_8859_11", "ISO8859_11", "8859_11", "8859-11", "885911", "thai", "csTIS620", "TIS-620"), + "ISO-8859-13" -> Set("ISO8859-13", "ISO_8859-13", "ISO885913", "ISO_8859_13", "ISO8859_13", "8859_13", "8859-13", "885913", "latin7", "L7", "csISO885913", "windows-28603"), + "ISO-8859-14" -> Set("ISO8859-14", "ISO_8859-14", "ISO885914", "ISO_8859_14", "ISO8859_14", "8859_14", "8859-14", "885914", "latin8", "L8", "csISO885914", "iso-ir-199", "iso-celtic"), + "ISO-8859-15" -> Set("ISO8859-15", "ISO_8859-15", "ISO885915", "ISO_8859_15", "ISO8859_15", "8859_15", "8859-15", "885915", "latin9", "L9", "csISO885915", "windows-28605"), + "ISO-8859-16" -> Set("ISO8859-16", "ISO_8859-16", "ISO885916", "ISO_8859_16", "ISO8859_16", "8859_16", "8859-16", "885916", "latin10", "L10", "csISO885916"), + "windows-1250" -> Set("cp1250", "cswindows1250"), + "windows-1251" -> Set("cp1251", "cswindows1251"), + "windows-1252" -> Set("cp1252", "cswindows1252"), + "windows-1253" -> Set("cp1253", "cswindows1253"), + "windows-1254" -> Set("cp1254", "cswindows1254"), + "windows-1255" -> Set("cp1255", "cswindows1255"), + "windows-1256" -> Set("cp1256", "cswindows1256"), + "windows-1257" -> Set("cp1257", "cswindows1257"), + "windows-1258" -> Set("cp1258", "cswindows1258"), + "IBM437" -> Set("cp437", "437", "csPC8CodePage437"), + "IBM866" -> Set("cp866", "866", "csIBM866"), + "Shift_JIS" -> Set("Shift-JIS", "ShiftJIS", "S-JIS", "S_JIS", "SJIS", "PCK", "csShiftJIS"), + "Big5" -> Set("csBig5"), + "EUC-KR" -> Set("EUCKR", "EUC_KR", "csEUCKR", "korean", "iso-ir-149"), ) } diff --git a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala index dd02b2997..8959d753e 100644 --- a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala @@ -70,16 +70,16 @@ class GoClassCompiler( lang.instanceHeader(className, instName, dataType, instSpec.isNullable) lang.instanceCheckCacheAndReturn(instName, dataType) + lang.instanceSetCalculated(instName) instSpec match { case vi: ValueInstanceSpec => lang.attrParseIfHeader(instName, vi.ifExpr) lang.instanceCalculate(instName, dataType, vi.value) lang.attrParseIfFooter(vi.ifExpr) - case i: ParseInstanceSpec => - lang.attrParse(i, instName, endian) + case pi: ParseInstanceSpec => + lang.attrParse(pi, instName, endian) } - lang.instanceSetCalculated(instName) lang.instanceReturn(instName, dataType) lang.instanceFooter } diff --git a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala index 0e8386cb1..22c746953 100644 --- a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala @@ -8,7 +8,7 @@ import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompiler import io.kaitai.struct.precompile.CalculateSeqSizes import io.kaitai.struct.translators.RubyTranslator -import scala.collection.mutable.ListBuffer +import scala.collection.mutable.{ListBuffer, LinkedHashSet} class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends AbstractCompiler { import GraphvizClassCompiler._ @@ -17,7 +17,7 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends val provider = new ClassTypeProvider(classSpecs, topClass) val translator = new RubyTranslator(provider) - val links = ListBuffer[(String, String, String)]() + val links = LinkedHashSet[(String, String, String)]() val extraClusterLines = new StringLanguageOutputWriter(indent) def nowClass: ClassSpec = provider.nowClass @@ -133,7 +133,9 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends val STYLE_EDGE_MISC = "color=\"#404040\"" val STYLE_EDGE_POS = STYLE_EDGE_MISC val STYLE_EDGE_SIZE = STYLE_EDGE_MISC + val STYLE_EDGE_VALID = STYLE_EDGE_MISC val STYLE_EDGE_REPEAT = STYLE_EDGE_MISC + val STYLE_EDGE_IF = STYLE_EDGE_MISC val STYLE_EDGE_VALUE = STYLE_EDGE_MISC def tableRow(curClass: List[String], pos: Option[String], attr: AttrLikeSpec, name: String): Unit = { @@ -145,7 +147,7 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends compileSwitch(name, st) s"switch (${expressionType(st.on, name)})" case _ => - dataTypeName(dataType) + dataTypeName(dataType, attr.valid) } out.puts("" + @@ -163,8 +165,48 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends // ignore, no links } + dataType match { + case blt: BytesLimitType if fixedBytes(blt, attr.valid).isDefined => + // No additional line for `valid` because it expresses the same simple + // constraint as if the `contents` key were used, and therefore was + // already displayed in the "type" column. + case _ => validTableRow(dataType, attr.valid, name) + } + repeatTableRow(dataType, attr.cond.repeat, name) + ifTableRow(attr.cond.ifExpr, name) + } + + def validTableRow(dataType: DataType, valid: Option[ValidationSpec], name: String): Unit = { + val portName = name + "__valid" + val fullPortName = s"$currentTable:$portName" + val text = valid match { + case Some(v) => + v match { + case ValidationEq(expected) => + s"must be equal to ${expression(expected, fullPortName, STYLE_EDGE_VALID)}" + case ValidationMin(min) => + s"must be at least ${expression(min, fullPortName, STYLE_EDGE_VALID)}" + case ValidationMax(max) => + s"must be at most ${expression(max, fullPortName, STYLE_EDGE_VALID)}" + case ValidationRange(min, max) => + s"must be between ${expression(min, fullPortName, STYLE_EDGE_VALID)} " + + s"and ${expression(max, fullPortName, STYLE_EDGE_VALID)}" + case ValidationAnyOf(values) => + s"must be any of ${values.map(expression(_, fullPortName, STYLE_EDGE_VALID)).mkString(", ")}" + case ValidationInEnum() => + "must be defined in the enum" + case ValidationExpr(expr) => + provider._currentIteratorType = Some(dataType) + s"must satisfy ${expression(expr, fullPortName, STYLE_EDGE_VALID)}" + } + case None => return + } + out.puts("" + text + "") + } + + def repeatTableRow(dataType: DataType, rep: RepeatSpec, name: String): Unit = { val portName = name + "__repeat" - attr.cond.repeat match { + rep match { case RepeatExpr(ex) => out.puts("repeat " + expression(ex, s"$currentTable:$portName", STYLE_EDGE_REPEAT) + @@ -177,7 +219,18 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends case RepeatEos => out.puts("repeat to end of stream") case NoRepeat => - // no additional line + // no additional line + } + } + + def ifTableRow(ifExpr: Option[Ast.expr], name: String): Unit = { + val portName = name + "__if" + ifExpr match { + case Some(e) => + out.puts("if " + + expression(e, s"$currentTable:$portName", STYLE_EDGE_IF) + + "") + case None => // ignore } } @@ -199,7 +252,6 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends } def compileSwitch(attrName: String, st: SwitchType): Unit = { - links += ((s"$currentTable:${attrName}_type", s"${currentTable}_${attrName}_switch", STYLE_EDGE_TYPE)) extraClusterLines.puts(s"${currentTable}_${attrName}_switch " + "[label=<") extraClusterLines.inc @@ -291,7 +343,6 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends // case expr.Dict(keys, values) => case Ast.expr.Compare(left, ops, right) => affectedVars(left) ++ affectedVars(right) - // case expr.Call(func, args) => case Ast.expr.IntNum(_) | Ast.expr.FloatNum(_) | Ast.expr.Str(_) | Ast.expr.Bool(_) => List() case _: Ast.expr.EnumByLabel => @@ -299,6 +350,30 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends case Ast.expr.EnumById(_, id, _) => affectedVars(id) case Ast.expr.Attribute(value, attr) => + if (attr.name == Identifier.SIZEOF) { + val vars = value match { + case Ast.expr.Name(id) if !id.name.startsWith("_") => + List(getGraphvizNode(nowClassName, nowClass, id.name) + s":${id.name}_size") + case Ast.expr.Attribute(valueNested, attrNested) if !attrNested.name.startsWith("_") => + val targetClass = translator.detectType(valueNested) + targetClass match { + case t: UserType => + val className = t.name + val classSpec = t.classSpec.get + List(getGraphvizNode(className, classSpec, attrNested.name) + s":${attrNested.name}_size") + case _ => + affectedVars(value) + } + case _ => + affectedVars(value) + } + return vars + } + + // special names like "_io", "_parent", "_root" + if (attr.name.startsWith("_")) + return affectedVars(value) + val targetClass = translator.detectType(value) targetClass match { case t: UserType => @@ -322,12 +397,9 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends fromFunc ::: affectedVars(Ast.expr.List(args)) case Ast.expr.Subscript(value, idx) => affectedVars(value) ++ affectedVars(idx) - case SwitchType.ELSE_CONST => - // "_" is a special const for - List() case Ast.expr.Name(id) => - if (id.name.charAt(0) == '_') { - // other special consts like "_io", "_index", etc + if (id.name.startsWith("_")) { + // other special names like "_", "_io", "_index", etc. List() } else { // this must be local name, resolve it @@ -335,6 +407,12 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends } case Ast.expr.List(elts) => elts.flatMap(affectedVars).toList + case Ast.expr.CastToType(expr, _) => + affectedVars(expr) + case Ast.expr.ByteSizeOfType(_) => + List() + case Ast.expr.BitSizeOfType(_) => + List() } } @@ -408,15 +486,14 @@ object GraphvizClassCompiler extends LanguageCompilerStatic { def type2class(name: List[String]) = name.last def type2display(name: List[String]) = name.map(Utils.upperCamelCase).mkString("::") - def dataTypeName(dataType: DataType): String = { + def dataTypeName(dataType: DataType, valid: Option[ValidationSpec]): String = { dataType match { case rt: ReadableType => rt.apiCall(None) // FIXME case ut: UserType => type2display(ut.name) - //case FixedBytesType(contents, _) => contents.map(_.formatted("%02X")).mkString(" ") case BytesTerminatedType(terminator, include, consume, eosError, _) => val args = ListBuffer[String]() - if (terminator != 0) - args += s"term=$terminator" + val termStr = terminator.map(_ & 0xff).mkString(", ") + args += "term=" + (if (terminator.length == 1) termStr else s"[$termStr]") if (include) args += "include" if (!consume) @@ -424,18 +501,32 @@ object GraphvizClassCompiler extends LanguageCompilerStatic { if (!eosError) args += "ignore EOS" args.mkString(", ") + case blt: BytesLimitType => fixedBytes(blt, valid).getOrElse("") case _: BytesType => "" case StrFromBytesType(basedOn, encoding, _) => - val bytesStr = dataTypeName(basedOn) + val bytesStr = dataTypeName(basedOn, valid) val comma = if (bytesStr.isEmpty) "" else ", " s"str($bytesStr$comma$encoding)" case EnumType(name, basedOn) => - s"${dataTypeName(basedOn)}→${type2display(name)}" - case BitsType(width, bitEndian) => s"b$width" + s"${dataTypeName(basedOn, valid)}→${type2display(name)}" + case BitsType(width, bitEndian) => s"b$width${bitEndian.toSuffix}" + case BitsType1(bitEndian) => s"b1${bitEndian.toSuffix}→bool" case _ => dataType.toString } } + private def fixedBytes(blt: BytesLimitType, valid: Option[ValidationSpec]): Option[String] = { + valid match { + case Some(ValidationEq(Ast.expr.List(contents))) + if blt.size == Ast.expr.IntNum(contents.length) => + Some(contents.map(_ match { + case Ast.expr.IntNum(byteVal) if byteVal >= 0x00 && byteVal <= 0xff => "%02X".format(byteVal) + case _ => return None + }).mkString(" ")) + case _ => None + } + } + def htmlEscape(s: String): String = { s.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """) } diff --git a/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala index 28202b40a..5c48a24f9 100644 --- a/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala @@ -147,6 +147,6 @@ object HtmlClassCompiler extends LanguageCompilerStatic { def kaitaiType2NativeType(attrType: DataType): String = attrType match { case ut: UserType => "" + type2str(ut.name.last) + "" - case _ => GraphvizClassCompiler.dataTypeName(attrType) + case _ => GraphvizClassCompiler.dataTypeName(attrType, None) } } diff --git a/shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala index a472baabf..b35dbff23 100644 --- a/shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala @@ -95,12 +95,12 @@ class NimClassCompiler( compileInstanceDoc(instName, instSpec) lang.instanceCheckCacheAndReturn(instName, dataType) + lang.instanceSetCalculated(instName) instSpec match { case vi: ValueInstanceSpec => lang.attrParseIfHeader(instName, vi.ifExpr) lang.instanceCalculate(instName, dataType, vi.value) lang.attrParseIfFooter(vi.ifExpr) - lang.instanceSetCalculated(instName) case pi: ParseInstanceSpec => lang.attrParse(pi, instName, endian) } diff --git a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala index 59ee86001..ef613946f 100644 --- a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala +++ b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala @@ -8,8 +8,8 @@ package io.kaitai.struct * @param stdStringFrontBack If true, allow use of `front()` and `back()` methods * on `std::string`. If false, come up with simulation. * @param useListInitializers If true, allows use of list initializers for - * `std::vector`. Otherwise, throw a fatal unimplemented - * error. + * `std::vector` and `std::set`. Otherwise, throw a fatal + * "not implemented" error. * @param pointers Choose which style of pointers to use. */ case class CppRuntimeConfig( diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala index 8be1aec4b..c0f0817dd 100644 --- a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala @@ -108,16 +108,16 @@ class RustClassCompiler( lang.instanceHeader(className, instName, dataType, instSpec.isNullable) lang.instanceCheckCacheAndReturn(instName, dataType) + lang.instanceSetCalculated(instName) instSpec match { case vi: ValueInstanceSpec => lang.attrParseIfHeader(instName, vi.ifExpr) lang.instanceCalculate(instName, dataType, vi.value) lang.attrParseIfFooter(vi.ifExpr) - case i: ParseInstanceSpec => - lang.attrParse(i, instName, None) // FIXME + case pi: ParseInstanceSpec => + lang.attrParse(pi, instName, None) // FIXME } - lang.instanceSetCalculated(instName) lang.instanceReturn(instName, dataType) lang.instanceFooter } diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala index 546c0afbe..96e6f4036 100644 --- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala +++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala @@ -4,6 +4,7 @@ import io.kaitai.struct.exprlang.{Ast, Expressions} import io.kaitai.struct.format._ import io.kaitai.struct.problems.KSYParseError import io.kaitai.struct.translators.TypeDetector +import io.kaitai.struct.precompile.CanonicalizeEncodingNames import scala.collection.immutable.SortedMap @@ -70,20 +71,20 @@ object DataType { override def process = None } case class BytesEosType( - terminator: Option[Int], + terminator: Option[Seq[Byte]], include: Boolean, padRight: Option[Int], override val process: Option[ProcessExpr] ) extends BytesType case class BytesLimitType( size: Ast.expr, - terminator: Option[Int], + terminator: Option[Seq[Byte]], include: Boolean, padRight: Option[Int], override val process: Option[ProcessExpr] ) extends BytesType case class BytesTerminatedType( - terminator: Int, + terminator: Seq[Byte], include: Boolean, consume: Boolean, eosError: Boolean, @@ -429,9 +430,27 @@ object DataType { case "str" | "strz" => val enc = getEncoding(arg.encoding, metaDef, path) - // "strz" makes terminator = 0 by default + // "strz" selects the appropriate null terminator depending on the "encoding", i.e. 2 zero + // bytes for UTF-16*, 4 zero bytes for UTF-32* and 1 zero byte for all other encodings val arg2 = if (dt == "strz") { - arg.copy(terminator = arg.terminator.orElse(Some(0))) + val term = arg.terminator match { + case Some(t) => + t + case None => + // FIXME: ideally, this null terminator resolution should not happen here in the + // "YAML parsing stage", but later on some intermediate representation after the + // `CanonicalizeEncodingNames` precompile step has run. See the discussion in + // https://github.com/kaitai-io/kaitai_struct_compiler/pull/278#discussion_r1527198115 + val (newEncoding, problem) = CanonicalizeEncodingNames.canonicalizeName(enc) + val nullTerm: Seq[Byte] = newEncoding match { + /** @note Must be kept in sync with [[EncodingList.canonicalToAlias]] */ + case "UTF-16LE" | "UTF-16BE" => Seq(0, 0) + case "UTF-32LE" | "UTF-32BE" => Seq(0, 0, 0, 0) + case _ => Seq(0) + } + nullTerm + } + arg.copy(terminator = Some(term)) } else { arg } diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/KSError.scala b/shared/src/main/scala/io/kaitai/struct/datatype/KSError.scala index 3f2f04142..2bbb2ba01 100644 --- a/shared/src/main/scala/io/kaitai/struct/datatype/KSError.scala +++ b/shared/src/main/scala/io/kaitai/struct/datatype/KSError.scala @@ -22,6 +22,7 @@ object KSError { case "ValidationLessThanError" => ValidationLessThanError case "ValidationGreaterThanError" => ValidationGreaterThanError case "ValidationNotAnyOfError" => ValidationNotAnyOfError + case "ValidationNotInEnumError" => ValidationNotInEnumError case "ValidationExprError" => ValidationExprError } excClass(dataType) @@ -66,6 +67,14 @@ case class ValidationNotAnyOfError(_dt: DataType) extends ValidationError(_dt) { def name = "ValidationNotAnyOfError" } +/** + * Error to be thrown when validation fails with actual not being in the enum. + * @param _dt data type used in validation process + */ +case class ValidationNotInEnumError(_dt: DataType) extends ValidationError(_dt) { + def name = "ValidationNotInEnumError" +} + /** * Error to be thrown when validation fails with actual not matching the custom * validation expression. diff --git a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala index 09e1b941f..d9e595dca 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala @@ -83,7 +83,7 @@ case class YamlAttrArgs( size: Option[Ast.expr], sizeEos: Boolean, encoding: Option[String], - terminator: Option[Int], + terminator: Option[Seq[Byte]], include: Boolean, consume: Boolean, eosError: Boolean, @@ -179,11 +179,11 @@ object AttrSpec { val sizeEos = ParseUtils.getOptValueBool(srcMap, "size-eos", path).getOrElse(false) val ifExpr = ParseUtils.getOptValueExpression(srcMap, "if", path) val encoding = ParseUtils.getOptValueStr(srcMap, "encoding", path) - val terminator = ParseUtils.getOptValueInt(srcMap, "terminator", path) + val terminator = ParseUtils.getOptValueByte(srcMap, "terminator", path).map(value => Seq(value.toByte)) val consume = ParseUtils.getOptValueBool(srcMap, "consume", path).getOrElse(true) val include = ParseUtils.getOptValueBool(srcMap, "include", path).getOrElse(false) val eosError = ParseUtils.getOptValueBool(srcMap, "eos-error", path).getOrElse(true) - val padRight = ParseUtils.getOptValueInt(srcMap, "pad-right", path) + val padRight = ParseUtils.getOptValueByte(srcMap, "pad-right", path) val enumOpt = ParseUtils.getOptValueStr(srcMap, "enum", path) val parent = ParseUtils.getOptValueExpression(srcMap, "parent", path) val valid = srcMap.get("valid").map(ValidationSpec.fromYaml(_, path ++ List("valid"))) diff --git a/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala b/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala index 9d35c47e0..2912eb9e9 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala @@ -70,6 +70,19 @@ object Identifier { } } + def itemExpr(id: Identifier, rep: RepeatSpec): Ast.expr = { + val astId = Ast.expr.InternalName(id) + rep match { + case NoRepeat => + astId + case _ => + Ast.expr.Subscript( + astId, + Ast.expr.Name(Ast.identifier(Identifier.INDEX)) + ) + } + } + // Constants for special names used in expression language val ROOT = "_root" val PARENT = "_parent" diff --git a/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala b/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala index 97652f03d..1481cddcb 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala @@ -72,6 +72,15 @@ object ParseUtils { } } + def getOptValueByte(src: Map[String, Any], field: String, path: List[String]): Option[Int] = { + getOptValueInt(src, field, path).map { value => + if (value < 0 || value > 255) { + throw KSYParseError.withText(s"expected an integer from 0 to 255, got ${value}", path ++ List(field)) + } + value + } + } + def getValueIdentifier(src: Map[String, Any], idx: Int, entityName: String, path: List[String]): Identifier = { getOptValueStr(src, "id", path) match { case Some(idStr) => diff --git a/shared/src/main/scala/io/kaitai/struct/format/ValidationSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ValidationSpec.scala index 5142517c4..3f404ce4f 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/ValidationSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/ValidationSpec.scala @@ -10,6 +10,7 @@ case class ValidationMin(min: Ast.expr) extends ValidationSpec case class ValidationMax(max: Ast.expr) extends ValidationSpec case class ValidationRange(min: Ast.expr, max: Ast.expr) extends ValidationSpec case class ValidationAnyOf(values: List[Ast.expr]) extends ValidationSpec +case class ValidationInEnum() extends ValidationSpec case class ValidationExpr(checkExpr: Ast.expr) extends ValidationSpec object ValidationEq { @@ -59,13 +60,30 @@ object ValidationAnyOf { } } +object ValidationInEnum { + val LEGAL_KEYS = Set("in-enum") + + def fromMap(src: Map[String, Any], path: List[String]): Option[ValidationInEnum] = + ParseUtils.getOptValueBool(src, "in-enum", path).map { case boolVal => + ParseUtils.ensureLegalKeys(src, LEGAL_KEYS, path) + if (!boolVal) { + throw KSYParseError.withText( + "only `true` is supported as value, got `false`" + + " (if you don't want any validation, omit the `valid` key)", + path ++ List("in-enum") + ) + } + ValidationInEnum() + } +} + object ValidationExpr { val LEGAL_KEYS = Set("expr") def fromMap(src: Map[String, Any], path: List[String]): Option[ValidationExpr] = - ParseUtils.getOptValueExpression(src, "expr", path).map { case eqExpr => + ParseUtils.getOptValueExpression(src, "expr", path).map { case expr => ParseUtils.ensureLegalKeys(src, LEGAL_KEYS, path) - ValidationExpr(eqExpr) + ValidationExpr(expr) } } @@ -74,6 +92,7 @@ object ValidationSpec { ValidationEq.LEGAL_KEYS ++ ValidationRange.LEGAL_KEYS ++ ValidationAnyOf.LEGAL_KEYS ++ + ValidationInEnum.LEGAL_KEYS ++ ValidationExpr.LEGAL_KEYS def fromYaml(src: Any, path: List[String]): ValidationSpec = { @@ -106,10 +125,12 @@ object ValidationSpec { val opt3 = ValidationAnyOf.fromMap(src, path) if (opt3.nonEmpty) return opt3.get - val opt4 = ValidationExpr.fromMap(src, path) + val opt4 = ValidationInEnum.fromMap(src, path) if (opt4.nonEmpty) return opt4.get - + val opt5 = ValidationExpr.fromMap(src, path) + if (opt5.nonEmpty) + return opt5.get // No validation templates matched, check for any bogus keys ParseUtils.ensureLegalKeys(src, LEGAL_KEYS, path) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala index b3570f1b6..97693c473 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala @@ -363,7 +363,12 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesEosType => s"$io.ReadBytesFull()" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io.ReadBytesTerm($terminator, $include, $consume, $eosError)" + if (terminator.length == 1) { + val term = terminator.head & 0xff + s"$io.ReadBytesTerm($term, $include, $consume, $eosError)" + } else { + s"$io.ReadBytesTermMulti(${translator.doByteArrayLiteral(terminator)}, $include, $consume, $eosError)" + } case BitsType1(bitEndian) => s"$io.ReadBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1) != 0" case BitsType(width: Int, bitEndian) => @@ -388,13 +393,19 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName.BytesStripRight($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.BytesTerminate($expr1, $term, $include)" + case Some(term) => + if (term.length == 1) { + val t = term.head & 0xff + s"$kstreamName.BytesTerminate($expr1, $t, $include)" + } else { + s"$kstreamName.BytesTerminateMulti($expr1, ${translator.doByteArrayLiteral(term)}, $include)" + } case None => expr1 } expr2 @@ -555,23 +566,18 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - def idToStr(id: Identifier): String = - id match { - case SpecialIdentifier(name) => name - case NamedIdentifier(name) => Utils.lowerCamelCase(name) - case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" - case InstanceIdentifier(name) => Utils.lowerCamelCase(name) - case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" - } - - override def publicMemberName(id: Identifier): String = CSharpCompiler.publicMemberName(id) + def idToStr(id: Identifier): String = CSharpCompiler.idToStr(id) - override def privateMemberName(id: Identifier): String = { + override def publicMemberName(id: Identifier): String = id match { - case SpecialIdentifier(name) => s"m${Utils.lowerCamelCase(name)}" - case _ => s"_${idToStr(id)}" + case SpecialIdentifier(name) => s"M${Utils.upperCamelCase(name)}" + case NamedIdentifier(name) => Utils.upperCamelCase(name) + case NumberedIdentifier(idx) => s"${NumberedIdentifier.TEMPLATE.capitalize}_$idx" + case InstanceIdentifier(name) => Utils.upperCamelCase(name) + case RawIdentifier(innerId) => s"M_Raw${publicMemberName(innerId)}" } - } + + override def privateMemberName(id: Identifier): String = CSharpCompiler.privateMemberName(id) override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" @@ -586,14 +592,35 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: KSError): String = CSharpCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr] + ): Unit = + attrValidate(s"!(${translator.translate(checkExpr)})", err, errArgs) + + override def attrValidateInEnum( + attr: AttrLikeSpec, + et: EnumType, + valueExpr: Ast.expr, + err: ValidationNotInEnumError, + errArgs: List[Ast.expr] ): Unit = { + // TODO: the non-generic overload `Enum.IsDefined(Type, object)` used here + // is supposedly slow because it uses reflection (see + // https://stackoverflow.com/q/13615#comment48178324_4807469). [This SO + // answer](https://stackoverflow.com/a/55028274) suggests to use the generic + // overload `Enum.IsDefined(TEnum)` instead, claiming that it fixes + // the performance issues. But it's only available since .NET 5, so we would + // need a command-line switch to allow the user to choose whether they need + // compabitility with older versions or not. + importList.add("System") + attrValidate(s"!Enum.IsDefined(typeof(${kaitaiType2NativeType(et)}), ${translator.translate(valueExpr)})", err, errArgs) + } + + private def attrValidate(failCondExpr: String, err: KSError, errArgs: List[Ast.expr]): Unit = { val errArgsStr = errArgs.map(translator.translate).mkString(", ") - out.puts(s"if (!(${translator.translate(checkExpr)}))") + out.puts(s"if ($failCondExpr)") out.puts("{") out.inc out.puts(s"throw new ${ksErrorName(err)}($errArgsStr);") @@ -611,13 +638,19 @@ object CSharpCompiler extends LanguageCompilerStatic config: RuntimeConfig ): LanguageCompiler = new CSharpCompiler(tp, config) - def publicMemberName(id: Identifier): String = + def idToStr(id: Identifier): String = id match { - case SpecialIdentifier(name) => s"M${Utils.upperCamelCase(name)}" - case NamedIdentifier(name) => Utils.upperCamelCase(name) - case NumberedIdentifier(idx) => s"${NumberedIdentifier.TEMPLATE.capitalize}_$idx" - case InstanceIdentifier(name) => Utils.upperCamelCase(name) - case RawIdentifier(innerId) => s"M_Raw${publicMemberName(innerId)}" + case SpecialIdentifier(name) => name + case NamedIdentifier(name) => Utils.lowerCamelCase(name) + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case InstanceIdentifier(name) => Utils.lowerCamelCase(name) + case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" + } + + def privateMemberName(id: Identifier): String = + id match { + case SpecialIdentifier(name) => s"m${Utils.lowerCamelCase(name)}" + case _ => s"_${idToStr(id)}" } /** diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala index 1aad69df6..b0fdee657 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala @@ -674,7 +674,12 @@ class CppCompiler( case _: BytesEosType => s"$io->read_bytes_full()" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io->read_bytes_term($terminator, $include, $consume, $eosError)" + if (terminator.length == 1) { + val term = terminator.head & 0xff + s"$io->read_bytes_term($term, $include, $consume, $eosError)" + } else { + s"$io->read_bytes_term_multi(${translator.doByteArrayLiteral(terminator)}, $include, $consume, $eosError)" + } case BitsType1(bitEndian) => s"$io->read_bits_int_${bitEndian.toSuffix}(1)" case BitsType(width: Int, bitEndian) => @@ -717,13 +722,19 @@ class CppCompiler( } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName::bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName::bytes_terminate($expr1, $term, $include)" + case Some(term) => + if (term.length == 1) { + val t = term.head & 0xff + s"$kstreamName::bytes_terminate($expr1, $t, $include)" + } else { + s"$kstreamName::bytes_terminate_multi($expr1, ${translator.doByteArrayLiteral(term)}, $include)" + } case None => expr1 } expr2 @@ -882,6 +893,44 @@ class CppCompiler( outHdr.dec outHdr.puts("};") + + outHdr.puts(s"static bool _is_defined_$enumClass($enumClass v);") + importListHdr.addSystem("set") + val inClassRef = types2class(curClass) + val enumClassAbs = s"$inClassRef::$enumClass" + val valuesSetAbsRef = s"$inClassRef::_values_$enumClass" + ensureMode(PrivateAccess) + // NOTE: declaration and definition must be separate in this case, + // see https://stackoverflow.com/a/12856069 + outHdr.puts(s"static const std::set<$enumClass> _values_$enumClass;") + if (config.cppConfig.useListInitializers) { + outSrc.puts(s"const std::set<$enumClassAbs> $valuesSetAbsRef{") + outSrc.inc + enumColl.foreach { case (_, label) => + outSrc.puts(s"$inClassRef::${value2Const(enumName, label.name)},") + } + outSrc.dec + outSrc.puts("};") + } else { + outHdr.puts(s"static std::set<$enumClass> _build_values_$enumClass();") + + outSrc.puts(s"std::set<$enumClassAbs> $inClassRef::_build_values_$enumClass() {") + outSrc.inc + outSrc.puts(s"std::set<$enumClassAbs> _t;") + enumColl.foreach { case (_, label) => + outSrc.puts(s"_t.insert($inClassRef::${value2Const(enumName, label.name)});") + } + outSrc.puts("return _t;") + outSrc.dec + outSrc.puts("}") + outSrc.puts(s"const std::set<$enumClassAbs> $valuesSetAbsRef = $inClassRef::_build_values_$enumClass();") + } + ensureMode(PublicAccess) + outSrc.puts(s"bool $inClassRef::_is_defined_$enumClass($enumClassAbs v) {") + outSrc.inc + outSrc.puts(s"return $valuesSetAbsRef.find(v) != $valuesSetAbsRef.end();") + outSrc.dec + outSrc.puts("}") } override def classToString(toStringExpr: Ast.expr): Unit = { @@ -932,9 +981,9 @@ class CppCompiler( override def idToStr(id: Identifier): String = CppCompiler.idToStr(id) - override def publicMemberName(id: Identifier): String = CppCompiler.publicMemberName(id) + override def publicMemberName(id: Identifier): String = idToStr(id) - override def privateMemberName(id: Identifier): String = s"m_${idToStr(id)}" + override def privateMemberName(id: Identifier): String = CppCompiler.privateMemberName(id) override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" @@ -996,21 +1045,37 @@ class CppCompiler( case _: ValidationLessThanError => "validation_less_than_error" case _: ValidationGreaterThanError => "validation_greater_than_error" case _: ValidationNotAnyOfError => "validation_not_any_of_error" + case _: ValidationNotInEnumError => "validation_not_in_enum_error" case _: ValidationExprError => "validation_expr_error" } s"kaitai::$cppErrName<$cppType>" } override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr] + ): Unit = + attrValidate(s"!(${translator.translate(checkExpr)})", err, errArgs) + + override def attrValidateInEnum( + attr: AttrLikeSpec, + et: EnumType, + valueExpr: Ast.expr, + err: ValidationNotInEnumError, + errArgs: List[Ast.expr] ): Unit = { + val enumSpec = et.enumSpec.get + val inClassRef = types2class(enumSpec.name.dropRight(1)) + val enumNameStr = type2class(enumSpec.name.last) + attrValidate(s"!$inClassRef::_is_defined_$enumNameStr(${translator.translate(valueExpr)})", err, errArgs) + } + + private def attrValidate(failCondExpr: String, err: KSError, errArgs: List[Ast.expr]): Unit = { val errArgsStr = errArgs.map(translator.translate).mkString(", ") importListSrc.addKaitai("kaitai/exceptions.h") - outSrc.puts(s"if (!(${translator.translate(checkExpr)})) {") + outSrc.puts(s"if ($failCondExpr) {") outSrc.inc outSrc.puts(s"throw ${ksErrorName(err)}($errArgsStr);") outSrc.dec @@ -1039,7 +1104,7 @@ object CppCompiler extends LanguageCompilerStatic case IoStorageIdentifier(inner) => s"_io_${idToStr(inner)}" } - def publicMemberName(id: Identifier): String = idToStr(id) + def privateMemberName(id: Identifier): String = s"m_${idToStr(id)}" override def kstructName = "kaitai::kstruct" override def kstreamName = "kaitai::kstream" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala index b5fbe88eb..49e4343e1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala @@ -301,7 +301,7 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = { - out.puts(s"for i := 1;; i++ {") + out.puts(s"for i := 0;; i++ {") out.inc val eofVar = translator.allocateLocalVar() @@ -405,7 +405,12 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesEosType => s"$io.ReadBytesFull()" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io.ReadBytesTerm($terminator, $include, $consume, $eosError)" + if (terminator.length == 1) { + val term = terminator.head & 0xff + s"$io.ReadBytesTerm($term, $include, $consume, $eosError)" + } else { + s"$io.ReadBytesTermMulti(${translator.resToStr(translator.doByteArrayLiteral(terminator))}, $include, $consume, $eosError)" + } case BitsType1(bitEndian) => s"$io.ReadBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1)" case BitsType(width: Int, bitEndian) => @@ -526,6 +531,15 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec out.puts(")") + // Inspired by https://gist.github.com/bgadrian/cb8b9344d9c66571ef331a14eb7a2e80 + val mapEntriesStr = enumColl.map { case (id, _) => s"$id: {}" }.mkString(", ") + out.puts(s"var values_$fullEnumNameStr = map[$fullEnumNameStr]struct{}{$mapEntriesStr}") + out.puts(s"func (v $fullEnumNameStr) isDefined() bool {") + out.inc + out.puts(s"_, ok := values_$fullEnumNameStr[v]") + out.puts("return ok") + out.dec + out.puts("}") } override def classToString(toStringExpr: Ast.expr): Unit = { @@ -539,9 +553,16 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def idToStr(id: Identifier): String = GoCompiler.idToStr(id) - override def publicMemberName(id: Identifier): String = GoCompiler.publicMemberName(id) + override def publicMemberName(id: Identifier): String = + id match { + case IoIdentifier => "_IO" + case RootIdentifier => "_Root" + case ParentIdentifier => "_Parent" + case InstanceIdentifier(name) => Utils.upperCamelCase(name) + case _ => idToStr(id) + } - override def privateMemberName(id: Identifier): String = s"this.${idToStr(id)}" + override def privateMemberName(id: Identifier): String = GoCompiler.privateMemberName(id) override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" @@ -552,14 +573,25 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: KSError): String = GoCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr] - ): Unit = { + ): Unit = + attrValidate(s"!(${translator.translate(checkExpr)})", err, errArgs) + + override def attrValidateInEnum( + attr: AttrLikeSpec, + et: EnumType, + valueExpr: Ast.expr, + err: ValidationNotInEnumError, + errArgs: List[Ast.expr] + ): Unit = + attrValidate(s"!${translator.translate(valueExpr)}.isDefined()", err, errArgs) + + private def attrValidate(failCondExpr: String, err: KSError, errArgs: List[Ast.expr]): Unit = { val errArgsStr = errArgs.map(translator.translate).mkString(", ") - out.puts(s"if !(${translator.translate(checkExpr)}) {") + out.puts(s"if $failCondExpr {") out.inc val errInst = s"kaitai.New${err.name}($errArgsStr)" val noValueAndErr = translator.returnRes match { @@ -601,14 +633,7 @@ object GoCompiler extends LanguageCompilerStatic case IoStorageIdentifier(innerId) => s"_io_${idToStr(innerId)}" } - def publicMemberName(id: Identifier): String = - id match { - case IoIdentifier => "_IO" - case RootIdentifier => "_Root" - case ParentIdentifier => "_Parent" - case InstanceIdentifier(name) => Utils.upperCamelCase(name) - case _ => idToStr(id) - } + def privateMemberName(id: Identifier): String = s"this.${idToStr(id)}" /** * Determine Go data type corresponding to a KS data type. diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index f2a22fa34..ee19058d3 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -438,7 +438,12 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesEosType => s"$io.readBytesFull()" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io.readBytesTerm((byte) $terminator, $include, $consume, $eosError)" + if (terminator.length == 1) { + val term = terminator.head & 0xff + s"$io.readBytesTerm((byte) $term, $include, $consume, $eosError)" + } else { + s"$io.readBytesTermMulti(${translator.doByteArrayLiteral(terminator)}, $include, $consume, $eosError)" + } case BitsType1(bitEndian) => s"$io.readBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1) != 0" case BitsType(width: Int, bitEndian) => @@ -491,13 +496,19 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName.bytesStripRight($expr0, (byte) $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.bytesTerminate($expr1, (byte) $term, $include)" + case Some(term) => + if (term.length == 1) { + val t = term.head & 0xff + s"$kstreamName.bytesTerminate($expr1, (byte) $t, $include)" + } else { + s"$kstreamName.bytesTerminateMulti($expr1, ${translator.doByteArrayLiteral(term)}, $include)" + } case None => expr1 } expr2 @@ -743,9 +754,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def idToStr(id: Identifier): String = JavaCompiler.idToStr(id) - override def publicMemberName(id: Identifier) = JavaCompiler.publicMemberName(id) + override def publicMemberName(id: Identifier): String = idToStr(id) - override def privateMemberName(id: Identifier): String = s"this.${idToStr(id)}" + override def privateMemberName(id: Identifier): String = JavaCompiler.privateMemberName(id) override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" @@ -768,14 +779,29 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr] + ): Unit = + attrValidate(s"!(${translator.translate(checkExpr)})", err, errArgs) + + override def attrValidateInEnum( + attr: AttrLikeSpec, + et: EnumType, + valueExpr: Ast.expr, + err: ValidationNotInEnumError, + errArgs: List[Ast.expr] ): Unit = { + // NOTE: this condition works for now because we haven't implemented + // https://github.com/kaitai-io/kaitai_struct/issues/778 for Java yet, but + // it will need to be changed when we do. + attrValidate(s"${translator.translate(valueExpr)} == null", err, errArgs) + } + + private def attrValidate(failCondExpr: String, err: KSError, errArgs: List[Ast.expr]): Unit = { val errArgsStr = errArgs.map(translator.translate).mkString(", ") - out.puts(s"if (!(${translator.translate(checkExpr)})) {") + out.puts(s"if ($failCondExpr) {") out.inc out.puts(s"throw new ${ksErrorName(err)}($errArgsStr);") out.dec @@ -801,7 +827,7 @@ object JavaCompiler extends LanguageCompilerStatic case IoStorageIdentifier(innerId) => s"_io_${idToStr(innerId)}" } - def publicMemberName(id: Identifier) = idToStr(id) + def privateMemberName(id: Identifier): String = s"this.${idToStr(id)}" def kaitaiType2JavaType(attrType: DataType, importList: ImportList): String = kaitaiType2JavaTypePrim(attrType, importList) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala index a553bd8ac..a039b4560 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala @@ -367,7 +367,12 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesEosType => s"$io.readBytesFull()" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io.readBytesTerm($terminator, $include, $consume, $eosError)" + if (terminator.length == 1) { + val term = terminator.head & 0xff + s"$io.readBytesTerm($term, $include, $consume, $eosError)" + } else { + s"$io.readBytesTermMulti(${translator.doByteArrayLiteral(terminator)}, $include, $consume, $eosError)" + } case BitsType1(bitEndian) => s"$io.readBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1) != 0" case BitsType(width: Int, bitEndian) => @@ -392,13 +397,19 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName.bytesStripRight($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.bytesTerminate($expr1, $term, $include)" + case Some(term) => + if (term.length == 1) { + val t = term.head & 0xff + s"$kstreamName.bytesTerminate($expr1, $t, $include)" + } else { + s"$kstreamName.bytesTerminateMulti($expr1, ${translator.doByteArrayLiteral(term)}, $include)" + } case None => expr1 } expr2 @@ -557,27 +568,45 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) def idToStr(id: Identifier): String = JavaScriptCompiler.idToStr(id) - override def publicMemberName(id: Identifier) = JavaCompiler.publicMemberName(id) + override def publicMemberName(id: Identifier): String = + id match { + case InstanceIdentifier(name) => Utils.lowerCamelCase(name) + case _ => idToStr(id) + } - override def privateMemberName(id: Identifier): String = s"this.${idToStr(id)}" + override def privateMemberName(id: Identifier): String = JavaScriptCompiler.privateMemberName(id) override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" override def ksErrorName(err: KSError): String = JavaScriptCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr] + ): Unit = + attrValidate(s"!(${translator.translate(checkExpr)})", attr, err, errArgs) + + override def attrValidateInEnum( + attr: AttrLikeSpec, + et: EnumType, + valueExpr: Ast.expr, + err: ValidationNotInEnumError, + errArgs: List[Ast.expr] ): Unit = { + val enumSpec = et.enumSpec.get + val enumRef = types2class(enumSpec.name, enumSpec.isExternal(typeProvider.nowClass)) + attrValidate(s"!Object.prototype.hasOwnProperty.call($enumRef, ${translator.translate(valueExpr)})", attr, err, errArgs) + } + + private def attrValidate(failCondExpr: String, attr: AttrLikeSpec, err: KSError, errArgs: List[Ast.expr]): Unit = { val errArgsStr = errArgs.map(translator.translate).mkString(", ") - out.puts(s"if (!(${translator.translate(checkExpr)})) {") + out.puts(s"if ($failCondExpr) {") out.inc val errObj = s"new ${ksErrorName(err)}($errArgsStr)" - if (attrDebugNeeded(attrId)) { - val debugName = attrDebugName(attrId, NoRepeat, true) + if (attrDebugNeeded(attr.id)) { + val debugName = attrDebugName(attr.id, attr.cond.repeat, true) out.puts(s"var _err = $errObj;") out.puts(s"$debugName.validationError = _err;") out.puts("throw _err;") @@ -617,11 +646,7 @@ object JavaScriptCompiler extends LanguageCompilerStatic case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" } - def publicMemberName(id: Identifier): String = - id match { - case InstanceIdentifier(name) => Utils.lowerCamelCase(name) - case _ => idToStr(id) - } + def privateMemberName(id: Identifier): String = s"this.${idToStr(id)}" override def kstreamName: String = "KaitaiStream" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala index 41895dbdd..1f24674e9 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala @@ -1,7 +1,7 @@ package io.kaitai.struct.languages import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils, ExternalType} -import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError, ValidationNotEqualError} +import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError, ValidationNotEqualError, ValidationNotInEnumError} import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ @@ -293,10 +293,14 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def idToStr(id: Identifier): String = LuaCompiler.idToStr(id) - override def publicMemberName(id: Identifier): String = LuaCompiler.publicMemberName(id) + override def publicMemberName(id: Identifier): String = + id match { + case InstanceIdentifier(name) => name + case _ => idToStr(id) + } + + override def privateMemberName(id: Identifier): String = LuaCompiler.privateMemberName(id) - override def privateMemberName(id: Identifier): String = - s"self.${idToStr(id)}" override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" @@ -323,7 +327,12 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesEosType => s"$io:read_bytes_full()" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io:read_bytes_term($terminator, $include, $consume, $eosError)" + if (terminator.length == 1) { + val term = terminator.head & 0xff + s"$io:read_bytes_term($term, $include, $consume, $eosError)" + } else { + s"$io:read_bytes_term_multi(${translator.doByteArrayLiteral(terminator)}, $include, $consume, $eosError)" + } case BitsType1(bitEndian) => s"$io:read_bits_int_${bitEndian.toSuffix}(1) ~= 0" case BitsType(width: Int, bitEndian) => @@ -346,13 +355,19 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } s"${types2class(t.classSpec.get.name)}($addParams$io$addArgs)" } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean): String = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName.bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.bytes_terminate($expr1, $term, $include)" + case Some(term) => + if (term.length == 1) { + val t = term.head & 0xff + s"$kstreamName.bytes_terminate($expr1, $t, $include)" + } else { + s"$kstreamName.bytes_terminate_multi($expr1, ${translator.doByteArrayLiteral(term)}, $include)" + } case None => expr1 } expr2 @@ -406,14 +421,29 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: KSError): String = LuaCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr] + ): Unit = + attrValidate(s"not(${translator.translate(checkExpr)})", err, errArgs) + + override def attrValidateInEnum( + attr: AttrLikeSpec, + et: EnumType, + valueExpr: Ast.expr, + err: ValidationNotInEnumError, + errArgs: List[Ast.expr] ): Unit = { + // NOTE: this condition works for now because we haven't implemented + // https://github.com/kaitai-io/kaitai_struct/issues/778 for Lua yet, but + // it will need to be changed when we do. + attrValidate(s"${translator.translate(valueExpr)} == nil", err, errArgs) + } + + private def attrValidate(failCondExpr: String, err: KSError, errArgs: List[Ast.expr]): Unit = { val errArgsCode = errArgs.map(translator.translate) - out.puts(s"if not(${translator.translate(checkExpr)}) then") + out.puts(s"if $failCondExpr then") out.inc val msg = err match { case _: ValidationNotEqualError => { @@ -457,11 +487,8 @@ object LuaCompiler extends LanguageCompilerStatic case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" } - def publicMemberName(id: Identifier): String = - id match { - case InstanceIdentifier(name) => name - case _ => idToStr(id) - } + def privateMemberName(id: Identifier): String = + s"self.${idToStr(id)}" override def kstructName: String = "KaitaiStruct" override def kstreamName: String = "KaitaiStream" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala index b5ac4366b..21868c146 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala @@ -349,13 +349,19 @@ class NimCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def publicMemberName(id: Identifier): String = idToStr(id) // Members declared in io.kaitai.struct.languages.components.EveryReadIsExpression - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean): String = { val expr1 = padRight match { case Some(padByte) => s"$expr0.bytesStripRight($padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$expr1.bytesTerminate($term, $include)" + case Some(term) => + if (term.length == 1) { + val t = term.head & 0xff + s"$expr1.bytesTerminate($t, $include)" + } else { + s"$expr1.bytesTerminateMulti(${translator.doByteArrayLiteral(term)}, $include)" + } case None => expr1 } expr2 @@ -394,7 +400,12 @@ class NimCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesEosType => s"$io.readBytesFull()" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io.readBytesTerm($terminator, $include, $consume, $eosError)" + if (terminator.length == 1) { + val term = terminator.head & 0xff + s"$io.readBytesTerm($term, $include, $consume, $eosError)" + } else { + s"$io.readBytesTermMulti(${translator.doByteArrayLiteral(terminator)}, $include, $consume, $eosError)" + } case BitsType1(bitEndian) => s"$io.readBitsInt${camelCase(bitEndian.toSuffix, true)}(1) != 0" case BitsType(width: Int, bitEndian) => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala index 3a2527dee..c2c4d1476 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala @@ -1,7 +1,7 @@ package io.kaitai.struct.languages import io.kaitai.struct.datatype.DataType._ -import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian, KSError, UndecidedEndiannessError} +import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian, KSError, UndecidedEndiannessError, ValidationNotInEnumError} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format.{NoRepeat, RepeatEos, RepeatExpr, RepeatSpec, _} import io.kaitai.struct.languages.components._ @@ -341,7 +341,12 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesEosType => s"$io->readBytesFull()" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io->readBytesTerm($terminator, $include, $consume, $eosError)" + if (terminator.length == 1) { + val term = terminator.head & 0xff + s"$io->readBytesTerm($term, $include, $consume, $eosError)" + } else { + s"$io->readBytesTermMulti(${translator.doByteArrayLiteral(terminator)}, $include, $consume, $eosError)" + } case BitsType1(bitEndian) => s"$io->readBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1) != 0" case BitsType(width: Int, bitEndian) => @@ -366,13 +371,19 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName::bytesStripRight($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName::bytesTerminate($expr1, $term, $include)" + case Some(term) => + if (term.length == 1) { + val t = term.head & 0xff + s"$kstreamName::bytesTerminate($expr1, $t, $include)" + } else { + s"$kstreamName::bytesTerminateMulti($expr1, ${translator.doByteArrayLiteral(term)}, $include)" + } case None => expr1 } expr2 @@ -428,6 +439,15 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) universalDoc(label.doc) out.puts(s"const ${value2Const(label.name)} = ${translator.doIntLiteral(id)};") } + out.puts + val arrayEntriesStr = enumColl.map { case (id, _) => s"${translator.doIntLiteral(id)} => true" }.mkString(", ") + out.puts(s"private const _VALUES = [$arrayEntriesStr];") + out.puts + out.puts("public static function isDefined(int $v): bool {") + out.inc + out.puts("return isset(self::_VALUES[$v]);") + out.dec + out.puts("}") classFooter(name) } @@ -444,16 +464,9 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def idToStr(id: Identifier): String = PHPCompiler.idToStr(id) - override def publicMemberName(id: Identifier) = PHPCompiler.publicMemberName(id) + override def publicMemberName(id: Identifier) = idToStr(id) - override def privateMemberName(id: Identifier): String = { - id match { - case IoIdentifier => s"$$this->_io" - case RootIdentifier => s"$$this->_root" - case ParentIdentifier => s"$$this->_parent" - case _ => s"$$this->_m_${idToStr(id)}" - } - } + override def privateMemberName(id: Identifier): String = PHPCompiler.privateMemberName(id) override def localTemporaryName(id: Identifier): String = s"$$_t_${idToStr(id)}" @@ -491,14 +504,28 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: KSError): String = PHPCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr] + ): Unit = + attrValidate(s"!(${translator.translate(checkExpr)})", err, errArgs) + + override def attrValidateInEnum( + attr: AttrLikeSpec, + et: EnumType, + valueExpr: Ast.expr, + err: ValidationNotInEnumError, + errArgs: List[Ast.expr] ): Unit = { + val enumSpec = et.enumSpec.get + val enumRef = translator.types2classAbs(enumSpec.name) + attrValidate(s"!$enumRef::isDefined(${translator.translate(valueExpr)})", err, errArgs) + } + + private def attrValidate(failCondExpr: String, err: KSError, errArgs: List[Ast.expr]): Unit = { val errArgsStr = errArgs.map(translator.translate).mkString(", ") - out.puts(s"if (!(${translator.translate(checkExpr)})) {") + out.puts(s"if ($failCondExpr) {") out.inc out.puts(s"throw new ${ksErrorName(err)}($errArgsStr);") out.dec @@ -524,7 +551,13 @@ object PHPCompiler extends LanguageCompilerStatic case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" } - def publicMemberName(id: Identifier) = idToStr(id) + def privateMemberName(id: Identifier): String = + id match { + case IoIdentifier => s"$$this->_io" + case RootIdentifier => s"$$this->_root" + case ParentIdentifier => s"$$this->_parent" + case _ => s"$$this->_m_${idToStr(id)}" + } override def kstreamName: String = "\\Kaitai\\Struct\\Stream" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala index d8c0cb3fb..c8bfe5b0c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala @@ -297,7 +297,12 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesEosType => s"$io->read_bytes_full()" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io->read_bytes_term($terminator, ${boolLiteral(include)}, ${boolLiteral(consume)}, ${boolLiteral(eosError)})" + if (terminator.length == 1) { + val term = terminator.head & 0xff + s"$io->read_bytes_term($term, ${boolLiteral(include)}, ${boolLiteral(consume)}, ${boolLiteral(eosError)})" + } else { + s"$io->read_bytes_term_multi(${translator.doByteArrayLiteral(terminator)}, ${boolLiteral(include)}, ${boolLiteral(consume)}, ${boolLiteral(eosError)})" + } case BitsType1(bitEndian) => s"$io->read_bits_int_${bitEndian.toSuffix}(1)" case BitsType(width: Int, bitEndian) => @@ -320,13 +325,19 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName::bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName::bytes_terminate($expr1, $term, ${boolLiteral(include)})" + case Some(term) => + if (term.length == 1) { + val t = term.head & 0xff + s"$kstreamName::bytes_terminate($expr1, $t, ${boolLiteral(include)})" + } else { + s"$kstreamName::bytes_terminate_multi($expr1, ${translator.doByteArrayLiteral(term)}, ${boolLiteral(include)})" + } case None => expr1 } expr2 @@ -409,9 +420,9 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def idToStr(id: Identifier): String = PerlCompiler.idToStr(id) - override def publicMemberName(id: Identifier): String = PerlCompiler.publicMemberName(id) + override def publicMemberName(id: Identifier): String = idToStr(id) - override def privateMemberName(id: Identifier): String = s"$$self->{${idToStr(id)}}" + override def privateMemberName(id: Identifier): String = PerlCompiler.privateMemberName(id) override def localTemporaryName(id: Identifier): String = s"$$_t_${idToStr(id)}" @@ -438,7 +449,7 @@ object PerlCompiler extends LanguageCompilerStatic case RawIdentifier(inner) => s"_raw_${idToStr(inner)}" } - def publicMemberName(id: Identifier): String = idToStr(id) + def privateMemberName(id: Identifier): String = s"$$self->{${idToStr(id)}}" def packageName: String = "IO::KaitaiStruct" override def kstreamName: String = s"$packageName::Stream" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala index bd8946e86..37a75f323 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala @@ -357,7 +357,12 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesEosType => s"$io.read_bytes_full()" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io.read_bytes_term($terminator, ${bool2Py(include)}, ${bool2Py(consume)}, ${bool2Py(eosError)})" + if (terminator.length == 1) { + val term = terminator.head & 0xff + s"$io.read_bytes_term($term, ${bool2Py(include)}, ${bool2Py(consume)}, ${bool2Py(eosError)})" + } else { + s"$io.read_bytes_term_multi(${translator.doByteArrayLiteral(terminator)}, ${bool2Py(include)}, ${bool2Py(consume)}, ${bool2Py(eosError)})" + } case BitsType1(bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}(1) != 0" case BitsType(width: Int, bitEndian) => @@ -382,13 +387,19 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName.bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName.bytes_terminate($expr1, $term, ${bool2Py(include)})" + case Some(term) => + if (term.length == 1) { + val t = term.head & 0xff + s"$kstreamName.bytes_terminate($expr1, $t, ${bool2Py(include)})" + } else { + s"$kstreamName.bytes_terminate_multi($expr1, ${translator.doByteArrayLiteral(term)}, ${bool2Py(include)})" + } case None => expr1 } expr2 @@ -474,23 +485,41 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def idToStr(id: Identifier): String = PythonCompiler.idToStr(id) - override def publicMemberName(id: Identifier): String = PythonCompiler.publicMemberName(id) + override def publicMemberName(id: Identifier): String = + id match { + case InstanceIdentifier(name) => name + case _ => idToStr(id) + } - override def privateMemberName(id: Identifier): String = s"self.${idToStr(id)}" + override def privateMemberName(id: Identifier): String = PythonCompiler.privateMemberName(id) override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" override def ksErrorName(err: KSError): String = PythonCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr] + ): Unit = + attrValidate(s"not ${translator.translate(checkExpr)}", err, errArgs) + + override def attrValidateInEnum( + attr: AttrLikeSpec, + et: EnumType, + valueExpr: Ast.expr, + err: ValidationNotInEnumError, + errArgs: List[Ast.expr] ): Unit = { + val enumSpec = et.enumSpec.get + val enumRef = types2class(enumSpec.name, enumSpec.isExternal(typeProvider.nowClass)) + attrValidate(s"not isinstance(${translator.translate(valueExpr)}, $enumRef)", err, errArgs) + } + + private def attrValidate(failCondExpr: String, err: KSError, errArgs: List[Ast.expr]): Unit = { val errArgsStr = errArgs.map(translator.translate).mkString(", ") - out.puts(s"if not ${translator.translate(checkExpr)}:") + out.puts(s"if $failCondExpr:") out.inc out.puts(s"raise ${ksErrorName(err)}($errArgsStr)") out.dec @@ -515,11 +544,7 @@ object PythonCompiler extends LanguageCompilerStatic case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" } - def publicMemberName(id: Identifier): String = - id match { - case InstanceIdentifier(name) => name - case _ => idToStr(id) - } + def privateMemberName(id: Identifier): String = s"self.${idToStr(id)}" override def kstreamName: String = "KaitaiStream" override def kstructName: String = "KaitaiStruct" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala index c223c3347..ccda2646e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala @@ -365,7 +365,12 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: BytesEosType => s"$io.read_bytes_full" case BytesTerminatedType(terminator, include, consume, eosError, _) => - s"$io.read_bytes_term($terminator, $include, $consume, $eosError)" + if (terminator.length == 1) { + val term = terminator.head & 0xff + s"$io.read_bytes_term($term, $include, $consume, $eosError)" + } else { + s"$io.read_bytes_term_multi(${translator.doByteArrayLiteral(terminator)}, $include, $consume, $eosError)" + } case BitsType1(bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}(1) != 0" case BitsType(width: Int, bitEndian) => @@ -396,13 +401,19 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ioName } - override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean) = { val expr1 = padRight match { case Some(padByte) => s"$kstreamName::bytes_strip_right($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"$kstreamName::bytes_terminate($expr1, $term, $include)" + case Some(term) => + if (term.length == 1) { + val t = term.head & 0xff + s"$kstreamName::bytes_terminate($expr1, $t, $include)" + } else { + s"$kstreamName::bytes_terminate_multi($expr1, ${translator.doByteArrayLiteral(term)}, $include)" + } case None => expr1 } expr2 @@ -477,23 +488,36 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def idToStr(id: Identifier): String = RubyCompiler.idToStr(id) - override def publicMemberName(id: Identifier): String = RubyCompiler.publicMemberName(id) + override def publicMemberName(id: Identifier) = idToStr(id) - override def privateMemberName(id: Identifier): String = s"@${idToStr(id)}" + override def privateMemberName(id: Identifier): String = RubyCompiler.privateMemberName(id) override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" override def ksErrorName(err: KSError): String = RubyCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr] + ): Unit = + attrValidate(s"not ${translator.translate(checkExpr)}", err, errArgs) + + override def attrValidateInEnum( + attr: AttrLikeSpec, + et: EnumType, + valueExpr: Ast.expr, + err: ValidationNotInEnumError, + errArgs: List[Ast.expr] ): Unit = { + val inverseMap = translator.enumInverseMap(et) + attrValidate(s"not ${inverseMap}.key?(${translator.translate(valueExpr)})", err, errArgs) + } + + private def attrValidate(failCondExpr: String, err: KSError, errArgs: List[Ast.expr]): Unit = { val errArgsStr = errArgs.map(translator.translate).mkString(", ") - out.puts(s"raise ${ksErrorName(err)}.new($errArgsStr) if not ${translator.translate(checkExpr)}") + out.puts(s"raise ${ksErrorName(err)}.new($errArgsStr) if $failCondExpr") } def types2class(names: List[String]) = names.map(type2class).mkString("::") @@ -516,7 +540,7 @@ object RubyCompiler extends LanguageCompilerStatic case RawIdentifier(inner) => s"_raw_${idToStr(inner)}" } - def publicMemberName(id: Identifier) = idToStr(id) + def privateMemberName(id: Identifier): String = s"@${idToStr(id)}" override def kstreamName: String = "Kaitai::Struct::Stream" override def kstructName: String = "Kaitai::Struct::Struct" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 52f9ff8e4..d8b2e7867 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -776,8 +776,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.read_${t.apiCall(defEndian)}()?.into()" } case _: BytesEosType => s"$io.read_bytes_full()?.into()" - case b: BytesTerminatedType => - s"$io.read_bytes_term(${b.terminator}, ${b.include}, ${b.consume}, ${b.eosError})?.into()" + case BytesTerminatedType(terminator, include, consume, eosError, _) => + val term = terminator.head & 0xff + s"$io.read_bytes_term($term, $include, $consume, $eosError)?.into()" case b: BytesLimitType => s"$io.read_bytes(${expression(b.size)} as usize)?.into()" case BitsType1(bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}(1)? != 0" case BitsType(width: Int, bitEndian) => s"$io.read_bits_int_${bitEndian.toSuffix}($width)?" @@ -832,20 +833,19 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def bytesPadTermExpr(expr0: String, - padRight: Option[Int], - terminator: Option[Int], - include: Boolean): String = { + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean): String = { val ioId = privateMemberName(IoIdentifier) - val expr = padRight match { - case Some(p) => s"BytesReader::bytes_strip_right(&$expr0, $p).into()" + val expr1 = padRight match { + case Some(padByte) => s"BytesReader::bytes_strip_right(&$expr0, $padByte).into()" case None => expr0 } - - terminator match { - case Some(term) => s"BytesReader::bytes_terminate(&$expr, $term, $include).into()" - case None => expr + val expr2 = terminator match { + case Some(term) => + val t = term.head & 0xff + s"BytesReader::bytes_terminate(&$expr1, $t, $include).into()" + case None => expr1 } + expr2 } override def attrFixedContentsParse(attrName: Identifier, @@ -1195,8 +1195,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr] diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala index 931286b6e..de049dc14 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala @@ -25,12 +25,11 @@ trait CommonReads extends LanguageCompiler { normalIO } - val needsDebug = attrDebugNeeded(id) + val needsArrayDebug = attrDebugNeeded(id) && attr.cond.repeat != NoRepeat - if (needsDebug) { + if (needsArrayDebug) { attrDebugStart(id, attr.dataType, Some(io), NoRepeat) - if (attr.cond.repeat != NoRepeat) - attrDebugArrInit(id, attr.dataType) + attrDebugArrInit(id, attr.dataType) } defEndian match { @@ -45,7 +44,7 @@ trait CommonReads extends LanguageCompiler { attrParse0(id, attr, io, Some(fe)) } - if (needsDebug) + if (needsArrayDebug) attrDebugEnd(id, attr.dataType, io, NoRepeat) // More position management + set calculated flag after parsing for ParseInstanceSpecs @@ -54,15 +53,9 @@ trait CommonReads extends LanguageCompiler { // Restore position, if applicable if (pis.pos.isDefined) popPos(io) - - // Mark parse instance as calculated - instanceSetCalculated(pis.id) case _ => // no seeking required for sequence attributes } - // Run validations (still inside "if", if applicable) - attrValidateAll(attr) - attrParseIfFooter(attr.cond.ifExpr) } @@ -78,7 +71,17 @@ trait CommonReads extends LanguageCompiler { condRepeatUntilHeader(id, io, attr.dataType, untilExpr) case NoRepeat => } + + val needsDebug = attrDebugNeeded(id) + if (needsDebug) + attrDebugStart(id, attr.dataType, Some(io), attr.cond.repeat) + attrParse2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) + + if (needsDebug) + attrDebugEnd(id, attr.dataType, io, attr.cond.repeat) + + attrValidateAll(attr) attr.cond.repeat match { case RepeatEos => condRepeatEosFooter @@ -96,12 +99,20 @@ trait CommonReads extends LanguageCompiler { def attrDebugArrInit(attrId: Identifier, attrType: DataType): Unit = {} def attrDebugEnd(attrName: Identifier, attrType: DataType, io: String, repeat: RepeatSpec): Unit = {} - def attrDebugNeeded(attrId: Identifier): Boolean = false + def attrDebugNeeded(attrId: Identifier): Boolean = { + if (!config.readStoresPos) + return false + + attrId match { + case _: NamedIdentifier | _: NumberedIdentifier | _: InstanceIdentifier => true + case _ => false + } + } /** * Runs all validation procedures requested for an attribute. * @param attr attribute to run validations for */ def attrValidateAll(attr: AttrLikeSpec) = - attr.valid.foreach(valid => attrValidate(attr.id, attr, valid)) + attr.valid.foreach(valid => attrValidate(attr, valid)) } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala index 9db2e5c44..887239370 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala @@ -31,10 +31,6 @@ trait EveryReadIsExpression assignTypeOpt: Option[DataType] = None ): Unit = { val assignType = assignTypeOpt.getOrElse(dataType) - val needsDebug = attrDebugNeeded(id) && rep != NoRepeat - - if (needsDebug) - attrDebugStart(id, dataType, Some(io), rep) dataType match { case t: UserType => @@ -61,9 +57,6 @@ trait EveryReadIsExpression val expr = parseExpr(dataType, assignType, io, defEndian) handleAssignment(id, expr, rep, isRaw) } - - if (needsDebug) - attrDebugEnd(id, dataType, io, rep) } def attrBytesTypeParse( @@ -247,7 +240,7 @@ trait EveryReadIsExpression def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = ??? def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String - def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String + def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean): String def userTypeDebugRead(id: String, dataType: DataType, assignType: DataType): Unit = ??? def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { @@ -255,14 +248,4 @@ trait EveryReadIsExpression attrDebugStart(instName, dataType, None, NoRepeat) handleAssignmentSimple(instName, expression(value)) } - - override def attrDebugNeeded(attrId: Identifier): Boolean = { - if (!config.readStoresPos) - return false - - attrId match { - case _: NamedIdentifier | _: NumberedIdentifier | _: InstanceIdentifier => true - case _ => super.attrDebugNeeded(attrId) - } - } } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala index 600d0cb28..9772ab647 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala @@ -85,14 +85,20 @@ trait GoReads extends CommonReads with ObjectOrientedLanguage with GoSwitchOps { } } - def bytesPadTermExpr(id: ResultLocalVar, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { + def bytesPadTermExpr(id: ResultLocalVar, padRight: Option[Int], terminator: Option[Seq[Byte]], include: Boolean): String = { val expr0 = translator.resToStr(id) val expr1 = padRight match { case Some(padByte) => s"kaitai.BytesStripRight($expr0, $padByte)" case None => expr0 } val expr2 = terminator match { - case Some(term) => s"kaitai.BytesTerminate($expr1, $term, $include)" + case Some(term) => + if (term.length == 1) { + val t = term.head & 0xff + s"kaitai.BytesTerminate($expr1, $t, $include)" + } else { + s"kaitai.BytesTerminateMulti($expr1, ${translator.resToStr(translator.doByteArrayLiteral(term))}, $include)" + } case None => expr1 } expr2 diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala index 39290579c..2c346726b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala @@ -2,6 +2,7 @@ package io.kaitai.struct.languages.components import io.kaitai.struct.ClassTypeProvider import io.kaitai.struct.datatype._ +import io.kaitai.struct.datatype.DataType.EnumType import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ import io.kaitai.struct.translators.AbstractTranslator @@ -13,23 +14,24 @@ trait ValidateOps extends ExceptionNames { val translator: AbstractTranslator val typeProvider: ClassTypeProvider - def attrValidate(attrId: Identifier, attr: AttrLikeSpec, valid: ValidationSpec): Unit = { + def attrValidate(attr: AttrLikeSpec, valid: ValidationSpec): Unit = { + val itemValue = Identifier.itemExpr(attr.id, attr.cond.repeat) valid match { case ValidationEq(expected) => - attrValidateExprCompare(attrId, attr, Ast.cmpop.Eq, expected, ValidationNotEqualError(attr.dataTypeComposite)) + attrValidateExprCompare(attr, Ast.cmpop.Eq, expected, ValidationNotEqualError(attr.dataType)) case ValidationMin(min) => - attrValidateExprCompare(attrId, attr, Ast.cmpop.GtE, min, ValidationLessThanError(attr.dataTypeComposite)) + attrValidateExprCompare(attr, Ast.cmpop.GtE, min, ValidationLessThanError(attr.dataType)) case ValidationMax(max) => - attrValidateExprCompare(attrId, attr, Ast.cmpop.LtE, max, ValidationGreaterThanError(attr.dataTypeComposite)) + attrValidateExprCompare(attr, Ast.cmpop.LtE, max, ValidationGreaterThanError(attr.dataType)) case ValidationRange(min, max) => - attrValidateExprCompare(attrId, attr, Ast.cmpop.GtE, min, ValidationLessThanError(attr.dataTypeComposite)) - attrValidateExprCompare(attrId, attr, Ast.cmpop.LtE, max, ValidationGreaterThanError(attr.dataTypeComposite)) + attrValidateExprCompare(attr, Ast.cmpop.GtE, min, ValidationLessThanError(attr.dataType)) + attrValidateExprCompare(attr, Ast.cmpop.LtE, max, ValidationGreaterThanError(attr.dataType)) case ValidationAnyOf(values) => val bigOrExpr = Ast.expr.BoolOp( Ast.boolop.Or, values.map(expected => Ast.expr.Compare( - Ast.expr.InternalName(attrId), + itemValue, Ast.cmpop.Eq, expected ) @@ -37,31 +39,41 @@ trait ValidateOps extends ExceptionNames { ) attrValidateExpr( - attrId, - attr.dataTypeComposite, + attr, checkExpr = bigOrExpr, - err = ValidationNotAnyOfError(attr.dataTypeComposite), + err = ValidationNotAnyOfError(attr.dataType), errArgs = List( - Ast.expr.InternalName(attrId), + itemValue, + Ast.expr.InternalName(IoIdentifier), + Ast.expr.Str(attr.path.mkString("/", "/", "")) + ) + ) + case ValidationInEnum() => + attrValidateInEnum( + attr, + attr.dataType.asInstanceOf[EnumType], + itemValue, + ValidationNotInEnumError(attr.dataType), + List( + itemValue, Ast.expr.InternalName(IoIdentifier), Ast.expr.Str(attr.path.mkString("/", "/", "")) ) ) case ValidationExpr(expr) => blockScopeHeader - typeProvider._currentIteratorType = Some(attr.dataTypeComposite) + typeProvider._currentIteratorType = Some(attr.dataType) handleAssignmentTempVar( - attr.dataTypeComposite, + attr.dataType, translator.translate(Ast.expr.Name(Ast.identifier(Identifier.ITERATOR))), - translator.translate(Ast.expr.InternalName(attrId)) + translator.translate(itemValue) ) attrValidateExpr( - attrId, - attr.dataTypeComposite, + attr, expr, - ValidationExprError(attr.dataTypeComposite), + ValidationExprError(attr.dataType), List( - Ast.expr.InternalName(attrId), + itemValue, Ast.expr.InternalName(IoIdentifier), Ast.expr.Str(attr.path.mkString("/", "/", "")) ) @@ -70,26 +82,27 @@ trait ValidateOps extends ExceptionNames { } } - def attrValidateExprCompare(attrId: Identifier, attr: AttrLikeSpec, op: Ast.cmpop, expected: Ast.expr, err: KSError): Unit = { + def attrValidateExprCompare(attr: AttrLikeSpec, op: Ast.cmpop, expected: Ast.expr, err: KSError): Unit = { + val itemValue = Identifier.itemExpr(attr.id, attr.cond.repeat) attrValidateExpr( - attrId, - attr.dataTypeComposite, + attr, checkExpr = Ast.expr.Compare( - Ast.expr.InternalName(attrId), + itemValue, op, expected ), err = err, errArgs = List( expected, - Ast.expr.InternalName(attrId), + itemValue, Ast.expr.InternalName(IoIdentifier), Ast.expr.Str(attr.path.mkString("/", "/", "")) ) ) } - def attrValidateExpr(attrId: Identifier, attrType: DataType, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr]): Unit = {} + def attrValidateExpr(attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr]): Unit = {} + def attrValidateInEnum(attr: AttrLikeSpec, et: EnumType, valueExpr: Ast.expr, err: ValidationNotInEnumError, errArgs: List[Ast.expr]): Unit = {} def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit def blockScopeHeader: Unit def blockScopeFooter: Unit diff --git a/shared/src/main/scala/io/kaitai/struct/problems/CompilationProblem.scala b/shared/src/main/scala/io/kaitai/struct/problems/CompilationProblem.scala index 0977e068f..2779887fb 100644 --- a/shared/src/main/scala/io/kaitai/struct/problems/CompilationProblem.scala +++ b/shared/src/main/scala/io/kaitai/struct/problems/CompilationProblem.scala @@ -4,6 +4,7 @@ import io.kaitai.struct.{JSON, Jsonable, Utils, problems} import io.kaitai.struct.datatype.DataType import io.kaitai.struct.exprlang.Expressions import io.kaitai.struct.format.{ClassSpec, Identifier, KSVersion} +import fastparse.Parsed.Failure /** * Abstract top-level trait common to all problems which might be raised during @@ -112,10 +113,11 @@ object KSYParseError { None }).map((x) => s", did you mean '$x'?").getOrElse("") - val expected = f.extra.trace().msg.replaceAll("\n", "\\n") + val found = Failure.formatTrailing(f.extra.input, f.index) + val expected = f.extra.trace().label.replaceAll("\n", "\\n") withText( - s"parsing expression '${epe.src}' failed on $pos, " + + s"parsing expression '${epe.src}' failed on $found at position $pos, " + s"expected $expected$suggestion", path ) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala index a105a1b00..9fdfedc99 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala @@ -59,7 +59,7 @@ class CSharpTranslator(provider: TypeProvider, importList: ImportList) extends B } override def doInternalName(id: Identifier): String = - s"${CSharpCompiler.publicMemberName(id)}" + CSharpCompiler.privateMemberName(id) override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = s"${enumClass(enumSpec.name)}.${Utils.upperCamelCase(label)}" @@ -103,7 +103,7 @@ class CSharpTranslator(provider: TypeProvider, importList: ImportList) extends B override def intToStr(i: expr): String = s"${translate(i, METHOD_PRECEDENCE)}.ToString()" override def bytesToStr(bytesExpr: String, encoding: String): String = - s"""System.Text.Encoding.GetEncoding("$encoding").GetString($bytesExpr)""" + s"""System.Text.Encoding.GetEncoding(${doStringLiteral(encoding)}).GetString($bytesExpr)""" override def strLength(s: expr): String = s"${translate(s, METHOD_PRECEDENCE)}.Length" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala index bedc1f052..dccb1c0ab 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala @@ -60,6 +60,8 @@ class CppTranslator(provider: TypeProvider, importListSrc: CppImportList, import } } + def doRawStringLiteral(s: String): String = super.doStringLiteral(s) + /** * Handles string literal for C++ by wrapping a C `const char*`-style string * into a std::string constructor. Note that normally std::string @@ -138,7 +140,7 @@ class CppTranslator(provider: TypeProvider, importListSrc: CppImportList, import } override def doInternalName(id: Identifier): String = - s"${CppCompiler.publicMemberName(id)}()" + CppCompiler.privateMemberName(id) override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = { val isExternal = enumSpec.isExternal(provider.nowClass) @@ -188,23 +190,24 @@ class CppTranslator(provider: TypeProvider, importListSrc: CppImportList, import //s"std::to_string(${translate(i)})" s"${CppCompiler.kstreamName}::to_string(${translate(i)})" override def bytesToStr(bytesExpr: String, encoding: String): String = - s"""${CppCompiler.kstreamName}::bytes_to_str($bytesExpr, "$encoding")""" + s"""${CppCompiler.kstreamName}::bytes_to_str($bytesExpr, ${doRawStringLiteral(encoding)})""" override def bytesLength(b: Ast.expr): String = s"${translate(b, METHOD_PRECEDENCE)}.length()" override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = - s"${translate(container)}[${translate(idx)}]" + s"${translate(container, METHOD_PRECEDENCE)}.at(${translate(idx)})" override def bytesFirst(b: Ast.expr): String = { + val bStr = translate(b, METHOD_PRECEDENCE) config.cppConfig.stdStringFrontBack match { - case true => s"${translate(b)}.front()" - case false => s"${translate(b)}[0]" + case true => s"$bStr.front()" + case false => s"$bStr.at(0)" } } override def bytesLast(b: Ast.expr): String = { val bStr = translate(b, METHOD_PRECEDENCE) config.cppConfig.stdStringFrontBack match { case true => s"$bStr.back()" - case false => s"$bStr[$bStr.length() - 1]" + case false => s"$bStr.at($bStr.length() - 1)" } } override def bytesMin(b: Ast.expr): String = diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala index c72567444..28647aec8 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala @@ -223,12 +223,7 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo } def trInternalName(id: Identifier): TranslatorResult = - id match { - case SpecialIdentifier(name) => trLocalName(name) - case NamedIdentifier(name) => trLocalName(name) - case InstanceIdentifier(name) => trLocalName(name) - case _ => ResultString(s"this.${GoCompiler.publicMemberName(id)}") - } + ResultString(GoCompiler.privateMemberName(id)) def specialName(id: String): String = id match { case Identifier.ROOT | Identifier.PARENT | Identifier.IO => @@ -290,15 +285,41 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo val IMPORT_CHARMAP = "golang.org/x/text/encoding/charmap" val ENCODINGS = Map( - "IBM437" -> ("charmap.CodePage437", IMPORT_CHARMAP), - "ISO-8859-1" -> ("charmap.ISO8859_1", IMPORT_CHARMAP), - "ISO-8859-2" -> ("charmap.ISO8859_2", IMPORT_CHARMAP), - "ISO-8859-3" -> ("charmap.ISO8859_3", IMPORT_CHARMAP), - "ISO-8859-4" -> ("charmap.ISO8859_4", IMPORT_CHARMAP), - "SJIS" -> ("japanese.ShiftJIS", "golang.org/x/text/encoding/japanese"), - "BIG5" -> ("traditionalchinese.Big5", "golang.org/x/text/encoding/traditionalchinese"), + "UTF-16BE" -> ("unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)", "golang.org/x/text/encoding/unicode"), "UTF-16LE" -> ("unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM)", "golang.org/x/text/encoding/unicode"), - "UTF-16BE" -> ("unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)", "golang.org/x/text/encoding/unicode") + "UTF-32BE" -> ("utf32.UTF32(utf32.BigEndian, utf32.IgnoreBOM)", "golang.org/x/text/encoding/unicode/utf32"), + "UTF-32LE" -> ("utf32.UTF32(utf32.LittleEndian, utf32.IgnoreBOM)", "golang.org/x/text/encoding/unicode/utf32"), + "ISO-8859-1" -> ("charmap.ISO8859_1", IMPORT_CHARMAP), + "ISO-8859-2" -> ("charmap.ISO8859_2", IMPORT_CHARMAP), + "ISO-8859-3" -> ("charmap.ISO8859_3", IMPORT_CHARMAP), + "ISO-8859-4" -> ("charmap.ISO8859_4", IMPORT_CHARMAP), + "ISO-8859-5" -> ("charmap.ISO8859_5", IMPORT_CHARMAP), + "ISO-8859-6" -> ("charmap.ISO8859_6", IMPORT_CHARMAP), + "ISO-8859-7" -> ("charmap.ISO8859_7", IMPORT_CHARMAP), + "ISO-8859-8" -> ("charmap.ISO8859_8", IMPORT_CHARMAP), + "ISO-8859-9" -> ("charmap.ISO8859_9", IMPORT_CHARMAP), + "ISO-8859-10" -> ("charmap.ISO8859_10", IMPORT_CHARMAP), + // The same note as in https://github.com/kaitai-io/kaitai_struct_cpp_stl_runtime/blob/07ff9cf91e8bdf3515c0efdda0a879c0021b5edb/kaitai/kaitaistream.cpp#L918-L922 + // applies here + "ISO-8859-11" -> ("charmap.Windows874", IMPORT_CHARMAP), + "ISO-8859-13" -> ("charmap.ISO8859_13", IMPORT_CHARMAP), + "ISO-8859-14" -> ("charmap.ISO8859_14", IMPORT_CHARMAP), + "ISO-8859-15" -> ("charmap.ISO8859_15", IMPORT_CHARMAP), + "ISO-8859-16" -> ("charmap.ISO8859_16", IMPORT_CHARMAP), + "windows-1250" -> ("charmap.Windows1250", IMPORT_CHARMAP), + "windows-1251" -> ("charmap.Windows1251", IMPORT_CHARMAP), + "windows-1252" -> ("charmap.Windows1252", IMPORT_CHARMAP), + "windows-1253" -> ("charmap.Windows1253", IMPORT_CHARMAP), + "windows-1254" -> ("charmap.Windows1254", IMPORT_CHARMAP), + "windows-1255" -> ("charmap.Windows1255", IMPORT_CHARMAP), + "windows-1256" -> ("charmap.Windows1256", IMPORT_CHARMAP), + "windows-1257" -> ("charmap.Windows1257", IMPORT_CHARMAP), + "windows-1258" -> ("charmap.Windows1258", IMPORT_CHARMAP), + "IBM437" -> ("charmap.CodePage437", IMPORT_CHARMAP), + "IBM866" -> ("charmap.CodePage866", IMPORT_CHARMAP), + "Shift_JIS" -> ("japanese.ShiftJIS", "golang.org/x/text/encoding/japanese"), + "Big5" -> ("traditionalchinese.Big5", "golang.org/x/text/encoding/traditionalchinese"), + "EUC-KR" -> ("korean.EUCKR", "golang.org/x/text/encoding/korean"), ) override def bytesToStr(value: Ast.expr, encoding: String): TranslatorResult = @@ -394,7 +415,7 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo override def intToStr(value: Ast.expr): TranslatorResult = { importList.add("strconv") - ResultString(s"strconv.Itoa(int64(${translate(value)}))") + ResultString(s"strconv.FormatInt(int64(${translate(value)}), 10)") } override def floatToInt(value: Ast.expr) = diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala index b26f47bab..867282002 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala @@ -54,7 +54,7 @@ class JavaScriptTranslator(provider: TypeProvider, importList: ImportList) exten } override def doInternalName(id: Identifier): String = - s"this.${JavaScriptCompiler.publicMemberName(id)}" + JavaScriptCompiler.privateMemberName(id) override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = { val isExternal = enumSpec.isExternal(provider.nowClass) @@ -114,7 +114,7 @@ class JavaScriptTranslator(provider: TypeProvider, importList: ImportList) exten s"(${translate(i)}).toString()" override def bytesToStr(bytesExpr: String, encoding: String): String = - s"""${JavaScriptCompiler.kstreamName}.bytesToStr($bytesExpr, "$encoding")""" + s"""${JavaScriptCompiler.kstreamName}.bytesToStr($bytesExpr, ${doStringLiteral(encoding)})""" override def strLength(s: expr): String = s"${translate(s, METHOD_PRECEDENCE)}.length" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index d303d3b9b..18eb78dda 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -64,7 +64,7 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas } override def doInternalName(id: Identifier): String = - s"${JavaCompiler.publicMemberName(id)}()" + JavaCompiler.privateMemberName(id) override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = s"${enumClass(enumSpec.name)}.${Utils.upperUnderscoreCase(label)}" @@ -132,7 +132,7 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas s"StandardCharsets.${charsetConst}" case None => importList.add("java.nio.charset.Charset") - s"""Charset.forName("$encoding")""" + s"""Charset.forName(${doStringLiteral(encoding)})""" } s"new String($bytesExpr, $charsetExpr)" } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala index 764bb1dae..5d45b0a3a 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala @@ -80,7 +80,7 @@ class LuaTranslator(provider: TypeProvider, importList: ImportList) extends Base override def doName(s: String): String = s override def doInternalName(id: Identifier): String = - s"self.${LuaCompiler.publicMemberName(id)}" + LuaCompiler.privateMemberName(id) override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = s"${LuaCompiler.types2class(enumSpec.name)}.$label" @@ -108,7 +108,7 @@ class LuaTranslator(provider: TypeProvider, importList: ImportList) extends Base override def bytesToStr(bytesExpr: String, encoding: String): String = { importList.add("local str_decode = require(\"string_decode\")") - s"""str_decode.decode($bytesExpr, "$encoding")""" + s"""str_decode.decode($bytesExpr, ${doStringLiteral(encoding)})""" } override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = { s"string.byte(${translate(container)}, ${translate(idx)} + 1)" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala index 7feaee04c..f82e00eb7 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala @@ -12,7 +12,7 @@ import io.kaitai.struct.languages.NimCompiler.{ksToNim, namespaced, camelCase} class NimTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { // Members declared in io.kaitai.struct.translators.BaseTranslator override def bytesToStr(bytesExpr: String, encoding: String): String = { - s"""encode($bytesExpr, "$encoding")""" + s"""encode($bytesExpr, ${doStringLiteral(encoding)})""" } override def doEnumById(enumSpec: EnumSpec, id: String): String = s"${namespaced(enumSpec.name)}($id)" // override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = s"${namespaced(enumSpec.name)}($label)" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala index e3b33e0bd..dafc23b21 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala @@ -68,7 +68,7 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT override def doName(s: String) = s"${Utils.lowerCamelCase(s)}()" override def doInternalName(id: Identifier): String = - s"$$this->${PHPCompiler.publicMemberName(id)}()" + PHPCompiler.privateMemberName(id) override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = { val enumClass = types2classAbs(enumSpec.name) @@ -103,7 +103,7 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT s"strval(${translate(i)})" override def bytesToStr(bytesExpr: String, encoding: String): String = - s"""${PHPCompiler.kstreamName}::bytesToStr($bytesExpr, "$encoding")""" + s"""${PHPCompiler.kstreamName}::bytesToStr($bytesExpr, ${doStringLiteral(encoding)})""" override def bytesLength(b: Ast.expr): String = s"strlen(${translate(b)})" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala index c0ef2c1a1..9c57890e5 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala @@ -74,7 +74,7 @@ class PerlTranslator(provider: TypeProvider, importList: ImportList) extends Bas } override def doInternalName(id: Identifier): String = - s"$$self->${PerlCompiler.publicMemberName(id)}()" + PerlCompiler.privateMemberName(id) override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = { val isExternal = enumSpec.isExternal(provider.nowClass) @@ -142,7 +142,7 @@ class PerlTranslator(provider: TypeProvider, importList: ImportList) extends Bas s"sprintf('%d', ${translate(i)})" override def bytesToStr(bytesExpr: String, encoding: String): String = { importList.add("Encode") - s"""Encode::decode("$encoding", $bytesExpr)""" + s"""Encode::decode(${doStringLiteral(encoding)}, $bytesExpr)""" } override def bytesLength(b: Ast.expr): String = strLength(b) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala index 8bdd599c7..8c91c010f 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala @@ -54,7 +54,7 @@ class PythonTranslator(provider: TypeProvider, importList: ImportList, config: R override def doName(s: String) = s override def doInternalName(id: Identifier): String = - s"self.${PythonCompiler.publicMemberName(id)}" + PythonCompiler.privateMemberName(id) override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = { val isExternal = enumSpec.isExternal(provider.nowClass) @@ -99,7 +99,7 @@ class PythonTranslator(provider: TypeProvider, importList: ImportList, config: R override def intToStr(i: Ast.expr): String = s"str(${translate(i)})" override def bytesToStr(bytesExpr: String, encoding: String): String = - s"""($bytesExpr).decode("$encoding")""" + s"""($bytesExpr).decode(${doStringLiteral(encoding)})""" override def bytesLength(value: Ast.expr): String = s"len(${translate(value)})" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala index 40b012c96..8018430dc 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala @@ -38,7 +38,7 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) } override def doInternalName(id: Identifier): String = - RubyCompiler.publicMemberName(id) + RubyCompiler.privateMemberName(id) override def doEnumByLabel(enumSpec: EnumSpec, label: String): String = RubyCompiler.enumValue(enumSpec.name.last, label) @@ -90,7 +90,7 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) override def bytesToStr(bytesExpr: String, encoding: String): String = { // We can skip "encode to UTF8" if we're 100% sure that the string we're handling is already // in UTF8. - s"""($bytesExpr).force_encoding("$encoding")""" + (if (encoding != "UTF-8") { + s"""($bytesExpr).force_encoding(${doStringLiteral(encoding)})""" + (if (encoding != "UTF-8") { ".encode('UTF-8')" } else { "" @@ -121,7 +121,7 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) override def strReverse(s: Ast.expr): String = s"${translate(s, METHOD_PRECEDENCE)}.reverse" override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = - s"${translate(s, METHOD_PRECEDENCE)}[${translate(from)}..${genericBinOp(to, Ast.operator.Sub, Ast.expr.IntNum(1), 0)}]" + s"${translate(s, METHOD_PRECEDENCE)}[${translate(from)}...${translate(to)}]" override def arrayFirst(a: Ast.expr): String = s"${translate(a, METHOD_PRECEDENCE)}.first"