From 29a55111789a0cc3a4cc05009c82ae98d32129a2 Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 15 Mar 2024 20:15:57 +0500 Subject: [PATCH 1/9] Rename pass MarkupClassNames to CalculateFullNamesAndSetSurroundingType The new name gives more information about what the pass does --- shared/src/main/scala/io/kaitai/struct/Main.scala | 2 +- ...cala => CalculateFullNamesAndSetSurroundingType.scala} | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) rename shared/src/main/scala/io/kaitai/struct/precompile/{MarkupClassNames.scala => CalculateFullNamesAndSetSurroundingType.scala} (67%) diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index 7ac34ef6e..7a03ba02b 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -39,7 +39,7 @@ object Main { * @return a list of compilation problems encountered during precompilation steps */ def precompile(specs: ClassSpecs, conf: RuntimeConfig): Iterable[CompilationProblem] = { - new MarkupClassNames(specs).run() + new CalculateFullNamesAndSetSurroundingType(specs).run() val resolveTypeProblems = specs.flatMap { case (_, topClass) => val config = updateConfigFromMeta(conf, topClass.meta) new ResolveTypes(specs, topClass, config.opaqueTypes).run() diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/MarkupClassNames.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala similarity index 67% rename from shared/src/main/scala/io/kaitai/struct/precompile/MarkupClassNames.scala rename to shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala index 083aec982..798ed957e 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/MarkupClassNames.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala @@ -3,13 +3,13 @@ package io.kaitai.struct.precompile import io.kaitai.struct.format.{ClassSpec, ClassSpecs} import io.kaitai.struct.problems.CompilationProblem -class MarkupClassNames(classSpecs: ClassSpecs) extends PrecompileStep { +class CalculateFullNamesAndSetSurroundingType(classSpecs: ClassSpecs) extends PrecompileStep { override def run(): Iterable[CompilationProblem] = { - classSpecs.foreach { case (_, curClass) => markupClassNames(curClass) } + classSpecs.foreach { case (_, curClass) => calculate(curClass) } None } - def markupClassNames(curClass: ClassSpec): Unit = { + private def calculate(curClass: ClassSpec): Unit = { curClass.enums.foreach { case (enumName, enumSpec) => enumSpec.name = curClass.name ::: List(enumName) } @@ -17,7 +17,7 @@ class MarkupClassNames(classSpecs: ClassSpecs) extends PrecompileStep { curClass.types.foreach { case (nestedName: String, nestedClass) => nestedClass.name = curClass.name ::: List(nestedName) nestedClass.upClass = Some(curClass) - markupClassNames(nestedClass) + calculate(nestedClass) } } } From 9c09f79bcddc3fd32dd016ff4e6204aa5b7d33e5 Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 15 Mar 2024 20:25:38 +0500 Subject: [PATCH 2/9] Document CalculateFullNamesAndSetSurroundingType pass and improve documentation of related ClassSpec and EnumSpec properties --- .../src/main/scala/io/kaitai/struct/format/ClassSpec.scala | 4 ++++ .../src/main/scala/io/kaitai/struct/format/EnumSpec.scala | 6 ++++++ .../CalculateFullNamesAndSetSurroundingType.scala | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala index 6a3cb0135..010516e00 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala @@ -58,6 +58,8 @@ case class ClassSpec( * Full absolute name of the class (including all names of classes that * it's nested into, as a namespace). Derived either from `meta`/`id` * (for top-level classes), or from keys in `types` (for nested classes). + * + * This name is calculated by the `CalculateFullNamesAndSetSurroundingType` pass. */ var name = List[String]() @@ -76,6 +78,8 @@ case class ClassSpec( /** * The class specification that this class is nested into, if it exists. * For top-level classes, it's None. + * + * This class is calculated by the `CalculateFullNamesAndSetSurroundingType` pass. */ var upClass: Option[ClassSpec] = None diff --git a/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala index 6fcaf791e..5c41442f5 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala @@ -6,6 +6,12 @@ import scala.collection.immutable.SortedMap import scala.collection.mutable case class EnumSpec(path: List[String], map: SortedMap[Long, EnumValueSpec]) extends YAMLPath { + /** + * Absolute name of the enum (includes the names of all classes inside which + * it is defined). Derived either from keys in `enums`. + * + * This name is calculated by the `CalculateFullNamesAndSetSurroundingType` pass. + */ var name = List[String]() /** diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala index 798ed957e..087a82f44 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala @@ -3,6 +3,10 @@ package io.kaitai.struct.precompile import io.kaitai.struct.format.{ClassSpec, ClassSpecs} import io.kaitai.struct.problems.CompilationProblem +/** + * Assigns a full name to the `.name` property to each KS type and enum. + * Also for each [[ClassSpec]] assigns [[ClassSpec.upClass]] to a surrounding type. + */ class CalculateFullNamesAndSetSurroundingType(classSpecs: ClassSpecs) extends PrecompileStep { override def run(): Iterable[CompilationProblem] = { classSpecs.foreach { case (_, curClass) => calculate(curClass) } From 9ceff30c40e7998fe03aa118bff37de3507c4204 Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 15 Mar 2024 23:24:42 +0500 Subject: [PATCH 3/9] Rename pass SpecsValueTypeDerive to DeriveValueInstanceTypes The new name gives more information about what the pass does --- shared/src/main/scala/io/kaitai/struct/Main.scala | 2 +- ...csValueTypeDerive.scala => DeriveValueInstanceTypes.scala} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename shared/src/main/scala/io/kaitai/struct/precompile/{SpecsValueTypeDerive.scala => DeriveValueInstanceTypes.scala} (82%) diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index 7a03ba02b..29978ff13 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -52,7 +52,7 @@ object Main { } new ParentTypes(specs).run() - new SpecsValueTypeDerive(specs).run() + new DeriveValueInstanceTypes(specs).run() new CalculateSeqSizes(specs).run() val typeValidatorProblems = new TypeValidator(specs).run() diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala similarity index 82% rename from shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala rename to shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala index 74686d1a1..d88f076e9 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala @@ -3,13 +3,13 @@ package io.kaitai.struct.precompile import io.kaitai.struct.Log import io.kaitai.struct.format.ClassSpecs -class SpecsValueTypeDerive(specs: ClassSpecs) { +class DeriveValueInstanceTypes(specs: ClassSpecs) { def run(): Unit = { var iterNum = 1 var hasChanged = false do { hasChanged = false - Log.typeProcValue.info(() => s"### SpecsValueTypeDerive: iteration #$iterNum") + Log.typeProcValue.info(() => s"### DeriveValueInstanceTypes: iteration #$iterNum") specs.foreach { case (specName, spec) => Log.typeProcValue.info(() => s"#### $specName") val thisChanged = new ValueTypesDeriver(specs, spec).run() From e544f70886a5682aa481fd376e183a64dc5fed62 Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 15 Mar 2024 23:31:03 +0500 Subject: [PATCH 4/9] Document DeriveValueInstanceTypes pass and add documentation to related ValueInstanceSpec property --- .../main/scala/io/kaitai/struct/format/InstanceSpec.scala | 4 ++++ .../kaitai/struct/precompile/DeriveValueInstanceTypes.scala | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala index 4a66307ed..db9b8169d 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala @@ -13,6 +13,10 @@ case class ValueInstanceSpec( path: List[String], value: Ast.expr, ifExpr: Option[Ast.expr] = None, + /** + * Type of the `value`. Calculated by the [[DeriveValueInstanceTypes]] pass. + * `None` means "un-calculated yet". + */ var dataTypeOpt: Option[DataType] = None, val _doc: DocSpec = DocSpec.EMPTY, ) extends InstanceSpec(_doc) { diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala index d88f076e9..0d7992851 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala @@ -3,6 +3,12 @@ package io.kaitai.struct.precompile import io.kaitai.struct.Log import io.kaitai.struct.format.ClassSpecs +/** + * Assign types to value instances by deriving them from their expressions. + * + * Calculates value of the [[ValueInstanceSpec.dataTypeOpt]] field, which is + * a type of value instance. + */ class DeriveValueInstanceTypes(specs: ClassSpecs) { def run(): Unit = { var iterNum = 1 From dc8f4547b4a8eee2e09f9328b78a673020dfc147 Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Oct 2024 16:54:21 +0500 Subject: [PATCH 5/9] hasUndecided variable do not used, remove it --- .../scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala index e66c5c016..4dbea47ff 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala @@ -15,7 +15,6 @@ class ValueTypesDeriver(specs: ClassSpecs, topClass: ClassSpec) { def deriveValueType(curClass: ClassSpec): Boolean = { Log.typeProcValue.info(() => s"deriveValueType(${curClass.nameAsStr})") var hasChanged = false - var hasUndecided = false provider.nowClass = curClass curClass.instances.foreach { @@ -32,7 +31,6 @@ class ValueTypesDeriver(specs: ClassSpecs, topClass: ClassSpec) { } catch { case tue: TypeUndecidedError => Log.typeProcValue.info(() => s"${instName.name} type undecided: ${tue.getMessage}") - hasUndecided = true // just ignore, we're not there yet, probably we'll get it on next iteration case err: ExpressionError => throw ErrorInInput(err, vi.path ++ List("value")).toException From d13db325de52a4d62349038f9ab5b76388cf88c4 Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Oct 2024 16:46:22 +0500 Subject: [PATCH 6/9] Merge ValueTypesDetector into DeriveValueInstanceTypes because it is used only by that pass --- .../precompile/DeriveValueInstanceTypes.scala | 56 ++++++++++++++++++- .../struct/precompile/ValueTypesDeriver.scala | 54 ------------------ 2 files changed, 53 insertions(+), 57 deletions(-) delete mode 100644 shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala index 0d7992851..e77d21cf0 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala @@ -1,7 +1,9 @@ package io.kaitai.struct.precompile -import io.kaitai.struct.Log -import io.kaitai.struct.format.ClassSpecs +import io.kaitai.struct.{ClassTypeProvider, Log} +import io.kaitai.struct.format.{ClassSpec, ClassSpecs, ValueInstanceSpec} +import io.kaitai.struct.problems.ErrorInInput +import io.kaitai.struct.translators.TypeDetector /** * Assign types to value instances by deriving them from their expressions. @@ -18,7 +20,11 @@ class DeriveValueInstanceTypes(specs: ClassSpecs) { Log.typeProcValue.info(() => s"### DeriveValueInstanceTypes: iteration #$iterNum") specs.foreach { case (specName, spec) => Log.typeProcValue.info(() => s"#### $specName") - val thisChanged = new ValueTypesDeriver(specs, spec).run() + + val provider = new ClassTypeProvider(specs, spec) + val detector = new TypeDetector(provider) + + val thisChanged = deriveValueType(spec, provider, detector) Log.typeProcValue.info(() => ".... => " + (if (thisChanged) "changed" else "no changes")) hasChanged |= thisChanged } @@ -26,4 +32,48 @@ class DeriveValueInstanceTypes(specs: ClassSpecs) { } while (hasChanged) Log.typeProcValue.info(() => s"## value type deriving finished in ${iterNum - 1} iteration(s)") } + + private def deriveValueType( + curClass: ClassSpec, + provider: ClassTypeProvider, + detector: TypeDetector + ): Boolean = { + Log.typeProcValue.info(() => s"deriveValueType(${curClass.nameAsStr})") + var hasChanged = false + + provider.nowClass = curClass; + curClass.instances.foreach { + case (instName, inst) => + inst match { + case vi: ValueInstanceSpec => + vi.dataTypeOpt match { + case None => + try { + val viType = detector.detectType(vi.value) + vi.dataTypeOpt = Some(viType) + Log.typeProcValue.info(() => s"${instName.name} derived type: $viType") + hasChanged = true + } catch { + case tue: TypeUndecidedError => + Log.typeProcValue.info(() => s"${instName.name} type undecided: ${tue.getMessage}") + // just ignore, we're not there yet, probably we'll get it on next iteration + case err: ExpressionError => + throw ErrorInInput(err, vi.path ++ List("value")).toException + } + case Some(_) => + // already derived, do nothing + } + case _ => + // do nothing + } + } + + // Continue with all nested types + curClass.types.foreach { + case (_, classSpec) => + hasChanged ||= deriveValueType(classSpec, provider, detector) + } + + hasChanged + } } diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala deleted file mode 100644 index 4dbea47ff..000000000 --- a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala +++ /dev/null @@ -1,54 +0,0 @@ -package io.kaitai.struct.precompile - -import io.kaitai.struct.format.{ClassSpec, ClassSpecs, ValueInstanceSpec} -import io.kaitai.struct.problems.ErrorInInput -import io.kaitai.struct.translators.TypeDetector -import io.kaitai.struct.{ClassTypeProvider, Log} - -class ValueTypesDeriver(specs: ClassSpecs, topClass: ClassSpec) { - val provider = new ClassTypeProvider(specs, topClass) - val detector = new TypeDetector(provider) - - def run(): Boolean = - deriveValueType(topClass) - - def deriveValueType(curClass: ClassSpec): Boolean = { - Log.typeProcValue.info(() => s"deriveValueType(${curClass.nameAsStr})") - var hasChanged = false - - provider.nowClass = curClass - curClass.instances.foreach { - case (instName, inst) => - inst match { - case vi: ValueInstanceSpec => - vi.dataTypeOpt match { - case None => - try { - val viType = detector.detectType(vi.value) - vi.dataTypeOpt = Some(viType) - Log.typeProcValue.info(() => s"${instName.name} derived type: $viType") - hasChanged = true - } catch { - case tue: TypeUndecidedError => - Log.typeProcValue.info(() => s"${instName.name} type undecided: ${tue.getMessage}") - // just ignore, we're not there yet, probably we'll get it on next iteration - case err: ExpressionError => - throw ErrorInInput(err, vi.path ++ List("value")).toException - } - case Some(_) => - // already derived, do nothing - } - case _ => - // do nothing - } - } - - // Continue with all nested types - curClass.types.foreach { - case (_, classSpec) => - hasChanged ||= deriveValueType(classSpec) - } - - hasChanged - } -} From 9c592cca7e18194537c925c53dae5deac57beeee Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Oct 2024 17:41:19 +0500 Subject: [PATCH 7/9] Convert all passes to the PrecompileStep passes and run them all uniformly --- .../main/scala/io/kaitai/struct/Main.scala | 19 ++++++++++--------- .../struct/precompile/CalculateSeqSizes.scala | 6 ++++-- .../precompile/DeriveValueInstanceTypes.scala | 7 ++++--- .../struct/precompile/ParentTypes.scala | 6 ++++-- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index 29978ff13..83bdaa854 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -51,16 +51,17 @@ object Main { return resolveTypeProblems } - new ParentTypes(specs).run() - new DeriveValueInstanceTypes(specs).run() - new CalculateSeqSizes(specs).run() - val typeValidatorProblems = new TypeValidator(specs).run() + var passes = Seq( + new ParentTypes(specs), + new DeriveValueInstanceTypes(specs), + new CalculateSeqSizes(specs), + new TypeValidator(specs), + // Warnings + new StyleCheckIds(specs), + new CanonicalizeEncodingNames(specs), + ) - // Warnings - val styleWarnings = new StyleCheckIds(specs).run() - val encodingProblems = new CanonicalizeEncodingNames(specs).run() - - resolveTypeProblems ++ typeValidatorProblems ++ styleWarnings ++ encodingProblems + resolveTypeProblems ++ passes.flatMap(_.run()) } /** diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala index 4d28c263b..688259fa0 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala @@ -5,10 +5,12 @@ import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ +import io.kaitai.struct.problems.CompilationProblem -class CalculateSeqSizes(specs: ClassSpecs) { - def run(): Unit = { +class CalculateSeqSizes(specs: ClassSpecs) extends PrecompileStep { + override def run(): Iterable[CompilationProblem] = { specs.forEachRec(CalculateSeqSizes.getSeqSize) + None } } diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala index e77d21cf0..a8133c503 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala @@ -2,7 +2,7 @@ package io.kaitai.struct.precompile import io.kaitai.struct.{ClassTypeProvider, Log} import io.kaitai.struct.format.{ClassSpec, ClassSpecs, ValueInstanceSpec} -import io.kaitai.struct.problems.ErrorInInput +import io.kaitai.struct.problems.{CompilationProblem, ErrorInInput} import io.kaitai.struct.translators.TypeDetector /** @@ -11,8 +11,8 @@ import io.kaitai.struct.translators.TypeDetector * Calculates value of the [[ValueInstanceSpec.dataTypeOpt]] field, which is * a type of value instance. */ -class DeriveValueInstanceTypes(specs: ClassSpecs) { - def run(): Unit = { +class DeriveValueInstanceTypes(specs: ClassSpecs) extends PrecompileStep { + override def run(): Iterable[CompilationProblem] = { var iterNum = 1 var hasChanged = false do { @@ -31,6 +31,7 @@ class DeriveValueInstanceTypes(specs: ClassSpecs) { iterNum += 1 } while (hasChanged) Log.typeProcValue.info(() => s"## value type deriving finished in ${iterNum - 1} iteration(s)") + None } private def deriveValueType( diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala index 2929f4e5f..4a1389cd5 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala @@ -4,18 +4,20 @@ import io.kaitai.struct.{ClassTypeProvider, Log} import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType.{ArrayTypeInStream, SwitchType, UserType} import io.kaitai.struct.format._ +import io.kaitai.struct.problems.CompilationProblem import io.kaitai.struct.translators.TypeDetector /** * Precompile step that calculates actual parent types of KSY-defined types * (the type of the `_parent` built-in property). */ -class ParentTypes(classSpecs: ClassSpecs) { - def run(): Unit = { +class ParentTypes(classSpecs: ClassSpecs) extends PrecompileStep { + override def run(): Iterable[CompilationProblem] = { classSpecs.foreach { case (_, curClass) => markup(curClass) } classSpecs.forEachTopLevel((_, spec) => { spec.parentClass = GenericStructClassSpec }) + None } def markup(curClass: ClassSpec): Unit = { From 75e8e5726a588d6818e5938800bd980ba907581a Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Oct 2024 18:33:43 +0500 Subject: [PATCH 8/9] Allow to run warning passes on model with erroneous value instances Currently the pass DeriveValueInstanceTypes throws error if type of instance cannot be calculated, but this will change in next commit. That means that accessing `dataType` of the ValueInstanceSpec will throw an error. If such error is thrown we just cannot analyze that instance and can just ignore it (review this commit in whitespace changes ignored mode) --- .../CanonicalizeEncodingNames.scala | 27 ++++++++------ .../struct/precompile/StyleCheckIds.scala | 35 +++++++++++-------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala index 80ff95fe2..3635cbb3f 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala @@ -32,16 +32,23 @@ object CanonicalizeEncodingNames { } def canonicalizeMember(member: MemberSpec): Iterable[CompilationProblem] = { - (member.dataType match { - case strType: StrFromBytesType => - val (newEncoding, problem1) = canonicalizeName(strType.encoding) - strType.encoding = newEncoding - // Do not report problem if encoding was derived from `meta/encoding` key - if (strType.isEncodingDerived) None else problem1 - case _ => - // not a string type = no problem - None - }).map(problem => problem.localizedInPath(member.path ++ List("encoding"))) + try { + (member.dataType match { + case strType: StrFromBytesType => + val (newEncoding, problem1) = canonicalizeName(strType.encoding) + strType.encoding = newEncoding + // Do not report problem if encoding was derived from `meta/encoding` key + if (strType.isEncodingDerived) None else problem1 + case _ => + // not a string type = no problem + None + }).map(problem => problem.localizedInPath(member.path ++ List("encoding"))) + } catch { + // This pass can be called on model with errors, in particular, types of + // value instances could not be calculated. In that case just ignore that + // instance + case _: ExpressionError => None + } } def canonicalizeName(original: String): (String, Option[CompilationProblem with PathLocalizable]) = { diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala b/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala index 4c4753847..597f3d9b4 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala @@ -40,20 +40,27 @@ class StyleCheckIds(specs: ClassSpecs) extends PrecompileStep { } def getSizeRefProblem(spec: ClassSpec, attr: MemberSpec): Option[CompilationProblem] = { - getSizeReference(spec, attr.dataType).flatMap(sizeRefAttr => { - val existingName = sizeRefAttr.id.humanReadable - val goodName = s"len_${attr.id.humanReadable}" - if (existingName != goodName) { - Some(StyleWarningSizeLen( - goodName, - existingName, - attr.id.humanReadable, - ProblemCoords(path = Some(sizeRefAttr.path ++ List("id"))) - )) - } else { - None - } - }) + try { + getSizeReference(spec, attr.dataType).flatMap(sizeRefAttr => { + val existingName = sizeRefAttr.id.humanReadable + val goodName = s"len_${attr.id.humanReadable}" + if (existingName != goodName) { + Some(StyleWarningSizeLen( + goodName, + existingName, + attr.id.humanReadable, + ProblemCoords(path = Some(sizeRefAttr.path ++ List("id"))) + )) + } else { + None + } + }) + } catch { + // This pass can be called on model with errors, in particular, types of + // value instances could not be calculated. In that case just ignore that + // instance + case _: ExpressionError => None + } } def getRepeatExprRefProblem(spec: ClassSpec, attr: AttrLikeSpec): Option[CompilationProblem] = { From 593b18245c661edd43a61cdf3a85fd92f42bfc74 Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Oct 2024 22:10:07 +0500 Subject: [PATCH 9/9] Ignore all errors in DeriveValueInstanceTypes, errors will be detected by `TypeValidator` If just return CompilationProblems from this pass, we will report some errors twice, because `TypeValidator` alro reports errors from the `TypeDetector` --- .../kaitai/struct/precompile/DeriveValueInstanceTypes.scala | 2 +- .../scala/io/kaitai/struct/precompile/TypeValidator.scala | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala index a8133c503..de42d4b4b 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala @@ -59,7 +59,7 @@ class DeriveValueInstanceTypes(specs: ClassSpecs) extends PrecompileStep { Log.typeProcValue.info(() => s"${instName.name} type undecided: ${tue.getMessage}") // just ignore, we're not there yet, probably we'll get it on next iteration case err: ExpressionError => - throw ErrorInInput(err, vi.path ++ List("value")).toException + // Ignore all errors, the validation will be performed in TypeValidator pass later } case Some(_) => // already derived, do nothing diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala b/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala index d80741484..b32b38609 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala @@ -105,6 +105,10 @@ class TypeValidator(specs: ClassSpecs) extends PrecompileStep { def validateValueInstance(vis: ValueInstanceSpec): Option[CompilationProblem] = { try { + // detectType performs some additional checks that validate does not (for example, + // applicability of operators to types, like `Not` to numbers). + // TODO: probably implement those checks in validate too? + detector.detectType(vis.value) detector.validate(vis.value) None } catch {