Collect all errors from all passes, rename and document some of them

shared/src/main/scala/io/kaitai/struct/Main.scala

@@ -39,7 +39,7 @@ object Main {
     * @return a list of compilation problems encountered during precompilation steps
     */
   def precompile(specs: ClassSpecs, conf: RuntimeConfig): Iterable[CompilationProblem] = {
-    new MarkupClassNames(specs).run()
+    new CalculateFullNamesAndSetSurroundingType(specs).run()
     val resolveTypeProblems = specs.flatMap { case (_, topClass) =>
       val config = updateConfigFromMeta(conf, topClass.meta)
       new ResolveTypes(specs, topClass, config.opaqueTypes).run()
@@ -51,16 +51,17 @@ object Main {
       return resolveTypeProblems
     }
 
-    new ParentTypes(specs).run()
-    new SpecsValueTypeDerive(specs).run()
-    new CalculateSeqSizes(specs).run()
-    val typeValidatorProblems = new TypeValidator(specs).run()
+    var passes = Seq(
+      new ParentTypes(specs),
+      new DeriveValueInstanceTypes(specs),
+      new CalculateSeqSizes(specs),
+      new TypeValidator(specs),
+      // Warnings
+      new StyleCheckIds(specs),
+      new CanonicalizeEncodingNames(specs),
+    )
 
-    // Warnings
-    val styleWarnings = new StyleCheckIds(specs).run()
-    val encodingProblems = new CanonicalizeEncodingNames(specs).run()
-
-    resolveTypeProblems ++ typeValidatorProblems ++ styleWarnings ++ encodingProblems
+    resolveTypeProblems ++ passes.flatMap(_.run())
   }
 
   /**

shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala

@@ -58,6 +58,8 @@ case class ClassSpec(
     * Full absolute name of the class (including all names of classes that
     * it's nested into, as a namespace). Derived either from `meta`/`id`
     * (for top-level classes), or from keys in `types` (for nested classes).
+    *
+    * This name is calculated by the `CalculateFullNamesAndSetSurroundingType` pass.
     */
   var name = List[String]()
 
@@ -76,6 +78,8 @@ case class ClassSpec(
   /**
     * The class specification that this class is nested into, if it exists.
     * For top-level classes, it's None.
+    *
+    * This class is calculated by the `CalculateFullNamesAndSetSurroundingType` pass.
     */
   var upClass: Option[ClassSpec] = None
 

shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala

@@ -6,6 +6,12 @@ import scala.collection.immutable.SortedMap
 import scala.collection.mutable
 
 case class EnumSpec(path: List[String], map: SortedMap[Long, EnumValueSpec]) extends YAMLPath {
+  /**
+    * Absolute name of the enum (includes the names of all classes inside which
+    * it is defined). Derived either from keys in `enums`.
+    *
+    * This name is calculated by the `CalculateFullNamesAndSetSurroundingType` pass.
+    */
   var name = List[String]()
 
   /**

shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala

@@ -13,6 +13,10 @@ case class ValueInstanceSpec(
   path: List[String],
   value: Ast.expr,
   ifExpr: Option[Ast.expr] = None,
+  /**
+    * Type of the `value`. Calculated by the [[DeriveValueInstanceTypes]] pass.
+    * `None` means "un-calculated yet".
+    */
   var dataTypeOpt: Option[DataType] = None,
   val _doc: DocSpec = DocSpec.EMPTY,
 ) extends InstanceSpec(_doc) {

shared/src/main/scala/io/kaitai/struct/precompile/MarkupClassNames.scala → shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala

@@ -3,21 +3,25 @@ package io.kaitai.struct.precompile
 import io.kaitai.struct.format.{ClassSpec, ClassSpecs}
 import io.kaitai.struct.problems.CompilationProblem
 
-class MarkupClassNames(classSpecs: ClassSpecs) extends PrecompileStep {
+/**
+  * Assigns a full name to the `.name` property to each KS type and enum.
+  * Also for each [[ClassSpec]] assigns [[ClassSpec.upClass]] to a surrounding type.
+  */
+class CalculateFullNamesAndSetSurroundingType(classSpecs: ClassSpecs) extends PrecompileStep {
   override def run(): Iterable[CompilationProblem] = {
-    classSpecs.foreach { case (_, curClass) => markupClassNames(curClass) }
+    classSpecs.foreach { case (_, curClass) => calculate(curClass) }
     None
   }
 
-  def markupClassNames(curClass: ClassSpec): Unit = {
+  private def calculate(curClass: ClassSpec): Unit = {
     curClass.enums.foreach { case (enumName, enumSpec) =>
       enumSpec.name = curClass.name ::: List(enumName)
     }
 
     curClass.types.foreach { case (nestedName: String, nestedClass) =>
       nestedClass.name = curClass.name ::: List(nestedName)
       nestedClass.upClass = Some(curClass)
-      markupClassNames(nestedClass)
+      calculate(nestedClass)
     }
   }
 }

shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala

@@ -5,10 +5,12 @@ import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format._
+import io.kaitai.struct.problems.CompilationProblem
 
-class CalculateSeqSizes(specs: ClassSpecs) {
-  def run(): Unit = {
+class CalculateSeqSizes(specs: ClassSpecs) extends PrecompileStep {
+  override def run(): Iterable[CompilationProblem] = {
     specs.forEachRec(CalculateSeqSizes.getSeqSize)
+    None
   }
 }
 

shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala

@@ -32,16 +32,23 @@ object CanonicalizeEncodingNames {
   }
 
   def canonicalizeMember(member: MemberSpec): Iterable[CompilationProblem] = {
-    (member.dataType match {
-      case strType: StrFromBytesType =>
-        val (newEncoding, problem1) = canonicalizeName(strType.encoding)
-        strType.encoding = newEncoding
-        // Do not report problem if encoding was derived from `meta/encoding` key
-        if (strType.isEncodingDerived) None else problem1
-      case _ =>
-        // not a string type = no problem
-        None
-    }).map(problem => problem.localizedInPath(member.path ++ List("encoding")))
+    try {
+      (member.dataType match {
+        case strType: StrFromBytesType =>
+          val (newEncoding, problem1) = canonicalizeName(strType.encoding)
+          strType.encoding = newEncoding
+          // Do not report problem if encoding was derived from `meta/encoding` key
+          if (strType.isEncodingDerived) None else problem1
+        case _ =>
+          // not a string type = no problem
+          None
+      }).map(problem => problem.localizedInPath(member.path ++ List("encoding")))
+    } catch {
+      // This pass can be called on model with errors, in particular, types of
+      // value instances could not be calculated. In that case just ignore that
+      // instance
+      case _: ExpressionError => None
+    }
   }
 
   def canonicalizeName(original: String): (String, Option[CompilationProblem with PathLocalizable]) = {

shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala

@@ -0,0 +1,80 @@
+package io.kaitai.struct.precompile
+
+import io.kaitai.struct.{ClassTypeProvider, Log}
+import io.kaitai.struct.format.{ClassSpec, ClassSpecs, ValueInstanceSpec}
+import io.kaitai.struct.problems.{CompilationProblem, ErrorInInput}
+import io.kaitai.struct.translators.TypeDetector
+
+/**
+  * Assign types to value instances by deriving them from their expressions.
+  *
+  * Calculates value of the [[ValueInstanceSpec.dataTypeOpt]] field, which is
+  * a type of value instance.
+  */
+class DeriveValueInstanceTypes(specs: ClassSpecs) extends PrecompileStep {
+  override def run(): Iterable[CompilationProblem] = {
+    var iterNum = 1
+    var hasChanged = false
+    do {
+      hasChanged = false
+      Log.typeProcValue.info(() => s"### DeriveValueInstanceTypes: iteration #$iterNum")
+      specs.foreach { case (specName, spec) =>
+        Log.typeProcValue.info(() => s"#### $specName")
+
+        val provider = new ClassTypeProvider(specs, spec)
+        val detector = new TypeDetector(provider)
+
+        val thisChanged = deriveValueType(spec, provider, detector)
+        Log.typeProcValue.info(() => ".... => " + (if (thisChanged) "changed" else "no changes"))
+        hasChanged |= thisChanged
+      }
+      iterNum += 1
+    } while (hasChanged)
+    Log.typeProcValue.info(() => s"## value type deriving finished in ${iterNum - 1} iteration(s)")
+    None
+  }
+
+  private def deriveValueType(
+    curClass: ClassSpec,
+    provider: ClassTypeProvider,
+    detector: TypeDetector
+  ): Boolean = {
+    Log.typeProcValue.info(() => s"deriveValueType(${curClass.nameAsStr})")
+    var hasChanged = false
+
+    provider.nowClass = curClass;
+    curClass.instances.foreach {
+      case (instName, inst) =>
+        inst match {
+          case vi: ValueInstanceSpec =>
+            vi.dataTypeOpt match {
+              case None =>
+                try {
+                  val viType = detector.detectType(vi.value)
+                  vi.dataTypeOpt = Some(viType)
+                  Log.typeProcValue.info(() => s"${instName.name} derived type: $viType")
+                  hasChanged = true
+                } catch {
+                  case tue: TypeUndecidedError =>
+                    Log.typeProcValue.info(() => s"${instName.name} type undecided: ${tue.getMessage}")
+                    // just ignore, we're not there yet, probably we'll get it on next iteration
+                  case err: ExpressionError =>
+                    // Ignore all errors, the validation will be performed in TypeValidator pass later
+                }
+              case Some(_) =>
+                // already derived, do nothing
+            }
+          case _ =>
+            // do nothing
+        }
+    }
+
+    // Continue with all nested types
+    curClass.types.foreach {
+      case (_, classSpec) =>
+        hasChanged ||= deriveValueType(classSpec, provider, detector)
+    }
+
+    hasChanged
+  }
+}

shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala

@@ -4,18 +4,20 @@ import io.kaitai.struct.{ClassTypeProvider, Log}
 import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType.{ArrayTypeInStream, SwitchType, UserType}
 import io.kaitai.struct.format._
+import io.kaitai.struct.problems.CompilationProblem
 import io.kaitai.struct.translators.TypeDetector
 
 /**
   * Precompile step that calculates actual parent types of KSY-defined types
   * (the type of the `_parent` built-in property).
   */
-class ParentTypes(classSpecs: ClassSpecs) {
-  def run(): Unit = {
+class ParentTypes(classSpecs: ClassSpecs) extends PrecompileStep {
+  override def run(): Iterable[CompilationProblem] = {
     classSpecs.foreach { case (_, curClass) => markup(curClass) }
     classSpecs.forEachTopLevel((_, spec) => {
       spec.parentClass = GenericStructClassSpec
     })
+    None
   }
 
   def markup(curClass: ClassSpec): Unit = {

shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala

@@ -40,20 +40,27 @@ class StyleCheckIds(specs: ClassSpecs) extends PrecompileStep {
   }
 
   def getSizeRefProblem(spec: ClassSpec, attr: MemberSpec): Option[CompilationProblem] = {
-    getSizeReference(spec, attr.dataType).flatMap(sizeRefAttr => {
-      val existingName = sizeRefAttr.id.humanReadable
-      val goodName = s"len_${attr.id.humanReadable}"
-      if (existingName != goodName) {
-        Some(StyleWarningSizeLen(
-          goodName,
-          existingName,
-          attr.id.humanReadable,
-          ProblemCoords(path = Some(sizeRefAttr.path ++ List("id")))
-        ))
-      } else {
-        None
-      }
-    })
+    try {
+      getSizeReference(spec, attr.dataType).flatMap(sizeRefAttr => {
+        val existingName = sizeRefAttr.id.humanReadable
+        val goodName = s"len_${attr.id.humanReadable}"
+        if (existingName != goodName) {
+          Some(StyleWarningSizeLen(
+            goodName,
+            existingName,
+            attr.id.humanReadable,
+            ProblemCoords(path = Some(sizeRefAttr.path ++ List("id")))
+          ))
+        } else {
+          None
+        }
+      })
+    } catch {
+      // This pass can be called on model with errors, in particular, types of
+      // value instances could not be calculated. In that case just ignore that
+      // instance
+      case _: ExpressionError => None
+    }
   }
 
   def getRepeatExprRefProblem(spec: ClassSpec, attr: AttrLikeSpec): Option[CompilationProblem] = {

shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala

@@ -105,6 +105,10 @@ class TypeValidator(specs: ClassSpecs) extends PrecompileStep {
 
   def validateValueInstance(vis: ValueInstanceSpec): Option[CompilationProblem] = {
     try {
+      // detectType performs some additional checks that validate does not (for example,
+      // applicability of operators to types, like `Not` to numbers).
+      // TODO: probably implement those checks in validate too?
+      detector.detectType(vis.value)
       detector.validate(vis.value)
       None
     } catch {