Merge branch 'master' into serialization

RELEASE_NOTES.md

@@ -1,3 +1,8 @@
+# 0.11 (TBD)
+
+* Packaging / infrastructure improvements:
+  * [npm package `kaitai-struct-compiler`](https://www.npmjs.com/package/kaitai-struct-compiler/) now returns the compiler object itself instead of a constructor function (called `KaitaiStructCompiler`). Make sure to adapt your code: replace `(new KaitaiStructCompiler()).compile(...)` with `KaitaiStructCompiler.compile(...)` ([#222](https://github.com/kaitai-io/kaitai_struct_compiler/pull/222))
+
 # 0.10 (2022-07-08)
 
 * General compilation improvements:

build.sbt

@@ -19,7 +19,8 @@ lazy val root = project.in(file(".")).
     publishLocal := {}
   )
 
-lazy val compiler = crossProject.in(file(".")).
+lazy val compiler = crossProject(JSPlatform, JVMPlatform).
+  in(file(".")).
   enablePlugins(JavaAppPackaging).
   settings(
     organization := "io.kaitai",
@@ -39,7 +40,7 @@ lazy val compiler = crossProject.in(file(".")).
       }
     },
     licenses := Seq(("GPL-3.0", url("https://opensource.org/licenses/GPL-3.0"))),
-    scalaVersion := "2.12.12",
+    scalaVersion := "2.12.18",
 
     // Repo publish options
     publishTo := version { (v: String) =>
@@ -70,17 +71,19 @@ lazy val compiler = crossProject.in(file(".")).
     Compile / sourceGenerators += generateVersionTask.taskValue, // update automatically on every rebuild
 
     libraryDependencies ++= Seq(
-      "com.github.scopt" %%% "scopt" % "3.6.0",
-      "com.lihaoyi" %%% "fastparse" % "1.0.0",
-      "org.yaml" % "snakeyaml" % "1.28"
+      "com.github.scopt" %%% "scopt" % "4.1.0",
+      "com.lihaoyi" %%% "fastparse" % "2.3.3",
+      "org.yaml" % "snakeyaml" % "2.0"
     )
   ).
   jvmSettings(
     name := NAME,
 
     Compile / mainClass := Some("io.kaitai.struct.JavaMain"),
     libraryDependencies ++= Seq(
-      "org.scalatest" %% "scalatest" % "3.2.12" % "test"
+      "org.scalatest" %% "scalatest-funspec" % "3.2.15" % "test",
+      "org.scalatest" %% "scalatest-funsuite" % "3.2.15" % "test",
+      "org.scalatest" %% "scalatest-shouldmatchers" % "3.2.15" % "test",
     ),
 
     Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-u", "target/test_out"),
@@ -228,13 +231,8 @@ lazy val buildNpmJsFileTask = Def.task {
        |  }
        |}(typeof self !== 'undefined' ? self : this, function () {
        |
-       |var exports = {};
-       |var __ScalaJSEnv = { exportsNamespace: exports };
-       |
        |$compiledFileContents
-       |
-       |return exports.io.kaitai.struct.MainJs;
-       |
+       |return MainJs;
        |}));
        |""".stripMargin
 

js/README.md

@@ -54,23 +54,24 @@ Our [examples repository](https://github.com/kaitai-io/kaitai_struct_examples) c
 
 ## Plugging in
 
+> **Note:** Before version 0.11.0, the `kaitai-struct-compiler` module returned a _compiler constructor_ (called `KaitaiStructCompiler`) and you would use it as `(new KaitaiStructCompiler()).compile(...)`, but since 0.11.0 it returns the _compiler object_ itself and you just need to do `KaitaiStructCompiler.compile(...)`. Make sure to adapt your code.
+
 We publish the compiler as a [UMD module](https://github.com/umdjs/umd), so it works from various environments, including server-side (e.g. Node.js) and client-side (e.g. web browser) ones.
 
 Note: currently we don't publish the compiler as standard ES module. This will probably change in the future. If you need ES module, please comment on [this issue](https://github.com/kaitai-io/kaitai_struct/issues/180).
 
 ### node
 
 ```javascript
-var KaitaiStructCompiler = require("kaitai-struct-compiler");
-var compiler = new KaitaiStructCompiler();
+var compiler = require("kaitai-struct-compiler");
 ```
 
 ### browser using script tags
 
 ```html
 <script src="node_modules/kaitai-struct-compiler/kaitai-struct-compiler.js"></script>
 <script>
-    var compiler = new KaitaiStructCompiler();
+    var compiler = KaitaiStructCompiler;
 </script>
 ```
 
@@ -85,8 +86,8 @@ var compiler = new KaitaiStructCompiler();
         }
     });
 
-    require(['kaitai-struct-compiler'], function(KaitaiStructCompiler){
-        var compiler = new KaitaiStructCompiler();
+    require(['kaitai-struct-compiler'], function(compiler){
+        // ...
     });
 </script>
 ```
@@ -204,5 +205,5 @@ Kaitai Struct compiler in JavaScript depends on the following libraries:
 
 Note that these clauses only apply only to compiler itself, not `.ksy`
 input files that one supplies in normal process of compilation, nor to
-compiler's output files — that consitutes normal usage process and you
+compiler's output files — that constitutes normal usage process and you
 obviously keep copyright to both.

js/src/main/scala/io/kaitai/struct/JavaScriptImporter.scala

@@ -2,8 +2,10 @@ package io.kaitai.struct
 
 import scala.scalajs.js
 import scala.scalajs.js.Promise
+import scala.scalajs.js.annotation.JSGlobal
 
 @js.native
+@JSGlobal
 class JavaScriptImporter extends js.Object {
   def importYaml(name: String, mode: String): Promise[js.Object] = js.native
 }

js/src/main/scala/io/kaitai/struct/MainJs.scala

@@ -7,9 +7,9 @@ import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
 import scala.scalajs.js
 import scala.scalajs.js.JSConverters._
-import scala.scalajs.js.annotation.JSExport
+import scala.scalajs.js.annotation.{JSExport, JSExportTopLevel}
 
-@JSExport
+@JSExportTopLevel("MainJs")
 object MainJs {
   KSVersion.current = Version.version
 

js/src/main/scala/io/kaitai/struct/format/JavaScriptClassSpecs.scala

@@ -1,8 +1,10 @@
 package io.kaitai.struct.format
 
 import io.kaitai.struct.JavaScriptImporter
+import io.kaitai.struct.problems.ErrorInInput
 
 import scala.concurrent.Future
+import scala.concurrent.ExecutionContext
 import scala.scalajs.concurrent.JSExecutionContext
 
 class JavaScriptClassSpecs(importer: JavaScriptImporter, firstSpec: ClassSpec)
@@ -12,13 +14,19 @@ class JavaScriptClassSpecs(importer: JavaScriptImporter, firstSpec: ClassSpec)
   val MODE_ABS = "abs"
 
   override def importRelative(name: String, path: List[String], inFile: Option[String]): Future[Option[ClassSpec]] =
-    doImport(name, MODE_REL)
+    doImport(name, path, MODE_REL)
   override def importAbsolute(name: String, path: List[String], inFile: Option[String]): Future[Option[ClassSpec]] =
-    doImport(name, MODE_ABS)
+    doImport(name, path, MODE_ABS)
 
-  def doImport(name: String, mode: String): Future[Option[ClassSpec]] =
-    importer.importYaml(name, mode).toFuture.map { (yaml) =>
-      val yamlScala = JavaScriptKSYParser.yamlJavascriptToScala(yaml)
-      Some(ClassSpec.fromYaml(yamlScala, Some(name)))
-    }(JSExecutionContext.queue)
+  def doImport(name: String, path: List[String], mode: String): Future[Option[ClassSpec]] = {
+    implicit val ec: ExecutionContext = JSExecutionContext.queue
+
+    importer.importYaml(name, mode).toFuture
+      .transform((yaml) => {
+        val yamlScala = JavaScriptKSYParser.yamlJavascriptToScala(yaml)
+        Some(ClassSpec.fromYaml(yamlScala, Some(name)))
+      }, (err) => {
+        throw ErrorInInput(err, path).toException
+      })
+  }
 }

js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala

@@ -18,7 +18,7 @@ object JavaScriptKSYParser {
     val yamlScala = yamlJavascriptToScala(yaml)
     val firstSpec = ClassSpec.fromYaml(yamlScala, None)
     val specs = new JavaScriptClassSpecs(importer, firstSpec)
-    Main.importAndPrecompile(specs, config).map{ problems =>
+    Main.importAndPrecompile(specs, config).map { problems =>
       // throw the first (if any) severe (not a warning) problem as an exception
       problems.find(p => p.severity != ProblemSeverity.Warning) match {
         case Some(problem) => throw problem.toException

jvm/src/main/scala/io/kaitai/struct/JavaMain.scala

@@ -31,7 +31,7 @@ object JavaMain {
 
   def parseCommandLine(args: Array[String]): Option[CLIConfig] = {
     val parser = new scopt.OptionParser[CLIConfig](Version.name) {
-      override def showUsageOnError = true
+      override def showUsageOnError = Some(true)
 
       head(Version.name, Version.version)
 

jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala

@@ -7,10 +7,8 @@ import io.kaitai.struct.JavaMain.CLIConfig
 import io.kaitai.struct.format.{ClassSpec, ClassSpecs}
 import io.kaitai.struct.problems.{CompilationProblem, CompilationProblemException, ProblemCoords, ProblemSeverity, YAMLParserError}
 import io.kaitai.struct.{Log, Main}
-import org.yaml.snakeyaml.constructor.SafeConstructor
 import org.yaml.snakeyaml.error.MarkedYAMLException
-import org.yaml.snakeyaml.representer.Representer
-import org.yaml.snakeyaml.{DumperOptions, LoaderOptions, Yaml}
+import org.yaml.snakeyaml.{LoaderOptions, Yaml}
 
 import scala.collection.JavaConverters._
 import scala.concurrent.Await
@@ -71,12 +69,7 @@ object JavaKSYParser {
   def getYamlLoader: Yaml = {
     val loaderOptions = new LoaderOptions
     loaderOptions.setAllowDuplicateKeys(false)
-    new Yaml(
-      new SafeConstructor,
-      new Representer,
-      new DumperOptions,
-      loaderOptions
-    )
+    new Yaml(loaderOptions)
   }
 
   def readerToYaml(reader: Reader): Any = {

jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala

@@ -297,7 +297,7 @@ class TranslatorSpec extends AnyFunSuite {
     JavaCompiler -> "new ArrayList<Integer>(Arrays.asList(0, 1, 100500))",
     JavaScriptCompiler -> "[0, 1, 100500]",
     LuaCompiler -> "{0, 1, 100500}",
-    PerlCompiler -> "(0, 1, 100500)",
+    PerlCompiler -> "[0, 1, 100500]",
     PHPCompiler -> "[0, 1, 100500]",
     PythonCompiler -> "[0, 1, 100500]",
     RubyCompiler -> "[0, 1, 100500]"
@@ -347,6 +347,7 @@ class TranslatorSpec extends AnyFunSuite {
     JavaCompiler -> "a().get((int) 42)",
     JavaScriptCompiler -> "this.a[42]",
     LuaCompiler -> "self.a[43]",
+    PerlCompiler -> "@{$self->a()}[42]",
     PHPCompiler -> "$this->a()[42]",
     PythonCompiler -> "self.a[42]",
     RubyCompiler -> "a[42]"
@@ -359,6 +360,7 @@ class TranslatorSpec extends AnyFunSuite {
     JavaCompiler -> "a().get((42 - 2))",
     JavaScriptCompiler -> "this.a[(42 - 2)]",
     LuaCompiler -> "self.a[(43 - 2)]",
+    PerlCompiler -> "@{$self->a()}[(42 - 2)]",
     PHPCompiler -> "$this->a()[(42 - 2)]",
     PythonCompiler -> "self.a[(42 - 2)]",
     RubyCompiler -> "a[(42 - 2)]"
@@ -371,6 +373,7 @@ class TranslatorSpec extends AnyFunSuite {
     JavaCompiler -> "a().get(0)",
     JavaScriptCompiler -> "this.a[0]",
     LuaCompiler -> "self.a[1]",
+    PerlCompiler -> "@{$self->a()}[0]",
     PHPCompiler -> "$this->a()[0]",
     PythonCompiler -> "self.a[0]",
     RubyCompiler -> "a.first"
@@ -383,6 +386,7 @@ class TranslatorSpec extends AnyFunSuite {
     JavaCompiler -> "a().get(a().size() - 1)",
     JavaScriptCompiler -> "this.a[this.a.length - 1]",
     LuaCompiler -> "self.a[#self.a]",
+    PerlCompiler -> "@{$self->a()}[-1]",
     PHPCompiler -> "$this->a()[count($this->a()) - 1]",
     PythonCompiler -> "self.a[-1]",
     RubyCompiler -> "a.last"
@@ -396,7 +400,7 @@ class TranslatorSpec extends AnyFunSuite {
     JavaScriptCompiler -> "this.a.length",
     LuaCompiler -> "#self.a",
     PHPCompiler -> "count($this->a())",
-    PerlCompiler -> "scalar($self->a())",
+    PerlCompiler -> "scalar(@{$self->a()})",
     PythonCompiler -> "len(self.a)",
     RubyCompiler -> "a.length"
   ))
@@ -520,7 +524,7 @@ class TranslatorSpec extends AnyFunSuite {
     JavaCompiler -> "Long.parseLong(\"12345\", 10)",
     JavaScriptCompiler -> "Number.parseInt(\"12345\", 10)",
     LuaCompiler -> "tonumber(\"12345\")",
-    PerlCompiler -> "\"12345\"",
+    PerlCompiler -> "\"12345\" + 0",
     PHPCompiler -> "intval(\"12345\", 10)",
     PythonCompiler -> "int(u\"12345\")",
     RubyCompiler -> "\"12345\".to_i"
@@ -601,7 +605,7 @@ class TranslatorSpec extends AnyFunSuite {
     JavaCompiler -> "new ArrayList<Integer>(Arrays.asList())",
     JavaScriptCompiler -> "[]",
     LuaCompiler -> "{}",
-    PerlCompiler -> "()",
+    PerlCompiler -> "[]",
     PHPCompiler -> "[]",
     PythonCompiler -> "[]",
     RubyCompiler -> "[]"
@@ -614,7 +618,7 @@ class TranslatorSpec extends AnyFunSuite {
     JavaCompiler -> "new ArrayList<Double>(Arrays.asList())",
     JavaScriptCompiler -> "[]",
     LuaCompiler -> "{}",
-    PerlCompiler -> "()",
+    PerlCompiler -> "[]",
     PHPCompiler -> "[]",
     PythonCompiler -> "[]",
     RubyCompiler -> "[]"
@@ -641,7 +645,7 @@ class TranslatorSpec extends AnyFunSuite {
     JavaCompiler -> "new ArrayList<Integer>(Arrays.asList(0, 1, 2))",
     JavaScriptCompiler -> "[0, 1, 2]",
     LuaCompiler -> "{0, 1, 2}",
-    PerlCompiler -> "(0, 1, 2)",
+    PerlCompiler -> "[0, 1, 2]",
     PHPCompiler -> "[0, 1, 2]",
     PythonCompiler -> "[0, 1, 2]",
     RubyCompiler -> "[0, 1, 2]"

project/build.properties

@@ -1 +1 @@
-sbt.version = 1.7.1
+sbt.version = 1.9.7

project/plugins.sbt

@@ -1,4 +1,5 @@
 logLevel := Level.Warn
 
-addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.12")
-addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.33")
+addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16")
+addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.13.1")
+addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.0.0")