From 18db1553de4d443777525b65bbc9f49488b01b33 Mon Sep 17 00:00:00 2001 From: Mingun Date: Wed, 25 Aug 2021 15:16:32 +0500 Subject: [PATCH 01/12] Unify style Change commenting pattern so stupid VSCode folding algorithm correcly fold class --- .../struct/format/JavaScriptKSYParser.scala | 2 +- .../kaitai/struct/exprlang/Expressions.scala | 64 +++++++++---------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala b/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala index 9a1317b06..0f7ee55a1 100644 --- a/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala +++ b/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala @@ -18,7 +18,7 @@ object JavaScriptKSYParser { val yamlScala = yamlJavascriptToScala(yaml) val firstSpec = ClassSpec.fromYaml(yamlScala, None) val specs = new JavaScriptClassSpecs(importer, firstSpec) - Main.importAndPrecompile(specs, config).map{ problems => + Main.importAndPrecompile(specs, config).map { problems => // throw the first (if any) severe (not a warning) problem as an exception problems.find(p => p.severity != ProblemSeverity.Warning) match { case Some(problem) => throw problem.toException diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala index 74bbafbcb..3b0737cb1 100644 --- a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala +++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala @@ -35,20 +35,20 @@ object Expressions { val FLOAT_NUMBER = Lexical.floatnumber val STRING: P[String] = Lexical.stringliteral - val test: P[Ast.expr] = P( or_test ~ ("?" ~ test ~ ":" ~ test).? ).map{ - case (x, None) => x - case (condition, Some((ifTrue, ifFalse))) => Ast.expr.IfExp(condition, ifTrue, ifFalse) - } - val or_test = P( and_test.rep(1, kw("or")) ).map{ + val test: P[Ast.expr] = P( or_test ~ ("?" ~ test ~ ":" ~ test).? ).map { + case (x, None) => x + case (condition, Some((ifTrue, ifFalse))) => Ast.expr.IfExp(condition, ifTrue, ifFalse) + } + val or_test = P( and_test.rep(1, kw("or")) ).map { case Seq(x) => x case xs => Ast.expr.BoolOp(Ast.boolop.Or, xs) } - val and_test = P( not_test.rep(1, kw("and")) ).map{ + val and_test = P( not_test.rep(1, kw("and")) ).map { case Seq(x) => x case xs => Ast.expr.BoolOp(Ast.boolop.And, xs) } val not_test: P[Ast.expr] = P( (kw("not") ~ not_test).map(Ast.expr.UnaryOp(Ast.unaryop.Not, _)) | comparison ) - val comparison: P[Ast.expr] = P( expr ~ (comp_op ~ expr).? ).map{ + val comparison: P[Ast.expr] = P( expr ~ (comp_op ~ expr).? ).map { case (lhs, None) => lhs case (lhs, Some(chunks)) => val (op, rhs) = chunks @@ -69,7 +69,7 @@ object Expressions { val comp_op = P( LtE|GtE|Eq|Gt|Lt|NotEq ) val Add = op("+", Ast.operator.Add) val Sub = op("-", Ast.operator.Sub) -// val Pow = op("**", Ast.operator.Pow) + // val Pow = op("**", Ast.operator.Pow) val Mult= op("*", Ast.operator.Mult) val Div = op("/", Ast.operator.Div) val Mod = op("%", Ast.operator.Mod) @@ -77,7 +77,7 @@ object Expressions { val BitAnd = op("&", Ast.operator.BitAnd) val BitXor = op("^", Ast.operator.BitXor) - def Chain(p: P[Ast.expr], op: P[Ast.operator]) = P( p ~ (op ~ p).rep ).map{ + def Chain(p: P[Ast.expr], op: P[Ast.operator]) = P( p ~ (op ~ p).rep ).map { case (lhs, chunks) => chunks.foldLeft(lhs){case (lhs, (op, rhs)) => Ast.expr.BinOp(lhs, op, rhs) @@ -96,27 +96,27 @@ object Expressions { ("~" ~ factor).map(Ast.expr.UnaryOp(Ast.unaryop.Invert, _)) | power ) -// val power: P[Ast.expr] = P( atom ~ trailer.rep ~ (Pow ~ factor).? ).map{ -// case (lhs, trailers, rhs) => -// val left = trailers.foldLeft(lhs)((l, t) => t(l)) -// rhs match{ -// case None => left -// case Some((op, right)) => Ast.expr.BinOp(left, op, right) -// } -// } + // val power: P[Ast.expr] = P( atom ~ trailer.rep ~ (Pow ~ factor).? ).map { + // case (lhs, trailers, rhs) => + // val left = trailers.foldLeft(lhs)((l, t) => t(l)) + // rhs match{ + // case None => left + // case Some((op, right)) => Ast.expr.BinOp(left, op, right) + // } + // } val power: P[Ast.expr] = P( atom ~ trailer.rep ).map { case (lhs, trailers) => trailers.foldLeft(lhs)((l, t) => t(l)) } val atom: P[Ast.expr] = { val empty_list = ("[" ~ "]").map(_ => Ast.expr.List(Nil)) -// val empty_dict = ("{" ~ "}").map(_ => Ast.expr.Dict(Nil, Nil)) + // val empty_dict = ("{" ~ "}").map(_ => Ast.expr.Dict(Nil, Nil)) P( empty_list | -// empty_dict | + // empty_dict | "(" ~ test ~ ")" | "[" ~ list ~ "]" | -// "{" ~ dictorsetmaker ~ "}" | + // "{" ~ dictorsetmaker ~ "}" | enumByName | byteSizeOfType | bitSizeOfType | @@ -134,8 +134,8 @@ object Expressions { val list = P( list_contents ).map(Ast.expr.List(_)) val trailer: P[Ast.expr => Ast.expr] = { - val call = P("(" ~ arglist ~ ")").map{ case (args) => (lhs: Ast.expr) => Ast.expr.Call(lhs, args)} - val slice = P("[" ~ test ~ "]").map{ case (args) => (lhs: Ast.expr) => Ast.expr.Subscript(lhs, args)} + val call = P("(" ~ arglist ~ ")").map { case (args) => (lhs: Ast.expr) => Ast.expr.Call(lhs, args)} + val slice = P("[" ~ test ~ "]").map { case (args) => (lhs: Ast.expr) => Ast.expr.Subscript(lhs, args)} val cast = P( "." ~ "as" ~ "<" ~ TYPE_NAME ~ ">" ).map( typeName => (lhs: Ast.expr) => Ast.expr.CastToType(lhs, typeName) ) @@ -145,16 +145,16 @@ object Expressions { val exprlist: P[Seq[Ast.expr]] = P( expr.rep(1, sep = ",") ~ ",".? ) val testlist: P[Seq[Ast.expr]] = P( test.rep(1, sep = ",") ~ ",".? ) -// val dictorsetmaker: P[Ast.expr] = { -// val dict_item = P( test ~ ":" ~ test ) -// val dict: P[Ast.expr.Dict] = P( -// (dict_item.rep(1, ",") ~ ",".?).map{x => -// val (keys, values) = x.unzip -// Ast.expr.Dict(keys, values) -// } -// ) -// P( /*dict_comp |*/ dict /*| set_comp | set*/) -// } + // val dictorsetmaker: P[Ast.expr] = { + // val dict_item = P( test ~ ":" ~ test ) + // val dict: P[Ast.expr.Dict] = P( + // (dict_item.rep(1, ",") ~ ",".?).map { x => + // val (keys, values) = x.unzip + // Ast.expr.Dict(keys, values) + // } + // ) + // P( /*dict_comp |*/ dict /*| set_comp | set*/) + // } val arglist: P[Seq[Ast.expr]] = P( (test).rep(0, ",") ) From ef035e83bb19a45dde93ddf5a0ada4f6cbf7bc12 Mon Sep 17 00:00:00 2001 From: Mingun Date: Mon, 23 Nov 2020 11:32:19 +0500 Subject: [PATCH 02/12] Upgrade FastParse to the latest version (1.0.0->2.3.3) Changes: https://com-lihaoyi.github.io/fastparse/#2.0.4 --- build.sbt | 2 +- .../kaitai/struct/exprlang/Expressions.scala | 203 +++++++++--------- .../io/kaitai/struct/exprlang/Lexical.scala | 70 +++--- .../struct/problems/CompilationProblem.scala | 7 +- 4 files changed, 138 insertions(+), 144 deletions(-) diff --git a/build.sbt b/build.sbt index 27a8fe60e..a40c7f5d6 100644 --- a/build.sbt +++ b/build.sbt @@ -71,7 +71,7 @@ lazy val compiler = crossProject.in(file(".")). libraryDependencies ++= Seq( "com.github.scopt" %%% "scopt" % "3.6.0", - "com.lihaoyi" %%% "fastparse" % "1.0.0", + "com.lihaoyi" %%% "fastparse" % "2.3.3", "org.yaml" % "snakeyaml" % "1.28" ) ). diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala index 3b0737cb1..33ab4cf77 100644 --- a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala +++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala @@ -1,9 +1,7 @@ package io.kaitai.struct.exprlang -import fastparse.noApi._ +import fastparse._ import Lexical.kw -import WsApi._ -import fastparse.StringReprOps /** * Loosely based on /pythonparse/shared/src/main/scala/pythonparse/ @@ -25,30 +23,32 @@ import fastparse.StringReprOps * IN THE SOFTWARE. */ object Expressions { + // Implicit rule which consume input in `~` and `.rep` operators + implicit val whitespace = { implicit ctx: ParsingRun[_] => Lexical.wscomment } - val NAME: P[Ast.identifier] = Lexical.identifier - val TYPE_NAME: P[Ast.typeId] = P("::".!.? ~ NAME.rep(1, "::") ~ ("[" ~ "]").!.?).map { + def NAME[_: P]: P[Ast.identifier] = Lexical.identifier + def TYPE_NAME[_: P]: P[Ast.typeId] = P("::".!.? ~ NAME.rep(1, "::") ~ ("[" ~ "]").!.?).map { case (first, names: Seq[Ast.identifier], arrayStr) => Ast.typeId(first.nonEmpty, names.map((el) => el.name), arrayStr.nonEmpty) } - val INT_NUMBER = Lexical.integer - val FLOAT_NUMBER = Lexical.floatnumber - val STRING: P[String] = Lexical.stringliteral + def INT_NUMBER[_: P] = Lexical.integer + def FLOAT_NUMBER[_: P] = Lexical.floatnumber + def STRING[_: P]: P[String] = Lexical.stringliteral - val test: P[Ast.expr] = P( or_test ~ ("?" ~ test ~ ":" ~ test).? ).map { + def test[_: P]: P[Ast.expr] = P( or_test ~ ("?" ~ test ~ ":" ~ test).? ).map { case (x, None) => x case (condition, Some((ifTrue, ifFalse))) => Ast.expr.IfExp(condition, ifTrue, ifFalse) } - val or_test = P( and_test.rep(1, kw("or")) ).map { + def or_test[_: P] = P( and_test.rep(1, kw("or")) ).map { case Seq(x) => x case xs => Ast.expr.BoolOp(Ast.boolop.Or, xs) } - val and_test = P( not_test.rep(1, kw("and")) ).map { + def and_test[_: P] = P( not_test.rep(1, kw("and")) ).map { case Seq(x) => x case xs => Ast.expr.BoolOp(Ast.boolop.And, xs) } - val not_test: P[Ast.expr] = P( (kw("not") ~ not_test).map(Ast.expr.UnaryOp(Ast.unaryop.Not, _)) | comparison ) - val comparison: P[Ast.expr] = P( expr ~ (comp_op ~ expr).? ).map { + def not_test[_: P]: P[Ast.expr] = P( (kw("not") ~ not_test).map(Ast.expr.UnaryOp(Ast.unaryop.Not, _)) | comparison ) + def comparison[_: P]: P[Ast.expr] = P( expr ~ (comp_op ~ expr).? ).map { case (lhs, None) => lhs case (lhs, Some(chunks)) => val (op, rhs) = chunks @@ -57,46 +57,46 @@ object Expressions { // Common operators, mapped from their // strings to their type-safe representations - def op[T](s: P0, rhs: T) = s.!.map(_ => rhs) - val LShift = op("<<", Ast.operator.LShift) - val RShift = op(">>", Ast.operator.RShift) - val Lt = op("<", Ast.cmpop.Lt) - val Gt = op(">", Ast.cmpop.Gt) - val Eq = op("==", Ast.cmpop.Eq) - val GtE = op(">=", Ast.cmpop.GtE) - val LtE = op("<=", Ast.cmpop.LtE) - val NotEq = op("!=", Ast.cmpop.NotEq) - val comp_op = P( LtE|GtE|Eq|Gt|Lt|NotEq ) - val Add = op("+", Ast.operator.Add) - val Sub = op("-", Ast.operator.Sub) - // val Pow = op("**", Ast.operator.Pow) - val Mult= op("*", Ast.operator.Mult) - val Div = op("/", Ast.operator.Div) - val Mod = op("%", Ast.operator.Mod) - val BitOr = op("|", Ast.operator.BitOr) - val BitAnd = op("&", Ast.operator.BitAnd) - val BitXor = op("^", Ast.operator.BitXor) - - def Chain(p: P[Ast.expr], op: P[Ast.operator]) = P( p ~ (op ~ p).rep ).map { + def op[_: P, T](s: P0, rhs: T) = s.!.map(_ => rhs) + def LShift[_: P] = op("<<", Ast.operator.LShift) + def RShift[_: P] = op(">>", Ast.operator.RShift) + def Lt[_: P] = op("<", Ast.cmpop.Lt) + def Gt[_: P] = op(">", Ast.cmpop.Gt) + def Eq[_: P] = op("==", Ast.cmpop.Eq) + def GtE[_: P] = op(">=", Ast.cmpop.GtE) + def LtE[_: P] = op("<=", Ast.cmpop.LtE) + def NotEq[_: P] = op("!=", Ast.cmpop.NotEq) + def comp_op[_: P] = P( LtE|GtE|Eq|Gt|Lt|NotEq ) + def Add[_: P] = op("+", Ast.operator.Add) + def Sub[_: P] = op("-", Ast.operator.Sub) + // def Pow[_: P] = op("**", Ast.operator.Pow) + def Mult[_: P]= op("*", Ast.operator.Mult) + def Div[_: P] = op("/", Ast.operator.Div) + def Mod[_: P] = op("%", Ast.operator.Mod) + def BitOr[_: P] = op("|", Ast.operator.BitOr) + def BitAnd[_: P] = op("&", Ast.operator.BitAnd) + def BitXor[_: P] = op("^", Ast.operator.BitXor) + + def Chain[_: P](p: => P[Ast.expr], op: => P[Ast.operator]) = P( p ~ (op ~ p).rep ).map { case (lhs, chunks) => chunks.foldLeft(lhs){case (lhs, (op, rhs)) => Ast.expr.BinOp(lhs, op, rhs) } } - val expr: P[Ast.expr] = P( Chain(xor_expr, BitOr) ) - val xor_expr: P[Ast.expr] = P( Chain(and_expr, BitXor) ) - val and_expr: P[Ast.expr] = P( Chain(shift_expr, BitAnd) ) - val shift_expr: P[Ast.expr] = P( Chain(arith_expr, LShift | RShift) ) - - val arith_expr: P[Ast.expr] = P( Chain(term, Add | Sub) ) - val term: P[Ast.expr] = P( Chain(factor, Mult | Div | Mod) ) - val factor: P[Ast.expr] = P( + def expr[_: P]: P[Ast.expr] = P( Chain(xor_expr, BitOr) ) + def xor_expr[_: P]: P[Ast.expr] = P( Chain(and_expr, BitXor) ) + def and_expr[_: P]: P[Ast.expr] = P( Chain(shift_expr, BitAnd) ) + def shift_expr[_: P]: P[Ast.expr] = P( Chain(arith_expr, LShift | RShift) ) + + def arith_expr[_: P]: P[Ast.expr] = P( Chain(term, Add | Sub) ) + def term[_: P]: P[Ast.expr] = P( Chain(factor, Mult | Div | Mod) ) + def factor[_: P]: P[Ast.expr] = P( ("+" ~ factor) | ("-" ~ factor).map(Ast.expr.UnaryOp(Ast.unaryop.Minus, _)) | ("~" ~ factor).map(Ast.expr.UnaryOp(Ast.unaryop.Invert, _)) | power ) - // val power: P[Ast.expr] = P( atom ~ trailer.rep ~ (Pow ~ factor).? ).map { + // def power[_: P]: P[Ast.expr] = P( atom ~ trailer.rep ~ (Pow ~ factor).? ).map { // case (lhs, trailers, rhs) => // val left = trailers.foldLeft(lhs)((l, t) => t(l)) // rhs match{ @@ -104,65 +104,60 @@ object Expressions { // case Some((op, right)) => Ast.expr.BinOp(left, op, right) // } // } - val power: P[Ast.expr] = P( atom ~ trailer.rep ).map { + def power[_: P]: P[Ast.expr] = P( atom ~ trailer.rep ).map { case (lhs, trailers) => trailers.foldLeft(lhs)((l, t) => t(l)) } - val atom: P[Ast.expr] = { - val empty_list = ("[" ~ "]").map(_ => Ast.expr.List(Nil)) - // val empty_dict = ("{" ~ "}").map(_ => Ast.expr.Dict(Nil, Nil)) - P( - empty_list | - // empty_dict | - "(" ~ test ~ ")" | - "[" ~ list ~ "]" | - // "{" ~ dictorsetmaker ~ "}" | - enumByName | - byteSizeOfType | - bitSizeOfType | - STRING.rep(1).map(_.mkString).map(Ast.expr.Str) | - NAME.map((x) => x.name match { - case "true" => Ast.expr.Bool(true) - case "false" => Ast.expr.Bool(false) - case _ => Ast.expr.Name(x) - }) | - FLOAT_NUMBER.map(Ast.expr.FloatNum) | - INT_NUMBER.map(Ast.expr.IntNum) - ) - } - val list_contents = P( test.rep(1, ",") ~ ",".? ) - val list = P( list_contents ).map(Ast.expr.List(_)) - - val trailer: P[Ast.expr => Ast.expr] = { - val call = P("(" ~ arglist ~ ")").map { case (args) => (lhs: Ast.expr) => Ast.expr.Call(lhs, args)} - val slice = P("[" ~ test ~ "]").map { case (args) => (lhs: Ast.expr) => Ast.expr.Subscript(lhs, args)} - val cast = P( "." ~ "as" ~ "<" ~ TYPE_NAME ~ ">" ).map( - typeName => (lhs: Ast.expr) => Ast.expr.CastToType(lhs, typeName) - ) - val attr = P("." ~ NAME).map(id => (lhs: Ast.expr) => Ast.expr.Attribute(lhs, id)) - P( call | slice | cast | attr ) - } + def empty_list[_: P] = P("[" ~ "]").map(_ => Ast.expr.List(Nil)) + // def empty_dict[_: P] = P("{" ~ "}").map(_ => Ast.expr.Dict(Nil, Nil)) + def atom[_: P]: P[Ast.expr] = P( + empty_list | + // empty_dict | + "(" ~ test ~ ")" | + "[" ~ list ~ "]" | + // "{" ~ dictorsetmaker ~ "}" | + enumByName | + byteSizeOfType | + bitSizeOfType | + STRING.rep(1).map(_.mkString).map(Ast.expr.Str) | + NAME.map((x) => x.name match { + case "true" => Ast.expr.Bool(true) + case "false" => Ast.expr.Bool(false) + case _ => Ast.expr.Name(x) + }) | + FLOAT_NUMBER.map(Ast.expr.FloatNum) | + INT_NUMBER.map(Ast.expr.IntNum) + ) + def list_contents[_: P] = P( test.rep(1, ",") ~ ",".? ) + def list[_: P] = P( list_contents ).map(Ast.expr.List(_)) - val exprlist: P[Seq[Ast.expr]] = P( expr.rep(1, sep = ",") ~ ",".? ) - val testlist: P[Seq[Ast.expr]] = P( test.rep(1, sep = ",") ~ ",".? ) - // val dictorsetmaker: P[Ast.expr] = { - // val dict_item = P( test ~ ":" ~ test ) - // val dict: P[Ast.expr.Dict] = P( - // (dict_item.rep(1, ",") ~ ",".?).map { x => - // val (keys, values) = x.unzip - // Ast.expr.Dict(keys, values) - // } - // ) - // P( /*dict_comp |*/ dict /*| set_comp | set*/) - // } + def call[_: P] = P("(" ~ arglist ~ ")").map { case (args) => (lhs: Ast.expr) => Ast.expr.Call(lhs, args)} + def slice[_: P] = P("[" ~ test ~ "]").map { case (args) => (lhs: Ast.expr) => Ast.expr.Subscript(lhs, args)} + def cast[_: P] = P( "." ~ "as" ~ "<" ~ TYPE_NAME ~ ">" ).map( + typeName => (lhs: Ast.expr) => Ast.expr.CastToType(lhs, typeName) + ) + def attr[_: P] = P("." ~ NAME).map(id => (lhs: Ast.expr) => Ast.expr.Attribute(lhs, id)) + def trailer[_: P]: P[Ast.expr => Ast.expr] = P( call | slice | cast | attr ) + + def exprlist[_: P]: P[Seq[Ast.expr]] = P( expr.rep(1, sep = ",") ~ ",".? ) + def testlist[_: P]: P[Seq[Ast.expr]] = P( test.rep(1, sep = ",") ~ ",".? ) + + // def dict_item[_: P] = P( test ~ ":" ~ test ) + // def dict[_: P]: P[Ast.expr.Dict] = P( + // (dict_item.rep(1, ",") ~ ",".?).map { x => + // val (keys, values) = x.unzip + // Ast.expr.Dict(keys, values) + // } + // ) + // def dictorsetmaker[_: P]: P[Ast.expr] = P( /*dict_comp |*/ dict /*| set_comp | set*/) - val arglist: P[Seq[Ast.expr]] = P( (test).rep(0, ",") ) + def arglist[_: P]: P[Seq[Ast.expr]] = P( (test).rep(0, ",") ) - val comp_if: P[Ast.expr] = P( "if" ~ test ) + def comp_if[_: P]: P[Ast.expr] = P( "if" ~ test ) - val testlist1: P[Seq[Ast.expr]] = P( test.rep(1, sep = ",") ) + def testlist1[_: P]: P[Seq[Ast.expr]] = P( test.rep(1, sep = ",") ) - val enumByName: P[Ast.expr.EnumByLabel] = P("::".!.? ~ NAME.rep(2, "::")).map { + def enumByName[_: P]: P[Ast.expr.EnumByLabel] = P("::".!.? ~ NAME.rep(2, "::")).map { case (first, names: Seq[Ast.identifier]) => val isAbsolute = first.nonEmpty val (enumName, enumLabel) = names.takeRight(2) match { @@ -176,16 +171,16 @@ object Expressions { } } - val byteSizeOfType: P[Ast.expr.ByteSizeOfType] = + def byteSizeOfType[_: P]: P[Ast.expr.ByteSizeOfType] = P("sizeof" ~ "<" ~ TYPE_NAME ~ ">").map(typeName => Ast.expr.ByteSizeOfType(typeName)) - val bitSizeOfType: P[Ast.expr.BitSizeOfType] = + def bitSizeOfType[_: P]: P[Ast.expr.BitSizeOfType] = P("bitsizeof" ~ "<" ~ TYPE_NAME ~ ">").map(typeName => Ast.expr.BitSizeOfType(typeName)) - val topExpr: P[Ast.expr] = P( test ~ End ) + def topExpr[_: P]: P[Ast.expr] = P( test ~ End ) - val topExprList: P[Seq[Ast.expr]] = P(testlist1 ~ End) + def topExprList[_: P]: P[Seq[Ast.expr]] = P(testlist1 ~ End) - val typeRef: P[Ast.TypeWithArguments] = P(Start ~ TYPE_NAME ~ ("(" ~ list ~ ")").? ~ End).map { + def typeRef[_: P]: P[Ast.TypeWithArguments] = P(Start ~ TYPE_NAME ~ ("(" ~ list ~ ")").? ~ End).map { case (path, None) => Ast.TypeWithArguments(path, Ast.expr.List(Seq())) case (path, Some(args)) => Ast.TypeWithArguments(path, args) } @@ -193,8 +188,8 @@ object Expressions { class ParseException(val src: String, val failure: Parsed.Failure) extends RuntimeException(failure.msg) - def parse(src: String): Ast.expr = realParse(src, topExpr) - def parseList(src: String): Seq[Ast.expr] = realParse(src, topExprList) + def parse(src: String): Ast.expr = realParse(src, topExpr(_)) + def parseList(src: String): Seq[Ast.expr] = realParse(src, topExprList(_)) /** * Parse string with reference to user-type definition, optionally in full path format @@ -204,10 +199,10 @@ object Expressions { * @return Tuple with path to type and type arguments. If arguments are not provided, * corresponding list is empty. List with path always contains at least one element */ - def parseTypeRef(src: String): Ast.TypeWithArguments = realParse(src, typeRef) + def parseTypeRef(src: String): Ast.TypeWithArguments = realParse(src, typeRef(_)) - private def realParse[T](src: String, parser: P[T]): T = { - val r = parser.parse(src.trim) + private def realParse[T](src: String, parser: P[_] => P[T]): T = { + val r = fastparse.parse(src.trim, parser) r match { case Parsed.Success(value, _) => value case f: Parsed.Failure => diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala index c60f285de..21c0ad72c 100644 --- a/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala +++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala @@ -1,7 +1,5 @@ package io.kaitai.struct.exprlang -object WsApi extends fastparse.WhitespaceApi.Wrapper(Lexical.wscomment) - /** * Loosely based on /pythonparse/shared/src/main/scala/pythonparse/ * from FastParse, Copyright (c) 2014 Li Haoyi (haoyi.sg@gmail.com) @@ -22,28 +20,30 @@ object WsApi extends fastparse.WhitespaceApi.Wrapper(Lexical.wscomment) * IN THE SOFTWARE. */ object Lexical { - import fastparse.all._ - def kw(s: String) = s ~ !namePart + import fastparse._ + import fastparse.NoWhitespace._ + + def kw[_: P](s: String) = P( s ~ !namePart ) - val wscomment = P( (CharsWhile(" \n".toSet, min = 1) | "\\\n").rep ) + def wscomment[_: P]: P[Unit] = P( (CharsWhile(" \n".toSet, 1) | "\\\n").rep ) - val nameStart = P( letter | "_" ) - val namePart = P( letter | digit | "_" ) - val identifier: P[Ast.identifier] = + def nameStart[_: P] = P( letter | "_" ) + def namePart[_: P] = P( letter | digit | "_" ) + def identifier[_: P]: P[Ast.identifier] = P( nameStart ~ namePart.rep ).!.map(Ast.identifier) - val letter = P( lowercase | uppercase ) - val lowercase = P( CharIn('a' to 'z') ) - val uppercase = P( CharIn('A' to 'Z') ) - val digit = P( CharIn('0' to '9') ) + def letter[_: P] = P( lowercase | uppercase ) + def lowercase[_: P] = P( CharIn("a-z") ) + def uppercase[_: P] = P( CharIn("A-Z") ) + def digit[_: P] = P( CharIn("0-9") ) - val stringliteral: P[String] = P( singlestring | doublestring ) - val singlestring = P("'" ~/ singlestringchar.rep.! ~ "'") - val singlestringchar = P( CharsWhile(!"'".contains(_)) ) + def stringliteral[_: P]: P[String] = P( singlestring | doublestring ) + def singlestring[_: P] = P("'" ~/ singlestringchar.rep.! ~ "'") + def singlestringchar[_: P] = P( CharsWhile(!"'".contains(_)) ) - val doublestring: P[String] = P("\"" ~/ doublestringitem.rep ~ "\"").map(_.mkString) - val doublestringitem = P( doublestringchar.! | escapeseq ) - val doublestringchar = P( CharsWhile(!"\\\"".contains(_)) ) - val escapeseq = P( "\\" ~/ (quotedchar | quotedoctal | quotedhex) ) + def doublestring[_: P]: P[String] = P("\"" ~/ doublestringitem.rep ~ "\"").map(_.mkString) + def doublestringitem[_: P] = P( doublestringchar.! | escapeseq ) + def doublestringchar[_: P] = P( CharsWhile(!"\\\"".contains(_)) ) + def escapeseq[_: P] = P( "\\" ~/ (quotedchar | quotedoctal | quotedhex) ) val QUOTED_CC = Map( "a" -> "\u0007", // bell, ASCII code 7 @@ -58,13 +58,13 @@ object Lexical { "\"" -> "\"", // double quote "\\" -> "\\" // backslash ) - val VALID_QUOTED = QUOTED_CC.keys.toList.sorted.mkString - val quotedchar = P( CharIn(VALID_QUOTED).! ).map(QUOTED_CC) - val quotedoctal: P[String] = P( octdigit.rep(1).! ).map { (digits) => + + def quotedchar[_: P] = P( CharIn("\"'\\abefnrtv").! ).map(QUOTED_CC) + def quotedoctal[_: P]: P[String] = P( octdigit.rep(1).! ).map { (digits) => val code = Integer.parseInt(digits, 8).toChar Character.toString(code) } - val quotedhex: P[String] = P( "u" ~/ hexdigit.rep(exactly = 4).! ).map { (digits) => + def quotedhex[_: P]: P[String] = P( "u" ~/ hexdigit.rep(exactly = 4).! ).map { (digits) => val code = Integer.parseInt(digits, 16).toChar Character.toString(code) } @@ -72,25 +72,25 @@ object Lexical { // probably underscore shouldn't be inside them, but somehow added separately // plus there's a problem with "0x_" and "0o_" being legal now - val integer: P[BigInt] = P( octinteger | hexinteger | bininteger | decimalinteger) - val decimalinteger: P[BigInt] = P( nonzerodigit ~ (digit | "_").rep | "0" ).!.map(parseNum(_, 10)) - val octinteger: P[BigInt] = P( "0" ~ ("o" | "O") ~ octdigit.rep(1).! ).map(parseNum(_, 8)) - val hexinteger: P[BigInt] = P( "0" ~ ("x" | "X") ~ hexdigit.rep(1).! ).map(parseNum(_, 16)) - val bininteger: P[BigInt] = P( "0" ~ ("b" | "B") ~ bindigit.rep(1).! ).map(parseNum(_, 2)) - val nonzerodigit: P0 = P( CharIn('1' to '9') ) - val octdigit: P0 = P( CharIn('0' to '7') | "_" ) - val bindigit: P0 = P( "0" | "1" | "_" ) - val hexdigit: P0 = P( digit | CharIn('a' to 'f', 'A' to 'F') | "_" ) + def integer[_: P]: P[BigInt] = P( octinteger | hexinteger | bininteger | decimalinteger) + def decimalinteger[_: P]: P[BigInt] = P( nonzerodigit ~ (digit | "_").rep | "0" ).!.map(parseNum(_, 10)) + def octinteger[_: P]: P[BigInt] = P( "0" ~ ("o" | "O") ~ octdigit.rep(1).! ).map(parseNum(_, 8)) + def hexinteger[_: P]: P[BigInt] = P( "0" ~ ("x" | "X") ~ hexdigit.rep(1).! ).map(parseNum(_, 16)) + def bininteger[_: P]: P[BigInt] = P( "0" ~ ("b" | "B") ~ bindigit.rep(1).! ).map(parseNum(_, 2)) + def nonzerodigit[_: P]: P0 = P( CharIn("1-9") ) + def octdigit[_: P]: P0 = P( CharIn("0-7") | "_" ) + def bindigit[_: P]: P0 = P( "0" | "1" | "_" ) + def hexdigit[_: P]: P0 = P( digit | CharIn("a-fA-F") | "_" ) - val floatnumber: P[BigDecimal] = P( + def floatnumber[_: P]: P[BigDecimal] = P( digit.rep(1) ~ exponent // Ex.: 4E2, 4E+2, 4e-2 | fixed ~ exponent.? // Ex.: 4.E2, .4e+2, 4.2e-0 ).!.map(BigDecimal(_)) - val fixed = P( + def fixed[_: P] = P( digit.rep ~ "." ~ digit.rep(1) // Ex.: 4.2, .42 | digit.rep(1) ~ "." ~ !(wscomment ~ nameStart) // Ex.: 42., but not '42.abc' or '42. def' ) - val exponent: P0 = P( ("e" | "E") ~ ("+" | "-").? ~ digit.rep(1) ) + def exponent[_: P]: P0 = P( ("e" | "E") ~ ("+" | "-").? ~ digit.rep(1) ) /** * Converts number literal from string form into BigInt, ignoring underscores that might be diff --git a/shared/src/main/scala/io/kaitai/struct/problems/CompilationProblem.scala b/shared/src/main/scala/io/kaitai/struct/problems/CompilationProblem.scala index d7ae7ee73..25ce8b445 100644 --- a/shared/src/main/scala/io/kaitai/struct/problems/CompilationProblem.scala +++ b/shared/src/main/scala/io/kaitai/struct/problems/CompilationProblem.scala @@ -1,6 +1,5 @@ package io.kaitai.struct.problems -import fastparse.StringReprOps import io.kaitai.struct.{JSON, Jsonable, Utils, problems} import io.kaitai.struct.datatype.DataType import io.kaitai.struct.exprlang.Expressions @@ -101,7 +100,7 @@ object KSYParseError { def expression(epe: Expressions.ParseException, path: List[String]) = { val f = epe.failure - val pos = StringReprOps.prettyIndex(f.extra.input, f.index) + val pos = f.extra.input.prettyIndex(f.index) // Try to diagnose most common errors and provide a friendly suggestion val lookup2 = Utils.safeLookup(epe.src, f.index, 2) @@ -113,11 +112,11 @@ object KSYParseError { None }).map((x) => s", did you mean '$x'?").getOrElse("") - f.extra.traced.expected + val expected = f.extra.trace().msg.replaceAll("\n", "\\n") withText( s"parsing expression '${epe.src}' failed on $pos, " + - s"expected ${f.extra.traced.expected.replaceAll("\n", "\\n")}$suggestion", + s"expected $expected$suggestion", path ) } From 92373c0db37c917ed618c30265f9776d7952c89d Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Dec 2020 22:20:25 +0500 Subject: [PATCH 03/12] Upgrade ScalaTest to the latest version (3.2.12->3.2.15) with sbt-scalafix Changes: https://www.scalatest.org/release_notes/3.2.13 https://www.scalatest.org/release_notes/3.2.14 https://www.scalatest.org/release_notes/3.2.15 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index a40c7f5d6..35c2468b0 100644 --- a/build.sbt +++ b/build.sbt @@ -80,7 +80,7 @@ lazy val compiler = crossProject.in(file(".")). Compile / mainClass := Some("io.kaitai.struct.JavaMain"), libraryDependencies ++= Seq( - "org.scalatest" %% "scalatest" % "3.2.12" % "test" + "org.scalatest" %% "scalatest" % "3.2.15" % "test" ), Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-u", "target/test_out"), From 7b806ded4ffcf5aa699dc0c95c7e742f18087d22 Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Dec 2020 22:42:25 +0500 Subject: [PATCH 04/12] Depend only on used ScalaTest modules: - scalatest-funspec - scalatest-funsuite - scalatest-shouldmatchers --- build.sbt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 35c2468b0..741caee42 100644 --- a/build.sbt +++ b/build.sbt @@ -80,7 +80,9 @@ lazy val compiler = crossProject.in(file(".")). Compile / mainClass := Some("io.kaitai.struct.JavaMain"), libraryDependencies ++= Seq( - "org.scalatest" %% "scalatest" % "3.2.15" % "test" + "org.scalatest" %% "scalatest-funspec" % "3.2.15" % "test", + "org.scalatest" %% "scalatest-funsuite" % "3.2.15" % "test", + "org.scalatest" %% "scalatest-shouldmatchers" % "3.2.15" % "test", ), Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-u", "target/test_out"), From d07272b81c7e9dd346b44972bafcf6b7d6d034e8 Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Dec 2020 23:14:28 +0500 Subject: [PATCH 05/12] Upgrade sbt-native-packager plugin (1.3.12->1.9.16) Changes: https://github.com/sbt/sbt-native-packager/compare/v1.3.12...v1.9.16 Organization name changed in the 1.9.0: com.typesafe.sbt -> com.github.sbt Java 8 is required since 1.9.12 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index c1cb6dfe9..7fe21292c 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ logLevel := Level.Warn -addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.12") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16") addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.33") From 0cc4fb3716dccf5b20918ca4708d8f6757a95543 Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Dec 2020 23:39:44 +0500 Subject: [PATCH 06/12] Use sbt-crossproject instead of deprecated crossProjectFromBuilder as warning suggests --- build.sbt | 5 ++++- project/plugins.sbt | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 741caee42..9c03a2875 100644 --- a/build.sbt +++ b/build.sbt @@ -4,6 +4,8 @@ import java.nio.file.Files import com.typesafe.sbt.packager.linux.{LinuxPackageMapping, LinuxSymlink} import sbt.Keys._ +// shadow sbt-scalajs' crossProject and CrossType from Scala.js 0.6.x +import sbtcrossproject.CrossPlugin.autoImport.{crossProject, CrossType} resolvers ++= Resolver.sonatypeOssRepos("public") @@ -19,7 +21,8 @@ lazy val root = project.in(file(".")). publishLocal := {} ) -lazy val compiler = crossProject.in(file(".")). +lazy val compiler = crossProject(JSPlatform, JVMPlatform). + in(file(".")). enablePlugins(JavaAppPackaging). settings( organization := "io.kaitai", diff --git a/project/plugins.sbt b/project/plugins.sbt index 7fe21292c..f5a8225de 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,3 +2,4 @@ logLevel := Level.Warn addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16") addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.33") +addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.0.0") \ No newline at end of file From 6b03028bf33cd72f3926dc6276fb09ccbbcb1cc0 Mon Sep 17 00:00:00 2001 From: Mingun Date: Sun, 6 Dec 2020 00:04:52 +0500 Subject: [PATCH 07/12] Upgrade scopt (3.6.0->4.1.0) and scala-js plugin (0.6.33->1.13.1) There dependencies depends on each other so updated simultaneously scala-js 0.6.x no longer maintained: http://www.scala-js.org/doc/internals/scalajs-0.6.x-eol.html scopt changelog: https://github.com/scopt/scopt/compare/v3.6.0...v4.1.0 scala-js changelog: https://www.scala-js.org/news/index.html - https://www.scala-js.org/news/2020/02/25/announcing-scalajs-1.0.0/ - https://www.scala-js.org/news/2020/03/10/announcing-scalajs-1.0.1/ - https://www.scala-js.org/news/2020/05/18/announcing-scalajs-1.1.0/ - https://www.scala-js.org/news/2020/07/02/announcing-scalajs-1.1.1/ - https://www.scala-js.org/news/2020/09/09/announcing-scalajs-1.2.0/ - https://www.scala-js.org/news/2020/10/16/announcing-scalajs-1.3.0/ - https://www.scala-js.org/news/2020/11/16/announcing-scalajs-1.3.1/ - https://www.scala-js.org/news/2021/01/12/announcing-scalajs-1.4.0/ - https://www.scala-js.org/news/2021/02/12/announcing-scalajs-1.5.0/ (Maven Central) - https://www.scala-js.org/news/2021/04/01/announcing-scalajs-1.5.1/ - https://www.scala-js.org/news/2021/06/09/announcing-scalajs-1.6.0/ (Standard Scala Library 2.12.13) - https://www.scala-js.org/news/2021/08/04/announcing-scalajs-1.7.0/ (Standard Scala Library 2.12.14) - https://www.scala-js.org/news/2021/08/04/announcing-scalajs-1.7.1/ (Standard Scala Library 2.12.15) - https://www.scala-js.org/news/2021/08/04/announcing-scalajs-1.8.0/ (Node.js 17) - https://www.scala-js.org/news/2021/08/04/announcing-scalajs-1.9.0/ (strict floats) - https://www.scala-js.org/news/2021/08/04/announcing-scalajs-1.10.0/ (address security vulnerability in java.util.UUID.randomUUID()) - https://www.scala-js.org/news/2021/08/04/announcing-scalajs-1.10.1/ (Standard Scala Library 2.12.16) - https://www.scala-js.org/news/2022/09/15/announcing-scalajs-1.11.0/ - https://www.scala-js.org/news/2022/11/23/announcing-scalajs-1.12.0/ (Standard Scala Library 2.12.17) - https://www.scala-js.org/news/2023/01/26/announcing-scalajs-1.13.0/ (Drop support Scala 2.11 and 2.12.1) - https://www.scala-js.org/news/2023/04/10/announcing-scalajs-1.13.1/ scala-js 1.x prerequisites: - Scala at least 2.11.12, 2.12.1 or 2.13.0 OK, current compiler is 2.12.12 - Scala-js at least 0.6.33 to address all deprecation warnings OK, upgrade from 0.6.33, no deprecation warnings - Usage of sbt-crossproject OK, upgraded in the previous commit - Usage of `scalaJSLinkerConfig` instead of couple of other settings OK, none of these settings are used - sbt 1.2.1 or later OK, current sbt is 1.7.1 scala-js artifacts hosted on the Maven Central since 1.5.0 --- build.sbt | 4 +--- jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 2 +- project/plugins.sbt | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/build.sbt b/build.sbt index 9c03a2875..8fce54b2c 100644 --- a/build.sbt +++ b/build.sbt @@ -4,8 +4,6 @@ import java.nio.file.Files import com.typesafe.sbt.packager.linux.{LinuxPackageMapping, LinuxSymlink} import sbt.Keys._ -// shadow sbt-scalajs' crossProject and CrossType from Scala.js 0.6.x -import sbtcrossproject.CrossPlugin.autoImport.{crossProject, CrossType} resolvers ++= Resolver.sonatypeOssRepos("public") @@ -73,7 +71,7 @@ lazy val compiler = crossProject(JSPlatform, JVMPlatform). Compile / sourceGenerators += generateVersionTask.taskValue, // update automatically on every rebuild libraryDependencies ++= Seq( - "com.github.scopt" %%% "scopt" % "3.6.0", + "com.github.scopt" %%% "scopt" % "4.1.0", "com.lihaoyi" %%% "fastparse" % "2.3.3", "org.yaml" % "snakeyaml" % "1.28" ) diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala index 2e0c6b57a..a8d896443 100644 --- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala +++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala @@ -31,7 +31,7 @@ object JavaMain { def parseCommandLine(args: Array[String]): Option[CLIConfig] = { val parser = new scopt.OptionParser[CLIConfig](Version.name) { - override def showUsageOnError = true + override def showUsageOnError = Some(true) head(Version.name, Version.version) diff --git a/project/plugins.sbt b/project/plugins.sbt index f5a8225de..ef64d7382 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,5 +1,5 @@ logLevel := Level.Warn addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16") -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.33") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.13.1") addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.0.0") \ No newline at end of file From 6ba2dc9a3f3e7c4066b278baa789caf67275af8f Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 20 Nov 2020 08:58:43 +0500 Subject: [PATCH 08/12] Upgrade SnakeYAML (1.28->2.0) Changes: - https://bitbucket.org/snakeyaml/snakeyaml/wiki/Changes - https://bitbucket.org/snakeyaml/snakeyaml/compare/snakeyaml-2.0%0Dsnakeyaml-1.28 Java 11 is required to build since 2.0 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 8fce54b2c..af093f451 100644 --- a/build.sbt +++ b/build.sbt @@ -73,7 +73,7 @@ lazy val compiler = crossProject(JSPlatform, JVMPlatform). libraryDependencies ++= Seq( "com.github.scopt" %%% "scopt" % "4.1.0", "com.lihaoyi" %%% "fastparse" % "2.3.3", - "org.yaml" % "snakeyaml" % "1.28" + "org.yaml" % "snakeyaml" % "2.0" ) ). jvmSettings( From 086d68749c71f674be0e6f8d4b742d8ea732cbb8 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 21 Oct 2023 20:50:35 +0200 Subject: [PATCH 09/12] project/plugins.sbt: add missing final newline --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index ef64d7382..5721e248e 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,4 +2,4 @@ logLevel := Level.Warn addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16") addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.13.1") -addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.0.0") \ No newline at end of file +addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.0.0") From cfc37f5a2f8e9de3094b0e8090a8d3a9b6137f09 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 21 Oct 2023 21:30:57 +0200 Subject: [PATCH 10/12] Adapt `snakeyaml.Yaml` constructor invocation to SnakeYAML 2.0 Note that we're not reducing security by no longer explicitly using `SafeConstructor`, as this is the default since SnakeYAML 2.0. See the references below: 1. https://snyk.io/blog/snakeyaml-unsafe-deserialization-vulnerability/ > SnakeYaml 2.0 was released in early 2023 to mitigate the default > behavior that can lead to possible arbitrary code execution. In > this version, the constructor that every new yaml() uses now > extends SafeConstructor. 2. https://www.veracode.com/blog/research/resolving-cve-2022-1471-snakeyaml-20-release-0 > Since 2.0, the SnakeYAML `Constructor` now inherits from > `SafeConstructor`, which prevents an attacker from accessing the > classpath by default. --- .../io/kaitai/struct/formats/JavaKSYParser.scala | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala b/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala index b514c00fc..ed1d0aecf 100644 --- a/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala +++ b/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala @@ -7,10 +7,8 @@ import io.kaitai.struct.JavaMain.CLIConfig import io.kaitai.struct.format.{ClassSpec, ClassSpecs} import io.kaitai.struct.problems.{CompilationProblem, CompilationProblemException, ProblemCoords, ProblemSeverity, YAMLParserError} import io.kaitai.struct.{Log, Main} -import org.yaml.snakeyaml.constructor.SafeConstructor import org.yaml.snakeyaml.error.MarkedYAMLException -import org.yaml.snakeyaml.representer.Representer -import org.yaml.snakeyaml.{DumperOptions, LoaderOptions, Yaml} +import org.yaml.snakeyaml.{LoaderOptions, Yaml} import scala.collection.JavaConverters._ import scala.concurrent.Await @@ -71,12 +69,7 @@ object JavaKSYParser { def getYamlLoader: Yaml = { val loaderOptions = new LoaderOptions loaderOptions.setAllowDuplicateKeys(false) - new Yaml( - new SafeConstructor, - new Representer, - new DumperOptions, - loaderOptions - ) + new Yaml(loaderOptions) } def readerToYaml(reader: Reader): Any = { From c28a56cc7f89abb3f426375d3c013a2c5f9b537b Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 22 Oct 2023 17:14:25 +0200 Subject: [PATCH 11/12] Expressions: fix unrecognized backslash in string literal Thankfully, this bug was revealed by our test StrLiterals (https://github.com/kaitai-io/kaitai_struct_tests/blob/3b242b185c15ea4e347711bc2eb7fe9d86b97557/formats/str_literals.ksy#L8-L9): ```ksy backslashes: value: '"\\\u005c\134"' ``` --- .../main/scala/io/kaitai/struct/exprlang/Lexical.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala index 21c0ad72c..2d2ed0c20 100644 --- a/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala +++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala @@ -59,7 +59,13 @@ object Lexical { "\\" -> "\\" // backslash ) - def quotedchar[_: P] = P( CharIn("\"'\\abefnrtv").! ).map(QUOTED_CC) + // Note: `CharIn("\\\\")` is necessary to include a single literal backslash, + // because the contents of `CharIn` are translated to a regex character class + // `[...]` (and as in regexes, ranges like `a-z` are also supported, etc.). + // Therefore, to match either `+` or `-` literally, you would need + // `CharIn("+\\-")`; consequently, a literal backslash is `CharIn("\\\\")`. + def quotedchar[_: P] = P( CharIn("\"'\\\\abefnrtv").! ).map(QUOTED_CC) + def quotedoctal[_: P]: P[String] = P( octdigit.rep(1).! ).map { (digits) => val code = Integer.parseInt(digits, 8).toChar Character.toString(code) From 0fe8b698e53c821a2e3e77046ce67ae6830a7094 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 22 Oct 2023 20:10:36 +0200 Subject: [PATCH 12/12] Adapt `buildNpmJsFile` task to the new Scala.js 1.x output See https://github.com/kaitai-io/kaitai_struct_compiler/pull/230#discussion_r1367945494 --- build.sbt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index af093f451..dc1792883 100644 --- a/build.sbt +++ b/build.sbt @@ -231,13 +231,8 @@ lazy val buildNpmJsFileTask = Def.task { | } |}(typeof self !== 'undefined' ? self : this, function () { | - |var exports = {}; - |var __ScalaJSEnv = { exportsNamespace: exports }; - | |$compiledFileContents - | - |return exports.MainJs; - | + |return MainJs; |})); |""".stripMargin