From 22db213b185f3568a175c88a2a2a553a9e4ab336 Mon Sep 17 00:00:00 2001 From: Sergei Winitzki Date: Fri, 12 Jul 2024 10:40:54 +0200 Subject: [PATCH] Feature/yaml perftest (#34) * add perftest files * add a perf test * start on nano-dhall * wip perftest * add perftest * Add memoization to grammar. * add sourcecode 0.3.0 dependency * Yaml perf test down to 4 seconds * use Memoize.parse in tests * wip * fixes * add test for LRUHashDict * fixes * add test file * update readme * update version to 0.2.1 * rename artifacts --- README.md | 10 + build.sbt | 102 +++- .../scala/io/chymyst/fastparse/Memoize.scala | 122 +++++ .../chymyst/fastparse/unit/MemoizeTest.scala | 60 +++ .../io/chymyst/nanodhall/LRUHashDict.scala | 25 + nano-dhall/src/test/resources/1.dhall | 1 + .../nanodhall/unit/LRUHashDictTest.scala | 19 + .../main/scala/io/chymyst/dhall/Main.scala | 10 +- .../test/resources/yaml-perftest/common.dhall | 89 ++++ .../resources/yaml-perftest/create_yaml.dhall | 8 + .../resources/yaml-perftest/create_yaml.yaml | 41 ++ .../yaml-perftest/get_item_name.dhall | 7 + .../test/resources/yaml-perftest/items.dhall | 138 ++++++ .../yaml-perftest/largeExpressionA.dhall | 272 +++++++++++ .../resources/yaml-perftest/renderAs.dhall | 458 ++++++++++++++++++ .../test/resources/yaml-perftest/schema.dhall | 145 ++++++ .../resources/yaml-perftest/yaml_record.dhall | 52 ++ .../io/chymyst/dhall/unit/PerfTest.scala | 64 +++ .../main/scala/io/chymyst/dhall/Grammar.scala | 29 +- .../io/chymyst/dhall/ImportResolution.scala | 7 +- .../main/scala/io/chymyst/dhall/Parser.scala | 19 +- .../dhall/unit/DhallParserAndCbor1Suite.scala | 2 +- .../dhall/unit/DhallParserAndCbor2Suite.scala | 6 +- .../chymyst/dhall/unit/DoNotationTest.scala | 6 + .../io/chymyst/dhall/unit/ParserTest.scala | 3 +- .../dhall/unit/SimpleExpressionTest.scala | 16 +- .../io/chymyst/dhall/unit/TestUtils.scala | 3 +- 27 files changed, 1665 insertions(+), 49 deletions(-) create mode 100644 fastparse-memoize/src/main/scala/io/chymyst/fastparse/Memoize.scala create mode 100644 fastparse-memoize/src/test/scala/io/chymyst/fastparse/unit/MemoizeTest.scala create mode 100644 nano-dhall/src/main/scala/io/chymyst/nanodhall/LRUHashDict.scala create mode 100644 nano-dhall/src/test/resources/1.dhall create mode 100644 nano-dhall/src/test/scala/io/chymyst/nanodhall/unit/LRUHashDictTest.scala create mode 100644 scall-cli/src/test/resources/yaml-perftest/common.dhall create mode 100644 scall-cli/src/test/resources/yaml-perftest/create_yaml.dhall create mode 100644 scall-cli/src/test/resources/yaml-perftest/create_yaml.yaml create mode 100644 scall-cli/src/test/resources/yaml-perftest/get_item_name.dhall create mode 100644 scall-cli/src/test/resources/yaml-perftest/items.dhall create mode 100644 scall-cli/src/test/resources/yaml-perftest/largeExpressionA.dhall create mode 100644 scall-cli/src/test/resources/yaml-perftest/renderAs.dhall create mode 100644 scall-cli/src/test/resources/yaml-perftest/schema.dhall create mode 100644 scall-cli/src/test/resources/yaml-perftest/yaml_record.dhall create mode 100644 scall-cli/src/test/scala/io/chymyst/dhall/unit/PerfTest.scala diff --git a/README.md b/README.md index 919b7a00..faf0d0e6 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,9 @@ Another feature is that some parses need to fail for others to succeed. For exam identifier. However, `missing` is a keyword and is matched first. To ensure correct parsing, negative lookahead is used for keywords. +To improve parsing performance, the parsing results for some sub-expressions are memoized. +This is implemented via an add-on library `fastparse-memoize`. + #### Limitations So far, there are some issues with the Unicode characters: @@ -238,8 +241,15 @@ So far, there are some issues with the Unicode characters: # Release version history + +## 0.2.1 + +- Implemented `fastparse-memoize` to speed up parsing (by 10x and more in some cases). +- Upgrade to fastparse 3.1.x + ## 0.2.0 +- First version published on Sonatype - Fixed the regression described in https://github.com/dhall-lang/dhall-haskell/issues/2597 - Support for Yaml and JSON export - Standalone JAR executable `dhall.jar` with command-line options similar to `dhall-haskell` diff --git a/build.sbt b/build.sbt index 9a83b0db..63dcb97e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,11 +1,9 @@ -import sbt.Keys.{developers, homepage, scmInfo} +import sbt.Keys.homepage import sbt.url import sbtassembly.AssemblyKeys.assembly -import xerial.sbt.Sonatype.{GitHubHosting, sonatypeCentralHost} +import xerial.sbt.Sonatype.GitHubHosting -import scala.collection.immutable.List - -val thisReleaseVersion = "0.2.0" +val thisReleaseVersion = "0.2.1" val scala2V = "2.13.13" val scala212V = "2.12.19" @@ -18,7 +16,7 @@ def munitFramework = new TestFramework("munit.Framework") val munitTest = "org.scalameta" %% "munit" % "0.7.29" % Test val assertVerboseTest = "com.eed3si9n.expecty" %% "expecty" % "0.16.0" % Test -val fastparse = "com.lihaoyi" %% "fastparse" % "3.0.2" +val fastparse = "com.lihaoyi" %% "fastparse" % "3.1.1" val antlr4 = "org.antlr" % "antlr4-runtime" % "4.13.1" val anltr4_formatter = "com.khubla.antlr4formatter" % "antlr4-formatter-standalone" % "1.2.1" % Provided @@ -34,6 +32,7 @@ val cbor1 = "co.nstant.in" % "cbor" % "0.9" val cbor2 = "com.upokecenter" % "cbor" % "4.5.3" val reflections = "org.reflections" % "reflections" % "0.10.2" val mainargs = "com.lihaoyi" %% "mainargs" % "0.7.0" +val sourcecode = "com.lihaoyi" %% "sourcecode" % "0.4.2" // Not used now: val flatlaf = "com.formdev" % "flatlaf" % "3.2.2" @@ -53,9 +52,6 @@ lazy val publishingOptions = Seq( description := "Implementation of the Dhall language in Scala, with Scala language bindings", publishTo := sonatypePublishToBundle.value, sonatypeProjectHosting := Some(GitHubHosting("winitzki", "scall", "winitzki@gmail.com")), -// homepage := Some(url("https://github.com/winitzki/scall")), -// scmInfo := Some(ScmInfo(url("https://github.com/winitzki/scall"), "scm:git@github.com:winitzki/scall.git")), -// developers := List(Developer(id = "winitzki", name = "Sergei Winitzki", email = "winitzki@gmail.com", url = url("https://sites.google.com/site/winitzki"))), ) lazy val noPublishing = @@ -71,11 +67,72 @@ lazy val jdkModuleOptions: Seq[String] = { lazy val root = (project in file(".")) .settings(noPublishing) .settings(scalaVersion := scalaV, crossScalaVersions := Seq(scalaV), name := "scall-root") - .aggregate(scall_core, scall_testutils, dhall_codec, abnf, scall_macros, scall_typeclasses, scall_cli) + .aggregate(scall_core, scall_testutils, dhall_codec, abnf, scall_macros, scall_typeclasses, scall_cli, nano_dhall, fastparse_memoize) + +lazy val nano_dhall = (project in file("nano-dhall")) // This is a POC project. + .settings(noPublishing) + .settings( + name := "nano-dhall", + scalaVersion := scalaV, + crossScalaVersions := supportedScalaVersions, + Test / parallelExecution := true, + Test / fork := true, + coverageEnabled := false, + scalafmtFailOnErrors := false, // Cannot disable the unicode surrogate pair error in Parser.scala? + testFrameworks += munitFramework, + Test / javaOptions ++= jdkModuleOptions, + Compile / scalacOptions ++= { + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((3, _)) => Seq("-Ydebug") + case Some((2, 12 | 13)) => Seq("-Ypatmat-exhaust-depth", "10") // Cannot make it smaller than 10. Want to speed up compilation. + } + }, + ThisBuild / scalacOptions ++= { + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((3, _)) => Seq("-Ykind-projector") // Seq("-Ykind-projector:underscores") + case Some((2, 12 | 13)) => Seq() // Seq("-Xsource:3", "-P:kind-projector:underscore-placeholders") + } + }, + // We need to run tests in forked JVM starting with the current directory set to the base resource directory. + // That base directory should contain `./dhall-lang` and all files below that. + Test / baseDirectory := (Test / resourceDirectory).value, + // addCompilerPlugin is a shortcut for libraryDependencies += compilerPlugin(dependency) + // See https://stackoverflow.com/questions/67579041 + libraryDependencies ++= + (CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, _)) => Seq(scala_reflect(scalaVersion.value), kindProjectorPlugin) + case Some((3, _)) => Seq.empty // No need for scala-reflect with Scala 3. + }), + libraryDependencies ++= Seq( + fastparse, + antlr4, + anltr4_formatter, + munitTest, + assertVerboseTest, + enumeratum, + cbor2, + // scalahashing, + // cbor3, + httpRequest, + os_lib % Test, + ), + ).dependsOn(scall_testutils % "test->compile", scall_typeclasses, fastparse_memoize) + +lazy val fastparse_memoize = (project in file("fastparse-memoize")) + .settings(publishingOptions) + .settings( + name := "fastparse-memoize", + scalaVersion := scalaV, + crossScalaVersions := supportedScalaVersions, + testFrameworks += munitFramework, + Test / javaOptions ++= jdkModuleOptions, + libraryDependencies ++= Seq(fastparse, sourcecode, munitTest, assertVerboseTest), + ).dependsOn(scall_testutils % "test->compile") lazy val scall_core = (project in file("scall-core")) .settings(publishingOptions) .settings( + name := "dhall-scala-core", scalaVersion := scalaV, crossScalaVersions := supportedScalaVersions, Test / parallelExecution := true, @@ -119,11 +176,12 @@ lazy val scall_core = (project in file("scall-core")) httpRequest, os_lib % Test, ), - ).dependsOn(scall_testutils % "test->compile", scall_typeclasses) + ).dependsOn(scall_testutils % "test->compile", scall_typeclasses, fastparse_memoize) lazy val scall_testutils = (project in file("scall-testutils")) .settings(publishingOptions) .settings( + name := "dhall-scala-testutils", scalaVersion := scalaV, crossScalaVersions := supportedScalaVersions, Test / parallelExecution := true, @@ -136,6 +194,7 @@ lazy val scall_testutils = (project in file("scall-testutils")) lazy val dhall_codec = (project in file("dhall-codec")) .settings(publishingOptions) .settings( + name := "dhall-scala-bindings", scalaVersion := scalaV, crossScalaVersions := supportedScalaVersions, Test / parallelExecution := true, @@ -155,6 +214,7 @@ lazy val dhall_codec = (project in file("dhall-codec")) lazy val scall_cli = (project in file("scall-cli")) .settings(publishingOptions) .settings( + name := "dhall-scala-cli", scalaVersion := scalaV, crossScalaVersions := supportedScalaVersions, Test / parallelExecution := true, @@ -178,7 +238,7 @@ lazy val scall_cli = (project in file("scall-cli")) lazy val abnf = (project in file("abnf")) .settings(noPublishing) .settings( - name := "scall-abnf", + name := "dhall-scala-abnf", scalaVersion := scalaV, crossScalaVersions := supportedScalaVersions, Test / parallelExecution := true, @@ -189,7 +249,7 @@ lazy val abnf = (project in file("abnf")) lazy val scall_macros = (project in file("scall-macros")) .settings(publishingOptions) .settings( - name := "scall-macros", + name := "dhall-scala-macros", scalaVersion := scalaV, crossScalaVersions := supportedScalaVersions, Test / parallelExecution := true, @@ -205,7 +265,7 @@ lazy val scall_macros = (project in file("scall-macros")) lazy val scall_typeclasses = (project in file("scall-typeclasses")) .settings(publishingOptions) .settings( - name := "scall-typeclasses", + name := "dhall-scala-typeclasses", scalaVersion := scalaV, crossScalaVersions := supportedScalaVersions, Test / parallelExecution := true, @@ -220,18 +280,10 @@ lazy val scall_typeclasses = (project in file("scall-typeclasses")) ///////////////////////////////////////////////////////////////////////////////////////////////////// // Publishing to Sonatype Maven repository -publishMavenStyle := true -publishTo := sonatypePublishToBundle.value -sonatypeProfileName := "io.chymyst" +publishMavenStyle := true +publishTo := sonatypePublishToBundle.value +sonatypeProfileName := "io.chymyst" //ThisBuild / sonatypeCredentialHost := sonatypeCentralHost // Not relevant because io.chymyst was created before 2021. - -/*{ - val nexus = "https://oss.sonatype.org/" - if (isSnapshot.value) - Some("snapshots" at nexus + "content/repositories/snapshots") - else - Some("releases" at nexus + "service/local/staging/deploy/maven2") -}*/ // Test / publishArtifact := false // diff --git a/fastparse-memoize/src/main/scala/io/chymyst/fastparse/Memoize.scala b/fastparse-memoize/src/main/scala/io/chymyst/fastparse/Memoize.scala new file mode 100644 index 00000000..fd3c6710 --- /dev/null +++ b/fastparse-memoize/src/main/scala/io/chymyst/fastparse/Memoize.scala @@ -0,0 +1,122 @@ +package io.chymyst.fastparse + +import fastparse.{P, Parsed, ParserInput, ParserInputSource, ParsingRun} +import fastparse.internal.{Instrument, Msgs} + +import scala.collection.mutable + +/* See discussion in https://github.com/com-lihaoyi/fastparse/discussions/301 */ + +final case class PRunData( // Copy all the mutable data from ParsingRun. + terminalMsgs: Msgs, + aggregateMsgs: Msgs, + shortMsg: Msgs, + lastFailureMsg: Msgs, + failureStack: List[(String, Int)], + isSuccess: Boolean, + logDepth: Int, + index: Int, + cut: Boolean, + successValue: Any, + verboseFailures: Boolean, + noDropBuffer: Boolean, + misc: collection.mutable.Map[Any, Any], +) { + override def toString: String = { + s"ParsingRun(index=$index, isSuccess = $isSuccess, successValue = $successValue)" + } + +} + +object PRunData { // Copy all the mutable data from a parsing run into a PRunData value. + def ofParsingRun[T](pr: ParsingRun[T]): PRunData = PRunData( + pr.terminalMsgs, + pr.aggregateMsgs, + pr.shortMsg, + pr.lastFailureMsg, + pr.failureStack, + pr.isSuccess, + pr.logDepth, + pr.index, + pr.cut, + pr.successValue, + pr.verboseFailures, + pr.noDropBuffer, + mutable.Map.from(pr.misc), + ) +} + +object Memoize { + val enable = true + + def assignToParsingRun[T](data: PRunData, pr: ParsingRun[T]): ParsingRun[T] = { // Assign the mutable data to a given ParsingRun value. + pr.terminalMsgs = data.terminalMsgs + pr.aggregateMsgs = data.aggregateMsgs + pr.shortMsg = data.shortMsg + pr.lastFailureMsg = data.lastFailureMsg + pr.failureStack = data.failureStack + pr.isSuccess = data.isSuccess + pr.logDepth = data.logDepth + pr.index = data.index + pr.cut = data.cut + pr.successValue = data.successValue + pr.verboseFailures = data.verboseFailures + pr.noDropBuffer = data.noDropBuffer + data.misc.foreach { case (k, v) => pr.misc.put(k, v) } + pr + } + + @inline private def cacheGrammar[R](cache: mutable.Map[Int, PRunData], parser: => P[_])(implicit p: P[_]): P[R] = { + // The `parser` has not yet been run! And it is mutable. Do not run it twice! + val cachedData: PRunData = cache.getOrElseUpdate(p.index, PRunData.ofParsingRun(parser)) + // After the `parser` has been run on `p`, the value of `p` changes and becomes equal to the result of running the parser. + // If the result was cached, we need to assign it to the current value of `p`. This will imitate the side effect of running the parser again. + assignToParsingRun(cachedData, p).asInstanceOf[P[R]] + } + + private val cache = new mutable.HashMap[(sourcecode.File, sourcecode.Line), mutable.Map[Int, PRunData]] + + private def getOrCreateCache(file: sourcecode.File, line: sourcecode.Line): mutable.Map[Int, PRunData] = { + cache.getOrElseUpdate((file, line), new mutable.HashMap[Int, PRunData]) + } + + implicit class MemoizeParser[A](parser: => P[A]) { + @inline def memoize(implicit file: sourcecode.File, line: sourcecode.Line, p: P[_]): P[A] = if (enable) { + val cache: mutable.Map[Int, PRunData] = getOrCreateCache(file, line) + cacheGrammar(cache, parser) + } else parser + } + + def clearAll(): Unit = cache.values.foreach(_.clear()) + + def statistics: String = cache.map { case ((file, line), c) => s"$file#$line: ${c.size} entries" }.mkString("\n") + + def parse[T]( + input: ParserInputSource, + parser: P[_] => P[T], + verboseFailures: Boolean = false, + startIndex: Int = 0, + instrument: Instrument = null, + ): Parsed[T] = { + clearAll() + val result = fastparse.parse(input, parser, verboseFailures, startIndex, instrument) + clearAll() + result + } + + def parseInputRaw[T]( + input: ParserInput, + parser: P[_] => P[T], + verboseFailures: Boolean = false, + startIndex: Int = 0, + traceIndex: Int = -1, + instrument: Instrument = null, + enableLogging: Boolean = true, + ): ParsingRun[T] = { + clearAll() + val result = fastparse.parseInputRaw(input, parser, verboseFailures, startIndex, traceIndex, instrument, enableLogging) + clearAll() + result + } + +} diff --git a/fastparse-memoize/src/test/scala/io/chymyst/fastparse/unit/MemoizeTest.scala b/fastparse-memoize/src/test/scala/io/chymyst/fastparse/unit/MemoizeTest.scala new file mode 100644 index 00000000..2169364e --- /dev/null +++ b/fastparse-memoize/src/test/scala/io/chymyst/fastparse/unit/MemoizeTest.scala @@ -0,0 +1,60 @@ +package io.chymyst.fastparse.unit + +import com.eed3si9n.expecty.Expecty.expect +import fastparse.NoWhitespace._ +import fastparse._ +import io.chymyst.fastparse.Memoize +import io.chymyst.test.TestTimings +import munit.FunSuite + +class MemoizeTest extends FunSuite with TestTimings { + + test("slow grammar becomes faster after memoization") { + // Integer calculator program: 1+2*3-(4-5)*6 and so on. No spaces, for simplicity. + def program1[$: P]: P[Int] = P(expr1 ~ End) + def expr1[$: P]: P[Int] = P(minus1 | plus1) + def minus1[$: P] = P(times1 ~ "-" ~ expr1).map { case (x, y) => x - y } + def plus1[$: P] = P(times1 ~ ("+" ~ expr1).rep).map { case (i, is) => i + is.sum } + def times1[$: P] = P(other1 ~ ("*" ~ other1).rep).map { case (i, is) => i * is.product } + def other1[$: P]: P[Int] = P(number | ("(" ~ expr1 ~ ")")) + def number[$: P] = P(CharIn("0-9").rep(1)).!.map(_.toInt) + // Verify that this works as expected. + assert(fastparse.parse("123*(1+1)", program1(_)).get.value == 246) + assert(fastparse.parse("123*1+1", program1(_)).get.value == 124) + assert(fastparse.parse("123*1-1", program1(_)).get.value == 122) + assert(fastparse.parse("123*(1-1)", program1(_)).get.value == 0) + + // Parse an expression of the form `(((((...(1)...)))))`. + val n = 23 + val (result1, elapsed1) = elapsedNanos(fastparse.parse("(" * (n - 1) + "1" + ")" * (n - 1), program1(_))) + assert(result1.get.value == 1) + + // The same parsing after memoization. + import io.chymyst.fastparse.Memoize.MemoizeParser + def program2[$: P]: P[Int] = P(expr2 ~ End) + def expr2[$: P]: P[Int] = P(minus2 | plus2) + def minus2[$: P] = P(times2 ~ "-" ~ expr2).map { case (x, y) => x - y } + def plus2[$: P] = P(times2 ~ ("+" ~ expr2).rep).map { case (i, is) => i + is.sum } + def times2[$: P] = P(other2 ~ ("*" ~ other2).rep).map { case (i, is) => i * is.product } + def other2[$: P]: P[Int] = P(number | ("(" ~ expr2 ~ ")")).memoize + + // Warm up JVM. + (1 to 10).foreach { _ => + Memoize.parse("(" * (n - 1) + "1" + ")" * (n - 1), program2(_)) + } + + val (result2, elapsed2) = elapsedNanos(Memoize.parse("(" * (n - 1) + "1" + ")" * (n - 1), program2(_))) + assert(result2.get.value == 1) + // Verify that the memoized parser works as expected. + assert(Memoize.parse("123*(1+1)", program2(_)).get.value == 246) + assert(Memoize.parse("123*1+1", program2(_)).get.value == 124) + assert(Memoize.parse("123*1-1", program2(_)).get.value == 122) + assert(Memoize.parse("123*(1-1)", program2(_)).get.value == 0) + + println( + s"before memoization: ${elapsed1 / 1e9}, after memoization: ${elapsed2 / 1e9}, speedup: ${elapsed1 / elapsed2}x, Memoize.statistics: ${Memoize.statistics}" + ) + // Memoization should speed up at least 200 times in this example, with JVM warmup. + expect(elapsed1 > elapsed2 * 200) + } +} diff --git a/nano-dhall/src/main/scala/io/chymyst/nanodhall/LRUHashDict.scala b/nano-dhall/src/main/scala/io/chymyst/nanodhall/LRUHashDict.scala new file mode 100644 index 00000000..79636a6a --- /dev/null +++ b/nano-dhall/src/main/scala/io/chymyst/nanodhall/LRUHashDict.scala @@ -0,0 +1,25 @@ +package io.chymyst.dhall + +import java.util.concurrent.{ConcurrentHashMap, ConcurrentMap} + +// Each distinct value of type A is mapped to a unique `Int` key. + +trait HashDict[A] { + def lookup(key: Int): Option[A] + def store(value: A): Int +} + +class LRUHashDict[A](maxSize: Int) extends HashDict[A] { + private val valueDict: ConcurrentMap[Int, A] = new ConcurrentHashMap[Int, A] + private val keyDict: ConcurrentMap[A, Int] = new ConcurrentHashMap[A, Int] + + override def lookup(key: Int): Option[A] = Option(valueDict.get(key)) + + override def store(value: A): Int = { + val key = keyDict.computeIfAbsent(value, _ => valueDict.size + 1) + valueDict.put(key, value) + key + } +} + +object LRUHashDict {} diff --git a/nano-dhall/src/test/resources/1.dhall b/nano-dhall/src/test/resources/1.dhall new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/nano-dhall/src/test/resources/1.dhall @@ -0,0 +1 @@ +1 diff --git a/nano-dhall/src/test/scala/io/chymyst/nanodhall/unit/LRUHashDictTest.scala b/nano-dhall/src/test/scala/io/chymyst/nanodhall/unit/LRUHashDictTest.scala new file mode 100644 index 00000000..fdb42b6a --- /dev/null +++ b/nano-dhall/src/test/scala/io/chymyst/nanodhall/unit/LRUHashDictTest.scala @@ -0,0 +1,19 @@ +package io.chymyst.nanodhall.unit + +import com.eed3si9n.expecty.Expecty.expect +import io.chymyst.dhall.LRUHashDict +import munit.FunSuite + +class LRUHashDictTest extends FunSuite { + test("hash strings") { + val dict = new LRUHashDict[String](10) + val abc = dict.store("abc") + val cde = dict.store("cde") + expect(abc != cde) + expect(dict.store("abc") == abc) + expect(dict.store("cde") == cde) + expect(dict.lookup(abc) == Some("abc")) + expect(dict.lookup(cde) == Some("cde")) + expect(dict.lookup(0) == None) + } +} diff --git a/scall-cli/src/main/scala/io/chymyst/dhall/Main.scala b/scall-cli/src/main/scala/io/chymyst/dhall/Main.scala index e89b40c5..cf9f7d0c 100644 --- a/scall-cli/src/main/scala/io/chymyst/dhall/Main.scala +++ b/scall-cli/src/main/scala/io/chymyst/dhall/Main.scala @@ -7,6 +7,8 @@ import mainargs.{Flag, Leftover, ParserForMethods, arg, main} import java.io.{FileInputStream, FileOutputStream, InputStream, OutputStream} import java.nio.file.{Path, Paths} +import java.time.LocalDateTime +import sourcecode.{File => SourceFile, Line => SourceLine} object Main { @@ -27,11 +29,15 @@ object Main { case OutputMode.Decode => // TODO streamline those APIs output.write((Expression(CBORmodel.decodeCbor2(CBOR.java8ReadInputStreamToByteArray(input)).toScheme).print + "\n").getBytes("UTF-8")) - case _ => + + case _ => // In all other modes, we need to evaluate the Dhall file to a normal form. val outputBytes = Parser.parseDhallStream(input) match { case Parsed.Success(dhallFile: DhallFile, _) => val resolved = dhallFile.value.resolveImports(path) - val valueType = resolved.inferType.map { t => (t, resolved.betaNormalized) } + val valueType = resolved.inferType.map { t => + val normalForm = resolved.betaNormalized + (t, normalForm) + } val result: Array[Byte] = valueType match { case TypecheckResult.Valid((tpe: Expression, expr: Expression)) => outputMode match { diff --git a/scall-cli/src/test/resources/yaml-perftest/common.dhall b/scall-cli/src/test/resources/yaml-perftest/common.dhall new file mode 100644 index 00000000..e89e1f0b --- /dev/null +++ b/scall-cli/src/test/resources/yaml-perftest/common.dhall @@ -0,0 +1,89 @@ +let List/map + : ∀(a : Type) → ∀(b : Type) → (a → b) → List a → List b + = λ(a : Type) → + λ(b : Type) → + λ(f : a → b) → + λ(xs : List a) → + List/build + b + ( λ(list : Type) → + λ(cons : b → list → list) → + List/fold a xs list (λ(x : a) → cons (f x)) + ) + +let Natural/lessThanEqual + : Natural → Natural → Bool + = λ(x : Natural) → λ(y : Natural) → Natural/isZero (Natural/subtract y x) + +let Natural/greaterThanEqual + : Natural → Natural → Bool + = λ(x : Natural) → λ(y : Natural) → Natural/lessThanEqual y x + +let drop + : ∀(n : Natural) → ∀(a : Type) → List a → List a + = λ(n : Natural) → + λ(a : Type) → + λ(xs : List a) → + List/fold + { index : Natural, value : a } + (List/indexed a xs) + (List a) + ( λ(x : { index : Natural, value : a }) → + λ(xs : List a) → + if Natural/greaterThanEqual x.index n + then [ x.value ] # xs + else xs + ) + ([] : List a) + +let List/index + : Natural → ∀(a : Type) → List a → Optional a + = λ(n : Natural) → λ(a : Type) → λ(xs : List a) → List/head a (drop n a xs) + +let Optional/getOrElse + : ∀(a : Type) → a → Optional a → a + = λ(a : Type) → + λ(default : a) → + λ(o : Optional a) → + merge { Some = λ(x : a) → x, None = default } o + +let toUppercase + : Text → Text + = List/fold + (Text → Text) + [ Text/replace "a" "A" + , Text/replace "b" "B" + , Text/replace "c" "C" + , Text/replace "d" "D" + , Text/replace "e" "E" + , Text/replace "f" "F" + , Text/replace "g" "G" + , Text/replace "h" "H" + , Text/replace "i" "I" + , Text/replace "j" "J" + , Text/replace "k" "K" + , Text/replace "l" "L" + , Text/replace "m" "M" + , Text/replace "n" "N" + , Text/replace "o" "O" + , Text/replace "p" "P" + , Text/replace "q" "Q" + , Text/replace "r" "R" + , Text/replace "s" "S" + , Text/replace "t" "T" + , Text/replace "u" "U" + , Text/replace "v" "V" + , Text/replace "w" "W" + , Text/replace "x" "X" + , Text/replace "y" "Y" + , Text/replace "z" "Z" + ] + Text + (λ(replacement : Text → Text) → replacement) + +in { List/map + , Natural/greaterThanEqual + , toUppercase + , Optional/getOrElse + , List/index + } diff --git a/scall-cli/src/test/resources/yaml-perftest/create_yaml.dhall b/scall-cli/src/test/resources/yaml-perftest/create_yaml.dhall new file mode 100644 index 00000000..95ca449d --- /dev/null +++ b/scall-cli/src/test/resources/yaml-perftest/create_yaml.dhall @@ -0,0 +1,8 @@ +-- nonk8s +let S = ./schema.dhall + +let items = ./items.dhall + +let item = S.get_item 2 items + +in ./yaml_record.dhall item diff --git a/scall-cli/src/test/resources/yaml-perftest/create_yaml.yaml b/scall-cli/src/test/resources/yaml-perftest/create_yaml.yaml new file mode 100644 index 00000000..8034e995 --- /dev/null +++ b/scall-cli/src/test/resources/yaml-perftest/create_yaml.yaml @@ -0,0 +1,41 @@ +# nonk8s +apiVersion: v1 +kind: k +metadata: + displayName: 0 name1 + labels: + alert_enabled: false + item_type: type1 + qwerty_type: type1 + version: 0.0.0 + name: 0-name1 +spec: + b10: method1 + description: Description of 0 + indicator: + metadata: + name: 0-name1 + spec: + ratioMetric: + counter: true + good: + metricSource: + spec: + query: "begin cluster1 continue \"1\" end" + queryType: type2 + source: source2 + type: type3 + total: + metricSource: + spec: + query: "begin cluster1 continue \"1\" end" + queryType: type4 + source: source3 + type: type5 + item: '0' + objectives: + - displayName: 0 name2 + target: 0.0 + timeWindow: + - duration: '0' + isRolling: true diff --git a/scall-cli/src/test/resources/yaml-perftest/get_item_name.dhall b/scall-cli/src/test/resources/yaml-perftest/get_item_name.dhall new file mode 100644 index 00000000..00b7ec46 --- /dev/null +++ b/scall-cli/src/test/resources/yaml-perftest/get_item_name.dhall @@ -0,0 +1,7 @@ +let S = ./schema.dhall + +let items = ./items.dhall + +let item = S.get_item 2 items + +in item.name diff --git a/scall-cli/src/test/resources/yaml-perftest/items.dhall b/scall-cli/src/test/resources/yaml-perftest/items.dhall new file mode 100644 index 00000000..6651ee50 --- /dev/null +++ b/scall-cli/src/test/resources/yaml-perftest/items.dhall @@ -0,0 +1,138 @@ +let S = ./schema.dhall + +let c1 = S.L.C + +let c2 = S.L.E + +in [ { name = "1" + , p10 = "2" + , l = c1 + , param1 = + { strategy = S.M.T + , objective = "3" + , metric = "4" + , max_value = "0.005" + , unit = "s" + , extra = ",6\"/7\"" + } + , other = S.OtherParams::{ error_q = "8", total_q = "9" } + } + , { name = "1" + , p10 = "2" + , l = c1 + , param1 = + { strategy = S.M.T + , objective = "3" + , metric = "4" + , max_value = "0.1" + , unit = "s" + , extra = ",5" + } + , other = S.OtherParams::{ error_q = "6", total_q = "7" } + } + , { name = "0" + , p10 = "1" + , l = c1 + , param1 = + { strategy = S.M.T + , objective = "2" + , metric = "1" + , max_value = "1200" + , unit = "s" + , extra = ",app=\"3\"" + } + , other = S.OtherParams::{ + , error_q = "2" + , total_q = "2" + , es = ",code=~\"^5.*\"" + } + } + , { name = "2" + , p10 = "3" + , l = c1 + , param1 = + { strategy = S.M.T + , objective = "2" + , metric = "1" + , max_value = "0.1" + , unit = "s" + , extra = ",2" + } + , other = S.OtherParams::{ error_q = "3", total_q = "2" } + } + , { name = "1" + , p10 = "2" + , l = c1 + , param1 = + { strategy = S.M.T + , metric = "1" + , objective = "2" + , max_value = "0.5" + , unit = "s" + , extra = ",path=\"/path\"" + } + , other = S.OtherParams::{ error_q = "3", total_q = "2" } + } + , { name = "a" + , p10 = "b" + , l = c2 + , param1 = + { strategy = S.M.E + , objective = "2" + , metric = "c" + , max_value = "500" + , unit = "ms" + , extra = "d" + } + , other = S.OtherParams::{ + , error_q = "e" + , total_q = "f" + , es = ",response=~\"Error\"" + } + } + , { name = "b" + , p10 = "c" + , l = c2 + , param1 = + { strategy = S.M.E + , objective = "2" + , metric = "a" + , max_value = "100" + , unit = "ms" + , extra = "url_path=\"/c\"" + } + , other = S.OtherParams::{ + , error_q = "d" + , total_q = "d" + , es = ",code=~\"^5.*\"" + } + } + , { name = "b" + , p10 = "b.*" + , l = c1 + , param1.strategy = S.M.E + , param1.objective = "d" + , param1.metric = "c" + , param1.max_value = "3600" + , param1.extra = "e" + , param1.unit = "s" + , other = S.OtherParams::{ + , error_q = "f" + , total_q = "g" + , es = ",pod=~\"h-.*\"" + , flag = True + } + } + , { name = "q" + , p10 = "w" + , l = c1 + , param1.strategy = S.M.E + , param1.objective = "2" + , param1.metric = "e" + , param1.max_value = "200" + , param1.extra = "url=\"r\"" + , param1.unit = "ms" + , other = S.OtherParams::{ error_q = "t", total_q = "u" } + } + ] + : List S.Item diff --git a/scall-cli/src/test/resources/yaml-perftest/largeExpressionA.dhall b/scall-cli/src/test/resources/yaml-perftest/largeExpressionA.dhall new file mode 100644 index 00000000..62d04ec8 --- /dev/null +++ b/scall-cli/src/test/resources/yaml-perftest/largeExpressionA.dhall @@ -0,0 +1,272 @@ + λ ( xs + : List + { cores : + Natural + , host : + Text + , key : + Text + , mandatoryFeatures : + List Text + , platforms : + List + < AArch64_Linux + | ARMv5tel_Linux + | ARMv7l_Linux + | I686_Cygwin + | I686_Linux + | MIPS64el_Linux + | PowerPC_Linux + | X86_64_Cygwin + | X86_64_Darwin + | X86_64_FreeBSD + | X86_64_Linux + | X86_64_Solaris + > + , speedFactor : + Natural + , supportedFeatures : + List Text + , user : + Optional Text + } + ) +→ List/fold + { cores : + Natural + , host : + Text + , key : + Text + , mandatoryFeatures : + List Text + , platforms : + List + < AArch64_Linux + | ARMv5tel_Linux + | ARMv7l_Linux + | I686_Cygwin + | I686_Linux + | MIPS64el_Linux + | PowerPC_Linux + | X86_64_Cygwin + | X86_64_Darwin + | X86_64_FreeBSD + | X86_64_Linux + | X86_64_Solaris + > + , speedFactor : + Natural + , supportedFeatures : + List Text + , user : + Optional Text + } + xs + Text + ( λ ( x + : { cores : + Natural + , host : + Text + , key : + Text + , mandatoryFeatures : + List Text + , platforms : + List + < AArch64_Linux + | ARMv5tel_Linux + | ARMv7l_Linux + | I686_Cygwin + | I686_Linux + | MIPS64el_Linux + | PowerPC_Linux + | X86_64_Cygwin + | X86_64_Darwin + | X86_64_FreeBSD + | X86_64_Linux + | X86_64_Solaris + > + , speedFactor : + Natural + , supportedFeatures : + List Text + , user : + Optional Text + } + ) + → λ(y : Text) + → merge + { None = x.host + , Some = λ(user : Text) → user ++ "@" ++ x.host ++ "" + } + x.user + ++ " " + ++ ( merge + { Empty = "", NonEmpty = λ(result : Text) → result } + ( List/fold + < AArch64_Linux + | ARMv5tel_Linux + | ARMv7l_Linux + | I686_Cygwin + | I686_Linux + | MIPS64el_Linux + | PowerPC_Linux + | X86_64_Cygwin + | X86_64_Darwin + | X86_64_FreeBSD + | X86_64_Linux + | X86_64_Solaris + > + x.platforms + < Empty | NonEmpty : Text > + ( λ ( element + : < AArch64_Linux + | ARMv5tel_Linux + | ARMv7l_Linux + | I686_Cygwin + | I686_Linux + | MIPS64el_Linux + | PowerPC_Linux + | X86_64_Cygwin + | X86_64_Darwin + | X86_64_FreeBSD + | X86_64_Linux + | X86_64_Solaris + > + ) + → λ(status : < Empty | NonEmpty : Text >) + → merge + { Empty = + < Empty | NonEmpty : Text >.NonEmpty + ( merge + { AArch64_Linux = + "aarch64-linux" + , ARMv5tel_Linux = + "armv5tel-linux" + , ARMv7l_Linux = + "armv7l-linux" + , I686_Cygwin = + "i686-cygwin" + , I686_Linux = + "i686-linux" + , MIPS64el_Linux = + "mips64el-linux" + , PowerPC_Linux = + "powerpc-linux" + , X86_64_Cygwin = + "x86_64-cygwin" + , X86_64_Darwin = + "x86_64-darwin" + , X86_64_FreeBSD = + "x86_64-freebsd" + , X86_64_Linux = + "x86_64-linux" + , X86_64_Solaris = + "x86_64-solaris" + } + element + ) + , NonEmpty = + λ(result : Text) + → < Empty | NonEmpty : Text >.NonEmpty + ( ( merge + { AArch64_Linux = + "aarch64-linux" + , ARMv5tel_Linux = + "armv5tel-linux" + , ARMv7l_Linux = + "armv7l-linux" + , I686_Cygwin = + "i686-cygwin" + , I686_Linux = + "i686-linux" + , MIPS64el_Linux = + "mips64el-linux" + , PowerPC_Linux = + "powerpc-linux" + , X86_64_Cygwin = + "x86_64-cygwin" + , X86_64_Darwin = + "x86_64-darwin" + , X86_64_FreeBSD = + "x86_64-freebsd" + , X86_64_Linux = + "x86_64-linux" + , X86_64_Solaris = + "x86_64-solaris" + } + element + ) + ++ "," + ++ result + ) + } + status + : < Empty | NonEmpty : Text > + ) + < Empty | NonEmpty : Text >.Empty + ) + : Text + ) + ++ " " + ++ x.key + ++ " " + ++ Integer/show (Natural/toInteger x.cores) + ++ " " + ++ Integer/show (Natural/toInteger x.speedFactor) + ++ " " + ++ ( merge + { Empty = "", NonEmpty = λ(result : Text) → result } + ( List/fold + Text + x.supportedFeatures + < Empty | NonEmpty : Text > + ( λ(element : Text) + → λ(status : < Empty | NonEmpty : Text >) + → merge + { Empty = + < Empty | NonEmpty : Text >.NonEmpty element + , NonEmpty = + λ(result : Text) + → < Empty | NonEmpty : Text >.NonEmpty + (element ++ "," ++ result) + } + status + : < Empty | NonEmpty : Text > + ) + < Empty | NonEmpty : Text >.Empty + ) + : Text + ) + ++ " " + ++ ( merge + { Empty = "", NonEmpty = λ(result : Text) → result } + ( List/fold + Text + x.mandatoryFeatures + < Empty | NonEmpty : Text > + ( λ(element : Text) + → λ(status : < Empty | NonEmpty : Text >) + → merge + { Empty = + < Empty | NonEmpty : Text >.NonEmpty element + , NonEmpty = + λ(result : Text) + → < Empty | NonEmpty : Text >.NonEmpty + (element ++ "," ++ result) + } + status + : < Empty | NonEmpty : Text > + ) + < Empty | NonEmpty : Text >.Empty + ) + : Text + ) + ++ '' + + '' + ++ y + ) + "" diff --git a/scall-cli/src/test/resources/yaml-perftest/renderAs.dhall b/scall-cli/src/test/resources/yaml-perftest/renderAs.dhall new file mode 100644 index 00000000..d65ab1a4 --- /dev/null +++ b/scall-cli/src/test/resources/yaml-perftest/renderAs.dhall @@ -0,0 +1,458 @@ +--| Render a `JSON` value as `Text` in either JSON or YAML format. +let JSON = + missing + sha256:5dc1135d5481cfd6fde625aaed9fcbdb7aa7c14f2e76726aa5fdef028a5c10f5 + ? ./core.dhall + +let Function/identity = + missing + sha256:f78b96792b459cb664f41c6119bd8897dd04353a3343521d436cd82ad71cb4d4 + ? ../Function/identity.dhall + +let Text/concatMap = + missing + sha256:7a0b0b99643de69d6f94ba49441cd0fa0507cbdfa8ace0295f16097af37e226f + ? ../Text/concatMap.dhall + +let List/map = + missing + sha256:dd845ffb4568d40327f2a817eb42d1c6138b929ca758d50bc33112ef3c885680 + ? ../List/map.dhall + +let NonEmpty = + missing + sha256:e2e247455a858317e470e0e4affca8ac07f9f130570ece9cb7ac1f4ea3deb87f + ? ../NonEmpty/Type.dhall + +let NonEmpty/toList = + missing + sha256:0977fe14b77232a4451dcf409c43df4589c4b3cdde7b613aab8df183be1b53f5 + ? ../NonEmpty/toList.dhall + +let NonEmpty/concat = + missing + sha256:6d55181938c06c6b806877028f6a241912e9c0935d9a10dd958775bf21e0f64d + ? ../NonEmpty/concat.dhall + +let NonEmpty/map = + missing + sha256:93d53afe874bb2eed946c21ca5ada3c9716b7d00e6d8edfaba6484cd9c5a00bd + ? ../NonEmpty/map.dhall + +let NonEmpty/singleton = + missing + sha256:c9197aabe97695f7ca66f7419bf172d806b2c915594a8fc0d2ff6495db496ff2 + ? ../NonEmpty/singleton.dhall + +let List/uncons + : ∀(a : Type) → List a → Optional (NonEmpty a) + = {- This version uses the `ls` argument only once to prevent cache blowups at the price + of performing two passes over the list: + A first one to reverse it, a second one with `List/fold` to determine + the head element. + See https://github.com/dhall-lang/dhall-lang/pull/1015#issuecomment-633381024 + for some context regarding the caching issue. + -} + λ(a : Type) → + λ(ls : List a) → + List/fold + a + (List/reverse a ls) + (Optional (NonEmpty a)) + ( λ(x : a) → + λ(acc : Optional (NonEmpty a)) → + merge + { None = Some (NonEmpty/singleton a x) + , Some = + λ(ne : NonEmpty a) → Some (ne ⫽ { tail = ne.tail # [ x ] }) + } + acc + ) + (None (NonEmpty a)) + +let NonEmpty/mapHead + : ∀(a : Type) → (a → a) → NonEmpty a → NonEmpty a + = λ(a : Type) → + λ(fn : a → a) → + λ(ls : NonEmpty a) → + ls ⫽ { head = fn ls.head } + +let NonEmpty/mapTail + : ∀(a : Type) → (a → a) → NonEmpty a → NonEmpty a + = λ(a : Type) → + λ(fn : a → a) → + λ(ls : NonEmpty a) → + ls ⫽ { tail = List/map a a fn ls.tail } + +let NonEmpty/prepend + : ∀(a : Type) → a → NonEmpty a → NonEmpty a + = λ(a : Type) → + λ(prefix : a) → + λ(ls : NonEmpty a) → + { head = prefix, tail = NonEmpty/toList a ls } + +let NonYtpme + : Type → Type + = λ(a : Type) → { init : List a, last : a } + +let List/unsnoc + : ∀(a : Type) → List a → Optional (NonYtpme a) + = λ(a : Type) → + λ(ls : List a) → + List/fold + a + ls + (Optional (NonYtpme a)) + ( λ(x : a) → + λ(acc : Optional (NonYtpme a)) → + merge + { None = Some { init = [] : List a, last = x } + , Some = + λ(ny : NonYtpme a) → Some (ny ⫽ { init = [ x ] # ny.init }) + } + acc + ) + (None (NonYtpme a)) + +let NonEmpty/mapLast + : ∀(a : Type) → (a → a) → NonEmpty a → NonEmpty a + = λ(a : Type) → + λ(fn : a → a) → + λ(ls : NonEmpty a) → + merge + { Some = λ(x : NonYtpme a) → ls ⫽ { tail = x.init # [ fn x.last ] } + , None = NonEmpty/singleton a (fn ls.head) + } + (List/unsnoc a ls.tail) + +let NonEmpty/mapLeading + : ∀(a : Type) → (a → a) → NonEmpty a → NonEmpty a + = λ(a : Type) → + λ(fn : a → a) → + λ(ls : NonEmpty a) → + merge + { Some = + λ(x : NonYtpme a) → + { head = fn ls.head + , tail = List/map a a fn x.init # [ x.last ] + } + , None = ls + } + (List/unsnoc a ls.tail) + +let Lines + : Type + = NonEmpty Text + +let Block + : Type + = < Simple : Text | Complex : Lines > + +let Block/toLines + : Block → Lines + = λ(block : Block) → + merge + { Simple = NonEmpty/singleton Text + , Complex = Function/identity Lines + } + block + +let manyBlocks + : ∀(a : Type) → Text → (NonEmpty a → Lines) → List a → Block + = λ(a : Type) → + λ(ifEmpty : Text) → + λ(render : NonEmpty a → Lines) → + λ(inputs : List a) → + merge + { Some = λ(inputs : NonEmpty a) → Block.Complex (render inputs) + , None = Block.Simple ifEmpty + } + (List/uncons a inputs) + +let blockToText + : Block → Text + = λ(block : Block) → + Text/concatMap + Text + (λ(line : Text) → line ++ "\n") + (NonEmpty/toList Text (Block/toLines block)) + +let addPrefix = λ(prefix : Text) → λ(line : Text) → prefix ++ line + +let addIndent = addPrefix " " + +let indentTail = NonEmpty/mapTail Text addIndent + +let Format = + missing + sha256:d7936b510cfc091faa994652af0eb5feb889cd44bc989edbe4f1eb8c5623caac + ? ./Format.dhall + +let ObjectField = { mapKey : Text, mapValue : Block } + +let -- Essentially the same thing as `Text/show`, except that this does not + -- escape `$` + escape = + List/fold + (Text → Text) + [ Text/replace "\"" "\\\"" + , Text/replace "\b" "\\b" + , Text/replace "\f" "\\f" + , Text/replace "\n" "\\n" + , Text/replace "\r" "\\r" + , Text/replace "\t" "\\t" + , Text/replace "\\" "\\\\" + ] + Text + (λ(replace : Text → Text) → λ(text : Text) → replace text) + +let renderJSONStruct = + λ(prefix : Text) → + λ(suffix : Text) → + λ(blocks : NonEmpty Lines) → + let indent = List/map Text Text addIndent + + let appendComma + : Lines → Lines + = NonEmpty/mapLast Text (λ(line : Text) → line ++ ",") + + let blocks = NonEmpty/mapLeading Lines appendComma blocks + + let block = NonEmpty/concat Text blocks + + in merge + { None = + NonEmpty/singleton Text "${prefix} ${block.head} ${suffix}" + , Some = + λ(ny : NonYtpme Text) → + { head = prefix + , tail = + indent ([ block.head ] # ny.init # [ ny.last ]) + # [ suffix ] + } + } + (List/unsnoc Text block.tail) + +let renderObject = + λ(format : Format) → + λ(fields : NonEmpty ObjectField) → + let keystr = λ(field : ObjectField) → "\"${escape field.mapKey}\":" + + let prefixKeyOnFirst = + λ(field : ObjectField) → + NonEmpty/mapHead + Text + (addPrefix "${keystr field} ") + (Block/toLines field.mapValue) + + let prependKeyLine = + λ(field : ObjectField) → + NonEmpty/prepend + Text + (keystr field) + (Block/toLines field.mapValue) + + let renderYAMLField = + λ(field : ObjectField) → + merge + { Simple = + λ(line : Text) → + NonEmpty/singleton Text "${keystr field} ${line}" + , Complex = λ(_ : Lines) → indentTail (prependKeyLine field) + } + field.mapValue + + in merge + { JSON = + renderJSONStruct + "{" + "}" + (NonEmpty/map ObjectField Lines prefixKeyOnFirst fields) + , YAML = + NonEmpty/concat + Text + (NonEmpty/map ObjectField Lines renderYAMLField fields) + } + format + +let renderYAMLArrayField = + λ(block : Block) → + NonEmpty/mapHead + Text + (addPrefix "- ") + (indentTail (Block/toLines block)) + +let renderArray = + λ(format : Format) → + λ(fields : NonEmpty Block) → + merge + { JSON = + renderJSONStruct + "[" + "]" + (NonEmpty/map Block Lines Block/toLines fields) + , YAML = + NonEmpty/concat + Text + (NonEmpty/map Block Lines renderYAMLArrayField fields) + } + format + +let renderAs + : Format → JSON.Type → Text + = λ(format : Format) → + λ(json : JSON.Type) → + blockToText + ( json + Block + { string = λ(x : Text) → Block.Simple "\"${escape x}\"" + , double = λ(x : Double) → Block.Simple (Double/show x) + , integer = λ(x : Integer) → Block.Simple (JSON.renderInteger x) + , object = manyBlocks ObjectField "{}" (renderObject format) + , array = manyBlocks Block "[]" (renderArray format) + , bool = + λ(x : Bool) → Block.Simple (if x then "true" else "false") + , null = Block.Simple "null" + } + ) + +let example0 = + let data = + JSON.array + [ JSON.bool True + , JSON.string "Hello" + , JSON.object + [ { mapKey = "foo", mapValue = JSON.null } + , { mapKey = "bar", mapValue = JSON.double 1.0 } + ] + ] + + let yaml = + assert + : renderAs Format.YAML data + ≡ '' + - true + - "Hello" + - "foo": null + "bar": 1.0 + '' + + let json = + assert + : renderAs Format.JSON data + ≡ '' + [ + true, + "Hello", + { + "foo": null, + "bar": 1.0 + } + ] + '' + + in True + +let example1 = + let data = + JSON.object + [ { mapKey = "zero", mapValue = JSON.array ([] : List JSON.Type) } + , { mapKey = "one", mapValue = JSON.array [ JSON.string "a" ] } + , { mapKey = "two" + , mapValue = JSON.array [ JSON.string "a", JSON.string "b" ] + } + ] + + let yaml = + assert + : renderAs Format.YAML data + ≡ '' + "zero": [] + "one": + - "a" + "two": + - "a" + - "b" + '' + + let json = + assert + : renderAs Format.JSON data + ≡ '' + { + "zero": [], + "one": [ "a" ], + "two": [ + "a", + "b" + ] + } + '' + + in True + +let example2 = + let data = + JSON.object + [ { mapKey = "zero" + , mapValue = + JSON.object + (toMap {=} : List { mapKey : Text, mapValue : JSON.Type }) + } + , { mapKey = "one" + , mapValue = JSON.object (toMap { a = JSON.null }) + } + , { mapKey = "two" + , mapValue = + JSON.object (toMap { a = JSON.null, b = JSON.null }) + } + ] + + let yaml = + assert + : renderAs Format.YAML data + ≡ '' + "zero": {} + "one": + "a": null + "two": + "a": null + "b": null + '' + + let json = + assert + : renderAs Format.JSON data + ≡ '' + { + "zero": {}, + "one": { "a": null }, + "two": { + "a": null, + "b": null + } + } + '' + + in True + +let example3 = + let specialCharacters = + '' + "\${"\b\f"} + ${"\r"} $'' + + let data = + JSON.object + [ { mapKey = specialCharacters + , mapValue = JSON.string specialCharacters + } + ] + + in assert + : renderAs Format.JSON data + ≡ '' + { "\"\\\b\f\n\r\t$": "\"\\\b\f\n\r\t$" } + '' + +in renderAs diff --git a/scall-cli/src/test/resources/yaml-perftest/schema.dhall b/scall-cli/src/test/resources/yaml-perftest/schema.dhall new file mode 100644 index 00000000..5b62e52f --- /dev/null +++ b/scall-cli/src/test/resources/yaml-perftest/schema.dhall @@ -0,0 +1,145 @@ +let Common = + ./common.dhall + +let L = < C | E > + +let M = < T | E > + +let Params = + { objective : Text + , metric : Text + , max_value : Text + , unit : Text + , strategy : M + , extra : Text + } + +let OtherParams = + { Type = + { error_q : Text + , total_q : Text + , es : Text + , flag : Bool + } + , default = + { error_q = "", total_q = "", es = "", flag = False } + } + +let emptyParams + : Params + = { strategy = M.T + , objective = "" + , metric = "" + , max_value = "0.1" + , unit = "s" + , extra = "" + } + +let Item = + { l : L + , name : Text + , p10 : Text + , param1 : Params + , other : OtherParams.Type + } + +let emptyItem = + { l = L.C + , name = "undefined" + , p10 = "undefined" + , param1 = emptyParams + , other = OtherParams.default + } + +let get_item = + λ(index : Natural) → + λ(items : List Item) → + Common.Optional/getOrElse + Item + emptyItem + (Common.List/index index Item items) + +let aggregator_for = + λ(item : Item) → merge { C = "cluster1", E = "env1" } item.l + +let aggregator_label_for = + λ(item : Item) → merge { C = "cluster2", E = "env2" } item.l + +let uppercased = + λ(item : Item) → Text/replace "-" " " (Common.toUppercase item.name) + +let S_record_type = + { apiVersion : Text + , kind : Text + , metadata : + { displayName : Text + , labels : + { alert_enabled : Bool + , item_type : Text + , qwerty_type : Text + , version : Text + } + , name : Text + } + , spec : + { b10 : Text + , description : Text + , indicator : + { metadata : { name : Text } + , spec : + { ratioMetric : + { counter : Bool + , good : + { metricSource : + { spec : + { query : Text + , queryType : Text + , source : Text + } + , type : Text + } + } + , total : + { metricSource : + { spec : + { query : Text + , queryType : Text + , source : Text + } + , type : Text + } + } + } + } + } + , objectives : List { displayName : Text, target : Double } + , item : Text + , timeWindow : List { duration : Text, isRolling : Bool } + } + } + +let E_type = + { apiVersion : Text + , kind : Text + , metadata : { labels : { owner : Text, item : Text }, name : Text } + , spec : + { envs : List Text + , s : List Text + , variables : + List { label : Text, name : Text, options : Text, type : Text } + } + } + +in { L + , Item + , M + , Params + , OtherParams + , emptyParams + , get_item + , aggregator_for + , aggregator_label_for + , uppercased + , S_record_type + , E_type + } diff --git a/scall-cli/src/test/resources/yaml-perftest/yaml_record.dhall b/scall-cli/src/test/resources/yaml-perftest/yaml_record.dhall new file mode 100644 index 00000000..59cb3092 --- /dev/null +++ b/scall-cli/src/test/resources/yaml-perftest/yaml_record.dhall @@ -0,0 +1,52 @@ +let S = ./schema.dhall + +in λ(item : S.Item) → + let aggr = S.aggregator_for item + + in { apiVersion = "v1" + , kind = "k" + , metadata = + { displayName = "${S.uppercased item} name1" + , labels = + { alert_enabled = False + , item_type = "type1" + , qwerty_type = "type1" + , version = "0.0.0" + } + , name = "${item.name}-name1" + } + , spec = + { b10 = "method1" + , description = "Description of ${S.uppercased item}" + , indicator = + { metadata.name = "${item.name}-name1" + , spec.ratioMetric + = + { counter = True + , good.metricSource + = + { spec = + { query = "begin ${aggr} continue \"${item.p10}\" end" + , queryType = "type2" + , source = "source2" + } + , type = "type3" + } + , total.metricSource + = + { spec = + { query = "begin ${aggr} continue \"${item.p10}\" end" + , queryType = "type4" + , source = "source3" + } + , type = "type5" + } + } + } + , objectives = + [ { displayName = "${S.uppercased item} name2", target = 0.0 } ] + , item = "${item.name}" + , timeWindow = [ { duration = "0", isRolling = True } ] + } + } + : S.S_record_type diff --git a/scall-cli/src/test/scala/io/chymyst/dhall/unit/PerfTest.scala b/scall-cli/src/test/scala/io/chymyst/dhall/unit/PerfTest.scala new file mode 100644 index 00000000..1c9c33df --- /dev/null +++ b/scall-cli/src/test/scala/io/chymyst/dhall/unit/PerfTest.scala @@ -0,0 +1,64 @@ +package io.chymyst.dhall.unit + +import com.eed3si9n.expecty.Expecty.expect +import io.chymyst.dhall.{Main, Parser} +import io.chymyst.dhall.Main.OutputMode +import io.chymyst.dhall.Yaml.YamlOptions +import io.chymyst.test.{ResourceFiles, TestTimings} +import munit.FunSuite + +import java.io.{ByteArrayOutputStream, FileInputStream} +import java.nio.file.{Files, Paths} + +class PerfTest extends FunSuite with ResourceFiles with TestTimings { + + test("create yaml from realistic example 1") { + val file = resourceAsFile("yaml-perftest/create_yaml.dhall").get + val options = YamlOptions() + val testOut = new ByteArrayOutputStream + val (_, elapsedNs) = elapsedNanos { + try { + Main.process(file.toPath, new FileInputStream(file), testOut, OutputMode.Yaml, options) + } finally { + testOut.close() + } + } + val resultYaml = new String(testOut.toByteArray) + val expectedYaml = new String(Files.readAllBytes(Paths.get(file.getAbsolutePath.replace(".dhall", ".yaml")))) + val elapsed = elapsedNs.toDouble / 1e9 + println(s"Yaml created in $elapsed seconds") + expect(resultYaml == expectedYaml) + expect(elapsed / 1e9 < 10.0) + } + + test("parse schema.dhall 20 times") { + val file = resourceAsFile("yaml-perftest/schema.dhall").get + val results = (1 to 20).map { i => + val (_, elapsed) = elapsedNanos(Parser.parseDhallStream(new FileInputStream(file)).get.value.value) + println(s"iteration $i : schema.dhall parsed in ${elapsed / 1e9} seconds") + expect(elapsed / 1e9 < 0.5) + } + } + + test("parse renderAs.dhall") { + val file = resourceAsFile("yaml-perftest/renderAs.dhall").get + val (_, elapsed) = elapsedNanos(Parser.parseDhallStream(new FileInputStream(file)).get.value.value) + println(s"Prelude/JSON/renderAs.dhall parsed in ${elapsed / 1e9} seconds") + expect(elapsed / 1e9 < 1.0) + } + + test("parse largeExpressionA.dhall") { + val file = resourceAsFile("yaml-perftest/largeExpressionA.dhall").get + val (_, elapsed) = elapsedNanos(Parser.parseDhallStream(new FileInputStream(file)).get.value.value) + println(s"largeExpressionA.dhall parsed in ${elapsed / 1e9} seconds") + expect(elapsed / 1e9 < 0.5) + } + + test("parse nested parentheses") { + val n = 30 // More than 30 gives stack overflow. + val input = "(" * n + "1" + ")" * n + val (_, elapsed) = elapsedNanos(Parser.parseDhall(input).get.value.value) + println(s"$n nested parentheses parsed in ${elapsed / 1e9} seconds") + expect(elapsed / 1e9 < 0.5) + } +} diff --git a/scall-core/src/main/scala/io/chymyst/dhall/Grammar.scala b/scall-core/src/main/scala/io/chymyst/dhall/Grammar.scala index a12c5334..c2c40258 100644 --- a/scall-core/src/main/scala/io/chymyst/dhall/Grammar.scala +++ b/scall-core/src/main/scala/io/chymyst/dhall/Grammar.scala @@ -9,6 +9,7 @@ import io.chymyst.dhall.TypeCheck._Type import java.time.LocalDate import scala.util.{Failure, Success, Try} +import io.chymyst.fastparse.Memoize.MemoizeParser object Grammar { @@ -111,12 +112,14 @@ object Grammar { // U+10FFFD = "\uDBFF\uDFFD" // %x10FFFE_10FFFF = non_characters ) + .memoize def tab[$: P] = P("\t") def block_comment[$: P] = P( "{-" ~/ block_comment_continue // Do not use cut here, because then block comment will fail the entire identifier when parsing "x {- -}" without a following @. ) + .memoize def block_comment_char[$: P] = P( CharIn("\u0020-\u007F") @@ -124,12 +127,14 @@ object Grammar { | tab | end_of_line ) + .memoize def block_comment_continue[$: P]: P[Unit] = P( "-}" | (block_comment ~/ block_comment_continue) | (block_comment_char ~ block_comment_continue) ) + .memoize def not_end_of_line[$: P] = P( CharIn("\u0020-\u007F") | valid_non_ascii | tab @@ -150,6 +155,7 @@ object Grammar { | line_comment | block_comment ) + .memoize def whsp[$: P]: P[Unit] = P( NoCut(whitespace_chunk.rep) @@ -293,7 +299,7 @@ object Grammar { def double_quote_literal[$: P]: P[TextLiteral[Expression]] = P( "\"" ~/ double_quote_chunk.rep ~ "\"" - ).map(_.map(literalOrInterp => literalOrInterp.map(TextLiteral.ofText).merge).fold(TextLiteral.empty)(_ ++ _)) + ).map(_.map(literalOrInterp => literalOrInterp.map(TextLiteral.ofText[Expression]).merge).fold(TextLiteral.empty[Expression])(_ ++ _)) def single_quote_continue[$: P]: P[TextLiteral[Expression]] = P( (interpolation ~ single_quote_continue).map { case (head, tail) => TextLiteral.ofExpression(head) ++ tail } @@ -407,6 +413,7 @@ object Grammar { (p1, p2) => implicit ctx: P[_] => P(p1(ctx) | p2(ctx)) }(implicitly[P[$]]).! } +// .memoize // Do not memoize: breaks parsing! //def keywordOrBuiltin[$: P]: P[String] = concatKeywords(simpleKeywords ++ builtinSymbolNames) @@ -987,6 +994,7 @@ object Grammar { // "x : t" | annotated_expression./ ) + .memoize def annotated_expression[$: P]: P[Expression] = P( operator_expression ~ (whsp ~ ":" ~ whsp1 ~/ expression).? @@ -1031,54 +1039,67 @@ object Grammar { def equivalent_expression[$: P]: P[Expression] = P( import_alt_expression ~ (whsp ~ equivalent ~ whsp ~/ import_alt_expression).rep ).withOperator(SyntaxConstants.Operator.Equivalent) + .memoize def import_alt_expression[$: P]: P[Expression] = P( or_expression ~ (whsp ~ opAlternative ~ whsp1 ~/ or_expression).rep ).withOperator(SyntaxConstants.Operator.Alternative) + .memoize def or_expression[$: P]: P[Expression] = P( plus_expression ~ (whsp ~ opOr ~ whsp ~/ plus_expression).rep ).withOperator(SyntaxConstants.Operator.Or) + .memoize def plus_expression[$: P]: P[Expression] = P( text_append_expression ~ (whsp ~ opPlus ~ whsp1 ~/ text_append_expression).rep ).withOperator(SyntaxConstants.Operator.Plus) + .memoize def text_append_expression[$: P]: P[Expression] = P( list_append_expression ~ (whsp ~ opTextAppend ~ whsp ~/ list_append_expression).rep ).withOperator(SyntaxConstants.Operator.TextAppend) + .memoize def list_append_expression[$: P]: P[Expression] = P( and_expression ~ (whsp ~ opListAppend ~ whsp ~/ and_expression).rep ).withOperator(SyntaxConstants.Operator.ListAppend) + .memoize def and_expression[$: P]: P[Expression] = P( combine_expression ~ (whsp ~ opAnd ~ whsp ~/ combine_expression).rep ).withOperator(SyntaxConstants.Operator.And) + .memoize def combine_expression[$: P]: P[Expression] = P( prefer_expression ~ (whsp ~ combine ~ whsp ~/ prefer_expression).rep ).withOperator(SyntaxConstants.Operator.CombineRecordTerms) + .memoize def prefer_expression[$: P]: P[Expression] = P( combine_types_expression ~ (whsp ~ prefer ~ whsp ~/ combine_types_expression).rep ).withOperator(SyntaxConstants.Operator.Prefer) + .memoize def combine_types_expression[$: P]: P[Expression] = P( times_expression ~ (whsp ~ combine_types ~ whsp ~/ times_expression).rep ).withOperator(SyntaxConstants.Operator.CombineRecordTypes) + .memoize def times_expression[$: P]: P[Expression] = P( equal_expression ~ (whsp ~ opTimes ~ whsp ~/ equal_expression).rep ).withOperator(SyntaxConstants.Operator.Times) + .memoize def equal_expression[$: P]: P[Expression] = P( not_equal_expression ~ (whsp ~ opEqual ~ whsp ~ not_equal_expression).rep // Should not cut because == can be confused with === ).withOperator(SyntaxConstants.Operator.Equal) + .memoize def not_equal_expression[$: P]: P[Expression] = P( application_expression ~ (whsp ~ opNotEqual ~ whsp ~/ application_expression).rep ).withOperator(SyntaxConstants.Operator.NotEqual) + .memoize def application_expression[$: P]: P[Expression] = P( first_application_expression ~ (whsp1 ~ import_expression).rep // Do not insert a cut after whsp1 here. @@ -1107,6 +1128,7 @@ object Grammar { def import_expression[$: P]: P[Expression] = P( import_only | completion_expression ) + .memoize def completion_expression[$: P]: P[Expression] = P( selector_expression ~ (whsp ~ complete ~ whsp ~ selector_expression).? @@ -1118,6 +1140,7 @@ object Grammar { def selector_expression[$: P]: P[Expression] = P( primitive_expression ~ (whsp ~ "." ~ whsp ~ /* No cut here, or else (List ./imported.file) cannot be parsed. */ selector).rep ).map { case (base, selectors) => selectors.foldLeft(base)((prev, selector) => selector.chooseExpression(prev)) } + .memoize sealed trait ExpressionSelector { def chooseExpression(base: Expression): Expression = this match { @@ -1186,6 +1209,7 @@ object Grammar { // "( e )" | P("(" ~/ complete_expression ~/ ")") ) + .memoize def record_type_or_literal[$: P]: P[Option[Expression]] = P( empty_record_literal.map(Expression.apply).map(Some.apply) @@ -1219,6 +1243,7 @@ object Grammar { def record_literal_normal_entry[$: P]: P[(Seq[FieldName], Expression)] = P( (whsp ~ "." ~ whsp ~/ any_label_or_some.map(FieldName)).rep ~ whsp ~ "=" ~ whsp ~/ expression ) + .memoize def union_type[$: P]: P[Expression] = P( (union_type_entry ~ (whsp ~ "|" ~ whsp ~ union_type_entry).rep ~ (whsp ~ "|").?).? @@ -1230,6 +1255,7 @@ object Grammar { def union_type_entry[$: P] = P( any_label_or_some.map(ConstructorName) ~ (whsp ~ ":" ~/ whsp1 ~/ expression).? ) + .memoize def non_empty_list_literal[$: P]: P[Expression] = P( "[" ~/ whsp ~ ("," ~ whsp).? ~ expression ~ whsp ~ ("," ~ whsp ~ /* No cut here, or else [, ,] cannot be parsed. */ expression ~ whsp).rep ~ ("," ~/ whsp).? ~ "]" @@ -1246,6 +1272,7 @@ object Grammar { def complete_expression[$: P] = P( whsp ~ expression ~ whsp ) + .memoize // Helpers to make sure we are using valid keyword and operator names. def requireKeyword[$: P](name: String): P[Unit] = { diff --git a/scall-core/src/main/scala/io/chymyst/dhall/ImportResolution.scala b/scall-core/src/main/scala/io/chymyst/dhall/ImportResolution.scala index 225c9fca..a75d377a 100644 --- a/scall-core/src/main/scala/io/chymyst/dhall/ImportResolution.scala +++ b/scall-core/src/main/scala/io/chymyst/dhall/ImportResolution.scala @@ -17,7 +17,9 @@ import io.chymyst.tc.Applicative import java.nio.file import java.nio.file.{Files, Paths} +import java.time.LocalDateTime import scala.util.{Failure, Success, Try} +import sourcecode.{File => SourceFile, Line => SourceLine} object ImportResolution { @@ -395,7 +397,8 @@ object ImportResolution { bytes <- Try(Files.readAllBytes(javaPath)) } yield bytes) match { case Failure(exception) => Left(TransientFailure(Seq(s"Failed to read imported file: $exception"))) - case Success(bytes) => Right(bytes) + case Success(bytes) => + Right(bytes) } case ImportType.Env(envVarName) => @@ -416,7 +419,7 @@ object ImportResolution { _ <- cyclicImportCheck _ <- referentialCheck readByImportMode <- resolveByImportMode - bytes <- missingOrData + bytes <- missingOrData // This is slow. expr <- Right(readByImportMode(bytes)) successfullyRead <- expr match { case Resolved(x) => Right(x) diff --git a/scall-core/src/main/scala/io/chymyst/dhall/Parser.scala b/scall-core/src/main/scala/io/chymyst/dhall/Parser.scala index 8f81dc3d..2935a2d8 100644 --- a/scall-core/src/main/scala/io/chymyst/dhall/Parser.scala +++ b/scall-core/src/main/scala/io/chymyst/dhall/Parser.scala @@ -4,6 +4,7 @@ import fastparse._ import io.chymyst.dhall.Syntax.ExpressionScheme._ import io.chymyst.dhall.Syntax.{DhallFile, Expression} import io.chymyst.dhall.SyntaxConstants.FieldName +import io.chymyst.fastparse.Memoize import java.io.InputStream @@ -13,15 +14,23 @@ object Parser { } def parseToExpression(input: String): Expression = parseDhall(input) match { - case Parsed.Success(value, index) => value.value - case failure: Parsed.Failure => throw new Exception(s"Dhall parser error: ${failure.extra.trace().longMsg}") + case Parsed.Success(value: DhallFile, index) => value.value + case failure: Parsed.Failure => + Memoize.clearAll() // Parser will be re-run on trace(). So, the parser cache needs to be cleared. + throw new Exception(s"Dhall parser error: ${failure.extra.trace().longMsg}") } - def parseDhallBytes(source: Array[Byte]): Parsed[DhallFile] = parse(source, Grammar.complete_dhall_file(_)) + def parseDhallBytes(source: Array[Byte]): Parsed[DhallFile] = { + val init = System.nanoTime() + val result = Memoize.parse(source, Grammar.complete_dhall_file(_)) - def parseDhall(source: String): Parsed[DhallFile] = parse(source, Grammar.complete_dhall_file(_)) + // println(s"DEBUG: elapsed time is ${(System.nanoTime() - init)/1e9} for dhall bytes: ${new String(source.slice(0, 100))}") + result + } + + def parseDhall(source: String): Parsed[DhallFile] = Memoize.parse(source, Grammar.complete_dhall_file(_)) - def parseDhallStream(source: InputStream): Parsed[DhallFile] = parse(source, Grammar.complete_dhall_file(_)) + def parseDhallStream(source: InputStream): Parsed[DhallFile] = Memoize.parse(source, Grammar.complete_dhall_file(_)) private def localDateTimeZone(dateOption: Option[DateLiteral], timeOption: Option[TimeLiteral], zoneOption: Option[Int]): Expression = { val dateR = dateOption.map { date => (FieldName("date"), Expression(date)) } diff --git a/scall-core/src/test/scala/io/chymyst/dhall/unit/DhallParserAndCbor1Suite.scala b/scall-core/src/test/scala/io/chymyst/dhall/unit/DhallParserAndCbor1Suite.scala index 4a05b1d7..c02faf5d 100644 --- a/scall-core/src/test/scala/io/chymyst/dhall/unit/DhallParserAndCbor1Suite.scala +++ b/scall-core/src/test/scala/io/chymyst/dhall/unit/DhallParserAndCbor1Suite.scala @@ -26,7 +26,7 @@ class DhallParserAndCbor1Suite extends DhallTest { result } println(s"Success count: ${results.count(_.isSuccess)}\nFailure count: ${results.count(_.isFailure)}") - TestUtils.requireSuccessAtLeast(284, results) + TestUtils.requireSuccessAtLeast(286, results) } test("validate CBOR writing for standard examples") { diff --git a/scall-core/src/test/scala/io/chymyst/dhall/unit/DhallParserAndCbor2Suite.scala b/scall-core/src/test/scala/io/chymyst/dhall/unit/DhallParserAndCbor2Suite.scala index f3c98e62..7a68c0fd 100644 --- a/scall-core/src/test/scala/io/chymyst/dhall/unit/DhallParserAndCbor2Suite.scala +++ b/scall-core/src/test/scala/io/chymyst/dhall/unit/DhallParserAndCbor2Suite.scala @@ -37,7 +37,7 @@ class DhallParserAndCbor2Suite extends DhallTest { } result } - TestUtils.requireSuccessAtLeast(281, results) + TestUtils.requireSuccessAtLeast(286, results) } test("parse standard examples for failed parsing") { @@ -66,7 +66,7 @@ class DhallParserAndCbor2Suite extends DhallTest { if (result.exists(_.isFailure)) println(s"${file.getName}: failed parsing or converting file to CBOR: ${result.get.failed.get.getMessage}") result } - TestUtils.requireSuccessAtLeast(282, results) + TestUtils.requireSuccessAtLeast(286, results) } test("validate CBOR writing for standard examples") { @@ -145,7 +145,7 @@ class DhallParserAndCbor2Suite extends DhallTest { println(s"Success count: ${results.count(_.isSuccess)}\nFailure count: ${results .count(_.isFailure)}\nCBOR expression mismatch count: ${results.filter(_.isFailure).count(_.failed.get.getMessage.contains("expression differs"))}") results.filter(_.isFailure).map(_.failed.get.getMessage).foreach(println) - TestUtils.requireSuccessAtLeast(283, results) + TestUtils.requireSuccessAtLeast(286, results) } test("validate binary decoding/success") { diff --git a/scall-core/src/test/scala/io/chymyst/dhall/unit/DoNotationTest.scala b/scall-core/src/test/scala/io/chymyst/dhall/unit/DoNotationTest.scala index a0d40cf9..6101b936 100644 --- a/scall-core/src/test/scala/io/chymyst/dhall/unit/DoNotationTest.scala +++ b/scall-core/src/test/scala/io/chymyst/dhall/unit/DoNotationTest.scala @@ -104,11 +104,17 @@ class DoNotationTest extends DhallTest { test("parse do notation correctly") { "as Optional Natural in if a then b else c then d".dhall // No test failures. "as Optional Natural in if a then b else c with x : Text in y then d".dhall // No test failures. + } + + test("parse do notation and detect error 1") { expect( Try( "as Optional Natural in if a then b".dhall ).failed.get.getMessage contains "Dhall parser error: Expected complete_dhall_file:1:1 / complete_expression:1:1 / expression:1:1 / expression_as_in:1:1 / expression:1:24 / expression_if_then_else:1:24 / requireKeyword:1:35 / \"else\":1:35" ) + } + + test("parse do notation and detect error 2") { expect( Try( "as Optional Natural in if a then b else c".dhall diff --git a/scall-core/src/test/scala/io/chymyst/dhall/unit/ParserTest.scala b/scall-core/src/test/scala/io/chymyst/dhall/unit/ParserTest.scala index d2e31d83..19f502c8 100644 --- a/scall-core/src/test/scala/io/chymyst/dhall/unit/ParserTest.scala +++ b/scall-core/src/test/scala/io/chymyst/dhall/unit/ParserTest.scala @@ -1,7 +1,8 @@ package io.chymyst.dhall.unit import com.eed3si9n.expecty.Expecty.expect -import fastparse._ +import fastparse.Parsed +import io.chymyst.fastparse.Memoize.parse import io.chymyst.dhall.Grammar import io.chymyst.dhall.Syntax.Expression import io.chymyst.dhall.Syntax.ExpressionScheme._ diff --git a/scall-core/src/test/scala/io/chymyst/dhall/unit/SimpleExpressionTest.scala b/scall-core/src/test/scala/io/chymyst/dhall/unit/SimpleExpressionTest.scala index 4856c451..e10fb3b7 100644 --- a/scall-core/src/test/scala/io/chymyst/dhall/unit/SimpleExpressionTest.scala +++ b/scall-core/src/test/scala/io/chymyst/dhall/unit/SimpleExpressionTest.scala @@ -2,7 +2,7 @@ package io.chymyst.dhall.unit import com.eed3si9n.expecty.Expecty.expect import com.upokecenter.cbor.CBORObject -import fastparse.{Parsed, parse} +import fastparse.{Parsed, SingleChar} import io.chymyst.dhall.Syntax.ExpressionScheme._ import io.chymyst.dhall.Syntax.{DhallFile, Expression} import io.chymyst.dhall.SyntaxConstants.Builtin.{Natural, Text} @@ -13,6 +13,7 @@ import io.chymyst.dhall.SyntaxConstants.Operator.Equivalent import io.chymyst.dhall.SyntaxConstants._ import io.chymyst.dhall._ import io.chymyst.dhall.unit.TestUtils.{check, toFail, v} +import io.chymyst.fastparse.Memoize.parse import io.chymyst.test.Throwables.printThrowable import scala.util.Try @@ -46,8 +47,8 @@ class SimpleExpressionTest extends DhallTest { expect(parse("1 in", Grammar.application_expression(_)).get.value.scheme == NaturalLiteral(1)) - import fastparse._ - import NoWhitespace._ + import fastparse.P + import fastparse.NoWhitespace._ def grammar1[$: P] = P(Grammar.let_binding) expect(parse("let x = 1 ", grammar1(_)).get.value == expected) @@ -97,8 +98,8 @@ class SimpleExpressionTest extends DhallTest { } test("expression followed by comment") { - import fastparse._ - import NoWhitespace._ + import fastparse.P + import fastparse.NoWhitespace._ val input = "x {- -}" val expected = v("x") @@ -129,7 +130,6 @@ class SimpleExpressionTest extends DhallTest { } test("parse x === y") { - import fastparse._ val input = "x === y" val expected = ExprOperator[Expression](v("x"), Equivalent, v("y")) @@ -230,8 +230,8 @@ class SimpleExpressionTest extends DhallTest { test("invalid utf-8") { // The byte sequence 0xED, 0xA0, 0x80 is not a valid UTF-8 sequence. val input = Array(0x20, 0xed, 0xa0, 0x80, 0x20).map(_.toByte) - import fastparse._ - import NoWhitespace._ + import fastparse.P + import fastparse.NoWhitespace._ def grammar[$: P] = P(SingleChar.rep) val result = parse(input, grammar(_)) diff --git a/scall-core/src/test/scala/io/chymyst/dhall/unit/TestUtils.scala b/scall-core/src/test/scala/io/chymyst/dhall/unit/TestUtils.scala index 822dd4a1..a24961d5 100644 --- a/scall-core/src/test/scala/io/chymyst/dhall/unit/TestUtils.scala +++ b/scall-core/src/test/scala/io/chymyst/dhall/unit/TestUtils.scala @@ -1,7 +1,8 @@ package io.chymyst.dhall.unit import com.eed3si9n.expecty.Expecty.expect -import fastparse._ +import fastparse.{P, Parsed} +import io.chymyst.fastparse.Memoize.parse import io.chymyst.dhall.Syntax.Expression import io.chymyst.dhall.Syntax.ExpressionScheme.Variable import io.chymyst.dhall.{Semantics, SyntaxConstants, TypeCheck}