Skip to content

Commit

Permalink
Support scrooge-generator compiler flags in thrift_library rule (#6)
Browse files Browse the repository at this point in the history
  * Add option parsing file

  * Add OptionParser

  * Add scopt dependency

  * Lint reformat
  • Loading branch information
cattibrie authored and Borja Lorente committed Feb 22, 2021
1 parent e619e64 commit 19c6a71
Show file tree
Hide file tree
Showing 11 changed files with 285 additions and 62 deletions.
8 changes: 8 additions & 0 deletions scala/private/macros/scala_repositories.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,16 @@ def scala_repositories(
"io_bazel_rules_scala_scalactic",
"io_bazel_rules_scala_scala_xml",
"io_bazel_rules_scala_scala_parser_combinators",
# Remove this dependency when ScroogeConfig and ScroogeOptionParser are added to the com.twitter.scrooge
"io_bazel_rules_scala_scopt",
],
maven_servers = maven_servers,
fetch_sources = fetch_sources,
overriden_artifacts = overriden_artifacts,
)

# Remove this dependency when ScroogeConfig and ScroogeOptionParser are added to the com.twitter.scrooge
native.bind(
name = "io_bazel_rules_scala/dependency/scala/scopt",
actual = "@io_bazel_rules_scala_scopt",
)
13 changes: 13 additions & 0 deletions src/scala/io/bazel/rules_scala/scrooge_support/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,18 @@ scala_library(
visibility = ["//visibility:public"],
deps = [
"//twitter_scrooge:compiler_classpath",
# Remove this dependency when ScroogeConfig and ScroogeOptionParser are added to the com.twitter.scrooge
":option_parser",
],
)

# Remove this target when ScroogeConfig and ScroogeOptionParser are added to the com.twitter.scrooge
scala_library(
name = "option_parser",
srcs = ["ScroogeOptionParser.scala"],
visibility = ["//visibility:public"],
deps = [
"//external:io_bazel_rules_scala/dependency/scala/scopt",
"//external:io_bazel_rules_scala/dependency/thrift/scrooge_generator",
],
)
59 changes: 23 additions & 36 deletions src/scala/io/bazel/rules_scala/scrooge_support/Compiler.scala
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@ package io.bazel.rules_scala.scrooge_support

import com.twitter.scrooge._
import com.twitter.scrooge.ast.Document
import com.twitter.scrooge.backend.{GeneratorFactory, ScalaGenerator, ServiceOption}
import com.twitter.scrooge.backend.{GeneratorFactory, ScalaGenerator}
import com.twitter.scrooge.frontend.{FileParseException, TypeResolver, ThriftParser, Importer, MultiImporter, ZipImporter}
import com.twitter.scrooge.java_generator.ApacheJavaGenerator
import java.io.{File, FileWriter}
import java.nio.file.Paths
import java.util.jar.{ JarFile, JarEntry }
Expand All @@ -44,9 +45,6 @@ import scala.collection.mutable
import scala.collection.JavaConverters._

object CompilerDefaults {
var language: String = "scala"
var defaultNamespace: String = "thrift"

def listJar(_jar: File): List[String] =
try {
val files = List.newBuilder[String]
Expand All @@ -66,46 +64,34 @@ object CompilerDefaults {
}
}

class Compiler {
val defaultDestFolder = "."
var destFolder: String = defaultDestFolder
class Compiler(val config: ScroogeConfig) {
// These are jars we are including, but are not compiling
val includeJars = new mutable.ListBuffer[String]
val includeJars = config.includePaths
// these are the jars we want to compile into scala source jars
val compileJars = new mutable.ListBuffer[String]
val flags = new mutable.HashSet[ServiceOption]
val namespaceMappings = new mutable.HashMap[String, String]
var verbose = false
var strict = true
var skipUnchanged = false
var experimentFlags = new mutable.ListBuffer[String]
var fileMapPath: scala.Option[String] = None
val compileJars = config.thriftFiles
val experimentFlags = config.languageFlags
var fileMapWriter: scala.Option[FileWriter] = None
var dryRun: Boolean = false
var language: String = CompilerDefaults.language
var defaultNamespace: String = CompilerDefaults.defaultNamespace
var scalaWarnOnJavaNSFallback: Boolean = false


def run() {
// if --gen-file-map is specified, prepare the map file.
fileMapWriter = fileMapPath.map { path =>
fileMapWriter = config.fileMapPath.map { path =>
val file = new File(path)
val dir = file.getParentFile
if (dir != null && !dir.exists()) {
dir.mkdirs()
}
if (verbose) {
if (config.verbose) {
println("+ Writing file mapping to %s".format(path))
}
new FileWriter(file)
}

val allJars: List[File] =
((includeJars.toList) ::: (compileJars.toList))
((includeJars) ::: (compileJars))
.map { path => (new File(path)).getCanonicalFile }

val isJava = language.equals("java")
val isJava = config.language.equals("java")
val documentCache = new TrieMap[String, Document]

// compile
Expand All @@ -126,36 +112,37 @@ class Compiler {
val importer = rootImporter.copy(focus = focus) +: rootImporter
val parser = new ThriftParser(
importer,
strict,
config.strict,
defaultOptional = isJava,
skipIncludes = false,
documentCache
)
parser.logger.setLevel(Level.OFF) // scrooge warns on file names with "/"
val doc = parser.parseFile(inputFile).mapNamespaces(namespaceMappings.toMap)
val doc = parser.parseFile(inputFile).mapNamespaces(config.namespaceMappings)

if (verbose) println("+ Compiling %s".format(inputFile))
if (config.verbose) println("+ Compiling %s".format(inputFile))
val resolvedDoc = TypeResolver()(doc)
val generator = GeneratorFactory(
language,
config.language,
resolvedDoc,
defaultNamespace,
experimentFlags.toList
)
config.defaultNamespace,
experimentFlags)

generator match {
case g: ScalaGenerator => g.warnOnJavaNamespaceFallback = scalaWarnOnJavaNSFallback
case g: ScalaGenerator => g.warnOnJavaNamespaceFallback = config.scalaWarnOnJavaNSFallback
case g: ApacheJavaGenerator => g.serEnumType = config.javaSerEnumType
case _ => ()
}

val generatedFiles = generator(
flags.toSet,
new File(destFolder),
dryRun
config.flags,
new File(config.destFolder),
config.dryRun,
config.genAdapt
).map {
_.getPath
}
if (verbose) {
if (config.verbose) {
println("+ Generated %s".format(generatedFiles.mkString(", ")))
}
fileMapWriter.foreach { w =>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
/*
* Copyright 2020 Twitter, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License. You may obtain
* a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.bazel.rules_scala.scrooge_support

import com.twitter.scrooge.backend.{
GeneratorFactory,
WithFinagle,
WithJavaPassThrough,
ServiceOption
}
import java.io.File
import java.util.Properties
import scopt.OptionParser

case class ScroogeConfig(
destFolder: String = ".",
includePaths: List[String] = List(),
thriftFiles: List[String] = List(),
flags: Set[ServiceOption] = Set(),
namespaceMappings: Map[String, String] = Map(),
verbose: Boolean = false,
strict: Boolean = true,
genAdapt: Boolean = false,
skipUnchanged: Boolean = false,
languageFlags: Seq[String] = Seq(),
fileMapPath: scala.Option[String] = None,
dryRun: Boolean = false,
language: String = "scala",
defaultNamespace: String = "thrift",
scalaWarnOnJavaNSFallback: Boolean = false,
javaSerEnumType: Boolean = false)

object ScroogeOptionParser {

/** Optionally returns config with parsed values from args.
* @param args command line arguments.
* @param defaultConfig config with configurable defaults that is used to store parsed args.
*/
def parseOptions(
args: Seq[String],
defaultConfig: ScroogeConfig = ScroogeConfig()
): Option[ScroogeConfig] = {
val buildProperties = new Properties
scala.Option(getClass.getResource("build.properties")) foreach { resource =>
buildProperties.load(resource.openStream)
}

val parser = new OptionParser[ScroogeConfig]("scrooge") {
help("help").text("show this help screen")

override def showUsageOnError: Option[Boolean] = Some(true)

opt[Unit]('V', "version")
.action { (_, c) =>
println("scrooge " + buildProperties.getProperty("version", "0.0"))
println(" build " + buildProperties.getProperty("build_name", "unknown"))
println(" git revision " + buildProperties.getProperty("build_revision", "unknown"))
System.exit(0)
c
}
.text("print version and quit")

opt[Unit]('v', "verbose")
.action((_, c) => c.copy(verbose = true))
.text("log verbose messages about progress")

opt[String]('d', "dest")
.valueName("<path>")
.action((d, c) => c.copy(destFolder = d))
.text("write generated code to a folder (default: %s)".format(defaultConfig.destFolder))

opt[String]("import-path")
.unbounded()
.valueName("<path>")
.action { (path, c) =>
val includePaths = path.split(File.pathSeparator) ++: c.includePaths
c.copy(includePaths = includePaths)
}
.text(
"[DEPRECATED] path(s) to search for included thrift files (may be used multiple times)"
)

opt[String]('i', "include-path")
.unbounded()
.valueName("<path>")
.action { (path, c) =>
val includePaths = path.split(File.pathSeparator) ++: c.includePaths
c.copy(includePaths = includePaths)
}
.text("path(s) to search for included thrift files (may be used multiple times)")

opt[String]('n', "namespace-map")
.unbounded()
.valueName("<oldname>=<newname>")
.action { (mapping, c) =>
mapping.split("=") match {
case Array(from, to) =>
c.copy(namespaceMappings = c.namespaceMappings + (from -> to))
}
}
.text("map old namespace to new (may be used multiple times)")

opt[String]("default-java-namespace")
.unbounded()
.valueName("<name>")
.action((name, c) => c.copy(defaultNamespace = name))
.text(
"Use <name> as default namespace if the thrift file doesn't define its own namespace. " +
"If this option is not specified either, then use \"thrift\" as default namespace"
)

opt[Unit]("disable-strict")
.action((_, c) => c.copy(strict = false))
.text("issue warnings on non-severe parse errors instead of aborting")

opt[String]("gen-file-map")
.valueName("<path>")
.action((path, c) => c.copy(fileMapPath = Some(path)))
.text(
"generate map.txt in the destination folder to specify the mapping from input thrift files to output Scala/Java files"
)

opt[Unit]("dry-run")
.action((_, c) => c.copy(dryRun = true))
.text(
"parses and validates source thrift files, reporting any errors, but" +
" does not emit any generated source code. can be used with " +
"--gen-file-mapping to get the file mapping"
)

opt[Unit]('s', "skip-unchanged")
.action((_, c) => c.copy(skipUnchanged = true))
.text("Don't re-generate if the target is newer than the input")

opt[String]('l', "language")
.action((languageString, c) => c.copy(language = languageString))
.validate { language =>
if (GeneratorFactory.languages.toList contains language.toLowerCase)
success
else
failure("language option %s not supported".format(language))
}
.text(
"name of language to generate code in (currently supported languages: " +
GeneratorFactory.languages.toList.mkString(", ") + ")"
)

opt[Unit]("java-ser-enum-type")
.action((_, c) => c.copy(javaSerEnumType = true))
.text("Encode a thrift enum as o.a.t.p.TType.ENUM instead of TType.I32")

opt[String]("language-flag")
.valueName("<flag>")
.unbounded()
.action { (flag, c) =>
val languageFlags = c.languageFlags :+ flag
c.copy(languageFlags = languageFlags)
}
.text(
"Pass arguments to supported language generators"
)

opt[Unit]("scala-warn-on-java-ns-fallback")
.action((_, c) => c.copy(scalaWarnOnJavaNSFallback = true))
.text("Print a warning when the scala generator falls back to the java namespace")

opt[Unit]("finagle")
.action((_, c) => c.copy(flags = c.flags + WithFinagle))
.text("generate finagle classes")

opt[Unit]("gen-adapt")
.action((_, c) => c.copy(genAdapt = true))
.text("Generate code for adaptive decoding for scala.")

opt[Unit]("java-passthrough")
.action((_, c) => c.copy(flags = c.flags + WithJavaPassThrough))
.text("Generate Java code with PassThrough")

arg[String]("<files...>")
.unbounded()
.action { (files, c) =>
// append files to preserve the order of thrift files from command line.
val thriftFiles = c.thriftFiles :+ files
c.copy(thriftFiles = thriftFiles)
}
.text("thrift files to compile")
}
parser.parse(args, defaultConfig)
}
}
2 changes: 2 additions & 0 deletions src/scala/scripts/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ scala_library(
"//src/java/io/bazel/rulesscala/worker",
"//src/scala/io/bazel/rules_scala/scrooge_support:compiler",
"//twitter_scrooge:scrooge_generator_classpath",
# Remove this dependency when ScroogeConfig and ScroogeOptionParser are added to the com.twitter.scrooge
"//src/scala/io/bazel/rules_scala/scrooge_support:option_parser",
],
)

Expand Down
Loading

0 comments on commit 19c6a71

Please sign in to comment.