From 973e16d78fab0cf8ad6751b6a94db7277286e93c Mon Sep 17 00:00:00 2001 From: Corbin McNeely-Smith <58151731+restingbull@users.noreply.github.com> Date: Tue, 26 Mar 2024 22:57:37 -0500 Subject: [PATCH] =?UTF-8?q?Reapply=20"[kt=5Fcompiler=5Fplugin]=20Add=20kt?= =?UTF-8?q?=5Fplugin=5Fcfg=20for=20reusable=20and=20comp=E2=80=A6=20=20=20?= =?UTF-8?q?=E2=80=A6osable=20op=E2=80=A6"=20(#1131)=20=20(#1135)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Reapply "[kt_compiler_plugin] Add kt_plugin_cfg for reusable and composable op…" (#1131) This reverts commit 00ef838f05ea9ce56dc7e44a19f4afc7f0cbadee. * Cleanup lint, etc --- MODULE.bazel | 2 + docs/kotlin.md | 27 +- examples/ksp/BUILD | 72 +++- examples/ksp/MODULE.bazel | 6 + kotlin/core.bzl | 2 + kotlin/internal/BUILD | 1 + kotlin/internal/compiler_plugins.bzl | 21 - kotlin/internal/defs.bzl | 27 +- kotlin/internal/jvm/compile.bzl | 139 +++++-- kotlin/internal/jvm/impl.bzl | 68 +++- kotlin/internal/jvm/jvm.bzl | 42 +- scripts/install_android_sdk | 2 +- scripts/release.sh | 9 +- .../builder/tasks/jvm/CompilationTask.kt | 6 +- src/main/starlark/core/BUILD.bazel | 3 +- src/main/starlark/core/options/BUILD.bazel | 7 +- src/main/starlark/core/plugin/BUILD.bazel | 18 + .../starlark/core/plugin/BUILD.release.bazel | 7 + src/main/starlark/core/plugin/providers.bzl | 34 ++ .../BUILD.com_github_jetbrains_kotlin.bazel | 7 + .../starlark/core/repositories/versions.bzl | 7 + src/test/starlark/case.bzl | 33 ++ src/test/starlark/core/plugin/BUILD.bazel | 5 + src/test/starlark/core/plugin/subjects.bzl | 37 ++ src/test/starlark/core/plugin/test.bzl | 380 ++++++++++++++++++ src/test/starlark/truth.bzl | 41 ++ 26 files changed, 899 insertions(+), 104 deletions(-) create mode 100644 examples/ksp/MODULE.bazel delete mode 100644 kotlin/internal/compiler_plugins.bzl create mode 100644 src/main/starlark/core/plugin/BUILD.bazel create mode 100644 src/main/starlark/core/plugin/BUILD.release.bazel create mode 100644 src/main/starlark/core/plugin/providers.bzl create mode 100644 src/test/starlark/case.bzl create mode 100644 src/test/starlark/core/plugin/BUILD.bazel create mode 100644 src/test/starlark/core/plugin/subjects.bzl create mode 100644 src/test/starlark/core/plugin/test.bzl create mode 100644 src/test/starlark/truth.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 18fa33121..c125a9b56 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -91,3 +91,5 @@ use_repo(maven, "kotlin_rules_maven", "unpinned_kotlin_rules_maven") bazel_dep(name = "rules_pkg", version = "0.7.0") bazel_dep(name = "stardoc", version = "0.5.6", repo_name = "io_bazel_stardoc") bazel_dep(name = "rules_proto", version = "5.3.0-21.7") + +bazel_dep(name = "rules_testing", version = "0.5.0", dev_dependency = True) diff --git a/docs/kotlin.md b/docs/kotlin.md index 293e3e7c0..f98c1e6be 100755 --- a/docs/kotlin.md +++ b/docs/kotlin.md @@ -361,7 +361,7 @@ kt_compiler_plugin(name, compile_phase | Runs the compiler plugin during kotlin compilation. Known examples: allopen, sam_with_reciever | Boolean | optional | True | |deps | The list of libraries to be added to the compiler's plugin classpath | List of labels | optional | [] | |id | The ID of the plugin | String | required | | -|options | Dictionary of options to be passed to the plugin. Supports the following template values:

- {generatedClasses}: directory for generated class output - {temp}: temporary directory, discarded between invocations - {generatedSources}: directory for generated source output | Dictionary: String -> String | optional | {} | +|options | Dictionary of options to be passed to the plugin. Supports the following template values:

- {generatedClasses}: directory for generated class output - {temp}: temporary directory, discarded between invocations - {generatedSources}: directory for generated source output - {classpath} : replaced with a list of jars separated by the filesystem appropriate separator. | Dictionary: String -> String | optional | {} | |stubs_phase | Runs the compiler plugin in kapt stub generation. | Boolean | optional | True | |target_embedded_compiler | Plugin was compiled against the embeddable kotlin compiler. These plugins expect shaded kotlinc dependencies, and will fail when running against a non-embeddable compiler. | Boolean | optional | False | @@ -482,6 +482,31 @@ kt_ksp_plugin(name, processor_class | The fully qualified class name that the Java compiler uses as an entry point to the annotation processor. | String | required | | + + +## kt_plugin_cfg + +kt_plugin_cfg(name, deps, options, plugin) + + + + Configurations for kt_compiler_plugin, ksp_plugin, and java_plugin. + + This allows setting options and dependencies independently from the initial plugin definition. + + + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +|name | A unique name for this target. | Name | required | | +|deps | Dependencies for this configuration. | List of labels | optional | [] | +|options | A dictionary of flag to values to be used as plugin configuration options. | Dictionary: String -> List of strings | optional | {} | +|plugin | The plugin to associate with this configuration | Label | required | | + + ## define_kt_toolchain diff --git a/examples/ksp/BUILD b/examples/ksp/BUILD index 0c4a5acdf..6722c96f2 100644 --- a/examples/ksp/BUILD +++ b/examples/ksp/BUILD @@ -14,7 +14,7 @@ load("@rules_java//java:defs.bzl", "java_binary", "java_plugin") # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -load("@rules_kotlin//kotlin:core.bzl", "define_kt_toolchain", "kt_ksp_plugin") +load("@rules_kotlin//kotlin:core.bzl", "define_kt_toolchain", "kt_compiler_plugin", "kt_ksp_plugin", "kt_plugin_cfg") load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") package(default_visibility = ["//visibility:public"]) @@ -81,3 +81,73 @@ build_test( "//:coffee_app_deploy.jar", ], ) + +kt_compiler_plugin( + name = "ksp", + compile_phase = True, + id = "com.google.devtools.ksp.symbol-processing", + options = { + "apclasspath": "{classpath}", + # projectBaseDir shouldn't matter because incremental is disabled + "projectBaseDir": "{temp}", + # Disable incremental mode + "incremental": "false", + # Directory where class files are written to. Files written to this directory are class + # files being written directly from the annotation processor, not Kotlinc + "classOutputDir": "{generatedClasses}", + # Directory where generated Java sources files are written to + "javaOutputDir": "{generatedSources}", + # Directory where generated Kotlin sources files are written to + "kotlinOutputDir": "{generatedSources}", + # Directory where META-INF data is written to. This might not be the most ideal place to + # write this. Maybe just directly to the classes directory? + "resourceOutputDir": "{generatedSources}", + # TODO(bencodes) Not sure what this directory is yet. + "kspOutputDir": "{temp}", + # Directory to write KSP caches. Shouldn't matter because incremental is disabled + "cachesDir": "{temp}", + # Include in compilation as an example. This should be processed in the stubs phase. + "withCompilation": "true", + # Set returnOkOnError to false because we want to fail the build if there are any errors + "returnOkOnError": "false", + "allWarningsAsErrors": "false", + }, + deps = [ + "@rules_kotlin//kotlin/compiler:symbol-processing-api", + "@rules_kotlin//kotlin/compiler:symbol-processing-cmdline", + ], +) + +kt_plugin_cfg( + name = "ksp_moshi", + options = { + }, + plugin = ":ksp", + deps = [ + "@maven//:com_squareup_moshi_moshi", + "@maven//:com_squareup_moshi_moshi_kotlin", + "@maven//:com_squareup_moshi_moshi_kotlin_codegen", + ], +) + +kt_jvm_library( + name = "raw_ksp_coffee_app_lib", + srcs = ["CoffeeAppModel.kt"], + plugins = [ + "//:ksp", + "//:ksp_moshi", + ], + deps = [ + "@maven//:com_google_auto_service_auto_service_annotations", + "@maven//:com_google_auto_value_auto_value_annotations", + "@maven//:com_squareup_moshi_moshi", + "@maven//:com_squareup_moshi_moshi_kotlin", + ], +) + +build_test( + name = "raw_ksp_lib_test", + targets = [ + "//:raw_ksp_coffee_app_lib", + ], +) diff --git a/examples/ksp/MODULE.bazel b/examples/ksp/MODULE.bazel new file mode 100644 index 000000000..00bb18361 --- /dev/null +++ b/examples/ksp/MODULE.bazel @@ -0,0 +1,6 @@ +############################################################################### +# Bazel now uses Bzlmod by default to manage external dependencies. +# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel. +# +# For more details, please check https://github.com/bazelbuild/bazel/issues/18958 +############################################################################### diff --git a/kotlin/core.bzl b/kotlin/core.bzl index a43a3eb3b..3b9dd46d8 100644 --- a/kotlin/core.bzl +++ b/kotlin/core.bzl @@ -12,6 +12,7 @@ load( "//kotlin/internal/jvm:jvm.bzl", _kt_compiler_plugin = "kt_compiler_plugin", _kt_ksp_plugin = "kt_ksp_plugin", + _kt_plugin_cfg = "kt_plugin_cfg", ) define_kt_toolchain = _define_kt_toolchain @@ -20,3 +21,4 @@ kt_javac_options = _kt_javac_options kt_kotlinc_options = _kt_kotlinc_options kt_compiler_plugin = _kt_compiler_plugin kt_ksp_plugin = _kt_ksp_plugin +kt_plugin_cfg = _kt_plugin_cfg diff --git a/kotlin/internal/BUILD b/kotlin/internal/BUILD index 223323d20..9cfa710f2 100644 --- a/kotlin/internal/BUILD +++ b/kotlin/internal/BUILD @@ -45,6 +45,7 @@ bzl_library( "//kotlin/internal/lint", "//kotlin/internal/utils", "//src/main/starlark", + "//src/main/starlark/core/options", "@rules_java//java:rules", ], ) diff --git a/kotlin/internal/compiler_plugins.bzl b/kotlin/internal/compiler_plugins.bzl deleted file mode 100644 index 4306a0d40..000000000 --- a/kotlin/internal/compiler_plugins.bzl +++ /dev/null @@ -1,21 +0,0 @@ -load( - "//kotlin/internal:defs.bzl", - _KtCompilerPluginInfo = "KtCompilerPluginInfo", -) - -def plugins_to_classpaths(providers_list): - flattened_files = [] - for providers in providers_list: - if _KtCompilerPluginInfo in providers: - provider = providers[_KtCompilerPluginInfo] - for e in provider.classpath: - flattened_files.append(e) - return flattened_files - -def plugins_to_options(providers_list): - kt_compiler_plugin_providers = [providers[_KtCompilerPluginInfo] for providers in providers_list if _KtCompilerPluginInfo in providers] - flattened_options = [] - for provider in kt_compiler_plugin_providers: - for option in provider.options: - flattened_options.append("%s:%s" % (option.id, option.value)) - return flattened_options diff --git a/kotlin/internal/defs.bzl b/kotlin/internal/defs.bzl index 498ce0645..7981fe438 100644 --- a/kotlin/internal/defs.bzl +++ b/kotlin/internal/defs.bzl @@ -11,6 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.# +load( + "//src/main/starlark/core/plugin:providers.bzl", + _KspPluginInfo = "KspPluginInfo", + _KtCompilerPluginInfo = "KtCompilerPluginInfo", + _KtCompilerPluginOption = "KtCompilerPluginOption", + _KtPluginConfiguration = "KtPluginConfiguration", +) # The Kotlin Toolchain type. TOOLCHAIN_TYPE = "%s" % Label("//kotlin/internal:kt_toolchain_type") @@ -50,18 +57,10 @@ KtJsInfo = provider( }, ) -KtCompilerPluginInfo = provider( - fields = { - "plugin_jars": "List of plugin jars.", - "classpath": "The kotlin compiler plugin classpath.", - "stubs": "Run this plugin during kapt stub generation.", - "compile": "Run this plugin during koltinc compilation.", - "options": "List of plugin options, represented as structs with an id and a value field, to be passed to the compiler", - }, -) +KtCompilerPluginInfo = _KtCompilerPluginInfo -KspPluginInfo = provider( - fields = { - "plugins": "List of JavaPLuginInfo providers for the plugins to run with KSP", - }, -) +KspPluginInfo = _KspPluginInfo + +KtCompilerPluginOption = _KtCompilerPluginOption + +KtPluginConfiguration = _KtPluginConfiguration diff --git a/kotlin/internal/jvm/compile.bzl b/kotlin/internal/jvm/compile.bzl index 93889affc..f13b938b3 100644 --- a/kotlin/internal/jvm/compile.bzl +++ b/kotlin/internal/jvm/compile.bzl @@ -1,9 +1,3 @@ -load( - "@bazel_tools//tools/jdk:toolchain_utils.bzl", - "find_java_runtime_toolchain", - "find_java_toolchain", -) - # Copyright 2018 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,11 +11,21 @@ load( # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -load("@rules_java//java:defs.bzl", "JavaInfo", "java_common") +load( + "@bazel_tools//tools/jdk:toolchain_utils.bzl", + "find_java_runtime_toolchain", + "find_java_toolchain", +) +load( + "@rules_java//java:defs.bzl", + "JavaInfo", + "java_common", +) load( "//kotlin/internal:defs.bzl", _KtCompilerPluginInfo = "KtCompilerPluginInfo", _KtJvmInfo = "KtJvmInfo", + _KtPluginConfiguration = "KtPluginConfiguration", _TOOLCHAIN_TYPE = "TOOLCHAIN_TYPE", ) load( @@ -178,9 +182,83 @@ def _adjust_resources_path(path, resource_strip_prefix): else: return _adjust_resources_path_by_default_prefixes(path) -def _format_compile_plugin_options(options): - """Format options into id:value for cmd line.""" - return ["%s:%s" % (o.id, o.value) for o in options] +def _format_compile_plugin_options(o): + """Format compiler option into id:value for cmd line.""" + return [ + "%s:%s" % (o.id, o.value), + ] + +def _new_plugins_from(targets): + """Returns a struct containing the plugin metadata for the given targets. + + Args: + targets: A list of targets. + Returns: + A struct containing the plugins for the given targets in the format: + { + stubs_phase = { + classpath = depset, + options= List[KtCompilerPluginOption], + ), + compile = { + classpath = depset, + options = List[KtCompilerPluginOption], + }, + } + """ + + all_plugins = {} + plugins_without_phase = [] + for t in targets: + if _KtCompilerPluginInfo not in t: + continue + plugin = t[_KtCompilerPluginInfo] + if not (plugin.stubs or plugin.compile): + plugins_without_phase.append("%s: %s" % (t.label, plugin.id)) + if plugin.id in all_plugins: + # This need a more robust error messaging. + fail("has multiple plugins with the same id: %s." % plugin.id) + all_plugins[plugin.id] = plugin + + if plugins_without_phase: + fail("has plugin without a phase defined: %s" % cfgs_without_plugin) + + all_plugin_cfgs = {} + cfgs_without_plugin = [] + for t in targets: + if _KtPluginConfiguration not in t: + continue + cfg = t[_KtPluginConfiguration] + if cfg.id not in all_plugins: + cfgs_without_plugin.append("%s: %s" % (t.label, cfg.id)) + all_plugin_cfgs[cfg.id] = cfg + + if cfgs_without_plugin: + fail("has plugin configurations without corresponding plugins: %s" % cfgs_without_plugin) + + return struct( + stubs_phase = _new_plugin_from(all_plugin_cfgs, [p for p in all_plugins.values() if p.stubs]), + compile_phase = _new_plugin_from(all_plugin_cfgs, [p for p in all_plugins.values() if p.compile]), + ) + +def _new_plugin_from(all_cfgs, plugins_for_phase): + classpath = [] + data = [] + options = [] + for p in plugins_for_phase: + classpath.append(p.classpath) + options.extend(p.options) + if p.id in all_cfgs: + cfg = all_cfgs[p.id] + classpath.append(cfg.classpath) + data.append(cfg.data) + options.extend(cfg.options) + + return struct( + classpath = depset(transitive = classpath), + data = depset(transitive = data), + options = options, + ) # INTERNAL ACTIONS ##################################################################################################### def _fold_jars_action(ctx, rule_kind, toolchains, output_jar, input_jars, action_type = ""): @@ -418,49 +496,28 @@ def _run_kt_builder_action( uniquify = True, ) - compiler_plugins = [ - p[_KtCompilerPluginInfo] - for p in plugins - if _KtCompilerPluginInfo in p and p[_KtCompilerPluginInfo] - ] - - stubs_compiler_plugins = [ - kcp - for kcp in compiler_plugins - if kcp.stubs - ] - - compiler_compiler_plugins = [ - ccp - for ccp in compiler_plugins - if ccp.compile - ] - - if compiler_plugins and not (stubs_compiler_plugins or compiler_compiler_plugins): - fail("plugins but no phase plugins: %s" % compiler_plugins) - args.add_all( "--stubs_plugin_classpath", - depset(transitive = [p.classpath for p in stubs_compiler_plugins]), + plugins.stubs_phase.classpath, omit_if_empty = True, ) args.add_all( "--stubs_plugin_options", - [p.options for p in stubs_compiler_plugins], + plugins.stubs_phase.options, map_each = _format_compile_plugin_options, omit_if_empty = True, ) args.add_all( "--compiler_plugin_classpath", - depset(transitive = [p.classpath for p in compiler_compiler_plugins]), + plugins.compile_phase.classpath, omit_if_empty = True, ) args.add_all( "--compiler_plugin_options", - [p.options for p in compiler_compiler_plugins], + plugins.compile_phase.options, map_each = _format_compile_plugin_options, omit_if_empty = True, ) @@ -479,7 +536,13 @@ def _run_kt_builder_action( mnemonic = mnemonic, inputs = depset( srcs.all_srcs + srcs.src_jars + generated_src_jars, - transitive = [compile_deps.compile_jars, transitive_runtime_jars, deps_artifacts] + [p.classpath for p in compiler_plugins], + transitive = [ + compile_deps.compile_jars, + transitive_runtime_jars, + deps_artifacts, + plugins.stubs_phase.classpath, + plugins.compile_phase.classpath, + ], ), tools = [ toolchains.kt.kotlinbuilder.files_to_run, @@ -521,10 +584,12 @@ def kt_jvm_produce_jar_actions(ctx, rule_kind): deps = ctx.attr.deps, runtime_deps = ctx.attr.runtime_deps, ) + annotation_processors = _plugin_mappers.targets_to_annotation_processors(ctx.attr.plugins + ctx.attr.deps) ksp_annotation_processors = _plugin_mappers.targets_to_ksp_annotation_processors(ctx.attr.plugins + ctx.attr.deps) transitive_runtime_jars = _plugin_mappers.targets_to_transitive_runtime_jars(ctx.attr.plugins + ctx.attr.deps) - plugins = ctx.attr.plugins + _exported_plugins(deps = ctx.attr.deps) + plugins = _new_plugins_from(ctx.attr.plugins + _exported_plugins(deps = ctx.attr.deps)) + deps_artifacts = _deps_artifacts(toolchains, ctx.attr.deps + associates.targets) generated_src_jars = [] diff --git a/kotlin/internal/jvm/impl.bzl b/kotlin/internal/jvm/impl.bzl index 1b2e51d4d..9d48e020d 100644 --- a/kotlin/internal/jvm/impl.bzl +++ b/kotlin/internal/jvm/impl.bzl @@ -1,12 +1,3 @@ -load("@rules_java//java:defs.bzl", "JavaInfo", "JavaPluginInfo", "java_common") -load( - "//kotlin/internal:defs.bzl", - _KspPluginInfo = "KspPluginInfo", - _KtCompilerPluginInfo = "KtCompilerPluginInfo", - _KtJvmInfo = "KtJvmInfo", - _TOOLCHAIN_TYPE = "TOOLCHAIN_TYPE", -) - # Copyright 2018 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,6 +11,17 @@ load( # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +load("@rules_java//java:defs.bzl", "JavaInfo", "JavaPluginInfo", "java_common") +load( + "//kotlin/internal:defs.bzl", + "KtCompilerPluginOption", + "KtPluginConfiguration", + _KspPluginInfo = "KspPluginInfo", + _KtCompilerPluginInfo = "KtCompilerPluginInfo", + _KtJvmInfo = "KtJvmInfo", + _TOOLCHAIN_TYPE = "TOOLCHAIN_TYPE", +) load( "//kotlin/internal/jvm:compile.bzl", "export_only_providers", @@ -392,16 +394,39 @@ def _deshade_embedded_kotlinc_jars(target, ctx, jars, deps): ], ) -def kt_compiler_plugin_impl(ctx): - plugin_id = ctx.attr.id +def _resolve_plugin_options(id, string_list_dict, expand_location): + """ + Resolves plugin options from a string dict to a dict of strings. + + Args: + id: the plugin id + string_list_dict: a dict of list[string]. + Returns: + a dict of strings + """ options = [] - for (k, v) in ctx.attr.options.items(): - if "=" in k: - fail("kt_compiler_plugin options keys cannot contain the = symbol") - options.append(struct(id = plugin_id, value = "%s=%s" % (k, v))) + for (k, vs) in string_list_dict.items(): + for v in vs: + if "=" in k: + fail("kotlin compiler option keys cannot contain the = symbol") + value = k + "=" + expand_location(v) if v else k + options.append(KtCompilerPluginOption(id = id, value = value)) + return options + +# This is naive reference implementation for resolving configurations. +# A more complicated plugin will need to provide its own implementation. +def _resolve_plugin_cfg(info, options, deps, expand_location): + ji = java_common.merge([dep[JavaInfo] for dep in deps if JavaInfo in dep]) + classpath = depset(ji.runtime_output_jars, transitive = [ji.transitive_runtime_jars]) + return KtPluginConfiguration( + id = info.id, + options = _resolve_plugin_options(info.id, options, expand_location), + classpath = classpath, + data = depset(), + ) - if not (ctx.attr.compile_phase or ctx.attr.stubs_phase): - fail("Plugin must execute during in one or more phases: stubs_phase, compile_phase") +def kt_compiler_plugin_impl(ctx): + plugin_id = ctx.attr.id deps = ctx.attr.deps info = None @@ -420,16 +445,25 @@ def kt_compiler_plugin_impl(ctx): classpath = depset(info.runtime_output_jars, transitive = [info.transitive_runtime_jars]) + # TODO(1035): Migrate kt_compiler_plugin.options to string_list_dict + options = _resolve_plugin_options(plugin_id, {k: [v] for (k, v) in ctx.attr.options.items()}, ctx.expand_location) + return [ DefaultInfo(files = classpath), _KtCompilerPluginInfo( + id = plugin_id, classpath = classpath, options = options, stubs = ctx.attr.stubs_phase, compile = ctx.attr.compile_phase, + resolve_cfg = _resolve_plugin_cfg, ), ] +def kt_plugin_cfg_impl(ctx): + plugin = ctx.attr.plugin[_KtCompilerPluginInfo] + return plugin.resolve_cfg(plugin, ctx.attr.options, ctx.attr.deps, ctx.expand_location) + def kt_ksp_plugin_impl(ctx): info = java_common.merge([dep[JavaInfo] for dep in ctx.attr.deps]) classpath = depset(info.runtime_output_jars, transitive = [info.transitive_runtime_jars]) diff --git a/kotlin/internal/jvm/jvm.bzl b/kotlin/internal/jvm/jvm.bzl index bef495e43..2f20de6d7 100644 --- a/kotlin/internal/jvm/jvm.bzl +++ b/kotlin/internal/jvm/jvm.bzl @@ -95,6 +95,7 @@ kt_jvm_binary( load("@rules_java//java:defs.bzl", "JavaInfo") load( "//kotlin/internal:defs.bzl", + "KtPluginConfiguration", _JAVA_RUNTIME_TOOLCHAIN_TYPE = "JAVA_RUNTIME_TOOLCHAIN_TYPE", _JAVA_TOOLCHAIN_TYPE = "JAVA_TOOLCHAIN_TYPE", _KspPluginInfo = "KspPluginInfo", @@ -109,6 +110,7 @@ load( ) load( "//kotlin/internal/jvm:impl.bzl", + "kt_plugin_cfg_impl", _kt_compiler_deps_aspect_impl = "kt_compiler_deps_aspect_impl", _kt_compiler_plugin_impl = "kt_compiler_plugin_impl", _kt_jvm_binary_impl = "kt_jvm_binary_impl", @@ -172,6 +174,7 @@ _common_attr = utils.add_dicts( [Attributes common to all build rules](https://docs.bazel.build/versions/master/be/common-definitions.html#common-attributes).""", providers = [ [JavaInfo], + [_KtJvmInfo], ], allow_files = False, ), @@ -211,6 +214,13 @@ _common_attr = utils.add_dicts( "plugins": attr.label_list( default = [], cfg = "exec", + providers = [ + [JavaPluginInfo], + [KtPluginConfiguration], + [_KspPluginInfo], + [_KtCompilerPluginInfo], + [_KtCompilerPluginInfo], + ], ), "module_name": attr.string( doc = """The name of the module, if not provided the module name is derived from the label. --e.g., @@ -253,7 +263,7 @@ Compiler plugins listed here will be treated as if they were added in the plugin of any targets that directly depend on this target. Unlike `java_plugin`s exported_plugins, this is not transitive""", default = [], - providers = [_KtCompilerPluginInfo], + providers = [[_KtCompilerPluginInfo], [KtPluginConfiguration]], ), "neverlink": attr.bool( doc = """If true only use this library for compilation and not at runtime.""", @@ -452,7 +462,7 @@ Compiler plugins listed here will be treated as if they were added in the plugin attribute of any targets that directly depend on this target. Unlike java_plugins' exported_plugins, this is not transitive""", default = [], - providers = [_KtCompilerPluginInfo], + providers = [[_KtCompilerPluginInfo], [KtPluginConfiguration], [_KspPluginInfo]], ), "neverlink": attr.bool( doc = """If true only use this library for compilation and not at runtime.""", @@ -532,6 +542,7 @@ Supports the following template values: - `{generatedClasses}`: directory for generated class output - `{temp}`: temporary directory, discarded between invocations - `{generatedSources}`: directory for generated source output +- `{classpath}` : replaced with a list of jars separated by the filesystem appropriate separator. """, default = {}, ), @@ -601,3 +612,30 @@ kt_jvm_library( implementation = _kt_ksp_plugin_impl, provides = [_KspPluginInfo], ) + +kt_plugin_cfg = rule( + implementation = kt_plugin_cfg_impl, + doc = """ + Configurations for kt_compiler_plugin, ksp_plugin, and java_plugin. + + This allows setting options and dependencies independently from the initial plugin definition. + """, + attrs = { + "plugin": attr.label( + doc = "The plugin to associate with this configuration", + providers = [_KtCompilerPluginInfo], + mandatory = True, + ), + "options": attr.string_list_dict( + doc = "A dictionary of flag to values to be used as plugin configuration options.", + ), + "deps": attr.label_list( + doc = "Dependencies for this configuration.", + providers = [ + [_KspPluginInfo], + [JavaInfo], + [JavaPluginInfo], + ], + ), + }, +) diff --git a/scripts/install_android_sdk b/scripts/install_android_sdk index 6b6d40fcd..72c96f6ac 100755 --- a/scripts/install_android_sdk +++ b/scripts/install_android_sdk @@ -6,7 +6,7 @@ if which brew; then brew install --cask android-commandlinetools sudo mkdir /usr/local/android-sdk sudo chown "$USER" /usr/local/android-sdk - sdkmanager --sdk_root=/usr/local/android-sdk "platform-tools" "platforms;android-33" "platforms;android-31" "build-tools;33.0.0" "emulator" "build-tools;30.0.3" + sdkmanager --sdk_root=/usr/local/android-sdk "platform-tools" "platforms;android-33" "platforms;android-31" "build-tools;33.0.0" "emulator" "build-tools;30.0.3" "ndk-bundle" else echo "home brew not installed, unsupported. Here is a chance to improve the script!" fi diff --git a/scripts/release.sh b/scripts/release.sh index 3a8de2930..e084914ed 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -47,6 +47,11 @@ if test ! -d examples; then fail "unable to find example directory: $PWD" fi +# generate stardoc +bazel build //kotlin:stardoc || fail "docs did not generate" + +cp -f bazel-bin/kotlin/kotlin.md docs/ || fail "couldn't copy" + bazel test //...:all || fail "tests failed" # build release @@ -74,10 +79,6 @@ done # clean up rm -rf $ARCHIVE_DIR -# generate stardoc -bazel build //kotlin:stardoc || fail "docs did not generate" - -cp -f bazel-bin/kotlin/kotlin.md docs/ || fail "couldn't copy" echo "Release artifact is good:" echo " bazel-bin/rules_kotlin_release.tgz" diff --git a/src/main/kotlin/io/bazel/kotlin/builder/tasks/jvm/CompilationTask.kt b/src/main/kotlin/io/bazel/kotlin/builder/tasks/jvm/CompilationTask.kt index 9f8dbf30a..679eae607 100644 --- a/src/main/kotlin/io/bazel/kotlin/builder/tasks/jvm/CompilationTask.kt +++ b/src/main/kotlin/io/bazel/kotlin/builder/tasks/jvm/CompilationTask.kt @@ -97,13 +97,15 @@ internal fun JvmCompilationTask.plugins( xFlag("plugin", it) } - val dirTokens = mapOf( + val optionTokens = mapOf( "{generatedClasses}" to directories.generatedClasses, "{stubs}" to directories.stubs, + "{temp}" to directories.temp, "{generatedSources}" to directories.generatedSources, + "{classpath}" to classpath.joinToString(File.pathSeparator), ) options.forEach { opt -> - val formatted = dirTokens.entries.fold(opt) { formatting, (token, value) -> + val formatted = optionTokens.entries.fold(opt) { formatting, (token, value) -> formatting.replace(token, value) } flag("-P", "plugin:$formatted") diff --git a/src/main/starlark/core/BUILD.bazel b/src/main/starlark/core/BUILD.bazel index 7cb934ba0..ce8674dab 100644 --- a/src/main/starlark/core/BUILD.bazel +++ b/src/main/starlark/core/BUILD.bazel @@ -11,6 +11,7 @@ release_archive( }, deps = [ "//src/main/starlark/core/options:pkg", + "//src/main/starlark/core/plugin:pkg", "//src/main/starlark/core/repositories:pkg", ], ) @@ -20,7 +21,7 @@ bzl_library( srcs = glob(["*.bzl"]), visibility = ["//:__subpackages__"], deps = [ - "//src/main/starlark/core/options", + "//src/main/starlark/core/plugin", "//src/main/starlark/core/repositories", ], ) diff --git a/src/main/starlark/core/options/BUILD.bazel b/src/main/starlark/core/options/BUILD.bazel index 6512aeb1b..7ef04865d 100644 --- a/src/main/starlark/core/options/BUILD.bazel +++ b/src/main/starlark/core/options/BUILD.bazel @@ -13,8 +13,9 @@ release_archive( bzl_library( name = "options", - srcs = glob(["*.bzl"]) + [ - "@com_github_jetbrains_kotlin//:capabilities.bzl", - ], + srcs = glob(["*.bzl"]), visibility = ["//:__subpackages__"], + deps = [ + "@com_github_jetbrains_kotlin//:capabilities", + ], ) diff --git a/src/main/starlark/core/plugin/BUILD.bazel b/src/main/starlark/core/plugin/BUILD.bazel new file mode 100644 index 000000000..3a04b6ed1 --- /dev/null +++ b/src/main/starlark/core/plugin/BUILD.bazel @@ -0,0 +1,18 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//src/main/starlark/release:packager.bzl", "release_archive") + +release_archive( + name = "pkg", + srcs = glob( + ["*.bzl"], + ), + src_map = { + "BUILD.release.bazel": "BUILD.bazel", + }, +) + +bzl_library( + name = "plugin", + srcs = glob(["*.bzl"]), + visibility = ["//:__subpackages__"], +) diff --git a/src/main/starlark/core/plugin/BUILD.release.bazel b/src/main/starlark/core/plugin/BUILD.release.bazel new file mode 100644 index 000000000..79697e76b --- /dev/null +++ b/src/main/starlark/core/plugin/BUILD.release.bazel @@ -0,0 +1,7 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +bzl_library( + name = "plugin", + srcs = glob(["*.bzl"]), + visibility = ["//:__subpackages__"], +) diff --git a/src/main/starlark/core/plugin/providers.bzl b/src/main/starlark/core/plugin/providers.bzl new file mode 100644 index 000000000..93c610bd7 --- /dev/null +++ b/src/main/starlark/core/plugin/providers.bzl @@ -0,0 +1,34 @@ +KtCompilerPluginOption = provider( + fields = { + "id": "The id of the option.", + "value": "The value of the option.", + }, +) + +KtCompilerPluginInfo = provider( + fields = { + "id": "The id of the plugin.", + "plugin_jars": "List of plugin jars.", + "classpath": "The kotlin compiler plugin classpath.", + "stubs": "Run this plugin during kapt stub generation.", + "compile": "Run this plugin during koltinc compilation.", + "options": "List of plugin options, represented as KtCompilerPluginOption, to be passed to the compiler", + "resolve_cfg": "A Callable[[KtCompilerPluginInfo, Dict[str,str], List[Target], KtPluginConfiguration]" + + " that resolves an associated plugin configuration.", + }, +) + +KtPluginConfiguration = provider( + fields = { + "id": "The id of the compiler plugin associated with this configuration.", + "options": "List of plugin options, represented KtCompilerPluginOption", + "classpath": "Depset of jars to add to the classpath when running the plugin.", + "data": "Depset of files to pass to the plugin as data.", + }, +) + +KspPluginInfo = provider( + fields = { + "plugins": "List of JavaPluginInfo providers for the plugins to run with KSP", + }, +) diff --git a/src/main/starlark/core/repositories/BUILD.com_github_jetbrains_kotlin.bazel b/src/main/starlark/core/repositories/BUILD.com_github_jetbrains_kotlin.bazel index 782b6e857..03467c3dc 100644 --- a/src/main/starlark/core/repositories/BUILD.com_github_jetbrains_kotlin.bazel +++ b/src/main/starlark/core/repositories/BUILD.com_github_jetbrains_kotlin.bazel @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + package(default_visibility = ["//visibility:public"]) # Kotlin home filegroup containing everything that is needed. @@ -18,3 +20,8 @@ filegroup( name = "home", srcs = glob(["**"]), ) + +bzl_library( + name = "capabilities", + srcs = ["capabilities.bzl"], +) diff --git a/src/main/starlark/core/repositories/versions.bzl b/src/main/starlark/core/repositories/versions.bzl index 383546f18..b9c7bc657 100644 --- a/src/main/starlark/core/repositories/versions.bzl +++ b/src/main/starlark/core/repositories/versions.bzl @@ -125,5 +125,12 @@ versions = struct( ], sha256 = None, ), + RULES_TESTING = version( + version = "0.5.0", + url_templates = [ + "https://github.com/bazelbuild/rules_testing/releases/download/v{version}/rules_testing-v{version}.tar.gz", + ], + sha256 = "b84ed8546f1969d700ead4546de9f7637e0f058d835e47e865dcbb13c4210aed", + ), use_repository = _use_repository, ) diff --git a/src/test/starlark/case.bzl b/src/test/starlark/case.bzl new file mode 100644 index 000000000..19827f91e --- /dev/null +++ b/src/test/starlark/case.bzl @@ -0,0 +1,33 @@ +load("@rules_testing//lib:util.bzl", "util") + +def _prepend(rule, name, **kwargs): + util.helper_target( + rule, + name = name, + **kwargs + ) + return ":" + name + +def case(namespace): + return struct( + name = namespace, + have = lambda rule, name, **kwargs: _prepend(rule, namespace + "_" + name, **kwargs), + artifact = lambda name, **kwargs: util.empty_file( + name = namespace + "_" + name, + **kwargs + ), + got = lambda rule, name, **kwargs: _prepend(rule, namespace + "_" + name, **kwargs), + ref = lambda name: ":" + namespace + "_" + name, + ) + +def suite(name, *tests): + test_targets = [] + for test in tests: + test_name = str(test).split(" ")[1] + test_targets.append(":" + test_name) + test(case(test_name)) + + native.test_suite( + name = name, + tests = test_targets, + ) diff --git a/src/test/starlark/core/plugin/BUILD.bazel b/src/test/starlark/core/plugin/BUILD.bazel new file mode 100644 index 000000000..9a78d311f --- /dev/null +++ b/src/test/starlark/core/plugin/BUILD.bazel @@ -0,0 +1,5 @@ +load(":test.bzl", "test_suite") + +test_suite( + name = "plugin", +) diff --git a/src/test/starlark/core/plugin/subjects.bzl b/src/test/starlark/core/plugin/subjects.bzl new file mode 100644 index 000000000..06541b3b5 --- /dev/null +++ b/src/test/starlark/core/plugin/subjects.bzl @@ -0,0 +1,37 @@ +load("@rules_testing//lib:truth.bzl", "subjects") + +def plugin_option_subject_factory(value, meta): + return subjects.struct( + value, + meta = meta.derive("option"), + attrs = { + "id": subjects.str, + "value": subjects.str, + }, + ) + +def plugin_subject_factory(value, meta): + return subjects.struct( + value, + meta = meta, + attrs = { + "id": subjects.str, + "plugin_jars": subjects.collection, + "classpath": subjects.collection, + "stubs": subjects.bool, + "compile": subjects.bool, + "options": subjects.collection, + }, + ) + +def plugin_configuration_subject_factory(value, meta): + return subjects.struct( + value, + meta = meta, + attrs = { + "id": subjects.str, + "options": subjects.collection, + "classpath": subjects.collection, + "data": subjects.collection, + }, + ) diff --git a/src/test/starlark/core/plugin/test.bzl b/src/test/starlark/core/plugin/test.bzl new file mode 100644 index 000000000..332d0a083 --- /dev/null +++ b/src/test/starlark/core/plugin/test.bzl @@ -0,0 +1,380 @@ +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("//kotlin:core.bzl", "kt_compiler_plugin", "kt_plugin_cfg") +load("//kotlin:jvm.bzl", "kt_jvm_import", "kt_jvm_library") +load("//src/main/starlark/core/plugin:providers.bzl", "KtCompilerPluginInfo", "KtPluginConfiguration") +load("//src/test/starlark:case.bzl", "suite") +load("//src/test/starlark:truth.bzl", "fail_messages_in", "flags_and_values_of") +load(":subjects.bzl", "plugin_configuration_subject_factory") + +def _provider_test_impl(env, target): + want_options = env.ctx.attr.want_options + got_target = env.expect.that_target(target) + got_target.has_provider(KtPluginConfiguration) + got_provider = got_target.provider(KtPluginConfiguration, plugin_configuration_subject_factory) + got_provider.options().transform(desc = "option.value", map_each = lambda o: o.value).contains_at_least(want_options) + got_provider.id().equals(env.ctx.attr.want_plugin[KtCompilerPluginInfo].id) + +def _action_test_impl(env, target): + action = env.expect.that_target(target).action_named(env.ctx.attr.on_action_mnemonic) + action.inputs().contains_at_least([f.short_path for f in env.ctx.files.want_inputs]) + flags_and_values_of(action).contains_at_least(env.ctx.attr.want_flags.items()) + +def _expect_failure(env, target): + fail_messages_in(env.expect.that_target(target)).contains_at_least(env.ctx.attr.want_failures) + +def plugin_for(test, name, deps = [], id = None, **kwargs): + plugin_jar = test.artifact( + name = name + ".jar", + ) + + deps = deps + [ + test.have( + kt_jvm_import, + name = name + "_plugin_jar", + jars = [ + plugin_jar, + ], + ), + ] + + plugin = test.have( + kt_compiler_plugin, + id = id if id else "plugin." + name, + name = name, + deps = deps, + **kwargs + ) + return (plugin, plugin_jar) + +def _test_kt_plugin_cfg(test): + plugin = test.have( + kt_compiler_plugin, + name = "plugin", + id = "test.stub", + options = { + "annotation": "plugin.StubForTesting", + }, + deps = [ + test.have( + kt_jvm_library, + name = "plugin_dep", + srcs = [ + test.artifact( + name = "plugin.kt", + ), + ], + ), + ], + ) + cfg_dep = test.have( + kt_jvm_library, + name = "cfg_dep", + srcs = [ + test.artifact( + name = "dependency.kt", + ), + ], + ) + + cfg = test.got( + kt_plugin_cfg, + name = "got", + plugin = plugin, + options = { + "extra": ["annotation"], + }, + deps = [ + cfg_dep, + ], + ) + + analysis_test( + name = test.name, + impl = _provider_test_impl, + target = cfg, + attr_values = { + "want_plugin": plugin, + "want_deps": [cfg_dep], + "want_options": [ + "extra=annotation", + ], + }, + attrs = { + "want_plugin": attr.label(providers = [KtCompilerPluginInfo]), + "want_options": attr.string_list(), + "want_deps": attr.label_list(providers = [JavaInfo]), + }, + ) + +def _test_compile_configuration(test): + plugin_jar = test.artifact( + name = "plugin.jar", + ) + + plugin = test.have( + kt_compiler_plugin, + name = "plugin", + id = "test.stub", + options = { + "annotation": "plugin.StubForTesting", + }, + deps = [ + test.have( + kt_jvm_import, + name = "plugin_jar", + jars = [ + plugin_jar, + ], + ), + ], + ) + + dep_jar = test.artifact( + name = "dep.jar", + ) + + cfg = test.have( + kt_plugin_cfg, + name = "cfg", + plugin = plugin, + options = { + "-Dop": ["koo"], + }, + deps = [ + test.have( + kt_jvm_import, + name = "dep_jar", + jars = [ + dep_jar, + ], + ), + ], + ) + + got = test.got( + kt_jvm_library, + name = "got_library", + srcs = [ + test.artifact( + name = "got_library.kt", + ), + ], + plugins = [ + plugin, + cfg, + ], + ) + + analysis_test( + name = test.name, + impl = _action_test_impl, + target = got, + attr_values = { + "on_action_mnemonic": "KotlinCompile", + "want_flags": { + "--compiler_plugin_options": ["test.stub:annotation=plugin.StubForTesting", "test.stub:-Dop=koo"], + "--stubs_plugin_options": ["test.stub:annotation=plugin.StubForTesting", "test.stub:-Dop=koo"], + }, + "want_inputs": [ + plugin_jar, + dep_jar, + ], + }, + attrs = { + "on_action_mnemonic": attr.string(), + "want_flags": attr.string_list_dict(), + "want_inputs": attr.label_list(providers = [DefaultInfo], allow_files = True), + }, + ) + +def _test_compile_configuration_single_phase(test): + stub, stub_jar = plugin_for( + test, + name = "stub", + id = "plugin.stub", + compile_phase = False, + stubs_phase = True, + ) + + compile, compile_jar = plugin_for( + test, + name = "compile", + id = "plugin.compile", + stubs_phase = False, + compile_phase = True, + ) + + stub_cfg = test.have( + kt_plugin_cfg, + name = "stub_cfg", + plugin = stub, + options = { + "-Dop": ["stub_only"], + }, + ) + + compile_cfg = test.have( + kt_plugin_cfg, + name = "compile_cfg", + plugin = compile, + options = { + "-Dop": ["compile_only"], + }, + ) + + got = test.got( + kt_jvm_library, + name = "got_library", + srcs = [ + test.artifact( + name = "got_library.kt", + ), + ], + plugins = [ + stub, + compile, + compile_cfg, + stub_cfg, + ], + ) + + analysis_test( + name = test.name, + impl = _action_test_impl, + target = got, + attr_values = { + "on_action_mnemonic": "KotlinCompile", + "want_flags": { + "--compiler_plugin_options": ["plugin.compile:-Dop=compile_only"], + "--stubs_plugin_options": ["plugin.stub:-Dop=stub_only"], + }, + "want_inputs": [ + stub_jar, + compile_jar, + ], + }, + attrs = { + "on_action_mnemonic": attr.string(), + "want_flags": attr.string_list_dict(), + "want_inputs": attr.label_list(providers = [DefaultInfo], allow_files = True), + }, + ) + +def _test_library_multiple_plugins_with_same_id(test): + got = test.got( + kt_jvm_library, + name = "got_library", + srcs = [ + test.artifact( + name = "got_library.kt", + ), + ], + plugins = [ + test.have( + kt_compiler_plugin, + name = "one", + id = "test.stub", + options = { + "annotation": "plugin.StubForTesting", + }, + deps = [ + test.have( + kt_jvm_import, + name = "one_plugin_jar", + jars = [ + test.artifact( + name = "one_plugin.jar", + ), + ], + ), + ], + ), + test.have( + kt_compiler_plugin, + name = "two", + id = "test.stub", + options = { + "annotation": "plugin.StubForTesting", + }, + deps = [ + test.have( + kt_jvm_import, + name = "two_plugin_jar", + jars = [ + test.artifact( + name = "two_plugin.jar", + ), + ], + ), + ], + ), + ], + ) + + analysis_test( + name = test.name, + impl = _expect_failure, + expect_failure = True, + target = got, + attr_values = { + "want_failures": [ + "has multiple plugins with the same id: test.stub.", + ], + }, + attrs = { + "want_failures": attr.string_list(), + }, + ) + +def _test_cfg_without_plugin(test): + adee, _ = plugin_for( + test, + name = "Adee", + id = "adee.see", + ) + adee_cfg = test.have( + kt_plugin_cfg, + name = "adee_cfg", + plugin = adee, + options = { + "-Dop": ["compile_only"], + }, + ) + + got = test.got( + kt_jvm_library, + name = "got_library", + srcs = [ + test.artifact( + name = "got_library.kt", + ), + ], + plugins = [ + adee_cfg, + ], + ) + + analysis_test( + name = test.name, + impl = _expect_failure, + expect_failure = True, + target = got, + attr_values = { + "want_failures": [ + "has plugin configurations without corresponding plugins: [\"%s: adee.see\"]" % Label(adee_cfg), + ], + }, + attrs = { + "want_failures": attr.string_list(), + }, + ) + +def test_suite(name): + suite( + name, + _test_kt_plugin_cfg, + _test_compile_configuration, + _test_library_multiple_plugins_with_same_id, + _test_compile_configuration_single_phase, + _test_cfg_without_plugin, + ) diff --git a/src/test/starlark/truth.bzl b/src/test/starlark/truth.bzl new file mode 100644 index 000000000..42cf81eb7 --- /dev/null +++ b/src/test/starlark/truth.bzl @@ -0,0 +1,41 @@ +""" +Collection of utility functions for the action subject +""" + +def fail_messages_in(target_subject): + return target_subject.failures().transform( + desc = "failure.message", + map_each = lambda f: f.partition("Error in fail:")[2].strip() if "Error in fail:" in f else f, + ) + +def flags_and_values_of(action_subject): + return action_subject.argv().transform(desc = "parsed()", loop = _action_subject_parse_flags) + +def _action_subject_parse_flags(argv): + parsed_flags = {} + + # argv might be none for e.g. builtin actions + if argv == None: + return parsed_flags + last_flag = None + for arg in argv: + value = None + if arg == "--": + # skip the rest of the arguments, this is standard end of the flags. + break + if arg.startswith("-"): + if "=" in arg: + last_flag, value = arg.split("=", 1) + else: + last_flag = arg + elif last_flag: + # have a flag, therefore this is probably an associated argument + value = arg + else: + # skip non-flag arguments + continue + + # only set the value if it exists + if value: + parsed_flags.setdefault(last_flag, []).append(value) + return parsed_flags.items()